diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 7c4b30b..1ae3631 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -19,14 +19,19 @@ Makefile targets below. ``` src/zenrows/ -├── __init__.py # re-exports both clients +├── __init__.py # re-exports the clients ├── client.py # ZenRowsClient (legacy sync scraper) -└── batch/ - ├── __init__.py # ZenRowsBatchClient + key models +├── batch/ +│ ├── __init__.py # ZenRowsBatchClient + key models +│ ├── client.py # hand-written typed facade +│ ├── _transport.py # httpx wrapper, RFC 7807 → exceptions +│ ├── errors.py # BatchAPIError, ProblemDetail +│ └── models.py # GENERATED — pydantic v2 (do not edit) +└── crawl/ + ├── __init__.py # ZenRowsCrawlClient + models ├── client.py # hand-written typed facade - ├── _transport.py # httpx wrapper, RFC 7807 → exceptions - ├── errors.py # BatchAPIError, ProblemDetail - └── models.py # GENERATED — pydantic v2 (do not edit) + ├── errors.py # CrawlAPIError + └── models.py # hand-written pydantic v2 models ``` The Batch SDK is split deliberately: @@ -42,12 +47,21 @@ This way the wire types stay in lockstep with the OpenAPI document while the ergonomic surface (method names, helpers, retries, URL override) stays in our control. +The Crawl client uses the shared HTTP transport (`_transport.py`: key +header, retries, problem+json) with `CrawlAPIError.from_response` as +its error mapping. Its models are +hand-written: the surface is a few small schemas, so there is no Crawl +codegen step. +Keep them tolerant: unknown fields are ignored, and every response enum +gets `_missing_ = classmethod(open_enum_missing)`. + ## Common tasks | Make target | What it does | |-----------------|--------------| | `make sync` | `uv sync --all-extras` | -| `make test` | `uv run pytest` | +| `make test` | `uv run pytest` (offline; e2e tests deselected) | +| `make test-e2e` | End-to-end tests against a live API (see below) | | `make check` | `ty check` + `ruff check` + `ruff format --check` (CI mode) | | `make typecheck`| `ty check src` (static types; `models.py` excluded) | | `make lint` | `ruff check --fix` | @@ -56,6 +70,38 @@ override) stays in our control. | `make build` | Build wheel + sdist via hatchling | | `make clean` | Drop caches + build outputs | +## Running the Crawl e2e test + +`tests/e2e/test_crawl_e2e.py` drives a real crawl through the client: +create (with `include_patterns`, `output_format="html"`), wait, read the +results, one page and the NDJSON download, list, stop, and a 404. It +costs a few credits. It is marked `e2e`, which `make test` deselects, +and it skips unless the three required variables below are set. + +| Variable | | +|---|---| +| `ZENROWS_API_KEY` | required: a key with Crawl access | +| `ZENROWS_CRAWL_BASE_URL` | required: the API base, e.g. `https://api.zenrows.com/v1`; the test passes it as `base_url=` | +| `ZENROWS_E2E_CRAWL_URL` | required: the page the crawl starts from | +| `ZENROWS_E2E_CRAWL_INCLUDE` | optional: an include pattern; every result must contain it | + +```bash +export ZENROWS_API_KEY=zr_... # a key with Crawl access +export ZENROWS_CRAWL_BASE_URL=https://api.zenrows.com/v1 +export ZENROWS_E2E_CRAWL_URL=https://example.com/products/ +export ZENROWS_E2E_CRAWL_INCLUDE=/product/ # optional +make test-e2e +``` + +Pick a start page whose links include a few pages matching the pattern: +the test crawls at depth 1 with `max_items=3` and `output_format="html"`. + +To test against a local or staging deployment, point +`ZENROWS_CRAWL_BASE_URL` at its `/v1` base instead. When the account +has reached its limit of active jobs (429 `too_many_crawls`: 3 by default, +shared with its Batch jobs), the test waits and retries (up to 5 minutes) +before creating its crawl. + ## Refreshing the OpenAPI spec `docs/openapi.yaml` is the SDK-local copy of the spec. `make generate` diff --git a/Makefile b/Makefile index 03233d9..216456f 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,18 @@ -.PHONY: install sync test lint format typecheck check generate docs clean build bump release +.PHONY: install sync test test-e2e lint format typecheck check generate docs clean build bump release # Bootstrap: install + dev deps, build the local venv. install sync: uv sync --all-extras -# Run the suite. +# Run the suite (offline; e2e tests are deselected). test: uv run pytest +# Run the end-to-end tests against a live API. Needs ZENROWS_API_KEY, +# ZENROWS_CRAWL_BASE_URL and ZENROWS_E2E_CRAWL_URL; see DEVELOPMENT.md. +test-e2e: + uv run pytest -m e2e -s tests/e2e + # Static type check (ty — Astral). Shipped surface only; tests are # covered by the suite. Generated models.py is excluded in pyproject. typecheck: @@ -15,14 +20,14 @@ typecheck: # Lint + format + type check (CI mode — no fixes). check: typecheck - uv run ruff check src/ tests/ - uv run ruff format --check src/ tests/ + uv run ruff check src/ tests/ acceptance/ + uv run ruff format --check src/ tests/ acceptance/ # Lint + format (writes fixes). lint: - uv run ruff check --fix src/ tests/ + uv run ruff check --fix src/ tests/ acceptance/ format: - uv run ruff format src/ tests/ + uv run ruff format src/ tests/ acceptance/ # Regenerate the pydantic v2 models from the backend's canonical spec. # docs/openapi.yaml is the SDK-local copy of the spec; refresh it from the diff --git a/README.md b/README.md index bf15f52..9550c8e 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ SDK to access [Zenrows](https://www.zenrows.com/) APIs directly from Python. Zenrows handles proxies rotation, headless browsers, and CAPTCHAs for you. -This package ships two clients: +This package ships three clients: - **`ZenRowsClient`** — the original synchronous scraping client. One URL in, one HTML/JSON response out. Best for ad-hoc scraping. @@ -18,6 +18,9 @@ This package ships two clients: thousands of URLs as one job, poll for results, optionally upload a CSV of URLs in one call. Best for offline / bulk pipelines. _(Private beta — [contact support](mailto:support@zenrows.com) for access.)_ + - **`ZenRowsCrawlClient`** — the Crawl API client. Give it one start + URL; it follows links and returns the URLs (and, optionally, the + pages) it finds behind it. _(New.)_ ## Table of Contents @@ -37,6 +40,10 @@ This package ships two clients: - [Act on an id without a GET](#act-on-an-id-without-a-get) - [Scheduled jobs & webhooks](#scheduled-jobs--webhooks) - [Error handling](#error-handling) +- [Quickstart — Crawl API (new) (`ZenRowsCrawlClient`)](#quickstart--crawl-api-new-zenrowscrawlclient) + - [Read the pages](#read-the-pages) + - [List and stop crawls](#list-and-stop-crawls) + - [Crawl errors](#crawl-errors) - [Contributing](#contributing) - [License](#license) @@ -485,6 +492,129 @@ The full Batch surface (jobs, runs, tasks, results, content, history, file_inputs, HMAC keys) is reachable via methods on `ZenRowsBatchClient`. See `src/zenrows/batch/client.py` or `help(ZenRowsBatchClient)`. +## Quickstart — Crawl API (new) (`ZenRowsCrawlClient`) + +Crawl is still evolving: new features are coming, limits may be tuned, and the changelog announces each change. + +A crawl starts from one URL, follows the links on each page, and keeps +the URLs that match your patterns, up to `max_items`. It stops at +`max_items` or `max_pages`, or when no link is left. `depth` limits the +link hops from the start URL; without it the crawl has no depth limit. +It stays on the start URL's registrable domain; +subdomains count. It runs in the background: `create` returns at once with +`status="running"`, and `wait` blocks until it ends. Without +`output_format` a crawl returns URLs only; `output_format="html"` also +returns each kept URL's page. + +```python +import os +from zenrows import ZenRowsCrawlClient + +client = ZenRowsCrawlClient(api_key=os.environ["ZENROWS_API_KEY"]) + +crawl = client.create( + "https://www.scrapingcourse.com/ecommerce/", + depth=1, # link hops from the start URL (default: no limit) + max_items=50, # stop after keeping 50 URLs (default 100) + max_pages=100, # stop after fetching 100 pages (default 100) + include_patterns=["/product/"], # keep only URLs containing this +) +crawl = client.wait(crawl.crawl_id) # polls up to timeout=600 s +print(crawl.status.value, crawl.stop_reason, crawl.coverage) + +for result in client.results(crawl.crawl_id): # follows next_cursor + print(result.url) +``` + +`wait` returns the crawl when it ends, or when `timeout` (seconds, +default 600) runs out. It does not raise: on timeout the crawl is still +`running` (call `wait` again, or `stop`), and a `failed` crawl comes +back with `crawl.error` set. `results` does not poll: it returns at the first empty page, so on a +running crawl it yields only the URLs kept so far. Call `wait` first to +read every result. `results(crawl_id, limit=...)` sets the page size of +each request. `get(crawl_id, cursor=..., limit=...)` returns one raw page. + +The client takes these options: `base_url` (default +`https://api.zenrows.com/v1`), `retries` (default 3; set 0 to disable) +and `timeout` (seconds per request, default 30): + +```python +client = ZenRowsCrawlClient(api_key, base_url="https://api.zenrows.com/v1", retries=3, timeout=30) +``` + +### Read the pages + +Pass `output_format="html"` to also fetch the page of every kept URL: + +```python +crawl = client.create(start_url, depth=1, output_format="html") +client.wait(crawl.crawl_id) + +for result in client.results(crawl.crawl_id): + if result.content_url: # content_status == fetched + html = client.content(crawl.crawl_id, result) + +# Or everything in one NDJSON file: {"url", "content_status", "content"} per line. +with client.download(crawl.crawl_id) as download: # streamed, parsed + print(download.status) # from X-Crawl-Status + for line in download.lines: + print(line.url, line.content_status) +``` + +`content` takes a result, its `content_url`, or its content id. +`download` returns a `CrawlDownload`: its `status` is the crawl's status +when the file was read, and its `lines` yields parsed lines. On a `running` crawl the +file holds only the results kept so far. + +### List and stop crawls + +```python +page = client.list(limit=20) # newest first, without results +for c in page.crawls: + print(c.crawl_id, c.status.value) +# Pass page.next_cursor as cursor= for the next page; it is None on the last. + +client.stop(crawl_id) # idempotent; read final counts with get() +``` + +### Crawl errors + +Every non-2xx raises `CrawlAPIError`. Branch on `status_code` and `code`: + +```python +from zenrows import CrawlAPIError +from zenrows.crawl import CRAWL_NOT_ENABLED_CODE + +try: + client.create(start_url, depth=1) +except CrawlAPIError as exc: + if exc.code == CRAWL_NOT_ENABLED_CODE: # 403 REQS008 + print("Crawl is not enabled for this account") + elif exc.code == "too_many_crawls": # 429: active jobs limit reached + print("retry in", exc.retry_after, "s") + elif exc.status_code in (400, 422): + print(exc.detail) + else: + raise +``` + +| Status | `code` | What to do | +|---|---|---| +| 400 | `invalid_request`, `unknown_parameter`, `invalid_cursor` | Fix the request. Do not retry it as is. | +| 403 | `REQS008` | Crawl is not enabled for the account (`CRAWL_NOT_ENABLED_CODE`). | +| 404 | `crawl_not_found`, `content_not_found` | Check the id. | +| 409 | `idempotency_request_in_flight` | Retry after the first request with this key finishes. | +| 422 | `invalid_parameter`, `invalid_start_url`, `domain_not_allowed` | Fix the request. Do not retry it as is. | +| 422 | `idempotency_key_reused` | Use a new key, or no key. Do not retry it as is. | +| 429 | `too_many_crawls` | The account has reached its limit of active jobs (3 by default), shared with its Batch jobs. Retry after `exc.retry_after` seconds. `create` does not retry it. | +| 503 | `crawl_busy` | `stop` could not be saved yet; the crawl is still running. `stop` retries it up to `retries` times, waiting `exc.retry_after` seconds, then raises it. Repeating the stop is safe. | + +`code` is None when the response has no problem body with a code (for +example a 502 from a proxy). A crawl that ended `failed` is not an error +response: read `crawl.error.code` and `crawl.error.detail` on it. Response enums (`CrawlStatus`, `StopReason`, +`ContentStatus`, ...) are open: a value added later parses as an +`UNKNOWN` member that keeps the raw value. + ## Contributing Pull requests are welcome. For significant changes, please open an issue first to discuss what you would like to change. diff --git a/acceptance/crawl/.gitignore b/acceptance/crawl/.gitignore new file mode 100644 index 0000000..89f9ac0 --- /dev/null +++ b/acceptance/crawl/.gitignore @@ -0,0 +1 @@ +out/ diff --git a/acceptance/crawl/README.md b/acceptance/crawl/README.md new file mode 100644 index 0000000..86d782f --- /dev/null +++ b/acceptance/crawl/README.md @@ -0,0 +1,113 @@ +# Crawl first-install acceptance test: Python SDK + +This folder tests the Python SDK (`zenrows` on PyPI) the way a new customer meets it. It builds the release +artifact, installs it in a clean container with nothing else, follows the README's Crawl +quickstart, and checks every Crawl feature against the live API. The result is one report. + +It tests this repo only, so it runs on its own. + +## What it needs + +- bash, git and Docker. Nothing else on the host. +- Network access to GitHub, the package registries and the Crawl API. +- A Zenrows API key with Crawl access, in `ZENROWS_API_KEY`. + +## How to run it + +From the repo root: + +```bash +export ZENROWS_API_KEY="$(cat path/to/key-file)" # keep it out of your shell history +bash acceptance/crawl/run.sh # build, run every check, write the report +bash acceptance/crawl/run.sh --no-key # only the checks that need no key; the live checks are SKIP +bash acceptance/crawl/run.sh --report # rebuild the report from out/ +REF=v1.2.3 bash acceptance/crawl/run.sh # test a tag, branch or SHA from GitHub instead of this checkout +``` + +Without `ZENROWS_API_KEY`, `run.sh` stops and asks for it. The report is +`acceptance/crawl/out/crawl-first-install-report.md`; `out/` is not committed. + +Every input is an environment variable with a default in `versions.env`: `CRAWL_URL` +(default `https://www.scrapingcourse.com/ecommerce/`), `CRAWL_INCLUDE` (default `/product/`), +`REF`, `CRAWL_API_BASE`, the images (`node:20`, `python:3.12-slim`, `golang:1.25`, pinned by +digest) and the tool versions. `OUT_DIR` moves the output. + +## What a run does + +1. **Source.** By default, this checkout, including changes not yet committed. With `REF`, the + repo at that ref from GitHub. +2. **Build.** In its own container, the release artifact: `pip install` of the wheel `python -m build` makes. +3. **Clean test container.** It gets only the artifact (read-only), the checks in `surface/` + (read-only), the inputs, and the key by name (`-e ZENROWS_API_KEY`). It has no source, no + caches from the build, and no tokens. +4. **README first.** It runs the README's Crawl quickstart as written. The only changes are the + ones the cost rules need: the key, `CRAWL_URL`, `CRAWL_INCLUDE`, `max_pages` 10 or less, and + no html. The report lists each change. +5. **Checks 1 to 21 and the extras.** Each writes one JSON result: + `{surface, check, title, status, expected, actual, evidence}`, with `status` PASS, FAIL or + SKIP. The evidence is the command and its trimmed output, with the key redacted. +6. **Clean-up.** It stops every crawl the run started and reads each one's final status and + `coverage.pages_fetched`. It runs even when a check fails, or on Ctrl-C. +7. **Report.** Verdict, environment, install steps, feature matrix, findings, cost and + clean-up. Last, the run checks that the key appears in no file under `out/`. + +Some checks don't use the live API, so they behave the same on every run. They point the +client at a small local stand-in server that records what it sends. These are: "only the +fields you set are sent", "the error code is empty when the body has none", and the timing +and signal checks. + +## The checks + +| # | Feature | Expected | +| --- | --- | --- | +| 1 | Create | returns at once: `running`, a `crawl_id`, the echoed parameters; only the fields set are sent, and a create without `depth` sends none and reads back none | +| 2 | Create with html | results get `content_status` and, once fetched, `content_url` | +| 3 | Idempotency | same key and body → same `crawl_id`; different body → 422 `idempotency_key_reused` (MCP, CLI: `CRAWL_INVALID_REQUEST` with the advice to send a new key) | +| 4 | Wait | returns the ended crawl; `failed` is returned, not raised (CLI: exit 1) | +| 5 | Wait runs out | a 1 s timeout on a running crawl returns it `running`, no error (CLI: exit 0; MCP, CLI: a `note`) | +| 6 | Get | `limit` 2, then the cursor: one page each; `next_cursor` moves on and is null on the last page | +| 7 | List | `limit` 2, then the next page: newest first; `next_cursor` absent or null on the last page | +| 8 | Results | every kept URL once, each matching `CRAWL_INCLUDE`; on a running crawl, what is kept so far (MCP, CLI: `partial: true`) | +| 9 | Content | by content id and by `content_url`: the same HTML | +| 10 | Download | a status, and one line per result with `url`, `content_status`, `content` | +| 11 | Stop | running → `stopped`, `stop_reason: user`; ended → answers as it ended | +| 12 | Not found | 404 `crawl_not_found` (MCP, CLI: `CRAWL_NOT_FOUND` with `crawl_id` and `server_code`) | +| 13 | Invalid input | `depth` 0 → 422 `invalid_parameter` (MCP, CLI: `CRAWL_INVALID_REQUEST`) | +| 14 | Gated option | `output_format` json → 422 `invalid_parameter`; if it is accepted, the key is in a beta segment, not a bug | +| 15 | Bad key | the key `invalid` → 401 (MCP, CLI: `AUTH_INVALID`) | +| 16 | Errors carry the server code | SDKs: `code` holds it, empty when the body has none; MCP, CLI: `server_code` | +| 17 | Naming | the API's operation and parameter names, in this surface's casing | +| 18 | Copy | Crawl is "new", never "beta"; one "still evolving" line; the 429 limit reads as the account's active jobs, shared with Batch | +| 19 | Failed crawl | an include pattern that matches nothing ends `failed` with `no_items_found`; SDKs return it, not raise it; MCP and CLI say `ok: false` (CLI: exit 1, `CRAWL_FAILED`) | +| 20 | Results after stop | the html crawl is stopped while its pages are still being fetched: reading its results ends at once (MCP and CLI: `partial: false`, no note to call again), and the same URLs come back once the pages in flight finish | +| 21 | Documented defaults | a create without `max_pages` and `max_items` gets the default the docs state (100) | + +Extra checks for this surface: `mypy --strict` sees the package as typed (`py.typed`) and reads the real signatures. + +## What it costs + +Every crawl uses `depth` 1. Every crawl but check 19's sets `max_pages` 10 or less; check 19's +leaves it out so check 21 can read the default (100). A run starts five crawls: the README run, +one URLs-only crawl, one idempotency crawl that is stopped at once, one html crawl that is +stopped with its pages in flight (check 20), and one that keeps nothing (check 19). On the default `CRAWL_URL`, a URLs-only crawl fetches 1 page and the +html crawl 6, so a run costs about **10 pages**. Check 19's crawl keeps nothing and fetches only +the start page at `depth` 1, so its default of 100 does not raise the cost. At most 140 if every +crawl reaches its `max_pages`. Check 9 starts a sixth crawl, one html item, when the stop in check 20 lands +before any page is fetched. It adds at most 10 pages. The report lists each crawl id and its `pages_fetched`. + +An account can run 3 active jobs at once, shared with Batch jobs. A run has at most 2 crawls +running at a time. If a create still gets a 429, the run waits `Retry-After` and tries again. +It never sends requests to trigger the limit. Don't run two copies against one account at the +same time. + +## How to read the report + +- **Verdict:** ready, unless there is a blocker or major finding. +- **Feature matrix:** PASS, FAIL or SKIP, with a short reason. SKIP means the check does not + apply to this surface, the client rejects the input before the API, a crawl ended before + the check could act on it, or there was no key. +- **Findings:** one entry per FAIL, most severe first, with the command and output. An install + or README failure is a blocker; a wrong answer from a feature is major; naming, copy, the + defaults and the extras are minor. +- `out/results/checks/*.json` holds every result; `out/results/container.log` is the full + output. diff --git a/acceptance/crawl/builders/build-python.sh b/acceptance/crawl/builders/build-python.sh new file mode 100644 index 0000000..ec399e7 --- /dev/null +++ b/acceptance/crawl/builders/build-python.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Runs in the Python image. Builds the wheel a release would publish. Only the wheel leaves. +set -euo pipefail +cp -a /out/source /build && cd /build +pip install --quiet --disable-pip-version-check --root-user-action=ignore "build==$PY_BUILD_VERSION" +python -m build --wheel --outdir /out/artifact +ls -l /out/artifact diff --git a/acceptance/crawl/builders/fetch-source.sh b/acceptance/crawl/builders/fetch-source.sh new file mode 100644 index 0000000..51f1209 --- /dev/null +++ b/acceptance/crawl/builders/fetch-source.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Runs in the Node image when REF is set: downloads ZenRows/$REPO at $REF as GitHub's source +# tarball and reads the full commit SHA from it. +set -euo pipefail +cd /out +curl -fsSL --retry 4 -o src.tar.gz "https://codeload.github.com/ZenRows/$REPO/tar.gz/$REF" +# git get-tar-commit-id stops reading after the header, so the pipe breaks: ignore that. +(gunzip -c src.tar.gz | git get-tar-commit-id > sha) || true +[ -s sha ] || { echo "no commit id in the tarball of $REPO@$REF" >&2; exit 1; } +rm -rf source && mkdir -p source && tar -xzf src.tar.gz -C source --strip-components 1 && rm src.tar.gz +echo "fetched $REPO@$REF = $(cat sha)" diff --git a/acceptance/crawl/run.sh b/acceptance/crawl/run.sh new file mode 100644 index 0000000..ec8cc1d --- /dev/null +++ b/acceptance/crawl/run.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +# Crawl first-install acceptance test for this repo. See README.md in this folder. +# +# ZENROWS_API_KEY=... bash run.sh build the release artifact, run every check, write the report +# bash run.sh --no-key only the checks that need no key; the live checks are SKIP +# bash run.sh --report rebuild the report from the results already in out/ +# +# Needs bash, git and Docker. The key reaches containers by name only (-e ZENROWS_API_KEY). +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=versions.env +source "$HERE/versions.env" +OUT_DIR="${OUT_DIR:-$HERE/out}" + +no_key=0; report_only=0 +for arg in "$@"; do + case "$arg" in + --no-key) no_key=1 ;; + --report) report_only=1 ;; + -h|--help) sed -n '2,8p' "$0"; exit 0 ;; + *) echo "unknown argument: $arg (expected --no-key or --report)" >&2; exit 2 ;; + esac +done + +if [ "$report_only" = 0 ] && [ "$no_key" = 0 ] && [ -z "${ZENROWS_API_KEY:-}" ]; then + echo "ZENROWS_API_KEY is not set. Set it (a key with Crawl access), or run 'bash run.sh --no-key' for the offline checks." >&2 + exit 2 +fi +if [ "$no_key" = 1 ]; then unset ZENROWS_API_KEY; fi +export ZENROWS_API_KEY="${ZENROWS_API_KEY:-}" CRAWL_URL CRAWL_INCLUDE CRAWL_API_BASE + +log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*" >&2; } +RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)" +CMDLINE="bash run.sh $*" +# Unique per run, so clean-up never touches the containers of another run. +LABEL="crawl-acceptance.run=$RUN_ID-$$-$RANDOM" +docker_run() { docker run --rm --label "$LABEL" "$@"; } + +# ---- 1. Source: this checkout (default), or the repo on GitHub at $REF ---- +prepare_source() { + rm -rf "$OUT_DIR/build"; mkdir -p "$OUT_DIR/build/source" + if [ -n "${REF:-}" ]; then + log "source: ZenRows/$REPO@$REF from GitHub" + docker_run -v "$OUT_DIR/build:/out" -v "$HERE/builders:/builders:ro" \ + -e REPO="$REPO" -e REF="$REF" "$NODE_IMAGE" bash /builders/fetch-source.sh >"$OUT_DIR/build/fetch.log" 2>&1 + echo "github:$REF" >"$OUT_DIR/build/origin" + else + local root; root="$(git -C "$HERE" rev-parse --show-toplevel)" + log "source: this checkout ($root)" + # Tracked and untracked-but-not-ignored files, as a release build would see them. + (cd "$root" && git ls-files -co --exclude-standard -z | tar --null -T - -cf -) | tar -C "$OUT_DIR/build/source" -xf - + local sha; sha="$(git -C "$root" rev-parse HEAD)" + if [ -n "$(git -C "$root" status --porcelain -- . ":(exclude)$(git -C "$HERE" rev-parse --show-prefix)")" ]; then + sha="$sha+local-changes" + fi + echo "$sha" >"$OUT_DIR/build/sha" + echo "checkout" >"$OUT_DIR/build/origin" + fi + # The test never reads the harness from the source; only the artifact goes in. + rm -rf "$OUT_DIR/build/source/acceptance" +} + +# ---- 2. Builder: the release artifact, in its own container ---- +build_artifact() { + mkdir -p "$OUT_DIR/build/artifact" + log "build: the wheel" + docker_run -v "$OUT_DIR/build:/out" -v "$HERE/builders:/builders:ro" \ + -e PY_BUILD_VERSION "$PYTHON_IMAGE" bash /builders/build-python.sh >"$OUT_DIR/build/build.log" 2>&1 +} + +# ---- 3. Clean test container: only the artifact, the checks and the inputs go in ---- +run_checks() { + rm -rf "$OUT_DIR/results"; mkdir -p "$OUT_DIR/results" + log "checks: a clean $PYTHON_IMAGE" + local rc=0 + docker_run --init \ + -v "$OUT_DIR/build/artifact:/artifact:ro" \ + -v "$HERE/surface:/harness:ro" \ + -v "$OUT_DIR/results:/results" \ + -e ZENROWS_API_KEY -e CRAWL_URL -e CRAWL_INCLUDE -e CRAWL_API_BASE \ + -e SURFACE \ + -e MCP_SDK_VERSION -e TYPESCRIPT_VERSION -e MYPY_VERSION \ + -w /work --tmpfs /work:exec,size=2g \ + "$PYTHON_IMAGE" bash /harness/run-in-container.sh >"$OUT_DIR/results/container.log" 2>&1 || rc=$? + log "checks: the container exited $rc (out/results/container.log)" +} + +# ---- 4. Clean-up: stop every crawl this run started. Runs even when a check fails. ---- +cleanup() { + [ "$report_only" = 1 ] && return 0 + docker ps -q --filter "label=$LABEL" | xargs -r docker kill >/dev/null 2>&1 || true + if [ -n "${ZENROWS_API_KEY:-}" ] && [ -d "$OUT_DIR/results" ]; then + log "clean-up: stopping every crawl this run started" + docker_run -v "$OUT_DIR:/out" -v "$HERE/tools:/tools:ro" \ + -e ZENROWS_API_KEY -e CRAWL_API_BASE "$PYTHON_IMAGE" python -I /tools/cleanup.py /out || log "clean-up failed" + fi +} + +report() { + log "report: out/crawl-first-install-report.md" + docker_run -v "$OUT_DIR:/out" -v "$HERE/tools:/tools:ro" \ + -e CMDLINE="$CMDLINE" -e SURFACE -e REPO -e CRAWL_URL -e CRAWL_INCLUDE -e CRAWL_API_BASE \ + -e NODE_IMAGE -e PYTHON_IMAGE -e GO_IMAGE -e MCP_SDK_VERSION -e TYPESCRIPT_VERSION \ + -e MYPY_VERSION -e PY_BUILD_VERSION -e PNPM_VERSION \ + "$PYTHON_IMAGE" python -I /tools/report.py /out + # The key must appear in nothing the run wrote. Checked by value, inside a container. + if [ -n "${ZENROWS_API_KEY:-}" ]; then + docker_run -v "$OUT_DIR:/out" -v "$HERE/tools:/tools:ro" -e ZENROWS_API_KEY "$PYTHON_IMAGE" \ + python -I /tools/scan_key.py /out + fi +} + +export SURFACE REPO NODE_IMAGE PYTHON_IMAGE GO_IMAGE PNPM_VERSION PY_BUILD_VERSION \ + MCP_SDK_VERSION TYPESCRIPT_VERSION MYPY_VERSION +if [ "$report_only" = 1 ]; then report; exit 0; fi + +mkdir -p "$OUT_DIR" +echo "$RUN_ID" >"$OUT_DIR/run_id" +trap cleanup EXIT +trap 'exit 130' INT TERM +if prepare_source && build_artifact; then + run_checks +else + log "the source or build step failed (out/build/*.log)" + mkdir -p "$OUT_DIR/results" + docker_run -v "$OUT_DIR:/out" -v "$HERE/tools:/tools:ro" -e SURFACE "$PYTHON_IMAGE" python -I /tools/build_failed.py /out +fi +cleanup +trap - EXIT +report +log "done: $OUT_DIR/crawl-first-install-report.md" diff --git a/acceptance/crawl/surface/run-in-container.sh b/acceptance/crawl/surface/run-in-container.sh new file mode 100644 index 0000000..e4598dd --- /dev/null +++ b/acceptance/crawl/surface/run-in-container.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Entry point in the clean python container: install the wheel as `pip install zenrows` would, +# install the pinned mypy for the typing check, record the versions, run the checks. +set -uo pipefail +export PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_ROOT_USER_ACTION=ignore PIP_NO_CACHE_DIR=1 +pip install /artifact/*.whl > /results/pip-install.log 2>&1 || { cat /results/pip-install.log; exit 1; } +pip install "mypy==$MYPY_VERSION" > /results/pip-mypy.log 2>&1 +printf '{"python":"%s","pip":"%s","mypy":"%s","os":"%s"}\n' "$(python -V 2>&1)" "$(pip -V | cut -d' ' -f2)" \ + "$MYPY_VERSION" "$(. /etc/os-release; echo "$PRETTY_NAME")" > /results/env.json +cd /work && python /harness/test.py diff --git a/acceptance/crawl/surface/test.py b/acceptance/crawl/surface/test.py new file mode 100644 index 0000000..1f1ba40 --- /dev/null +++ b/acceptance/crawl/surface/test.py @@ -0,0 +1,914 @@ +"""Python SDK (`zenrows`): the README walk-through, checks 1-21 and the Python extra check. + +Runs in a clean python image. run-in-container.sh installs the wheel first; this script then +imports it the way a user would. +""" + +import http.server +import json +import os +import re +import subprocess +import sys +import threading +import time +import traceback +import urllib.request +import uuid +from datetime import UTC, datetime +from importlib import metadata + +SURFACE = "python" +KEY = os.environ.get("ZENROWS_API_KEY", "") +HAS_KEY = bool(KEY) +CRAWL_URL = os.environ["CRAWL_URL"] +INCLUDE = os.environ["CRAWL_INCLUDE"] +API_BASE = os.environ.get("CRAWL_API_BASE", "https://api.zenrows.com/v1") +NO_MATCH = ( + "/crawl-acceptance-matches-nothing/" # no URL contains it: the crawl keeps nothing (check 19) +) +DOCUMENTED_DEFAULT = 100 # the default the README documents for max_pages and max_items (check 21) +RESULTS = "/results" +PASS, FAIL, SKIP = "PASS", "FAIL", "SKIP" +TERMINAL = {"completed", "stopped", "failed"} +os.makedirs(f"{RESULTS}/checks", exist_ok=True) +_seq = 0 + + +def redact(v): + s = v if isinstance(v, str) else json.dumps(v, default=str) + return s.replace(KEY, "") if KEY else s + + +def trim(v, n): + s = redact(v) + return s if len(s) <= n else f"{s[:n]}… [{len(s) - n} more chars]" + + +def record(check, title, status, expected, actual, evidence=""): + global _seq + _seq += 1 + r = { + "surface": SURFACE, + "check": str(check), + "title": title, + "status": status, + "expected": expected, + "actual": trim(actual, 800), + "evidence": trim(evidence, 2500), + } + name = f"{_seq:03d}-{re.sub(r'[^A-Za-z0-9_.-]+', '_', str(check))}.json" + with open(f"{RESULTS}/checks/{name}", "w") as f: + json.dump(r, f, indent=2) + first = r["actual"].splitlines()[0][:200] if r["actual"] else "" + print(f"[{status}] python {check} {title}: {first}", flush=True) + + +def pass_if(ok): + return PASS if ok else FAIL + + +def run_check(check, title, expected, fn, live=False): + if live and not HAS_KEY: + return record( + check, title, SKIP, expected, "ZENROWS_API_KEY not set (run without --no-key)" + ) + try: + status, actual, evidence = fn() + record(check, title, status, expected, actual, evidence) + except Exception as e: # an unexpected exception is a FAIL with its traceback + record( + check, title, FAIL, expected, f"raised {type(e).__name__}: {e}", traceback.format_exc() + ) + + +def ledger(crawl_id, purpose): + if crawl_id: + with open(f"{RESULTS}/crawls.jsonl", "a") as f: + f.write(json.dumps({"crawl_id": crawl_id, "purpose": purpose}) + "\n") + + +def sh(args, **kw): + p = subprocess.run(args, capture_output=True, text=True, **kw) + return p.returncode, p.stdout + p.stderr + + +def crawls_since(t): + if not HAS_KEY: + return [] + req = urllib.request.Request(f"{API_BASE}/crawls?limit=100", headers={"X-API-Key": KEY}) + with urllib.request.urlopen(req, timeout=60) as resp: + body = json.load(resp) + out = [] + for c in body["crawls"]: + created = datetime.strptime(c["created_at"][:19], "%Y-%m-%dT%H:%M:%S").replace(tzinfo=UTC) + if created.timestamp() >= t: + out.append(c["crawl_id"]) + return out + + +# ---------- README: install (run-in-container.sh) and the Crawl quickstart ---------- +with open(f"{RESULTS}/pip-install.log") as f: + install_log = f.read() +record( + "readme-1", + "Install (README: `pip install zenrows`; here the release wheel)", + PASS, + "the package installs", + f"zenrows {metadata.version('zenrows')} installed", + f"$ pip install \n{install_log[-1200:]}", +) + +readme = metadata.metadata("zenrows").get_payload() or "" +HEADING = "## Quickstart — Crawl API (new)" +section = readme.split(HEADING, 1)[1].split("\n## ", 1)[0] if HEADING in readme else "" +m = re.search(r"```python\n(.*?)```", section, re.S) +block = m.group(1) if m else None +record( + "readme-2", + "README has a Crawl quickstart", + pass_if(bool(block)), + "a ```python block under the Crawl quickstart", + f"{len(block.splitlines())} lines" if block else "not found", + block or section[:600], +) + +subs = [ + ('"https://example.com/products/"', json.dumps(CRAWL_URL), "start URL → CRAWL_URL"), + ("max_items=50,", "max_items=5,", "max_items 50 → 5"), + ("max_pages=100,", "max_pages=10,", "max_pages 100 → 10 (at most 10)"), + ( + 'include_patterns=["/product/"]', + f"include_patterns=[{json.dumps(INCLUDE)}]", + "include → CRAWL_INCLUDE", + ), +] +quick, applied = block or "", [] +for a, b, why in subs: + if a in quick: + quick = quick.replace(a, b) + applied.append(why) + else: + applied.append(f"not found, left as is: {why}") +applied.append("the key comes from ZENROWS_API_KEY, as the quickstart reads it") +with open("/work/quickstart.py", "w") as f: + f.write(quick) + + +def readme_run(): + since = time.time() - 5 + code, out = sh([sys.executable, "/work/quickstart.py"], timeout=12 * 60) + mine = crawls_since(since) + for i in mine: + ledger(i, "README quickstart") + return ( + pass_if(code == 0), + f"exit {code}; crawls: {', '.join(mine) or 'none'}", + f"substitutions: {'; '.join(applied)}\n$ python quickstart.py\n{out}", + ) + + +run_check( + "readme-3", + "Run the README Crawl quickstart", + "it runs to the end without an error", + readme_run, + live=True, +) + + +# ---------- extra: a typed package (py.typed) under mypy --strict ---------- +def mypy_check(): + code, out = sh( + [sys.executable, "-m", "mypy", "--strict", "--no-incremental", "/harness/typecheck.py"] + ) + untyped = ( + "missing library stubs or py.typed" in out or "module is installed, but missing" in out + ) + revealed = "def (url: builtins.str" in out + errors = [line for line in out.splitlines() if ": error:" in line] + return ( + pass_if(not untyped and revealed and not errors), + f"mypy exit {code}; py.typed seen {not untyped}; " + f"real signature revealed {revealed}; {len(errors)} errors", + f"$ mypy --strict typecheck.py (mypy {metadata.version('mypy')})\n{out}", + ) + + +run_check( + "x-python-typed", + "mypy --strict sees the package as typed (py.typed)", + "no missing-stubs error; the real signatures", + mypy_check, +) + +# ---------- check 1, offline half: only the fields you set are sent ---------- +captured = [] + + +class StandIn(http.server.BaseHTTPRequestHandler): + """A local stand-in for the Crawl API that records what the client sends.""" + + def log_message(self, *a): + pass + + def do_POST(self): + n = int(self.headers.get("Content-Length") or 0) + body = json.loads(self.rfile.read(n) or b"null") + captured.append(body) + self.send_response(201) + self.send_header("Content-Type", "application/json") + self.end_headers() + crawl = { + "crawl_id": "c_capture", + "status": "running", + "url": body.get("url"), + "max_items": DOCUMENTED_DEFAULT, + "max_pages": DOCUMENTED_DEFAULT, + "created_at": datetime.now(UTC).isoformat(), + "coverage": {"pages_fetched": 0, "pages_failed": 0, "items_found": 0}, + } + if "depth" in body: # as the API: a crawl created without depth omits it + crawl["depth"] = body["depth"] + self.wfile.write(json.dumps(crawl).encode()) + + def do_GET(self): + if "c_nobody" in self.path: + self.send_response(502) + self.send_header("Content-Length", "0") + self.end_headers() + return + self.send_response(404) + self.send_header("Content-Type", "application/problem+json") + self.end_headers() + self.wfile.write(b'{"status":404,"title":"Not Found","code":"crawl_not_found"}') + + +srv = http.server.ThreadingHTTPServer(("127.0.0.1", 0), StandIn) +threading.Thread(target=srv.serve_forever, daemon=True).start() + +from zenrows import ZenRowsCrawlClient # noqa: E402 (imported once installed, as a user would) +from zenrows.crawl import CrawlAPIError # noqa: E402 + +cap = ZenRowsCrawlClient( + "capture-key", base_url=f"http://127.0.0.1:{srv.server_address[1]}/v1", retries=0 +) +minimal = cap.create("https://example.com/") # depth is optional: no depth limit +cap.create( + "https://example.com/", + depth=1, + max_items=5, + max_pages=10, + include_patterns=["/p/"], + idempotency_key="idem-1", +) +sent_ok = ( + sorted(captured[0]) == ["url"] + and minimal.depth is None + and sorted(captured[1]) + == sorted(["url", "depth", "max_items", "max_pages", "include_patterns"]) +) +sent_line = ( + f"sent (stand-in server): minimal={json.dumps(captured[0])} full={json.dumps(captured[1])}" +) + +# ---------- live ---------- +client = ZenRowsCrawlClient(KEY or "none", base_url=API_BASE) +BASE = {"depth": 1, "max_items": 5, "max_pages": 10, "include_patterns": [INCLUDE]} +state = {} + + +def create(**kw): + for _ in range(10): + try: + c = client.create(CRAWL_URL, **kw) + ledger(c.crawl_id, "check") + return c + except CrawlAPIError as e: + if e.status_code != 429: + raise + time.sleep(e.retry_after or 30) # the account's active jobs are shared: wait as told + raise RuntimeError("still 429 after 10 tries") + + +def err_of(fn): + try: + return fn(), None + except CrawlAPIError as e: + return None, e + + +def esum(e): + return None if e is None else {"status_code": e.status_code, "code": e.code, "detail": e.detail} + + +def c1(): + if not HAS_KEY: + return ( + (SKIP if sent_ok else FAIL), + f"live part skipped (no key); only set fields sent: {sent_ok}", + sent_line, + ) + t = time.time() + a = state["A"] = create(**BASE) + echo = { + "url": a.url, + "depth": a.depth, + "max_items": a.max_items, + "max_pages": a.max_pages, + "include_patterns": a.include_patterns, + } + want = {"url": CRAWL_URL, **BASE} + ok = sent_ok and a.status.value == "running" and a.crawl_id.startswith("c_") and echo == want + return ( + pass_if(ok), + f"{a.status.value}, {a.crawl_id}, {int((time.time() - t) * 1000)} ms, " + f"echo {json.dumps(echo)}; only set fields sent: {sent_ok}", + f"{a.model_dump_json()}\n{sent_line}", + ) + + +def c5(): + a = state.get("A") + if not a: + return SKIP, "no crawl from check 1", "" + t = time.time() + w = client.wait(a.crawl_id, timeout=1) + ms = int((time.time() - t) * 1000) + if w.status.value != "running": + return SKIP, f"the crawl had already ended ({w.status.value}) after {ms} ms", "" + return pass_if(ms < 5000), f"{w.status.value} after {ms} ms, no exception", w.model_dump_json() + + +def partial_read(): + a = state.get("A") + if not a: + return + running = client.get(a.crawl_id, limit=1).status.value == "running" + t = time.time() + items, e = err_of(lambda: [r.url for r in client.results(a.crawl_id)]) + state["partial"] = (running, len(items or []), int((time.time() - t) * 1000), e) + + +def c3(): + key = f"acc-{uuid.uuid4()}" + body = dict(BASE, max_items=3, max_pages=3) + c = state["C"] = create(idempotency_key=key, **body) + again = create(idempotency_key=key, **body) + v, e = err_of(lambda: client.create(CRAWL_URL, idempotency_key=key, **dict(body, max_items=4))) + if v: + ledger(v.crawl_id, "idempotency, different body (unexpected create)") + ok = ( + c.crawl_id == again.crawl_id + and e is not None + and e.status_code == 422 + and e.code == "idempotency_key_reused" + ) + third = f"{e.status_code} {e.code}" if e else f"created {v.crawl_id}" + return ( + pass_if(ok), + f"first {c.crawl_id}, second {again.crawl_id}; different body → {third}", + json.dumps(esum(e)), + ) + + +def c11(): + c = state.get("C") + if not c: + return SKIP, "no crawl from check 3", "" + first = client.stop(c.crawl_id) + reason = first.stop_reason.value if first.stop_reason else None + if first.status.value == "completed" and reason != "user": + return SKIP, f"the crawl had already ended on its own ({first.status.value}/{reason})", "" + second, e = err_of(lambda: client.stop(c.crawl_id)) + ok = ( + first.status.value == "stopped" + and reason == "user" + and e is None + and second.status == first.status + ) + two = f"error {e}" if e else second.status.value + return ( + pass_if(ok), + f"first: {first.status.value}/{reason}; second: {two}", + first.model_dump_json(), + ) + + +def c4(): + a = state.get("A") + if not a: + return SKIP, "no crawl from check 1", "" + e = state["ended"] = client.wait(a.crawl_id) + reason = f"/{e.stop_reason.value}" if e.stop_reason else "" + return ( + pass_if(e.status.value in TERMINAL), + f"{e.status.value}{reason}; coverage {e.coverage.model_dump_json()}", + e.model_dump_json(), + ) + + +def c6(): + e = state.get("ended") + if not e: + return SKIP, "no ended crawl", "" + pages, cursor = [], None + for _ in range(20): + p = client.get(e.crawl_id, limit=2, cursor=cursor) + pages.append((len(p.results), p.next_cursor, [r.url for r in p.results])) + if p.next_cursor is None: + break + cursor = p.next_cursor + urls = [u for _, _, us in pages for u in us] + curs = [c for _, c, _ in pages if c is not None] + ok = ( + all(n <= 2 for n, _, _ in pages) + and pages[-1][1] is None + and len(set(curs)) == len(curs) + and len(set(urls)) == len(urls) + and (len(pages) >= 2 or e.coverage.items_found <= 2) + ) + sizes = ",".join(str(n) for n, _, _ in pages) + return ( + pass_if(ok), + f"{len(pages)} pages of {sizes}; last next_cursor {pages[-1][1]}; {len(urls)} URLs", + "", + ) + + +def c7(): + p1 = client.list(limit=2) + p2 = client.list(limit=2, cursor=p1.next_cursor) if p1.next_cursor else None + allc = p1.crawls + (p2.crawls if p2 else []) + times = [c.created_at for c in allc] + newest = all(times[i - 1] >= times[i] for i in range(1, len(times))) + overlap = len({c.crawl_id for c in allc}) != len(allc) + cursor, pages = (p2.next_cursor if p2 else None), 2 + while cursor and pages < 60: + cursor = client.list(limit=100, cursor=cursor).next_cursor + pages += 1 + ok = len(p1.crawls) == 2 and newest and not overlap and not cursor + return ( + pass_if(ok), + f"newest first {newest}, no overlap {not overlap}; last page after {pages} pages: " + f"next_cursor {json.dumps(cursor)}", + "", + ) + + +def c8(): + e = state.get("ended") + if not e: + return SKIP, "no ended crawl", "" + urls = [r.url for r in client.results(e.crawl_id, limit=2)] + distinct, match = len(set(urls)) == len(urls), all(INCLUDE in u for u in urls) + p = state.get("partial") + run = ( + ( + f"while running ({'running' if p[0] else 'already ended'}): {p[1]} URLs in {p[2]} ms, " + f"{'error ' + str(p[3]) if p[3] else 'returned'}" + ) + if p + else "not read while running" + ) + ok = ( + bool(urls) + and distinct + and match + and len(urls) == e.coverage.items_found + and not (p and p[3]) + ) + return ( + pass_if(ok), + f"{len(urls)} URLs (items_found {e.coverage.items_found}), distinct {distinct}, " + f"all match {match}; {run}", + "\n".join(urls), + ) + + +def settle(crawl_id): + """Poll until no result is still pending (the pages in flight finished), at most 90 s.""" + for _ in range(45): + page = client.get(crawl_id, limit=100) + if not any(r.content_status and r.content_status.value == "pending" for r in page.results): + return "settled" + time.sleep(2) + return "still pending after 90 s" + + +def stop_mid_flight(): + """Create the html crawl and stop it as soon as it has kept URLs. + + The stop lands while their pages are fetched. + """ + try: + b = state["B"] = create(output_format="html", **BASE) + g = None + for _ in range(100): + g = client.get(b.crawl_id, limit=1) + if g.status.value != "running" or g.coverage.items_found > 0: + break + time.sleep(0.15) + s = client.stop(b.crawl_id) + state["stop"] = {"items": g.coverage.items_found, "status": s.status.value} + except Exception as e: # recorded by check 20 + state["stop"] = {"error": e} + + +def c20(): + b, st = state.get("B"), state.get("stop") or {} + if not b or "error" in st: + return FAIL, f"setup failed: {st.get('error')}", "" + if st["status"] != "stopped": + return SKIP, f"the crawl had already ended ({st['status']}) before the stop", "" + t = time.time() + box = {} + # A daemon thread, so a results() that pages forever fails the check instead of hanging the run. + reader = threading.Thread( + target=lambda: box.update(now=list(client.results(b.crawl_id))), daemon=True + ) + reader.start() + reader.join(60) + if "now" not in box: + return ( + FAIL, + "results() did not finish within 60 s on a crawl stopped with pages in flight", + "", + ) + now, ms = box["now"], int((time.time() - t) * 1000) + settled = settle(b.crawl_id) + if settled != "settled": + settled += " (an API issue: the stop left these pages pending on an ended crawl)" + later = [r.url for r in client.results(b.crawl_id)] + same = sorted(r.url for r in now) == sorted(later) + pending = sum(1 for r in now if r.content_status and r.content_status.value == "pending") + return ( + pass_if(same and bool(now)), + f"stopped with {st['items']} kept; read {len(now)} URLs ({pending} pending) in {ms} ms, " + f"then {settled}: {len(later)} URLs, same set {same}", + "", + ) + + +def c2(): + b = state.get("B") + if not b: + return SKIP, "no html crawl", "" + if (state.get("stop") or {}).get("status") == "stopped": + settle(b.crawl_id) + rs = list(client.results(b.crawl_id)) + ok = ( + b.output_format is not None + and b.output_format.value == "html" + and bool(rs) + and all( + r.content_status and (r.content_status.value != "fetched" or r.content_url) for r in rs + ) + ) + with_url = sum(1 for r in rs if r.content_url) + return ( + pass_if(ok), + f"{b.crawl_id} ({(state.get('stop') or {}).get('status', '-')}); {len(rs)} results; " + f"with content_url {with_url}", + json.dumps([r.model_dump(mode="json") for r in rs[:2]]), + ) + + +def c9(): + b = state.get("B") + if not b: + return SKIP, "no html crawl", "" + + def first_fetched(crawl_id): + return next((x for x in client.results(crawl_id) if x.content_url), None) + + # The stop in check 20 can land before any page is fetched. Then read a + # one-item html crawl that runs to the end instead. + src = b.crawl_id + r = first_fetched(src) + if not r: + own = create(output_format="html", **{**BASE, "max_items": 1}) + client.wait(own.crawl_id) + src = own.crawl_id + r = first_fetched(src) + if not r: + return FAIL, f"no result with content_url on {src}", "" + cid = r.content_url.rstrip("/").rsplit("/", 1)[-1] + by_id, by_url = client.content(src, cid), client.content(src, r.content_url) + ok = by_id == by_url and re.search(r"= 0 else 1, 30)) + + +crawls = {} +ledger = os.path.join(OUT, "results", "crawls.jsonl") +if os.path.exists(ledger): + with open(ledger) as f: + for line in f: + if line.strip(): + row = json.loads(line) + entry = crawls.setdefault(row["crawl_id"], {"purposes": []}) + if row["purpose"] not in entry["purposes"]: + entry["purposes"].append(row["purpose"]) + +rows = [] +for crawl_id, entry in crawls.items(): + stopped = False + try: + status, body, _ = call("GET", f"/crawls/{crawl_id}?limit=1") + if status == 200 and body and body.get("status") == "running": + stopped = stop(crawl_id) < 300 + time.sleep(2) + status, body, _ = call("GET", f"/crawls/{crawl_id}?limit=1") + except (OSError, http.client.HTTPException, ValueError) as e: + # One unreachable crawl must not keep the others running. + rows.append( + {"crawl_id": crawl_id, **entry, "stopped_by_cleanup": stopped, "error": repr(e)} + ) + print(f"{crawl_id} not checked: {e!r}") + continue + row = {"crawl_id": crawl_id, **entry, "stopped_by_cleanup": stopped, "http": status} + if status == 200 and body: + row.update( + status=body.get("status"), + stop_reason=body.get("stop_reason"), + pages_fetched=body.get("coverage", {}).get("pages_fetched"), + output_format=body.get("output_format"), + ) + rows.append(row) + print( + f"{crawl_id} {row.get('status')} pages_fetched={row.get('pages_fetched')}" + f"{' (stopped by clean-up)' if stopped else ''}" + ) + +still = [r["crawl_id"] for r in rows if r.get("status") == "running" or "error" in r] +with open(os.path.join(OUT, "cleanup.json"), "w") as f: + json.dump( + { + "crawls": rows, + "still_running": still, + "checked_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + }, + f, + indent=2, + ) +print(f"clean-up: {len(rows)} crawls, {len(still)} still running") diff --git a/acceptance/crawl/tools/report.py b/acceptance/crawl/tools/report.py new file mode 100644 index 0000000..37eccf4 --- /dev/null +++ b/acceptance/crawl/tools/report.py @@ -0,0 +1,285 @@ +"""Turn out/results/checks/*.json into out/crawl-first-install-report.md and out/results.json. + +Also computes check 17: the names this surface uses against the API's names. +""" + +import glob +import json +import os +import re +import sys + +OUT = sys.argv[1] +SURFACE = os.environ["SURFACE"] +REPO = os.environ["REPO"] +NAMES = { + "node": "Node SDK (`zenrows` on npm)", + "python": "Python SDK (`zenrows` on PyPI)", + "go": "Go SDK (`github.com/zenrows/zenrows-go-sdk/service/crawl`)", + "mcp": "MCP server (`@zenrows/mcp`)", + "cli": "CLI (`@zenrows/cli`)", +} +CHECK_TITLES = { + "1": "Create", + "2": "Create with html", + "3": "Idempotency", + "4": "Wait", + "5": "Wait runs out", + "6": "Get", + "7": "List", + "8": "Results", + "9": "Content", + "10": "Download", + "11": "Stop", + "12": "Not found", + "13": "Invalid input", + "14": "Gated option", + "15": "Bad key", + "16": "Errors carry the server code", + "17": "Naming", + "18": "Copy", + "19": "Failed crawl", + "20": "Results after stop", + "21": "Documented defaults", +} +# The API's operation and parameter names. Each surface uses them in its own casing. +CANON_OPS = ["create", "get", "list", "results", "content", "download", "stop", "wait"] +CANON_PARAMS = [ + "url", + "depth", + "max_items", + "max_pages", + "include_patterns", + "exclude_patterns", + "output_format", + "idempotency_key", + "cursor", + "limit", + "timeout", + "content_id", +] + + +def read(path, default=None): + try: + with open(path) as f: + return f.read().strip() + except OSError: + return default + + +def snake(name): + n = re.sub(r"^crawl[._ ]", "", name.strip()) + n = n.replace("()", "").replace("<", "").replace(">", "").lstrip("-").replace("-", "_") + return re.sub(r"(?<=[a-z0-9])(?=[A-Z])", "_", n).lower() + + +def naming(rows): + p = os.path.join(OUT, "results", "naming.json") + if not os.path.exists(p): + return + with open(p) as f: + d = json.load(f) + diffs = [] + for op in CANON_OPS: + got = d["operations"].get(op) + if got is None: + diffs.append(f"operation `{op}` missing") + elif snake(got) != op: + diffs.append(f"operation `{op}` is `{got}`") + for prm in CANON_PARAMS: + got = d["params"].get(prm) + if got is None: + diffs.append(f"parameter `{prm}` missing") + elif snake(got) != prm: + diffs.append(f"parameter `{prm}` is `{got}`") + rows["17"] = { + "surface": SURFACE, + "check": "17", + "title": "Naming", + "status": "FAIL" if diffs else "PASS", + "expected": "the API's operation and parameter names, in this surface's casing", + "actual": "; ".join(diffs) or "matches", + "evidence": json.dumps({k: d[k] for k in ("operations", "params")}), + } + + +def severity(check): + if check in ("17", "18", "21") or check.startswith("x-"): + return "minor" + if check.startswith("readme") or check == "build": + return "blocker" + return "major" + + +def first_line(s, n=110): + line = re.sub(r"\s+", " ", (s or "").split("\n")[0]) + return line if len(line) <= n else line[: n - 1] + "…" + + +def main(): + rows = {} + for p in sorted(glob.glob(os.path.join(OUT, "results", "checks", "*.json"))): + with open(p) as f: + r = json.load(f) + rows[r["check"]] = r + naming(rows) + env = json.loads(read(os.path.join(OUT, "results", "env.json"), "{}") or "{}") + cleanup = json.loads(read(os.path.join(OUT, "cleanup.json"), "null") or "null") + sha = read(os.path.join(OUT, "build", "sha"), "-") + origin = read(os.path.join(OUT, "build", "origin"), "-") + artifacts = [ + os.path.basename(p) for p in glob.glob(os.path.join(OUT, "build", "artifact", "*")) + ] + gomod = read(os.path.join(OUT, "results", "go-module.txt")) + + findings = [ + {"check": c, "severity": severity(c), "r": r} + for c, r in rows.items() + if r["status"] == "FAIL" + ] + order = {"blocker": 0, "major": 1, "minor": 2} + findings.sort(key=lambda f: order[f["severity"]]) + blocking = [f for f in findings if f["severity"] != "minor"] + counts = {} + for r in rows.values(): + counts[r["status"]] = counts.get(r["status"], 0) + 1 + + lines = [] + w = lines.append + w(f"# Crawl first-install test: {NAMES[SURFACE]}\n") + w( + f"Run `{read(os.path.join(OUT, 'run_id'), '-')}`. The client was built as a release " + "artifact, installed in a clean container, set up from its README, then tested against " + "the live Crawl API.\n" + ) + + w("## 1. Verdict\n") + if blocking: + w( + "**Not ready.** " + + "; ".join( + f"{f['check']} {f['r']['title']}: {first_line(f['r']['actual'], 90)}" + for f in blocking + ) + ) + else: + w("**Ready.**" + (f" {len(findings)} minor finding(s), see Findings." if findings else "")) + w("\n" + ", ".join(f"{v} {k}" for k, v in sorted(counts.items())) + ".\n") + + w("## 2. Environment\n") + w(f"- Command: `{os.environ.get('CMDLINE') or 'bash run.sh'}`") + w(f"- Repo: ZenRows/{REPO}; source: {origin}; SHA tested: `{sha}`") + w(f"- Artifact: {', '.join(artifacts) or ('go get ' + gomod.split()[0] if gomod else '-')}") + w( + f"- `CRAWL_URL`: {os.environ.get('CRAWL_URL')}; " + f"`CRAWL_INCLUDE`: `{os.environ.get('CRAWL_INCLUDE')}`; " + f"API: {os.environ.get('CRAWL_API_BASE')}" + ) + image = { + "node": "NODE_IMAGE", + "mcp": "NODE_IMAGE", + "cli": "NODE_IMAGE", + "python": "PYTHON_IMAGE", + "go": "GO_IMAGE", + }[SURFACE] + w( + f"- Image: `{os.environ.get(image)}`; container: " + + ", ".join(f"{k} {v}" for k, v in env.items()) + ) + tools = { + "node": f"typescript {os.environ.get('TYPESCRIPT_VERSION')}, " + f"pnpm {os.environ.get('PNPM_VERSION')}", + "mcp": f"@modelcontextprotocol/sdk {os.environ.get('MCP_SDK_VERSION')}", + "cli": "none", + "python": f"mypy {os.environ.get('MYPY_VERSION')}, " + f"build {os.environ.get('PY_BUILD_VERSION')}", + "go": "none", + }[SURFACE] + w(f"- Pinned tools: {tools}\n") + + w("## 3. First-time install\n") + for check, r in rows.items(): + if check.startswith("readme") or check == "build": + w(f"- **{r['status']}** {r['title']}: {first_line(r['actual'], 200)}") + ev = r.get("evidence") or "" + if ev and (r["status"] != "PASS" or "substitutions" in ev): + w("\n ```\n " + "\n ".join(ev.splitlines()[:16]) + "\n ```") + w("") + + w("## 4. Feature matrix\n") + w("| # | Check | Result |") + w("| --- | --- | --- |") + for c in [str(i) for i in range(1, 22)]: + r = rows.get(c) + cell = f"**{r['status']}** {first_line(r['actual'])}".replace("|", "\\|") if r else "—" + w(f"| {c} | {CHECK_TITLES[c]} | {cell} |") + for c, r in rows.items(): + if c.startswith("x-"): + w( + f"| extra | {r['title']} | " + + f"**{r['status']}** {first_line(r['actual'])}".replace("|", "\\|") + + " |" + ) + w("") + + w("## 5. Findings\n") + if not findings: + w("None.\n") + for i, f in enumerate(findings, 1): + r = f["r"] + w(f"### {i}. {r['title']} ({f['severity']})\n") + w(f"- What happened: {r['actual']}") + w(f"- Expected: {r['expected']}") + w( + "- Evidence:\n\n ```\n " + + "\n ".join((r.get("evidence") or "").splitlines()[:20]) + + "\n ```\n" + ) + + w("## 6. Cost\n") + if cleanup and cleanup["crawls"]: + w("| Crawl | Purpose | Final status | pages_fetched |") + w("| --- | --- | --- | --- |") + total = 0 + for c in cleanup["crawls"]: + total += c.get("pages_fetched") or 0 + html = " (html)" if c.get("output_format") == "html" else "" + reason = "/" + c["stop_reason"] if c.get("stop_reason") else "" + w( + f"| `{c['crawl_id']}` | {', '.join(c['purposes'])}{html} | " + f"{c.get('status')}{reason} | {c.get('pages_fetched')} |" + ) + w(f"\nTotal pages fetched: **{total}**.\n") + else: + w("No crawls.\n") + + w("## 7. Clean-up\n") + if cleanup and cleanup["crawls"]: + stopped = [c["crawl_id"] for c in cleanup["crawls"] if c.get("stopped_by_cleanup")] + w( + f"Checked at {cleanup['checked_at']}: " + f"{len(cleanup['crawls'])} crawls started by this run, " + f"{', '.join(cleanup['still_running']) or 'none'} still running. " + + ( + f"Clean-up stopped {', '.join(stopped)}." + if stopped + else "Clean-up had nothing to stop." + ) + + "\n" + ) + else: + w("No crawls to clean up.\n") + + with open(os.path.join(OUT, "crawl-first-install-report.md"), "w") as f: + f.write("\n".join(lines)) + with open(os.path.join(OUT, "results.json"), "w") as f: + json.dump(list(rows.values()), f, indent=2) + print( + f"{SURFACE}: " + + ", ".join(f"{k} {v}" for k, v in sorted(counts.items())) + + f"; findings: {len(findings)}" + ) + + +main() diff --git a/acceptance/crawl/tools/scan_key.py b/acceptance/crawl/tools/scan_key.py new file mode 100644 index 0000000..e332651 --- /dev/null +++ b/acceptance/crawl/tools/scan_key.py @@ -0,0 +1,25 @@ +"""Fail the run if the API key appears in any file under out/. Reads the key from the env.""" + +import os +import sys + +key = os.environ.get("ZENROWS_API_KEY", "").encode() +# The copy of the source under test holds the repo, not anything the run wrote. +source_dir = os.path.join(sys.argv[1], "build", "source") +hits = [] +for root, dirs, files in os.walk(sys.argv[1]): + if root == source_dir: + dirs.clear() + continue + for f in files: + p = os.path.join(root, f) + try: + with open(p, "rb") as fh: + if key and key in fh.read(): + hits.append(p) + except OSError: + pass +if hits: + print("THE API KEY APPEARS IN:", *hits, sep="\n ") + sys.exit(1) +print("key scan: the key appears in no result, log or report file") diff --git a/acceptance/crawl/versions.env b/acceptance/crawl/versions.env new file mode 100644 index 0000000..dc1f030 --- /dev/null +++ b/acceptance/crawl/versions.env @@ -0,0 +1,23 @@ +# Pinned inputs for the Crawl acceptance test. Every value can be overridden from the environment. +SURFACE=python +REPO=zenrows-python-sdk + +# The official images, pinned by digest so two runs agree. +NODE_IMAGE="${NODE_IMAGE:-node:20@sha256:8f693eaa7e0a8e71560c9a82b55fd54c2ae920a2ba5d2cde28bac7d1c01c9ba5}" +PYTHON_IMAGE="${PYTHON_IMAGE:-python:3.12-slim@sha256:a6e34c598f2467ed0e9a8d349809fcd8b5c603269512df273a0bb1784edc11b1}" +GO_IMAGE="${GO_IMAGE:-golang:1.25@sha256:699337d620559a59b4a2bb298ad59611e535d2ee755a34cf2d2a98f37578dc80}" + +# Tools the build and the checks install. +PNPM_VERSION="${PNPM_VERSION:-9.4.0}" +PY_BUILD_VERSION="${PY_BUILD_VERSION:-1.2.2.post1}" +MCP_SDK_VERSION="${MCP_SDK_VERSION:-1.27.1}" +TYPESCRIPT_VERSION="${TYPESCRIPT_VERSION:-5.7.3}" +MYPY_VERSION="${MYPY_VERSION:-1.15.0}" + +# What to crawl. +CRAWL_URL="${CRAWL_URL:-https://www.scrapingcourse.com/ecommerce/}" +CRAWL_INCLUDE="${CRAWL_INCLUDE:-/product/}" +CRAWL_API_BASE="${CRAWL_API_BASE:-https://api.zenrows.com/v1}" + +# REF unset tests this checkout. Set it to a SHA, branch or tag to test that commit from GitHub. +REF="${REF:-}" diff --git a/examples/11_crawl.py b/examples/11_crawl.py new file mode 100644 index 0000000..1832256 --- /dev/null +++ b/examples/11_crawl.py @@ -0,0 +1,78 @@ +"""11: Crawl a site → wait → read URLs and pages. + +Demonstrates the Crawl client (`ZenRowsCrawlClient`): + - `create(url, depth=..., include_patterns=..., output_format="html")` + starts a crawl and returns at once, `running`. + - `wait(crawl_id)` polls until it ends, or 600 s pass, and returns + the `Crawl` (still `running` on timeout; `error` set when `failed`). + - `results(crawl_id)` pages through the kept URLs. + - `content(crawl_id, result)` reads one kept URL's HTML. + - `download(crawl_id)` reads every result as NDJSON, with the crawl's + status; the example saves the lines to a file. + +Run with: + export ZENROWS_API_KEY=zr_... + python examples/11_crawl.py --url https://example.com/products/ +""" + +import argparse +import json +import os + +from zenrows import ZenRowsCrawlClient +from zenrows.crawl import CRAWL_NOT_ENABLED_CODE, CrawlAPIError, CrawlStatus + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--url", required=True, help="the page to start from") + parser.add_argument("--depth", type=int, default=1) + parser.add_argument("--max-items", type=int, default=5) + parser.add_argument( + "--include", action="append", help="keep URLs containing this (default /product/)" + ) + parser.add_argument("--out", default="crawl.jsonl") + args = parser.parse_args() + + client = ZenRowsCrawlClient(api_key=os.environ["ZENROWS_API_KEY"]) + + try: + crawl = client.create( + args.url, + depth=args.depth, + max_items=args.max_items, + include_patterns=args.include or ["/product/"], + output_format="html", + ) + except CrawlAPIError as exc: + if exc.code == CRAWL_NOT_ENABLED_CODE: + raise SystemExit("Crawl is not enabled for this account.") from exc + raise + print(f"started {crawl.crawl_id}") + + crawl = client.wait(crawl.crawl_id) + cov = crawl.coverage + print( + f"{crawl.crawl_id} {crawl.status.value}: {cov.items_found} URLs kept, " + f"{cov.pages_fetched} pages fetched ({cov.pages_failed} failed)" + ) + if crawl.status is CrawlStatus.FAILED and crawl.error: + raise SystemExit(f"crawl failed: {crawl.error.code.value}: {crawl.error.detail}") + if crawl.status is CrawlStatus.RUNNING: + print("still running: the results below are what it has kept so far") + + for result in client.results(crawl.crawl_id): + status = result.content_status.value if result.content_status else "-" + print(f" [{status}] {result.url}") + if result.content_url: + html = client.content(crawl.crawl_id, result) + print(f" {len(html)} chars of HTML") + + with client.download(crawl.crawl_id) as download, open(args.out, "w") as f: + for line in download.lines: + f.write(json.dumps(line.model_dump(mode="json", exclude_none=True)) + "\n") + print(f"wrote {args.out} (crawl {download.status.value if download.status else '-'})") + + +if __name__ == "__main__": + main() diff --git a/examples/README.md b/examples/README.md index ca93890..555f5b4 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,6 +1,7 @@ # Examples -Runnable samples for the Batch API client (`ZenRowsBatchClient`). +Runnable samples for the Batch API client (`ZenRowsBatchClient`) and +the Crawl API client (`ZenRowsCrawlClient`, `11_crawl.py`). Each script reads the API key from the environment: ```bash @@ -27,6 +28,7 @@ uv run python examples/02_download_to_dir.py --job-id 01J... | `08_download_all_results.py` | `download_all_results()` — server-side export zip of a whole run | | `09_scheduled_jobs.py` | scheduled cadences (`Rate`/`Calendar`/`At`), `pause`/`resume`/`update_schedule`, per-fire runs, `retry_failed` | | `10_lightweight_handles.py` | `client.job(id)` / `client.run(id, rid)` — act on a known id without a GET (`.data` lazily fetches) | +| `11_crawl.py` | Crawl: `create` → `wait` → `results` + `content` → `download` (NDJSON) | The samples are deliberately small — each demonstrates one feature end-to-end so they double as living documentation. diff --git a/pyproject.toml b/pyproject.toml index f697fb2..4cb04f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ packages = ["src/zenrows"] [tool.hatch.build.targets.sdist] include = ["src/zenrows", "README.md", "LICENSE"] +exclude = ["/acceptance"] # the acceptance test harness, never shipped # --- uv --- [tool.uv] @@ -112,5 +113,9 @@ exclude = ["src/zenrows/batch/models.py"] # --- pytest --- [tool.pytest.ini_options] testpaths = ["tests"] -addopts = "-ra --strict-markers" +# e2e tests hit a live API; `make test-e2e` selects them with `-m e2e`. +addopts = "-ra --strict-markers -m 'not e2e'" asyncio_mode = "auto" +markers = [ + "e2e: end-to-end tests against a live ZenRows API (opt-in, see DEVELOPMENT.md)", +] diff --git a/src/zenrows/__init__.py b/src/zenrows/__init__.py index 9bbf8c1..f9e7159 100644 --- a/src/zenrows/__init__.py +++ b/src/zenrows/__init__.py @@ -1,6 +1,6 @@ """Zenrows Python SDK. -Two clients live here: +Three clients live here: - `ZenRowsClient` — the original synchronous scraping API client. Backward-compatible with pre-1.4 usage; freshened-up internals. @@ -8,10 +8,20 @@ - `ZenRowsBatchClient` — the new async-job / batch API client. Built on a generated OpenAPI core with a hand-written ergonomic facade on top. + + - `ZenRowsCrawlClient` — the Crawl API client: start a crawl from + one URL and read back the URLs (and pages) it finds. """ from zenrows.__version__ import __version__ from zenrows.batch import ZenRowsBatchClient from zenrows.client import ZenRowsClient +from zenrows.crawl import CrawlAPIError, ZenRowsCrawlClient -__all__ = ["ZenRowsBatchClient", "ZenRowsClient", "__version__"] +__all__ = [ + "CrawlAPIError", + "ZenRowsBatchClient", + "ZenRowsClient", + "ZenRowsCrawlClient", + "__version__", +] diff --git a/src/zenrows/batch/_transport.py b/src/zenrows/batch/_transport.py index 20e45c5..b29ccc7 100644 --- a/src/zenrows/batch/_transport.py +++ b/src/zenrows/batch/_transport.py @@ -4,10 +4,19 @@ User-Agent, automatic retries for transient failures, and the RFC 7807 → `BatchAPIError` mapping. The facade (`client.py`) is thin glue over this — one method per endpoint, all typed via pydantic v2 models. + +The Crawl client (`zenrows.crawl`) reuses it with its own error type +and logger: both APIs share the key header, problem+json errors and +retry policy. """ +import json +import logging +import math import random import time +from collections.abc import Callable, Iterator +from contextlib import contextmanager from typing import Any, TypeVar import httpx @@ -32,6 +41,9 @@ # capped at 10s. _BACKOFF_BASE_MS = 250 _BACKOFF_CAP_MS = 10_000 +# A `Retry-After` above this waits only this long, so a large server value +# cannot hang a call. +_RETRY_AFTER_CAP_MS = 30_000 _DEFAULT_RETRIES = 3 @@ -46,7 +58,8 @@ def _backoff_ms(attempt: int) -> float: def _retry_after_ms(response: httpx.Response) -> float | None: - """Parse a `Retry-After` header (delta-seconds only) to milliseconds.""" + """Parse a `Retry-After` header (delta-seconds only) to milliseconds, + at most `_RETRY_AFTER_CAP_MS`.""" raw = response.headers.get("Retry-After") if not raw: return None @@ -54,7 +67,9 @@ def _retry_after_ms(response: httpx.Response) -> float | None: secs = float(raw) except ValueError: return None - return secs * 1000 if secs >= 0 else None + if not math.isfinite(secs) or secs < 0: + return None + return min(secs * 1000, _RETRY_AFTER_CAP_MS) class _Transport: @@ -78,8 +93,15 @@ def __init__( retries: int = _DEFAULT_RETRIES, verify: bool | str = True, httpx_args: dict[str, Any] | None = None, + error_from_response: Callable[[httpx.Response], Exception] = BatchAPIError.from_response, + log: logging.Logger = _log, ): + """`error_from_response` turns a non-2xx response into the + exception to raise; it should expose `.code` and `.detail` for + the error log. `log` receives the per-request records.""" self._retries = max(0, retries) + self._error_from_response = error_from_response + self._log = log self._client = httpx.Client( base_url=base_url.rstrip("/"), headers={ @@ -102,17 +124,18 @@ def _send( params: dict[str, Any] | None, headers: dict[str, str] | None, content: bytes | None = None, + retry_statuses: frozenset[int] = _RETRYABLE_STATUSES, ) -> httpx.Response: """Issue the request, retrying transient failures on idempotent requests. - Retries `{429, 502, 503, 504}` and transient network errors up - to `retries` times, with jittered exponential backoff (honoring - `Retry-After` when present). Only idempotent requests are - replayed — `GET`/`PUT`/`DELETE`/`HEAD`/`OPTIONS`, plus `POST` - when the caller supplied an `Idempotency-Key`. Our own timeouts - (`httpx.TimeoutException`) are never retried: the caller set - that budget. + Retries `retry_statuses` (default `{429, 502, 503, 504}`) and + transient network errors up to `retries` times, with jittered + exponential backoff (honoring `Retry-After` up to 30s). Only + idempotent requests are replayed — `GET`/`PUT`/`DELETE`/`HEAD`/ + `OPTIONS`, plus `POST` when the caller supplied an + `Idempotency-Key`. Our own timeouts (`httpx.TimeoutException`) + are never retried: the caller set that budget. """ idempotent = method.upper() in _IDEMPOTENT_METHODS or ( method.upper() == "POST" and _has_idempotency_key(headers) @@ -134,11 +157,7 @@ def _send( continue raise - if ( - idempotent - and attempt < self._retries - and response.status_code in _RETRYABLE_STATUSES - ): + if idempotent and attempt < self._retries and response.status_code in retry_statuses: wait_ms = _retry_after_ms(response) or _backoff_ms(attempt) response.close() time.sleep(wait_ms / 1000) @@ -185,11 +204,26 @@ def request_json( payload = body.model_dump( mode="json", exclude_unset=True, exclude_none=True, by_alias=True ) - import json - content = json.dumps(payload).encode("utf-8") headers = {**(headers or {}), "Content-Type": "application/json"} + response = self.request(method, path, params=params, headers=headers, content=content) + if response.status_code == 204 or not response.content: + return None + return response.json() + + def request( + self, + method: str, + path: str, + *, + params: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, + content: bytes | None = None, + retry_statuses: frozenset[int] = _RETRYABLE_STATUSES, + ) -> httpx.Response: + """Send a request (with retries) and return the raw response, + raising the mapped error on non-2xx. For non-JSON bodies.""" start = time.monotonic() response = self._send( method, @@ -197,27 +231,44 @@ def request_json( params=_drop_none(params), headers=headers, content=content, + retry_statuses=retry_statuses, ) elapsed_ms = (time.monotonic() - start) * 1000 - - if response.status_code >= 400: - err = BatchAPIError.from_response(response) - log_error( - _log, - method=method, - path=path, - status=response.status_code, - code=err.code, - detail=err.problem.detail if err.problem else None, - ) - raise err - + self._raise_for_status(method, path, response) log_request( - _log, method=method, path=path, status=response.status_code, elapsed_ms=elapsed_ms + self._log, method=method, path=path, status=response.status_code, elapsed_ms=elapsed_ms ) - if response.status_code == 204 or not response.content: - return None - return response.json() + return response + + @contextmanager + def stream( + self, + method: str, + path: str, + *, + params: dict[str, Any] | None = None, + ) -> Iterator[httpx.Response]: + """Open a streamed response, raising the mapped error on + non-2xx. Not retried: a stream may be partly consumed.""" + with self._client.stream(method, path, params=_drop_none(params)) as response: + if response.status_code >= 400: + response.read() + self._raise_for_status(method, path, response) + yield response + + def _raise_for_status(self, method: str, path: str, response: httpx.Response) -> None: + if response.status_code < 400: + return + err = self._error_from_response(response) + log_error( + self._log, + method=method, + path=path, + status=response.status_code, + code=getattr(err, "code", "internal"), + detail=getattr(err, "detail", None), + ) + raise err def _drop_none(d: dict[str, Any] | None) -> dict[str, Any] | None: diff --git a/src/zenrows/crawl/__init__.py b/src/zenrows/crawl/__init__.py new file mode 100644 index 0000000..696a9d7 --- /dev/null +++ b/src/zenrows/crawl/__init__.py @@ -0,0 +1,55 @@ +"""Public surface for the ZenRows Crawl API (New). + +Crawl is still evolving: new features are coming, limits may be tuned, +and the changelog announces each change. + +What's where: + - `client.ZenRowsCrawlClient` — the typed facade: create, get, list, + results, content, download, stop, and wait. + - `models` — hand-written pydantic v2 response models with open enums. + - `errors.CrawlAPIError` — RFC 9457 problem+json mapping. + +`create(output_format="html")` also returns each kept URL's page; +without it a crawl returns URLs only. +""" + +from zenrows.crawl.client import ( + CrawlDownload, + ZenRowsCrawlClient, +) +from zenrows.crawl.errors import CRAWL_NOT_ENABLED_CODE, CrawlAPIError +from zenrows.crawl.models import ( + ContentStatus, + Coverage, + Crawl, + CrawlErrorCode, + CrawlList, + CrawlResult, + CrawlStatus, + CrawlStop, + CrawlWithResults, + DownloadLine, + OutputFormat, + RunError, + StopReason, +) + +__all__ = [ + "CRAWL_NOT_ENABLED_CODE", + "ContentStatus", + "Coverage", + "Crawl", + "CrawlAPIError", + "CrawlDownload", + "CrawlErrorCode", + "CrawlList", + "CrawlResult", + "CrawlStatus", + "CrawlStop", + "CrawlWithResults", + "DownloadLine", + "OutputFormat", + "RunError", + "StopReason", + "ZenRowsCrawlClient", +] diff --git a/src/zenrows/crawl/client.py b/src/zenrows/crawl/client.py new file mode 100644 index 0000000..20bf213 --- /dev/null +++ b/src/zenrows/crawl/client.py @@ -0,0 +1,375 @@ +"""ZenRowsCrawlClient — the typed facade over the Crawl API (New). + +A crawl starts from one URL, follows links (up to `depth` hops when +set), and keeps the URLs that match its patterns, up to `max_items`. It +stops at `max_items` or `max_pages`, or when no link is left. It runs as +a job: `create` returns at once with `status=running`, and the URLs +grow as pages finish. + + client = ZenRowsCrawlClient(api_key) + crawl = client.create("https://example.com/shop/", depth=1, + include_patterns=["/product/"]) + client.wait(crawl.crawl_id) + for result in client.results(crawl.crawl_id): + print(result.url) + +Requests go through the SDK's shared HTTP transport (auth header, +retries, problem+json mapping), which raises `CrawlAPIError`. +""" + +import json +import logging +import math +import random +import time +from collections.abc import Callable, Iterator +from contextlib import ExitStack +from typing import Any, Literal + +import httpx + +from zenrows.__version__ import __version__ +from zenrows.batch._transport import _RETRYABLE_STATUSES, _Transport +from zenrows.crawl.errors import CrawlAPIError +from zenrows.crawl.models import ( + Crawl, + CrawlList, + CrawlResult, + CrawlStatus, + CrawlStop, + CrawlWithResults, + DownloadLine, +) + +_DEFAULT_BASE_URL = "https://api.zenrows.com/v1" +DEFAULT_USER_AGENT = f"zenrows-crawl-python/{__version__}" + +_log = logging.getLogger("zenrows.crawl.transport") + +# 429 `too_many_crawls` is a capacity limit, not a transient failure. +_CREATE_RETRY_STATUSES = _RETRYABLE_STATUSES - {429} + +# A stop is safe to repeat (an ended crawl answers as it ended), so +# `stop` retries these statuses itself even though it is a POST without +# an idempotency key. 503 includes `crawl_busy`: nothing was saved and +# the crawl is still running. +_STOP_RETRY_STATUSES = _RETRYABLE_STATUSES +# The longest a `Retry-After` may make a stop wait, so a large server +# value cannot hang it. +_MAX_STOP_RETRY_AFTER = 30.0 + +_POLL_INTERVAL = 2.0 +_POLL_BACKOFF = 1.5 +_MAX_POLL_INTERVAL = 15.0 + + +class CrawlDownload: + """A crawl's NDJSON download, open and ready to read. + + `status` is the crawl's status when the file was read + (`X-Crawl-Status`): `running` means the file holds only what the + crawl has kept so far. Iterate `lines` once for the parsed lines; + the connection closes when the iteration ends. Use it as a context + manager to close it without reading to the end. + """ + + def __init__(self, response: httpx.Response, close: Callable[[], None]): + self.status: CrawlStatus | None = _crawl_status(response) + self._response = response + self._close = close + + @property + def lines(self) -> Iterator[DownloadLine]: + return self._lines() + + def _lines(self) -> Iterator[DownloadLine]: + try: + for line in self._response.iter_lines(): + if line.strip(): + yield DownloadLine.model_validate_json(line) + finally: + self.close() + + def close(self) -> None: + self._close() + + def __enter__(self) -> "CrawlDownload": + return self + + def __exit__(self, *exc: object) -> None: + self.close() + + +class ZenRowsCrawlClient: + """Synchronous, typed client for the ZenRows Crawl API (New). + + `api_key` is required: + + client = ZenRowsCrawlClient(api_key=os.environ["ZENROWS_API_KEY"]) + + Every non-2xx raises `CrawlAPIError`; branch on its `status_code` + and `code`. An account without Crawl enabled gets 403 `REQS008` + (`CRAWL_NOT_ENABLED_CODE`). + """ + + def __init__( + self, + api_key: str, + *, + base_url: str | None = None, + timeout: float | httpx.Timeout = 30.0, + retries: int = 3, + verify_ssl: bool | str = True, + user_agent: str = DEFAULT_USER_AGENT, + httpx_args: dict[str, Any] | None = None, + ): + """`base_url` defaults to `https://api.zenrows.com/v1`. `retries` bounds + automatic retries of transient failures (HTTP 429/502/503/504 + and network errors) on idempotent requests — GETs, and a + `create` that carries an `idempotency_key` (never on 429 for + `create`). Retries use jittered exponential backoff and honor + `Retry-After`; set `retries=0` to disable. It also bounds the + retries of a `stop` answered 429, 502, 503 or 504.""" + if not api_key: + raise ValueError("ZenRowsCrawlClient: api_key is required.") + self._retries = max(0, retries) + self._t = _Transport( + base_url=base_url or _DEFAULT_BASE_URL, + api_key=api_key, + user_agent=user_agent, + timeout=timeout, + retries=retries, + verify=verify_ssl, + httpx_args=httpx_args, + error_from_response=CrawlAPIError.from_response, + log=_log, + ) + + # ----- lifecycle ----- + + def __enter__(self) -> "ZenRowsCrawlClient": + return self + + def __exit__(self, *exc: object) -> None: + self.close() + + def close(self) -> None: + self._t.close() + + @property + def base_url(self) -> str: + return self._t.base_url + + # ===== crawls ===== + + def create( + self, + url: str, + *, + depth: int | None = None, + max_items: int | None = None, + max_pages: int | None = None, + include_patterns: list[str] | None = None, + exclude_patterns: list[str] | None = None, + output_format: Literal["html"] | None = None, + idempotency_key: str | None = None, + ) -> Crawl: + """`POST /crawls` — start a crawl and return it, `running`. + + - `depth`: link hops to follow from `url` (1 to 100,000). + Omit it for no depth limit: the crawl follows links until + `max_items` or `max_pages` stops it, or no link is left. + - `max_items`: stop after keeping this many URLs (server + default 100). + - `max_pages`: stop after fetching this many pages, which + bounds the cost (server default 100). + - `include_patterns` / `exclude_patterns`: substrings matched + against each normalized URL. A URL is kept when it matches + an include pattern (if any) and no exclude pattern. A crawl + stays on the start URL's registrable domain; subdomains count. + - `output_format="html"`: also fetch every kept URL's page, + readable with `content` / `download`. Omit for URLs only. + - `idempotency_key`: a retry with the same key and body answers + with the crawl the first request created. + + Only the arguments you pass are sent. When the account has + reached its limit of active jobs (3 by default), shared with its + Batch jobs, `create` raises 429 `too_many_crawls` without + retrying; retry after the error's `retry_after` seconds. + """ + body: dict[str, Any] = {"url": url} + optional: dict[str, Any] = { + "depth": depth, + "max_items": max_items, + "max_pages": max_pages, + "include_patterns": include_patterns, + "exclude_patterns": exclude_patterns, + "output_format": output_format, + } + body.update({k: v for k, v in optional.items() if v is not None}) + headers = {"Content-Type": "application/json"} + if idempotency_key: + headers["Idempotency-Key"] = idempotency_key + response = self._t.request( + "POST", + "/crawls", + headers=headers, + content=json.dumps(body).encode("utf-8"), + retry_statuses=_CREATE_RETRY_STATUSES, + ) + return Crawl.model_validate(response.json()) + + def get( + self, + crawl_id: str, + *, + cursor: str | None = None, + limit: int | None = None, + ) -> CrawlWithResults: + """`GET /crawls/{crawl_id}` — the crawl and one page of its + results (`limit` 1 to 10,000, server default 1,000). Pass the + page's `next_cursor` as `cursor` for the URLs kept since. For + every result, prefer `results`.""" + return CrawlWithResults.model_validate( + self._t.request_json( + "GET", f"/crawls/{crawl_id}", params={"cursor": cursor, "limit": limit} + ) + ) + + def list( + self, + *, + cursor: str | None = None, + limit: int | None = None, + ) -> CrawlList: + """`GET /crawls` — one page of the account's crawls, newest + first (`limit` 1 to 100, server default 20), without results. + Pass the page's `next_cursor` as `cursor` for the next page; it + is None on the last page.""" + return CrawlList.model_validate( + self._t.request_json("GET", "/crawls", params={"cursor": cursor, "limit": limit}) + ) + + def results( + self, + crawl_id: str, + *, + limit: int | None = None, + ) -> Iterator[CrawlResult]: + """Yield the URLs the crawl has kept, in order, each once. + + Follows `next_cursor`, `limit` URLs per request, and returns at + the first empty page or null cursor. It does not poll: on a + running crawl it yields only what the crawl has kept so far. + Call `wait` first to read every result. + """ + cursor: str | None = None + while True: + page = self.get(crawl_id, cursor=cursor, limit=limit) + yield from page.results + if not page.results or page.next_cursor is None: + return + cursor = page.next_cursor + + def stop(self, crawl_id: str) -> CrawlStop: + """`POST /crawls/{crawl_id}/stop` — stop a running crawl. + + Idempotent: a crawl that already ended answers as it ended. + The answer carries no counts; read the final coverage and + results with `get` once pages already in flight have finished. + + Because repeating a stop is safe, `stop` retries a 429, 502, + 503 (including `crawl_busy`, which means the stop was not saved + and the crawl is still running) or 504 up to `retries` times, + waiting `Retry-After` seconds (at most 30; 1 when missing or + unusable) each time, then raises the last error. Network errors + are not retried. + """ + attempt = 0 + while True: + try: + return CrawlStop.model_validate( + self._t.request_json("POST", f"/crawls/{crawl_id}/stop") + ) + except CrawlAPIError as err: + if err.status_code not in _STOP_RETRY_STATUSES or attempt >= self._retries: + raise + time.sleep(_stop_retry_wait(err.retry_after)) + attempt += 1 + + # ===== contents ===== + + def content(self, crawl_id: str, content_id: str | CrawlResult) -> str: + """`GET /crawls/{crawl_id}/contents/{content_id}` — one kept + URL's page, as HTML text. + + `content_id` is a `CrawlResult` whose `content_status` is + `fetched`, its `content_url`, or its content id (the last + segment of the `content_url`). Only crawls created with an + `output_format` have contents; anything else is 404 + `content_not_found`. + """ + if isinstance(content_id, CrawlResult): + if not content_id.content_url: + raise ValueError( + f"content: {content_id.url} has no content_url " + f"(content_status={content_id.content_status})" + ) + content_id = content_id.content_url + segment = content_id.rstrip("/").rsplit("/", 1)[-1] + return self._t.request("GET", f"/crawls/{crawl_id}/contents/{segment}").text + + def download(self, crawl_id: str) -> CrawlDownload: + """`GET /crawls/{crawl_id}/download` — open the NDJSON file and + return it as a `CrawlDownload`: iterate its `lines` for parsed + `DownloadLine`s (url, content_status, content), and read its + `status`. + + On a running crawl the file holds what the crawl has kept so + far, and `status` is `running`; call `wait` first for all of it. + """ + stack = ExitStack() + response = stack.enter_context(self._t.stream("GET", f"/crawls/{crawl_id}/download")) + return CrawlDownload(response, stack.close) + + # ===== waiter ===== + + def wait(self, crawl_id: str, *, timeout: float = 600.0) -> Crawl: + """Poll the crawl until it ends (`completed`, `stopped` or + `failed`) or `timeout` seconds pass, and return it. + + A `failed` crawl is returned, not raised: read `error` on it. + On timeout the crawl is returned still `running`; call `wait` + again to keep waiting, or `stop` to end it. + """ + deadline = time.monotonic() + timeout + interval = _POLL_INTERVAL + while True: + # limit=1 keeps each poll cheap; results are not returned. + page = self.get(crawl_id, limit=1) + crawl = Crawl.model_validate(page.model_dump(exclude={"results", "next_cursor"})) + remaining = deadline - time.monotonic() + if crawl.status is not CrawlStatus.RUNNING or remaining <= 0: + return crawl + time.sleep(min(interval * random.uniform(0.8, 1.2), remaining)) + interval = min(interval * _POLL_BACKOFF, _MAX_POLL_INTERVAL) + + +def _stop_retry_wait(retry_after: float | None) -> float: + """Seconds `stop` waits after a 429/502/503/504: the server's + `Retry-After`, 1s when it is missing or unusable, at most 30s.""" + if retry_after is None or not math.isfinite(retry_after) or retry_after < 0: + return 1.0 + return min(retry_after, _MAX_STOP_RETRY_AFTER) + + +def _crawl_status(response: httpx.Response) -> CrawlStatus | None: + status = response.headers.get("X-Crawl-Status") + return CrawlStatus(status) if status else None + + +__all__ = [ + "DEFAULT_USER_AGENT", + "CrawlDownload", + "ZenRowsCrawlClient", +] diff --git a/src/zenrows/crawl/errors.py b/src/zenrows/crawl/errors.py new file mode 100644 index 0000000..2b70fb4 --- /dev/null +++ b/src/zenrows/crawl/errors.py @@ -0,0 +1,84 @@ +"""RFC 9457 problem+json → `CrawlAPIError`. + +The Crawl API answers errors as `application/problem+json` with a +stable `code`. Branch on +`status_code` + `code`; display `detail`, don't parse it. +""" + +import json +import math + +import httpx + +from zenrows.batch.errors import ProblemDetail + +# The code of the 403 answer to a key whose account does not have +# Crawl enabled. +CRAWL_NOT_ENABLED_CODE = "REQS008" + + +def _problem_code(raw: bytes) -> str | None: + # Read from the body: Batch's `ProblemDetail.code` defaults a missing + # code to "internal". + try: + code = json.loads(raw).get("code") + except (ValueError, AttributeError): + return None + return code if isinstance(code, str) else None + + +class CrawlAPIError(Exception): + """A non-2xx response from the Crawl API. + + `code` is the problem's `code` member: `crawl_not_found`, + `content_not_found`, `invalid_parameter`, `too_many_crawls`, ..., + `REQS008` when Crawl is not enabled for the account, and the API's + usual auth and credit codes; None when the body has no code. + `problem` holds the whole body, or None when it was not JSON. + """ + + def __init__( + self, + status_code: int, + problem: ProblemDetail | None, + raw: bytes, + *, + retry_after: float | None = None, + ): + self.status_code = status_code + # Seconds the server asked to wait (`Retry-After`), e.g. on 429 + # `too_many_crawls`. + self.retry_after = retry_after + self.problem = problem + self.raw = raw + self.code: str | None = _problem_code(raw) if problem else None + self.detail: str | None = problem.detail if problem else None + if problem and self.code == CRAWL_NOT_ENABLED_CODE: + msg = f"{status_code} Crawl is not enabled for this account ({self.code})" + ( + f": {problem.detail}" if problem.detail else "" + ) + elif problem: + reason = problem.detail or self.code + msg = f"{status_code} {problem.title}" + (f": {reason}" if reason else "") + else: + msg = f"{status_code} (no problem body)" + super().__init__(msg) + + @classmethod + def from_response(cls, response: httpx.Response) -> "CrawlAPIError": + try: + retry_after: float | None = float(response.headers.get("Retry-After", "")) + except ValueError: + retry_after = None + # A negative, NaN or infinite value is no wait a caller can sleep for. + if retry_after is not None and (not math.isfinite(retry_after) or retry_after < 0): + retry_after = None + return cls( + status_code=response.status_code, + problem=ProblemDetail.from_response(response), + raw=response.content, + retry_after=retry_after, + ) + + +__all__ = ["CRAWL_NOT_ENABLED_CODE", "CrawlAPIError"] diff --git a/src/zenrows/crawl/models.py b/src/zenrows/crawl/models.py new file mode 100644 index 0000000..e1ab707 --- /dev/null +++ b/src/zenrows/crawl/models.py @@ -0,0 +1,185 @@ +"""Pydantic v2 models for the Crawl API's responses. + +Hand-written: the surface is a handful of small schemas. + +Forward compatible: unknown response fields +are ignored (pydantic's default), and every enum is open: a value the +server adds later parses as an `UNKNOWN` member that keeps the raw +value (see `zenrows.batch._open_enum`). +""" + +from datetime import datetime +from enum import Enum +from typing import Any + +from pydantic import BaseModel + +from zenrows.batch._open_enum import open_enum_missing + + +class CrawlStatus(Enum): + """Where a crawl stands. `running` until it ends; the other values + are terminal. Clients must accept values not listed here.""" + + RUNNING = "running" + COMPLETED = "completed" + STOPPED = "stopped" + FAILED = "failed" + + _missing_ = classmethod(open_enum_missing) + + +class StopReason(Enum): + """What ended the crawl before nothing was left to open: a limit + (`max_items` / `max_pages`, with `completed`) or the caller (`user`, + with `stopped`).""" + + MAX_ITEMS = "max_items" + MAX_PAGES = "max_pages" + USER = "user" + + _missing_ = classmethod(open_enum_missing) + + +class ContentStatus(Enum): + """Where a kept URL's page stands, when the crawl has an + `output_format`.""" + + PENDING = "pending" + FETCHED = "fetched" + FAILED = "failed" + + _missing_ = classmethod(open_enum_missing) + + +class OutputFormat(Enum): + """The format the crawl returns each kept URL's page in.""" + + HTML = "html" + + _missing_ = classmethod(open_enum_missing) + + +class CrawlErrorCode(Enum): + """Why a crawl failed (`Crawl.error.code`).""" + + INSUFFICIENT_CREDITS = "insufficient_credits" + SEED_UNREACHABLE = "seed_unreachable" + DOMAIN_NOT_ALLOWED = "domain_not_allowed" + NO_ITEMS_FOUND = "no_items_found" + INTERNAL_ERROR = "internal_error" + + _missing_ = classmethod(open_enum_missing) + + +class RunError(BaseModel): + """Why a crawl failed. Present only when `status` is `failed`; part + of the crawl, not an error response.""" + + code: CrawlErrorCode + detail: str + + +class Coverage(BaseModel): + """How far the crawl has got.""" + + pages_fetched: int + """Pages fetched so far, successful or not.""" + pages_failed: int + """Fetched pages whose fetch failed (not charged).""" + items_found: int + """URLs kept so far, bounded by `max_items`.""" + + +class Crawl(BaseModel): + """One crawl, with the parameters it runs with.""" + + crawl_id: str + status: CrawlStatus + url: str + """The page the crawl started from.""" + depth: int | None = None + """Link hops the crawl follows from `url`. Absent when the crawl was + created without one: it has no depth limit.""" + max_items: int + max_pages: int + coverage: Coverage + created_at: datetime + stop_reason: StopReason | None = None + error: RunError | None = None + include_patterns: list[str] | None = None + exclude_patterns: list[str] | None = None + output_format: OutputFormat | None = None + duplicates_removed: int | None = None + """Links met again and dropped as already seen.""" + finished_at: datetime | None = None + """When the crawl reached a terminal status. Absent while it runs.""" + + +class CrawlResult(BaseModel): + """One URL the crawl kept.""" + + url: str + content_status: ContentStatus | None = None + """Present only when the crawl has an `output_format`.""" + content_url: str | None = None + """The page's path (e.g. `/v1/crawls/c_x/contents/ct_y`), present + once `content_status` is `fetched`. Pass the result itself to + `ZenRowsCrawlClient.content`.""" + + +class CrawlWithResults(Crawl): + """A crawl and one page of the URLs it has kept (`GET /crawls/{id}`). + + `next_cursor` is never null while the crawl runs: polling with it + returns only URLs kept since. It is null once the crawl has ended + and this page holds its last URLs.""" + + results: list[CrawlResult] + next_cursor: str | None = None + + +class CrawlList(BaseModel): + """One page of the account's crawls, newest first. `next_cursor` is + absent on the last page.""" + + crawls: list[Crawl] + next_cursor: str | None = None + + +class CrawlStop(BaseModel): + """What `stop` answers: where the crawl stands, without counts. Read + the coverage and results with `get`.""" + + crawl_id: str + status: CrawlStatus + stop_reason: StopReason | None = None + error: RunError | None = None + finished_at: datetime | None = None + + +class DownloadLine(BaseModel): + """One line of a crawl's NDJSON download.""" + + url: str + content_status: ContentStatus | None = None + content: str | dict[str, Any] | None = None + """The page, present when `content_status` is `fetched`: HTML text, + or an object for a crawl whose output format returns parsed data.""" + + +__all__ = [ + "ContentStatus", + "Coverage", + "Crawl", + "CrawlErrorCode", + "CrawlList", + "CrawlResult", + "CrawlStatus", + "CrawlStop", + "CrawlWithResults", + "DownloadLine", + "OutputFormat", + "RunError", + "StopReason", +] diff --git a/src/zenrows/py.typed b/src/zenrows/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/tests/e2e/test_crawl_e2e.py b/tests/e2e/test_crawl_e2e.py new file mode 100644 index 0000000..0e95212 --- /dev/null +++ b/tests/e2e/test_crawl_e2e.py @@ -0,0 +1,108 @@ +"""End-to-end test of ZenRowsCrawlClient against a live Crawl API. + +Opt-in: runs only with `make test-e2e` (pytest `-m e2e`), and only when +`ZENROWS_API_KEY` (a key with Crawl access), `ZENROWS_CRAWL_BASE_URL` +(the API base, e.g. https://api.zenrows.com/v1) and +`ZENROWS_E2E_CRAWL_URL` (the page to crawl from) are set. Otherwise it is +skipped, so `make test` stays offline. When `ZENROWS_E2E_CRAWL_INCLUDE` +is set, the crawl keeps only URLs containing it, and the test checks +that every result does. It starts one small crawl (a few credits) and +reads it back every way the client can. +""" + +import os +import time + +import pytest + +from zenrows.crawl import CrawlAPIError, CrawlStatus, ZenRowsCrawlClient + +_REQUIRED = ("ZENROWS_API_KEY", "ZENROWS_CRAWL_BASE_URL", "ZENROWS_E2E_CRAWL_URL") + +pytestmark = [ + pytest.mark.e2e, + pytest.mark.skipif( + not all(os.environ.get(v) for v in _REQUIRED), + reason="set " + ", ".join(_REQUIRED) + " to run the Crawl e2e test", + ), +] + +START_URL = os.environ.get("ZENROWS_E2E_CRAWL_URL", "") +INCLUDE = os.environ.get("ZENROWS_E2E_CRAWL_INCLUDE") or None +# An account has a limit of active jobs (3 by default, shared with its +# Batch jobs), so a create can meet 429 too_many_crawls while others finish. +SLOT_WAIT_SECONDS = 300 + + +@pytest.fixture(scope="module") +def client(): + with ZenRowsCrawlClient( + api_key=os.environ["ZENROWS_API_KEY"], base_url=os.environ["ZENROWS_CRAWL_BASE_URL"] + ) as c: + yield c + + +def create_when_a_slot_frees(client: ZenRowsCrawlClient): + deadline = time.monotonic() + SLOT_WAIT_SECONDS + while True: + try: + return client.create( + START_URL, + depth=1, + max_items=3, + max_pages=5, + include_patterns=[INCLUDE] if INCLUDE else None, + output_format="html", + ) + except CrawlAPIError as err: + if err.code != "too_many_crawls" or time.monotonic() >= deadline: + raise + wait = err.retry_after or 30.0 + print(f"too_many_crawls: retrying create in {wait:.0f}s") + time.sleep(wait) + + +def test_crawl_end_to_end(client: ZenRowsCrawlClient): + created = create_when_a_slot_frees(client) + print(f"created {created.crawl_id} status={created.status.value}") + assert created.crawl_id + + crawl = client.wait(created.crawl_id, timeout=600) + print( + f"finished {crawl.crawl_id} status={crawl.status.value} " + f"stop_reason={crawl.stop_reason and crawl.stop_reason.value} " + f"pages_fetched={crawl.coverage.pages_fetched} items_found={crawl.coverage.items_found}" + ) + assert crawl.status is CrawlStatus.COMPLETED, crawl.error + + results = list(client.results(crawl.crawl_id)) + print(f"results: {len(results)}") + assert results + if INCLUDE: + assert all(INCLUDE in r.url for r in results), [r.url for r in results] + + fetched = [r for r in results if r.content_url] + assert fetched, "no result has a fetched page" + html = client.content(crawl.crawl_id, fetched[0]) + print(f"content of {fetched[0].url}: {len(html)} chars") + assert " ZenRowsCrawlClient: + return ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=0) + + +@pytest.fixture +def no_sleep(monkeypatch): + """Record + swallow sleeps (waiter and retry backoff both call + `time.sleep`) so tests run instantly.""" + slept: list[float] = [] + monkeypatch.setattr("time.sleep", slept.append) + return slept + + +# ----- construction ----- + + +def test_missing_api_key_raises(): + with pytest.raises(ValueError): + ZenRowsCrawlClient(api_key="") + + +def test_base_url_is_a_constructor_option_only(monkeypatch): + monkeypatch.setenv("ZENROWS_CRAWL_BASE_URL", "http://env-set/v1") + assert ZenRowsCrawlClient(api_key=API_KEY).base_url.rstrip("/") == "https://api.zenrows.com/v1" + assert ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL).base_url.rstrip("/") == BASE_URL + + +# ----- create ----- + + +@respx.mock +def test_create_sends_only_required_fields(client: ZenRowsCrawlClient): + route = respx.post(f"{BASE_URL}/crawls").mock( + return_value=Response( + 202, headers={"Location": "/v1/crawls/c_1"}, json=crawl_body(omit=("depth",)) + ) + ) + + crawl = client.create("https://example.com/") + + assert crawl.crawl_id == "c_1" + assert crawl.status is CrawlStatus.RUNNING + assert crawl.depth is None # created without depth: no depth limit, none echoed + sent = route.calls.last.request + assert sent.headers["X-API-Key"] == API_KEY + assert sent.headers["Content-Type"] == "application/json" + assert "Idempotency-Key" not in sent.headers + assert json.loads(sent.content) == {"url": "https://example.com/"} + + +@respx.mock +def test_create_sends_depth_when_given(client: ZenRowsCrawlClient): + route = respx.post(f"{BASE_URL}/crawls").mock(return_value=Response(202, json=crawl_body())) + + crawl = client.create("https://example.com/", depth=1) + + assert crawl.depth == 1 + assert json.loads(route.calls.last.request.content) == { + "url": "https://example.com/", + "depth": 1, + } + + +@respx.mock +def test_create_sends_every_option_and_idempotency_key(client: ZenRowsCrawlClient): + route = respx.post(f"{BASE_URL}/crawls").mock( + return_value=Response(202, json=crawl_body(output_format="html")) + ) + + crawl = client.create( + "https://example.com/", + depth=2, + max_items=3, + max_pages=5, + include_patterns=["/product/"], + exclude_patterns=["/cart"], + output_format="html", + idempotency_key="idem-1", + ) + + assert crawl.output_format is OutputFormat.HTML + sent = route.calls.last.request + assert sent.headers["Idempotency-Key"] == "idem-1" + body = json.loads(sent.content) + assert body == { + "url": "https://example.com/", + "depth": 2, + "max_items": 3, + "max_pages": 5, + "include_patterns": ["/product/"], + "exclude_patterns": ["/cart"], + "output_format": "html", + } + assert "discovery" not in body + + +# ----- get / list / stop ----- + + +@respx.mock +def test_get_parses_results_and_open_enums(client: ZenRowsCrawlClient): + route = respx.get(f"{BASE_URL}/crawls/c_1").mock( + return_value=Response( + 200, + json=crawl_body( + status="completed", + stop_reason="max_items", + output_format="xml", + finished_at="2026-10-08T09:02:00Z", + some_new_field=True, + results=[ + { + "url": "https://example.com/product/a", + "content_status": "fetched", + "content_url": "/v1/crawls/c_1/contents/ct_a", + }, + {"url": "https://example.com/product/b", "content_status": "archived"}, + ], + next_cursor=None, + ), + ) + ) + + page = client.get("c_1", cursor="cur_1", limit=50) + + assert route.calls.last.request.url.params["cursor"] == "cur_1" + assert route.calls.last.request.url.params["limit"] == "50" + assert page.status is CrawlStatus.COMPLETED + assert page.stop_reason is StopReason.MAX_ITEMS + assert is_unknown(page.output_format) # type: ignore[arg-type] + assert page.next_cursor is None + assert page.results[0].content_status is ContentStatus.FETCHED + assert page.results[0].content_url == "/v1/crawls/c_1/contents/ct_a" + unknown = page.results[1].content_status + assert unknown is not None and is_unknown(unknown) and unknown.value == "archived" + + +@respx.mock +def test_get_parses_failed_crawl_error(client: ZenRowsCrawlClient): + respx.get(f"{BASE_URL}/crawls/c_1").mock( + return_value=Response( + 200, + json=crawl_body( + status="failed", + error={"code": "seed_unreachable", "detail": "The site may be blocking it."}, + results=[], + next_cursor=None, + ), + ) + ) + page = client.get("c_1") + assert page.status is CrawlStatus.FAILED + assert page.error is not None and page.error.code is CrawlErrorCode.SEED_UNREACHABLE + + +@respx.mock +def test_get_omits_unset_query_params(client: ZenRowsCrawlClient): + route = respx.get(f"{BASE_URL}/crawls/c_1").mock( + return_value=Response(200, json=crawl_body(results=[], next_cursor="cur_0")) + ) + client.get("c_1") + assert dict(route.calls.last.request.url.params) == {} + + +@respx.mock +def test_list_returns_one_page(client: ZenRowsCrawlClient): + route = respx.get(f"{BASE_URL}/crawls").mock( + side_effect=[ + Response(200, json={"crawls": [crawl_body(crawl_id="c_2")], "next_cursor": "n1"}), + Response(200, json={"crawls": [crawl_body(crawl_id="c_1", omit=("depth",))]}), + ] + ) + + first = client.list(limit=1) + last = client.list(cursor=first.next_cursor, limit=1) + + assert [c.crawl_id for c in first.crawls] == ["c_2"] + assert [c.crawl_id for c in last.crawls] == ["c_1"] + assert first.crawls[0].depth == 1 + assert last.crawls[0].depth is None # created without depth + assert last.next_cursor is None + assert dict(route.calls[0].request.url.params) == {"limit": "1"} + assert route.calls[1].request.url.params["cursor"] == "n1" + + +@respx.mock +def test_stop_posts_without_body(client: ZenRowsCrawlClient): + route = respx.post(f"{BASE_URL}/crawls/c_1/stop").mock( + return_value=Response( + 200, + json={ + "crawl_id": "c_1", + "status": "stopped", + "stop_reason": "user", + "finished_at": "2026-10-08T09:01:00Z", + }, + ) + ) + + stopped = client.stop("c_1") + + assert stopped.status is CrawlStatus.STOPPED + assert stopped.stop_reason is StopReason.USER + assert route.calls.last.request.content == b"" + + +@respx.mock +def test_stop_retries_crawl_busy_after_retry_after(no_sleep): + client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=2) + route = respx.post(f"{BASE_URL}/crawls/c_1/stop").mock( + side_effect=[ + problem(503, "crawl_busy", headers={"Retry-After": "1"}), + Response(200, json={"crawl_id": "c_1", "status": "stopped", "stop_reason": "user"}), + ] + ) + + stopped = client.stop("c_1") + + assert stopped.status is CrawlStatus.STOPPED + assert route.call_count == 2 + assert no_sleep == [1.0] + + +@respx.mock +def test_stop_surfaces_crawl_busy_after_retries(no_sleep): + client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=2) + route = respx.post(f"{BASE_URL}/crawls/c_1/stop").mock( + return_value=problem(503, "crawl_busy", headers={"Retry-After": "1"}) + ) + + with pytest.raises(CrawlAPIError) as exc_info: + client.stop("c_1") + + assert exc_info.value.status_code == 503 + assert exc_info.value.code == "crawl_busy" + assert route.call_count == 3 + assert no_sleep == [1.0, 1.0] + + +@pytest.mark.parametrize( + ("status", "code"), + [(429, "rate_limited"), (502, "bad_gateway"), (503, "service_unavailable"), (504, None)], +) +@respx.mock +def test_stop_retries_transient_status(no_sleep, status: int, code: str | None): + client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=2) + first = ( + problem(status, code, headers={"Retry-After": "2"}) + if code + else Response(status, headers={"Retry-After": "2"}) + ) + route = respx.post(f"{BASE_URL}/crawls/c_1/stop").mock( + side_effect=[ + first, + Response(200, json={"crawl_id": "c_1", "status": "stopped", "stop_reason": "user"}), + ] + ) + + stopped = client.stop("c_1") + + assert stopped.status is CrawlStatus.STOPPED + assert route.call_count == 2 + assert no_sleep == [2.0] + + +@respx.mock +def test_stop_surfaces_502_after_retries(no_sleep): + client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=3) + route = respx.post(f"{BASE_URL}/crawls/c_1/stop").mock(return_value=problem(502, "bad_gateway")) + + with pytest.raises(CrawlAPIError) as exc_info: + client.stop("c_1") + + assert exc_info.value.status_code == 502 + assert route.call_count == 4 + assert no_sleep == [1.0, 1.0, 1.0] + + +@respx.mock +def test_stop_does_not_retry_other_status(no_sleep): + client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=2) + route = respx.post(f"{BASE_URL}/crawls/c_1/stop").mock( + return_value=problem(500, "internal_error") + ) + + with pytest.raises(CrawlAPIError): + client.stop("c_1") + + assert route.call_count == 1 + assert no_sleep == [] + + +# ----- results scanner ----- + + +@respx.mock +def test_results_stops_on_null_cursor(client: ZenRowsCrawlClient): + route = respx.get(f"{BASE_URL}/crawls/c_1").mock( + side_effect=[ + Response(200, json=crawl_body(results=[{"url": "https://a"}], next_cursor="k1")), + Response( + 200, + json=crawl_body( + status="completed", results=[{"url": "https://b"}], next_cursor=None + ), + ), + ] + ) + + urls = [r.url for r in client.results("c_1", limit=1)] + + assert urls == ["https://a", "https://b"] + assert route.call_count == 2 + assert dict(route.calls[0].request.url.params) == {"limit": "1"} + assert route.calls[1].request.url.params["cursor"] == "k1" + + +@respx.mock +def test_results_returns_at_first_empty_page_of_running_crawl(client: ZenRowsCrawlClient, no_sleep): + route = respx.get(f"{BASE_URL}/crawls/c_1").mock( + side_effect=[ + Response(200, json=crawl_body(results=[{"url": "https://a"}], next_cursor="k1")), + # Caught up with a running crawl: no results, same cursor back. + Response(200, json=crawl_body(results=[], next_cursor="k1")), + ] + ) + + urls = [r.url for r in client.results("c_1")] + + assert urls == ["https://a"] + assert route.call_count == 2 + assert no_sleep == [] + + +# ----- contents / download ----- + + +@respx.mock +def test_content_accepts_result_url_or_id(client: ZenRowsCrawlClient): + route = respx.get(f"{BASE_URL}/crawls/c_1/contents/ct_a").mock( + return_value=Response( + 200, headers={"Content-Type": "text/html"}, text="product" + ) + ) + result = CrawlResult( + url="https://a", content_status="fetched", content_url="/v1/crawls/c_1/contents/ct_a" + ) # type: ignore[arg-type] + + assert client.content("c_1", result) == "product" + assert client.content("c_1", "/v1/crawls/c_1/contents/ct_a") == "product" + assert client.content("c_1", "ct_a") == "product" + assert route.call_count == 3 + + +def test_content_without_content_url_raises(client: ZenRowsCrawlClient): + with pytest.raises(ValueError, match="content_url"): + client.content("c_1", CrawlResult(url="https://a", content_status="pending")) # type: ignore[arg-type] + + +NDJSON = ( + b'{"url":"https://a","content_status":"fetched","content":"a"}\n' + b'{"url":"https://b","content_status":"failed"}\n' + b'{"url":"https://c","content_status":"fetched","content":{"title":"C"}}\n' +) + + +@respx.mock +def test_download_parses_lines_and_status(client: ZenRowsCrawlClient): + respx.get(f"{BASE_URL}/crawls/c_1/download").mock( + return_value=Response( + 200, + headers={"Content-Type": "application/x-ndjson", "X-Crawl-Status": "completed"}, + content=NDJSON, + ) + ) + download = client.download("c_1") + assert download.status is CrawlStatus.COMPLETED + lines = list(download.lines) + assert [line.url for line in lines] == ["https://a", "https://b", "https://c"] + assert lines[0].content == "a" + assert lines[1].content_status is ContentStatus.FAILED + assert lines[2].content == {"title": "C"} + + +@respx.mock +def test_download_without_status_header(client: ZenRowsCrawlClient): + respx.get(f"{BASE_URL}/crawls/c_1/download").mock(return_value=Response(200, content=b"")) + with client.download("c_1") as download: + assert download.status is None + assert list(download.lines) == [] + + +@respx.mock +def test_download_maps_errors(client: ZenRowsCrawlClient): + respx.get(f"{BASE_URL}/crawls/nope/download").mock(return_value=problem(404, "crawl_not_found")) + with pytest.raises(CrawlAPIError) as exc_info: + client.download("nope") + assert exc_info.value.code == "crawl_not_found" + + +# ----- errors ----- + + +@respx.mock +def test_not_found_maps_to_crawl_api_error(client: ZenRowsCrawlClient): + respx.get(f"{BASE_URL}/crawls/nope").mock(return_value=problem(404, "crawl_not_found")) + with pytest.raises(CrawlAPIError) as exc_info: + client.get("nope") + err = exc_info.value + assert err.status_code == 404 + assert err.code == "crawl_not_found" + assert err.detail == "detail for crawl_not_found" + assert err.code != CRAWL_NOT_ENABLED_CODE + + +@respx.mock +def test_validation_error_maps_to_crawl_api_error(client: ZenRowsCrawlClient): + respx.post(f"{BASE_URL}/crawls").mock(return_value=problem(422, "invalid_start_url")) + with pytest.raises(CrawlAPIError) as exc_info: + client.create("http://10.0.0.1/", depth=1) + assert exc_info.value.status_code == 422 + assert exc_info.value.code == "invalid_start_url" + + +@respx.mock +def test_too_many_crawls_carries_retry_after(client: ZenRowsCrawlClient): + respx.post(f"{BASE_URL}/crawls").mock( + return_value=problem(429, "too_many_crawls", headers={"Retry-After": "30"}) + ) + with pytest.raises(CrawlAPIError) as exc_info: + client.create("https://example.com/", depth=1) + assert exc_info.value.code == "too_many_crawls" + assert exc_info.value.retry_after == 30.0 + + +@respx.mock +def test_crawl_not_enabled_has_clear_message(client: ZenRowsCrawlClient): + respx.post(f"{BASE_URL}/crawls").mock( + return_value=problem( + 403, + "REQS008", + title="Crawl is not enabled for this account.", + type="https://docs.zenrows.com/api-error-codes#REQS008", + ) + ) + with pytest.raises(CrawlAPIError) as exc_info: + client.create("https://example.com/", depth=1) + err = exc_info.value + assert err.code == CRAWL_NOT_ENABLED_CODE + assert "Crawl is not enabled for this account" in str(err) + + +@pytest.mark.parametrize( + "response", + [Response(502, text="bad gateway"), Response(500, json={"title": "Server error"})], +) +@respx.mock +def test_error_without_code_has_none_code(client: ZenRowsCrawlClient, response): + respx.get(f"{BASE_URL}/crawls/c_1").mock(return_value=response) + with pytest.raises(CrawlAPIError) as exc_info: + client.get("c_1") + assert exc_info.value.status_code == response.status_code + assert exc_info.value.code is None + + +@respx.mock +def test_keyed_create_does_not_retry_too_many_crawls(no_sleep): + client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=2) + route = respx.post(f"{BASE_URL}/crawls").mock( + return_value=problem(429, "too_many_crawls", headers={"Retry-After": "1"}) + ) + with pytest.raises(CrawlAPIError) as exc_info: + client.create("https://example.com/", depth=1, idempotency_key="idem-1") + assert exc_info.value.code == "too_many_crawls" + assert route.call_count == 1 + assert no_sleep == [] + + +@respx.mock +def test_keyed_create_retries_transient_status(no_sleep): + client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=2) + route = respx.post(f"{BASE_URL}/crawls").mock( + side_effect=[Response(503), Response(202, json=crawl_body())] + ) + client.create("https://example.com/", depth=1, idempotency_key="idem-1") + assert route.call_count == 2 + assert len(no_sleep) == 1 + + +@respx.mock +def test_get_retries_transient_status(no_sleep): + client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=2) + route = respx.get(f"{BASE_URL}/crawls/c_1").mock( + side_effect=[Response(503), Response(200, json=crawl_body(results=[], next_cursor="k"))] + ) + client.get("c_1") + assert route.call_count == 2 + assert len(no_sleep) == 1 + + +# ----- waiter ----- + + +@respx.mock +def test_wait_polls_until_terminal(client: ZenRowsCrawlClient, no_sleep): + route = respx.get(f"{BASE_URL}/crawls/c_1").mock( + side_effect=[ + Response(200, json=crawl_body(results=[], next_cursor="k")), + Response( + 200, + json=crawl_body( + status="completed", + finished_at="2026-10-08T09:02:00Z", + results=[{"url": "https://a"}], + next_cursor="k2", + ), + ), + ] + ) + + crawl = client.wait("c_1") + + assert crawl.status is CrawlStatus.COMPLETED + assert not hasattr(crawl, "results") + assert route.call_count == 2 + assert all(c.request.url.params["limit"] == "1" for c in route.calls) + assert len(no_sleep) == 1 + + +@respx.mock +def test_wait_returns_running_crawl_on_timeout(client: ZenRowsCrawlClient, monkeypatch): + route = respx.get(f"{BASE_URL}/crawls/c_1").mock( + return_value=Response(200, json=crawl_body(results=[], next_cursor="k")) + ) + now = [0.0] + slept: list[float] = [] + + def sleep(seconds: float) -> None: + slept.append(seconds) + now[0] += seconds + + monkeypatch.setattr("time.monotonic", lambda: now[0]) + monkeypatch.setattr("time.sleep", sleep) + + crawl = client.wait("c_1", timeout=25) + + assert crawl.status is CrawlStatus.RUNNING + assert route.call_count == len(slept) + 1 + assert sum(slept) == pytest.approx(25) + assert max(slept) <= 15.0 * 1.2 + + +@respx.mock +def test_wait_returns_failed_crawl(client: ZenRowsCrawlClient, no_sleep): + respx.get(f"{BASE_URL}/crawls/c_1").mock( + return_value=Response( + 200, + json=crawl_body( + status="failed", + error={"code": "no_items_found", "detail": "No URL matched."}, + results=[], + next_cursor=None, + ), + ) + ) + crawl = client.wait("c_1") + assert crawl.status is CrawlStatus.FAILED + assert crawl.error is not None and crawl.error.code is CrawlErrorCode.NO_ITEMS_FOUND + assert no_sleep == [] + + +@pytest.mark.parametrize( + ("retry_after", "slept"), [("3600", 30.0), ("-1", 1.0), ("nan", 1.0), ("2", 2.0)] +) +@pytest.mark.parametrize(("status", "code"), [(503, "crawl_busy"), (429, "rate_limited")]) +@respx.mock +def test_stop_waits_at_most_30_seconds( + no_sleep, status: int, code: str, retry_after: str, slept: float +): + client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=1) + respx.post(f"{BASE_URL}/crawls/c_1/stop").mock( + side_effect=[ + problem(status, code, headers={"Retry-After": retry_after}), + Response(200, json={"crawl_id": "c_1", "status": "stopped", "stop_reason": "user"}), + ] + ) + client.stop("c_1") + assert no_sleep == [slept] + + +@respx.mock +def test_retry_after_above_30_seconds_waits_30_seconds(no_sleep): + client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=2) + respx.get(f"{BASE_URL}/crawls/c_1").mock( + side_effect=[ + problem(429, "rate_limited", headers={"Retry-After": "3600"}), + Response(200, json=crawl_body(status="completed", results=[], next_cursor=None)), + ] + ) + client.get("c_1") + assert no_sleep == [30.0] + + +@pytest.mark.parametrize("raw", ["-1", "nan", "inf", "soon"]) +@respx.mock +def test_an_unusable_retry_after_is_not_reported(client: ZenRowsCrawlClient, raw: str): + respx.post(f"{BASE_URL}/crawls").mock( + return_value=problem(429, "too_many_crawls", headers={"Retry-After": raw}) + ) + with pytest.raises(CrawlAPIError) as exc_info: + client.create("https://example.com/", depth=1) + assert exc_info.value.retry_after is None + + +def test_crawl_api_error_imports_from_the_package_root(): + import zenrows + from zenrows.crawl import CrawlAPIError as FromCrawl + + assert zenrows.CrawlAPIError is FromCrawl + + +@respx.mock +def test_content_takes_content_id_by_keyword(client: ZenRowsCrawlClient): + respx.get(f"{BASE_URL}/crawls/c_1/contents/ct_9").mock( + return_value=Response(200, text="") + ) + assert client.content("c_1", content_id="/v1/crawls/c_1/contents/ct_9") == ""