diff --git a/acceptance/crawl/.gitignore b/acceptance/crawl/.gitignore new file mode 100644 index 0000000..89f9ac0 --- /dev/null +++ b/acceptance/crawl/.gitignore @@ -0,0 +1 @@ +out/ diff --git a/acceptance/crawl/README.md b/acceptance/crawl/README.md new file mode 100644 index 0000000..9a45870 --- /dev/null +++ b/acceptance/crawl/README.md @@ -0,0 +1,111 @@ +# Crawl first-install acceptance test: Python SDK + +This folder tests the Python SDK (`zenrows` on PyPI) the way a new customer meets it. It builds the release +artifact, installs it in a clean container with nothing else, follows the README's Crawl +quickstart, and checks every Crawl feature against the live API. The result is one report. + +It tests this repo only, so it runs on its own. + +## What it needs + +- bash, git and Docker. Nothing else on the host. +- Network access to GitHub, the package registries and the Crawl API. +- A Zenrows API key with Crawl access, in `ZENROWS_API_KEY`. + +## How to run it + +From the repo root: + +```bash +export ZENROWS_API_KEY="$(cat path/to/key-file)" # keep it out of your shell history +bash acceptance/crawl/run.sh # build, run every check, write the report +bash acceptance/crawl/run.sh --no-key # only the checks that need no key; the live checks are SKIP +bash acceptance/crawl/run.sh --report # rebuild the report from out/ +REF=v1.2.3 bash acceptance/crawl/run.sh # test a tag, branch or SHA from GitHub instead of this checkout +``` + +Without `ZENROWS_API_KEY`, `run.sh` stops and asks for it. The report is +`acceptance/crawl/out/crawl-first-install-report.md`; `out/` is not committed. + +Every input is an environment variable with a default in `versions.env`: `CRAWL_URL` +(default `https://www.scrapingcourse.com/ecommerce/`), `CRAWL_INCLUDE` (default `/product/`), +`REF`, `CRAWL_API_BASE`, the images (`node:20`, `python:3.12-slim`, `golang:1.25`, pinned by +digest) and the tool versions. `OUT_DIR` moves the output. + +## What a run does + +1. **Source.** By default, this checkout, including changes not yet committed. With `REF`, the + repo at that ref from GitHub. +2. **Build.** In its own container, the release artifact: `pip install` of the wheel `python -m build` makes. +3. **Clean test container.** It gets only the artifact (read-only), the checks in `surface/` + (read-only), the inputs, and the key by name (`-e ZENROWS_API_KEY`). It has no source, no + caches from the build, and no tokens. +4. **README first.** It runs the README's Crawl quickstart as written. The only changes are the + ones the cost rules need: the key, `CRAWL_URL`, `CRAWL_INCLUDE`, `max_pages` 10 or less, and + no html. The report lists each change. +5. **Checks 1 to 21 and the extras.** Each writes one JSON result: + `{surface, check, title, status, expected, actual, evidence}`, with `status` PASS, FAIL or + SKIP. The evidence is the command and its trimmed output, with the key redacted. +6. **Clean-up.** It stops every crawl the run started and reads each one's final status and + `coverage.pages_fetched`. It runs even when a check fails, or on Ctrl-C. +7. **Report.** Verdict, environment, install steps, feature matrix, findings, cost and + clean-up. Last, the run checks that the key appears in no file under `out/`. + +Some checks don't use the live API, so they behave the same on every run. They point the +client at a small local stand-in server that records what it sends. These are: "only the +fields you set are sent", "the error code is empty when the body has none", and the timing +and signal checks. + +## The checks + +| # | Feature | Expected | +| --- | --- | --- | +| 1 | Create | returns at once: `running`, a `crawl_id`, the echoed parameters; only the fields set are sent | +| 2 | Create with html | results get `content_status` and, once fetched, `content_url` | +| 3 | Idempotency | same key and body → same `crawl_id`; different body → 422 `idempotency_key_reused` (MCP, CLI: `CRAWL_INVALID_REQUEST` with the advice to send a new key) | +| 4 | Wait | returns the ended crawl; `failed` is returned, not raised (CLI: exit 1) | +| 5 | Wait runs out | a 1 s timeout on a running crawl returns it `running`, no error (CLI: exit 0; MCP, CLI: a `note`) | +| 6 | Get | `limit` 2, then the cursor: one page each; `next_cursor` moves on and is null on the last page | +| 7 | List | `limit` 2, then the next page: newest first; `next_cursor` absent or null on the last page | +| 8 | Results | every kept URL once, each matching `CRAWL_INCLUDE`; on a running crawl, what is kept so far (MCP, CLI: `partial: true`) | +| 9 | Content | by content id and by `content_url`: the same HTML | +| 10 | Download | a status, and one line per result with `url`, `content_status`, `content` | +| 11 | Stop | running → `stopped`, `stop_reason: user`; ended → answers as it ended | +| 12 | Not found | 404 `crawl_not_found` (MCP, CLI: `CRAWL_NOT_FOUND` with `crawl_id` and `server_code`) | +| 13 | Invalid input | `depth` 0 → 422 `invalid_parameter` (MCP, CLI: `CRAWL_INVALID_REQUEST`) | +| 14 | Gated option | `output_format` json → 422 `invalid_parameter`; if it is accepted, the key is in a beta segment, not a bug | +| 15 | Bad key | the key `invalid` → 401 (MCP, CLI: `AUTH_INVALID`) | +| 16 | Errors carry the server code | SDKs: `code` holds it, empty when the body has none; MCP, CLI: `server_code` | +| 17 | Naming | the API's operation and parameter names, in this surface's casing | +| 18 | Copy | Crawl is "new", never "beta"; one "still evolving" line; the 429 limit reads as the account's active jobs, shared with Batch | +| 19 | Failed crawl | an include pattern that matches nothing ends `failed` with `no_items_found`; SDKs return it, not raise it; MCP and CLI say `ok: false` (CLI: exit 1, `CRAWL_FAILED`) | +| 20 | Results after stop | the html crawl is stopped while its pages are still being fetched: reading its results ends at once (MCP and CLI: `partial: false`, no note to call again), and the same URLs come back once the pages in flight finish | +| 21 | Documented defaults | a create without `max_pages` and `max_items` gets the default the docs state (10) | + +Extra checks for this surface: `mypy --strict` sees the package as typed (`py.typed`) and reads the real signatures. + +## What it costs + +Every crawl uses `depth` 1 and `max_pages` 10 or less. A run starts five crawls: the README run, +one URLs-only crawl, one idempotency crawl that is stopped at once, one html crawl that is +stopped with its pages in flight (check 20), and one that keeps nothing (check 19). On the default `CRAWL_URL`, a URLs-only crawl fetches 1 page and the +html crawl 6, so a run costs about **10 pages**, and at most 50 if every crawl reaches its +`max_pages`. Check 9 starts a sixth crawl, one html item, when the stop in check 20 lands +before any page is fetched. It adds at most 10 pages. The report lists each crawl id and its `pages_fetched`. + +An account can run 3 active jobs at once, shared with Batch jobs. A run has at most 2 crawls +running at a time. If a create still gets a 429, the run waits `Retry-After` and tries again. +It never sends requests to trigger the limit. Don't run two copies against one account at the +same time. + +## How to read the report + +- **Verdict:** ready, unless there is a blocker or major finding. +- **Feature matrix:** PASS, FAIL or SKIP, with a short reason. SKIP means the check does not + apply to this surface, the client rejects the input before the API, a crawl ended before + the check could act on it, or there was no key. +- **Findings:** one entry per FAIL, most severe first, with the command and output. An install + or README failure is a blocker; a wrong answer from a feature is major; naming, copy, the + defaults and the extras are minor. +- `out/results/checks/*.json` holds every result; `out/results/container.log` is the full + output. diff --git a/acceptance/crawl/builders/build-python.sh b/acceptance/crawl/builders/build-python.sh new file mode 100644 index 0000000..ec399e7 --- /dev/null +++ b/acceptance/crawl/builders/build-python.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Runs in the Python image. Builds the wheel a release would publish. Only the wheel leaves. +set -euo pipefail +cp -a /out/source /build && cd /build +pip install --quiet --disable-pip-version-check --root-user-action=ignore "build==$PY_BUILD_VERSION" +python -m build --wheel --outdir /out/artifact +ls -l /out/artifact diff --git a/acceptance/crawl/builders/fetch-source.sh b/acceptance/crawl/builders/fetch-source.sh new file mode 100644 index 0000000..51f1209 --- /dev/null +++ b/acceptance/crawl/builders/fetch-source.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Runs in the Node image when REF is set: downloads ZenRows/$REPO at $REF as GitHub's source +# tarball and reads the full commit SHA from it. +set -euo pipefail +cd /out +curl -fsSL --retry 4 -o src.tar.gz "https://codeload.github.com/ZenRows/$REPO/tar.gz/$REF" +# git get-tar-commit-id stops reading after the header, so the pipe breaks: ignore that. +(gunzip -c src.tar.gz | git get-tar-commit-id > sha) || true +[ -s sha ] || { echo "no commit id in the tarball of $REPO@$REF" >&2; exit 1; } +rm -rf source && mkdir -p source && tar -xzf src.tar.gz -C source --strip-components 1 && rm src.tar.gz +echo "fetched $REPO@$REF = $(cat sha)" diff --git a/acceptance/crawl/run.sh b/acceptance/crawl/run.sh new file mode 100644 index 0000000..03d63bb --- /dev/null +++ b/acceptance/crawl/run.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash +# Crawl first-install acceptance test for this repo. See README.md in this folder. +# +# ZENROWS_API_KEY=... bash run.sh build the release artifact, run every check, write the report +# bash run.sh --no-key only the checks that need no key; the live checks are SKIP +# bash run.sh --report rebuild the report from the results already in out/ +# +# Needs bash, git and Docker. The key reaches containers by name only (-e ZENROWS_API_KEY). +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=versions.env +source "$HERE/versions.env" +OUT_DIR="${OUT_DIR:-$HERE/out}" + +no_key=0; report_only=0 +for arg in "$@"; do + case "$arg" in + --no-key) no_key=1 ;; + --report) report_only=1 ;; + -h|--help) sed -n '2,8p' "$0"; exit 0 ;; + *) echo "unknown argument: $arg (expected --no-key or --report)" >&2; exit 2 ;; + esac +done + +if [ "$report_only" = 0 ] && [ "$no_key" = 0 ] && [ -z "${ZENROWS_API_KEY:-}" ]; then + echo "ZENROWS_API_KEY is not set. Set it (a key with Crawl access), or run 'bash run.sh --no-key' for the offline checks." >&2 + exit 2 +fi +if [ "$no_key" = 1 ]; then unset ZENROWS_API_KEY; fi +export ZENROWS_API_KEY="${ZENROWS_API_KEY:-}" CRAWL_URL CRAWL_INCLUDE CRAWL_API_BASE + +log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*" >&2; } +RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)" +CMDLINE="bash run.sh $*" +# Unique per run, so clean-up never touches the containers of another run. +LABEL="crawl-acceptance.run=$RUN_ID-$$-$RANDOM" +docker_run() { docker run --rm --label "$LABEL" "$@"; } + +# ---- 1. Source: this checkout (default), or the repo on GitHub at $REF ---- +prepare_source() { + rm -rf "$OUT_DIR/build"; mkdir -p "$OUT_DIR/build/source" + if [ -n "${REF:-}" ]; then + log "source: ZenRows/$REPO@$REF from GitHub" + docker_run -v "$OUT_DIR/build:/out" -v "$HERE/builders:/builders:ro" \ + -e REPO="$REPO" -e REF="$REF" "$NODE_IMAGE" bash /builders/fetch-source.sh >"$OUT_DIR/build/fetch.log" 2>&1 + echo "github:$REF" >"$OUT_DIR/build/origin" + else + local root; root="$(git -C "$HERE" rev-parse --show-toplevel)" + log "source: this checkout ($root)" + # Tracked and untracked-but-not-ignored files, as a release build would see them. + (cd "$root" && git ls-files -co --exclude-standard -z | tar --null -T - -cf -) | tar -C "$OUT_DIR/build/source" -xf - + local sha; sha="$(git -C "$root" rev-parse HEAD)" + if [ -n "$(git -C "$root" status --porcelain -- . ":(exclude)$(realpath --relative-to="$root" "$HERE")")" ]; then + sha="$sha+local-changes" + fi + echo "$sha" >"$OUT_DIR/build/sha" + echo "checkout" >"$OUT_DIR/build/origin" + fi + # The test never reads the harness from the source; only the artifact goes in. + rm -rf "$OUT_DIR/build/source/acceptance" +} + +# ---- 2. Builder: the release artifact, in its own container ---- +build_artifact() { + mkdir -p "$OUT_DIR/build/artifact" + case "$SURFACE" in + node|mcp|cli) + log "build: npm pack" + docker_run -v "$OUT_DIR/build:/out" -v "$HERE/builders:/builders:ro" \ + -e SURFACE -e PNPM_VERSION "$NODE_IMAGE" bash /builders/build-npm.sh >"$OUT_DIR/build/build.log" 2>&1 ;; + python) + log "build: the wheel" + docker_run -v "$OUT_DIR/build:/out" -v "$HERE/builders:/builders:ro" \ + -e PY_BUILD_VERSION "$PYTHON_IMAGE" bash /builders/build-python.sh >"$OUT_DIR/build/build.log" 2>&1 ;; + go) + # Go has no artifact: the clean container runs `go get` at a pushed commit, as a user would. + local sha; sha="$(cat "$OUT_DIR/build/sha")" + case "$sha" in *+local-changes) + log "go: local changes are not tested; go get installs the commit ${sha%+local-changes} from GitHub" ;; + esac + echo "${sha%+local-changes}" >"$OUT_DIR/build/go_ref" ;; + esac +} + +# ---- 3. Clean test container: only the artifact, the checks and the inputs go in ---- +run_checks() { + rm -rf "$OUT_DIR/results"; mkdir -p "$OUT_DIR/results" + local image + case "$SURFACE" in node|mcp|cli) image="$NODE_IMAGE" ;; python) image="$PYTHON_IMAGE" ;; go) image="$GO_IMAGE" ;; esac + log "checks: a clean $image" + local rc=0 + docker_run --init \ + -v "$OUT_DIR/build/artifact:/artifact:ro" \ + -v "$HERE/surface:/harness:ro" \ + -v "$OUT_DIR/results:/results" \ + -e ZENROWS_API_KEY -e CRAWL_URL -e CRAWL_INCLUDE -e CRAWL_API_BASE \ + -e SURFACE -e GO_REF="$(cat "$OUT_DIR/build/go_ref" 2>/dev/null || true)" \ + -e MCP_SDK_VERSION -e TYPESCRIPT_VERSION -e MYPY_VERSION \ + -w /work --tmpfs /work:exec,size=2g \ + "$image" bash /harness/run-in-container.sh >"$OUT_DIR/results/container.log" 2>&1 || rc=$? + log "checks: the container exited $rc (out/results/container.log)" +} + +# ---- 4. Clean-up: stop every crawl this run started. Runs even when a check fails. ---- +cleanup() { + [ "$report_only" = 1 ] && return 0 + docker ps -q --filter "label=$LABEL" | xargs -r docker kill >/dev/null 2>&1 || true + if [ -n "${ZENROWS_API_KEY:-}" ] && [ -d "$OUT_DIR/results" ]; then + log "clean-up: stopping every crawl this run started" + docker_run -v "$OUT_DIR:/out" -v "$HERE/tools:/tools:ro" \ + -e ZENROWS_API_KEY -e CRAWL_API_BASE "$PYTHON_IMAGE" python -I /tools/cleanup.py /out || log "clean-up failed" + fi +} + +report() { + log "report: out/crawl-first-install-report.md" + docker_run -v "$OUT_DIR:/out" -v "$HERE/tools:/tools:ro" \ + -e CMDLINE="$CMDLINE" -e SURFACE -e REPO -e CRAWL_URL -e CRAWL_INCLUDE -e CRAWL_API_BASE \ + -e NODE_IMAGE -e PYTHON_IMAGE -e GO_IMAGE -e MCP_SDK_VERSION -e TYPESCRIPT_VERSION \ + -e MYPY_VERSION -e PY_BUILD_VERSION -e PNPM_VERSION \ + "$PYTHON_IMAGE" python -I /tools/report.py /out + # The key must appear in nothing the run wrote. Checked by value, inside a container. + if [ -n "${ZENROWS_API_KEY:-}" ]; then + docker_run -v "$OUT_DIR:/out" -v "$HERE/tools:/tools:ro" -e ZENROWS_API_KEY "$PYTHON_IMAGE" \ + python -I /tools/scan_key.py /out + fi +} + +export SURFACE REPO NODE_IMAGE PYTHON_IMAGE GO_IMAGE PNPM_VERSION PY_BUILD_VERSION \ + MCP_SDK_VERSION TYPESCRIPT_VERSION MYPY_VERSION +if [ "$report_only" = 1 ]; then report; exit 0; fi + +mkdir -p "$OUT_DIR" +echo "$RUN_ID" >"$OUT_DIR/run_id" +trap cleanup EXIT +trap 'exit 130' INT TERM +if prepare_source && build_artifact; then + run_checks +else + log "the source or build step failed (out/build/*.log)" + mkdir -p "$OUT_DIR/results" + docker_run -v "$OUT_DIR:/out" -v "$HERE/tools:/tools:ro" -e SURFACE "$PYTHON_IMAGE" python -I /tools/build_failed.py /out +fi +cleanup +trap - EXIT +report +log "done: $OUT_DIR/crawl-first-install-report.md" diff --git a/acceptance/crawl/surface/run-in-container.sh b/acceptance/crawl/surface/run-in-container.sh new file mode 100644 index 0000000..e4598dd --- /dev/null +++ b/acceptance/crawl/surface/run-in-container.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Entry point in the clean python container: install the wheel as `pip install zenrows` would, +# install the pinned mypy for the typing check, record the versions, run the checks. +set -uo pipefail +export PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_ROOT_USER_ACTION=ignore PIP_NO_CACHE_DIR=1 +pip install /artifact/*.whl > /results/pip-install.log 2>&1 || { cat /results/pip-install.log; exit 1; } +pip install "mypy==$MYPY_VERSION" > /results/pip-mypy.log 2>&1 +printf '{"python":"%s","pip":"%s","mypy":"%s","os":"%s"}\n' "$(python -V 2>&1)" "$(pip -V | cut -d' ' -f2)" \ + "$MYPY_VERSION" "$(. /etc/os-release; echo "$PRETTY_NAME")" > /results/env.json +cd /work && python /harness/test.py diff --git a/acceptance/crawl/surface/test.py b/acceptance/crawl/surface/test.py new file mode 100644 index 0000000..06888ff --- /dev/null +++ b/acceptance/crawl/surface/test.py @@ -0,0 +1,581 @@ +"""Python SDK (`zenrows`): the README walk-through, checks 1-21 and the Python extra check. + +Runs in a clean python image. run-in-container.sh installs the wheel first; this script then +imports it the way a user would. +""" + +import http.server +import json +import os +import re +import subprocess +import sys +import threading +import time +import traceback +import urllib.request +import uuid +from datetime import UTC, datetime +from importlib import metadata + +SURFACE = "python" +KEY = os.environ.get("ZENROWS_API_KEY", "") +HAS_KEY = bool(KEY) +CRAWL_URL = os.environ["CRAWL_URL"] +INCLUDE = os.environ["CRAWL_INCLUDE"] +API_BASE = os.environ.get("CRAWL_API_BASE", "https://api.zenrows.com/v1") +NO_MATCH = "/crawl-acceptance-matches-nothing/" # no URL contains it: the crawl keeps nothing (check 19) +DOCUMENTED_DEFAULT = 10 # the default the README documents for max_pages and max_items (check 21) +RESULTS = "/results" +PASS, FAIL, SKIP = "PASS", "FAIL", "SKIP" +TERMINAL = {"completed", "stopped", "failed"} +os.makedirs(f"{RESULTS}/checks", exist_ok=True) +_seq = 0 + + +def redact(v): + s = v if isinstance(v, str) else json.dumps(v, default=str) + return s.replace(KEY, "") if KEY else s + + +def trim(v, n): + s = redact(v) + return s if len(s) <= n else f"{s[:n]}… [{len(s) - n} more chars]" + + +def record(check, title, status, expected, actual, evidence=""): + global _seq + _seq += 1 + r = {"surface": SURFACE, "check": str(check), "title": title, "status": status, "expected": expected, + "actual": trim(actual, 800), "evidence": trim(evidence, 2500)} + name = f"{_seq:03d}-{re.sub(r'[^A-Za-z0-9_.-]+', '_', str(check))}.json" + with open(f"{RESULTS}/checks/{name}", "w") as f: + json.dump(r, f, indent=2) + print(f"[{status}] python {check} {title}: {r['actual'].splitlines()[0][:200] if r['actual'] else ''}", flush=True) + + +def pass_if(ok): + return PASS if ok else FAIL + + +def run_check(check, title, expected, fn, live=False): + if live and not HAS_KEY: + return record(check, title, SKIP, expected, "ZENROWS_API_KEY not set (run without --no-key)") + try: + status, actual, evidence = fn() + record(check, title, status, expected, actual, evidence) + except Exception as e: # an unexpected exception is a FAIL with its traceback + record(check, title, FAIL, expected, f"raised {type(e).__name__}: {e}", traceback.format_exc()) + + +def ledger(crawl_id, purpose): + if crawl_id: + with open(f"{RESULTS}/crawls.jsonl", "a") as f: + f.write(json.dumps({"crawl_id": crawl_id, "purpose": purpose}) + "\n") + + +def sh(args, **kw): + p = subprocess.run(args, capture_output=True, text=True, **kw) + return p.returncode, p.stdout + p.stderr + + +def crawls_since(t): + if not HAS_KEY: + return [] + req = urllib.request.Request(f"{API_BASE}/crawls?limit=100", headers={"X-API-Key": KEY}) + with urllib.request.urlopen(req, timeout=60) as resp: + body = json.load(resp) + out = [] + for c in body["crawls"]: + created = datetime.strptime(c["created_at"][:19], "%Y-%m-%dT%H:%M:%S").replace(tzinfo=UTC) + if created.timestamp() >= t: + out.append(c["crawl_id"]) + return out + + +# ---------- README: install (run-in-container.sh) and the Crawl quickstart ---------- +with open(f"{RESULTS}/pip-install.log") as f: + install_log = f.read() +record("readme-1", "Install (README: `pip install zenrows`; here the release wheel)", PASS, "the package installs", + f"zenrows {metadata.version('zenrows')} installed", f"$ pip install \n{install_log[-1200:]}") + +readme = metadata.metadata("zenrows").get_payload() or "" +HEADING = "## Quickstart — Crawl API (new)" +section = readme.split(HEADING, 1)[1].split("\n## ", 1)[0] if HEADING in readme else "" +m = re.search(r"```python\n(.*?)```", section, re.S) +block = m.group(1) if m else None +record("readme-2", "README has a Crawl quickstart", pass_if(bool(block)), "a ```python block under the Crawl quickstart", + f"{len(block.splitlines())} lines" if block else "not found", block or section[:600]) + +subs = [ + ('"https://example.com/products/"', json.dumps(CRAWL_URL), "start URL → CRAWL_URL"), + ("max_items=50,", "max_items=5,", "max_items 50 → 5"), + ("max_pages=100,", "max_pages=10,", "max_pages 100 → 10 (at most 10)"), + ('include_patterns=["/product/"]', f"include_patterns=[{json.dumps(INCLUDE)}]", "include → CRAWL_INCLUDE"), +] +quick, applied = block or "", [] +for a, b, why in subs: + if a in quick: + quick = quick.replace(a, b) + applied.append(why) + else: + applied.append(f"not found, left as is: {why}") +applied.append("the key comes from ZENROWS_API_KEY, as the quickstart reads it") +with open("/work/quickstart.py", "w") as f: + f.write(quick) + + +def readme_run(): + since = time.time() - 5 + code, out = sh([sys.executable, "/work/quickstart.py"], timeout=12 * 60) + mine = crawls_since(since) + for i in mine: + ledger(i, "README quickstart") + return pass_if(code == 0), f"exit {code}; crawls: {', '.join(mine) or 'none'}", \ + f"substitutions: {'; '.join(applied)}\n$ python quickstart.py\n{out}" + + +run_check("readme-3", "Run the README Crawl quickstart", "it runs to the end without an error", readme_run, live=True) + + +# ---------- extra: a typed package (py.typed) under mypy --strict ---------- +def mypy_check(): + code, out = sh([sys.executable, "-m", "mypy", "--strict", "--no-incremental", "/harness/typecheck.py"]) + untyped = "missing library stubs or py.typed" in out or "module is installed, but missing" in out + revealed = "def (url: builtins.str" in out + errors = [line for line in out.splitlines() if ": error:" in line] + return pass_if(not untyped and revealed and not errors), \ + f"mypy exit {code}; py.typed seen {not untyped}; real signature revealed {revealed}; {len(errors)} errors", \ + f"$ mypy --strict typecheck.py (mypy {metadata.version('mypy')})\n{out}" + + +run_check("x-python-typed", "mypy --strict sees the package as typed (py.typed)", "no missing-stubs error; the real signatures", + mypy_check) + +# ---------- check 1, offline half: only the fields you set are sent ---------- +captured = [] + + +class StandIn(http.server.BaseHTTPRequestHandler): + """A local stand-in for the Crawl API that records what the client sends.""" + + def log_message(self, *a): + pass + + def do_POST(self): + n = int(self.headers.get("Content-Length") or 0) + body = json.loads(self.rfile.read(n) or b"null") + captured.append(body) + self.send_response(201) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps({ + "crawl_id": "c_capture", "status": "running", "url": body.get("url"), "depth": body.get("depth"), + "max_items": 10, "max_pages": 10, "created_at": datetime.now(UTC).isoformat(), + "coverage": {"pages_fetched": 0, "pages_failed": 0, "items_found": 0}}).encode()) + + def do_GET(self): + if "c_nobody" in self.path: + self.send_response(502) + self.send_header("Content-Length", "0") + self.end_headers() + return + self.send_response(404) + self.send_header("Content-Type", "application/problem+json") + self.end_headers() + self.wfile.write(b'{"status":404,"title":"Not Found","code":"crawl_not_found"}') + + +srv = http.server.ThreadingHTTPServer(("127.0.0.1", 0), StandIn) +threading.Thread(target=srv.serve_forever, daemon=True).start() + +from zenrows import ZenRowsCrawlClient # noqa: E402 (imported once installed, as a user would) +from zenrows.crawl import CrawlAPIError # noqa: E402 + +cap = ZenRowsCrawlClient("capture-key", base_url=f"http://127.0.0.1:{srv.server_address[1]}/v1", retries=0) +cap.create("https://example.com/", depth=1) +cap.create("https://example.com/", depth=1, max_items=5, max_pages=10, include_patterns=["/p/"], idempotency_key="idem-1") +sent_ok = sorted(captured[0]) == ["depth", "url"] and \ + sorted(captured[1]) == sorted(["url", "depth", "max_items", "max_pages", "include_patterns"]) +sent_line = f"sent (stand-in server): minimal={json.dumps(captured[0])} full={json.dumps(captured[1])}" + +# ---------- live ---------- +client = ZenRowsCrawlClient(KEY or "none", base_url=API_BASE) +BASE = {"depth": 1, "max_items": 5, "max_pages": 10, "include_patterns": [INCLUDE]} +state = {} + + +def create(**kw): + for _ in range(10): + try: + c = client.create(CRAWL_URL, **kw) + ledger(c.crawl_id, "check") + return c + except CrawlAPIError as e: + if e.status_code != 429: + raise + time.sleep(e.retry_after or 30) # the account's active jobs are shared: wait as told + raise RuntimeError("still 429 after 10 tries") + + +def err_of(fn): + try: + return fn(), None + except CrawlAPIError as e: + return None, e + + +def esum(e): + return None if e is None else {"status_code": e.status_code, "code": e.code, "detail": e.detail} + + +def c1(): + if not HAS_KEY: + return (SKIP if sent_ok else FAIL), f"live part skipped (no key); only set fields sent: {sent_ok}", sent_line + t = time.time() + a = state["A"] = create(**BASE) + echo = {"url": a.url, "depth": a.depth, "max_items": a.max_items, "max_pages": a.max_pages, + "include_patterns": a.include_patterns} + want = {"url": CRAWL_URL, **BASE} + ok = sent_ok and a.status.value == "running" and a.crawl_id.startswith("c_") and echo == want + return pass_if(ok), f"{a.status.value}, {a.crawl_id}, {int((time.time() - t) * 1000)} ms, echo {json.dumps(echo)}; " \ + f"only set fields sent: {sent_ok}", f"{a.model_dump_json()}\n{sent_line}" + + +def c5(): + a = state.get("A") + if not a: + return SKIP, "no crawl from check 1", "" + t = time.time() + w = client.wait(a.crawl_id, timeout=1) + ms = int((time.time() - t) * 1000) + if w.status.value != "running": + return SKIP, f"the crawl had already ended ({w.status.value}) after {ms} ms", "" + return pass_if(ms < 5000), f"{w.status.value} after {ms} ms, no exception", w.model_dump_json() + + +def partial_read(): + a = state.get("A") + if not a: + return + running = client.get(a.crawl_id, limit=1).status.value == "running" + t = time.time() + items, e = err_of(lambda: [r.url for r in client.results(a.crawl_id)]) + state["partial"] = (running, len(items or []), int((time.time() - t) * 1000), e) + + +def c3(): + key = f"acc-{uuid.uuid4()}" + body = dict(BASE, max_items=3, max_pages=3) + c = state["C"] = create(idempotency_key=key, **body) + again = create(idempotency_key=key, **body) + v, e = err_of(lambda: client.create(CRAWL_URL, idempotency_key=key, **dict(body, max_items=4))) + if v: + ledger(v.crawl_id, "idempotency, different body (unexpected create)") + ok = c.crawl_id == again.crawl_id and e is not None and e.status_code == 422 and e.code == "idempotency_key_reused" + third = f"{e.status_code} {e.code}" if e else f"created {v.crawl_id}" + return pass_if(ok), f"first {c.crawl_id}, second {again.crawl_id}; different body → {third}", json.dumps(esum(e)) + + +def c11(): + c = state.get("C") + if not c: + return SKIP, "no crawl from check 3", "" + first = client.stop(c.crawl_id) + reason = first.stop_reason.value if first.stop_reason else None + if first.status.value == "completed" and reason != "user": + return SKIP, f"the crawl had already ended on its own ({first.status.value}/{reason})", "" + second, e = err_of(lambda: client.stop(c.crawl_id)) + ok = first.status.value == "stopped" and reason == "user" and e is None and second.status == first.status + two = f"error {e}" if e else second.status.value + return pass_if(ok), f"first: {first.status.value}/{reason}; second: {two}", first.model_dump_json() + + +def c4(): + a = state.get("A") + if not a: + return SKIP, "no crawl from check 1", "" + e = state["ended"] = client.wait(a.crawl_id) + reason = f"/{e.stop_reason.value}" if e.stop_reason else "" + return pass_if(e.status.value in TERMINAL), f"{e.status.value}{reason}; coverage {e.coverage.model_dump_json()}", \ + e.model_dump_json() + + +def c6(): + e = state.get("ended") + if not e: + return SKIP, "no ended crawl", "" + pages, cursor = [], None + for _ in range(20): + p = client.get(e.crawl_id, limit=2, cursor=cursor) + pages.append((len(p.results), p.next_cursor, [r.url for r in p.results])) + if p.next_cursor is None: + break + cursor = p.next_cursor + urls = [u for _, _, us in pages for u in us] + curs = [c for _, c, _ in pages if c is not None] + ok = all(n <= 2 for n, _, _ in pages) and pages[-1][1] is None and len(set(curs)) == len(curs) \ + and len(set(urls)) == len(urls) and (len(pages) >= 2 or e.coverage.items_found <= 2) + sizes = ",".join(str(n) for n, _, _ in pages) + return pass_if(ok), f"{len(pages)} pages of {sizes}; last next_cursor {pages[-1][1]}; {len(urls)} URLs", "" + + +def c7(): + p1 = client.list(limit=2) + p2 = client.list(limit=2, cursor=p1.next_cursor) if p1.next_cursor else None + allc = p1.crawls + (p2.crawls if p2 else []) + times = [c.created_at for c in allc] + newest = all(times[i - 1] >= times[i] for i in range(1, len(times))) + overlap = len({c.crawl_id for c in allc}) != len(allc) + cursor, pages = (p2.next_cursor if p2 else None), 2 + while cursor and pages < 60: + cursor = client.list(limit=100, cursor=cursor).next_cursor + pages += 1 + ok = len(p1.crawls) == 2 and newest and not overlap and not cursor + return pass_if(ok), f"newest first {newest}, no overlap {not overlap}; last page after {pages} pages: " \ + f"next_cursor {json.dumps(cursor)}", "" + + +def c8(): + e = state.get("ended") + if not e: + return SKIP, "no ended crawl", "" + urls = [r.url for r in client.results(e.crawl_id, limit=2)] + distinct, match = len(set(urls)) == len(urls), all(INCLUDE in u for u in urls) + p = state.get("partial") + run = (f"while running ({'running' if p[0] else 'already ended'}): {p[1]} URLs in {p[2]} ms, " + f"{'error ' + str(p[3]) if p[3] else 'returned'}") if p else "not read while running" + ok = bool(urls) and distinct and match and len(urls) == e.coverage.items_found and not (p and p[3]) + return pass_if(ok), f"{len(urls)} URLs (items_found {e.coverage.items_found}), distinct {distinct}, " \ + f"all match {match}; {run}", "\n".join(urls) + + +def settle(crawl_id): + """Poll until no result is still pending (the pages in flight finished), at most 90 s.""" + for _ in range(45): + page = client.get(crawl_id, limit=100) + if not any(r.content_status and r.content_status.value == "pending" for r in page.results): + return "settled" + time.sleep(2) + return "still pending after 90 s" + + +def stop_mid_flight(): + """Create the html crawl and stop it as soon as it has kept URLs, while their pages are fetched.""" + try: + b = state["B"] = create(output_format="html", **BASE) + g = None + for _ in range(100): + g = client.get(b.crawl_id, limit=1) + if g.status.value != "running" or g.coverage.items_found > 0: + break + time.sleep(0.15) + s = client.stop(b.crawl_id) + state["stop"] = {"items": g.coverage.items_found, "status": s.status.value} + except Exception as e: # recorded by check 20 + state["stop"] = {"error": e} + + +def c20(): + b, st = state.get("B"), state.get("stop") or {} + if not b or "error" in st: + return FAIL, f"setup failed: {st.get('error')}", "" + if st["status"] != "stopped": + return SKIP, f"the crawl had already ended ({st['status']}) before the stop", "" + t = time.time() + box = {} + # A daemon thread, so a results() that pages forever fails the check instead of hanging the run. + reader = threading.Thread(target=lambda: box.update(now=list(client.results(b.crawl_id))), daemon=True) + reader.start() + reader.join(60) + if "now" not in box: + return FAIL, "results() did not finish within 60 s on a crawl stopped with pages in flight", "" + now, ms = box["now"], int((time.time() - t) * 1000) + settled = settle(b.crawl_id) + if settled != "settled": + settled += " (an API issue: the stop left these pages pending on an ended crawl)" + later = [r.url for r in client.results(b.crawl_id)] + same = sorted(r.url for r in now) == sorted(later) + pending = sum(1 for r in now if r.content_status and r.content_status.value == "pending") + return pass_if(same and bool(now)), f"stopped with {st['items']} kept; read {len(now)} URLs ({pending} pending) in {ms} ms, " \ + f"then {settled}: {len(later)} URLs, same set {same}", "" + + +def c2(): + b = state.get("B") + if not b: + return SKIP, "no html crawl", "" + if (state.get("stop") or {}).get("status") == "stopped": + settle(b.crawl_id) + rs = list(client.results(b.crawl_id)) + ok = b.output_format is not None and b.output_format.value == "html" and bool(rs) and all( + r.content_status and (r.content_status.value != "fetched" or r.content_url) for r in rs) + with_url = sum(1 for r in rs if r.content_url) + return pass_if(ok), f"{b.crawl_id} ({(state.get('stop') or {}).get('status', '-')}); {len(rs)} results; " \ + f"with content_url {with_url}", json.dumps([r.model_dump(mode="json") for r in rs[:2]]) + + +def c9(): + b = state.get("B") + if not b: + return SKIP, "no html crawl", "" + def first_fetched(crawl_id): + return next((x for x in client.results(crawl_id) if x.content_url), None) + + # The stop in check 20 can land before any page is fetched. Then read a + # one-item html crawl that runs to the end instead. + src = b.crawl_id + r = first_fetched(src) + if not r: + own = create(output_format="html", **{**BASE, "max_items": 1}) + client.wait(own.crawl_id) + src = own.crawl_id + r = first_fetched(src) + if not r: + return FAIL, f"no result with content_url on {src}", "" + cid = r.content_url.rstrip("/").rsplit("/", 1)[-1] + by_id, by_url = client.content(src, cid), client.content(src, r.content_url) + ok = by_id == by_url and re.search(r"", "").lstrip("-").replace("-", "_") + return re.sub(r"(?<=[a-z0-9])(?=[A-Z])", "_", n).lower() + + +def naming(rows): + p = os.path.join(OUT, "results", "naming.json") + if not os.path.exists(p): + return + with open(p) as f: + d = json.load(f) + diffs = [] + for op in CANON_OPS: + got = d["operations"].get(op) + if got is None: + diffs.append(f"operation `{op}` missing") + elif snake(got) != op: + diffs.append(f"operation `{op}` is `{got}`") + for prm in CANON_PARAMS: + got = d["params"].get(prm) + if got is None: + diffs.append(f"parameter `{prm}` missing") + elif snake(got) != prm: + diffs.append(f"parameter `{prm}` is `{got}`") + rows["17"] = {"surface": SURFACE, "check": "17", "title": "Naming", + "status": "FAIL" if diffs else "PASS", + "expected": "the API's operation and parameter names, in this surface's casing", + "actual": "; ".join(diffs) or "matches", + "evidence": json.dumps({k: d[k] for k in ("operations", "params")})} + + +def severity(check): + if check in ("17", "18", "21") or check.startswith("x-"): + return "minor" + if check.startswith("readme") or check == "build": + return "blocker" + return "major" + + +def first_line(s, n=110): + line = re.sub(r"\s+", " ", (s or "").split("\n")[0]) + return line if len(line) <= n else line[: n - 1] + "…" + + +def main(): + rows = {} + for p in sorted(glob.glob(os.path.join(OUT, "results", "checks", "*.json"))): + with open(p) as f: + r = json.load(f) + rows[r["check"]] = r + naming(rows) + env = json.loads(read(os.path.join(OUT, "results", "env.json"), "{}") or "{}") + cleanup = json.loads(read(os.path.join(OUT, "cleanup.json"), "null") or "null") + sha = read(os.path.join(OUT, "build", "sha"), "-") + origin = read(os.path.join(OUT, "build", "origin"), "-") + artifacts = [os.path.basename(p) for p in glob.glob(os.path.join(OUT, "build", "artifact", "*"))] + gomod = read(os.path.join(OUT, "results", "go-module.txt")) + + findings = [{"check": c, "severity": severity(c), "r": r} for c, r in rows.items() if r["status"] == "FAIL"] + order = {"blocker": 0, "major": 1, "minor": 2} + findings.sort(key=lambda f: order[f["severity"]]) + blocking = [f for f in findings if f["severity"] != "minor"] + counts = {} + for r in rows.values(): + counts[r["status"]] = counts.get(r["status"], 0) + 1 + + lines = [] + w = lines.append + w(f"# Crawl first-install test: {NAMES[SURFACE]}\n") + w(f"Run `{read(os.path.join(OUT, 'run_id'), '-')}`. The client was built as a release artifact, installed in a " + "clean container, set up from its README, then tested against the live Crawl API.\n") + + w("## 1. Verdict\n") + if blocking: + w("**Not ready.** " + "; ".join(f"{f['check']} {f['r']['title']}: {first_line(f['r']['actual'], 90)}" for f in blocking)) + else: + w("**Ready.**" + (f" {len(findings)} minor finding(s), see Findings." if findings else "")) + w("\n" + ", ".join(f"{v} {k}" for k, v in sorted(counts.items())) + ".\n") + + w("## 2. Environment\n") + w(f"- Command: `{os.environ.get('CMDLINE') or 'bash run.sh'}`") + w(f"- Repo: ZenRows/{REPO}; source: {origin}; SHA tested: `{sha}`") + w(f"- Artifact: {', '.join(artifacts) or ('go get ' + gomod.split()[0] if gomod else '-')}") + w(f"- `CRAWL_URL`: {os.environ.get('CRAWL_URL')}; `CRAWL_INCLUDE`: `{os.environ.get('CRAWL_INCLUDE')}`; " + f"API: {os.environ.get('CRAWL_API_BASE')}") + image = {"node": "NODE_IMAGE", "mcp": "NODE_IMAGE", "cli": "NODE_IMAGE", "python": "PYTHON_IMAGE", "go": "GO_IMAGE"}[SURFACE] + w(f"- Image: `{os.environ.get(image)}`; container: " + ", ".join(f"{k} {v}" for k, v in env.items())) + tools = {"node": f"typescript {os.environ.get('TYPESCRIPT_VERSION')}, pnpm {os.environ.get('PNPM_VERSION')}", + "mcp": f"@modelcontextprotocol/sdk {os.environ.get('MCP_SDK_VERSION')}", "cli": "none", + "python": f"mypy {os.environ.get('MYPY_VERSION')}, build {os.environ.get('PY_BUILD_VERSION')}", "go": "none"}[SURFACE] + w(f"- Pinned tools: {tools}\n") + + w("## 3. First-time install\n") + for check, r in rows.items(): + if check.startswith("readme") or check == "build": + w(f"- **{r['status']}** {r['title']}: {first_line(r['actual'], 200)}") + ev = r.get("evidence") or "" + if ev and (r["status"] != "PASS" or "substitutions" in ev): + w("\n ```\n " + "\n ".join(ev.splitlines()[:16]) + "\n ```") + w("") + + w("## 4. Feature matrix\n") + w("| # | Check | Result |") + w("| --- | --- | --- |") + for c in [str(i) for i in range(1, 22)]: + r = rows.get(c) + cell = f"**{r['status']}** {first_line(r['actual'])}".replace("|", "\\|") if r else "—" + w(f"| {c} | {CHECK_TITLES[c]} | {cell} |") + for c, r in rows.items(): + if c.startswith("x-"): + w(f"| extra | {r['title']} | " + f"**{r['status']}** {first_line(r['actual'])}".replace("|", "\\|") + " |") + w("") + + w("## 5. Findings\n") + if not findings: + w("None.\n") + for i, f in enumerate(findings, 1): + r = f["r"] + w(f"### {i}. {r['title']} ({f['severity']})\n") + w(f"- What happened: {r['actual']}") + w(f"- Expected: {r['expected']}") + w("- Evidence:\n\n ```\n " + "\n ".join((r.get("evidence") or "").splitlines()[:20]) + "\n ```\n") + + w("## 6. Cost\n") + if cleanup and cleanup["crawls"]: + w("| Crawl | Purpose | Final status | pages_fetched |") + w("| --- | --- | --- | --- |") + total = 0 + for c in cleanup["crawls"]: + total += c.get("pages_fetched") or 0 + w(f"| `{c['crawl_id']}` | {', '.join(c['purposes'])}{' (html)' if c.get('output_format') == 'html' else ''} | " + f"{c.get('status')}{'/' + c['stop_reason'] if c.get('stop_reason') else ''} | {c.get('pages_fetched')} |") + w(f"\nTotal pages fetched: **{total}**.\n") + else: + w("No crawls.\n") + + w("## 7. Clean-up\n") + if cleanup and cleanup["crawls"]: + stopped = [c["crawl_id"] for c in cleanup["crawls"] if c.get("stopped_by_cleanup")] + w(f"Checked at {cleanup['checked_at']}: {len(cleanup['crawls'])} crawls started by this run, " + f"{', '.join(cleanup['still_running']) or 'none'} still running. " + + (f"Clean-up stopped {', '.join(stopped)}." if stopped else "Clean-up had nothing to stop.") + "\n") + else: + w("No crawls to clean up.\n") + + with open(os.path.join(OUT, "crawl-first-install-report.md"), "w") as f: + f.write("\n".join(lines)) + with open(os.path.join(OUT, "results.json"), "w") as f: + json.dump(list(rows.values()), f, indent=2) + print(f"{SURFACE}: " + ", ".join(f"{k} {v}" for k, v in sorted(counts.items())) + f"; findings: {len(findings)}") + + +main() diff --git a/acceptance/crawl/tools/scan_key.py b/acceptance/crawl/tools/scan_key.py new file mode 100644 index 0000000..f6f0419 --- /dev/null +++ b/acceptance/crawl/tools/scan_key.py @@ -0,0 +1,21 @@ +"""Fail the run if the API key appears in any file under out/. Reads the key from the env.""" +import os +import sys + +key = os.environ.get("ZENROWS_API_KEY", "").encode() +hits = [] +for root, _, files in os.walk(sys.argv[1]): + if os.sep + "build" in root: + continue + for f in files: + p = os.path.join(root, f) + try: + with open(p, "rb") as fh: + if key and key in fh.read(): + hits.append(p) + except OSError: + pass +if hits: + print("THE API KEY APPEARS IN:", *hits, sep="\n ") + sys.exit(1) +print("key scan: the key appears in no result, log or report file") diff --git a/acceptance/crawl/versions.env b/acceptance/crawl/versions.env new file mode 100644 index 0000000..dc1f030 --- /dev/null +++ b/acceptance/crawl/versions.env @@ -0,0 +1,23 @@ +# Pinned inputs for the Crawl acceptance test. Every value can be overridden from the environment. +SURFACE=python +REPO=zenrows-python-sdk + +# The official images, pinned by digest so two runs agree. +NODE_IMAGE="${NODE_IMAGE:-node:20@sha256:8f693eaa7e0a8e71560c9a82b55fd54c2ae920a2ba5d2cde28bac7d1c01c9ba5}" +PYTHON_IMAGE="${PYTHON_IMAGE:-python:3.12-slim@sha256:a6e34c598f2467ed0e9a8d349809fcd8b5c603269512df273a0bb1784edc11b1}" +GO_IMAGE="${GO_IMAGE:-golang:1.25@sha256:699337d620559a59b4a2bb298ad59611e535d2ee755a34cf2d2a98f37578dc80}" + +# Tools the build and the checks install. +PNPM_VERSION="${PNPM_VERSION:-9.4.0}" +PY_BUILD_VERSION="${PY_BUILD_VERSION:-1.2.2.post1}" +MCP_SDK_VERSION="${MCP_SDK_VERSION:-1.27.1}" +TYPESCRIPT_VERSION="${TYPESCRIPT_VERSION:-5.7.3}" +MYPY_VERSION="${MYPY_VERSION:-1.15.0}" + +# What to crawl. +CRAWL_URL="${CRAWL_URL:-https://www.scrapingcourse.com/ecommerce/}" +CRAWL_INCLUDE="${CRAWL_INCLUDE:-/product/}" +CRAWL_API_BASE="${CRAWL_API_BASE:-https://api.zenrows.com/v1}" + +# REF unset tests this checkout. Set it to a SHA, branch or tag to test that commit from GitHub. +REF="${REF:-}" diff --git a/pyproject.toml b/pyproject.toml index fbad15b..4cb04f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ packages = ["src/zenrows"] [tool.hatch.build.targets.sdist] include = ["src/zenrows", "README.md", "LICENSE"] +exclude = ["/acceptance"] # the acceptance test harness, never shipped # --- uv --- [tool.uv]