refactor: split lib.rs, add rustdoc and enforce 90% coverage

.github/workflows/ci.yml

@@ -5,6 +5,7 @@ on:
     paths:
       - "src/**"
       - "tests/**"
+      - "scripts/**"
       - "Cargo.toml"
       - "Cargo.lock"
       - "pyproject.toml"
@@ -17,6 +18,7 @@ on:
     paths:
       - "src/**"
       - "tests/**"
+      - "scripts/**"
       - "Cargo.toml"
       - "Cargo.lock"
       - "pyproject.toml"
@@ -59,6 +61,10 @@ jobs:
       - run: cargo fmt --check
       - run: cargo check --locked --all-targets
       - run: cargo clippy --locked --all-targets -- -D warnings
+      - name: Check rustdoc
+        env:
+          RUSTDOCFLAGS: "-D warnings"
+        run: cargo doc --locked --no-deps --document-private-items
 
   tests:
     runs-on: ubuntu-24.04
@@ -92,3 +98,37 @@ jobs:
           python -m pip install dist/*.whl
       - name: Test public Python API
         run: python -m unittest discover -s tests -v
+
+  coverage:
+    name: Coverage (>=90%)
+    runs-on: ubuntu-24.04
+    timeout-minutes: 25
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
+        with:
+          python-version: "3.14"
+      - name: Install native build dependencies
+        run: sudo apt-get update && sudo apt-get install -y cmake clang libclang-dev
+      - name: Install coverage tooling
+        run: |
+          rustup toolchain install stable --profile minimal --component llvm-tools-preview
+          rustup default stable
+          cargo install cargo-llvm-cov --version 0.9.1 --locked
+          python -m pip install "maturin>=1.7,<2"
+      - uses: actions/cache@v5
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target/coverage
+          key: coverage-${{ runner.os }}-${{ hashFiles('Cargo.lock', 'Cargo.toml') }}
+      - name: Measure Rust + Python API coverage and enforce 90%
+        run: bash scripts/coverage.sh
+      - name: Upload coverage report and badge
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: rust-coverage
+          path: target/coverage/report/
+          if-no-files-found: warn

CHANGELOG.md

@@ -10,10 +10,15 @@ Unreleased entry.
 ## [Unreleased]
 
 ### Added
+- Internal Rust module documentation and a rustdoc warning check.
+- Unit and local Python integration tests for stream messages, error propagation, body handling, and bounded concurrency.
+- A combined Rust/Python coverage check requiring at least 90% production Rust line coverage, with a CI progress bar and downloadable reports/badge.
 - CI for relevant pull requests, pushes to `master`, and manual runs, with path filters and cancellation of superseded runs.
 - Rust checks, regression tests, pedantic Clippy, and Python API tests on Python 3.10 and 3.15.
 
 ### Changed
+- Split the extension implementation into API, configuration, checker, worker, and stream modules without changing the public Python API.
+- Enable PyO3 extension-module mode through Maturin so Rust unit tests can embed Python normally.
 - Update the Rust `rsloop` dependency to 0.1.56 and `wreq` to the stable 0.16.1 series.
 - Update PyO3 to 0.29.3 for compatibility with `rsloop`.
 - Require Python 3.10 or newer and Rust 1.98 or newer for the updated dependencies.

Cargo.toml

@@ -12,7 +12,7 @@ crate-type = ["cdylib"]
 [dependencies]
 async-std = "1"
 futures = "0.3"
-pyo3 = { version = "0.29.3", features = ["extension-module"] }
+pyo3 = { version = "0.29.3" }
 rsloop = { version = "0.1.56" }
 tokio = { version = "1", features = ["full"] }
 wreq = { version = "0.16.1", features = ["socks", "stream"] }

README.md

@@ -1,5 +1,7 @@
 # Rust Proxy Checker Example
 
+[![CI](https://github.com/RustedBytes/proxychecker/actions/workflows/ci.yml/badge.svg)](https://github.com/RustedBytes/proxychecker/actions/workflows/ci.yml)
+
 This example is a standalone PyO3 extension built on top of `rsloop::rust_async`
 that checks many proxies concurrently and yields Python-friendly result objects
 as soon as each proxy finishes.
@@ -104,6 +106,35 @@ async for result in stream:
         failed.append(proxy_result)
 ```
 
+## Rust module layout
+
+- `api`: Python arguments, validation, and stream creation.
+- `config`: shared settings and defaults.
+- `checker`: proxy parsing, HTTP client, and single-proxy outcomes.
+- `worker`: Tokio worker lifecycle and bounded concurrency.
+- `stream`: Python queue delivery, result dictionaries, and async iteration.
+- `lib`: module registration.
+
+Build the internal Rust documentation with `cargo doc --no-deps --document-private-items`.
+
+## Coverage
+
+The CI check **Coverage (>=90%)** requires at least 90% line coverage across all
+production Rust modules, including the Python binding and worker code. It combines
+Rust unit tests with Python tests against an instrumented wheel; tests and external
+dependencies are excluded from the denominator. It measures line coverage, not branch
+coverage. The CI summary shows a progress bar; the `rust-coverage` artifact includes
+an HTML report, JSON metrics, and a badge with the measured percentage.
+
+To reproduce locally in an activated Python virtual environment:
+
+```bash
+rustup component add llvm-tools-preview
+cargo install cargo-llvm-cov --version 0.9.1 --locked
+python -m pip install "maturin>=1.7,<2"
+bash scripts/coverage.sh
+```
+
 ## Supported proxy strings
 
 The example passes the proxy string directly to `wreq`, so support follows the

pyproject.toml

@@ -11,6 +11,7 @@ dependencies = [
 ]
 
 [tool.maturin]
+features = ["pyo3/extension-module"]
 module-name = "rsloop_rust_proxychecker"
 
 [dependency-groups]

scripts/coverage.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Measure all production Rust modules using both unit and Python API tests.
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+export CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-$PWD/target/coverage}"
+# Rust unit tests embed Python; some distributions need its shared-library path.
+python_libdir="$(python -c 'import sysconfig; print(sysconfig.get_config_var("LIBDIR") or "")')"
+export LD_LIBRARY_PATH="$python_libdir${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
+source <(cargo llvm-cov show-env --sh)
+cargo llvm-cov clean --workspace
+cargo test --locked
+maturin build --locked --out "$CARGO_TARGET_DIR/wheels"
+python -m pip install --force-reinstall "$CARGO_TARGET_DIR"/wheels/*.whl
+python -m unittest discover -s tests -v
+
+mkdir -p "$CARGO_TARGET_DIR/report"
+cargo llvm-cov report --json --output-path "$CARGO_TARGET_DIR/report/coverage.json"
+cargo llvm-cov report --html --output-dir "$CARGO_TARGET_DIR/report/html"
+python scripts/coverage_summary.py "$CARGO_TARGET_DIR/report/coverage.json"
+cargo llvm-cov report --fail-under-lines 90

scripts/coverage_summary.py

@@ -0,0 +1,31 @@
+"""Render the measured Rust line coverage as a badge and CI progress bar."""
+
+import json
+import os
+import sys
+from pathlib import Path
+
+report_path = Path(sys.argv[1])
+report = json.loads(report_path.read_text())
+lines = report["data"][0]["totals"]["lines"]
+percent = lines["percent"]
+covered, count = lines["covered"], lines["count"]
+color = "#4c1" if percent >= 90 else "#e05d44"
+label = f"{percent:.2f}%"
+svg = f'''<svg xmlns="http://www.w3.org/2000/svg" width="150" height="20" role="img" aria-label="coverage: {label}">
+<rect width="90" height="20" fill="#555"/><rect x="90" width="60" height="20" fill="{color}"/>
+<g fill="white" font-family="Verdana, sans-serif" font-size="11" text-anchor="middle">
+<text x="45" y="14">Rust coverage</text><text x="120" y="14">{label}</text></g></svg>'''
+report_path.with_name("coverage.svg").write_text(svg)
+filled = min(20, round(percent / 5))
+bar = "▰" * filled + "▱" * (20 - filled)
+summary = (
+    f"### Rust line coverage: {label}\n\n"
+    f"{bar} **{label}** — required: **90%**\n\n"
+    f"{covered}/{count} production Rust lines covered by unit and Python API tests.\n"
+    "All production modules are included; test code and dependencies are excluded.\n"
+)
+print(summary)
+if summary_path := os.environ.get("GITHUB_STEP_SUMMARY"):
+    with Path(summary_path).open("a") as output:
+        output.write(summary)

src/api.rs

@@ -0,0 +1,76 @@
+//! Python entry point and synchronous argument validation.
+
+use std::time::Duration;
+
+use pyo3::exceptions::PyValueError;
+use pyo3::prelude::*;
+
+use crate::config::{CheckerConfig, DEFAULT_CHECK_URL, DEFAULT_CONCURRENCY, DEFAULT_TIMEOUT_MS};
+use crate::stream::PyProxyCheckStream;
+use crate::worker::spawn_proxy_checks;
+
+/// Check proxy URLs and resolve to an asynchronous stream of result dictionaries.
+///
+/// Arguments are validated synchronously. A nonempty user agent, positive timeout,
+/// positive concurrency limit, and parseable target URL are required. Network and
+/// body-read errors are returned as failed results rather than raised exceptions.
+/// `return_response` controls body collection; concurrency controls in-flight work.
+///
+/// # Errors
+///
+/// Raises `ValueError` for invalid arguments, or propagates Python event-loop and
+/// queue setup errors. Iterating the returned stream can raise `RuntimeError` when
+/// the worker cannot initialize or deliver results.
+#[pyfunction]
+#[pyo3(signature=(proxies, *, user_agent, check_url=None, timeout_ms=DEFAULT_TIMEOUT_MS, concurrency=DEFAULT_CONCURRENCY, return_response=false))]
+pub(crate) fn check_proxies(
+    py: Python<'_>,
+    proxies: Vec<String>,
+    user_agent: String,
+    check_url: Option<String>,
+    timeout_ms: u64,
+    concurrency: usize,
+    return_response: bool,
+) -> PyResult<Bound<'_, PyAny>> {
+    if user_agent.trim().is_empty() {
+        return Err(PyValueError::new_err("user_agent must not be empty"));
+    }
+    if timeout_ms == 0 {
+        return Err(PyValueError::new_err("timeout_ms must be greater than 0"));
+    }
+    if concurrency == 0 {
+        return Err(PyValueError::new_err("concurrency must be greater than 0"));
+    }
+
+    let check_url = check_url.unwrap_or_else(|| DEFAULT_CHECK_URL.to_string());
+    url::Url::parse(&check_url)
+        .map_err(|err| PyValueError::new_err(format!("invalid check_url: {err}")))?;
+
+    let config = CheckerConfig {
+        check_url,
+        user_agent,
+        timeout: Duration::from_millis(timeout_ms),
+        concurrency,
+        return_response,
+    };
+    let locals = rsloop::rust_async::get_current_locals(py)?;
+    let queue = py.import("asyncio")?.getattr("Queue")?.call0()?.unbind();
+    let stream = Py::new(
+        py,
+        PyProxyCheckStream {
+            queue: queue.clone_ref(py),
+        },
+    )?
+    .into_any();
+
+    spawn_proxy_checks(
+        proxies,
+        config.clone(),
+        locals.event_loop(py).unbind(),
+        queue,
+    );
+    rsloop::rust_async::future_into_py(py, async move {
+        let _ = config;
+        Ok(stream)
+    })
+}