diff --git a/.github/workflows/CI-test_image_action.yml b/.github/workflows/CI-test_image_action.yml new file mode 100644 index 0000000..7021278 --- /dev/null +++ b/.github/workflows/CI-test_image_action.yml @@ -0,0 +1,22 @@ +name: CI | test-image action + +on: + pull_request: + paths: + - "test-image/**" + - ".github/workflows/CI-test_image_action.yml" + push: + branches: + - main + paths: + - "test-image/**" + +jobs: + unit-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + - name: Run unit tests + working-directory: test-image + run: uv run -p 3.12 --no-project --with pytest --with runpod==2.1.0 pytest -q tests diff --git a/.github/workflows/TEMPLATE-run_worker_tests.yml b/.github/workflows/TEMPLATE-run_worker_tests.yml deleted file mode 100644 index 1777fba..0000000 --- a/.github/workflows/TEMPLATE-run_worker_tests.yml +++ /dev/null @@ -1,93 +0,0 @@ -name: CI | Test Worker - -# Requires the following secrets: -# - RUNPOD_API_KEY: The API key to authenticate with the Runpod API -# - GH_PAT: A GitHub Personal Access Token with access to the repository - -# Requires the following variables: -# - RUNPOD_ENDPOINT: The endpoint of the Runpod API - -on: - # push: - # branches: - # - main - - # pull_request: - # branches: - # - main - - workflow_dispatch: - -jobs: - initialize_runner: - runs-on: ubuntu-latest - outputs: - id: ${{ steps.extract_id.outputs.runpod_job_id }} - - steps: - - name: Deploy Worker - id: deploy - uses: fjogeleit/http-request-action@v1 - with: - url: "https://api.runpod.ai/v2/${{ vars.RUNPOD_ENDPOINT }}/run" - method: "POST" - customHeaders: '{"Content-Type": "application/json"}' - bearerToken: ${{ secrets.RUNPOD_API_KEY }} - data: '{"input":{"github_pat": "${{ secrets.GH_PAT }}", "github_org":"${{ github.repository_owner }}"}}' - - - name: Extract Job ID - id: extract_id - run: | - ID=$(echo '${{ steps.deploy.outputs.response }}' | jq -r '.id') - echo "::set-output name=runpod_job_id::$ID" - - run_tests: - needs: initialize_runner - runs-on: runpod - - steps: - - uses: actions/checkout@v3 - - - name: Set up Python 3.11 & install dependencies - uses: actions/setup-python@v4 - with: - python-version: "3.11" - - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install -r builder/requirements.txt - - - name: Execute Tests - run: | - python src/handler.py --test_input='{"input": "test"}' - - terminate_runner: - if: always() - needs: [run_tests, initialize_runner] - runs-on: ubuntu-latest - - steps: - - name: Get Job Status - id: get_status - uses: fjogeleit/http-request-action@v1 - with: - url: "https://api.runpod.ai/v2/${{ vars.RUNPOD_ENDPOINT }}/status/${{ needs.initialize_runner.outputs.id }}" - method: "GET" - customHeaders: '{"Content-Type": "application/json"}' - bearerToken: ${{ secrets.RUNPOD_API_KEY }} - - - name: Parse Status - id: parse_status - run: | - STATUS=$(echo '${{ steps.get_status.outputs.response }}' | jq -r '.status') - echo "::set-output name=status::$STATUS" - - - name: Shutdown Worker - if: ${{ steps.parse_status.outputs.status != 'COMPLETED' }} - uses: fjogeleit/http-request-action@v1 - with: - url: "https://api.runpod.ai/v2/${{ vars.RUNPOD_ENDPOINT }}/cancel/${{ needs.initialize_runner.outputs.id }}" - method: "POST" - customHeaders: '{"Content-Type": "application/json"}' - bearerToken: ${{ secrets.RUNPOD_API_KEY }} diff --git a/README.md b/README.md index 2493252..495be05 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,10 @@ This repository hosts the code for deploying a GitHub Actions worker on the Runp 5. Deploy your worker to the Runpod platform. +## Testing worker images in CI + +The [`test-image`](./test-image) action runs a worker image's `.runpod/tests.json` cases on a temporary endpoint and deletes it afterwards. See [docs/usage.md](./docs/usage.md). + ## Documentation For more details on how this GitHub Actions worker functions and how to integrate it into your CI/CD pipeline, please refer to the [Documentation](./docs). diff --git a/docs/usage.md b/docs/usage.md index 1b5f88a..f0141e4 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -1,53 +1,88 @@ # Documentation -## CI/CD with GitHub Actions and Runpod - -This repository uses GitHub Actions to run a series of tests on the Runpod serverless platform. The workflow is defined in the `.github/workflows/test-worker.yml` file, with the primary purpose to deploy a worker to the Runpod platform, execute the tests, and then terminate the worker. - -Here is a step-by-step breakdown of the `test-worker.yml` file: - -### Triggers - -This workflow runs whenever: - -- A push is made to the main branch. -- A pull request is opened to the main branch. -- The workflow is manually triggered (workflow_dispatch). - -### Jobs - -#### `initialize_worker` - -This job deploys a worker to the Runpod platform. It: - -- Sends a POST request to the Runpod API to start a new job (uses `fjogeleit/http-request-action@v1`). -- Extracts the job ID from the response and sets it as a job output. - -#### `run_tests` - -This job runs the tests in the worker environment. It: - -- Checks out the repository code. -- Sets up Python 3.11 and installs dependencies. -- Executes tests by running `handler.py` with a test input. - -#### `terminate_worker` - -This job terminates the worker in the Runpod platform. It: - -- Sends a POST request to the Runpod API to cancel the job (uses `fjogeleit/http-request-action@v1`). - -It only runs if the previous job (`initialize_worker` or `run_tests`) was cancelled or failed. - -### Secrets - -The workflow requires the following secrets to be set in the repository settings: - -- `RUNPOD_API_KEY`: The API key to authenticate requests to the Runpod API. -- `RUNPOD_ENDPOINT`: The endpoint of the Runpod API (not including the base URL). - -### Customizing the Workflow - -This workflow can be customized by editing the `.github/workflows/test-worker.yml` file. For example, to change the Python version, edit the `python-version` input in the `Set up Python 3.11 & install dependencies` step. To add or modify tests, edit the `Execute Tests` step. - -**Note:** Always ensure that the necessary permissions are granted when modifying GitHub workflows. It's also a good practice to validate your changes with a trusted colleague or through a pull request to prevent accidental disruptions to the workflow. +## Testing a worker image in CI + +The `test-image` action runs a worker image's test cases on a temporary serverless endpoint and deletes the endpoint afterwards. Each run gets its own endpoint, so CI does not depend on a long-lived endpoint that can be paused for being idle. + +It uses Flash from runpod-python (`rp flash dev --once`): + +1. The CLI creates a live endpoint from your image, with at most one worker. +2. For each test in your tests file, it sends a job to that endpoint. What the job does on the worker depends on the mode: + - **Handler mode** (default, queue workers): load your handler file, capture the handler it registers with `runpod.serverless.start(...)`, and call it with the test input. + - **Server mode** (load-balancer workers, selected by `start-command`): start your server once with `PORT` set, wait until its health path returns 200, and send the test's HTTP request to it. +3. A handler-mode test fails if the job fails, times out, returns no output, or returns a dict with an `error` key. A server-mode test fails if the server exits or never becomes healthy, or the response status differs from the expected one. +4. The endpoint is deleted when the tests finish, fail, or the run is cancelled. + +### Usage + +Build and push the image first, then: + +```yaml +- uses: runpod-workers/worker-github_runner/test-image@main + with: + image: runpod/worker-example:ci-pr-${{ github.event.number }} + runpod-api-key: ${{ secrets.RUNPOD_API_KEY }} + handler: /handler.py +``` + +| Input | Default | Description | +|---|---|---| +| `image` | (required) | Worker image to test | +| `runpod-api-key` | (required) | API key used to create and delete the endpoint | +| `tests-file` | `.runpod/tests.json` | Hub-format tests file in your checkout | +| `handler` | `/handler.py` | Path of the handler file inside the image (handler mode) | +| `start-command` | (none) | Command that starts your HTTP server; selects server mode. Runs on the worker through `sh -c` | +| `port` | `8000` | Port your server listens on, passed to it as `PORT` (server mode) | +| `health-path` | `/ping` | Path that returns 200 once the server is ready (server mode) | +| `gpu` | `config.gpuTypeId` from the tests file | GPU type for the endpoint | +| `min-cuda` | lowest `config.allowedCudaVersions` | Minimum host CUDA version for the endpoint | +| `startup-timeout` | `900` | Seconds allowed for the worker to pull the image and start, and in server mode for the server to become healthy | +| `runpod-version` | `2.1.0` | runpod-python version for the CLI and the worker | +| `runtime-package-spec` | pinned `runpod-sdk-runtime` | Flash worker runtime installed into the image | + +### Tests file + +The action reads the same format as the Hub: + +```json +{ + "tests": [ + {"name": "basic", "input": {"prompt": "hello"}, "timeout": 10000} + ], + "config": { + "gpuTypeId": "NVIDIA GeForce RTX 4090", + "env": [{"key": "MODEL", "value": "base"}] + } +} +``` + +`timeout` is in milliseconds and also sets the job's execution timeout, together with `startup-timeout`. Entries in `config.env` are set on the endpoint. The lowest entry of `config.allowedCudaVersions` becomes the endpoint's minimum CUDA version. + +In server mode, each test's `input` is an HTTP request. `method` defaults to `GET` and `status` (the expected response status) to `200`: + +```json +{ + "tests": [ + { + "name": "completion", + "input": {"method": "POST", "path": "/v1/completions", "body": {"prompt": "Hello", "max_tokens": 8}}, + "timeout": 120000 + } + ] +} +``` + +```yaml +- uses: runpod-workers/worker-github_runner/test-image@main + with: + image: runpod/worker-example-lb:ci-pr-${{ github.event.number }} + runpod-api-key: ${{ secrets.RUNPOD_API_KEY }} + start-command: cd /src && python3 handler.py +``` + +### What can go wrong + +- **The image's start command does not run.** A live endpoint starts the Flash runtime instead of the image's start command. In handler mode the action calls your handler directly, so code that runs only in the start command (for example, a shell script that runs before `python handler.py`) is not exercised. In server mode, `start-command` should match the image's start command. +- **The worker runs `runpod==`.** The endpoint installs that version into the image before starting, even if the image pins another one. A handler that only works with an older SDK will fail here. +- **The image needs Python 3.10 or newer, with `pip` on its `python3`,** so the runtime can be installed. Images built on a uv-managed venv without pip need `uv pip install pip` in their Dockerfile. +- **Pull requests from forks** do not get repository secrets, so the action cannot create an endpoint for them. Skip the job for forks. diff --git a/test-image/action.yml b/test-image/action.yml new file mode 100644 index 0000000..5186eeb --- /dev/null +++ b/test-image/action.yml @@ -0,0 +1,92 @@ +name: "Test a Runpod worker image" +description: > + Run a worker image's Hub test cases on a temporary Flash live endpoint, + then delete the endpoint. +inputs: + image: + description: "Worker image to test, e.g. runpod/worker-faster_whisper:ci-pr-12" + required: true + runpod-api-key: + description: "Runpod API key used to create and delete the endpoint" + required: true + tests-file: + description: "Hub-format tests file in the caller's checkout" + required: false + default: ".runpod/tests.json" + handler: + description: "Path of the handler file inside the image (handler mode)" + required: false + default: "/handler.py" + start-command: + description: > + Command that starts the image's HTTP server, e.g. "cd /src && python3 handler.py". + Setting it selects server mode, for load-balancer workers. It runs on the + worker through `sh -c`, so shell syntax such as `cd` and `&&` works. + required: false + default: "" + port: + description: "Port the server listens on; passed to it as PORT (server mode)" + required: false + default: "8000" + health-path: + description: "Path that returns 200 once the server is ready (server mode)" + required: false + default: "/ping" + gpu: + description: "GPU type; defaults to config.gpuTypeId in the tests file" + required: false + default: "" + min-cuda: + description: "Minimum host CUDA version; defaults to the lowest config.allowedCudaVersions" + required: false + default: "" + startup-timeout: + description: "Seconds allowed for the worker to pull the image and start (and, in server mode, for the server to become healthy)" + required: false + default: "900" + runpod-version: + description: "runpod-python version used by the CLI and installed into the worker" + required: false + default: "2.1.0" + runtime-package-spec: + description: "pip spec of the Flash worker runtime installed into the image" + required: false + # runpod-sdk-runtime v0.2.0; not published to PyPI + default: "https://github.com/runpod-workers/sdk-runtime/archive/14f70227008c1ec317e3fe1865a4688b14f5d432.tar.gz" +runs: + using: "composite" + steps: + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install runpod-python + shell: bash + env: + RUNPOD_VERSION: ${{ inputs.runpod-version }} + run: python -m pip install --quiet "runpod==${RUNPOD_VERSION}" + + - name: Run tests on a temporary endpoint + shell: bash + env: + RUNPOD_API_KEY: ${{ inputs.runpod-api-key }} + RP_TEST_IMAGE: ${{ inputs.image }} + RP_TEST_FILE: ${{ inputs.tests-file }} + RP_TEST_HANDLER: ${{ inputs.handler }} + RP_TEST_START_COMMAND: ${{ inputs.start-command }} + RP_TEST_PORT: ${{ inputs.port }} + RP_TEST_HEALTH_PATH: ${{ inputs.health-path }} + RP_TEST_GPU: ${{ inputs.gpu }} + RP_TEST_MIN_CUDA: ${{ inputs.min-cuda }} + RP_TEST_STARTUP_S: ${{ inputs.startup-timeout }} + # The live endpoint's launcher force-installs these into the image, + # so the worker runs the same SDK as the CLI. + RUNPOD_PACKAGE_SPEC: runpod==${{ inputs.runpod-version }} + RUNPOD_RUNTIME_PACKAGE_SPEC: ${{ inputs.runtime-package-spec }} + run: | + export RP_TEST_FILE="$(realpath "$RP_TEST_FILE")" + export RP_TEST_APP_NAME="$(python -c 'import os, sys; sys.path.insert(0, sys.argv[1]); from cases import app_name; print(app_name(os.environ["GITHUB_REPOSITORY"], os.environ["GITHUB_RUN_ID"], os.environ["GITHUB_RUN_ATTEMPT"]))' "$GITHUB_ACTION_PATH")" + echo "Flash app: $RP_TEST_APP_NAME" + # --once runs main(), deletes the endpoint, and exits non-zero if + # main() raised. stdin from /dev/null keeps the session non-interactive. + rp flash dev --once "$GITHUB_ACTION_PATH/flash_app.py" < /dev/null diff --git a/test-image/cases.py b/test-image/cases.py new file mode 100644 index 0000000..6761162 --- /dev/null +++ b/test-image/cases.py @@ -0,0 +1,140 @@ +"""Test-suite parsing and result checks for the test-image action. + +Reads the Hub test format (`.runpod/tests.json`): + + {"tests": [{"name": "basic", "input": {...}, "timeout": 10000}], + "config": {"gpuTypeId": "NVIDIA GeForce RTX 4090", + "allowedCudaVersions": ["12.8", "13.0"], + "env": [{"key": "K", "value": "V"}]}} + +`timeout` is in milliseconds, as on the Hub. The lowest entry of +`allowedCudaVersions` becomes the endpoint's minimum CUDA version. + +In server mode each test's input is an HTTP request instead of a job input: + {"method": "POST", "path": "/v1/completions", "body": {...}, "status": 200} +""" + +import json +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional + +DEFAULT_TIMEOUT_S = 300.0 +MS_PER_S = 1000.0 +# endpoint names carry a "dev-" prefix, a resource name and a hash suffix, +# so the app name gets a conservative share of the length. +MAX_APP_NAME = 48 + + +class SuiteError(ValueError): + """The tests file is missing or malformed.""" + + +@dataclass(frozen=True) +class Case: + name: str + input: Dict[str, Any] + timeout_s: float + + +@dataclass(frozen=True) +class Suite: + cases: List[Case] + gpu: Optional[str] = None + env: Dict[str, str] = field(default_factory=dict) + min_cuda: Optional[str] = None + + +def load_suite(path: Path) -> Suite: + path = Path(path) + if not path.is_file(): + raise SuiteError(f"tests file not found: {path}") + data = json.loads(path.read_text()) + + raw_cases = data.get("tests") or [] + if not raw_cases: + raise SuiteError(f"{path} defines no tests") + + cases = [] + for index, raw in enumerate(raw_cases): + if "name" not in raw or "input" not in raw: + raise SuiteError(f"{path}: test #{index} needs both 'name' and 'input'") + timeout_ms = raw.get("timeout") + timeout_s = timeout_ms / MS_PER_S if timeout_ms else DEFAULT_TIMEOUT_S + cases.append(Case(name=raw["name"], input=raw["input"], timeout_s=timeout_s)) + + config = data.get("config") or {} + env = _parse_env(path, config.get("env") or []) + return Suite( + cases=cases, + gpu=config.get("gpuTypeId"), + env=env, + min_cuda=_lowest_version(config.get("allowedCudaVersions") or []), + ) + + +def _parse_env(path: Path, entries: Any) -> Dict[str, str]: + """Read config.env, a list of {"key": ..., "value": ...} entries.""" + if not isinstance(entries, list): + raise SuiteError(f"{path}: config.env must be a list of key/value entries") + env = {} + for index, entry in enumerate(entries): + if not isinstance(entry, dict) or "key" not in entry or "value" not in entry: + raise SuiteError( + f"{path}: config.env entry #{index} needs both 'key' and 'value'" + ) + env[entry["key"]] = str(entry["value"]) + return env + + +def _lowest_version(versions: List[str]) -> Optional[str]: + if not versions: + return None + return min(versions, key=lambda v: tuple(int(part) for part in v.split("."))) + + +def http_request(case_input: Dict[str, Any]) -> Dict[str, Any]: + """Normalize a server-mode test input into method, path, body and expected status.""" + path = case_input.get("path") + if not isinstance(path, str) or not path.startswith("/"): + raise SuiteError( + f"server-mode test input needs an absolute 'path', got {path!r}" + ) + return { + "method": str(case_input.get("method", "GET")).upper(), + "path": path, + "body": case_input.get("body"), + "status": int(case_input.get("status", 200)), + } + + +def app_name(repository: str, run_id: str, attempt: str) -> str: + """A Flash app name unique to one workflow run attempt.""" + repo = repository.rsplit("/", 1)[-1].lower() + repo = re.sub(r"[^a-z0-9]+", "-", repo).strip("-") + suffix = f"-{run_id}-{attempt}" + budget = MAX_APP_NAME - len("ci-") - len(suffix) + return f"ci-{repo[:budget].rstrip('-')}{suffix}" + + +def check_output(result: Dict[str, Any]) -> Optional[str]: + """Return why a handler result fails the test, or None if it passes.""" + output = result.get("output") + if output is None: + return "handler returned no output" + if isinstance(output, dict) and "error" in output: + return f"handler returned an error: {output['error']}" + return None + + +def check_http_output(result: Dict[str, Any], expected_status: int) -> Optional[str]: + """Return why a server response fails the test, or None if it passes.""" + output = result.get("output") + if output is None: + return "server returned no response" + if output["status"] != expected_status: + return ( + f"expected HTTP {expected_status}, got {output['status']}: {output['body']}" + ) + return None diff --git a/test-image/flash_app.py b/test-image/flash_app.py new file mode 100644 index 0000000..78b2cda --- /dev/null +++ b/test-image/flash_app.py @@ -0,0 +1,342 @@ +"""Flash app that tests a worker image on a temporary live endpoint. + +Run by the test-image action as `rp flash dev --once flash_app.py`. The dev +session creates a live endpoint from the worker image, runs `main`, and +deletes the endpoint when `main` returns or raises. + +A live endpoint replaces the image's start command with the Flash runtime, +so the image's own entrypoint never runs. `run_case` drives the worker from +inside the image instead, in one of two modes: + +- handler mode: load the handler file, capture the handler it passes to + `runpod.serverless.start(...)`, and call it with each test input. +- server mode (load-balancer workers): start the image's HTTP server once, + wait for its health path to return 200, and send each test's request to it. + +Configured through environment variables (set by action.yml): + RP_TEST_IMAGE worker image to test (required) + RP_TEST_FILE Hub-format tests file (required) + RP_TEST_APP_NAME Flash app name, unique per run (required) + RP_TEST_HANDLER handler file inside the image (default /handler.py) + RP_TEST_START_COMMAND server command; setting it selects server mode + RP_TEST_PORT port the server listens on, passed as PORT (default 8000) + RP_TEST_HEALTH_PATH server health path (default /ping) + RP_TEST_GPU GPU type; defaults to config.gpuTypeId in the tests file + RP_TEST_MIN_CUDA minimum CUDA; defaults to the lowest allowedCudaVersions + RP_TEST_STARTUP_S seconds allowed for the worker or server to start (default 900) +""" + +import asyncio +import concurrent.futures +import inspect +import json +import os +import runpy +import signal +import subprocess +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path +from typing import Optional + +import runpod + +DEFAULT_HANDLER = "/handler.py" +DEFAULT_PORT = 8000 +DEFAULT_HEALTH_PATH = "/ping" +DEFAULT_STARTUP_S = 900.0 +# one worker at most; a CI endpoint never needs to scale +CI_WORKERS = (0, 1) +CI_IDLE_TIMEOUT_S = 5 +HEALTH_POLL_S = 1.0 +HEALTH_PROBE_TIMEOUT_S = 2.0 +SERVER_LOG_TAIL_LINES = 50 +# Flash execs this whole module on the worker too. The endpoint env carries +# this marker plus the values below, so the worker can rebuild the app +# without the tests file or cases.py, which exist only on the CI runner. +WORKER_MARKER = "RP_TEST_ON_WORKER" +FORWARDED = ("RP_TEST_IMAGE", "RP_TEST_GPU", "RP_TEST_APP_NAME") + + +def _required(name: str) -> str: + value = os.environ.get(name) + if not value: + raise RuntimeError(f"{name} must be set") + return value + + +if os.environ.get(WORKER_MARKER): + SUITE = None + IMAGE, GPU, APP_NAME = (_required(name) for name in FORWARDED) + MIN_CUDA = os.environ.get("RP_TEST_MIN_CUDA") or None + WORKER_ENV = {} +else: + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from cases import check_http_output, check_output, http_request, load_suite + + SUITE = load_suite(Path(_required("RP_TEST_FILE"))) + IMAGE = _required("RP_TEST_IMAGE") + APP_NAME = _required("RP_TEST_APP_NAME") + GPU = os.environ.get("RP_TEST_GPU") or SUITE.gpu + if not GPU: + raise RuntimeError("set RP_TEST_GPU or config.gpuTypeId in the tests file") + MIN_CUDA = os.environ.get("RP_TEST_MIN_CUDA") or SUITE.min_cuda + WORKER_ENV = { + **SUITE.env, + WORKER_MARKER: "1", + "RP_TEST_IMAGE": IMAGE, + "RP_TEST_GPU": GPU, + "RP_TEST_APP_NAME": APP_NAME, + **({"RP_TEST_MIN_CUDA": MIN_CUDA} if MIN_CUDA else {}), + } + +HANDLER = os.environ.get("RP_TEST_HANDLER") or DEFAULT_HANDLER +START_COMMAND = os.environ.get("RP_TEST_START_COMMAND") or None +STARTUP_S = float(os.environ.get("RP_TEST_STARTUP_S") or DEFAULT_STARTUP_S) +SERVER = { + "command": START_COMMAND, + "port": int(os.environ.get("RP_TEST_PORT") or DEFAULT_PORT), + "health_path": os.environ.get("RP_TEST_HEALTH_PATH") or DEFAULT_HEALTH_PATH, + "startup_s": STARTUP_S, +} + +app = runpod.App(APP_NAME) + + +# ---- worker side: handler mode --------------------------------------------- + + +def _run_coroutine(coro): + """Run a coroutine to completion from sync code, even inside a running loop. + + The Flash runtime calls run_case from inside its event loop, where + asyncio.run() refuses to start, so the coroutine gets its own loop on a + separate thread. + """ + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(coro) + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + return pool.submit(asyncio.run, coro).result() + + +async def _drain(result): + if inspect.isawaitable(result): + result = await result + if inspect.isasyncgen(result): + return [chunk async for chunk in result] + return result + + +def _run_handler(handler_path: str, job: dict) -> dict: + import runpod.serverless + + captured = {} + original_start = runpod.serverless.start + handler_dir = os.path.dirname(os.path.abspath(handler_path)) + original_cwd = os.getcwd() + runpod.serverless.start = captured.update + sys.path.insert(0, handler_dir) + os.chdir(handler_dir) + try: + # run_name="__main__" also covers handlers that start the worker + # under an `if __name__ == "__main__":` guard. + runpy.run_path(handler_path, run_name="__main__") + handler = captured.get("handler") + if handler is None: + raise RuntimeError(f"{handler_path} never called runpod.serverless.start") + + result = handler(job) + if inspect.isgenerator(result): + result = list(result) + elif inspect.isawaitable(result) or inspect.isasyncgen(result): + result = _run_coroutine(_drain(result)) + return {"output": result} + finally: + runpod.serverless.start = original_start + sys.path.remove(handler_dir) + os.chdir(original_cwd) + + +# ---- worker side: server mode ---------------------------------------------- + + +def _server_files(port: int): + # Keyed by port and kept on disk, so a server started by one job is + # found again by the next even if the runtime re-execs this module. + return f"/tmp/rp-test-server-{port}.pid", f"/tmp/rp-test-server-{port}.log" + + +def _exit_code(pid: int): + """The server's exit code if it has exited, else None.""" + try: + done, status = os.waitpid(pid, os.WNOHANG) + except ChildProcessError: # not our child; fall back to a liveness probe + try: + os.kill(pid, 0) + return None + except ProcessLookupError: + return -1 + return os.waitstatus_to_exitcode(status) if done else None + + +def _log_tail(log_file: str) -> str: + try: + with open(log_file, errors="replace") as f: + return "".join(f.readlines()[-SERVER_LOG_TAIL_LINES:]) + except FileNotFoundError: + return "" + + +def _health_status(port: int, path: str): + try: + url = f"http://127.0.0.1:{port}{path}" + with urllib.request.urlopen(url, timeout=HEALTH_PROBE_TIMEOUT_S) as response: + return response.status + except urllib.error.HTTPError as exc: + return exc.code + except OSError: # not listening yet + return None + + +def _read_pid(pid_file: str) -> Optional[int]: + """Return the recorded server pid, or None when none is recorded. + + An empty or corrupt file (a worker that died mid-write) counts as no + server, so the next case starts a fresh one instead of failing forever. + """ + try: + return int(Path(pid_file).read_text()) + except (FileNotFoundError, ValueError): + return None + + +def _ensure_server(server: dict) -> None: + port = server["port"] + pid_file, log_file = _server_files(port) + pid = _read_pid(pid_file) + if pid is None or _exit_code(pid) is not None: + with open(log_file, "w") as log: + process = subprocess.Popen( + server["command"], + shell=True, + env={**os.environ, "PORT": str(port)}, + stdout=log, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + pid = process.pid + Path(pid_file).write_text(str(pid)) + + deadline = time.monotonic() + server["startup_s"] + while True: + code = _exit_code(pid) + if code is not None: + os.remove(pid_file) + raise RuntimeError( + f"server command exited with code {code} before becoming healthy:\n" + f"{_log_tail(log_file)}" + ) + if _health_status(port, server["health_path"]) == 200: + return + if time.monotonic() > deadline: + os.killpg(pid, signal.SIGTERM) + os.remove(pid_file) + raise TimeoutError( + f"server was not healthy within {server['startup_s']:g}s:\n" + f"{_log_tail(log_file)}" + ) + time.sleep(HEALTH_POLL_S) + + +def _send(port: int, request: dict) -> dict: + data = None if request["body"] is None else json.dumps(request["body"]).encode() + http_request = urllib.request.Request( + f"http://127.0.0.1:{port}{request['path']}", + data=data, + method=request["method"], + headers={"Content-Type": "application/json"}, + ) + try: + with urllib.request.urlopen( + http_request, timeout=request["timeout_s"] + ) as response: + status, raw = response.status, response.read() + except urllib.error.HTTPError as exc: + status, raw = exc.code, exc.read() + text = raw.decode(errors="replace") + try: + body = json.loads(text) + except ValueError: + body = text + return {"status": status, "body": body} + + +@app.queue( + name="worker", + image=IMAGE, + gpu=GPU, + workers=CI_WORKERS, + idle_timeout=CI_IDLE_TIMEOUT_S, + env=WORKER_ENV, + min_cuda_version=MIN_CUDA, +) +def run_case(task: dict) -> dict: + """Run one test on the worker. `task` is either + {"handler": path, "job": {...}} or {"server": {...}, "request": {...}}.""" + if "server" in task: + _ensure_server(task["server"]) + return {"output": _send(task["server"]["port"], task["request"])} + return _run_handler(task["handler"], task["job"]) + + +# ---- CI runner side ---------------------------------------------------------- + + +def _task_and_check(case): + """The worker task for a test case and the function that checks its result.""" + if START_COMMAND: + request = http_request(case.input) + task = { + "server": SERVER, + "request": {**request, "timeout_s": case.timeout_s}, + } + return task, lambda result: check_http_output(result, request["status"]) + task = {"handler": HANDLER, "job": {"id": f"ci-{case.name}", "input": case.input}} + return task, check_output + + +@runpod.local_entrypoint +async def main(): + failures = [] + for case in SUITE.cases: + task, check = _task_and_check(case) + budget_s = STARTUP_S + case.timeout_s + print(f"[{case.name}] submitting", flush=True) + try: + # without an explicit limit the endpoint's default execution + # timeout can kill long tests (model downloads, training) + caller = run_case.with_options(execution_timeout=int(budget_s * 1000)) + job = await caller.spawn.aio(task) + result = await job.result.aio(timeout=budget_s) + except Exception as exc: # the job, the worker, or the wait failed + failures.append(f"{case.name}: {type(exc).__name__}: {exc}") + print(f"[{case.name}] FAILED: {exc}", flush=True) + continue + + reason = check(result) + if reason: + failures.append(f"{case.name}: {reason}") + print(f"[{case.name}] FAILED: {reason}", flush=True) + else: + print(f"[{case.name}] passed", flush=True) + + print( + f"{len(SUITE.cases) - len(failures)}/{len(SUITE.cases)} tests passed", + flush=True, + ) + if failures: + raise RuntimeError("worker tests failed:\n" + "\n".join(failures)) diff --git a/test-image/tests/conftest.py b/test-image/tests/conftest.py new file mode 100644 index 0000000..ece8e0c --- /dev/null +++ b/test-image/tests/conftest.py @@ -0,0 +1,4 @@ +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) diff --git a/test-image/tests/test_cases.py b/test-image/tests/test_cases.py new file mode 100644 index 0000000..a82c2d6 --- /dev/null +++ b/test-image/tests/test_cases.py @@ -0,0 +1,187 @@ +import json + +import pytest + +from cases import ( + Case, + SuiteError, + app_name, + check_http_output, + check_output, + http_request, + load_suite, +) + + +def write_suite(tmp_path, suite): + path = tmp_path / "tests.json" + path.write_text(json.dumps(suite)) + return path + + +def test_load_suite_reads_hub_format(tmp_path): + path = write_suite( + tmp_path, + { + "tests": [{"name": "basic", "input": {"audio": "a.wav"}, "timeout": 10000}], + "config": {"gpuTypeId": "NVIDIA GeForce RTX 4090"}, + }, + ) + + suite = load_suite(path) + + assert suite.cases == [Case(name="basic", input={"audio": "a.wav"}, timeout_s=10.0)] + assert suite.gpu == "NVIDIA GeForce RTX 4090" + + +def test_load_suite_reads_config_env_as_dict(tmp_path): + path = write_suite( + tmp_path, + { + "tests": [{"name": "basic", "input": {}}], + "config": {"env": [{"key": "MODEL", "value": "turbo"}]}, + }, + ) + + assert load_suite(path).env == {"MODEL": "turbo"} + + +def test_load_suite_defaults_timeout_and_missing_config(tmp_path): + path = write_suite(tmp_path, {"tests": [{"name": "basic", "input": {}}]}) + + suite = load_suite(path) + + assert suite.cases[0].timeout_s == 300.0 + assert suite.gpu is None + assert suite.env == {} + + +@pytest.mark.parametrize( + "suite", + [ + {"tests": []}, + {}, + {"tests": [{"input": {}}]}, + {"tests": [{"name": "no-input"}]}, + ], +) +def test_load_suite_rejects_invalid_suites(tmp_path, suite): + with pytest.raises(SuiteError): + load_suite(write_suite(tmp_path, suite)) + + +@pytest.mark.parametrize( + "env", + [ + [{"key": "K"}], + [{"value": "V"}], + ["K=V"], + {"K": "V"}, + ], +) +def test_load_suite_rejects_malformed_config_env(tmp_path, env): + suite = {"tests": [{"name": "t", "input": {}}], "config": {"env": env}} + + with pytest.raises(SuiteError, match="config.env"): + load_suite(write_suite(tmp_path, suite)) + + +def test_load_suite_rejects_missing_file(tmp_path): + with pytest.raises(SuiteError, match="not found"): + load_suite(tmp_path / "absent.json") + + +def test_app_name_is_unique_per_run_attempt_and_api_safe(): + name = app_name("runpod-workers/worker-faster_whisper", "42459297624", "2") + + assert name == "ci-worker-faster-whisper-42459297624-2" + + +def test_app_name_truncates_long_repo_names_but_keeps_run_ids(): + name = app_name("org/" + "x" * 80, "123", "1") + + assert name.endswith("-123-1") + assert len(name) <= 48 + + +@pytest.mark.parametrize( + ("result", "expected"), + [ + ({"output": {"text": "four score"}}, None), + ({"output": [1, 2]}, None), + ( + {"output": {"error": "model missing"}}, + "handler returned an error: model missing", + ), + ({"output": None}, "handler returned no output"), + ], +) +def test_check_output(result, expected): + assert check_output(result) == expected + + +def test_load_suite_takes_lowest_allowed_cuda_version(tmp_path): + path = write_suite( + tmp_path, + { + "tests": [{"name": "basic", "input": {}}], + "config": {"allowedCudaVersions": ["12.7", "12.10", "12.3"]}, + }, + ) + + assert load_suite(path).min_cuda == "12.3" + + +def test_load_suite_without_cuda_versions_sets_no_minimum(tmp_path): + path = write_suite(tmp_path, {"tests": [{"name": "basic", "input": {}}]}) + + assert load_suite(path).min_cuda is None + + +def test_http_request_reads_method_path_body_and_expected_status(): + request = http_request( + { + "method": "post", + "path": "/v1/completions", + "body": {"prompt": "hi"}, + "status": 201, + } + ) + + assert request == { + "method": "POST", + "path": "/v1/completions", + "body": {"prompt": "hi"}, + "status": 201, + } + + +def test_http_request_defaults_to_get_and_200(): + assert http_request({"path": "/ping"}) == { + "method": "GET", + "path": "/ping", + "body": None, + "status": 200, + } + + +def test_http_request_requires_an_absolute_path(): + with pytest.raises(SuiteError, match="path"): + http_request({"method": "GET"}) + with pytest.raises(SuiteError, match="path"): + http_request({"path": "v1/completions"}) + + +@pytest.mark.parametrize( + ("result", "expected"), + [ + ({"output": {"status": 200, "body": {"choices": []}}}, None), + ( + {"output": {"status": 500, "body": {"detail": "boom"}}}, + "expected HTTP 200, got 500: {'detail': 'boom'}", + ), + ({"output": None}, "server returned no response"), + ], +) +def test_check_http_output(result, expected): + assert check_http_output(result, 200) == expected diff --git a/test-image/tests/test_run_case.py b/test-image/tests/test_run_case.py new file mode 100644 index 0000000..a178b6a --- /dev/null +++ b/test-image/tests/test_run_case.py @@ -0,0 +1,397 @@ +import importlib +import json +import sys +import textwrap +from pathlib import Path + +import pytest + + +@pytest.fixture +def flash_app(tmp_path, monkeypatch): + suite = tmp_path / "tests.json" + suite.write_text(json.dumps({"tests": [{"name": "basic", "input": {}}]})) + monkeypatch.setenv("RP_TEST_IMAGE", "runpod/worker-example:ci") + monkeypatch.setenv("RP_TEST_FILE", str(suite)) + monkeypatch.setenv("RP_TEST_GPU", "NVIDIA GeForce RTX 4090") + monkeypatch.setenv("RP_TEST_APP_NAME", "ci-example-1-1") + import flash_app as module + + return importlib.reload(module) + + +def handler_task(handler, job): + return {"handler": handler, "job": job} + + +def write_handler(tmp_path, source, name="rp_handler.py"): + path = tmp_path / name + path.write_text(textwrap.dedent(source)) + return str(path) + + +def test_runs_handler_registered_at_module_level(flash_app, tmp_path): + handler = write_handler( + tmp_path, + """ + import runpod + + def handler(job): + return {"echo": job["input"]["text"], "id": job["id"]} + + runpod.serverless.start({"handler": handler}) + """, + ) + + result = flash_app.run_case.local( + handler_task(handler, {"id": "ci-basic", "input": {"text": "hi"}}) + ) + + assert result == {"output": {"echo": "hi", "id": "ci-basic"}} + + +def test_runs_handler_registered_under_main_guard(flash_app, tmp_path): + handler = write_handler( + tmp_path, + """ + import runpod + + def handler(job): + return "ok" + + if __name__ == "__main__": + runpod.serverless.start({"handler": handler}) + """, + ) + + assert flash_app.run_case.local( + handler_task(handler, {"id": "x", "input": {}}) + ) == {"output": "ok"} + + +def test_handler_can_import_sibling_modules(flash_app, tmp_path): + write_handler(tmp_path, "VALUE = 42\n", name="predict.py") + handler = write_handler( + tmp_path, + """ + import runpod + import predict + + runpod.serverless.start({"handler": lambda job: predict.VALUE}) + """, + ) + + assert flash_app.run_case.local( + handler_task(handler, {"id": "x", "input": {}}) + ) == {"output": 42} + + +def test_collects_generator_output(flash_app, tmp_path): + handler = write_handler( + tmp_path, + """ + import runpod + + def handler(job): + yield 1 + yield 2 + + runpod.serverless.start({"handler": handler}) + """, + ) + + assert flash_app.run_case.local( + handler_task(handler, {"id": "x", "input": {}}) + ) == {"output": [1, 2]} + + +def test_awaits_async_handler(flash_app, tmp_path): + handler = write_handler( + tmp_path, + """ + import runpod + + async def handler(job): + return "async ok" + + runpod.serverless.start({"handler": handler}) + """, + ) + + assert flash_app.run_case.local( + handler_task(handler, {"id": "x", "input": {}}) + ) == {"output": "async ok"} + + +def test_collects_async_generator_output(flash_app, tmp_path): + handler = write_handler( + tmp_path, + """ + import runpod + + async def handler(job): + yield "a" + yield "b" + + runpod.serverless.start({"handler": handler}) + """, + ) + + assert flash_app.run_case.local( + handler_task(handler, {"id": "x", "input": {}}) + ) == {"output": ["a", "b"]} + + +def test_fails_when_handler_file_never_starts_a_worker(flash_app, tmp_path): + handler = write_handler(tmp_path, "def handler(job):\n return 1\n") + + with pytest.raises(RuntimeError, match="never called runpod.serverless.start"): + flash_app.run_case.local(handler_task(handler, {"id": "x", "input": {}})) + + +def test_restores_serverless_start_after_the_call(flash_app, tmp_path): + import runpod.serverless + + original = runpod.serverless.start + handler = write_handler( + tmp_path, + "import runpod\nrunpod.serverless.start({'handler': lambda job: 1})\n", + ) + + flash_app.run_case.local(handler_task(handler, {"id": "x", "input": {}})) + + assert runpod.serverless.start is original + + +def test_shipped_module_loads_on_worker_without_local_files( + flash_app, tmp_path, monkeypatch +): + # Live endpoints exec the whole module on the worker, where cases.py, + # the tests file, and the action's env vars do not exist. Only the env + # the endpoint was created with is there. + from runpod.apps.serialization import get_function_source + + source = get_function_source(flash_app.run_case._fn) + worker_env = flash_app.run_case.spec.env + for name in ("RP_TEST_IMAGE", "RP_TEST_FILE", "RP_TEST_GPU", "RP_TEST_APP_NAME"): + monkeypatch.delenv(name, raising=False) + for name, value in worker_env.items(): + monkeypatch.setenv(name, value) + monkeypatch.setitem(sys.modules, "cases", None) # any import of it fails + + namespace = {"__name__": "shipped"} + exec(compile(source, "shipped.py", "exec"), namespace) + handler = write_handler( + tmp_path, + "import runpod\nrunpod.serverless.start({'handler': lambda job: 'shipped'})\n", + ) + + assert namespace["run_case"].local( + handler_task(handler, {"id": "x", "input": {}}) + ) == {"output": "shipped"} + + +def test_worker_env_carries_suite_env(tmp_path, monkeypatch): + suite = tmp_path / "tests.json" + suite.write_text( + json.dumps( + { + "tests": [{"name": "basic", "input": {}}], + "config": {"env": [{"key": "MODEL", "value": "turbo"}]}, + } + ) + ) + monkeypatch.setenv("RP_TEST_IMAGE", "runpod/worker-example:ci") + monkeypatch.setenv("RP_TEST_FILE", str(suite)) + monkeypatch.setenv("RP_TEST_GPU", "NVIDIA GeForce RTX 4090") + monkeypatch.setenv("RP_TEST_APP_NAME", "ci-example-1-1") + import flash_app as module + + env = importlib.reload(module).run_case.spec.env + + assert env["MODEL"] == "turbo" + assert env["RP_TEST_IMAGE"] == "runpod/worker-example:ci" + + +# ---- server mode ------------------------------------------------------------ + +SERVER = """ +import json, os, sys +from http.server import BaseHTTPRequestHandler, HTTPServer + +with open(os.environ["STARTS_FILE"], "a") as f: + f.write("start\\n") + +class Handler(BaseHTTPRequestHandler): + def _send(self, code, body): + data = json.dumps(body).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def do_GET(self): + if self.path == "/ping": + return self._send(200, {"status": "healthy"}) + self._send(404, {"detail": "not found"}) + + def do_POST(self): + length = int(self.headers.get("Content-Length", 0)) + body = json.loads(self.rfile.read(length) or b"null") + if self.path == "/fail": + return self._send(500, {"detail": "boom"}) + self._send(200, {"echo": body}) + + def log_message(self, *args): + pass + +HTTPServer(("127.0.0.1", int(os.environ["PORT"])), Handler).serve_forever() +""" + + +def free_port(): + import socket + + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +@pytest.fixture +def server(tmp_path, monkeypatch): + script = tmp_path / "server.py" + script.write_text(SERVER) + starts = tmp_path / "starts.txt" + monkeypatch.setenv("STARTS_FILE", str(starts)) + port = free_port() + spec = { + "command": f"{sys.executable} {script}", + "port": port, + "health_path": "/ping", + "startup_s": 20, + } + yield spec, starts + import os + import signal + + pid_file = f"/tmp/rp-test-server-{port}.pid" + if os.path.exists(pid_file): + try: + os.killpg(int(Path(pid_file).read_text()), signal.SIGTERM) + except ProcessLookupError: + pass + os.remove(pid_file) + + +def server_task(spec, method="GET", path="/ping", body=None, timeout_s=10): + return { + "server": spec, + "request": { + "method": method, + "path": path, + "body": body, + "timeout_s": timeout_s, + }, + } + + +def test_server_mode_starts_server_and_returns_response(flash_app, server): + spec, _ = server + + result = flash_app.run_case.local( + server_task(spec, "POST", "/echo", {"prompt": "hi"}) + ) + + assert result == {"output": {"status": 200, "body": {"echo": {"prompt": "hi"}}}} + + +def test_server_mode_returns_error_statuses_instead_of_raising(flash_app, server): + spec, _ = server + + result = flash_app.run_case.local(server_task(spec, "POST", "/fail", {})) + + assert result == {"output": {"status": 500, "body": {"detail": "boom"}}} + + +def test_server_mode_reuses_a_running_server(flash_app, server): + spec, starts = server + + flash_app.run_case.local(server_task(spec)) + flash_app.run_case.local(server_task(spec)) + + assert starts.read_text().count("start") == 1 + + +def test_server_mode_starts_a_server_over_an_unreadable_pid_file(flash_app, server): + spec, starts = server + Path(f"/tmp/rp-test-server-{spec['port']}.pid").write_text("") + + result = flash_app.run_case.local(server_task(spec)) + + assert result["output"]["status"] == 200 + assert starts.read_text().count("start") == 1 + + +def test_server_mode_reports_a_crashed_server_with_its_log(flash_app, tmp_path): + crash = tmp_path / "crash.py" + crash.write_text( + "raise ImportError('attempted relative import with no known parent package')\n" + ) + spec = { + "command": f"{sys.executable} {crash}", + "port": free_port(), + "health_path": "/ping", + "startup_s": 20, + } + + with pytest.raises( + RuntimeError, match="exited with code 1(.|\\n)*attempted relative import" + ): + flash_app.run_case.local(server_task(spec)) + + +def test_server_mode_times_out_when_never_healthy(flash_app, tmp_path): + idle = tmp_path / "idle.py" + idle.write_text("import time\ntime.sleep(60)\n") + port = free_port() + spec = { + "command": f"{sys.executable} {idle}", + "port": port, + "health_path": "/ping", + "startup_s": 2, + } + + try: + with pytest.raises(TimeoutError, match="not healthy within 2"): + flash_app.run_case.local(server_task(spec)) + finally: + import os + import signal + + pid_file = f"/tmp/rp-test-server-{port}.pid" + if os.path.exists(pid_file): + os.killpg(int(Path(pid_file).read_text()), signal.SIGTERM) + os.remove(pid_file) + + +def test_awaits_async_handler_when_called_inside_a_running_loop(flash_app, tmp_path): + # The Flash runtime calls run_case from inside its own event loop. + import asyncio + + handler = write_handler( + tmp_path, + """ + import runpod + + async def handler(job): + return "inside loop ok" + + runpod.serverless.start({"handler": handler}) + """, + ) + + async def runtime(): + return flash_app.run_case.local(handler_task(handler, {"id": "x", "input": {}})) + + assert asyncio.run(runtime()) == {"output": "inside loop ok"}