From 44f823ccfc56e5f1d4356b3be803c2783d0a5a51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dean=20Qui=C3=B1anola?= Date: Sun, 8 Nov 2026 10:24:21 -0800 Subject: [PATCH 1/5] feat(ci): add test-image action for per-run endpoint tests Worker repos ran GPU tests by POSTing to a long-lived runner endpoint. Once that endpoint idles it is paused (max_workers=0) and every run fails with 409 ENDPOINT_PAUSED. The test-image action runs a worker image's .runpod/tests.json on a temporary Flash live endpoint (rp flash dev --once) and deletes it afterwards: - the live endpoint force-installs runpod and the pinned runpod-sdk-runtime into the image - on the worker, the job loads the handler file, captures the handler passed to runpod.serverless.start, and calls it - the module is exec'd on the worker too, so its config travels in the endpoint env Removes TEMPLATE-run_worker_tests.yml, which sent the GitHub PAT in job input to the long-lived endpoint. --- .../workflows/TEMPLATE-run_worker_tests.yml | 93 ------------------- 1 file changed, 93 deletions(-) delete mode 100644 .github/workflows/TEMPLATE-run_worker_tests.yml diff --git a/.github/workflows/TEMPLATE-run_worker_tests.yml b/.github/workflows/TEMPLATE-run_worker_tests.yml deleted file mode 100644 index 1777fba..0000000 --- a/.github/workflows/TEMPLATE-run_worker_tests.yml +++ /dev/null @@ -1,93 +0,0 @@ -name: CI | Test Worker - -# Requires the following secrets: -# - RUNPOD_API_KEY: The API key to authenticate with the Runpod API -# - GH_PAT: A GitHub Personal Access Token with access to the repository - -# Requires the following variables: -# - RUNPOD_ENDPOINT: The endpoint of the Runpod API - -on: - # push: - # branches: - # - main - - # pull_request: - # branches: - # - main - - workflow_dispatch: - -jobs: - initialize_runner: - runs-on: ubuntu-latest - outputs: - id: ${{ steps.extract_id.outputs.runpod_job_id }} - - steps: - - name: Deploy Worker - id: deploy - uses: fjogeleit/http-request-action@v1 - with: - url: "https://api.runpod.ai/v2/${{ vars.RUNPOD_ENDPOINT }}/run" - method: "POST" - customHeaders: '{"Content-Type": "application/json"}' - bearerToken: ${{ secrets.RUNPOD_API_KEY }} - data: '{"input":{"github_pat": "${{ secrets.GH_PAT }}", "github_org":"${{ github.repository_owner }}"}}' - - - name: Extract Job ID - id: extract_id - run: | - ID=$(echo '${{ steps.deploy.outputs.response }}' | jq -r '.id') - echo "::set-output name=runpod_job_id::$ID" - - run_tests: - needs: initialize_runner - runs-on: runpod - - steps: - - uses: actions/checkout@v3 - - - name: Set up Python 3.11 & install dependencies - uses: actions/setup-python@v4 - with: - python-version: "3.11" - - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install -r builder/requirements.txt - - - name: Execute Tests - run: | - python src/handler.py --test_input='{"input": "test"}' - - terminate_runner: - if: always() - needs: [run_tests, initialize_runner] - runs-on: ubuntu-latest - - steps: - - name: Get Job Status - id: get_status - uses: fjogeleit/http-request-action@v1 - with: - url: "https://api.runpod.ai/v2/${{ vars.RUNPOD_ENDPOINT }}/status/${{ needs.initialize_runner.outputs.id }}" - method: "GET" - customHeaders: '{"Content-Type": "application/json"}' - bearerToken: ${{ secrets.RUNPOD_API_KEY }} - - - name: Parse Status - id: parse_status - run: | - STATUS=$(echo '${{ steps.get_status.outputs.response }}' | jq -r '.status') - echo "::set-output name=status::$STATUS" - - - name: Shutdown Worker - if: ${{ steps.parse_status.outputs.status != 'COMPLETED' }} - uses: fjogeleit/http-request-action@v1 - with: - url: "https://api.runpod.ai/v2/${{ vars.RUNPOD_ENDPOINT }}/cancel/${{ needs.initialize_runner.outputs.id }}" - method: "POST" - customHeaders: '{"Content-Type": "application/json"}' - bearerToken: ${{ secrets.RUNPOD_API_KEY }} From 757ffa21a01a55d9fa9bbf8d1adc4f4b2135c3e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dean=20Qui=C3=B1anola?= Date: Sun, 8 Nov 2026 10:24:40 -0800 Subject: [PATCH 2/5] feat(ci): add test-image action for per-run endpoint tests The test-image action runs a worker image's .runpod/tests.json on a temporary Flash live endpoint (rp flash dev --once) and deletes it afterwards, replacing the long-lived runner endpoint that fails with 409 ENDPOINT_PAUSED once it idles. - the live endpoint force-installs runpod and the pinned runpod-sdk-runtime into the image - on the worker, the job loads the handler file, captures the handler passed to runpod.serverless.start, and calls it - the module is exec'd on the worker too, so its config travels in the endpoint env - docs/usage.md documents inputs, the tests file, and limitations --- .github/workflows/CI-test_image_action.yml | 22 +++ README.md | 4 + docs/usage.md | 109 ++++++----- test-image/action.yml | 69 +++++++ test-image/cases.py | 82 ++++++++ test-image/flash_app.py | 153 +++++++++++++++ test-image/tests/conftest.py | 4 + test-image/tests/test_cases.py | 96 ++++++++++ test-image/tests/test_run_case.py | 206 +++++++++++++++++++++ 9 files changed, 694 insertions(+), 51 deletions(-) create mode 100644 .github/workflows/CI-test_image_action.yml create mode 100644 test-image/action.yml create mode 100644 test-image/cases.py create mode 100644 test-image/flash_app.py create mode 100644 test-image/tests/conftest.py create mode 100644 test-image/tests/test_cases.py create mode 100644 test-image/tests/test_run_case.py diff --git a/.github/workflows/CI-test_image_action.yml b/.github/workflows/CI-test_image_action.yml new file mode 100644 index 0000000..7021278 --- /dev/null +++ b/.github/workflows/CI-test_image_action.yml @@ -0,0 +1,22 @@ +name: CI | test-image action + +on: + pull_request: + paths: + - "test-image/**" + - ".github/workflows/CI-test_image_action.yml" + push: + branches: + - main + paths: + - "test-image/**" + +jobs: + unit-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + - name: Run unit tests + working-directory: test-image + run: uv run -p 3.12 --no-project --with pytest --with runpod==2.1.0 pytest -q tests diff --git a/README.md b/README.md index 2493252..495be05 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,10 @@ This repository hosts the code for deploying a GitHub Actions worker on the Runp 5. Deploy your worker to the Runpod platform. +## Testing worker images in CI + +The [`test-image`](./test-image) action runs a worker image's `.runpod/tests.json` cases on a temporary endpoint and deletes it afterwards. See [docs/usage.md](./docs/usage.md). + ## Documentation For more details on how this GitHub Actions worker functions and how to integrate it into your CI/CD pipeline, please refer to the [Documentation](./docs). diff --git a/docs/usage.md b/docs/usage.md index 1b5f88a..c59547a 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -1,53 +1,60 @@ # Documentation -## CI/CD with GitHub Actions and Runpod - -This repository uses GitHub Actions to run a series of tests on the Runpod serverless platform. The workflow is defined in the `.github/workflows/test-worker.yml` file, with the primary purpose to deploy a worker to the Runpod platform, execute the tests, and then terminate the worker. - -Here is a step-by-step breakdown of the `test-worker.yml` file: - -### Triggers - -This workflow runs whenever: - -- A push is made to the main branch. -- A pull request is opened to the main branch. -- The workflow is manually triggered (workflow_dispatch). - -### Jobs - -#### `initialize_worker` - -This job deploys a worker to the Runpod platform. It: - -- Sends a POST request to the Runpod API to start a new job (uses `fjogeleit/http-request-action@v1`). -- Extracts the job ID from the response and sets it as a job output. - -#### `run_tests` - -This job runs the tests in the worker environment. It: - -- Checks out the repository code. -- Sets up Python 3.11 and installs dependencies. -- Executes tests by running `handler.py` with a test input. - -#### `terminate_worker` - -This job terminates the worker in the Runpod platform. It: - -- Sends a POST request to the Runpod API to cancel the job (uses `fjogeleit/http-request-action@v1`). - -It only runs if the previous job (`initialize_worker` or `run_tests`) was cancelled or failed. - -### Secrets - -The workflow requires the following secrets to be set in the repository settings: - -- `RUNPOD_API_KEY`: The API key to authenticate requests to the Runpod API. -- `RUNPOD_ENDPOINT`: The endpoint of the Runpod API (not including the base URL). - -### Customizing the Workflow - -This workflow can be customized by editing the `.github/workflows/test-worker.yml` file. For example, to change the Python version, edit the `python-version` input in the `Set up Python 3.11 & install dependencies` step. To add or modify tests, edit the `Execute Tests` step. - -**Note:** Always ensure that the necessary permissions are granted when modifying GitHub workflows. It's also a good practice to validate your changes with a trusted colleague or through a pull request to prevent accidental disruptions to the workflow. +## Testing a worker image in CI + +The `test-image` action runs a worker image's test cases on a temporary serverless endpoint and deletes the endpoint afterwards. Each run gets its own endpoint, so CI does not depend on a long-lived endpoint that can be paused for being idle. + +It uses Flash from runpod-python (`rp flash dev --once`): + +1. The CLI creates a live endpoint from your image, with at most one worker. +2. For each test in your tests file, it sends a job to that endpoint. On the worker, the job loads your handler file, captures the handler it registers with `runpod.serverless.start(...)`, and calls it with the test input. +3. A test fails if the job fails, times out, returns no output, or returns a dict with an `error` key. +4. The endpoint is deleted when the tests finish, fail, or the run is cancelled. + +### Usage + +Build and push the image first, then: + +```yaml +- uses: runpod-workers/worker-github_runner/test-image@main + with: + image: runpod/worker-example:ci-pr-${{ github.event.number }} + runpod-api-key: ${{ secrets.RUNPOD_API_KEY }} + handler: /handler.py +``` + +| Input | Default | Description | +|---|---|---| +| `image` | (required) | Worker image to test | +| `runpod-api-key` | (required) | API key used to create and delete the endpoint | +| `tests-file` | `.runpod/tests.json` | Hub-format tests file in your checkout | +| `handler` | `/handler.py` | Path of the handler file inside the image | +| `gpu` | `config.gpuTypeId` from the tests file | GPU type for the endpoint | +| `startup-timeout` | `900` | Seconds allowed for the first worker to pull the image and start | +| `runpod-version` | `2.1.0` | runpod-python version for the CLI and the worker | +| `runtime-package-spec` | pinned `runpod-sdk-runtime` | Flash worker runtime installed into the image | + +### Tests file + +The action reads the same format as the Hub: + +```json +{ + "tests": [ + {"name": "basic", "input": {"prompt": "hello"}, "timeout": 10000} + ], + "config": { + "gpuTypeId": "NVIDIA GeForce RTX 4090", + "env": [{"key": "MODEL", "value": "base"}] + } +} +``` + +`timeout` is in milliseconds. Entries in `config.env` are set on the endpoint. + +### What can go wrong + +- **The image's start command does not run.** A live endpoint starts the Flash runtime instead of the image's start command. The action calls your handler directly, so code that runs only in the start command (for example, a shell script that runs before `python handler.py`) is not exercised. +- **The worker runs `runpod==`.** The endpoint installs that version into the image before starting, even if the image pins another one. A handler that only works with an older SDK will fail here. +- **The image needs Python 3.10 or newer**, with `pip`, so the runtime can be installed. +- **Pull requests from forks** do not get repository secrets, so the action cannot create an endpoint for them. Skip the job for forks. diff --git a/test-image/action.yml b/test-image/action.yml new file mode 100644 index 0000000..6584c2a --- /dev/null +++ b/test-image/action.yml @@ -0,0 +1,69 @@ +name: "Test a Runpod worker image" +description: > + Run a worker image's Hub test cases on a temporary Flash live endpoint, + then delete the endpoint. +inputs: + image: + description: "Worker image to test, e.g. runpod/worker-faster_whisper:ci-pr-12" + required: true + runpod-api-key: + description: "Runpod API key used to create and delete the endpoint" + required: true + tests-file: + description: "Hub-format tests file in the caller's checkout" + required: false + default: ".runpod/tests.json" + handler: + description: "Path of the handler file inside the image" + required: false + default: "/handler.py" + gpu: + description: "GPU type; defaults to config.gpuTypeId in the tests file" + required: false + default: "" + startup-timeout: + description: "Seconds allowed for the first worker to pull the image and start" + required: false + default: "900" + runpod-version: + description: "runpod-python version used by the CLI and installed into the worker" + required: false + default: "2.1.0" + runtime-package-spec: + description: "pip spec of the Flash worker runtime installed into the image" + required: false + # runpod-sdk-runtime v0.2.0; not published to PyPI + default: "https://github.com/runpod-workers/sdk-runtime/archive/14f70227008c1ec317e3fe1865a4688b14f5d432.tar.gz" +runs: + using: "composite" + steps: + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install runpod-python + shell: bash + env: + RUNPOD_VERSION: ${{ inputs.runpod-version }} + run: python -m pip install --quiet "runpod==${RUNPOD_VERSION}" + + - name: Run tests on a temporary endpoint + shell: bash + env: + RUNPOD_API_KEY: ${{ inputs.runpod-api-key }} + RP_TEST_IMAGE: ${{ inputs.image }} + RP_TEST_FILE: ${{ inputs.tests-file }} + RP_TEST_HANDLER: ${{ inputs.handler }} + RP_TEST_GPU: ${{ inputs.gpu }} + RP_TEST_STARTUP_S: ${{ inputs.startup-timeout }} + # The live endpoint's launcher force-installs these into the image, + # so the worker runs the same SDK as the CLI. + RUNPOD_PACKAGE_SPEC: runpod==${{ inputs.runpod-version }} + RUNPOD_RUNTIME_PACKAGE_SPEC: ${{ inputs.runtime-package-spec }} + run: | + export RP_TEST_FILE="$(realpath "$RP_TEST_FILE")" + export RP_TEST_APP_NAME="$(python -c 'import os, sys; sys.path.insert(0, sys.argv[1]); from cases import app_name; print(app_name(os.environ["GITHUB_REPOSITORY"], os.environ["GITHUB_RUN_ID"], os.environ["GITHUB_RUN_ATTEMPT"]))' "$GITHUB_ACTION_PATH")" + echo "Flash app: $RP_TEST_APP_NAME" + # --once runs main(), deletes the endpoint, and exits non-zero if + # main() raised. stdin from /dev/null keeps the session non-interactive. + rp flash dev --once "$GITHUB_ACTION_PATH/flash_app.py" < /dev/null diff --git a/test-image/cases.py b/test-image/cases.py new file mode 100644 index 0000000..dbf5a87 --- /dev/null +++ b/test-image/cases.py @@ -0,0 +1,82 @@ +"""Test-suite parsing and result checks for the test-image action. + +Reads the Hub test format (`.runpod/tests.json`): + + {"tests": [{"name": "basic", "input": {...}, "timeout": 10000}], + "config": {"gpuTypeId": "NVIDIA GeForce RTX 4090", + "env": [{"key": "K", "value": "V"}]}} + +`timeout` is in milliseconds, as on the Hub. +""" + +import json +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional + +DEFAULT_TIMEOUT_S = 300.0 +MS_PER_S = 1000.0 +# endpoint names carry a "dev-" prefix, a resource name and a hash suffix, +# so the app name gets a conservative share of the length. +MAX_APP_NAME = 48 + + +class SuiteError(ValueError): + """The tests file is missing or malformed.""" + + +@dataclass(frozen=True) +class Case: + name: str + input: Dict[str, Any] + timeout_s: float + + +@dataclass(frozen=True) +class Suite: + cases: List[Case] + gpu: Optional[str] = None + env: Dict[str, str] = field(default_factory=dict) + + +def load_suite(path: Path) -> Suite: + path = Path(path) + if not path.is_file(): + raise SuiteError(f"tests file not found: {path}") + data = json.loads(path.read_text()) + + raw_cases = data.get("tests") or [] + if not raw_cases: + raise SuiteError(f"{path} defines no tests") + + cases = [] + for index, raw in enumerate(raw_cases): + if "name" not in raw or "input" not in raw: + raise SuiteError(f"{path}: test #{index} needs both 'name' and 'input'") + timeout_ms = raw.get("timeout") + timeout_s = timeout_ms / MS_PER_S if timeout_ms else DEFAULT_TIMEOUT_S + cases.append(Case(name=raw["name"], input=raw["input"], timeout_s=timeout_s)) + + config = data.get("config") or {} + env = {item["key"]: str(item["value"]) for item in config.get("env") or []} + return Suite(cases=cases, gpu=config.get("gpuTypeId"), env=env) + + +def app_name(repository: str, run_id: str, attempt: str) -> str: + """A Flash app name unique to one workflow run attempt.""" + repo = repository.rsplit("/", 1)[-1].lower() + repo = re.sub(r"[^a-z0-9]+", "-", repo).strip("-") + suffix = f"-{run_id}-{attempt}" + budget = MAX_APP_NAME - len("ci-") - len(suffix) + return f"ci-{repo[:budget].rstrip('-')}{suffix}" + + +def check_output(result: Dict[str, Any]) -> Optional[str]: + """Return why a handler result fails the test, or None if it passes.""" + output = result.get("output") + if output is None: + return "handler returned no output" + if isinstance(output, dict) and "error" in output: + return f"handler returned an error: {output['error']}" + return None diff --git a/test-image/flash_app.py b/test-image/flash_app.py new file mode 100644 index 0000000..429090d --- /dev/null +++ b/test-image/flash_app.py @@ -0,0 +1,153 @@ +"""Flash app that tests a worker image on a temporary live endpoint. + +Run by the test-image action as `rp flash dev --once flash_app.py`. The dev +session creates a live endpoint from the worker image, runs `main`, and +deletes the endpoint when `main` returns or raises. + +A live endpoint replaces the image's start command with the Flash runtime, +so the image's own `runpod.serverless.start(...)` never runs. Instead, +`run_case` loads the worker's handler file inside the image and calls its +handler directly. The handler still runs with the image's code, models, and +libraries. + +Configured through environment variables (set by action.yml): + RP_TEST_IMAGE worker image to test (required) + RP_TEST_FILE Hub-format tests file (required) + RP_TEST_APP_NAME Flash app name, unique per run (required) + RP_TEST_HANDLER handler file inside the image (default /handler.py) + RP_TEST_GPU GPU type; defaults to config.gpuTypeId in the tests file + RP_TEST_STARTUP_S seconds allowed for the first worker to start (default 900) +""" + +import os +import sys +from pathlib import Path + +import runpod + +DEFAULT_HANDLER = "/handler.py" +DEFAULT_STARTUP_S = 900.0 +# one worker at most; a CI endpoint never needs to scale +CI_WORKERS = (0, 1) +CI_IDLE_TIMEOUT_S = 5 +# Flash execs this whole module on the worker too. The endpoint env carries +# this marker plus the values below, so the worker can rebuild the app +# without the tests file or cases.py, which exist only on the CI runner. +WORKER_MARKER = "RP_TEST_ON_WORKER" +FORWARDED = ("RP_TEST_IMAGE", "RP_TEST_GPU", "RP_TEST_APP_NAME") + + +def _required(name: str) -> str: + value = os.environ.get(name) + if not value: + raise RuntimeError(f"{name} must be set") + return value + + +if os.environ.get(WORKER_MARKER): + SUITE = None + IMAGE, GPU, APP_NAME = (_required(name) for name in FORWARDED) + WORKER_ENV = {} +else: + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from cases import check_output, load_suite + + SUITE = load_suite(Path(_required("RP_TEST_FILE"))) + IMAGE = _required("RP_TEST_IMAGE") + APP_NAME = _required("RP_TEST_APP_NAME") + GPU = os.environ.get("RP_TEST_GPU") or SUITE.gpu + if not GPU: + raise RuntimeError("set RP_TEST_GPU or config.gpuTypeId in the tests file") + WORKER_ENV = { + **SUITE.env, + WORKER_MARKER: "1", + "RP_TEST_IMAGE": IMAGE, + "RP_TEST_GPU": GPU, + "RP_TEST_APP_NAME": APP_NAME, + } + +HANDLER = os.environ.get("RP_TEST_HANDLER") or DEFAULT_HANDLER +STARTUP_S = float(os.environ.get("RP_TEST_STARTUP_S") or DEFAULT_STARTUP_S) + +app = runpod.App(APP_NAME) + + +@app.queue( + name="worker", + image=IMAGE, + gpu=GPU, + workers=CI_WORKERS, + idle_timeout=CI_IDLE_TIMEOUT_S, + env=WORKER_ENV, +) +def run_case(handler_path: str, job: dict) -> dict: + # Runs on the worker, so imports stay local to this function. + import asyncio + import inspect + import os + import runpy + import sys + + import runpod.serverless + + async def _drain(result): + if inspect.isawaitable(result): + result = await result + if inspect.isasyncgen(result): + return [chunk async for chunk in result] + return result + + captured = {} + original_start = runpod.serverless.start + handler_dir = os.path.dirname(os.path.abspath(handler_path)) + original_cwd = os.getcwd() + runpod.serverless.start = captured.update + sys.path.insert(0, handler_dir) + os.chdir(handler_dir) + try: + # run_name="__main__" also covers handlers that start the worker + # under an `if __name__ == "__main__":` guard. + runpy.run_path(handler_path, run_name="__main__") + handler = captured.get("handler") + if handler is None: + raise RuntimeError(f"{handler_path} never called runpod.serverless.start") + + result = handler(job) + if inspect.isgenerator(result): + result = list(result) + elif inspect.isawaitable(result) or inspect.isasyncgen(result): + result = asyncio.run(_drain(result)) + return {"output": result} + finally: + runpod.serverless.start = original_start + sys.path.remove(handler_dir) + os.chdir(original_cwd) + + +@runpod.local_entrypoint +async def main(): + failures = [] + for case in SUITE.cases: + job_input = {"id": f"ci-{case.name}", "input": case.input} + print(f"[{case.name}] submitting", flush=True) + try: + job = await run_case.spawn.aio(HANDLER, job_input) + result = await job.result.aio(timeout=STARTUP_S + case.timeout_s) + except Exception as exc: # the job, the worker, or the wait failed + failures.append(f"{case.name}: {type(exc).__name__}: {exc}") + print(f"[{case.name}] FAILED: {exc}", flush=True) + continue + + reason = check_output(result) + if reason: + failures.append(f"{case.name}: {reason}") + print(f"[{case.name}] FAILED: {reason}", flush=True) + else: + print(f"[{case.name}] passed", flush=True) + + print( + f"{len(SUITE.cases) - len(failures)}/{len(SUITE.cases)} tests passed", + flush=True, + ) + if failures: + raise RuntimeError("worker tests failed:\n" + "\n".join(failures)) diff --git a/test-image/tests/conftest.py b/test-image/tests/conftest.py new file mode 100644 index 0000000..ece8e0c --- /dev/null +++ b/test-image/tests/conftest.py @@ -0,0 +1,4 @@ +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) diff --git a/test-image/tests/test_cases.py b/test-image/tests/test_cases.py new file mode 100644 index 0000000..1a52d7f --- /dev/null +++ b/test-image/tests/test_cases.py @@ -0,0 +1,96 @@ +import json + +import pytest + +from cases import Case, SuiteError, app_name, check_output, load_suite + + +def write_suite(tmp_path, suite): + path = tmp_path / "tests.json" + path.write_text(json.dumps(suite)) + return path + + +def test_load_suite_reads_hub_format(tmp_path): + path = write_suite( + tmp_path, + { + "tests": [{"name": "basic", "input": {"audio": "a.wav"}, "timeout": 10000}], + "config": {"gpuTypeId": "NVIDIA GeForce RTX 4090"}, + }, + ) + + suite = load_suite(path) + + assert suite.cases == [Case(name="basic", input={"audio": "a.wav"}, timeout_s=10.0)] + assert suite.gpu == "NVIDIA GeForce RTX 4090" + + +def test_load_suite_reads_config_env_as_dict(tmp_path): + path = write_suite( + tmp_path, + { + "tests": [{"name": "basic", "input": {}}], + "config": {"env": [{"key": "MODEL", "value": "turbo"}]}, + }, + ) + + assert load_suite(path).env == {"MODEL": "turbo"} + + +def test_load_suite_defaults_timeout_and_missing_config(tmp_path): + path = write_suite(tmp_path, {"tests": [{"name": "basic", "input": {}}]}) + + suite = load_suite(path) + + assert suite.cases[0].timeout_s == 300.0 + assert suite.gpu is None + assert suite.env == {} + + +@pytest.mark.parametrize( + "suite", + [ + {"tests": []}, + {}, + {"tests": [{"input": {}}]}, + {"tests": [{"name": "no-input"}]}, + ], +) +def test_load_suite_rejects_invalid_suites(tmp_path, suite): + with pytest.raises(SuiteError): + load_suite(write_suite(tmp_path, suite)) + + +def test_load_suite_rejects_missing_file(tmp_path): + with pytest.raises(SuiteError, match="not found"): + load_suite(tmp_path / "absent.json") + + +def test_app_name_is_unique_per_run_attempt_and_api_safe(): + name = app_name("runpod-workers/worker-faster_whisper", "42459297624", "2") + + assert name == "ci-worker-faster-whisper-42459297624-2" + + +def test_app_name_truncates_long_repo_names_but_keeps_run_ids(): + name = app_name("org/" + "x" * 80, "123", "1") + + assert name.endswith("-123-1") + assert len(name) <= 48 + + +@pytest.mark.parametrize( + ("result", "expected"), + [ + ({"output": {"text": "four score"}}, None), + ({"output": [1, 2]}, None), + ( + {"output": {"error": "model missing"}}, + "handler returned an error: model missing", + ), + ({"output": None}, "handler returned no output"), + ], +) +def test_check_output(result, expected): + assert check_output(result) == expected diff --git a/test-image/tests/test_run_case.py b/test-image/tests/test_run_case.py new file mode 100644 index 0000000..d0a28ce --- /dev/null +++ b/test-image/tests/test_run_case.py @@ -0,0 +1,206 @@ +import importlib +import json +import sys +import textwrap + +import pytest + + +@pytest.fixture +def flash_app(tmp_path, monkeypatch): + suite = tmp_path / "tests.json" + suite.write_text(json.dumps({"tests": [{"name": "basic", "input": {}}]})) + monkeypatch.setenv("RP_TEST_IMAGE", "runpod/worker-example:ci") + monkeypatch.setenv("RP_TEST_FILE", str(suite)) + monkeypatch.setenv("RP_TEST_GPU", "NVIDIA GeForce RTX 4090") + monkeypatch.setenv("RP_TEST_APP_NAME", "ci-example-1-1") + import flash_app as module + + return importlib.reload(module) + + +def write_handler(tmp_path, source, name="rp_handler.py"): + path = tmp_path / name + path.write_text(textwrap.dedent(source)) + return str(path) + + +def test_runs_handler_registered_at_module_level(flash_app, tmp_path): + handler = write_handler( + tmp_path, + """ + import runpod + + def handler(job): + return {"echo": job["input"]["text"], "id": job["id"]} + + runpod.serverless.start({"handler": handler}) + """, + ) + + result = flash_app.run_case.local( + handler, {"id": "ci-basic", "input": {"text": "hi"}} + ) + + assert result == {"output": {"echo": "hi", "id": "ci-basic"}} + + +def test_runs_handler_registered_under_main_guard(flash_app, tmp_path): + handler = write_handler( + tmp_path, + """ + import runpod + + def handler(job): + return "ok" + + if __name__ == "__main__": + runpod.serverless.start({"handler": handler}) + """, + ) + + assert flash_app.run_case.local(handler, {"id": "x", "input": {}}) == { + "output": "ok" + } + + +def test_handler_can_import_sibling_modules(flash_app, tmp_path): + write_handler(tmp_path, "VALUE = 42\n", name="predict.py") + handler = write_handler( + tmp_path, + """ + import runpod + import predict + + runpod.serverless.start({"handler": lambda job: predict.VALUE}) + """, + ) + + assert flash_app.run_case.local(handler, {"id": "x", "input": {}}) == {"output": 42} + + +def test_collects_generator_output(flash_app, tmp_path): + handler = write_handler( + tmp_path, + """ + import runpod + + def handler(job): + yield 1 + yield 2 + + runpod.serverless.start({"handler": handler}) + """, + ) + + assert flash_app.run_case.local(handler, {"id": "x", "input": {}}) == { + "output": [1, 2] + } + + +def test_awaits_async_handler(flash_app, tmp_path): + handler = write_handler( + tmp_path, + """ + import runpod + + async def handler(job): + return "async ok" + + runpod.serverless.start({"handler": handler}) + """, + ) + + assert flash_app.run_case.local(handler, {"id": "x", "input": {}}) == { + "output": "async ok" + } + + +def test_collects_async_generator_output(flash_app, tmp_path): + handler = write_handler( + tmp_path, + """ + import runpod + + async def handler(job): + yield "a" + yield "b" + + runpod.serverless.start({"handler": handler}) + """, + ) + + assert flash_app.run_case.local(handler, {"id": "x", "input": {}}) == { + "output": ["a", "b"] + } + + +def test_fails_when_handler_file_never_starts_a_worker(flash_app, tmp_path): + handler = write_handler(tmp_path, "def handler(job):\n return 1\n") + + with pytest.raises(RuntimeError, match="never called runpod.serverless.start"): + flash_app.run_case.local(handler, {"id": "x", "input": {}}) + + +def test_restores_serverless_start_after_the_call(flash_app, tmp_path): + import runpod.serverless + + original = runpod.serverless.start + handler = write_handler( + tmp_path, + "import runpod\nrunpod.serverless.start({'handler': lambda job: 1})\n", + ) + + flash_app.run_case.local(handler, {"id": "x", "input": {}}) + + assert runpod.serverless.start is original + + +def test_shipped_module_loads_on_worker_without_local_files( + flash_app, tmp_path, monkeypatch +): + # Live endpoints exec the whole module on the worker, where cases.py, + # the tests file, and the action's env vars do not exist. Only the env + # the endpoint was created with is there. + from runpod.apps.serialization import get_function_source + + source = get_function_source(flash_app.run_case._fn) + worker_env = flash_app.run_case.spec.env + for name in ("RP_TEST_IMAGE", "RP_TEST_FILE", "RP_TEST_GPU", "RP_TEST_APP_NAME"): + monkeypatch.delenv(name, raising=False) + for name, value in worker_env.items(): + monkeypatch.setenv(name, value) + monkeypatch.setitem(sys.modules, "cases", None) # any import of it fails + + namespace = {"__name__": "shipped"} + exec(compile(source, "shipped.py", "exec"), namespace) + handler = write_handler( + tmp_path, + "import runpod\nrunpod.serverless.start({'handler': lambda job: 'shipped'})\n", + ) + + assert namespace["run_case"].local(handler, {"id": "x", "input": {}}) == { + "output": "shipped" + } + + +def test_worker_env_carries_suite_env(tmp_path, monkeypatch): + suite = tmp_path / "tests.json" + suite.write_text( + json.dumps( + { + "tests": [{"name": "basic", "input": {}}], + "config": {"env": [{"key": "MODEL", "value": "turbo"}]}, + } + ) + ) + monkeypatch.setenv("RP_TEST_IMAGE", "runpod/worker-example:ci") + monkeypatch.setenv("RP_TEST_FILE", str(suite)) + monkeypatch.setenv("RP_TEST_GPU", "NVIDIA GeForce RTX 4090") + monkeypatch.setenv("RP_TEST_APP_NAME", "ci-example-1-1") + import flash_app as module + + env = importlib.reload(module).run_case.spec.env + + assert env["MODEL"] == "turbo" + assert env["RP_TEST_IMAGE"] == "runpod/worker-example:ci" From 0f06aab66170ecd93ebf2074398d174e164854ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dean=20Qui=C3=B1anola?= Date: Sun, 8 Nov 2026 12:11:30 -0800 Subject: [PATCH 3/5] feat(ci): add server mode, CUDA minimum and job timeout to test-image - server mode (start-command input) for load-balancer workers: the worker starts the image's server once with PORT set, waits for the health path to return 200, and sends each test's HTTP request to it. A crashed server fails the test with its exit code and log tail. - the lowest config.allowedCudaVersions entry becomes the endpoint's minimum CUDA version (min-cuda input overrides it) - each job's execution timeout is startup-timeout plus the test's timeout, so long tests are not cut off by the endpoint default - run_case takes a single task dict, so both modes share one endpoint --- docs/usage.md | 42 +++++- test-image/action.yml | 26 +++- test-image/cases.py | 48 +++++- test-image/flash_app.py | 234 +++++++++++++++++++++++++----- test-image/tests/test_cases.py | 77 +++++++++- test-image/tests/test_run_case.py | 196 ++++++++++++++++++++++--- 6 files changed, 555 insertions(+), 68 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index c59547a..cb42dae 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -7,8 +7,10 @@ The `test-image` action runs a worker image's test cases on a temporary serverle It uses Flash from runpod-python (`rp flash dev --once`): 1. The CLI creates a live endpoint from your image, with at most one worker. -2. For each test in your tests file, it sends a job to that endpoint. On the worker, the job loads your handler file, captures the handler it registers with `runpod.serverless.start(...)`, and calls it with the test input. -3. A test fails if the job fails, times out, returns no output, or returns a dict with an `error` key. +2. For each test in your tests file, it sends a job to that endpoint. What the job does on the worker depends on the mode: + - **Handler mode** (default, queue workers): load your handler file, capture the handler it registers with `runpod.serverless.start(...)`, and call it with the test input. + - **Server mode** (load-balancer workers, selected by `start-command`): start your server once with `PORT` set, wait until its health path returns 200, and send the test's HTTP request to it. +3. A handler-mode test fails if the job fails, times out, returns no output, or returns a dict with an `error` key. A server-mode test fails if the server exits or never becomes healthy, or the response status differs from the expected one. 4. The endpoint is deleted when the tests finish, fail, or the run is cancelled. ### Usage @@ -28,9 +30,13 @@ Build and push the image first, then: | `image` | (required) | Worker image to test | | `runpod-api-key` | (required) | API key used to create and delete the endpoint | | `tests-file` | `.runpod/tests.json` | Hub-format tests file in your checkout | -| `handler` | `/handler.py` | Path of the handler file inside the image | +| `handler` | `/handler.py` | Path of the handler file inside the image (handler mode) | +| `start-command` | (none) | Command that starts your HTTP server; selects server mode | +| `port` | `8000` | Port your server listens on, passed to it as `PORT` (server mode) | +| `health-path` | `/ping` | Path that returns 200 once the server is ready (server mode) | | `gpu` | `config.gpuTypeId` from the tests file | GPU type for the endpoint | -| `startup-timeout` | `900` | Seconds allowed for the first worker to pull the image and start | +| `min-cuda` | lowest `config.allowedCudaVersions` | Minimum host CUDA version for the endpoint | +| `startup-timeout` | `900` | Seconds allowed for the worker to pull the image and start, and in server mode for the server to become healthy | | `runpod-version` | `2.1.0` | runpod-python version for the CLI and the worker | | `runtime-package-spec` | pinned `runpod-sdk-runtime` | Flash worker runtime installed into the image | @@ -50,11 +56,33 @@ The action reads the same format as the Hub: } ``` -`timeout` is in milliseconds. Entries in `config.env` are set on the endpoint. +`timeout` is in milliseconds and also sets the job's execution timeout, together with `startup-timeout`. Entries in `config.env` are set on the endpoint. The lowest entry of `config.allowedCudaVersions` becomes the endpoint's minimum CUDA version. + +In server mode, each test's `input` is an HTTP request. `method` defaults to `GET` and `status` (the expected response status) to `200`: + +```json +{ + "tests": [ + { + "name": "completion", + "input": {"method": "POST", "path": "/v1/completions", "body": {"prompt": "Hello", "max_tokens": 8}}, + "timeout": 120000 + } + ] +} +``` + +```yaml +- uses: runpod-workers/worker-github_runner/test-image@main + with: + image: runpod/worker-example-lb:ci-pr-${{ github.event.number }} + runpod-api-key: ${{ secrets.RUNPOD_API_KEY }} + start-command: cd /src && python3 handler.py +``` ### What can go wrong -- **The image's start command does not run.** A live endpoint starts the Flash runtime instead of the image's start command. The action calls your handler directly, so code that runs only in the start command (for example, a shell script that runs before `python handler.py`) is not exercised. +- **The image's start command does not run.** A live endpoint starts the Flash runtime instead of the image's start command. In handler mode the action calls your handler directly, so code that runs only in the start command (for example, a shell script that runs before `python handler.py`) is not exercised. In server mode, `start-command` should match the image's start command. - **The worker runs `runpod==`.** The endpoint installs that version into the image before starting, even if the image pins another one. A handler that only works with an older SDK will fail here. -- **The image needs Python 3.10 or newer**, with `pip`, so the runtime can be installed. +- **The image needs Python 3.10 or newer, with `pip` on its `python3`,** so the runtime can be installed. Images built on a uv-managed venv without pip need `uv pip install pip` in their Dockerfile. - **Pull requests from forks** do not get repository secrets, so the action cannot create an endpoint for them. Skip the job for forks. diff --git a/test-image/action.yml b/test-image/action.yml index 6584c2a..cb9ac92 100644 --- a/test-image/action.yml +++ b/test-image/action.yml @@ -14,15 +14,33 @@ inputs: required: false default: ".runpod/tests.json" handler: - description: "Path of the handler file inside the image" + description: "Path of the handler file inside the image (handler mode)" required: false default: "/handler.py" + start-command: + description: > + Command that starts the image's HTTP server, e.g. "cd /src && python3 handler.py". + Setting it selects server mode, for load-balancer workers. + required: false + default: "" + port: + description: "Port the server listens on; passed to it as PORT (server mode)" + required: false + default: "8000" + health-path: + description: "Path that returns 200 once the server is ready (server mode)" + required: false + default: "/ping" gpu: description: "GPU type; defaults to config.gpuTypeId in the tests file" required: false default: "" + min-cuda: + description: "Minimum host CUDA version; defaults to the lowest config.allowedCudaVersions" + required: false + default: "" startup-timeout: - description: "Seconds allowed for the first worker to pull the image and start" + description: "Seconds allowed for the worker to pull the image and start (and, in server mode, for the server to become healthy)" required: false default: "900" runpod-version: @@ -54,7 +72,11 @@ runs: RP_TEST_IMAGE: ${{ inputs.image }} RP_TEST_FILE: ${{ inputs.tests-file }} RP_TEST_HANDLER: ${{ inputs.handler }} + RP_TEST_START_COMMAND: ${{ inputs.start-command }} + RP_TEST_PORT: ${{ inputs.port }} + RP_TEST_HEALTH_PATH: ${{ inputs.health-path }} RP_TEST_GPU: ${{ inputs.gpu }} + RP_TEST_MIN_CUDA: ${{ inputs.min-cuda }} RP_TEST_STARTUP_S: ${{ inputs.startup-timeout }} # The live endpoint's launcher force-installs these into the image, # so the worker runs the same SDK as the CLI. diff --git a/test-image/cases.py b/test-image/cases.py index dbf5a87..038ae36 100644 --- a/test-image/cases.py +++ b/test-image/cases.py @@ -4,9 +4,14 @@ {"tests": [{"name": "basic", "input": {...}, "timeout": 10000}], "config": {"gpuTypeId": "NVIDIA GeForce RTX 4090", + "allowedCudaVersions": ["12.8", "13.0"], "env": [{"key": "K", "value": "V"}]}} -`timeout` is in milliseconds, as on the Hub. +`timeout` is in milliseconds, as on the Hub. The lowest entry of +`allowedCudaVersions` becomes the endpoint's minimum CUDA version. + +In server mode each test's input is an HTTP request instead of a job input: + {"method": "POST", "path": "/v1/completions", "body": {...}, "status": 200} """ import json @@ -38,6 +43,7 @@ class Suite: cases: List[Case] gpu: Optional[str] = None env: Dict[str, str] = field(default_factory=dict) + min_cuda: Optional[str] = None def load_suite(path: Path) -> Suite: @@ -60,7 +66,33 @@ def load_suite(path: Path) -> Suite: config = data.get("config") or {} env = {item["key"]: str(item["value"]) for item in config.get("env") or []} - return Suite(cases=cases, gpu=config.get("gpuTypeId"), env=env) + return Suite( + cases=cases, + gpu=config.get("gpuTypeId"), + env=env, + min_cuda=_lowest_version(config.get("allowedCudaVersions") or []), + ) + + +def _lowest_version(versions: List[str]) -> Optional[str]: + if not versions: + return None + return min(versions, key=lambda v: tuple(int(part) for part in v.split("."))) + + +def http_request(case_input: Dict[str, Any]) -> Dict[str, Any]: + """Normalize a server-mode test input into method, path, body and expected status.""" + path = case_input.get("path") + if not isinstance(path, str) or not path.startswith("/"): + raise SuiteError( + f"server-mode test input needs an absolute 'path', got {path!r}" + ) + return { + "method": str(case_input.get("method", "GET")).upper(), + "path": path, + "body": case_input.get("body"), + "status": int(case_input.get("status", 200)), + } def app_name(repository: str, run_id: str, attempt: str) -> str: @@ -80,3 +112,15 @@ def check_output(result: Dict[str, Any]) -> Optional[str]: if isinstance(output, dict) and "error" in output: return f"handler returned an error: {output['error']}" return None + + +def check_http_output(result: Dict[str, Any], expected_status: int) -> Optional[str]: + """Return why a server response fails the test, or None if it passes.""" + output = result.get("output") + if output is None: + return "server returned no response" + if output["status"] != expected_status: + return ( + f"expected HTTP {expected_status}, got {output['status']}: {output['body']}" + ) + return None diff --git a/test-image/flash_app.py b/test-image/flash_app.py index 429090d..2072a1a 100644 --- a/test-image/flash_app.py +++ b/test-image/flash_app.py @@ -5,31 +5,52 @@ deletes the endpoint when `main` returns or raises. A live endpoint replaces the image's start command with the Flash runtime, -so the image's own `runpod.serverless.start(...)` never runs. Instead, -`run_case` loads the worker's handler file inside the image and calls its -handler directly. The handler still runs with the image's code, models, and -libraries. +so the image's own entrypoint never runs. `run_case` drives the worker from +inside the image instead, in one of two modes: + +- handler mode: load the handler file, capture the handler it passes to + `runpod.serverless.start(...)`, and call it with each test input. +- server mode (load-balancer workers): start the image's HTTP server once, + wait for its health path to return 200, and send each test's request to it. Configured through environment variables (set by action.yml): - RP_TEST_IMAGE worker image to test (required) - RP_TEST_FILE Hub-format tests file (required) - RP_TEST_APP_NAME Flash app name, unique per run (required) - RP_TEST_HANDLER handler file inside the image (default /handler.py) - RP_TEST_GPU GPU type; defaults to config.gpuTypeId in the tests file - RP_TEST_STARTUP_S seconds allowed for the first worker to start (default 900) + RP_TEST_IMAGE worker image to test (required) + RP_TEST_FILE Hub-format tests file (required) + RP_TEST_APP_NAME Flash app name, unique per run (required) + RP_TEST_HANDLER handler file inside the image (default /handler.py) + RP_TEST_START_COMMAND server command; setting it selects server mode + RP_TEST_PORT port the server listens on, passed as PORT (default 8000) + RP_TEST_HEALTH_PATH server health path (default /ping) + RP_TEST_GPU GPU type; defaults to config.gpuTypeId in the tests file + RP_TEST_MIN_CUDA minimum CUDA; defaults to the lowest allowedCudaVersions + RP_TEST_STARTUP_S seconds allowed for the worker or server to start (default 900) """ +import asyncio +import inspect +import json import os +import runpy +import signal +import subprocess import sys +import time +import urllib.error +import urllib.request from pathlib import Path import runpod DEFAULT_HANDLER = "/handler.py" +DEFAULT_PORT = 8000 +DEFAULT_HEALTH_PATH = "/ping" DEFAULT_STARTUP_S = 900.0 # one worker at most; a CI endpoint never needs to scale CI_WORKERS = (0, 1) CI_IDLE_TIMEOUT_S = 5 +HEALTH_POLL_S = 1.0 +HEALTH_PROBE_TIMEOUT_S = 2.0 +SERVER_LOG_TAIL_LINES = 50 # Flash execs this whole module on the worker too. The endpoint env carries # this marker plus the values below, so the worker can rebuild the app # without the tests file or cases.py, which exist only on the CI runner. @@ -47,10 +68,11 @@ def _required(name: str) -> str: if os.environ.get(WORKER_MARKER): SUITE = None IMAGE, GPU, APP_NAME = (_required(name) for name in FORWARDED) + MIN_CUDA = os.environ.get("RP_TEST_MIN_CUDA") or None WORKER_ENV = {} else: sys.path.insert(0, str(Path(__file__).resolve().parent)) - from cases import check_output, load_suite + from cases import check_http_output, check_output, http_request, load_suite SUITE = load_suite(Path(_required("RP_TEST_FILE"))) IMAGE = _required("RP_TEST_IMAGE") @@ -58,44 +80,42 @@ def _required(name: str) -> str: GPU = os.environ.get("RP_TEST_GPU") or SUITE.gpu if not GPU: raise RuntimeError("set RP_TEST_GPU or config.gpuTypeId in the tests file") + MIN_CUDA = os.environ.get("RP_TEST_MIN_CUDA") or SUITE.min_cuda WORKER_ENV = { **SUITE.env, WORKER_MARKER: "1", "RP_TEST_IMAGE": IMAGE, "RP_TEST_GPU": GPU, "RP_TEST_APP_NAME": APP_NAME, + **({"RP_TEST_MIN_CUDA": MIN_CUDA} if MIN_CUDA else {}), } HANDLER = os.environ.get("RP_TEST_HANDLER") or DEFAULT_HANDLER +START_COMMAND = os.environ.get("RP_TEST_START_COMMAND") or None STARTUP_S = float(os.environ.get("RP_TEST_STARTUP_S") or DEFAULT_STARTUP_S) +SERVER = { + "command": START_COMMAND, + "port": int(os.environ.get("RP_TEST_PORT") or DEFAULT_PORT), + "health_path": os.environ.get("RP_TEST_HEALTH_PATH") or DEFAULT_HEALTH_PATH, + "startup_s": STARTUP_S, +} app = runpod.App(APP_NAME) -@app.queue( - name="worker", - image=IMAGE, - gpu=GPU, - workers=CI_WORKERS, - idle_timeout=CI_IDLE_TIMEOUT_S, - env=WORKER_ENV, -) -def run_case(handler_path: str, job: dict) -> dict: - # Runs on the worker, so imports stay local to this function. - import asyncio - import inspect - import os - import runpy - import sys +# ---- worker side: handler mode --------------------------------------------- - import runpod.serverless - async def _drain(result): - if inspect.isawaitable(result): - result = await result - if inspect.isasyncgen(result): - return [chunk async for chunk in result] - return result +async def _drain(result): + if inspect.isawaitable(result): + result = await result + if inspect.isasyncgen(result): + return [chunk async for chunk in result] + return result + + +def _run_handler(handler_path: str, job: dict) -> dict: + import runpod.serverless captured = {} original_start = runpod.serverless.start @@ -124,21 +144,161 @@ async def _drain(result): os.chdir(original_cwd) +# ---- worker side: server mode ---------------------------------------------- + + +def _server_files(port: int): + # Keyed by port and kept on disk, so a server started by one job is + # found again by the next even if the runtime re-execs this module. + return f"/tmp/rp-test-server-{port}.pid", f"/tmp/rp-test-server-{port}.log" + + +def _exit_code(pid: int): + """The server's exit code if it has exited, else None.""" + try: + done, status = os.waitpid(pid, os.WNOHANG) + except ChildProcessError: # not our child; fall back to a liveness probe + try: + os.kill(pid, 0) + return None + except ProcessLookupError: + return -1 + return os.waitstatus_to_exitcode(status) if done else None + + +def _log_tail(log_file: str) -> str: + try: + with open(log_file, errors="replace") as f: + return "".join(f.readlines()[-SERVER_LOG_TAIL_LINES:]) + except FileNotFoundError: + return "" + + +def _health_status(port: int, path: str): + try: + url = f"http://127.0.0.1:{port}{path}" + with urllib.request.urlopen(url, timeout=HEALTH_PROBE_TIMEOUT_S) as response: + return response.status + except urllib.error.HTTPError as exc: + return exc.code + except OSError: # not listening yet + return None + + +def _ensure_server(server: dict) -> None: + port = server["port"] + pid_file, log_file = _server_files(port) + pid = int(Path(pid_file).read_text()) if os.path.exists(pid_file) else None + if pid is None or _exit_code(pid) is not None: + with open(log_file, "w") as log: + process = subprocess.Popen( + server["command"], + shell=True, + env={**os.environ, "PORT": str(port)}, + stdout=log, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + pid = process.pid + Path(pid_file).write_text(str(pid)) + + deadline = time.monotonic() + server["startup_s"] + while True: + code = _exit_code(pid) + if code is not None: + os.remove(pid_file) + raise RuntimeError( + f"server command exited with code {code} before becoming healthy:\n" + f"{_log_tail(log_file)}" + ) + if _health_status(port, server["health_path"]) == 200: + return + if time.monotonic() > deadline: + os.killpg(pid, signal.SIGTERM) + os.remove(pid_file) + raise TimeoutError( + f"server was not healthy within {server['startup_s']:g}s:\n" + f"{_log_tail(log_file)}" + ) + time.sleep(HEALTH_POLL_S) + + +def _send(port: int, request: dict) -> dict: + data = None if request["body"] is None else json.dumps(request["body"]).encode() + http_request = urllib.request.Request( + f"http://127.0.0.1:{port}{request['path']}", + data=data, + method=request["method"], + headers={"Content-Type": "application/json"}, + ) + try: + with urllib.request.urlopen( + http_request, timeout=request["timeout_s"] + ) as response: + status, raw = response.status, response.read() + except urllib.error.HTTPError as exc: + status, raw = exc.code, exc.read() + text = raw.decode(errors="replace") + try: + body = json.loads(text) + except ValueError: + body = text + return {"status": status, "body": body} + + +@app.queue( + name="worker", + image=IMAGE, + gpu=GPU, + workers=CI_WORKERS, + idle_timeout=CI_IDLE_TIMEOUT_S, + env=WORKER_ENV, + min_cuda_version=MIN_CUDA, +) +def run_case(task: dict) -> dict: + """Run one test on the worker. `task` is either + {"handler": path, "job": {...}} or {"server": {...}, "request": {...}}.""" + if "server" in task: + _ensure_server(task["server"]) + return {"output": _send(task["server"]["port"], task["request"])} + return _run_handler(task["handler"], task["job"]) + + +# ---- CI runner side ---------------------------------------------------------- + + +def _task_and_check(case): + """The worker task for a test case and the function that checks its result.""" + if START_COMMAND: + request = http_request(case.input) + task = { + "server": SERVER, + "request": {**request, "timeout_s": case.timeout_s}, + } + return task, lambda result: check_http_output(result, request["status"]) + task = {"handler": HANDLER, "job": {"id": f"ci-{case.name}", "input": case.input}} + return task, check_output + + @runpod.local_entrypoint async def main(): failures = [] for case in SUITE.cases: - job_input = {"id": f"ci-{case.name}", "input": case.input} + task, check = _task_and_check(case) + budget_s = STARTUP_S + case.timeout_s print(f"[{case.name}] submitting", flush=True) try: - job = await run_case.spawn.aio(HANDLER, job_input) - result = await job.result.aio(timeout=STARTUP_S + case.timeout_s) + # without an explicit limit the endpoint's default execution + # timeout can kill long tests (model downloads, training) + caller = run_case.with_options(execution_timeout=int(budget_s * 1000)) + job = await caller.spawn.aio(task) + result = await job.result.aio(timeout=budget_s) except Exception as exc: # the job, the worker, or the wait failed failures.append(f"{case.name}: {type(exc).__name__}: {exc}") print(f"[{case.name}] FAILED: {exc}", flush=True) continue - reason = check_output(result) + reason = check(result) if reason: failures.append(f"{case.name}: {reason}") print(f"[{case.name}] FAILED: {reason}", flush=True) diff --git a/test-image/tests/test_cases.py b/test-image/tests/test_cases.py index 1a52d7f..df6bcd7 100644 --- a/test-image/tests/test_cases.py +++ b/test-image/tests/test_cases.py @@ -2,7 +2,15 @@ import pytest -from cases import Case, SuiteError, app_name, check_output, load_suite +from cases import ( + Case, + SuiteError, + app_name, + check_http_output, + check_output, + http_request, + load_suite, +) def write_suite(tmp_path, suite): @@ -94,3 +102,70 @@ def test_app_name_truncates_long_repo_names_but_keeps_run_ids(): ) def test_check_output(result, expected): assert check_output(result) == expected + + +def test_load_suite_takes_lowest_allowed_cuda_version(tmp_path): + path = write_suite( + tmp_path, + { + "tests": [{"name": "basic", "input": {}}], + "config": {"allowedCudaVersions": ["12.7", "12.10", "12.3"]}, + }, + ) + + assert load_suite(path).min_cuda == "12.3" + + +def test_load_suite_without_cuda_versions_sets_no_minimum(tmp_path): + path = write_suite(tmp_path, {"tests": [{"name": "basic", "input": {}}]}) + + assert load_suite(path).min_cuda is None + + +def test_http_request_reads_method_path_body_and_expected_status(): + request = http_request( + { + "method": "post", + "path": "/v1/completions", + "body": {"prompt": "hi"}, + "status": 201, + } + ) + + assert request == { + "method": "POST", + "path": "/v1/completions", + "body": {"prompt": "hi"}, + "status": 201, + } + + +def test_http_request_defaults_to_get_and_200(): + assert http_request({"path": "/ping"}) == { + "method": "GET", + "path": "/ping", + "body": None, + "status": 200, + } + + +def test_http_request_requires_an_absolute_path(): + with pytest.raises(SuiteError, match="path"): + http_request({"method": "GET"}) + with pytest.raises(SuiteError, match="path"): + http_request({"path": "v1/completions"}) + + +@pytest.mark.parametrize( + ("result", "expected"), + [ + ({"output": {"status": 200, "body": {"choices": []}}}, None), + ( + {"output": {"status": 500, "body": {"detail": "boom"}}}, + "expected HTTP 200, got 500: {'detail': 'boom'}", + ), + ({"output": None}, "server returned no response"), + ], +) +def test_check_http_output(result, expected): + assert check_http_output(result, 200) == expected diff --git a/test-image/tests/test_run_case.py b/test-image/tests/test_run_case.py index d0a28ce..43729f2 100644 --- a/test-image/tests/test_run_case.py +++ b/test-image/tests/test_run_case.py @@ -19,6 +19,10 @@ def flash_app(tmp_path, monkeypatch): return importlib.reload(module) +def handler_task(handler, job): + return {"handler": handler, "job": job} + + def write_handler(tmp_path, source, name="rp_handler.py"): path = tmp_path / name path.write_text(textwrap.dedent(source)) @@ -39,7 +43,7 @@ def handler(job): ) result = flash_app.run_case.local( - handler, {"id": "ci-basic", "input": {"text": "hi"}} + handler_task(handler, {"id": "ci-basic", "input": {"text": "hi"}}) ) assert result == {"output": {"echo": "hi", "id": "ci-basic"}} @@ -59,9 +63,9 @@ def handler(job): """, ) - assert flash_app.run_case.local(handler, {"id": "x", "input": {}}) == { - "output": "ok" - } + assert flash_app.run_case.local( + handler_task(handler, {"id": "x", "input": {}}) + ) == {"output": "ok"} def test_handler_can_import_sibling_modules(flash_app, tmp_path): @@ -76,7 +80,9 @@ def test_handler_can_import_sibling_modules(flash_app, tmp_path): """, ) - assert flash_app.run_case.local(handler, {"id": "x", "input": {}}) == {"output": 42} + assert flash_app.run_case.local( + handler_task(handler, {"id": "x", "input": {}}) + ) == {"output": 42} def test_collects_generator_output(flash_app, tmp_path): @@ -93,9 +99,9 @@ def handler(job): """, ) - assert flash_app.run_case.local(handler, {"id": "x", "input": {}}) == { - "output": [1, 2] - } + assert flash_app.run_case.local( + handler_task(handler, {"id": "x", "input": {}}) + ) == {"output": [1, 2]} def test_awaits_async_handler(flash_app, tmp_path): @@ -111,9 +117,9 @@ async def handler(job): """, ) - assert flash_app.run_case.local(handler, {"id": "x", "input": {}}) == { - "output": "async ok" - } + assert flash_app.run_case.local( + handler_task(handler, {"id": "x", "input": {}}) + ) == {"output": "async ok"} def test_collects_async_generator_output(flash_app, tmp_path): @@ -130,16 +136,16 @@ async def handler(job): """, ) - assert flash_app.run_case.local(handler, {"id": "x", "input": {}}) == { - "output": ["a", "b"] - } + assert flash_app.run_case.local( + handler_task(handler, {"id": "x", "input": {}}) + ) == {"output": ["a", "b"]} def test_fails_when_handler_file_never_starts_a_worker(flash_app, tmp_path): handler = write_handler(tmp_path, "def handler(job):\n return 1\n") with pytest.raises(RuntimeError, match="never called runpod.serverless.start"): - flash_app.run_case.local(handler, {"id": "x", "input": {}}) + flash_app.run_case.local(handler_task(handler, {"id": "x", "input": {}})) def test_restores_serverless_start_after_the_call(flash_app, tmp_path): @@ -151,7 +157,7 @@ def test_restores_serverless_start_after_the_call(flash_app, tmp_path): "import runpod\nrunpod.serverless.start({'handler': lambda job: 1})\n", ) - flash_app.run_case.local(handler, {"id": "x", "input": {}}) + flash_app.run_case.local(handler_task(handler, {"id": "x", "input": {}})) assert runpod.serverless.start is original @@ -179,9 +185,9 @@ def test_shipped_module_loads_on_worker_without_local_files( "import runpod\nrunpod.serverless.start({'handler': lambda job: 'shipped'})\n", ) - assert namespace["run_case"].local(handler, {"id": "x", "input": {}}) == { - "output": "shipped" - } + assert namespace["run_case"].local( + handler_task(handler, {"id": "x", "input": {}}) + ) == {"output": "shipped"} def test_worker_env_carries_suite_env(tmp_path, monkeypatch): @@ -204,3 +210,155 @@ def test_worker_env_carries_suite_env(tmp_path, monkeypatch): assert env["MODEL"] == "turbo" assert env["RP_TEST_IMAGE"] == "runpod/worker-example:ci" + + +# ---- server mode ------------------------------------------------------------ + +SERVER = """ +import json, os, sys +from http.server import BaseHTTPRequestHandler, HTTPServer + +with open(os.environ["STARTS_FILE"], "a") as f: + f.write("start\\n") + +class Handler(BaseHTTPRequestHandler): + def _send(self, code, body): + data = json.dumps(body).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def do_GET(self): + if self.path == "/ping": + return self._send(200, {"status": "healthy"}) + self._send(404, {"detail": "not found"}) + + def do_POST(self): + length = int(self.headers.get("Content-Length", 0)) + body = json.loads(self.rfile.read(length) or b"null") + if self.path == "/fail": + return self._send(500, {"detail": "boom"}) + self._send(200, {"echo": body}) + + def log_message(self, *args): + pass + +HTTPServer(("127.0.0.1", int(os.environ["PORT"])), Handler).serve_forever() +""" + + +def free_port(): + import socket + + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +@pytest.fixture +def server(tmp_path, monkeypatch): + script = tmp_path / "server.py" + script.write_text(SERVER) + starts = tmp_path / "starts.txt" + monkeypatch.setenv("STARTS_FILE", str(starts)) + port = free_port() + spec = { + "command": f"{sys.executable} {script}", + "port": port, + "health_path": "/ping", + "startup_s": 20, + } + yield spec, starts + import os + import signal + + pid_file = f"/tmp/rp-test-server-{port}.pid" + if os.path.exists(pid_file): + try: + os.killpg(int(open(pid_file).read()), signal.SIGTERM) + except ProcessLookupError: + pass + os.remove(pid_file) + + +def server_task(spec, method="GET", path="/ping", body=None, timeout_s=10): + return { + "server": spec, + "request": { + "method": method, + "path": path, + "body": body, + "timeout_s": timeout_s, + }, + } + + +def test_server_mode_starts_server_and_returns_response(flash_app, server): + spec, _ = server + + result = flash_app.run_case.local( + server_task(spec, "POST", "/echo", {"prompt": "hi"}) + ) + + assert result == {"output": {"status": 200, "body": {"echo": {"prompt": "hi"}}}} + + +def test_server_mode_returns_error_statuses_instead_of_raising(flash_app, server): + spec, _ = server + + result = flash_app.run_case.local(server_task(spec, "POST", "/fail", {})) + + assert result == {"output": {"status": 500, "body": {"detail": "boom"}}} + + +def test_server_mode_reuses_a_running_server(flash_app, server): + spec, starts = server + + flash_app.run_case.local(server_task(spec)) + flash_app.run_case.local(server_task(spec)) + + assert starts.read_text().count("start") == 1 + + +def test_server_mode_reports_a_crashed_server_with_its_log(flash_app, tmp_path): + crash = tmp_path / "crash.py" + crash.write_text( + "raise ImportError('attempted relative import with no known parent package')\n" + ) + spec = { + "command": f"{sys.executable} {crash}", + "port": free_port(), + "health_path": "/ping", + "startup_s": 20, + } + + with pytest.raises( + RuntimeError, match="exited with code 1(.|\\n)*attempted relative import" + ): + flash_app.run_case.local(server_task(spec)) + + +def test_server_mode_times_out_when_never_healthy(flash_app, tmp_path): + idle = tmp_path / "idle.py" + idle.write_text("import time\ntime.sleep(60)\n") + port = free_port() + spec = { + "command": f"{sys.executable} {idle}", + "port": port, + "health_path": "/ping", + "startup_s": 2, + } + + try: + with pytest.raises(TimeoutError, match="not healthy within 2"): + flash_app.run_case.local(server_task(spec)) + finally: + import os + import signal + + pid_file = f"/tmp/rp-test-server-{port}.pid" + if os.path.exists(pid_file): + os.killpg(int(open(pid_file).read()), signal.SIGTERM) + os.remove(pid_file) From 4976704200e1fa87092140f780fae9f23edd632a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dean=20Qui=C3=B1anola?= Date: Sun, 8 Nov 2026 12:35:15 -0800 Subject: [PATCH 4/5] fix(ci): run async handlers inside the runtime's event loop The Flash runtime calls run_case from inside its event loop, so asyncio.run() raised "cannot be called from a running event loop" for async handlers (seen with llm-fine-tuning). Run the coroutine on a separate thread with its own loop when a loop is already running. --- test-image/flash_app.py | 18 +++++++++++++++++- test-image/tests/test_run_case.py | 22 ++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/test-image/flash_app.py b/test-image/flash_app.py index 2072a1a..598be02 100644 --- a/test-image/flash_app.py +++ b/test-image/flash_app.py @@ -27,6 +27,7 @@ """ import asyncio +import concurrent.futures import inspect import json import os @@ -106,6 +107,21 @@ def _required(name: str) -> str: # ---- worker side: handler mode --------------------------------------------- +def _run_coroutine(coro): + """Run a coroutine to completion from sync code, even inside a running loop. + + The Flash runtime calls run_case from inside its event loop, where + asyncio.run() refuses to start, so the coroutine gets its own loop on a + separate thread. + """ + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(coro) + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + return pool.submit(asyncio.run, coro).result() + + async def _drain(result): if inspect.isawaitable(result): result = await result @@ -136,7 +152,7 @@ def _run_handler(handler_path: str, job: dict) -> dict: if inspect.isgenerator(result): result = list(result) elif inspect.isawaitable(result) or inspect.isasyncgen(result): - result = asyncio.run(_drain(result)) + result = _run_coroutine(_drain(result)) return {"output": result} finally: runpod.serverless.start = original_start diff --git a/test-image/tests/test_run_case.py b/test-image/tests/test_run_case.py index 43729f2..0a0d397 100644 --- a/test-image/tests/test_run_case.py +++ b/test-image/tests/test_run_case.py @@ -362,3 +362,25 @@ def test_server_mode_times_out_when_never_healthy(flash_app, tmp_path): if os.path.exists(pid_file): os.killpg(int(open(pid_file).read()), signal.SIGTERM) os.remove(pid_file) + + +def test_awaits_async_handler_when_called_inside_a_running_loop(flash_app, tmp_path): + # The Flash runtime calls run_case from inside its own event loop. + import asyncio + + handler = write_handler( + tmp_path, + """ + import runpod + + async def handler(job): + return "inside loop ok" + + runpod.serverless.start({"handler": handler}) + """, + ) + + async def runtime(): + return flash_app.run_case.local(handler_task(handler, {"id": "x", "input": {}})) + + assert asyncio.run(runtime()) == {"output": "inside loop ok"} From 92fbfd5775680735df3c60d006f173d4d2b051d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dean=20Qui=C3=B1anola?= Date: Mon, 9 Nov 2026 11:36:06 -0800 Subject: [PATCH 5/5] fix(ci): validate config.env and tolerate a bad pid file - load_suite raises SuiteError for a config.env that is not a list or has an entry without key and value, instead of a bare KeyError or TypeError. - An empty or corrupt server pid file counts as no running server, so the next case starts a fresh one instead of failing on every call. - Document that start-command runs through sh -c. - Read pid files in test teardown with Path.read_text. --- docs/usage.md | 2 +- test-image/action.yml | 3 ++- test-image/cases.py | 16 +++++++++++++++- test-image/flash_app.py | 15 ++++++++++++++- test-image/tests/test_cases.py | 16 ++++++++++++++++ test-image/tests/test_run_case.py | 15 +++++++++++++-- 6 files changed, 61 insertions(+), 6 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index cb42dae..f0141e4 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -31,7 +31,7 @@ Build and push the image first, then: | `runpod-api-key` | (required) | API key used to create and delete the endpoint | | `tests-file` | `.runpod/tests.json` | Hub-format tests file in your checkout | | `handler` | `/handler.py` | Path of the handler file inside the image (handler mode) | -| `start-command` | (none) | Command that starts your HTTP server; selects server mode | +| `start-command` | (none) | Command that starts your HTTP server; selects server mode. Runs on the worker through `sh -c` | | `port` | `8000` | Port your server listens on, passed to it as `PORT` (server mode) | | `health-path` | `/ping` | Path that returns 200 once the server is ready (server mode) | | `gpu` | `config.gpuTypeId` from the tests file | GPU type for the endpoint | diff --git a/test-image/action.yml b/test-image/action.yml index cb9ac92..5186eeb 100644 --- a/test-image/action.yml +++ b/test-image/action.yml @@ -20,7 +20,8 @@ inputs: start-command: description: > Command that starts the image's HTTP server, e.g. "cd /src && python3 handler.py". - Setting it selects server mode, for load-balancer workers. + Setting it selects server mode, for load-balancer workers. It runs on the + worker through `sh -c`, so shell syntax such as `cd` and `&&` works. required: false default: "" port: diff --git a/test-image/cases.py b/test-image/cases.py index 038ae36..6761162 100644 --- a/test-image/cases.py +++ b/test-image/cases.py @@ -65,7 +65,7 @@ def load_suite(path: Path) -> Suite: cases.append(Case(name=raw["name"], input=raw["input"], timeout_s=timeout_s)) config = data.get("config") or {} - env = {item["key"]: str(item["value"]) for item in config.get("env") or []} + env = _parse_env(path, config.get("env") or []) return Suite( cases=cases, gpu=config.get("gpuTypeId"), @@ -74,6 +74,20 @@ def load_suite(path: Path) -> Suite: ) +def _parse_env(path: Path, entries: Any) -> Dict[str, str]: + """Read config.env, a list of {"key": ..., "value": ...} entries.""" + if not isinstance(entries, list): + raise SuiteError(f"{path}: config.env must be a list of key/value entries") + env = {} + for index, entry in enumerate(entries): + if not isinstance(entry, dict) or "key" not in entry or "value" not in entry: + raise SuiteError( + f"{path}: config.env entry #{index} needs both 'key' and 'value'" + ) + env[entry["key"]] = str(entry["value"]) + return env + + def _lowest_version(versions: List[str]) -> Optional[str]: if not versions: return None diff --git a/test-image/flash_app.py b/test-image/flash_app.py index 598be02..78b2cda 100644 --- a/test-image/flash_app.py +++ b/test-image/flash_app.py @@ -39,6 +39,7 @@ import urllib.error import urllib.request from pathlib import Path +from typing import Optional import runpod @@ -201,10 +202,22 @@ def _health_status(port: int, path: str): return None +def _read_pid(pid_file: str) -> Optional[int]: + """Return the recorded server pid, or None when none is recorded. + + An empty or corrupt file (a worker that died mid-write) counts as no + server, so the next case starts a fresh one instead of failing forever. + """ + try: + return int(Path(pid_file).read_text()) + except (FileNotFoundError, ValueError): + return None + + def _ensure_server(server: dict) -> None: port = server["port"] pid_file, log_file = _server_files(port) - pid = int(Path(pid_file).read_text()) if os.path.exists(pid_file) else None + pid = _read_pid(pid_file) if pid is None or _exit_code(pid) is not None: with open(log_file, "w") as log: process = subprocess.Popen( diff --git a/test-image/tests/test_cases.py b/test-image/tests/test_cases.py index df6bcd7..a82c2d6 100644 --- a/test-image/tests/test_cases.py +++ b/test-image/tests/test_cases.py @@ -70,6 +70,22 @@ def test_load_suite_rejects_invalid_suites(tmp_path, suite): load_suite(write_suite(tmp_path, suite)) +@pytest.mark.parametrize( + "env", + [ + [{"key": "K"}], + [{"value": "V"}], + ["K=V"], + {"K": "V"}, + ], +) +def test_load_suite_rejects_malformed_config_env(tmp_path, env): + suite = {"tests": [{"name": "t", "input": {}}], "config": {"env": env}} + + with pytest.raises(SuiteError, match="config.env"): + load_suite(write_suite(tmp_path, suite)) + + def test_load_suite_rejects_missing_file(tmp_path): with pytest.raises(SuiteError, match="not found"): load_suite(tmp_path / "absent.json") diff --git a/test-image/tests/test_run_case.py b/test-image/tests/test_run_case.py index 0a0d397..a178b6a 100644 --- a/test-image/tests/test_run_case.py +++ b/test-image/tests/test_run_case.py @@ -2,6 +2,7 @@ import json import sys import textwrap +from pathlib import Path import pytest @@ -277,7 +278,7 @@ def server(tmp_path, monkeypatch): pid_file = f"/tmp/rp-test-server-{port}.pid" if os.path.exists(pid_file): try: - os.killpg(int(open(pid_file).read()), signal.SIGTERM) + os.killpg(int(Path(pid_file).read_text()), signal.SIGTERM) except ProcessLookupError: pass os.remove(pid_file) @@ -322,6 +323,16 @@ def test_server_mode_reuses_a_running_server(flash_app, server): assert starts.read_text().count("start") == 1 +def test_server_mode_starts_a_server_over_an_unreadable_pid_file(flash_app, server): + spec, starts = server + Path(f"/tmp/rp-test-server-{spec['port']}.pid").write_text("") + + result = flash_app.run_case.local(server_task(spec)) + + assert result["output"]["status"] == 200 + assert starts.read_text().count("start") == 1 + + def test_server_mode_reports_a_crashed_server_with_its_log(flash_app, tmp_path): crash = tmp_path / "crash.py" crash.write_text( @@ -360,7 +371,7 @@ def test_server_mode_times_out_when_never_healthy(flash_app, tmp_path): pid_file = f"/tmp/rp-test-server-{port}.pid" if os.path.exists(pid_file): - os.killpg(int(open(pid_file).read()), signal.SIGTERM) + os.killpg(int(Path(pid_file).read_text()), signal.SIGTERM) os.remove(pid_file)