diff --git a/.github/workflows/CD-publish_to_pypi.yml b/.github/workflows/CD-publish_to_pypi.yml deleted file mode 100644 index 9c73a1f9..00000000 --- a/.github/workflows/CD-publish_to_pypi.yml +++ /dev/null @@ -1,116 +0,0 @@ -name: CD | Publish Python 🐍 distributions đŸ“Ļ to PyPI - -on: - push: - branches: [main] - workflow_dispatch: - inputs: - force_publish: - description: 'Force publish without release-please' - required: true - default: 'false' - type: choice - options: - - 'true' - - 'false' - -permissions: - contents: write - pull-requests: write - actions: write - issues: write - id-token: write - -jobs: - release-please: - runs-on: ubuntu-latest - outputs: - release_created: ${{ steps.release.outputs.release_created }} - tag_name: ${{ steps.release.outputs.tag_name }} - steps: - - uses: google-github-actions/release-please-action@v3 - id: release - with: - token: ${{ secrets.GITHUB_TOKEN }} - release-type: python - package-name: runpod - - # PyPI publishing - pypi-publish: - name: PyPI Publish - needs: release-please - if: ${{ needs.release-please.outputs.release_created || (github.event_name == 'workflow_dispatch' && inputs.force_publish == 'true') }} - runs-on: ubuntu-latest - - environment: - name: pypi-production - url: https://pypi.org/project/runpod/ - - steps: - - name: Checkout code - uses: actions/checkout@v5 - with: - fetch-depth: 0 - - - name: Install uv - uses: astral-sh/setup-uv@v7 - with: - enable-cache: true - - - name: Set up Python 3.11.10 - run: uv python install 3.11.10 - - - name: Install dependencies - run: uv sync --all-groups - - - name: Build package - run: uv build - - - name: Verify package - run: uv run twine check dist/* - - - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - verbose: true - - notify-runpod-workers: - name: Notify workers - needs: [release-please, pypi-publish] - if: ${{ needs.release-please.outputs.release_created || (github.event_name == 'workflow_dispatch' && inputs.force_publish == 'true') }} - strategy: - matrix: - repo: - [ - "runpod-workers/worker-faster_whisper", - "runpod-workers/worker-stable_diffusion_v1", - "runpod-workers/worker-kandinsky", - "runpod-workers/worker-stable_diffusion_v2", - "runpod-workers/worker-template", - "runpod-workers/worker-whisper", - "runpod-workers/worker-esrgan", - "runpod-workers/worker-github_runner", - "runpod-workers/worker-a1111", - "runpod-workers/worker-dreambooth", - "runpod-workers/worker-bark", - "runpod-workers/worker-gpt", - "runpod-workers/worker-iseven", - "runpod-workers/worker-controlnet", - "runpod-workers/worker-blip", - "runpod-workers/worker-deforum", - runpod-workers/mock-worker, - ] - - runs-on: ubuntu-latest - - steps: - - name: Wait for propagation - run: sleep 300s - shell: bash - - - name: Repository Dispatch - uses: peter-evans/repository-dispatch@v4 - with: - token: ${{ secrets.RUNPOD_WORKERS_PAT }} - repository: ${{ matrix.repo }} - event-type: python-package-release diff --git a/.github/workflows/CD-test_publish_to_pypi.yml b/.github/workflows/CD-test_publish_to_pypi.yml deleted file mode 100644 index e9d707c5..00000000 --- a/.github/workflows/CD-test_publish_to_pypi.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: CD | Test Publish Python 🐍 distributions đŸ“Ļ to TestPyPI - -on: - push: - branches: - - "staging" - - workflow_dispatch: - -jobs: - build-n-publish: - name: Build and publish Python 🐍 distributions đŸ“Ļ to TestPyPI - runs-on: ubuntu-latest - environment: test-release - permissions: - contents: read - id-token: write # Required for OIDC - - steps: - - uses: actions/checkout@v5 - - name: Install uv - uses: astral-sh/setup-uv@v7 - with: - enable-cache: true - - - name: Set up Python 3.11.10 - run: uv python install 3.11.10 - - - name: Build package - run: uv build - - - name: Publish distribution đŸ“Ļ to Test PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - repository-url: https://test.pypi.org/legacy/ diff --git a/.github/workflows/CI-codeql.yml b/.github/workflows/CI-codeql.yml deleted file mode 100644 index a65a80e9..00000000 --- a/.github/workflows/CI-codeql.yml +++ /dev/null @@ -1,68 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "CI | CodeQL" - -on: - pull_request: - # The branches below must be a subset of the branches above - branches: ["main"] - schedule: - - cron: "15 1 * * 1" - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: ["python"] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] - # Use only 'java' to analyze code written in Java, Kotlin or both - # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both - # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support - - steps: - - name: Checkout repository - uses: actions/checkout@v5 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v4 - with: - languages: ${{ matrix.language }} - queries: security-extended,security-and-quality - - # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v4 - - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - - # If the Autobuild fails above, remove it and uncomment the following three lines. - # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. - - # - run: | - # echo "Run, Build Application using script" - # ./location_of_script_within_repo/buildscript.sh - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v4 - with: - category: "/language:${{matrix.language}}" diff --git a/.github/workflows/CI-e2e.yml b/.github/workflows/CI-e2e.yml deleted file mode 100644 index 224ca881..00000000 --- a/.github/workflows/CI-e2e.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: CI-e2e -on: - push: - branches: [main] - pull_request: - branches: [main] - workflow_dispatch: - -jobs: - e2e: - if: github.repository == 'runpod/runpod-python' - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - uses: actions/checkout@v5 - - - uses: astral-sh/setup-uv@v6 - - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - - name: Install dependencies - run: | - uv venv - source .venv/bin/activate - uv pip install -e ".[test]" --quiet || uv pip install -e . - uv pip install runpod-flash pytest pytest-asyncio pytest-timeout pytest-rerunfailures httpx - uv pip install -e . --reinstall --no-deps - python -c "import runpod; print(f'runpod: {runpod.__version__} from {runpod.__file__}')" - - - name: Run e2e tests - run: | - source .venv/bin/activate - pytest tests/e2e/ -v -p no:xdist --timeout=600 --reruns 1 --reruns-delay 5 --log-cli-level=INFO -o "addopts=" - env: - RUNPOD_API_KEY: ${{ secrets.RUNPOD_API_KEY }} - RUNPOD_SDK_GIT_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} diff --git a/.github/workflows/CI-pytests.yml b/.github/workflows/CI-pytests.yml deleted file mode 100644 index cb9b7bb7..00000000 --- a/.github/workflows/CI-pytests.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: CI | Unit Tests - -on: - push: - branches: - - main - - pull_request: - branches: - - main - - workflow_dispatch: - -jobs: - run_tests: - strategy: - matrix: - python-version: ["3.10", "3.11", "3.12"] - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v5 - - - name: Install uv - uses: astral-sh/setup-uv@v7 - with: - enable-cache: true - - - name: Set up Python ${{ matrix.python-version }} - run: uv python install ${{ matrix.python-version }} - - - name: Install Dependencies - run: uv sync --group test - - - name: Run Tests - run: uv run pytest diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml new file mode 100644 index 00000000..1d3e1d8e --- /dev/null +++ b/.github/workflows/cd.yml @@ -0,0 +1,142 @@ +name: CD + +on: + push: + branches: [main, staging] + workflow_dispatch: + inputs: + force_publish: + description: "Force publish without release-please" + required: true + default: "false" + type: choice + options: + - "true" + - "false" + +permissions: + contents: write + pull-requests: write + actions: write + issues: write + id-token: write + +jobs: + # ────────────────────────────────────────────── + # Staging: TestPyPI + # ────────────────────────────────────────────── + + test-publish: + if: github.ref == 'refs/heads/staging' + runs-on: ubuntu-latest + environment: test-release + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@v5 + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + + - name: Set up Python 3.11.10 + run: uv python install 3.11.10 + + - name: Build package + run: uv build + + - name: Publish to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + + # ────────────────────────────────────────────── + # Production: Release + PyPI + Worker notifications + # ────────────────────────────────────────────── + + release-please: + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + outputs: + release_created: ${{ steps.release.outputs.release_created }} + tag_name: ${{ steps.release.outputs.tag_name }} + steps: + - uses: google-github-actions/release-please-action@v3 + id: release + with: + token: ${{ secrets.GITHUB_TOKEN }} + release-type: python + package-name: runpod + + pypi-publish: + name: PyPI Publish + needs: release-please + if: ${{ always() && (needs.release-please.outputs.release_created || (github.event_name == 'workflow_dispatch' && inputs.force_publish == 'true')) }} + runs-on: ubuntu-latest + environment: + name: pypi-production + url: https://pypi.org/project/runpod/ + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + + - name: Set up Python 3.11.10 + run: uv python install 3.11.10 + + - name: Install dependencies + run: uv sync --all-groups + + - name: Build and verify package + run: make verify + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + verbose: true + + # TODO: Re-enable after optimizing (17 parallel jobs each sleeping 5min is wasteful). + # Consider a single job that sleeps once then dispatches sequentially. + # notify-workers: + # name: Notify workers + # needs: [release-please, pypi-publish] + # if: ${{ needs.release-please.outputs.release_created || (github.event_name == 'workflow_dispatch' && inputs.force_publish == 'true') }} + # runs-on: ubuntu-latest + # strategy: + # matrix: + # repo: + # - runpod-workers/worker-faster_whisper + # - runpod-workers/worker-stable_diffusion_v1 + # - runpod-workers/worker-kandinsky + # - runpod-workers/worker-stable_diffusion_v2 + # - runpod-workers/worker-template + # - runpod-workers/worker-whisper + # - runpod-workers/worker-esrgan + # - runpod-workers/worker-github_runner + # - runpod-workers/worker-a1111 + # - runpod-workers/worker-dreambooth + # - runpod-workers/worker-bark + # - runpod-workers/worker-gpt + # - runpod-workers/worker-iseven + # - runpod-workers/worker-controlnet + # - runpod-workers/worker-blip + # - runpod-workers/worker-deforum + # - runpod-workers/mock-worker + # steps: + # - name: Wait for PyPI propagation + # run: sleep 300s + # shell: bash + # + # - name: Repository Dispatch + # uses: peter-evans/repository-dispatch@v4 + # with: + # token: ${{ secrets.RUNPOD_WORKERS_PAT }} + # repository: ${{ matrix.repo }} + # event-type: python-package-release diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..1ce31570 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,118 @@ +name: CI + +on: + pull_request: + branches: [main] + push: + branches: [main] + schedule: + - cron: "15 1 * * 1" + workflow_dispatch: + +permissions: + contents: read + +jobs: + # ────────────────────────────────────────────── + # Quality gates (run in parallel) + # ────────────────────────────────────────────── + + test: + if: github.event_name != 'schedule' + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10", "3.11", "3.12"] + steps: + - uses: actions/checkout@v5 + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + + - name: Set up Python ${{ matrix.python-version }} + run: uv python install ${{ matrix.python-version }} + + - name: Setup dependencies + run: make setup + env: + UV_PYTHON: ${{ matrix.python-version }} + + - name: Run tests + run: make test + env: + UV_PYTHON: ${{ matrix.python-version }} + + e2e: + if: github.event_name != 'schedule' && github.repository == 'runpod/runpod-python' + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v5 + + - uses: astral-sh/setup-uv@v7 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies + run: | + uv venv + source .venv/bin/activate + uv pip install -e ".[test]" --quiet || uv pip install -e . + uv pip install runpod-flash pytest pytest-asyncio pytest-timeout pytest-rerunfailures httpx + uv pip install -e . --reinstall --no-deps + python -c "import runpod; print(f'runpod: {runpod.__version__} from {runpod.__file__}')" + + - name: Run e2e tests + run: | + source .venv/bin/activate + pytest tests/e2e/ -v -p no:xdist --timeout=600 --reruns 1 --reruns-delay 5 --log-cli-level=INFO -o "addopts=" + env: + RUNPOD_API_KEY: ${{ secrets.RUNPOD_API_KEY }} + RUNPOD_SDK_GIT_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + + codeql: + if: github.event_name == 'schedule' || github.event_name == 'pull_request' + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + steps: + - uses: actions/checkout@v5 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v4 + with: + languages: python + queries: security-extended,security-and-quality + + - name: Autobuild + uses: github/codeql-action/autobuild@v4 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v4 + with: + category: "/language:python" + + # ────────────────────────────────────────────── + # Validation gate + # ────────────────────────────────────────────── + + validation: + runs-on: ubuntu-latest + needs: [test, e2e, codeql] + if: always() + steps: + - name: Check all jobs succeeded + run: | + results=("${{ needs.test.result }}" "${{ needs.e2e.result }}" "${{ needs.codeql.result }}") + for result in "${results[@]}"; do + if [[ "$result" != "success" && "$result" != "skipped" ]]; then + echo "One or more quality checks failed (got: $result)" + exit 1 + fi + done diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..95145287 --- /dev/null +++ b/Makefile @@ -0,0 +1,22 @@ +.PHONY: setup test test-coverage build verify quality-check + +# --- Setup --- +setup: + uv sync --group test --group dev + +# --- Testing --- +test: + uv run pytest + +test-coverage: + uv run pytest --cov=runpod --cov-report=term-missing + +# --- Build --- +build: + uv build + +verify: build + uv run twine check dist/* + +# --- Quality Gate (expand with lint/format when ruff is configured) --- +quality-check: test-coverage