python-publish.yml

# Build, test and publish lance-csr wheels: Linux x86-64 and aarch64 (manylinux_2_28),
# native Apple Silicon (macOS arm64) and Windows x86-64, plus an sdist for every other platform.
#
# Pull requests that change how wheels are built (this workflow, the wheel check, the
# release build profile) build and test them without publishing. A release builds and tests
# every wheel before publishing, so it needs no separate dry run; see RELEASE.md.
# A published GitHub release (tag vX.Y.Z) publishes to PyPI after every build and test
# passes, through PyPI trusted publishing and the protected `pypi` environment; so does
# a manual run with mode=release. mode=dry_run builds and tests only. Once PyPI has the
# release, the crates go to crates.io through crates.io trusted publishing and the
# protected `crates-io` environment.
name: Python Publish

on:
  release:
    types:
      - published
  pull_request:
    paths:
      - .github/workflows/python-publish.yml
      - ci/check_wheel.py
      - python/.cargo/**
  workflow_dispatch:
    inputs:
      mode:
        description: "dry_run: build and test only; release: build, test and publish to PyPI"
        required: true
        default: dry_run
        type: choice
        options:
          - dry_run
          - release

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  CARGO_TERM_COLOR: always
  CARGO_INCREMENTAL: "0"

jobs:
  build-linux:
    # Built on the host with zig as the linker, targeting glibc 2.28 (RHEL 8, Ubuntu 20.04
    # and newer); the same command produced the alpha's tested Linux wheels. Each
    # architecture builds natively on its own runner.
    strategy:
      fail-fast: false
      matrix:
        include:
          - arch: x86_64
            runner: ubuntu-24.04
          - arch: aarch64
            runner: ubuntu-24.04-arm
    runs-on: ${{ matrix.runner }}
    timeout-minutes: 90
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
        with:
          python-version: "3.12"

      - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3

        with:

          enable-cache: false
      - name: Install build dependencies
        run: |
          # protoc is needed by Lance's build scripts, not by wheel users.
          sudo apt-get update
          sudo apt-get install -y protobuf-compiler
          # The ziglang package ships a zig binary: put its directory on PATH, where maturin
          # looks first. (Not the tool's python3, which has no pip for maturin-action to use.)
          uv tool install ziglang
          "$(uv tool dir)/ziglang/bin/python" -c "import os, ziglang; print(os.path.dirname(ziglang.__file__))" >> "$GITHUB_PATH"

      - name: Build manylinux_2_28 ${{ matrix.arch }} wheel
        uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0
        with:
          command: build
          working-directory: python
          target: ${{ matrix.arch }}-unknown-linux-gnu
          container: "off"
          # No sccache: release wheels are built without restoring any cache.
          sccache: "false"
          args: --release --locked --strip --zig --compatibility manylinux_2_28 --out dist

      - name: Build sdist
        if: matrix.arch == 'x86_64'
        uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0
        with:
          command: sdist
          working-directory: python
          args: --out dist

      - name: Inspect distributions
        working-directory: python
        run: |
          uvx twine check dist/*
          sha256sum dist/*

      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        with:
          name: python-dist-linux-${{ matrix.arch }}
          path: python/dist/*
          if-no-files-found: error

  build-macos:
    # GitHub's macos-15 runners are Apple Silicon, so this builds natively.
    runs-on: macos-15
    timeout-minutes: 90
    env:
      MACOSX_DEPLOYMENT_TARGET: "11.0"
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
        with:
          python-version: "3.12"

      - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3

        with:

          enable-cache: false
      - name: Install build dependencies
        run: brew install protobuf

      - name: Build Apple Silicon wheel
        uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0
        with:
          command: build
          working-directory: python
          target: aarch64-apple-darwin
          # No sccache: release wheels are built without restoring any cache.
          sccache: "false"
          args: --release --locked --strip --out dist

      - name: Inspect wheel
        working-directory: python
        run: |
          uvx twine check dist/*.whl
          shasum -a 256 dist/*.whl

      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        with:
          name: python-dist-macos-arm64
          path: python/dist/*.whl
          if-no-files-found: error

  build-windows:
    runs-on: windows-2022
    timeout-minutes: 90
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
        with:
          python-version: "3.12"

      - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3

        with:

          enable-cache: false
      - name: Install build dependencies
        # protoc comes from its GitHub release: Chocolatey's feed is intermittently unreachable from runners.
        run: |
          $zip = "$env:RUNNER_TEMP\protoc.zip"
          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v29.3/protoc-29.3-win64.zip -OutFile $zip
          if ((Get-FileHash $zip -Algorithm SHA256).Hash -ne "57EA59E9F551AD8D71FFAA9B5CFBE0CA1F4E720972A1DB7EC2D12AB44BFF9383") { throw "protoc checksum mismatch" }
          Expand-Archive $zip "$env:RUNNER_TEMP\protoc"
          "$env:RUNNER_TEMP\protoc\bin" | Out-File -Append -Encoding utf8 $env:GITHUB_PATH

      - name: Build Windows x86-64 wheel
        uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0
        with:
          command: build
          working-directory: python
          target: x86_64-pc-windows-msvc
          # No sccache: release wheels are built without restoring any cache.
          sccache: "false"
          args: --release --locked --strip --out dist

      - name: Inspect wheel
        working-directory: python
        shell: bash
        run: |
          uvx twine check dist/*.whl
          sha256sum dist/*.whl

      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        with:
          name: python-dist-windows-x64
          path: python/dist/*.whl
          if-no-files-found: error

  test:
    # The wheels are abi3 (CPython 3.12+), so each platform's single wheel is tested on
    # every supported Python, in a clean environment with Rust removed from PATH: a wheel
    # that silently needed compiling would fail here.
    needs: [build-linux, build-macos, build-windows]
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-24.04, ubuntu-24.04-arm, macos-15, windows-2022]
        python: ["3.12", "3.13", "3.14"]
    runs-on: ${{ matrix.os }}
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
          sparse-checkout: |
            ci
            python/python/tests

      - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3

        with:

          enable-cache: false
      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
        with:
          pattern: python-dist-*
          merge-multiple: true
          path: dist

      - name: Install the wheel and run the test suite outside the source tree
        shell: bash
        run: |
          export PATH=$(echo "$PATH" | tr ':' '\n' | grep -v -e cargo -e rustup | paste -sd: -)
          ! command -v cargo && ! command -v rustc
          # This platform's wheel, by path, so PyPI's copy of the same version can't stand in.
          case "$RUNNER_OS" in
            Linux) wheel=$(ls "$PWD"/dist/*manylinux*_"$(uname -m)".whl) ;;
            macOS) wheel=$(ls "$PWD"/dist/*macosx*.whl) ;;
            Windows) wheel=$(cygpath -w "$(ls "$PWD"/dist/*win_amd64*.whl)") ;;
          esac
          # A fresh environment with the wheel and test tools; --no-build refuses any sdist.
          run=(uv run --isolated --no-project --no-build --python ${{ matrix.python }}
               --with "$wheel" --with pytest --with pandas)
          "${run[@]}" python -I ci/check_wheel.py
          cp -r python/python/tests "$RUNNER_TEMP/tests"
          cd "$RUNNER_TEMP"
          "${run[@]}" python -I -m pytest "$RUNNER_TEMP/tests" -q -p no:cacheprovider

  publish:
    needs: [build-linux, build-macos, build-windows, test]
    if: >
      github.event_name == 'release' ||
      (github.event_name == 'workflow_dispatch' && github.event.inputs.mode == 'release')
    runs-on: ubuntu-24.04
    environment:
      name: pypi
      url: https://pypi.org/project/lance-csr/
    permissions:
      id-token: write
      contents: read
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
          sparse-checkout: python/pyproject.toml

      - name: Check the release tag matches the package version
        if: github.event_name == 'release'
        env:
          TAG: ${{ github.event.release.tag_name }}
        run: |
          version=$(grep -m1 '^version = ' python/pyproject.toml | cut -d'"' -f2)
          test "$TAG" = "v$version" || {
            echo "Release tag $TAG does not match version $version"; exit 1; }

      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
        with:
          pattern: python-dist-*
          merge-multiple: true
          path: dist

      - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3

        with:

          enable-cache: false
      - name: Publish to PyPI
        run: uv publish --trusted-publishing always dist/*

  publish-crates:
    # After PyPI, so the two registries never disagree. If only this job fails, rerun it on
    # its own; the wheels don't need rebuilding. The crates.io trusted publisher for the
    # crate is registered for this workflow and environment `crates-io`.
    needs: publish
    runs-on: ubuntu-24.04
    timeout-minutes: 60
    environment:
      name: crates-io
      url: https://crates.io/crates/lance-csr
    permissions:
      id-token: write
      contents: read
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - name: Install build dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y protobuf-compiler

      # Verifying compiles lance, so do it before getting the token, which lasts 30 minutes.
      - name: Package and verify the crate
        run: cargo package --locked -p lance-csr

      - uses: rust-lang/crates-io-auth-action@3268eb413133e2960fa4ae0e963ca76380c54cd9 # v1.0.6
        id: auth

      - name: Publish to crates.io
        run: cargo publish --locked --no-verify -p lance-csr
        env:
          CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }}