diff --git a/.github/workflows/build-pyramids-gis.yml b/.github/workflows/build-pyramids-gis.yml new file mode 100644 index 000000000000..f7331f9b29b5 --- /dev/null +++ b/.github/workflows/build-pyramids-gis.yml @@ -0,0 +1,288 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +--- +# This workflow is based on the `build-linux-wheels`/`test-wheels` jobs of +# https://github.com/serapeum-org/pyramids/blob/0.59.0/.github/workflows/bundle-pypi-wheels.yml +name: Build pyramids-gis wheels (riscv64) + +on: + workflow_dispatch: + inputs: + version: + description: 'Version glob to (re)build; empty builds every version of docs/packages/pyramids-gis.yaml not released yet' + required: false + default: '' + pull_request: + branches: [main] + paths: + - '.github/workflows/build-pyramids-gis.yml' + - 'docs/packages/pyramids-gis.yaml' + - 'patches/pyramids-gis/**' + push: + branches: [main] + paths: + - '.github/workflows/build-pyramids-gis.yml' + - 'docs/packages/pyramids-gis.yaml' + - 'patches/pyramids-gis/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read # to fetch code (actions/checkout) + +env: + BASEIMAGE: quay.io/pypa/manylinux_2_39_riscv64 + GDAL_IMAGE: manylinux_2_39_riscv64_gdal_pyramids_gis + +jobs: + setup: + uses: $/.github/workflows/_setup.yml + with: + package: pyramids-gis + version: ${{ inputs.version }} + + build_image: + needs: [setup] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + name: Build the GDAL build image + runs-on: ubuntu-24.04-riscv + timeout-minutes: 1440 + + env: + PYRAMIDS_GIS_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout pyramids ${{ env.PYRAMIDS_GIS_VERSION }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: serapeum-org/pyramids + ref: ${{ env.PYRAMIDS_GIS_VERSION }} + persist-credentials: false + + - name: Checkout python-wheels + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: python-wheels + persist-credentials: false + + - name: Patch pyramids source + run: git apply python-wheels/patches/pyramids-gis/${{ env.PYRAMIDS_GIS_VERSION }}/*.patch + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + with: + install: true + buildkitd-flags: --debug + + - name: Write the GDAL image Dockerfile + run: | + mkdir -p "$RUNNER_TEMP/gdal-ctx" + cat > "$RUNNER_TEMP/gdal-ctx/Dockerfile" <<'DOCKER_EOF' + # SPDX-FileCopyrightText: 2026 The RISE Project + # SPDX-License-Identifier: MIT + # + # upstream's before-all (ci/source-build/build-gdal-stack.sh) compiles ~26 + # libraries from source; run it once into an image rather than once per + # interpreter, with upstream's [tool.cibuildwheel.linux.environment]. + ARG BASEIMAGE=quay.io/pypa/manylinux_2_39_riscv64:latest + FROM ${BASEIMAGE} + + ENV BUILD_PREFIX=/usr/local \ + GDAL_DATA=/usr/local/share/gdal \ + PROJ_DATA=/usr/local/share/proj \ + PROJ_LIB=/usr/local/share/proj \ + LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64 + + COPY . /opt/pyramids/ci/source-build/ + RUN bash /opt/pyramids/ci/source-build/build-gdal-stack.sh \ + && rm -rf /opt/pyramids /tmp/gdal-src-build /tmp/probe.* + DOCKER_EOF + + - name: Build Docker image with GDAL + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: ci/source-build + file: ${{ runner.temp }}/gdal-ctx/Dockerfile + platforms: linux/riscv64 + build-args: | + BASEIMAGE=${{ env.BASEIMAGE }} + push: false + cache-from: type=gha,scope=${{ env.GDAL_IMAGE }}-${{ env.PYRAMIDS_GIS_VERSION }} + cache-to: type=gha,mode=max,scope=${{ env.GDAL_IMAGE }}-${{ env.PYRAMIDS_GIS_VERSION }} + env: + BUILDKIT_PROGRESS: plain + + build_wheels: + name: Build pyramids-gis ${{ matrix.version }} ${{ matrix.python }}-manylinux_riscv64 + needs: [setup, build_image] + if: needs.setup.outputs.versions != '[]' + runs-on: ubuntu-24.04-riscv + timeout-minutes: 600 + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + python: ["cp312", "cp313", "cp314"] + + env: + PYRAMIDS_GIS_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout pyramids ${{ env.PYRAMIDS_GIS_VERSION }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: serapeum-org/pyramids + ref: ${{ env.PYRAMIDS_GIS_VERSION }} + persist-credentials: false + + - name: Checkout python-wheels + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: python-wheels + persist-credentials: false + + - name: Patch pyramids source + run: git apply python-wheels/patches/pyramids-gis/${{ env.PYRAMIDS_GIS_VERSION }}/*.patch + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + with: + install: true + buildkitd-flags: --debug + + - name: Write the GDAL image Dockerfile + run: | + mkdir -p "$RUNNER_TEMP/gdal-ctx" + cat > "$RUNNER_TEMP/gdal-ctx/Dockerfile" <<'DOCKER_EOF' + # SPDX-FileCopyrightText: 2026 The RISE Project + # SPDX-License-Identifier: MIT + # + # upstream's before-all (ci/source-build/build-gdal-stack.sh) compiles ~26 + # libraries from source; run it once into an image rather than once per + # interpreter, with upstream's [tool.cibuildwheel.linux.environment]. + ARG BASEIMAGE=quay.io/pypa/manylinux_2_39_riscv64:latest + FROM ${BASEIMAGE} + + ENV BUILD_PREFIX=/usr/local \ + GDAL_DATA=/usr/local/share/gdal \ + PROJ_DATA=/usr/local/share/proj \ + PROJ_LIB=/usr/local/share/proj \ + LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64 + + COPY . /opt/pyramids/ci/source-build/ + RUN bash /opt/pyramids/ci/source-build/build-gdal-stack.sh \ + && rm -rf /opt/pyramids /tmp/gdal-src-build /tmp/probe.* + DOCKER_EOF + + - name: Load the GDAL build image from the cache + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: ci/source-build + file: ${{ runner.temp }}/gdal-ctx/Dockerfile + platforms: linux/riscv64 + tags: ${{ env.GDAL_IMAGE }}:${{ env.PYRAMIDS_GIS_VERSION }} + build-args: | + BASEIMAGE=${{ env.BASEIMAGE }} + push: false + load: true + cache-from: type=gha,scope=${{ env.GDAL_IMAGE }}-${{ env.PYRAMIDS_GIS_VERSION }} + env: + BUILDKIT_PROGRESS: plain + + - name: Build wheels + uses: pypa/cibuildwheel@1828c10ab37f080699c7b81cea34097c684a7074 # v4.2.0 + with: + output-dir: wheelhouse/ + only: ${{ matrix.python }}-manylinux_riscv64 + env: + CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.GDAL_IMAGE }}:${{ env.PYRAMIDS_GIS_VERSION }} + # the GDAL stack upstream's before-all builds is already in the image + CIBW_BEFORE_ALL: '' + CIBW_ENVIRONMENT: >- + BUILD_PREFIX=/usr/local + GDAL_DATA=/usr/local/share/gdal + PROJ_DATA=/usr/local/share/proj + PROJ_LIB=/usr/local/share/proj + PACKAGE_DATA=1 + LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64 + PIP_EXTRA_INDEX_URL=https://pypi.riseproject.dev/simple/ + # upstream pins --plat manylinux_2_28, below the riscv64 image's glibc floor + CIBW_REPAIR_WHEEL_COMMAND: auditwheel repair -w {dest_dir} {wheel} + # upstream skips cibuildwheel's test step and runs the same commands in its + # separate test-wheels job, which needs riscv64 hardware here anyway + CIBW_TEST_SKIP: '' + CIBW_TEST_SOURCES: tests ci pyproject.toml pixi.lock + CIBW_TEST_REQUIRES: pytest pytest-env pytest-mock pytest-timeout + CIBW_TEST_ENVIRONMENT: 'PIP_ONLY_BINARY=:all:' + # the two vsi tests need userfaultfd, which this runner's kernel denies regardless + # of container seccomp/capabilities (gotcha 662); the third is an independent, + # riscv64-specific hang in GDAL's curvilinear-grid read path (gotcha 663) + CIBW_TEST_COMMAND: >- + python ci/verify-wheel.py && + pytest -vvv -o "pythonpath=" --timeout=60 --timeout-method=thread -m "core and not vfs and not slow" tests/ + --deselect "tests/netcdf/io/test_netcdf_archive.py::TestNetCDFReadFileArchiveLinux::test_open_zipped_nc_via_vsi_auto" + --deselect "tests/netcdf/io/test_netcdf_archive.py::TestNetCDFReadFileArchiveLinux::test_open_zipped_nc_via_vsi_zip" + --deselect "tests/netcdf/samples/test_array_io.py::test_read_array_returns_spatial_array[none__5v__1d2-2d2-3d1__curv]" + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: pyramids-gis-${{ env.PYRAMIDS_GIS_VERSION }}-${{ matrix.python }}-manylinux_riscv64 + path: wheelhouse/*.whl + if-no-files-found: error + + gpl_sources: + needs: [setup] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + name: Collect GPL sources + runs-on: ubuntu-24.04-riscv + + env: + PYRAMIDS_GIS_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout python-wheels + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # libgdal links OpenMP, so auditwheel vendors the image's libgomp. + - uses: ./actions/collect-gpl-sources + with: + image: ${{ env.BASEIMAGE }} + packages: gcc + output: gpl-sources.tar + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: pyramids-gis-${{ env.PYRAMIDS_GIS_VERSION }}-gpl-sources + path: gpl-sources.tar + if-no-files-found: error + + publish: + name: Publish pyramids-gis ${{ matrix.version }} + needs: [setup, build_wheels, gpl_sources] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + permissions: + contents: write + pull-requests: write + uses: $/.github/workflows/_publish-wheel.yml + secrets: + app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} + with: + artifact-pattern: pyramids-gis-${{ matrix.version }}-*-manylinux_riscv64 + gpl-sources-artifact: pyramids-gis-${{ matrix.version }}-gpl-sources + gpl-sources-description: gcc diff --git a/docs/packages/pyramids-gis.yaml b/docs/packages/pyramids-gis.yaml new file mode 100644 index 000000000000..86b8be668b8d --- /dev/null +++ b/docs/packages/pyramids-gis.yaml @@ -0,0 +1,5 @@ +package-name: pyramids-gis +source-code: https://github.com/serapeum-org/pyramids +license: GPL-3.0-only +versions: +- version: 0.59.0 diff --git a/patches/pyramids-gis/0.59.0/0001-source-build-add-the-riscv64-OpenSSL-target.patch b/patches/pyramids-gis/0.59.0/0001-source-build-add-the-riscv64-OpenSSL-target.patch new file mode 100644 index 000000000000..8ad514958cbb --- /dev/null +++ b/patches/pyramids-gis/0.59.0/0001-source-build-add-the-riscv64-OpenSSL-target.patch @@ -0,0 +1,31 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Wed, 14 Oct 2026 18:00:00 +0000 +Subject: [PATCH] source-build: add the riscv64 OpenSSL target + +ci/source-build/config.sh maps `uname -m` to the target name OpenSSL's +Configure expects and aborts on anything other than x86_64/aarch64, so +the from-source GDAL stack (and with it the Linux wheel) cannot be built +on riscv64 at all. OpenSSL has shipped a linux64-riscv64 target since +3.0; map riscv64 to it. + +Upstream-Status: To upstream [one-line arch mapping, applies unchanged to upstream main] +--- + ci/source-build/config.sh | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/ci/source-build/config.sh b/ci/source-build/config.sh +index 8cad992fe..98ad64a00 100644 +--- a/ci/source-build/config.sh ++++ b/ci/source-build/config.sh +@@ -42,6 +42,7 @@ ARCH="$(uname -m)" + case "${ARCH}" in + x86_64) OPENSSL_TARGET="linux-x86_64" ;; + aarch64) OPENSSL_TARGET="linux-aarch64" ;; ++ riscv64) OPENSSL_TARGET="linux64-riscv64" ;; + *) echo "ERROR: unsupported architecture ${ARCH}" >&2; exit 1 ;; + esac + +-- +2.43.0 + diff --git a/skills/python-project-porting/references/gotchas-index.md b/skills/python-project-porting/references/gotchas-index.md index 12cc82f5f639..c5db9bc39c17 100644 --- a/skills/python-project-porting/references/gotchas-index.md +++ b/skills/python-project-porting/references/gotchas-index.md @@ -1,6 +1,6 @@ # Gotchas index — router for the themed gotcha files -The porting gotchas (559 of them) live in [`references/gotchas/`](gotchas/), split by theme so only the relevant slice loads. Every gotcha keeps a **permanent number** cited elsewhere as "gotcha N" (and in workflow comments as "CLAUDE.md gotcha N"). Numbers are stable IDs — **not sequential**, and four are **reused** with different content (two each of 33, 55, 56, 57), disambiguated by theme below. +The porting gotchas (561 of them) live in [`references/gotchas/`](gotchas/), split by theme so only the relevant slice loads. Every gotcha keeps a **permanent number** cited elsewhere as "gotcha N" (and in workflow comments as "CLAUDE.md gotcha N"). Numbers are stable IDs — **not sequential**, and four are **reused** with different content (two each of 33, 55, 56, 57), disambiguated by theme below. ## How to find the gotcha you need @@ -1403,6 +1403,14 @@ The porting gotchas (559 of them) live in [`references/gotchas/`](gotchas/), spl - **643** — A test suite that needs Node.js (an npm mock server) still runs on riscv64: `actions/setup-node` with `mirror: https://unofficial-builds.nodejs.org/download/release` installs a riscv64 Node (the eppo-server-sdk case). +- **662** — A netCDF `/vsizip/`/`/vsi*`-auto open needs `userfaultfd(2)`, gated by the host + kernel's `vm.unprivileged_userfaultfd` sysctl, not by anything a container's seccomp + profile or capability set reaches — don't treat gotcha 552's `CAP_SYS_PTRACE` fix as a + template for this syscall too; deselect rather than keep widening the container (pyramids-gis). +- **663** — A hang reproducible at one exact parametrize id, with sibling parametrizations + passing clean on either side, is a per-input library bug, not corrupted GDAL global state + left over from an earlier failed test; `--timeout-method=thread` can't interrupt a + blocking GIL-held C call, so only the job's own wall clock ends the hang (pyramids-gis). ### Test failures, flakes & arch-specific bugs — [`gotchas/test-failures-and-flakes.md`](gotchas/test-failures-and-flakes.md) diff --git a/skills/python-project-porting/references/gotchas/pytest-config-servers-and-selection.md b/skills/python-project-porting/references/gotchas/pytest-config-servers-and-selection.md index 3b4065bf3cdf..5605889001bf 100644 --- a/skills/python-project-porting/references/gotchas/pytest-config-servers-and-selection.md +++ b/skills/python-project-porting/references/gotchas/pytest-config-servers-and-selection.md @@ -57,6 +57,12 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/pytest-config-servers- - **643** — An upstream test suite that needs a Node.js helper (a mock server via `npm ci`/`npm run`) still runs on the riscv64 runner: `actions/setup-node` with `mirror: https://unofficial-builds.nodejs.org/download/release` installs a riscv64 Node. +- **662** — A netCDF `/vsizip/`/`/vsi*` open needs `userfaultfd(2)`, which a fully unconfined + seccomp profile and `CAP_SYS_PTRACE` do not restore — that syscall is gated by the host + kernel's `vm.unprivileged_userfaultfd` sysctl, not by anything a container flag reaches. +- **663** — A hang reproducible at one exact parametrize id, with sibling parametrizations + passing clean on either side, is a per-input library bug, not corrupted global state from + an earlier failure; `--timeout-method=thread` can't interrupt a blocking GIL-held C call. --- @@ -964,3 +970,50 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/pytest-config-servers- filter entries whose `files` include `linux-riscv64`. This covers Node used as a *test-time tool* only. A package that bundles or downloads a Node binary into the wheel is a different question (feasibility gotchas on playwright). + +662. **A netCDF `/vsizip/`/`/vsi*`-auto open genuinely needs Linux `userfaultfd(2)`, which + neither a fully unconfined seccomp profile nor `CAP_SYS_PTRACE` restores — don't treat + gotcha 552's fix as a template for every container-syscall `EPERM` (the pyramids-gis + case).** GDAL's netCDF driver opens any `/vsi*` virtual-filesystem path (zip, auto + archive detection) through a `userfaultfd`-backed memory-mapping shim. + `tests/netcdf/io/test_netcdf_archive.py::TestNetCDFReadFileArchiveLinux::test_open_zipped_nc_via_vsi_auto` + and `...::test_open_zipped_nc_via_vsi_zip` failed identically, byte-for-byte, across + three separate riscv64 CI runs: the baseline, a round adding + `CIBW_CONTAINER_ENGINE: "docker; create_args: --security-opt=seccomp=unconfined"`, and a + further round adding `--cap-add=SYS_PTRACE` on top. Gotcha 552's fix (that same + capability) worked for a *different* syscall — `pidfd_getfd(2)`, blocked by Docker's + *default*, non-unconfined seccomp profile. `userfaultfd(2)` is a different mechanism, + gated by the **host kernel's** `vm.unprivileged_userfaultfd` sysctl, checked before a + container's own seccomp filter or capability set are ever consulted. No `docker run` + flag reachable from an unprivileged cibuildwheel container flips that sysctl — it needs + `--privileged` or a host/runner-level `sysctl -w`, both out of a package workflow's + reach. **Fix: deselect both tests and revert any seccomp/capability widening added + chasing this** once a second round lands the identical failure — the generalizable tell + is the syscall itself (the traceback or a `strace` names `userfaultfd`), not the shape + of the failure. + +663. **A hang reproducible at one exact pytest parametrize id, with sibling parametrizations + of the same test — including other same-shaped ones — passing clean on either side of + it, is a per-input bug in the library's C code, not corrupted global state left over + from an earlier failed test; `--timeout-method=thread` reports `Timeout` without + actually freeing the runner (the pyramids-gis case).** + `tests/netcdf/samples/test_array_io.py::test_read_array_returns_spatial_array[none__5v__1d2-2d2-3d1__curv]` + hung for roughly 800-850s before pytest-timeout's thread watchdog finally fired, + identically across CI runs, even though the suite's own `--timeout=60` is far shorter — + while many neighboring parametrizations, including other `curv`-suffixed ones, passed + normally just before and after it in collection order. That pattern (not "everything + after the first failure gets slower or flakier", not "only this grid type across the + board") rules out the tempting but wrong theory that the two FAILED tests earlier in the + same run (gotcha 662) left GDAL's driver registry or PROJ context corrupted for the rest + of the process: state corruption reads as noisy and order-dependent, not reproducibly + keyed to one specific data shape (5 variables, dims `1d2-2d2-3d1`, curvilinear grid). + **`--timeout-method=thread` can't save you here** — its watcher thread still needs the + GIL to act, and a blocking call stuck inside GDAL's curvilinear-grid raster-read path in + C never yields it, so the configured timeout is not what actually ends the hang; only + the job's own wall clock (`timeout-minutes`) does, burning the full hang's length on + every rebuild before the run is even reported as failed. Not yet root-caused at the C + level. **Fix: deselect the exact parametrize id** (`--deselect + "path::test[none__5v__1d2-2d2-3d1__curv]"`, copied verbatim from what pytest itself + reports and quoted for the brackets) rather than widening a marker or `-k` expression + that would also drop the passing sibling parametrizations, and rather than leaving it to + hang and burn CI minutes on every run.