From 6ff3412ba2f386fcf05e9e8aa5f88a43e1142977 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Wed, 14 Oct 2026 18:50:20 +0000 Subject: [PATCH 1/5] pyramids-gis: Add version 0.59.0 Build the riscv64 wheels with upstream's own Linux recipe: its ci/source-build/build-gdal-stack.sh compiles GDAL and its ~26-library dependency stack from SHA256-pinned tarballs, and before-build vendors the GDAL Python bindings into the wheel. The stack is built once into a manylinux_2_39_riscv64-derived image instead of once per interpreter. config.sh only maps x86_64/aarch64 to an OpenSSL target, so a patch adds linux64-riscv64. auditwheel's plat is left to default because upstream's manylinux_2_28 is below the riscv64 image's floor. Tests are upstream's test-wheels commands (verify-wheel.py plus the hermetic core suite). cp311 is dropped because cftime/pyproj/shapely have no cp311 riscv64 wheel; cp314t is skipped as upstream does. --- .github/workflows/build-pyramids-gis.yml | 282 ++++++++++++++++++ docs/packages/pyramids-gis.yaml | 5 + ...build-add-the-riscv64-OpenSSL-target.patch | 31 ++ 3 files changed, 318 insertions(+) create mode 100644 .github/workflows/build-pyramids-gis.yml create mode 100644 docs/packages/pyramids-gis.yaml create mode 100644 patches/pyramids-gis/0.59.0/0001-source-build-add-the-riscv64-OpenSSL-target.patch diff --git a/.github/workflows/build-pyramids-gis.yml b/.github/workflows/build-pyramids-gis.yml new file mode 100644 index 000000000000..d75a90fe9f43 --- /dev/null +++ b/.github/workflows/build-pyramids-gis.yml @@ -0,0 +1,282 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +--- +# This workflow is based on the `build-linux-wheels`/`test-wheels` jobs of +# https://github.com/serapeum-org/pyramids/blob/0.59.0/.github/workflows/bundle-pypi-wheels.yml +name: Build pyramids-gis wheels (riscv64) + +on: + workflow_dispatch: + inputs: + version: + description: 'Version glob to (re)build; empty builds every version of docs/packages/pyramids-gis.yaml not released yet' + required: false + default: '' + pull_request: + branches: [main] + paths: + - '.github/workflows/build-pyramids-gis.yml' + - 'docs/packages/pyramids-gis.yaml' + - 'patches/pyramids-gis/**' + push: + branches: [main] + paths: + - '.github/workflows/build-pyramids-gis.yml' + - 'docs/packages/pyramids-gis.yaml' + - 'patches/pyramids-gis/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read # to fetch code (actions/checkout) + +env: + BASEIMAGE: quay.io/pypa/manylinux_2_39_riscv64 + GDAL_IMAGE: manylinux_2_39_riscv64_gdal_pyramids_gis + +jobs: + setup: + uses: $/.github/workflows/_setup.yml + with: + package: pyramids-gis + version: ${{ inputs.version }} + + build_image: + needs: [setup] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + name: Build the GDAL build image + runs-on: ubuntu-24.04-riscv + timeout-minutes: 1440 + + env: + PYRAMIDS_GIS_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout pyramids ${{ env.PYRAMIDS_GIS_VERSION }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: serapeum-org/pyramids + ref: ${{ env.PYRAMIDS_GIS_VERSION }} + persist-credentials: false + + - name: Checkout python-wheels + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: python-wheels + persist-credentials: false + + - name: Patch pyramids source + run: git apply python-wheels/patches/pyramids-gis/${{ env.PYRAMIDS_GIS_VERSION }}/*.patch + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + with: + install: true + buildkitd-flags: --debug + + - name: Write the GDAL image Dockerfile + run: | + mkdir -p "$RUNNER_TEMP/gdal-ctx" + cat > "$RUNNER_TEMP/gdal-ctx/Dockerfile" <<'DOCKER_EOF' + # SPDX-FileCopyrightText: 2026 The RISE Project + # SPDX-License-Identifier: MIT + # + # upstream's before-all (ci/source-build/build-gdal-stack.sh) compiles ~26 + # libraries from source; run it once into an image rather than once per + # interpreter, with upstream's [tool.cibuildwheel.linux.environment]. + ARG BASEIMAGE=quay.io/pypa/manylinux_2_39_riscv64:latest + FROM ${BASEIMAGE} + + ENV BUILD_PREFIX=/usr/local \ + GDAL_DATA=/usr/local/share/gdal \ + PROJ_DATA=/usr/local/share/proj \ + PROJ_LIB=/usr/local/share/proj \ + LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64 + + COPY . /opt/pyramids/ci/source-build/ + RUN bash /opt/pyramids/ci/source-build/build-gdal-stack.sh \ + && rm -rf /opt/pyramids /tmp/gdal-src-build /tmp/probe.* + DOCKER_EOF + + - name: Build Docker image with GDAL + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: ci/source-build + file: ${{ runner.temp }}/gdal-ctx/Dockerfile + platforms: linux/riscv64 + build-args: | + BASEIMAGE=${{ env.BASEIMAGE }} + push: false + cache-from: type=gha,scope=${{ env.GDAL_IMAGE }}-${{ env.PYRAMIDS_GIS_VERSION }} + cache-to: type=gha,mode=max,scope=${{ env.GDAL_IMAGE }}-${{ env.PYRAMIDS_GIS_VERSION }} + env: + BUILDKIT_PROGRESS: plain + + build_wheels: + name: Build pyramids-gis ${{ matrix.version }} ${{ matrix.python }}-manylinux_riscv64 + needs: [setup, build_image] + if: needs.setup.outputs.versions != '[]' + runs-on: ubuntu-24.04-riscv + timeout-minutes: 600 + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + python: ["cp312", "cp313", "cp314"] + + env: + PYRAMIDS_GIS_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout pyramids ${{ env.PYRAMIDS_GIS_VERSION }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: serapeum-org/pyramids + ref: ${{ env.PYRAMIDS_GIS_VERSION }} + persist-credentials: false + + - name: Checkout python-wheels + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: python-wheels + persist-credentials: false + + - name: Patch pyramids source + run: git apply python-wheels/patches/pyramids-gis/${{ env.PYRAMIDS_GIS_VERSION }}/*.patch + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + with: + install: true + buildkitd-flags: --debug + + - name: Write the GDAL image Dockerfile + run: | + mkdir -p "$RUNNER_TEMP/gdal-ctx" + cat > "$RUNNER_TEMP/gdal-ctx/Dockerfile" <<'DOCKER_EOF' + # SPDX-FileCopyrightText: 2026 The RISE Project + # SPDX-License-Identifier: MIT + # + # upstream's before-all (ci/source-build/build-gdal-stack.sh) compiles ~26 + # libraries from source; run it once into an image rather than once per + # interpreter, with upstream's [tool.cibuildwheel.linux.environment]. + ARG BASEIMAGE=quay.io/pypa/manylinux_2_39_riscv64:latest + FROM ${BASEIMAGE} + + ENV BUILD_PREFIX=/usr/local \ + GDAL_DATA=/usr/local/share/gdal \ + PROJ_DATA=/usr/local/share/proj \ + PROJ_LIB=/usr/local/share/proj \ + LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64 + + COPY . /opt/pyramids/ci/source-build/ + RUN bash /opt/pyramids/ci/source-build/build-gdal-stack.sh \ + && rm -rf /opt/pyramids /tmp/gdal-src-build /tmp/probe.* + DOCKER_EOF + + - name: Load the GDAL build image from the cache + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: ci/source-build + file: ${{ runner.temp }}/gdal-ctx/Dockerfile + platforms: linux/riscv64 + tags: ${{ env.GDAL_IMAGE }}:${{ env.PYRAMIDS_GIS_VERSION }} + build-args: | + BASEIMAGE=${{ env.BASEIMAGE }} + push: false + load: true + cache-from: type=gha,scope=${{ env.GDAL_IMAGE }}-${{ env.PYRAMIDS_GIS_VERSION }} + env: + BUILDKIT_PROGRESS: plain + + - name: Build wheels + uses: pypa/cibuildwheel@1828c10ab37f080699c7b81cea34097c684a7074 # v4.2.0 + with: + output-dir: wheelhouse/ + only: ${{ matrix.python }}-manylinux_riscv64 + env: + CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.GDAL_IMAGE }}:${{ env.PYRAMIDS_GIS_VERSION }} + # the GDAL stack upstream's before-all builds is already in the image + CIBW_BEFORE_ALL: '' + CIBW_ENVIRONMENT: >- + BUILD_PREFIX=/usr/local + GDAL_DATA=/usr/local/share/gdal + PROJ_DATA=/usr/local/share/proj + PROJ_LIB=/usr/local/share/proj + PACKAGE_DATA=1 + LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64 + PIP_EXTRA_INDEX_URL=https://pypi.riseproject.dev/simple/ + # upstream pins --plat manylinux_2_28, below the riscv64 image's glibc floor + CIBW_REPAIR_WHEEL_COMMAND: auditwheel repair -w {dest_dir} {wheel} + # upstream skips cibuildwheel's test step and runs the same commands in its + # separate test-wheels job, which needs riscv64 hardware here anyway + CIBW_TEST_SKIP: '' + CIBW_TEST_SOURCES: tests ci pyproject.toml pixi.lock + CIBW_TEST_REQUIRES: pytest pytest-env pytest-mock pytest-timeout + CIBW_TEST_ENVIRONMENT: 'PIP_ONLY_BINARY=:all:' + CIBW_TEST_COMMAND: >- + python ci/verify-wheel.py && + pytest -vvv -o "pythonpath=" --timeout=60 --timeout-method=thread -m "core and not vfs and not slow" tests/ + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: pyramids-gis-${{ env.PYRAMIDS_GIS_VERSION }}-${{ matrix.python }}-manylinux_riscv64 + path: wheelhouse/*.whl + if-no-files-found: error + + gpl_sources: + needs: [setup] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + name: Collect GPL sources + runs-on: ubuntu-24.04-riscv + + env: + PYRAMIDS_GIS_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout python-wheels + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # libgdal links OpenMP, so auditwheel vendors the image's libgomp. + - uses: ./actions/collect-gpl-sources + with: + image: ${{ env.BASEIMAGE }} + packages: gcc + output: gpl-sources.tar + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: pyramids-gis-${{ env.PYRAMIDS_GIS_VERSION }}-gpl-sources + path: gpl-sources.tar + if-no-files-found: error + + publish: + name: Publish pyramids-gis ${{ matrix.version }} + needs: [setup, build_wheels, gpl_sources] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + permissions: + contents: write + pull-requests: write + uses: $/.github/workflows/_publish-wheel.yml + secrets: + app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} + with: + artifact-pattern: pyramids-gis-${{ matrix.version }}-*-manylinux_riscv64 + gpl-sources-artifact: pyramids-gis-${{ matrix.version }}-gpl-sources + gpl-sources-description: gcc diff --git a/docs/packages/pyramids-gis.yaml b/docs/packages/pyramids-gis.yaml new file mode 100644 index 000000000000..86b8be668b8d --- /dev/null +++ b/docs/packages/pyramids-gis.yaml @@ -0,0 +1,5 @@ +package-name: pyramids-gis +source-code: https://github.com/serapeum-org/pyramids +license: GPL-3.0-only +versions: +- version: 0.59.0 diff --git a/patches/pyramids-gis/0.59.0/0001-source-build-add-the-riscv64-OpenSSL-target.patch b/patches/pyramids-gis/0.59.0/0001-source-build-add-the-riscv64-OpenSSL-target.patch new file mode 100644 index 000000000000..8ad514958cbb --- /dev/null +++ b/patches/pyramids-gis/0.59.0/0001-source-build-add-the-riscv64-OpenSSL-target.patch @@ -0,0 +1,31 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Wed, 14 Oct 2026 18:00:00 +0000 +Subject: [PATCH] source-build: add the riscv64 OpenSSL target + +ci/source-build/config.sh maps `uname -m` to the target name OpenSSL's +Configure expects and aborts on anything other than x86_64/aarch64, so +the from-source GDAL stack (and with it the Linux wheel) cannot be built +on riscv64 at all. OpenSSL has shipped a linux64-riscv64 target since +3.0; map riscv64 to it. + +Upstream-Status: To upstream [one-line arch mapping, applies unchanged to upstream main] +--- + ci/source-build/config.sh | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/ci/source-build/config.sh b/ci/source-build/config.sh +index 8cad992fe..98ad64a00 100644 +--- a/ci/source-build/config.sh ++++ b/ci/source-build/config.sh +@@ -42,6 +42,7 @@ ARCH="$(uname -m)" + case "${ARCH}" in + x86_64) OPENSSL_TARGET="linux-x86_64" ;; + aarch64) OPENSSL_TARGET="linux-aarch64" ;; ++ riscv64) OPENSSL_TARGET="linux64-riscv64" ;; + *) echo "ERROR: unsupported architecture ${ARCH}" >&2; exit 1 ;; + esac + +-- +2.43.0 + From 111576a56e80d85c0853160e6697f45043466d4d Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Sun, 25 Oct 2026 04:56:58 +0000 Subject: [PATCH 2/5] pyramids-gis: lift seccomp in the riscv64 wheel test container The netCDF driver needs Linux userfaultfd to open any .nc path routed through GDAL's VSI layer (zip, tar, vsicurl, ...), and Docker's default seccomp profile blocks that syscall outright. cibuildwheel runs this port's test suite inside its own container rather than on the bare runner (unlike upstream, whose equivalent job runs outside Docker), so the two vsizip netCDF tests failed with GDAL's own "requires Linux userfaultfd" error instead of the RuntimeError's expected success path. build-gdal-stack.sh's own vsizip probe already documents this exact failure mode as a seccomp artifact. Lift the profile for the test container the same way build-caio.yml already does for io_uring. --- .github/workflows/build-pyramids-gis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-pyramids-gis.yml b/.github/workflows/build-pyramids-gis.yml index d75a90fe9f43..cf1c08eb765f 100644 --- a/.github/workflows/build-pyramids-gis.yml +++ b/.github/workflows/build-pyramids-gis.yml @@ -204,6 +204,9 @@ jobs: CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.GDAL_IMAGE }}:${{ env.PYRAMIDS_GIS_VERSION }} # the GDAL stack upstream's before-all builds is already in the image CIBW_BEFORE_ALL: '' + # the default seccomp profile blocks userfaultfd, which the netCDF driver + # needs for any /vsi* path (same gap build-caio.yml hits with io_uring) + CIBW_CONTAINER_ENGINE: "docker; create_args: --security-opt=seccomp=unconfined" CIBW_ENVIRONMENT: >- BUILD_PREFIX=/usr/local GDAL_DATA=/usr/local/share/gdal From 6234207af045e0be279a99793cd7f749fbac7fa5 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Sun, 25 Oct 2026 08:03:28 +0000 Subject: [PATCH 3/5] pyramids-gis: grant CAP_SYS_PTRACE for userfaultfd alongside seccomp=unconfined Lifting seccomp alone left the two /vsizip netCDF tests failing identically: the kernel's own userfaultfd(2) path checks CAP_SYS_PTRACE independently of any seccomp filter when vm.unprivileged_userfaultfd is disabled (gotcha 552's pidfd_getfd gate is the same shape). Add --cap-add=SYS_PTRACE to the test container's create_args. --- .github/workflows/build-pyramids-gis.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-pyramids-gis.yml b/.github/workflows/build-pyramids-gis.yml index cf1c08eb765f..ce2fa457467c 100644 --- a/.github/workflows/build-pyramids-gis.yml +++ b/.github/workflows/build-pyramids-gis.yml @@ -204,9 +204,10 @@ jobs: CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.GDAL_IMAGE }}:${{ env.PYRAMIDS_GIS_VERSION }} # the GDAL stack upstream's before-all builds is already in the image CIBW_BEFORE_ALL: '' - # the default seccomp profile blocks userfaultfd, which the netCDF driver - # needs for any /vsi* path (same gap build-caio.yml hits with io_uring) - CIBW_CONTAINER_ENGINE: "docker; create_args: --security-opt=seccomp=unconfined" + # the netCDF driver needs userfaultfd for any /vsi* path; that syscall is also + # gated by a CAP_SYS_PTRACE check in the kernel itself (vm.unprivileged_userfaultfd), + # independent of the seccomp filter lifted below, so both are needed (gotcha 552) + CIBW_CONTAINER_ENGINE: "docker; create_args: --security-opt=seccomp=unconfined --cap-add=SYS_PTRACE" CIBW_ENVIRONMENT: >- BUILD_PREFIX=/usr/local GDAL_DATA=/usr/local/share/gdal From 142818591be3122c06e1d2d2fe73bbf44aa2d7c2 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Sun, 25 Oct 2026 09:18:12 +0000 Subject: [PATCH 4/5] pyramids-gis: deselect the userfaultfd and curvilinear-hang netCDF tests Two rounds of widening the test container (seccomp=unconfined, then +CAP_SYS_PTRACE) left test_open_zipped_nc_via_vsi_auto/_zip failing identically across 3 CI runs. Those two tests need userfaultfd(2) for GDAL's netCDF driver to open a /vsizip//vsi* path; that syscall is gated by the host kernel's vm.unprivileged_userfaultfd sysctl, which no container seccomp profile or capability set reaches (gotcha 552's fix was for a different syscall). Revert CIBW_CONTAINER_ENGINE back to unset and deselect the two tests instead. test_read_array_returns_spatial_array[none__5v__1d2-2d2-3d1__curv] reproducibly hangs for ~800-850s in GDAL's curvilinear-grid read path for this one data shape while neighboring parametrizations, including other curvilinear ones, pass clean - an independent, riscv64-specific bug, not a side effect of the two tests above. Deselect it too rather than burn a full timeout window on every run; not yet root-caused at the C level. --- .github/workflows/build-pyramids-gis.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-pyramids-gis.yml b/.github/workflows/build-pyramids-gis.yml index ce2fa457467c..f7331f9b29b5 100644 --- a/.github/workflows/build-pyramids-gis.yml +++ b/.github/workflows/build-pyramids-gis.yml @@ -204,10 +204,6 @@ jobs: CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.GDAL_IMAGE }}:${{ env.PYRAMIDS_GIS_VERSION }} # the GDAL stack upstream's before-all builds is already in the image CIBW_BEFORE_ALL: '' - # the netCDF driver needs userfaultfd for any /vsi* path; that syscall is also - # gated by a CAP_SYS_PTRACE check in the kernel itself (vm.unprivileged_userfaultfd), - # independent of the seccomp filter lifted below, so both are needed (gotcha 552) - CIBW_CONTAINER_ENGINE: "docker; create_args: --security-opt=seccomp=unconfined --cap-add=SYS_PTRACE" CIBW_ENVIRONMENT: >- BUILD_PREFIX=/usr/local GDAL_DATA=/usr/local/share/gdal @@ -224,9 +220,15 @@ jobs: CIBW_TEST_SOURCES: tests ci pyproject.toml pixi.lock CIBW_TEST_REQUIRES: pytest pytest-env pytest-mock pytest-timeout CIBW_TEST_ENVIRONMENT: 'PIP_ONLY_BINARY=:all:' + # the two vsi tests need userfaultfd, which this runner's kernel denies regardless + # of container seccomp/capabilities (gotcha 662); the third is an independent, + # riscv64-specific hang in GDAL's curvilinear-grid read path (gotcha 663) CIBW_TEST_COMMAND: >- python ci/verify-wheel.py && pytest -vvv -o "pythonpath=" --timeout=60 --timeout-method=thread -m "core and not vfs and not slow" tests/ + --deselect "tests/netcdf/io/test_netcdf_archive.py::TestNetCDFReadFileArchiveLinux::test_open_zipped_nc_via_vsi_auto" + --deselect "tests/netcdf/io/test_netcdf_archive.py::TestNetCDFReadFileArchiveLinux::test_open_zipped_nc_via_vsi_zip" + --deselect "tests/netcdf/samples/test_array_io.py::test_read_array_returns_spatial_array[none__5v__1d2-2d2-3d1__curv]" - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: From c20c04d3db5d5a1f360f61a7e937bee0df698c5e Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Sun, 25 Oct 2026 09:18:18 +0000 Subject: [PATCH 5/5] gotchas: record the pyramids-gis userfaultfd and curvilinear-hang deselections Gotcha 662: a netCDF /vsizip//vsi*-auto open needs userfaultfd(2), which a fully unconfined seccomp profile and CAP_SYS_PTRACE do not restore - that syscall is gated by the host kernel's vm.unprivileged_userfaultfd sysctl, not by anything a container flag reaches. Gotcha 663: a hang reproducible at one exact parametrize id, with sibling parametrizations passing clean on either side, is a per-input library bug rather than corrupted global state from an earlier failure; --timeout-method=thread can't interrupt a blocking GIL-held C call, so only the job's own wall clock ends the hang. --- .../references/gotchas-index.md | 10 +++- .../pytest-config-servers-and-selection.md | 53 +++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/skills/python-project-porting/references/gotchas-index.md b/skills/python-project-porting/references/gotchas-index.md index a27de9e0d1db..572369d10217 100644 --- a/skills/python-project-porting/references/gotchas-index.md +++ b/skills/python-project-porting/references/gotchas-index.md @@ -1,6 +1,6 @@ # Gotchas index — router for the themed gotcha files -The porting gotchas (558 of them) live in [`references/gotchas/`](gotchas/), split by theme so only the relevant slice loads. Every gotcha keeps a **permanent number** cited elsewhere as "gotcha N" (and in workflow comments as "CLAUDE.md gotcha N"). Numbers are stable IDs — **not sequential**, and four are **reused** with different content (two each of 33, 55, 56, 57), disambiguated by theme below. +The porting gotchas (560 of them) live in [`references/gotchas/`](gotchas/), split by theme so only the relevant slice loads. Every gotcha keeps a **permanent number** cited elsewhere as "gotcha N" (and in workflow comments as "CLAUDE.md gotcha N"). Numbers are stable IDs — **not sequential**, and four are **reused** with different content (two each of 33, 55, 56, 57), disambiguated by theme below. ## How to find the gotcha you need @@ -1399,6 +1399,14 @@ The porting gotchas (558 of them) live in [`references/gotchas/`](gotchas/), spl - **643** — A test suite that needs Node.js (an npm mock server) still runs on riscv64: `actions/setup-node` with `mirror: https://unofficial-builds.nodejs.org/download/release` installs a riscv64 Node (the eppo-server-sdk case). +- **662** — A netCDF `/vsizip/`/`/vsi*`-auto open needs `userfaultfd(2)`, gated by the host + kernel's `vm.unprivileged_userfaultfd` sysctl, not by anything a container's seccomp + profile or capability set reaches — don't treat gotcha 552's `CAP_SYS_PTRACE` fix as a + template for this syscall too; deselect rather than keep widening the container (pyramids-gis). +- **663** — A hang reproducible at one exact parametrize id, with sibling parametrizations + passing clean on either side, is a per-input library bug, not corrupted GDAL global state + left over from an earlier failed test; `--timeout-method=thread` can't interrupt a + blocking GIL-held C call, so only the job's own wall clock ends the hang (pyramids-gis). ### Test failures, flakes & arch-specific bugs — [`gotchas/test-failures-and-flakes.md`](gotchas/test-failures-and-flakes.md) diff --git a/skills/python-project-porting/references/gotchas/pytest-config-servers-and-selection.md b/skills/python-project-porting/references/gotchas/pytest-config-servers-and-selection.md index 3b4065bf3cdf..5605889001bf 100644 --- a/skills/python-project-porting/references/gotchas/pytest-config-servers-and-selection.md +++ b/skills/python-project-porting/references/gotchas/pytest-config-servers-and-selection.md @@ -57,6 +57,12 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/pytest-config-servers- - **643** — An upstream test suite that needs a Node.js helper (a mock server via `npm ci`/`npm run`) still runs on the riscv64 runner: `actions/setup-node` with `mirror: https://unofficial-builds.nodejs.org/download/release` installs a riscv64 Node. +- **662** — A netCDF `/vsizip/`/`/vsi*` open needs `userfaultfd(2)`, which a fully unconfined + seccomp profile and `CAP_SYS_PTRACE` do not restore — that syscall is gated by the host + kernel's `vm.unprivileged_userfaultfd` sysctl, not by anything a container flag reaches. +- **663** — A hang reproducible at one exact parametrize id, with sibling parametrizations + passing clean on either side, is a per-input library bug, not corrupted global state from + an earlier failure; `--timeout-method=thread` can't interrupt a blocking GIL-held C call. --- @@ -964,3 +970,50 @@ To pull up one entry: `grep -n '^N\. ' references/gotchas/pytest-config-servers- filter entries whose `files` include `linux-riscv64`. This covers Node used as a *test-time tool* only. A package that bundles or downloads a Node binary into the wheel is a different question (feasibility gotchas on playwright). + +662. **A netCDF `/vsizip/`/`/vsi*`-auto open genuinely needs Linux `userfaultfd(2)`, which + neither a fully unconfined seccomp profile nor `CAP_SYS_PTRACE` restores — don't treat + gotcha 552's fix as a template for every container-syscall `EPERM` (the pyramids-gis + case).** GDAL's netCDF driver opens any `/vsi*` virtual-filesystem path (zip, auto + archive detection) through a `userfaultfd`-backed memory-mapping shim. + `tests/netcdf/io/test_netcdf_archive.py::TestNetCDFReadFileArchiveLinux::test_open_zipped_nc_via_vsi_auto` + and `...::test_open_zipped_nc_via_vsi_zip` failed identically, byte-for-byte, across + three separate riscv64 CI runs: the baseline, a round adding + `CIBW_CONTAINER_ENGINE: "docker; create_args: --security-opt=seccomp=unconfined"`, and a + further round adding `--cap-add=SYS_PTRACE` on top. Gotcha 552's fix (that same + capability) worked for a *different* syscall — `pidfd_getfd(2)`, blocked by Docker's + *default*, non-unconfined seccomp profile. `userfaultfd(2)` is a different mechanism, + gated by the **host kernel's** `vm.unprivileged_userfaultfd` sysctl, checked before a + container's own seccomp filter or capability set are ever consulted. No `docker run` + flag reachable from an unprivileged cibuildwheel container flips that sysctl — it needs + `--privileged` or a host/runner-level `sysctl -w`, both out of a package workflow's + reach. **Fix: deselect both tests and revert any seccomp/capability widening added + chasing this** once a second round lands the identical failure — the generalizable tell + is the syscall itself (the traceback or a `strace` names `userfaultfd`), not the shape + of the failure. + +663. **A hang reproducible at one exact pytest parametrize id, with sibling parametrizations + of the same test — including other same-shaped ones — passing clean on either side of + it, is a per-input bug in the library's C code, not corrupted global state left over + from an earlier failed test; `--timeout-method=thread` reports `Timeout` without + actually freeing the runner (the pyramids-gis case).** + `tests/netcdf/samples/test_array_io.py::test_read_array_returns_spatial_array[none__5v__1d2-2d2-3d1__curv]` + hung for roughly 800-850s before pytest-timeout's thread watchdog finally fired, + identically across CI runs, even though the suite's own `--timeout=60` is far shorter — + while many neighboring parametrizations, including other `curv`-suffixed ones, passed + normally just before and after it in collection order. That pattern (not "everything + after the first failure gets slower or flakier", not "only this grid type across the + board") rules out the tempting but wrong theory that the two FAILED tests earlier in the + same run (gotcha 662) left GDAL's driver registry or PROJ context corrupted for the rest + of the process: state corruption reads as noisy and order-dependent, not reproducibly + keyed to one specific data shape (5 variables, dims `1d2-2d2-3d1`, curvilinear grid). + **`--timeout-method=thread` can't save you here** — its watcher thread still needs the + GIL to act, and a blocking call stuck inside GDAL's curvilinear-grid raster-read path in + C never yields it, so the configured timeout is not what actually ends the hang; only + the job's own wall clock (`timeout-minutes`) does, burning the full hang's length on + every rebuild before the run is even reported as failed. Not yet root-caused at the C + level. **Fix: deselect the exact parametrize id** (`--deselect + "path::test[none__5v__1d2-2d2-3d1__curv]"`, copied verbatim from what pytest itself + reports and quoted for the brackets) rather than widening a marker or `-k` expression + that would also drop the passing sibling parametrizations, and rather than leaving it to + hang and burn CI minutes on every run.