Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
288 changes: 288 additions & 0 deletions .github/workflows/build-pyramids-gis.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,288 @@
# SPDX-FileCopyrightText: 2026 The RISE Project
# SPDX-License-Identifier: MIT
---
# This workflow is based on the `build-linux-wheels`/`test-wheels` jobs of
# https://github.com/serapeum-org/pyramids/blob/0.59.0/.github/workflows/bundle-pypi-wheels.yml
name: Build pyramids-gis wheels (riscv64)

on:
workflow_dispatch:
inputs:
version:
description: 'Version glob to (re)build; empty builds every version of docs/packages/pyramids-gis.yaml not released yet'
required: false
default: ''
pull_request:
branches: [main]
paths:
- '.github/workflows/build-pyramids-gis.yml'
- 'docs/packages/pyramids-gis.yaml'
- 'patches/pyramids-gis/**'
push:
branches: [main]
paths:
- '.github/workflows/build-pyramids-gis.yml'
- 'docs/packages/pyramids-gis.yaml'
- 'patches/pyramids-gis/**'

concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true

permissions:
contents: read # to fetch code (actions/checkout)

env:
BASEIMAGE: quay.io/pypa/manylinux_2_39_riscv64
GDAL_IMAGE: manylinux_2_39_riscv64_gdal_pyramids_gis

jobs:
setup:
uses: $/.github/workflows/_setup.yml
with:
package: pyramids-gis
version: ${{ inputs.version }}

build_image:
needs: [setup]
if: needs.setup.outputs.versions != '[]'
strategy:
fail-fast: false
matrix:
version: ${{ fromJSON(needs.setup.outputs.versions) }}
name: Build the GDAL build image
runs-on: ubuntu-24.04-riscv
timeout-minutes: 1440

env:
PYRAMIDS_GIS_VERSION: ${{ matrix.version }}

steps:
- name: Checkout pyramids ${{ env.PYRAMIDS_GIS_VERSION }}
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: serapeum-org/pyramids
ref: ${{ env.PYRAMIDS_GIS_VERSION }}
persist-credentials: false

- name: Checkout python-wheels
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
path: python-wheels
persist-credentials: false

- name: Patch pyramids source
run: git apply python-wheels/patches/pyramids-gis/${{ env.PYRAMIDS_GIS_VERSION }}/*.patch

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
with:
install: true
buildkitd-flags: --debug

- name: Write the GDAL image Dockerfile
run: |
mkdir -p "$RUNNER_TEMP/gdal-ctx"
cat > "$RUNNER_TEMP/gdal-ctx/Dockerfile" <<'DOCKER_EOF'
# SPDX-FileCopyrightText: 2026 The RISE Project
# SPDX-License-Identifier: MIT
#
# upstream's before-all (ci/source-build/build-gdal-stack.sh) compiles ~26
# libraries from source; run it once into an image rather than once per
# interpreter, with upstream's [tool.cibuildwheel.linux.environment].
ARG BASEIMAGE=quay.io/pypa/manylinux_2_39_riscv64:latest
FROM ${BASEIMAGE}

ENV BUILD_PREFIX=/usr/local \
GDAL_DATA=/usr/local/share/gdal \
PROJ_DATA=/usr/local/share/proj \
PROJ_LIB=/usr/local/share/proj \
LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64

COPY . /opt/pyramids/ci/source-build/
RUN bash /opt/pyramids/ci/source-build/build-gdal-stack.sh \
&& rm -rf /opt/pyramids /tmp/gdal-src-build /tmp/probe.*
DOCKER_EOF

- name: Build Docker image with GDAL
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
with:
context: ci/source-build
file: ${{ runner.temp }}/gdal-ctx/Dockerfile
platforms: linux/riscv64
build-args: |
BASEIMAGE=${{ env.BASEIMAGE }}
push: false
cache-from: type=gha,scope=${{ env.GDAL_IMAGE }}-${{ env.PYRAMIDS_GIS_VERSION }}
cache-to: type=gha,mode=max,scope=${{ env.GDAL_IMAGE }}-${{ env.PYRAMIDS_GIS_VERSION }}
env:
BUILDKIT_PROGRESS: plain

build_wheels:
name: Build pyramids-gis ${{ matrix.version }} ${{ matrix.python }}-manylinux_riscv64
needs: [setup, build_image]
if: needs.setup.outputs.versions != '[]'
runs-on: ubuntu-24.04-riscv
timeout-minutes: 600
strategy:
fail-fast: false
matrix:
version: ${{ fromJSON(needs.setup.outputs.versions) }}
python: ["cp312", "cp313", "cp314"]

env:
PYRAMIDS_GIS_VERSION: ${{ matrix.version }}

steps:
- name: Checkout pyramids ${{ env.PYRAMIDS_GIS_VERSION }}
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: serapeum-org/pyramids
ref: ${{ env.PYRAMIDS_GIS_VERSION }}
persist-credentials: false

- name: Checkout python-wheels
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
path: python-wheels
persist-credentials: false

- name: Patch pyramids source
run: git apply python-wheels/patches/pyramids-gis/${{ env.PYRAMIDS_GIS_VERSION }}/*.patch

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
with:
install: true
buildkitd-flags: --debug

- name: Write the GDAL image Dockerfile
run: |
mkdir -p "$RUNNER_TEMP/gdal-ctx"
cat > "$RUNNER_TEMP/gdal-ctx/Dockerfile" <<'DOCKER_EOF'
# SPDX-FileCopyrightText: 2026 The RISE Project
# SPDX-License-Identifier: MIT
#
# upstream's before-all (ci/source-build/build-gdal-stack.sh) compiles ~26
# libraries from source; run it once into an image rather than once per
# interpreter, with upstream's [tool.cibuildwheel.linux.environment].
ARG BASEIMAGE=quay.io/pypa/manylinux_2_39_riscv64:latest
FROM ${BASEIMAGE}

ENV BUILD_PREFIX=/usr/local \
GDAL_DATA=/usr/local/share/gdal \
PROJ_DATA=/usr/local/share/proj \
PROJ_LIB=/usr/local/share/proj \
LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64

COPY . /opt/pyramids/ci/source-build/
RUN bash /opt/pyramids/ci/source-build/build-gdal-stack.sh \
&& rm -rf /opt/pyramids /tmp/gdal-src-build /tmp/probe.*
DOCKER_EOF

- name: Load the GDAL build image from the cache
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
with:
context: ci/source-build
file: ${{ runner.temp }}/gdal-ctx/Dockerfile
platforms: linux/riscv64
tags: ${{ env.GDAL_IMAGE }}:${{ env.PYRAMIDS_GIS_VERSION }}
build-args: |
BASEIMAGE=${{ env.BASEIMAGE }}
push: false
load: true
cache-from: type=gha,scope=${{ env.GDAL_IMAGE }}-${{ env.PYRAMIDS_GIS_VERSION }}
env:
BUILDKIT_PROGRESS: plain

- name: Build wheels
uses: pypa/cibuildwheel@1828c10ab37f080699c7b81cea34097c684a7074 # v4.2.0
with:
output-dir: wheelhouse/
only: ${{ matrix.python }}-manylinux_riscv64
env:
CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.GDAL_IMAGE }}:${{ env.PYRAMIDS_GIS_VERSION }}
# the GDAL stack upstream's before-all builds is already in the image
CIBW_BEFORE_ALL: ''
CIBW_ENVIRONMENT: >-
BUILD_PREFIX=/usr/local
GDAL_DATA=/usr/local/share/gdal
PROJ_DATA=/usr/local/share/proj
PROJ_LIB=/usr/local/share/proj
PACKAGE_DATA=1
LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64
PIP_EXTRA_INDEX_URL=https://pypi.riseproject.dev/simple/
# upstream pins --plat manylinux_2_28, below the riscv64 image's glibc floor
CIBW_REPAIR_WHEEL_COMMAND: auditwheel repair -w {dest_dir} {wheel}
# upstream skips cibuildwheel's test step and runs the same commands in its
# separate test-wheels job, which needs riscv64 hardware here anyway
CIBW_TEST_SKIP: ''
CIBW_TEST_SOURCES: tests ci pyproject.toml pixi.lock
CIBW_TEST_REQUIRES: pytest pytest-env pytest-mock pytest-timeout
CIBW_TEST_ENVIRONMENT: 'PIP_ONLY_BINARY=:all:'
# the two vsi tests need userfaultfd, which this runner's kernel denies regardless
# of container seccomp/capabilities (gotcha 662); the third is an independent,
# riscv64-specific hang in GDAL's curvilinear-grid read path (gotcha 663)
CIBW_TEST_COMMAND: >-
python ci/verify-wheel.py &&
pytest -vvv -o "pythonpath=" --timeout=60 --timeout-method=thread -m "core and not vfs and not slow" tests/
--deselect "tests/netcdf/io/test_netcdf_archive.py::TestNetCDFReadFileArchiveLinux::test_open_zipped_nc_via_vsi_auto"
--deselect "tests/netcdf/io/test_netcdf_archive.py::TestNetCDFReadFileArchiveLinux::test_open_zipped_nc_via_vsi_zip"
--deselect "tests/netcdf/samples/test_array_io.py::test_read_array_returns_spatial_array[none__5v__1d2-2d2-3d1__curv]"

- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: pyramids-gis-${{ env.PYRAMIDS_GIS_VERSION }}-${{ matrix.python }}-manylinux_riscv64
path: wheelhouse/*.whl
if-no-files-found: error

gpl_sources:
needs: [setup]
if: needs.setup.outputs.versions != '[]'
strategy:
fail-fast: false
matrix:
version: ${{ fromJSON(needs.setup.outputs.versions) }}
name: Collect GPL sources
runs-on: ubuntu-24.04-riscv

env:
PYRAMIDS_GIS_VERSION: ${{ matrix.version }}

steps:
- name: Checkout python-wheels
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false

# libgdal links OpenMP, so auditwheel vendors the image's libgomp.
- uses: ./actions/collect-gpl-sources
with:
image: ${{ env.BASEIMAGE }}
packages: gcc
output: gpl-sources.tar

- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: pyramids-gis-${{ env.PYRAMIDS_GIS_VERSION }}-gpl-sources
path: gpl-sources.tar
if-no-files-found: error

publish:
name: Publish pyramids-gis ${{ matrix.version }}
needs: [setup, build_wheels, gpl_sources]
if: needs.setup.outputs.versions != '[]'
strategy:
fail-fast: false
matrix:
version: ${{ fromJSON(needs.setup.outputs.versions) }}
permissions:
contents: write
pull-requests: write
uses: $/.github/workflows/_publish-wheel.yml
secrets:
app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }}
with:
artifact-pattern: pyramids-gis-${{ matrix.version }}-*-manylinux_riscv64
gpl-sources-artifact: pyramids-gis-${{ matrix.version }}-gpl-sources
gpl-sources-description: gcc
5 changes: 5 additions & 0 deletions docs/packages/pyramids-gis.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
package-name: pyramids-gis
source-code: https://github.com/serapeum-org/pyramids
license: GPL-3.0-only
versions:
- version: 0.59.0
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Ludovic Henry <git@ludovic.dev>
Date: Wed, 14 Oct 2026 18:00:00 +0000
Subject: [PATCH] source-build: add the riscv64 OpenSSL target

ci/source-build/config.sh maps `uname -m` to the target name OpenSSL's
Configure expects and aborts on anything other than x86_64/aarch64, so
the from-source GDAL stack (and with it the Linux wheel) cannot be built
on riscv64 at all. OpenSSL has shipped a linux64-riscv64 target since
3.0; map riscv64 to it.

Upstream-Status: To upstream [one-line arch mapping, applies unchanged to upstream main]
---
ci/source-build/config.sh | 1 +
1 file changed, 1 insertion(+)

diff --git a/ci/source-build/config.sh b/ci/source-build/config.sh
index 8cad992fe..98ad64a00 100644
--- a/ci/source-build/config.sh
+++ b/ci/source-build/config.sh
@@ -42,6 +42,7 @@ ARCH="$(uname -m)"
case "${ARCH}" in
x86_64) OPENSSL_TARGET="linux-x86_64" ;;
aarch64) OPENSSL_TARGET="linux-aarch64" ;;
+ riscv64) OPENSSL_TARGET="linux64-riscv64" ;;
*) echo "ERROR: unsupported architecture ${ARCH}" >&2; exit 1 ;;
esac

--
2.43.0

10 changes: 9 additions & 1 deletion skills/python-project-porting/references/gotchas-index.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Gotchas index — router for the themed gotcha files

The porting gotchas (559 of them) live in [`references/gotchas/`](gotchas/), split by theme so only the relevant slice loads. Every gotcha keeps a **permanent number** cited elsewhere as "gotcha N" (and in workflow comments as "CLAUDE.md gotcha N"). Numbers are stable IDs — **not sequential**, and four are **reused** with different content (two each of 33, 55, 56, 57), disambiguated by theme below.
The porting gotchas (561 of them) live in [`references/gotchas/`](gotchas/), split by theme so only the relevant slice loads. Every gotcha keeps a **permanent number** cited elsewhere as "gotcha N" (and in workflow comments as "CLAUDE.md gotcha N"). Numbers are stable IDs — **not sequential**, and four are **reused** with different content (two each of 33, 55, 56, 57), disambiguated by theme below.

## How to find the gotcha you need

Expand Down Expand Up @@ -1403,6 +1403,14 @@ The porting gotchas (559 of them) live in [`references/gotchas/`](gotchas/), spl
- **643** — A test suite that needs Node.js (an npm mock server) still runs on riscv64:
`actions/setup-node` with `mirror: https://unofficial-builds.nodejs.org/download/release`
installs a riscv64 Node (the eppo-server-sdk case).
- **662** — A netCDF `/vsizip/`/`/vsi*`-auto open needs `userfaultfd(2)`, gated by the host
kernel's `vm.unprivileged_userfaultfd` sysctl, not by anything a container's seccomp
profile or capability set reaches — don't treat gotcha 552's `CAP_SYS_PTRACE` fix as a
template for this syscall too; deselect rather than keep widening the container (pyramids-gis).
- **663** — A hang reproducible at one exact parametrize id, with sibling parametrizations
passing clean on either side, is a per-input library bug, not corrupted GDAL global state
left over from an earlier failed test; `--timeout-method=thread` can't interrupt a
blocking GIL-held C call, so only the job's own wall clock ends the hang (pyramids-gis).

### Test failures, flakes & arch-specific bugs — [`gotchas/test-failures-and-flakes.md`](gotchas/test-failures-and-flakes.md)

Expand Down
Loading
Loading