diff --git a/.github/workflows/big-endian.yml b/.github/workflows/big-endian.yml index 0119f04ffa..7bf78eacf5 100644 --- a/.github/workflows/big-endian.yml +++ b/.github/workflows/big-endian.yml @@ -57,7 +57,10 @@ jobs: run: | docker run --rm --platform linux/s390x -v "$PWD":/src:ro zarr-big-endian bash -euc ' cp -r /src /work && cd /work - pip install -q --no-deps --no-build-isolation . + # Resolve dependencies against pyproject.toml rather than trusting whatever the + # image provides, then fail if any installed package falls outside the constraints. + pip install -q --no-build-isolation . + pip check python -c "import sys; assert sys.byteorder == \"big\", sys.byteorder" python -m pytest -p no:cacheprovider --hypothesis-profile ci -n auto \ tests/test_dtype tests/test_dtype_registry.py \ diff --git a/changes/4547.misc.md b/changes/4547.misc.md new file mode 100644 index 0000000000..f98644f479 --- /dev/null +++ b/changes/4547.misc.md @@ -0,0 +1 @@ +The big-endian (s390x) CI job now installs zarr with its dependencies resolved against `pyproject.toml` and runs `pip check`, instead of trusting whatever versions the Debian-based test image provides. The image builds a numcodecs that satisfies zarr's constraints from source where Debian's package does not. diff --git a/ci/big-endian/Dockerfile b/ci/big-endian/Dockerfile index 973ce698be..8c70535fc7 100644 --- a/ci/big-endian/Dockerfile +++ b/ci/big-endian/Dockerfile @@ -1,11 +1,20 @@ # Test image for a big-endian host (s390x, run under QEMU by .github/workflows/big-endian.yml). -# Debian packages supply numpy and numcodecs, so nothing heavy compiles under emulation; -# this is also the environment Debian tests zarr in. Debian does not package Google's -# crc32c library, so it is built here for the google-crc32c C extension. +# +# Debian packages are used as a source of prebuilt binaries for s390x, which PyPI does not +# have, so that numpy and friends do not compile under emulation. They do not define the +# test environment: zarr's pyproject.toml does. The workflow installs zarr with its +# dependencies resolved and runs `pip check`, so a Debian package that falls outside the +# declared constraints fails the job at install time instead of through unrelated-looking +# test failures. Anything Debian cannot satisfy is built here from source so the compile is +# cached as an image layer rather than repeated in every job. +# +# Debian does not package Google's crc32c library, so it is built here for the +# google-crc32c C extension. FROM debian:sid RUN apt-get update -qq && apt-get install -y -qq --no-install-recommends \ - build-essential cmake git python3 python3-dev python3-pip python3-venv \ + build-essential cmake cython3 git meson ninja-build pkgconf python3 python3-dev \ + python3-mesonpy python3-pip python3-setuptools-scm python3-venv \ python3-crc32c python3-donfig python3-fsspec python3-hatch-vcs python3-hatchling \ python3-hypothesis python3-numcodecs python3-numpy python3-numpydoc python3-packaging \ python3-pytest python3-pytest-asyncio python3-pytest-benchmark python3-pytest-xdist \ @@ -24,3 +33,9 @@ RUN git clone -q --depth 1 --branch 1.1.2 https://github.com/google/crc32c /tmp/ RUN python3 -m venv --system-site-packages /venv ENV PATH=/venv/bin:$PATH RUN pip install -q --no-binary google-crc32c google-crc32c msgspec pytest-accept +# Debian sid ships numcodecs 0.17.0, which zarr excludes (see pyproject.toml; GHSA-jqc8-qjfv-5m6r), +# and PyPI has no s390x wheels, so build a satisfying numcodecs from source into the venv, where +# it shadows the Debian package. Built without isolation so the Debian meson, Cython and numpy +# are used instead of compiling a build-time numpy under emulation. Remove once Debian ships a +# version that satisfies zarr's constraint; `pip check` in the workflow will say when it does not. +RUN pip install -q --no-binary numcodecs --no-build-isolation "numcodecs>=0.16.6,!=0.17.0"