From bc75322391428cf20b45e1b8e046d568457aaecd Mon Sep 17 00:00:00 2001 From: AN Long Date: Tue, 2 Jun 2026 21:20:15 +0900 Subject: [PATCH 1/2] Fix incomplete Python dev requirements Closes #8082 --- .github/workflows/paimon-python-checks.yml | 28 +++++++++------------- paimon-python/dev/requirements-dev.txt | 15 ++++++++---- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/.github/workflows/paimon-python-checks.yml b/.github/workflows/paimon-python-checks.yml index f27219e6facb..1aaf8d0d0f91 100755 --- a/.github/workflows/paimon-python-checks.yml +++ b/.github/workflows/paimon-python-checks.yml @@ -127,26 +127,27 @@ jobs: df -h if [[ "${{ matrix.python-version }}" == "3.6.15" ]]; then python -m pip install --upgrade pip==21.3.1 - python --version - python -m pip install --no-cache-dir pyroaring readerwriterlock==1.0.9 'fsspec==2021.10.1' 'cachetools==4.2.4' 'ossfs==2021.8.0' pyarrow==6.0.1 pandas==1.1.5 'polars==0.9.12' 'fastavro==1.4.7' zstandard==0.19.0 dataclasses==0.8.0 flake8 pytest py4j==0.10.9.9 requests parameterized==0.8.1 2>&1 >/dev/null - python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}' -i https://pypi.org/simple/ else python -m pip install --upgrade pip - pip install torch --index-url https://download.pytorch.org/whl/cpu - python -m pip install pyroaring readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 ray==2.54.0 fastavro==1.11.1 pyarrow==16.0.0 zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2 numpy==1.24.3 pandas==2.0.3 pylance==0.39.0 cramjam flake8==4.0.1 pytest~=7.0 py4j==0.10.9.9 requests parameterized==0.9.0 'daft>=0.7.6' pypaimon-rust==0.2.0 'datafusion>=52' - python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}' -i https://pypi.org/simple/ + fi + python --version + python -m pip install --no-cache-dir -r paimon-python/dev/requirements-dev.txt + if [[ "${{ matrix.python-version }}" == "3.6.15" ]]; then + python -m pip install --no-cache-dir ossfs==2021.8.0 + else + python -m pip install --no-cache-dir 'daft>=0.7.6' ossfs==2023.12.0 pylance==0.39.0 maturin[patchelf] if python -c "import sys; sys.exit(0 if sys.version_info >= (3, 11) else 1)"; then - python -m pip install vortex-data==0.70.0 + python -m pip install --no-cache-dir vortex-data==0.70.0 fi python -m pip install 'paimon-mosaic>=0.1.0' fi + python -m pip install --no-cache-dir 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}' -i https://pypi.org/simple/ df -h - name: Build and install tantivy-py from source if: matrix.python-version != '3.6.15' shell: bash run: | - pip install maturin[patchelf] git clone -b support_directory https://github.com/JingsongLi/tantivy-py.git /tmp/tantivy-py cd /tmp/tantivy-py maturin build --release @@ -184,9 +185,8 @@ jobs: shell: bash run: | python -m pip install --upgrade pip + python -m pip install --no-cache-dir -r paimon-python/dev/requirements-dev.txt pip install torch --index-url https://download.pytorch.org/whl/cpu - python -m pip install pyroaring readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 ray==2.54.0 fastavro==1.11.1 pyarrow==16.0.0 zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2 numpy==1.24.3 pandas==2.0.3 pylance==0.39.0 flake8==4.0.1 pytest~=7.0 py4j==0.10.9.9 requests parameterized==0.9.0 - python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}' -i https://pypi.org/simple/ - name: Run lint-python.sh shell: bash run: | @@ -279,13 +279,7 @@ jobs: - name: Install base Python dependencies run: | python -m pip install --upgrade pip - pip install torch --index-url https://download.pytorch.org/whl/cpu - python -m pip install --no-cache-dir \ - pyroaring readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 \ - fastavro==1.11.1 pyarrow==16.0.0 zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2 \ - numpy==1.24.3 pandas==2.0.3 cramjam pytest~=7.0 py4j==0.10.9.9 requests \ - parameterized==0.9.0 packaging - python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}' -i https://pypi.org/simple/ + python -m pip install --no-cache-dir -r paimon-python/dev/requirements-dev.txt - name: Test Ray version compatibility run: | cd paimon-python diff --git a/paimon-python/dev/requirements-dev.txt b/paimon-python/dev/requirements-dev.txt index c83a2e44b8b6..4b4e3573ddd9 100644 --- a/paimon-python/dev/requirements-dev.txt +++ b/paimon-python/dev/requirements-dev.txt @@ -15,17 +15,22 @@ # See the License for the specific language governing permissions and # limitations under the License. ################################################################################ -# Core dependencies for pypaimon are in requirements.txt +# Core dependencies for pypaimon +-r requirements.txt # Test dependencies for pypaimon are as follows -duckdb==1.3.2 +duckdb==1.3.2; python_version >= "3.8" flake8==4.0.1 -pytest~=7.0 +parameterized==0.8.1; python_version < "3.7" +parameterized==0.9.0; python_version >= "3.7" +py4j==0.10.9.9 +pypaimon-rust==0.2.0; python_version >= "3.10" +pytest<7; python_version < "3.7" +pytest~=7.0; python_version >= "3.7" # merge_into needs Dataset.join (added in Ray 2.50). Python 3.8 has no 2.50 wheel. -ray>=2.10.0; python_version < "3.9" +ray>=2.10.0; python_version >= "3.8" and python_version < "3.9" ray>=2.50.0; python_version >= "3.9" requests -parameterized # Vortex 0.71.0 regresses native predicate pushdown on single-row files. vortex-data==0.70.0; python_version >= "3.11" # merge_into condition expressions (optional, for condition tests) From eebbd44b0c1bb9a56c39d72785a40a387593d4c1 Mon Sep 17 00:00:00 2001 From: AN Long Date: Wed, 3 Jun 2026 00:34:25 +0900 Subject: [PATCH 2/2] Fix pyroaring requirement for BitMap64 --- paimon-python/dev/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paimon-python/dev/requirements.txt b/paimon-python/dev/requirements.txt index 49cdfef0340f..08165b08bd8e 100644 --- a/paimon-python/dev/requirements.txt +++ b/paimon-python/dev/requirements.txt @@ -30,8 +30,8 @@ polars>=1,<2; python_version=="3.8" polars>=1.32,<2; python_version>="3.9" pyarrow>=6,<7; python_version < "3.8" pyarrow>=16,<20; python_version >= "3.8" -pyroaring<=0.3.3; python_version < "3.7" -pyroaring<=0.4.5; python_version == "3.7" +# BitMap64 is required by RoaringBitmap64 and is available since pyroaring 1.0.0. +pyroaring==1.0.0; python_version < "3.8" pyroaring>=1.0.0; python_version >= "3.8" readerwriterlock>=1,<2 requests>=2.21.0,<3