From 25722bbb38194e110923fa8e8d47b26d26e00e23 Mon Sep 17 00:00:00 2001 From: Orest Tokovenko Date: Fri, 30 Oct 2026 10:19:38 -0700 Subject: [PATCH 1/8] dev: move every check into one noxfile The Makefile, the pre-commit config, and the CI workflows each described the same checks in their own way, and tool versions were pinned in two of them. One noxfile now holds the sessions, the justfile names them, and CI runs the same sessions a developer runs locally. --- .github/workflows/python-ci-docs.yml | 2 +- .github/workflows/python-ci.yml | 37 +- .github/workflows/python-integration.yml | 18 +- .github/workflows/python-release-docs.yml | 2 +- .gitignore | 8 + .pre-commit-config.yaml | 64 --- AGENTS.md | 24 +- MANIFEST.in | 4 +- Makefile | 203 ---------- dev/release/verify_rc.sh | 9 +- justfile | 141 +++++++ mkdocs/README.md | 2 +- mkdocs/docs/api.md | 2 +- mkdocs/docs/community.md | 6 +- mkdocs/docs/configuration.md | 2 +- mkdocs/docs/contributing.md | 73 ++-- mkdocs/docs/expression-dsl.md | 4 +- mkdocs/docs/verify-release.md | 4 +- notebooks/spark_integration_example.ipynb | 4 +- noxfile.py | 451 ++++++++++++++++++++++ pyproject.toml | 37 +- tests/test_version.py | 2 +- uv.lock | 362 ++++++++++++++++- 23 files changed, 1082 insertions(+), 379 deletions(-) delete mode 100644 .pre-commit-config.yaml delete mode 100644 Makefile create mode 100644 justfile create mode 100644 noxfile.py diff --git a/.github/workflows/python-ci-docs.yml b/.github/workflows/python-ci-docs.yml index 65c15cd51a..358d42cb74 100644 --- a/.github/workflows/python-ci-docs.yml +++ b/.github/workflows/python-ci-docs.yml @@ -45,4 +45,4 @@ jobs: - name: Install UV uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 - name: Build docs - run: make docs-build + run: uvx --from rust-just just docs-build diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 1abd4c8600..401dae881d 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -33,20 +33,12 @@ concurrency: jobs: lint-and-unit-test: runs-on: ubuntu-24.04 - timeout-minutes: 10 - strategy: - max-parallel: 15 - fail-fast: true - matrix: - python: ['3.11', '3.12', '3.13', '3.14'] + timeout-minutes: 25 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: ${{ matrix.python }} - name: Install UV uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 with: @@ -54,41 +46,36 @@ jobs: - name: Install system dependencies run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos - name: Check uv.lock is up to date - if: matrix.python == '3.11' run: uv lock --check - name: Install - run: make install + run: uv run nox -s install - name: Run linters - if: matrix.python == '3.11' - run: make lint + run: uv run nox + # The Python version matrix lives in noxfile.py, so nox runs every supported + # version here instead of a job matrix. --parallel runs them concurrently. - name: Run unit tests with coverage - run: COVERAGE=1 make test + env: + COVERAGE: "1" + run: uv run nox -s tests -j auto - name: Generate coverage report (85%) # Coverage threshold should only increase over time — never decrease it! - run: COVERAGE_FAIL_UNDER=85 make coverage-report + run: uv run nox -s coverage windows-unit-test: runs-on: windows-latest - timeout-minutes: 15 - strategy: - fail-fast: true - matrix: - python: ['3.12'] + timeout-minutes: 20 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: ${{ matrix.python }} - name: Install UV uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 with: enable-cache: true - name: Install - run: uv sync --all-extras --no-extra hive-kerberos + run: uv run nox -s install - name: Run unit tests - run: uv run python -m pytest tests/ -m unit --ignore=tests/integration -v -x + run: uv run nox -s tests -p 3.12 cibw-dev-env-smoke-test: runs-on: ubuntu-slim diff --git a/.github/workflows/python-integration.yml b/.github/workflows/python-integration.yml index 42f76fd035..4a1d526833 100644 --- a/.github/workflows/python-integration.yml +++ b/.github/workflows/python-integration.yml @@ -48,9 +48,9 @@ jobs: - name: Install system dependencies run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos - name: Install - run: make install + run: uvx --from rust-just just install - name: Run integration tests with coverage - run: COVERAGE=1 make test-integration + run: COVERAGE=1 uvx --from rust-just just test-integration - name: Show debug logs if: ${{ failure() }} run: docker compose -f dev/docker-compose-integration.yml logs @@ -78,9 +78,9 @@ jobs: - name: Install system dependencies run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos - name: Install - run: make install + run: uvx --from rust-just just install - name: Run s3 integration tests with coverage - run: COVERAGE=1 make test-s3 + run: COVERAGE=1 uvx --from rust-just just test-s3 - name: Show debug logs if: ${{ failure() }} run: docker compose -f dev/docker-compose.yml logs @@ -108,9 +108,9 @@ jobs: - name: Install system dependencies run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos - name: Install - run: make install + run: uvx --from rust-just just install - name: Run adls integration tests with coverage - run: COVERAGE=1 make test-adls + run: COVERAGE=1 uvx --from rust-just just test-adls - name: Show debug logs if: ${{ failure() }} run: docker compose -f dev/docker-compose-azurite.yml logs @@ -138,9 +138,9 @@ jobs: - name: Install system dependencies run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos - name: Install - run: make install + run: uvx --from rust-just just install - name: Run gcs integration tests with coverage - run: COVERAGE=1 make test-gcs + run: COVERAGE=1 uvx --from rust-just just test-gcs - name: Show debug logs if: ${{ failure() }} run: docker compose -f dev/docker-compose-gcs-server.yml logs @@ -173,7 +173,7 @@ jobs: pattern: coverage-* merge-multiple: true - name: Generate coverage report (75%) # Coverage threshold should only increase over time — never decrease it! - run: COVERAGE_FAIL_UNDER=75 make coverage-report + run: COVERAGE_FAIL_UNDER=75 uvx --from rust-just just coverage-report python-integration-required: if: ${{ always() }} diff --git a/.github/workflows/python-release-docs.yml b/.github/workflows/python-release-docs.yml index 0f9343f11a..236d58fef6 100644 --- a/.github/workflows/python-release-docs.yml +++ b/.github/workflows/python-release-docs.yml @@ -44,7 +44,7 @@ jobs: - name: Install UV uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 - name: Build docs - run: make docs-build + run: uvx --from rust-just just docs-build - name: Copy working-directory: ./mkdocs run: mv ./site /tmp/site diff --git a/.gitignore b/.gitignore index 49476b3f37..a9f2bcc4d5 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ build/ dist/ sdist/ .coverage +.coverage.* coverage.xml .pytest_cache/ @@ -50,3 +51,10 @@ htmlcov pyiceberg/avro/decoder_fast.c pyiceberg/avro/*.html pyiceberg/avro/*.so + +# nox session environments +.nox/ + +# rumdl lint cache +.rumdl_cache/ +.ruff_cache/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index f0dedd0804..0000000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,64 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. ---- -exclude: ^vendor/ - -repos: - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v6.0.0 - hooks: - - id: trailing-whitespace - - id: end-of-file-fixer - - id: check-yaml - - id: check-ast - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.14.3 - hooks: - - id: ruff - args: [ --fix, --exit-non-zero-on-fix ] - - id: ruff-format - - repo: https://github.com/nbQA-dev/nbQA - rev: 1.9.1 - hooks: - - id: nbqa-ruff - args: [ --fix, --exit-non-zero-on-fix ] - - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.18.2 - hooks: - - id: mypy - args: - [--config=pyproject.toml] - additional_dependencies: - - types-cachetools - - types-pytest-lazy-fixture - - types-pytz - - types-requests - - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.45.0 - hooks: - - id: markdownlint - args: ["--fix"] - - repo: https://github.com/codespell-project/codespell - rev: v2.4.1 - hooks: - - id: codespell - - repo: https://github.com/astral-sh/uv-pre-commit - # uv version. - rev: 0.10.9 - hooks: - # Update the uv lockfile - - id: uv-lock diff --git a/AGENTS.md b/AGENTS.md index 3dc1d79c72..d62dd89479 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -46,7 +46,7 @@ lives under `pyiceberg/`, organized by concern rather than by engine: ### Style & Typing -- Formatting and linting are enforced by `ruff` via `prek` (pre-commit): line length **130**, double-quoted strings, isort with `pyiceberg`/`tests` as first-party. Run `make lint` — ruff autofixes most issues. +- Formatting and linting are enforced by `ruff` and `mypy`, driven by `nox` sessions defined in `noxfile.py`: line length **130**, double-quoted strings, isort with `pyiceberg`/`tests` as first-party. Run `just fix` then `just check` — ruff autofixes most issues. - Full type annotations are required (`mypy` runs in strict mode: `disallow_untyped_defs`, `no_implicit_optional`, `warn_unused_ignores`). Avoid Any types. - Docstrings follow the project's pydocstyle config; one-line summaries on public functions. No personal pronouns in comments. - Use domain-specific exceptions from pyiceberg/exceptions.py for Iceberg-specific error conditions. For invalid arguments/values, ValueError is acceptable. Don't raise bare Exception. @@ -76,15 +76,17 @@ lives under `pyiceberg/`, organized by concern rather than by engine: ## Commands -- **Install / set up dev env:** `make install` (installs `uv`, syncs all extras, builds Cython, installs pre-commit hooks) -- **Run unit tests:** `make test` -- **Run a subset:** `make test PYTEST_ARGS="-v -k "` -- **Integration tests (Spark/Docker):** `make test-integration` (rebuild with `make test-integration-rebuild`) -- **Cloud storage suites:** `make test-s3` / `make test-adls` / `make test-gcs` -- **Lint & format:** `make lint` -- **Docs preview / build:** `make docs-serve` / `make docs-build` -- **Clean build artifacts:** `make clean` -- **Use a specific Python:** prefix with `PYTHON=3.12`, e.g. `PYTHON=3.12 make install` +- **Install / set up dev env:** `just install` (installs `uv`, syncs all extras, builds Cython) +- **Run unit tests:** `just test` +- **Run a subset:** `just test -v -k ` +- **Integration tests (Spark/Docker):** `just test-integration` (rebuild with `just test-integration-rebuild`) +- **Cloud storage suites:** `just test-s3` / `just test-adls` / `just test-gcs` +- **Every CI gate (format, lint, types, hygiene):** `just check` +- **Autofix format and lint:** `just fix` +- **Docs preview / build:** `just docs-serve` / `just docs-build` +- **Clean build artifacts:** `just clean` +- **Use a specific Python:** pass nox's `--force-python`, e.g. `just install --force-python 3.12` +- **Run one nox session:** `uv run nox -s `, list them with `uv run nox --list` ## PR & Commit Conventions @@ -93,7 +95,7 @@ lives under `pyiceberg/`, organized by concern rather than by engine: - Keep the first version of a PR minimal; defer optimizations and edge cases to follow-ups. - Commit messages explain the *what* and *why*, not line-by-line implementation. Be as succinct as possible. - The Apache License header is required on every new source file (enforced by `./dev/check-license`). -- Run `make lint` and `make test` before pushing; CI runs both plus the lockfile check. +- Run `just check` and `just test` before pushing; CI runs both plus the lockfile check. ## GitHub Write Approval diff --git a/MANIFEST.in b/MANIFEST.in index 7215e0d6e3..56f8a7e0f9 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -25,7 +25,9 @@ exclude pyiceberg/avro/decoder_fast.c recursive-include tests * # Include development files -include Makefile +include .gitignore +include justfile +include noxfile.py include uv.lock recursive-include dev * diff --git a/Makefile b/Makefile deleted file mode 100644 index 9f0282c65b..0000000000 --- a/Makefile +++ /dev/null @@ -1,203 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -.PHONY: help install install-uv check-license lint \ - test test-integration test-integration-setup test-integration-exec test-integration-cleanup test-integration-rebuild \ - test-s3 test-adls test-gcs test-coverage coverage-report test test-notebook\ - docs-serve docs-build notebook notebook-infra \ - clean - -.DEFAULT_GOAL := help -# ======================== -# Configuration Variables -# ======================== - -PYTHON ?= # Override with e.g. PYTHON=3.11 to use specific Python version -PYTEST_ARGS ?= -v -x # Override with e.g. PYTEST_ARGS="-vv --tb=short" -COVERAGE ?= 0 # Set COVERAGE=1 to enable coverage: make test COVERAGE=1 -COVERAGE_FAIL_UNDER ?= 85 # Minimum coverage % to pass: make coverage-report COVERAGE_FAIL_UNDER=70 -KEEP_COMPOSE ?= 0 # Set KEEP_COMPOSE=1 to keep containers after integration tests - -# Set Python argument for uv commands if PYTHON is specified -ifneq ($(PYTHON),) - PYTHON_ARG = --python $(PYTHON) -else - PYTHON_ARG = -endif - -ifeq ($(COVERAGE),1) - TEST_RUNNER = uv run $(PYTHON_ARG) python -m coverage run --parallel-mode --source=pyiceberg -m -else - TEST_RUNNER = uv run $(PYTHON_ARG) python -m -endif - -ifeq ($(KEEP_COMPOSE),1) - CLEANUP_COMMAND = echo "Keeping containers running for debugging (KEEP_COMPOSE=1)" -else - CLEANUP_COMMAND = docker compose -f dev/docker-compose-integration.yml down -v --remove-orphans --timeout 0 2>/dev/null || true -endif - -# ============ -# Help Section -# ============ - -##@ General - -help: ## Display this help message - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-25s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) - -# ================== -# Installation Tasks -# ================== - -##@ Setup - -install-uv: ## Ensure uv is installed - @if ! command -v uv > /dev/null 2>&1; then \ - echo "uv not found. Installing..."; \ - curl -LsSf https://astral.sh/uv/install.sh | sh; \ - else \ - echo "uv is already installed."; \ - fi - -install: install-uv ## Install uv, dependencies, and pre-commit hooks - uv sync $(PYTHON_ARG) --all-extras - @# Reinstall pyiceberg if Cython extensions (.so) are missing after `make clean` (see #2869) - @if ! find pyiceberg -name "*.so" 2>/dev/null | grep -q .; then \ - echo "Cython extensions not found, reinstalling pyiceberg..."; \ - uv sync $(PYTHON_ARG) --all-extras --reinstall-package pyiceberg; \ - fi - @# Install pre-commit hooks (skipped outside git repo, e.g. release tarballs) - @if [ -d .git ]; then \ - uv run $(PYTHON_ARG) prek install; \ - fi - -# =============== -# Code Validation -# =============== - -##@ Quality - -check-license: ## Check license headers - ./dev/check-license - -lint: ## Run code linters via prek (pre-commit hooks) - uv run $(PYTHON_ARG) prek run -a - -# =============== -# Testing Section -# =============== - -##@ Testing - -test: ## Run all unit tests (excluding integration) - $(TEST_RUNNER) pytest tests/ -m unit --ignore=tests/integration $(PYTEST_ARGS) - -test-integration: test-integration-setup test-integration-exec test-integration-cleanup ## Run integration tests - -test-integration-setup: install ## Start Docker services for integration tests - docker compose -f dev/docker-compose-integration.yml kill - docker compose -f dev/docker-compose-integration.yml rm -f - docker compose -f dev/docker-compose-integration.yml up -d --build --wait - uv run $(PYTHON_ARG) python dev/provision.py - -test-integration-exec: ## Run integration tests (excluding provision) - $(TEST_RUNNER) pytest tests/ -m integration $(PYTEST_ARGS) - -test-integration-cleanup: ## Clean up integration test environment - @if [ "${KEEP_COMPOSE}" != "1" ]; then \ - echo "Cleaning up Docker containers..."; \ - fi - $(CLEANUP_COMMAND) - -test-integration-rebuild: ## Rebuild integration Docker services from scratch - docker compose -f dev/docker-compose-integration.yml kill - docker compose -f dev/docker-compose-integration.yml rm -f - docker compose -f dev/docker-compose-integration.yml build --no-cache - -test-s3: ## Run tests marked with @pytest.mark.s3 - sh ./dev/run-s3.sh - $(TEST_RUNNER) pytest tests/ -m s3 $(PYTEST_ARGS) - -test-adls: ## Run tests marked with @pytest.mark.adls - sh ./dev/run-azurite.sh - $(TEST_RUNNER) pytest tests/ -m adls $(PYTEST_ARGS) - -test-gcs: ## Run tests marked with @pytest.mark.gcs - sh ./dev/run-gcs-server.sh - # gcsfs 2026.6.0 sends HNS/Zonal bucket detection to endpoint_url. - # fake-gcs-server does not support that API, so repeated detection retries - # make GCS sanity tests take close to an hour. Disable it here. - # Workaround documented by gcsfs: - # https://github.com/fsspec/gcsfs/blob/2026.6.0/docs/source/hns_buckets.rst#L99-L101 - GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT=false $(TEST_RUNNER) pytest tests/ -m gcs $(PYTEST_ARGS) - -test-coverage: ## Run all tests with coverage and report - $(MAKE) COVERAGE=1 test test-integration test-s3 test-adls test-gcs - $(MAKE) coverage-report - -coverage-report: ## Combine and report coverage - uv run $(PYTHON_ARG) coverage combine - uv run $(PYTHON_ARG) coverage report -m --fail-under=$(COVERAGE_FAIL_UNDER) - uv run $(PYTHON_ARG) coverage html - uv run $(PYTHON_ARG) coverage xml - -test-notebook: ## Run notebook tests (pyiceberg_example and spark_integration_example) via papermill - $(TEST_RUNNER) pytest tests/notebooks/test_pyiceberg_example.py tests/notebooks/test_spark_integration_example.py -m notebook $(PYTEST_ARGS) - -# ================ -# Documentation -# ================ - -##@ Documentation - -docs-serve: ## Serve local docs preview (hot reload) - uv run $(PYTHON_ARG) --no-default-groups --group docs mkdocs serve -f mkdocs/mkdocs.yml --livereload - -docs-build: ## Build the static documentation site - uv run $(PYTHON_ARG) --no-default-groups --group docs mkdocs build -f mkdocs/mkdocs.yml --strict - -# ======================== -# Experimentation -# ======================== - -##@ Experimentation - -notebook: ## Launch notebook for experimentation - uv run $(PYTHON_ARG) --all-extras --group notebook jupyter lab --notebook-dir=notebooks - -notebook-infra: test-integration-setup ## Launch notebook with integration test infra (Spark, Iceberg Rest Catalog, object storage, etc.) - uv run $(PYTHON_ARG) --all-extras --group notebook jupyter lab --notebook-dir=notebooks - -# =================== -# Project Maintenance -# =================== - -##@ Maintenance - -clean: ## Remove build artifacts and caches - @echo "Cleaning up Cython and Python cached files..." - @rm -rf build dist *.egg-info .venv - @find . -name "*.so" -exec echo Deleting {} \; -delete - @find . -name "*.pyc" -exec echo Deleting {} \; -delete - @find . -name "__pycache__" -exec echo Deleting {} \; -exec rm -rf {} + - @find . -name "*.pyd" -exec echo Deleting {} \; -delete - @find . -name "*.pyo" -exec echo Deleting {} \; -delete - @echo "Cleaning up Jupyter notebook checkpoints..." - @find . -name ".ipynb_checkpoints" -exec echo Deleting {} \; -exec rm -rf {} + - @echo "Cleaning up coverage files..." - @rm -rf .coverage .coverage.* htmlcov/ coverage.xml - @echo "Cleanup complete." diff --git a/dev/release/verify_rc.sh b/dev/release/verify_rc.sh index 744f9d5a19..95ce24ccc1 100755 --- a/dev/release/verify_rc.sh +++ b/dev/release/verify_rc.sh @@ -23,6 +23,11 @@ set -euo pipefail +if ! command -v just > /dev/null 2>&1; then + echo "just is required to verify a release candidate. Install it with: uv tool install rust-just" + exit 1 +fi + if [ "$#" -ne 1 ]; then echo "Usage: $0 " echo " e.g.: $0 0.6.1rc3" @@ -82,8 +87,8 @@ verify_license_documentation() { test_source_distribution() { echo "--- Installing and running the tests, this spins up Docker containers" - make install - make test-coverage + just install + just test-coverage } echo "Verifying pyiceberg-${PYICEBERG_VERSION} in ${PYICEBERG_VERIFICATION_DIR}" diff --git a/justfile b/justfile new file mode 100644 index 0000000000..dd1676fe59 --- /dev/null +++ b/justfile @@ -0,0 +1,141 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Every recipe delegates to noxfile.py, which holds the checks and the orchestration. +# Run `just --list` for the full set. + +# Show available recipes +default: + @just --list + +# ----------------------------------------------------------------- setup + +# Install uv, sync all extras and build the Cython extensions +install *ARGS: install-uv + uv run nox -s install {{ARGS}} + +# Install uv when it is missing +install-uv: + command -v uv > /dev/null || curl -LsSf https://astral.sh/uv/install.sh | sh + +# ----------------------------------------------------------------- quality + +# Run every CI gate without modifying files +check: + uv run nox -- --check + +# Run every CI gate, fixing what can be fixed +fix: + uv run nox + +# Format Python and notebooks +format: + uv run nox -s format + +# Format noxfile.py directly, so it works before nox can load it +format-noxfile: + uv run ruff format noxfile.py + +# Lint Python and notebooks, fixing what can be fixed +lint: + uv run nox -s lint + +# Check whitespace, final newlines, YAML and Python syntax +hygiene: + uv run nox -s hygiene + +# Type-check with mypy +typecheck-mypy: + uv run nox -s typecheck-mypy + +# Type-check with ty (experimental) +typecheck-ty: + uv run nox -s typecheck-ty + +# Verify uv.lock is in sync with pyproject.toml +lock-check: + uv lock --check + +# Check license headers +check-license: + ./dev/check-license + +# ----------------------------------------------------------------- testing + +# Run unit tests on every supported Python version, e.g. `just test -k schema` +test *ARGS: + uv run nox -s tests -j auto -- {{ARGS}} + +# Run integration tests against Docker services +test-integration: + uv run nox -s integration + +# Rebuild the integration Docker services from scratch +test-integration-rebuild: + uv run nox -s integration-rebuild + +# Run tests marked with @pytest.mark.s3 +test-s3: + uv run nox -s s3 + +# Run tests marked with @pytest.mark.adls +test-adls: + uv run nox -s adls + +# Run tests marked with @pytest.mark.gcs +test-gcs: + uv run nox -s gcs + +# Combine coverage data and report +coverage-report: + uv run nox -s coverage + +# Run the full suite with coverage and report +test-coverage: + COVERAGE=1 uv run nox -s tests -j auto + COVERAGE=1 uv run nox -s integration s3 adls gcs + just coverage-report + +# Run notebook tests via papermill +test-notebook: + uv run nox -s test-notebook + +# ----------------------------------------------------------------- docs + +# Serve the docs with hot reload +docs-serve: + uv run nox -s docs-serve + +# Build the static documentation site +docs-build: + uv run nox -s docs-build + +# ----------------------------------------------------------------- experimentation + +# Launch JupyterLab for experimentation +notebook: + uv run nox -s notebook + +# Launch JupyterLab with the integration infrastructure running +notebook-infra: + uv run nox -s notebook-infra + +# ----------------------------------------------------------------- maintenance + +# Remove build artifacts and caches +clean: + uv run nox -s clean diff --git a/mkdocs/README.md b/mkdocs/README.md index f20a2ed152..2f83f5e37f 100644 --- a/mkdocs/README.md +++ b/mkdocs/README.md @@ -22,5 +22,5 @@ The pyiceberg docs are stored in `docs/`. ## Running docs locally ```sh -make docs-serve +just docs-serve ``` diff --git a/mkdocs/docs/api.md b/mkdocs/docs/api.md index a732d2c0a3..8c36d539e5 100644 --- a/mkdocs/docs/api.md +++ b/mkdocs/docs/api.md @@ -235,7 +235,7 @@ static_table = StaticTable.from_metadata( ) ``` -The static-table does not allow for write operations. If your table metadata directory contains a `version-hint.text` file, you can just specify the table root path, and the latest `metadata.json` file will be resolved automatically: +The static-table does not allow for write operations. If your table metadata directory contains a `version-hint.text` file, you can just specify the table root path, and the latest `metadata.json` file will be resolved automatically: ```python from pyiceberg.table import StaticTable diff --git a/mkdocs/docs/community.md b/mkdocs/docs/community.md index 3bb5079982..e1d7aef4e4 100644 --- a/mkdocs/docs/community.md +++ b/mkdocs/docs/community.md @@ -48,14 +48,14 @@ encourage engagement and contributions, the community is not a venue for marketi Any vendor who wants to participate in the Apache Iceberg community Slack workspace should create a dedicated vendor channel for their organization prefixed by `vendor-`. -This space can be used to discuss features and integration with Iceberg related to the vendor offering. This space should not -be used to promote competing vendor products/services or disparage other vendor offerings. Discussion should be focused on +This space can be used to discuss features and integration with Iceberg related to the vendor offering. This space should not +be used to promote competing vendor products/services or disparage other vendor offerings. Discussion should be focused on questions asked by the community and not to expand/introduce/redirect users to alternate offerings. ### Marketing / Solicitation / Recruiting The Apache Iceberg community is a space for everyone to operate free of influence. The development lists, Slack workspace, -and GitHub should not be used to market products or services. Solicitation or overt promotion should not be performed in common +and GitHub should not be used to market products or services. Solicitation or overt promotion should not be performed in common channels or through direct messages. Recruitment of community members should not be conducted through direct messages or community channels, but opportunities diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md index eba97908b3..b70af56dc5 100644 --- a/mkdocs/docs/configuration.md +++ b/mkdocs/docs/configuration.md @@ -671,7 +671,7 @@ catalog: !!! Note "Metastore Authentication Models" - If your BigLake Metastore catalog is configured for "user credentials" authentication instead of "vendor credentials", set the `header.X-Iceberg-Access-Delegation` header to an empty string as shown above. Standard GCP Application Default Credentials (ADC) will be used to authenticate requests to the BigLake Metastore REST API. + If your BigLake Metastore catalog is configured for "user credentials" authentication instead of "vendor credentials", set the `header.X-Iceberg-Access-Delegation` header to an empty string as shown above. Standard GCP Application Default Credentials (ADC) will be used to authenticate requests to the BigLake Metastore REST API. You can retrieve the configuration details for your BigLake Iceberg catalog at the [GCP Console BigLake Metastore page](https://console.cloud.google.com/biglake/metastore/catalogs). Select your catalog, then find the necessary parameters such as `uri`, `warehouse`, and authentication method (e.g. user-creds or vendor). diff --git a/mkdocs/docs/contributing.md b/mkdocs/docs/contributing.md index 2e9cf126d1..f438cd86f1 100644 --- a/mkdocs/docs/contributing.md +++ b/mkdocs/docs/contributing.md @@ -36,12 +36,22 @@ The PyIceberg Project is hosted on GitHub at **Tip:** `uv python list` shows available interpreters. `uv python install 3.12` can install one if needed. ### IDE Setup -After running `make install`, configure your IDE to use the Python interpreter at `.venv/bin/python`. +After running `just install`, configure your IDE to use the Python interpreter at `.venv/bin/python`. **To set up IDEA with uv:** @@ -94,8 +105,8 @@ For full control over your environment, you can use uv commands directly. See th `uv.lock` is a cross-platform lockfile that contains the exact versions of all project dependencies. See the [uv.lock documentation](https://docs.astral.sh/uv/guides/projects/#uvlock) for more details. -**Automatic lock file management:** -When you commit changes to `pyproject.toml` or dependencies, the [`uv-pre-commit`](https://github.com/astral-sh/uv-pre-commit) pre-commit hook will automatically update `uv.lock` if needed. If the lockfile changes, your commit will be stopped and you will be prompted to add the updated `uv.lock` to your commit. +**Keeping it in sync:** +After editing `pyproject.toml` or any dependency, regenerate the lockfile with `uv lock` and commit it alongside the change. CI runs `uv lock --check` (also available as `just lock-check`) and fails when the two disagree. **Manual update:** If you want to update the lockfile yourself (for example, after editing `pyproject.toml`), run: @@ -125,24 +136,26 @@ pip install "git+https://github.com/apache/iceberg-python.git#egg=pyiceberg[pyar ## Linting -`prek` is used for autoformatting and linting: +`ruff`, `nbqa`, `codespell` and `mypy` are driven by [nox](https://nox.thea.codes) through the recipes in the `justfile`. Every tool is pinned in `pyproject.toml` and `uv.lock`, so there is nothing to keep in sync by hand: ```bash -make lint +just format # ruff format over Python and notebooks +just lint # ruff and nbqa, fixing what they can +just typecheck-mypy # mypy +just hygiene # whitespace, final newlines, YAML and Python syntax +just check # everything CI gates on, without writing ``` -`prek` will automatically fix the violations such as import orders, formatting etc. Pylint errors you need to fix yourself. - -In addition to manually running `make lint`, you can install the pre-commit hooks in your local repo with `prek install`. By doing this, linting is run automatically every time you make a commit. +`just fix` applies the automatic fixes (import order, formatting, whitespace) in place. Errors that ruff cannot fix need to be fixed by hand. -You can bump the integrations to the latest version using `prek auto-update`. This will check if there is a newer version of `{ruff,mypy,...}` and update the yaml. +There are no git hooks. Run `just check` before pushing; CI runs the same sessions. Run `just --list` for the full set of recipes. ## Cleaning Removal of old cached files generated during the Cython build process: ```bash -make clean +just clean ``` Helps prevent build failures and unexpected behavior by removing outdated files, ensuring that only up-to-date sources are used & the build environment is always clean. @@ -152,50 +165,50 @@ Helps prevent build failures and unexpected behavior by removing outdated files, For Python, `pytest` is used a testing framework in combination with `coverage` to enforce 90%+ code coverage. ```bash -make test +just test ``` By default, S3 and ADLS tests are ignored because that require RustFS and azurite to be running. To run the S3 suite: ```bash -make test-s3 +just test-s3 ``` To run the ADLS suite: ```bash -make test-adls +just test-adls ``` -To pass additional arguments to pytest, you can use `PYTEST_ARGS`. +To pass additional arguments to pytest, add them after the recipe name; they are forwarded to pytest. ### Run pytest in verbose mode ```sh -make test PYTEST_ARGS="-v" +just test -v ``` ### Run pytest with pdb enabled ```sh -make test PYTEST_ARGS="--pdb" +just test --pdb ``` -To see all available pytest arguments, run `make test PYTEST_ARGS="--help"`. +To see all available pytest arguments, run `just test --help`. ### Integration tests PyIceberg has integration tests with Apache Spark. Spark will create a new database and provision some tables that PyIceberg can query against. ```sh -make test-integration +just test-integration ``` This will restart the containers, to get to a clean state, and then run the PyTest suite. In case something changed in the Dockerfile or the provision script, you can run: ```sh -make test-integration-rebuild +just test-integration-rebuild ``` To rebuild the containers from scratch. @@ -230,26 +243,26 @@ export PYICEBERG_CATALOG__TEST_CATALOG__SECRET_ACCESS_KEY=password ## Notebooks for Experimentation -PyIceberg provides Jupyter notebooks for quick experimentation and learning. Two Make commands are available depending on your needs: +PyIceberg provides Jupyter notebooks for quick experimentation and learning. Two recipes are available depending on your needs: -### PyIceberg Examples (`make notebook`) +### PyIceberg Examples (`just notebook`) For basic PyIceberg experimentation without additional infrastructure: ```bash -make notebook +just notebook ``` This will install notebook dependencies and launch Jupyter Lab in the `notebooks/` directory. **PyIceberg Example Notebook** (`notebooks/pyiceberg_example.ipynb`) is based on the [Getting Started with PyIceberg](https://py.iceberg.apache.org/#getting-started-with-pyiceberg) page. It demonstrates basic PyIceberg operations like creating catalogs, schemas, and querying tables without requiring any external services. -### Spark Integration Examples (`make notebook-infra`) +### Spark Integration Examples (`just notebook-infra`) For working with PyIceberg alongside Spark, use the infrastructure-enabled notebook environment: ```bash -make notebook-infra +just notebook-infra ``` This command spins up the full integration test infrastructure via Docker Compose, including: @@ -261,7 +274,7 @@ This command spins up the full integration test infrastructure via Docker Compos **Spark Example Notebook** (`notebooks/spark_integration_example.ipynb`) is based on the [Spark Getting Started](https://iceberg.apache.org/docs/nightly/spark-getting-started/) guide. This notebook demonstrates how to work with PyIceberg alongside Spark, leveraging the Docker-based testing setup for a complete local development environment. -After running `make notebook-infra`, open `spark_integration_example.ipynb` in the Jupyter Lab interface to explore Spark integration capabilities. +After running `just notebook-infra`, open `spark_integration_example.ipynb` in the Jupyter Lab interface to explore Spark integration capabilities. ## Code standards diff --git a/mkdocs/docs/expression-dsl.md b/mkdocs/docs/expression-dsl.md index 420043bd55..fe41f6a36a 100644 --- a/mkdocs/docs/expression-dsl.md +++ b/mkdocs/docs/expression-dsl.md @@ -189,8 +189,8 @@ complex_filter = And(age_range, status_filter) ## Common Pitfalls 1. **Null Handling**: Be careful when using `IsNull` and `NotNull` predicates with required fields. The expression DSL will automatically optimize these cases: - - `IsNull` (and `IsNaN` for doubles/floats) on a required field will always return `False` - - `NotNull` (and `NotNaN` for doubles/floats) on a required field will always return `True` + - `IsNull` (and `IsNaN` for doubles/floats) on a required field will always return `False` + - `NotNull` (and `NotNaN` for doubles/floats) on a required field will always return `True` 2. **String Comparisons**: When using string predicates like `StartsWith`, ensure that the field type is a string type. diff --git a/mkdocs/docs/verify-release.md b/mkdocs/docs/verify-release.md index 3726b39570..91836a0b02 100644 --- a/mkdocs/docs/verify-release.md +++ b/mkdocs/docs/verify-release.md @@ -115,13 +115,13 @@ This section explains how to run the tests of the source distribution. First step is to install the package: ```sh -make install +just install ``` To run the full test coverage, with both unit tests and integration tests: ```sh -make test-coverage +just test-coverage ``` This will spin up Docker containers to facilitate running test coverage. diff --git a/notebooks/spark_integration_example.ipynb b/notebooks/spark_integration_example.ipynb index c416c2b8df..51cc8faf66 100644 --- a/notebooks/spark_integration_example.ipynb +++ b/notebooks/spark_integration_example.ipynb @@ -14,8 +14,8 @@ "**⚠️ This notebook requires the integration test infrastructure to be running.**\n", "\n", "To start the infrastructure, use one of these commands:\n", - "- `make test-integration-setup` - Start just the infrastructure\n", - "- `make notebook-infra` - Start infrastructure and launch JupyterLab\n", + "- `just test-integration-setup` - Start just the infrastructure\n", + "- `just notebook-infra` - Start infrastructure and launch JupyterLab\n", "\n", "The infrastructure includes:\n", "- Spark Connect server (port 15002)\n", diff --git a/noxfile.py b/noxfile.py new file mode 100644 index 0000000000..95136e7395 --- /dev/null +++ b/noxfile.py @@ -0,0 +1,451 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import os +import shutil +import sys +from collections.abc import Iterator +from enum import StrEnum +from pathlib import Path + +import nox +from identify import identify +from pathspec import GitIgnoreSpec + +nox.options.default_venv_backend = "uv" + +# Sessions without `default=False` are the CI gate, so `just check` runs exactly those. +nox.options.stop_on_first_error = True + +PYPROJECT = nox.project.load_toml("pyproject.toml") +PYTHON_VERSIONS = nox.project.python_versions(PYPROJECT, max_version="3.14") + +SOURCE = ("pyiceberg", "tests", "dev", "mkdocs", "noxfile.py", "setup.py") +NOTEBOOKS = ("notebooks", "tests/notebooks") +SKIP_DIRS = (".git", ".nox", ".venv", "build", "dist", "vendor") +COMPOSE = ("docker", "compose", "-f", "dev/docker-compose-integration.yml") + +IGNORE = GitIgnoreSpec.from_lines(Path(".gitignore").read_text(encoding="utf-8").splitlines()) + + +class Env(StrEnum): + """Environment variables that change how the sessions run.""" + + CI = "CI" + COVERAGE = "COVERAGE" + COVERAGE_FAIL_UNDER = "COVERAGE_FAIL_UNDER" + GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT = "GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT" + KEEP_COMPOSE = "KEEP_COMPOSE" + UV_PROJECT_ENVIRONMENT = "UV_PROJECT_ENVIRONMENT" + + +# The environment is fixed for the whole run, so the switches are read once at import +CI = Env.CI in os.environ +COVERAGE = os.environ.get(Env.COVERAGE) == "1" +COVERAGE_FAIL_UNDER = os.environ.get(Env.COVERAGE_FAIL_UNDER, "85") +KEEP_COMPOSE = os.environ.get(Env.KEEP_COMPOSE) == "1" +COVERAGE_ARGS = ["-m", "coverage", "run", "--parallel-mode", "--source=pyiceberg"] if COVERAGE else [] + + +def _ignored(path: Path) -> bool: + """Return whether .gitignore excludes the path.""" + relative = path.as_posix() + return IGNORE.match_file(relative + "/" if path.is_dir() else relative) + + +def _tracked(*suffixes: str) -> Iterator[Path]: + """Yield project files with the given suffixes, skipping ignored and vendored paths. + + .gitignore decides which build artifacts are left out, so a tool that starts writing a + new cache directory needs no change here. Files that are not ignored are included + whether or not they are committed yet, so a new file is checked before you stage it. + """ + for root, dirnames, filenames in os.walk("."): + base = Path(root) + dirnames[:] = sorted(name for name in dirnames if name not in SKIP_DIRS and not _ignored(base / name)) + for name in sorted(filenames): + path = base / name + if (not suffixes or path.suffix in suffixes) and not _ignored(path): + yield path + + +def _tracked_text() -> list[str]: + """Return tracked text files, matching what the pre-commit `types: [text]` filter gave.""" + texts = [] + for path in _tracked(): + name = str(path) + if path.is_file() and "text" in identify.tags_from_path(name): + texts.append(name) + return texts + + +def _unsupported_extras() -> list[str]: + """Return flags for extras this platform cannot build. + + `hive-kerberos` pulls the `kerberos` package, which publishes no Windows wheels, so the + runner would have to build it against MIT Kerberos headers it does not have. The previous + workflow excluded the same extra. + """ + return ["--no-extra", "hive-kerberos"] if sys.platform == "win32" else [] + + +def _check_only(session: nox.Session) -> bool: + """Return whether this run reports problems instead of fixing them.""" + return CI or "--check" in session.posargs + + +def _python_arg(session: nox.Session) -> list[str]: + """Return the interpreter that nox was asked for, when it was asked for one. + + `nox -s install --force-python 3.12` sets `session.python` without creating a virtualenv, + which is how a session that shells out to `uv` learns which interpreter to sync. + """ + return ["--python", session.python] if isinstance(session.python, str) else [] + + +def _posargs(session: nox.Session) -> list[str]: + """Return the arguments meant for the tool, without the flags this file consumes.""" + return [argument for argument in session.posargs if argument != "--check"] + + +def _pytest(session: nox.Session, *args: str, env: dict[str, str] | None = None) -> None: + """Run pytest in the project environment, collecting coverage when it is enabled. + + The extras are synced here because pytest runs in the project environment, not in the + session environment, so these sessions do not depend on a prior `just install`. + """ + session.run( + "uv", + "run", + "--all-extras", + *_unsupported_extras(), + *_python_arg(session), + "python", + *COVERAGE_ARGS, + "-m", + "pytest", + *args, + *_posargs(session), + external=True, + env=env, + ) + + +def _start_services(session: nox.Session) -> None: + """Recreate the integration services from a clean state.""" + session.run(*COMPOSE, "kill", external=True) + session.run(*COMPOSE, "rm", "-f", external=True) + session.run(*COMPOSE, "up", "-d", "--build", "--wait", external=True) + + +def _stop_services(session: nox.Session) -> None: + """Tear the services down unless KEEP_COMPOSE=1 asks to keep them.""" + if KEEP_COMPOSE: + session.log("Keeping containers running for debugging (KEEP_COMPOSE=1)") + return + session.run(*COMPOSE, "down", "-v", "--remove-orphans", "--timeout", "0", external=True) + + +# ----------------------------------------------------------------- setup +@nox.session(tags=["setup"], default=False, venv_backend="none") +def install(session: nox.Session) -> None: + """Sync all extras and rebuild the Cython extensions when they are missing.""" + session.run("uv", "sync", "--all-extras", *_python_arg(session), *_unsupported_extras(), *session.posargs) + compiled = list(Path("pyiceberg").rglob("*.so")) + list(Path("pyiceberg").rglob("*.pyd")) + if not compiled: + session.log("Cython extensions not found, reinstalling pyiceberg...") + session.run( + "uv", + "sync", + "--all-extras", + "--reinstall-package", + "pyiceberg", + *_python_arg(session), + *_unsupported_extras(), + *session.posargs, + ) + + +# ----------------------------------------------------------------- quality +@nox.session(tags=["check"], allow_parallel=True) +def format(session: nox.Session) -> None: + """Format Python and notebook sources with ruff.""" + session.install("--group", "lint") + session.run("ruff", "format", *(["--check"] if _check_only(session) else []), *SOURCE, *NOTEBOOKS) + + +@nox.session(tags=["check"], allow_parallel=True) +def lint(session: nox.Session) -> None: + """Lint Python and notebook sources with ruff, fixing what it can. + + E402 is ignored in notebooks: their imports legitimately follow the markdown and setup cells. + """ + session.install("--group", "lint") + fix = [] if _check_only(session) else ["--fix"] + session.run("ruff", "check", "--no-fix" if not fix else "--fix", "--exit-non-zero-on-fix", *SOURCE) + session.run("nbqa", "ruff", *fix, "--exit-non-zero-on-fix", "--ignore=E402", *NOTEBOOKS) + + +@nox.session(tags=["check"], allow_parallel=True) +def hygiene(session: nox.Session) -> None: + """Check whitespace, final newlines, YAML and Python syntax with pre-commit-hooks. + + The two fixers rewrite files and fail the session when they change something, which is + the same behaviour the hooks had. `check-yaml --unsafe` parses instead of loading, so the + `!!python/name:` tag in mkdocs/mkdocs.yml is accepted. + """ + session.install("--group", "lint") + text = _tracked_text() + session.run("trailing-whitespace-fixer", *text, silent=True, log=False) + session.run("end-of-file-fixer", *text, silent=True, log=False) + session.run("check-yaml", "--unsafe", *[str(path) for path in _tracked(".yaml", ".yml")], log=False) + session.run("check-ast", *[str(path) for path in _tracked(".py", ".pyi")], log=False) + + +@nox.session(name="typecheck-mypy", tags=["check"], allow_parallel=True) +def typecheck_mypy(session: nox.Session) -> None: + """Type-check every tracked Python file with mypy. + + Only mypy and the type stubs are installed; the project's runtime dependencies stay + out of the environment, which keeps the gate in line with the previous pre-commit hook. + Missing third-party imports are ignored by configuration, so the tests and dev scripts + can be checked without their dependencies present. + """ + session.install("--group", "types") + files = [str(path) for path in _tracked(".py", ".pyi")] + session.run("mypy", "--config-file=pyproject.toml", "--scripts-are-modules", *files, *_posargs(session)) + + +@nox.session(name="typecheck-ty", tags=["experimental"], allow_parallel=True, default=False) +def typecheck_ty(session: nox.Session) -> None: + """Type-check with ty. Experimental: not a gate yet.""" + session.install("--group", "ty", "--group", "noxfile") + session.run("ty", "check", *_posargs(session)) + + +@nox.session(tags=["check"], allow_parallel=True) +def codespell(session: nox.Session) -> None: + """Check spelling in tracked text files, so build artifacts are skipped.""" + session.install("--group", "lint") + session.run("codespell", "--", *_tracked_text(), silent=True, log=False) + + +@nox.session(tags=["check"], allow_parallel=True) +def markdownlint(session: nox.Session) -> None: + """Lint Markdown with rumdl, which reads the existing .markdownlint.yaml. + + MD057 is disabled: it checks that relative links resolve, but the `reference/` pages are + generated by mkdocstrings at build time and do not exist in the source tree. + """ + session.install("--group", "lint") + session.run( + "rumdl", + "check", + *([] if _check_only(session) else ["--fix"]), + "--disable", + "MD057", + *[str(path) for path in _tracked(".md")], + ) + + +# ----------------------------------------------------------------- testing +@nox.session(python=PYTHON_VERSIONS, tags=["test"], allow_parallel=True, default=False) +def tests(session: nox.Session) -> None: + """Run the unit test suite on every supported Python version. + + The environment is synced from uv.lock so each version runs the same pins. The project is + installed into the session environment, which builds the Cython extensions for that + interpreter (the compiled modules are ABI-tagged, so each version needs its own). + """ + # pytest runs with the repository on sys.path, so the in-place Cython build must match + # this interpreter. Rebuild only when its own extension file is missing from the tree. + suffix = ( + session.run("python", "-c", "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))", silent=True) or "" + ).strip() + compiled = list(Path("pyiceberg").rglob(f"decoder_fast{suffix}")) + sync = ["uv", "sync", "--locked", "--all-extras", *_unsupported_extras()] + if not compiled: + sync += ["--reinstall-package", "pyiceberg"] + session.run_install(*sync, env={Env.UV_PROJECT_ENVIRONMENT: str(session.env_dir)}) + extra = [] if session.posargs else ["-v", "-x"] + session.run( + "python", *COVERAGE_ARGS, "-m", "pytest", "tests/", "-m", "unit", "--ignore=tests/integration", *extra, *_posargs(session) + ) + + +@nox.session(tags=["test"], default=False) +def coverage(session: nox.Session) -> None: + """Combine coverage data and enforce the coverage threshold.""" + session.install("--group", "coverage") + fail_under = COVERAGE_FAIL_UNDER + session.run("coverage", "combine") + session.run("coverage", "report", "-m", f"--fail-under={fail_under}") + session.run("coverage", "html") + session.run("coverage", "xml") + + +@nox.session(tags=["test"], default=False, venv_backend="none") +def integration(session: nox.Session) -> None: + """Run the integration suite against local Docker services.""" + try: + _start_services(session) + session.run("uv", "run", "python", "dev/provision.py", external=True) + _pytest(session, "tests/", "-m", "integration", "-v", "-x") + finally: + _stop_services(session) + + +@nox.session(name="integration-rebuild", tags=["test"], default=False, venv_backend="none") +def integration_rebuild(session: nox.Session) -> None: + """Rebuild the integration services from scratch, without a cache.""" + session.run(*COMPOSE, "kill", external=True) + session.run(*COMPOSE, "rm", "-f", external=True) + session.run(*COMPOSE, "build", "--no-cache", external=True) + + +@nox.session(tags=["test"], default=False, venv_backend="none") +def s3(session: nox.Session) -> None: + """Run the S3 suite against a local MinIO.""" + session.run("sh", "./dev/run-s3.sh", external=True) + _pytest(session, "tests/", "-m", "s3", "-v", "-x") + + +@nox.session(tags=["test"], default=False, venv_backend="none") +def adls(session: nox.Session) -> None: + """Run the ADLS suite against a local Azurite.""" + session.run("sh", "./dev/run-azurite.sh", external=True) + _pytest(session, "tests/", "-m", "adls", "-v", "-x") + + +@nox.session(tags=["test"], default=False, venv_backend="none") +def gcs(session: nox.Session) -> None: + """Run the GCS suite against a local fake-gcs-server.""" + session.run("sh", "./dev/run-gcs-server.sh", external=True) + # gcsfs 2026.6.0 sends HNS/Zonal bucket detection to endpoint_url, which fake-gcs-server + # does not support; leaving it on makes the GCS sanity tests take close to an hour. + _pytest(session, "tests/", "-m", "gcs", "-v", "-x", env={Env.GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT: "false"}) + + +@nox.session(name="test-notebook", tags=["test"], default=False, venv_backend="none") +def test_notebook(session: nox.Session) -> None: + """Run the notebook tests with papermill.""" + _pytest( + session, + "tests/notebooks/test_pyiceberg_example.py", + "tests/notebooks/test_spark_integration_example.py", + "-m", + "notebook", + "-v", + "-x", + ) + + +# ----------------------------------------------------------------- docs +@nox.session(name="docs-build", default=False, venv_backend="none") +def docs_build(session: nox.Session) -> None: + """Build the static documentation site, failing on warnings.""" + session.run( + "uv", + "run", + *_python_arg(session), + "--no-default-groups", + "--group", + "docs", + "mkdocs", + "build", + "-f", + "mkdocs/mkdocs.yml", + "--strict", + external=True, + ) + + +@nox.session(name="docs-serve", default=False, venv_backend="none") +def docs_serve(session: nox.Session) -> None: + """Serve the documentation with hot reload.""" + session.run( + "uv", + "run", + *_python_arg(session), + "--no-default-groups", + "--group", + "docs", + "mkdocs", + "serve", + "-f", + "mkdocs/mkdocs.yml", + "--livereload", + external=True, + ) + + +# ----------------------------------------------------------------- experimentation +@nox.session(default=False, venv_backend="none") +def notebook(session: nox.Session) -> None: + """Start JupyterLab for experimentation.""" + session.run( + "uv", + "run", + *_python_arg(session), + "--all-extras", + "--group", + "notebook", + "jupyter", + "lab", + "--notebook-dir=notebooks", + external=True, + ) + + +@nox.session(name="notebook-infra", default=False, venv_backend="none") +def notebook_infra(session: nox.Session) -> None: + """Start JupyterLab with the integration services running. + + The services stop when JupyterLab exits, unless KEEP_COMPOSE=1 asks to keep them. + """ + try: + _start_services(session) + session.run( + "uv", + "run", + *_python_arg(session), + "--all-extras", + "--group", + "notebook", + "jupyter", + "lab", + "--notebook-dir=notebooks", + external=True, + ) + finally: + _stop_services(session) + + +# ----------------------------------------------------------------- maintenance +@nox.session(default=False, venv_backend="none") +def clean(session: nox.Session) -> None: + """Remove build artifacts, caches and session environments.""" + root = Path.cwd() + for directory in ("build", "dist", ".venv", ".nox", "htmlcov"): + shutil.rmtree(root / directory, ignore_errors=True) + for path in [*root.glob("*.egg-info"), *root.glob("**/__pycache__"), *root.glob("**/.ipynb_checkpoints")]: + shutil.rmtree(path, ignore_errors=True) + for pattern in ("**/*.so", "**/*.pyd", "**/*.pyc", "**/*.pyo", ".coverage", ".coverage.*", "coverage.xml"): + for path in root.glob(pattern): + path.unlink(missing_ok=True) diff --git a/pyproject.toml b/pyproject.toml index 8268338fa1..2ba90aa8f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -103,14 +103,45 @@ geoarrow = ["geoarrow-pyarrow>=0.2.0"] encryption = ["cryptography>=42.0.0"] [dependency-groups] +# Imported by noxfile.py itself, so every session that type-checks it needs them. +noxfile = [ + "nox>=2026.10.20", + "identify==2.6.20", + "pathspec>=1.1.1", +] +# Run as commands inside nox sessions, so each session installs only what it runs. +lint = [ + "ruff==0.14.3", + "codespell==2.4.1", + "nbqa==1.9.1", + "rumdl==0.3.2", + "pre-commit-hooks==6.0.0", +] +types = [ + "mypy==1.18.2", + "types-cachetools==7.0.0.20260713", + "types-pytest-lazy-fixture==0.6.3.20260518", + "types-pytz==2026.5.0.20261006", + "types-requests==2.33.0.20261006", +] +ty = [ + "ty==0.0.84", +] +coverage = [ + "coverage[toml]>=7.4.2", +] dev = [ + {include-group = "noxfile"}, + {include-group = "lint"}, + {include-group = "types"}, + {include-group = "ty"}, + {include-group = "coverage"}, "pytest==9.1.1", "pytest-checkdocs==2.14.0", "setuptools>=83.0.0", - "prek>=0.2.1", + # imported by noxfile.py, so they must be present wherever nox runs "pytest-lazy-fixtures==1.4.1", "fastavro==1.12.2", - "coverage[toml]>=7.4.2", "requests-mock==1.12.1", "moto[server]>=5.0.2", "typing-extensions==4.16.0", @@ -128,7 +159,6 @@ dev = [ "papermill>=2.6.0", "nbformat>=5.10.0", "ipykernel>=6.29.0", - "ty==0.0.84", ] # for mkdocs docs = [ @@ -150,6 +180,7 @@ notebook = [ ] [tool.uv] +required-version = ">=0.13.4" default-groups = [ "dev", ] diff --git a/tests/test_version.py b/tests/test_version.py index dcaa3f0735..6870d89842 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -26,5 +26,5 @@ def test_version_format() -> None: assert __version__ == installed_version, ( f"The installed version ({installed_version}) does not match the current codebase version ({__version__})." "This failure could be due to a recent version bump in the Pyiceberg library. " - "Please ensure you have the latest version installed by rerunning `make install` command." + "Please ensure you have the latest version installed by rerunning `just install` command." ) diff --git a/uv.lock b/uv.lock index 7901c5ea12..4be21cfe5a 100644 --- a/uv.lock +++ b/uv.lock @@ -2,9 +2,12 @@ version = 1 revision = 5 requires-python = ">=3.11" resolution-markers = [ - "python_full_version >= '3.14' and sys_platform == 'win32'", - "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.15' and sys_platform == 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", + "python_full_version >= '3.15' and sys_platform == 'emscripten'", + "python_full_version == '3.14.*' and sys_platform == 'emscripten'", + "python_full_version >= '3.15' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version == '3.13.*' and sys_platform == 'win32'", "python_full_version == '3.13.*' and sys_platform == 'emscripten'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", @@ -289,6 +292,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/29/5ecc3a15d5a33e31b26c11426c45c501e439cb865d0bff96315d86443b78/appnope-0.1.4-py2.py3-none-any.whl", hash = "sha256:502575ee11cd7a28c0205f379b525beefebab9d161b7c964670864014ed7213c", size = 4321, upload-time = "2024-02-06T09:43:09.663Z" }, ] +[[package]] +name = "argcomplete" +version = "3.7.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/87/6f/5a73f04007ca950701765949209f068da628bd11f9c2da287278ce91e0ee/argcomplete-3.7.2.tar.gz", hash = "sha256:aad8b69a0b9969edb62db0d1752354c0d50717b10e0cbb00e2a958381b9fc6b9", size = 74473, upload-time = "2026-08-06T04:53:21.662Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/46/bd/551ee6af426af84ca33e02622be722925c196608e9127d731ef17c47f06e/argcomplete-3.7.2-py3-none-any.whl", hash = "sha256:6029205678bdd9c1c728a155f5f9ecf5812393f969eef58807641a2bc2aa5b19", size = 43294, upload-time = "2026-08-06T04:53:20.246Z" }, +] + [[package]] name = "argon2-cffi" version = "25.1.0" @@ -372,6 +384,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, ] +[[package]] +name = "autopep8" +version = "2.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycodestyle" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/50/d8/30873d2b7b57dee9263e53d142da044c4600a46f2d28374b3e38b023df16/autopep8-2.3.2.tar.gz", hash = "sha256:89440a4f969197b69a995e4ce0661b031f455a9f776d2c5ba3dbd83466931758", size = 92210, upload-time = "2025-01-14T14:46:18.454Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/43/53afb8ba17218f19b77c7834128566c5bbb100a0ad9ba2e8e89d089d7079/autopep8-2.3.2-py2.py3-none-any.whl", hash = "sha256:ce8ad498672c845a0c3de2629c15b635ec2b05ef8177a6e7c91c74f3e9b51128", size = 45807, upload-time = "2025-01-14T14:46:15.466Z" }, +] + [[package]] name = "aws-sam-translator" version = "1.110.0" @@ -821,6 +845,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/39/799be3f2f0f38cc727ee3b4f1445fe6d5e4133064ec2e4115069418a5bb6/cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a", size = 22228, upload-time = "2025-11-03T09:25:25.534Z" }, ] +[[package]] +name = "codespell" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/15/e0/709453393c0ea77d007d907dd436b3ee262e28b30995ea1aa36c6ffbccaf/codespell-2.4.1.tar.gz", hash = "sha256:299fcdcb09d23e81e35a671bbe746d5ad7e8385972e65dbb833a2eaac33c01e5", size = 344740, upload-time = "2025-01-28T18:52:39.411Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/01/b394922252051e97aab231d416c86da3d8a6d781eeadcdca1082867de64e/codespell-2.4.1-py3-none-any.whl", hash = "sha256:3dadafa67df7e4a3dbf51e0d7315061b80d265f9552ebd699b3dd6834b47e425", size = 344501, upload-time = "2025-01-28T18:52:37.057Z" }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -830,6 +863,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "colorlog" +version = "6.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8c/55/ba79756cb90c8d69d599d57785398ac87bba7b19c80e87f4e8a562197c93/colorlog-6.12.0.tar.gz", hash = "sha256:2a7924c1dadf18b22a0eb8b06d1c7b01d5341707ec1641eb6fcc4fde0c3e8e5f", size = 18151, upload-time = "2026-07-23T13:40:40.71Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/19/0b6647bf5e331521e55d2b63bfbdc210bd9cd605189273f03614a05f702d/colorlog-6.12.0-py3-none-any.whl", hash = "sha256:30d392604e9110045a2c2aeefc27d7a017abbab63f3a8aee594eac0801df784e", size = 12239, upload-time = "2026-07-23T13:40:39.562Z" }, +] + [[package]] name = "comm" version = "0.2.3" @@ -1189,6 +1234,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, ] +[[package]] +name = "dependency-groups" +version = "1.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b5/16/65e61d8e837e0d70a99a0d4dd76e2af53357c3a03ed1db8ad9b5786d7f25/dependency_groups-1.3.2.tar.gz", hash = "sha256:c81831f43828dbc3987ee247eb198241a0152b8e7ceab10977cc3808eb388ac7", size = 13309, upload-time = "2026-08-22T02:43:30.137Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a1/e4/da5f2f176050efcfb9222a76a3e81c41f69f775be8e563b76ca4292a134f/dependency_groups-1.3.2-py3-none-any.whl", hash = "sha256:8c337f7b92445823b0d0ea507f611390c58dcd2112be00aebcd738a2ca7a1c64", size = 9802, upload-time = "2026-08-22T02:43:28.853Z" }, +] + [[package]] name = "deptry" version = "0.25.1" @@ -2125,6 +2182,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5f/f9/d752d5756ce0405ba29405b65517b287d2baf9f77a1cf5329266f3d577f0/huggingface_hub-2.0.0-py3-none-any.whl", hash = "sha256:b3eecb60540c17e8ef94c187104f4a5c0c451afcb9510ba8e17739bedcc2d6dd", size = 828498, upload-time = "2026-09-24T12:01:21.358Z" }, ] +[[package]] +name = "humanize" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0a/ea/13a1ef3c12d12662905801495283530251918b70d62d368f1d2e0272c70d/humanize-4.16.0.tar.gz", hash = "sha256:7dc2244a2f84a4bfb1d36c37bac80cd78e35cdc5c119206d87b018e1445f3a3f", size = 89515, upload-time = "2026-06-30T16:17:29.859Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b0/aa/0b7365d30fed43e7a3449aba1fe20a0a7174d9cf13e282af4e69ac825441/humanize-4.16.0-py3-none-any.whl", hash = "sha256:353eb2f34c09d098b2880eee8bef21832eae6d174f48c5762fff7e5fcb74d01d", size = 137209, upload-time = "2026-06-30T16:17:28.36Z" }, +] + +[[package]] +name = "identify" +version = "2.6.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/35/d70c0006c7cee65999ea94a6273e60b2094f600a3d8b71b04318253fc643/identify-2.6.20.tar.gz", hash = "sha256:ad729860a923858d26917c2f4fb0a1d83d27a75b1e090c06440c573f048f3285", size = 99681, upload-time = "2026-09-26T20:29:07.187Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/70/fffc9613501877c0a10a1ef73a165e6ded3e53f6e7502227fef56d973933/identify-2.6.20-py2.py3-none-any.whl", hash = "sha256:6a16b69b93187244e0548cbfd25b3e4a6f9a7a2ad784625c3bec2b8d27b81aaa", size = 99506, upload-time = "2026-09-26T20:29:06.054Z" }, +] + [[package]] name = "idna" version = "3.18" @@ -3448,6 +3523,44 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/be/59/e26cb779be4c591d1a910f59d29aca9fba4de70349840a833beba2652371/multidict-6.9.1-py3-none-any.whl", hash = "sha256:7bf6478188f4e47bf5686e8a33da4ae28bf43b1b2528d9ee144d28492bfac60b", size = 19176, upload-time = "2026-09-21T17:59:03.501Z" }, ] +[[package]] +name = "mypy" +version = "1.18.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mypy-extensions" }, + { name = "pathspec" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/77/8f0d0001ffad290cef2f7f216f96c814866248a0b92a722365ed54648e7e/mypy-1.18.2.tar.gz", hash = "sha256:06a398102a5f203d7477b2923dda3634c36727fa5c237d8f859ef90c42a9924b", size = 3448846, upload-time = "2025-09-19T00:11:10.519Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/87/cafd3ae563f88f94eec33f35ff722d043e09832ea8530ef149ec1efbaf08/mypy-1.18.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:807d9315ab9d464125aa9fcf6d84fde6e1dc67da0b6f80e7405506b8ac72bc7f", size = 12731198, upload-time = "2025-09-19T00:09:44.857Z" }, + { url = "https://files.pythonhosted.org/packages/0f/e0/1e96c3d4266a06d4b0197ace5356d67d937d8358e2ee3ffac71faa843724/mypy-1.18.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:776bb00de1778caf4db739c6e83919c1d85a448f71979b6a0edd774ea8399341", size = 11817879, upload-time = "2025-09-19T00:09:47.131Z" }, + { url = "https://files.pythonhosted.org/packages/72/ef/0c9ba89eb03453e76bdac5a78b08260a848c7bfc5d6603634774d9cd9525/mypy-1.18.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1379451880512ffce14505493bd9fe469e0697543717298242574882cf8cdb8d", size = 12427292, upload-time = "2025-09-19T00:10:22.472Z" }, + { url = "https://files.pythonhosted.org/packages/1a/52/ec4a061dd599eb8179d5411d99775bec2a20542505988f40fc2fee781068/mypy-1.18.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1331eb7fd110d60c24999893320967594ff84c38ac6d19e0a76c5fd809a84c86", size = 13163750, upload-time = "2025-09-19T00:09:51.472Z" }, + { url = "https://files.pythonhosted.org/packages/c4/5f/2cf2ceb3b36372d51568f2208c021870fe7834cf3186b653ac6446511839/mypy-1.18.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3ca30b50a51e7ba93b00422e486cbb124f1c56a535e20eff7b2d6ab72b3b2e37", size = 13351827, upload-time = "2025-09-19T00:09:58.311Z" }, + { url = "https://files.pythonhosted.org/packages/c8/7d/2697b930179e7277529eaaec1513f8de622818696857f689e4a5432e5e27/mypy-1.18.2-cp311-cp311-win_amd64.whl", hash = "sha256:664dc726e67fa54e14536f6e1224bcfce1d9e5ac02426d2326e2bb4e081d1ce8", size = 9757983, upload-time = "2025-09-19T00:10:09.071Z" }, + { url = "https://files.pythonhosted.org/packages/07/06/dfdd2bc60c66611dd8335f463818514733bc763e4760dee289dcc33df709/mypy-1.18.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:33eca32dd124b29400c31d7cf784e795b050ace0e1f91b8dc035672725617e34", size = 12908273, upload-time = "2025-09-19T00:10:58.321Z" }, + { url = "https://files.pythonhosted.org/packages/81/14/6a9de6d13a122d5608e1a04130724caf9170333ac5a924e10f670687d3eb/mypy-1.18.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a3c47adf30d65e89b2dcd2fa32f3aeb5e94ca970d2c15fcb25e297871c8e4764", size = 11920910, upload-time = "2025-09-19T00:10:20.043Z" }, + { url = "https://files.pythonhosted.org/packages/5f/a9/b29de53e42f18e8cc547e38daa9dfa132ffdc64f7250e353f5c8cdd44bee/mypy-1.18.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5d6c838e831a062f5f29d11c9057c6009f60cb294fea33a98422688181fe2893", size = 12465585, upload-time = "2025-09-19T00:10:33.005Z" }, + { url = "https://files.pythonhosted.org/packages/77/ae/6c3d2c7c61ff21f2bee938c917616c92ebf852f015fb55917fd6e2811db2/mypy-1.18.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01199871b6110a2ce984bde85acd481232d17413868c9807e95c1b0739a58914", size = 13348562, upload-time = "2025-09-19T00:10:11.51Z" }, + { url = "https://files.pythonhosted.org/packages/4d/31/aec68ab3b4aebdf8f36d191b0685d99faa899ab990753ca0fee60fb99511/mypy-1.18.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a2afc0fa0b0e91b4599ddfe0f91e2c26c2b5a5ab263737e998d6817874c5f7c8", size = 13533296, upload-time = "2025-09-19T00:10:06.568Z" }, + { url = "https://files.pythonhosted.org/packages/9f/83/abcb3ad9478fca3ebeb6a5358bb0b22c95ea42b43b7789c7fb1297ca44f4/mypy-1.18.2-cp312-cp312-win_amd64.whl", hash = "sha256:d8068d0afe682c7c4897c0f7ce84ea77f6de953262b12d07038f4d296d547074", size = 9828828, upload-time = "2025-09-19T00:10:28.203Z" }, + { url = "https://files.pythonhosted.org/packages/5f/04/7f462e6fbba87a72bc8097b93f6842499c428a6ff0c81dd46948d175afe8/mypy-1.18.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:07b8b0f580ca6d289e69209ec9d3911b4a26e5abfde32228a288eb79df129fcc", size = 12898728, upload-time = "2025-09-19T00:10:01.33Z" }, + { url = "https://files.pythonhosted.org/packages/99/5b/61ed4efb64f1871b41fd0b82d29a64640f3516078f6c7905b68ab1ad8b13/mypy-1.18.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ed4482847168439651d3feee5833ccedbf6657e964572706a2adb1f7fa4dfe2e", size = 11910758, upload-time = "2025-09-19T00:10:42.607Z" }, + { url = "https://files.pythonhosted.org/packages/3c/46/d297d4b683cc89a6e4108c4250a6a6b717f5fa96e1a30a7944a6da44da35/mypy-1.18.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3ad2afadd1e9fea5cf99a45a822346971ede8685cc581ed9cd4d42eaf940986", size = 12475342, upload-time = "2025-09-19T00:11:00.371Z" }, + { url = "https://files.pythonhosted.org/packages/83/45/4798f4d00df13eae3bfdf726c9244bcb495ab5bd588c0eed93a2f2dd67f3/mypy-1.18.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a431a6f1ef14cf8c144c6b14793a23ec4eae3db28277c358136e79d7d062f62d", size = 13338709, upload-time = "2025-09-19T00:11:03.358Z" }, + { url = "https://files.pythonhosted.org/packages/d7/09/479f7358d9625172521a87a9271ddd2441e1dab16a09708f056e97007207/mypy-1.18.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7ab28cc197f1dd77a67e1c6f35cd1f8e8b73ed2217e4fc005f9e6a504e46e7ba", size = 13529806, upload-time = "2025-09-19T00:10:26.073Z" }, + { url = "https://files.pythonhosted.org/packages/71/cf/ac0f2c7e9d0ea3c75cd99dff7aec1c9df4a1376537cb90e4c882267ee7e9/mypy-1.18.2-cp313-cp313-win_amd64.whl", hash = "sha256:0e2785a84b34a72ba55fb5daf079a1003a34c05b22238da94fcae2bbe46f3544", size = 9833262, upload-time = "2025-09-19T00:10:40.035Z" }, + { url = "https://files.pythonhosted.org/packages/5a/0c/7d5300883da16f0063ae53996358758b2a2df2a09c72a5061fa79a1f5006/mypy-1.18.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:62f0e1e988ad41c2a110edde6c398383a889d95b36b3e60bcf155f5164c4fdce", size = 12893775, upload-time = "2025-09-19T00:10:03.814Z" }, + { url = "https://files.pythonhosted.org/packages/50/df/2cffbf25737bdb236f60c973edf62e3e7b4ee1c25b6878629e88e2cde967/mypy-1.18.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8795a039bab805ff0c1dfdb8cd3344642c2b99b8e439d057aba30850b8d3423d", size = 11936852, upload-time = "2025-09-19T00:10:51.631Z" }, + { url = "https://files.pythonhosted.org/packages/be/50/34059de13dd269227fb4a03be1faee6e2a4b04a2051c82ac0a0b5a773c9a/mypy-1.18.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6ca1e64b24a700ab5ce10133f7ccd956a04715463d30498e64ea8715236f9c9c", size = 12480242, upload-time = "2025-09-19T00:11:07.955Z" }, + { url = "https://files.pythonhosted.org/packages/5b/11/040983fad5132d85914c874a2836252bbc57832065548885b5bb5b0d4359/mypy-1.18.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d924eef3795cc89fecf6bedc6ed32b33ac13e8321344f6ddbf8ee89f706c05cb", size = 13326683, upload-time = "2025-09-19T00:09:55.572Z" }, + { url = "https://files.pythonhosted.org/packages/e9/ba/89b2901dd77414dd7a8c8729985832a5735053be15b744c18e4586e506ef/mypy-1.18.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20c02215a080e3a2be3aa50506c67242df1c151eaba0dcbc1e4e557922a26075", size = 13514749, upload-time = "2025-09-19T00:10:44.827Z" }, + { url = "https://files.pythonhosted.org/packages/25/bc/cc98767cffd6b2928ba680f3e5bc969c4152bf7c2d83f92f5a504b92b0eb/mypy-1.18.2-cp314-cp314-win_amd64.whl", hash = "sha256:749b5f83198f1ca64345603118a6f01a4e99ad4bf9d103ddc5a3200cc4614adf", size = 9982959, upload-time = "2025-09-19T00:10:37.344Z" }, + { url = "https://files.pythonhosted.org/packages/87/e3/be76d87158ebafa0309946c4a73831974d4d6ab4f4ef40c3b53a385a66fd/mypy-1.18.2-py3-none-any.whl", hash = "sha256:22a1748707dd62b58d2ae53562ffc4d7f8bcc727e8ac7cbc69c053ddc874d47e", size = 2352367, upload-time = "2025-09-19T00:10:15.489Z" }, +] + [[package]] name = "mypy-boto3-dynamodb" version = "1.43.64" @@ -3472,6 +3585,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/bf/781db91d144e0f2a4c91649499722de9b57f6a3c251b34d339820f3f7f0f/mypy_boto3_glue-1.43.100-py3-none-any.whl", hash = "sha256:104b3d95133e10b4ad825432cfeeeb97ec56d5bb7ee70509ffc2a6a749b45a03", size = 154974, upload-time = "2026-09-22T20:27:52.045Z" }, ] +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + [[package]] name = "natsort" version = "8.4.0" @@ -3536,6 +3658,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1a/69/ee613f74085ca7103f79cd08d579c4f3177d1d26e5d3d9528d2d6536a707/nbformat-5.11.1-py3-none-any.whl", hash = "sha256:cc6698fa75f4fab8755ead786317815f13a6fee3b53311c0abb1a8b51d52f7ec", size = 79849, upload-time = "2026-08-17T08:10:50.18Z" }, ] +[[package]] +name = "nbqa" +version = "1.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "autopep8" }, + { name = "ipython" }, + { name = "tokenize-rt" }, + { name = "tomli" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/aa/76/62d2609924cf34445148cd6b5de694cf64c179cc416cac93182579620e57/nbqa-1.9.1.tar.gz", hash = "sha256:a1f4bcf587c597302fed295951001fc4e1be4ce0e77e1ab1b25ac2fbe3db0cdd", size = 38348, upload-time = "2024-11-10T12:21:58.333Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/28/88/4789719fbbe166d12d345b3ac66b96105f10001b16e00a9765ba29261a21/nbqa-1.9.1-py3-none-any.whl", hash = "sha256:95552d2f6c2c038136252a805aa78d85018aef922586270c3a074332737282e5", size = 35259, upload-time = "2024-11-10T12:21:56.731Z" }, +] + [[package]] name = "nest-asyncio2" version = "1.7.2" @@ -3566,6 +3703,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f9/33/bd5b9137445ea4b680023eb0469b2bb969d61303dedb2aac6560ff3d14a1/notebook_shim-0.2.4-py3-none-any.whl", hash = "sha256:411a5be4e9dc882a074ccbcae671eda64cceb068767e9a3419096986560e1cef", size = 13307, upload-time = "2024-02-14T23:35:16.286Z" }, ] +[[package]] +name = "nox" +version = "2026.10.20" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "argcomplete" }, + { name = "attrs" }, + { name = "colorlog" }, + { name = "dependency-groups" }, + { name = "humanize" }, + { name = "packaging" }, + { name = "platformdirs" }, + { name = "python-discovery" }, + { name = "virtualenv" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8c/ae/44b519e4074993256fa713a56d97a9e50e2afe0452e8486eaf41e235171f/nox-2026.10.20.tar.gz", hash = "sha256:5da57c009c3dfdd762c068d9079b17966ec967a9eb02cd8ebcf016a946eaf67f", size = 4085873, upload-time = "2026-10-20T22:07:44.499Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/5e/91e4a29b20c652283d9dc055400e8259a16a7f957db935f0d9814699a032/nox-2026.10.20-py3-none-any.whl", hash = "sha256:7aff12bb01471f5b087ada34cf0dbe452b02d840593f2e2afecc4bf6e6d6cd3f", size = 99068, upload-time = "2026-10-20T22:07:42.255Z" }, +] + [[package]] name = "numba" version = "0.65.1" @@ -3931,19 +4088,15 @@ wheels = [ ] [[package]] -name = "prek" -version = "0.5.3" +name = "pre-commit-hooks" +version = "6.0.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/54/93/d1e5afc996b9fde04d71c37fd28bdd404e54daf1da6c76b883c5cdbdb411/prek-0.5.3.tar.gz", hash = "sha256:06d88bed9a5b2886cd3796957e4cecf05fa9b06724b625df31dbf0a48ff2330c", size = 551929, upload-time = "2026-09-13T08:38:37.823Z" } +dependencies = [ + { name = "ruamel-yaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/36/4d/93e63e48f8fd16d6c1e4cef5dabadcade4d1325c7fd6f29f075a4d2284f3/pre_commit_hooks-6.0.0.tar.gz", hash = "sha256:76d8370c006f5026cdd638a397a678d26dda735a3c88137e05885a020f824034", size = 28293, upload-time = "2025-08-09T19:25:04.6Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/db/5d/8731dc49cb5424d37424db3ea2f3ff6a0b20caa971db72796e822ce48b24/prek-0.5.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:2d7240c5e6a996ef5bebe5d57d43049803e41c225b343681c020bf0c87dab2b2", size = 6003518, upload-time = "2026-09-13T08:38:20.584Z" }, - { url = "https://files.pythonhosted.org/packages/51/45/ca413aefb3ea2411bbf0adb6eebfac1e9bfc5ac7679dbe26336fc651c347/prek-0.5.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:1dd6df8235ca361dbbeec89c305ccb41a138088a2fc548484b4cb89f66a80d20", size = 5538754, upload-time = "2026-09-13T08:38:23.004Z" }, - { url = "https://files.pythonhosted.org/packages/dc/92/874d58ba40d2fe7cfec2789ae8bf415f3751666ad5c59928821fe53c3c01/prek-0.5.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:61ff791bb850ba52cc0d86576498df7e1bd1e9ed20df63dae113a4a6aa495e19", size = 5841568, upload-time = "2026-09-13T08:38:25.008Z" }, - { url = "https://files.pythonhosted.org/packages/8f/40/167037b8eef75f6f036c60ff7e93f42defebb7c3733677e2f30e5a479257/prek-0.5.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a863379a2668c4ef079d820ffd5b14073145380dfbe0e70fef498dc73c568f86", size = 6226960, upload-time = "2026-09-13T08:38:26.788Z" }, - { url = "https://files.pythonhosted.org/packages/42/9a/bfcd3c1fe1fdefee8dc144a92c4b0b93b59f31c7df60ee1720c49223a90a/prek-0.5.3-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:086d1b77557f0a089568dc5b93c22ccd793351524f3eeb29d36781421a350e72", size = 5854832, upload-time = "2026-09-13T08:38:28.954Z" }, - { url = "https://files.pythonhosted.org/packages/15/41/1450b995b18bbf0d8ab65016860198052baad5c79c231151fa1dcca4a2de/prek-0.5.3-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:4f09b947255124e9591a7f7ef16eb5340a36ba94a275d788d7ea804fab35f0e2", size = 6343055, upload-time = "2026-09-13T08:38:30.96Z" }, - { url = "https://files.pythonhosted.org/packages/9c/e8/8f45950b8a8ddcf22efc7183e9d6b1d7e30373229ff7d9e56f918a55922c/prek-0.5.3-py3-none-win_amd64.whl", hash = "sha256:5b74a9742c3e8f8688d5dad1439688f112e2e1746fcb91016015dae38df3eb42", size = 5744293, upload-time = "2026-09-13T08:38:33.104Z" }, - { url = "https://files.pythonhosted.org/packages/24/cd/ce637f4b6cc6c583516dc16b539408dab593672f7c6fe7723a37caf378ff/prek-0.5.3-py3-none-win_arm64.whl", hash = "sha256:20d92aef53a5e237f4659ecc6669c0fbc98a1b83f0e2476530400a72d42ca390", size = 5512958, upload-time = "2026-09-13T08:38:36.123Z" }, + { url = "https://files.pythonhosted.org/packages/12/46/eba9be9daa403fa94854ce16a458c29df9a01c6c047931c3d8be6016cd9a/pre_commit_hooks-6.0.0-py2.py3-none-any.whl", hash = "sha256:76161b76d321d2f8ee2a8e0b84c30ee8443e01376121fd1c90851e33e3bd7ee2", size = 41338, upload-time = "2025-08-09T19:25:03.513Z" }, ] [[package]] @@ -4337,6 +4490,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, ] +[[package]] +name = "pycodestyle" +version = "2.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/86/df/178e41c9ed0ff33c23b3f3757e4e658c889f9abc5ad76ece6cd607b12e9c/pycodestyle-2.15.0.tar.gz", hash = "sha256:318f5db083869b4c4dad922d0b11124fb27ab181b6730b93371da671e31bd50e", size = 39765, upload-time = "2026-09-22T22:15:34.926Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/e2/480bcdb153feecde4d736b18c908c7eee4846f11d7547d2e0c71499af630/pycodestyle-2.15.0-py2.py3-none-any.whl", hash = "sha256:12fd2f73c7b8ee8845a0431111df8faf4c1a07d6e64e2ee7f0c74014dab14181", size = 31583, upload-time = "2026-09-22T22:15:34.007Z" }, +] + [[package]] name = "pycparser" version = "3.0" @@ -4593,20 +4755,29 @@ sql-sqlite = [ ] [package.dev-dependencies] +coverage = [ + { name = "coverage", extra = ["toml"] }, +] dev = [ + { name = "codespell" }, { name = "coverage", extra = ["toml"] }, { name = "cython" }, { name = "deptry" }, { name = "docutils" }, { name = "fastavro" }, { name = "google-cloud-bigquery" }, + { name = "identify" }, { name = "ipykernel" }, { name = "moto", extra = ["server"] }, + { name = "mypy" }, { name = "mypy-boto3-dynamodb" }, { name = "mypy-boto3-glue" }, { name = "nbformat" }, + { name = "nbqa" }, + { name = "nox" }, { name = "papermill" }, - { name = "prek" }, + { name = "pathspec" }, + { name = "pre-commit-hooks" }, { name = "protobuf" }, { name = "pyarrow-stubs" }, { name = "pyspark", extra = ["connect"] }, @@ -4615,9 +4786,15 @@ dev = [ { name = "pytest-lazy-fixtures" }, { name = "pytest-mock" }, { name = "requests-mock" }, + { name = "ruff" }, + { name = "rumdl" }, { name = "setuptools" }, { name = "sqlalchemy" }, { name = "ty" }, + { name = "types-cachetools" }, + { name = "types-pytest-lazy-fixture" }, + { name = "types-pytz" }, + { name = "types-requests" }, { name = "typing-extensions" }, ] docs = [ @@ -4633,9 +4810,31 @@ docs = [ { name = "mkdocstrings" }, { name = "mkdocstrings-python" }, ] +lint = [ + { name = "codespell" }, + { name = "nbqa" }, + { name = "pre-commit-hooks" }, + { name = "ruff" }, + { name = "rumdl" }, +] notebook = [ { name = "jupyterlab" }, ] +noxfile = [ + { name = "identify" }, + { name = "nox" }, + { name = "pathspec" }, +] +ty = [ + { name = "ty" }, +] +types = [ + { name = "mypy" }, + { name = "types-cachetools" }, + { name = "types-pytest-lazy-fixture" }, + { name = "types-pytz" }, + { name = "types-requests" }, +] [package.metadata] requires-dist = [ @@ -4688,20 +4887,27 @@ requires-dist = [ provides-extras = ["pyarrow", "pandas", "duckdb", "ray", "bodo", "daft", "polars", "snappy", "hive", "hive-kerberos", "s3fs", "glue", "adlfs", "dynamodb", "bigquery", "sql-postgres", "sql-sqlite", "gcsfs", "rest-sigv4", "hf", "pyiceberg-core", "datafusion", "gcp-auth", "entra-auth", "geoarrow", "encryption"] [package.metadata.requires-dev] +coverage = [{ name = "coverage", extras = ["toml"], specifier = ">=7.4.2" }] dev = [ + { name = "codespell", specifier = "==2.4.1" }, { name = "coverage", extras = ["toml"], specifier = ">=7.4.2" }, { name = "cython", specifier = ">=3.0.0" }, { name = "deptry", specifier = ">=0.14" }, { name = "docutils", specifier = "!=0.21.post1" }, { name = "fastavro", specifier = "==1.12.2" }, { name = "google-cloud-bigquery", specifier = ">=3.33.0,<4" }, + { name = "identify", specifier = "==2.6.20" }, { name = "ipykernel", specifier = ">=6.29.0" }, { name = "moto", extras = ["server"], specifier = ">=5.0.2" }, + { name = "mypy", specifier = "==1.18.2" }, { name = "mypy-boto3-dynamodb", specifier = ">=1.28.18" }, { name = "mypy-boto3-glue", specifier = ">=1.28.18" }, { name = "nbformat", specifier = ">=5.10.0" }, + { name = "nbqa", specifier = "==1.9.1" }, + { name = "nox", specifier = ">=2026.10.20" }, { name = "papermill", specifier = ">=2.6.0" }, - { name = "prek", specifier = ">=0.2.1" }, + { name = "pathspec", specifier = ">=1.1.1" }, + { name = "pre-commit-hooks", specifier = "==6.0.0" }, { name = "protobuf", specifier = "==7.36.2" }, { name = "pyarrow-stubs", specifier = ">=20.0.0.20251107" }, { name = "pyspark", extras = ["connect"], specifier = "==4.1.3" }, @@ -4710,9 +4916,15 @@ dev = [ { name = "pytest-lazy-fixtures", specifier = "==1.4.1" }, { name = "pytest-mock", specifier = "==3.15.1" }, { name = "requests-mock", specifier = "==1.12.1" }, + { name = "ruff", specifier = "==0.14.3" }, + { name = "rumdl", specifier = "==0.3.2" }, { name = "setuptools", specifier = ">=83.0.0" }, { name = "sqlalchemy", specifier = ">=2.0.18,<3" }, { name = "ty", specifier = "==0.0.84" }, + { name = "types-cachetools", specifier = "==7.0.0.20260713" }, + { name = "types-pytest-lazy-fixture", specifier = "==0.6.3.20260518" }, + { name = "types-pytz", specifier = "==2026.5.0.20261006" }, + { name = "types-requests", specifier = "==2.33.0.20261006" }, { name = "typing-extensions", specifier = "==4.16.0" }, ] docs = [ @@ -4728,7 +4940,27 @@ docs = [ { name = "mkdocstrings", specifier = "==1.0.6" }, { name = "mkdocstrings-python", specifier = "==2.0.9" }, ] +lint = [ + { name = "codespell", specifier = "==2.4.1" }, + { name = "nbqa", specifier = "==1.9.1" }, + { name = "pre-commit-hooks", specifier = "==6.0.0" }, + { name = "ruff", specifier = "==0.14.3" }, + { name = "rumdl", specifier = "==0.3.2" }, +] notebook = [{ name = "jupyterlab", specifier = ">=4.0.0" }] +noxfile = [ + { name = "identify", specifier = "==2.6.20" }, + { name = "nox", specifier = ">=2026.10.20" }, + { name = "pathspec", specifier = ">=1.1.1" }, +] +ty = [{ name = "ty", specifier = "==0.0.84" }] +types = [ + { name = "mypy", specifier = "==1.18.2" }, + { name = "types-cachetools", specifier = "==7.0.0.20260713" }, + { name = "types-pytest-lazy-fixture", specifier = "==0.6.3.20260518" }, + { name = "types-pytz", specifier = "==2026.5.0.20261006" }, + { name = "types-requests", specifier = "==2.33.0.20261006" }, +] [[package]] name = "pyiceberg-core" @@ -5559,6 +5791,56 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ea/ea/e7b0251441da9adfeaebcf29601d10f2a1455fcf0772fae9e7e19032bd96/rpds_py-2026.5.1-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:8c43a8a973270fd173bf48cdf80bbe66312421cba68d40845034f174f2389049", size = 586326, upload-time = "2026-05-28T12:02:11.47Z" }, ] +[[package]] +name = "ruamel-yaml" +version = "0.19.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/3b/ebda527b56beb90cb7652cb1c7e4f91f48649fbcd8d2eb2fb6e77cd3329b/ruamel_yaml-0.19.1.tar.gz", hash = "sha256:53eb66cd27849eff968ebf8f0bf61f46cdac2da1d1f3576dd4ccee9b25c31993", size = 142709, upload-time = "2026-01-02T16:50:31.84Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b8/0c/51f6841f1d84f404f92463fc2b1ba0da357ca1e3db6b7fbda26956c3b82a/ruamel_yaml-0.19.1-py3-none-any.whl", hash = "sha256:27592957fedf6e0b62f281e96effd28043345e0e66001f97683aa9a40c667c93", size = 118102, upload-time = "2026-01-02T16:50:29.201Z" }, +] + +[[package]] +name = "ruff" +version = "0.14.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/75/62/50b7727004dfe361104dfbf898c45a9a2fdfad8c72c04ae62900224d6ecf/ruff-0.14.3.tar.gz", hash = "sha256:4ff876d2ab2b161b6de0aa1f5bd714e8e9b4033dc122ee006925fbacc4f62153", size = 5558687, upload-time = "2025-10-31T00:26:26.878Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/8e/0c10ff1ea5d4360ab8bfca4cb2c9d979101a391f3e79d2616c9bf348cd26/ruff-0.14.3-py3-none-linux_armv6l.whl", hash = "sha256:876b21e6c824f519446715c1342b8e60f97f93264012de9d8d10314f8a79c371", size = 12535613, upload-time = "2025-10-31T00:25:44.302Z" }, + { url = "https://files.pythonhosted.org/packages/d3/c8/6724f4634c1daf52409fbf13fefda64aa9c8f81e44727a378b7b73dc590b/ruff-0.14.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b6fd8c79b457bedd2abf2702b9b472147cd860ed7855c73a5247fa55c9117654", size = 12855812, upload-time = "2025-10-31T00:25:47.793Z" }, + { url = "https://files.pythonhosted.org/packages/de/03/db1bce591d55fd5f8a08bb02517fa0b5097b2ccabd4ea1ee29aa72b67d96/ruff-0.14.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:71ff6edca490c308f083156938c0c1a66907151263c4abdcb588602c6e696a14", size = 11944026, upload-time = "2025-10-31T00:25:49.657Z" }, + { url = "https://files.pythonhosted.org/packages/0b/75/4f8dbd48e03272715d12c87dc4fcaaf21b913f0affa5f12a4e9c6f8a0582/ruff-0.14.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:786ee3ce6139772ff9272aaf43296d975c0217ee1b97538a98171bf0d21f87ed", size = 12356818, upload-time = "2025-10-31T00:25:51.949Z" }, + { url = "https://files.pythonhosted.org/packages/ec/9b/506ec5b140c11d44a9a4f284ea7c14ebf6f8b01e6e8917734a3325bff787/ruff-0.14.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cd6291d0061811c52b8e392f946889916757610d45d004e41140d81fb6cd5ddc", size = 12336745, upload-time = "2025-10-31T00:25:54.248Z" }, + { url = "https://files.pythonhosted.org/packages/c7/e1/c560d254048c147f35e7f8131d30bc1f63a008ac61595cf3078a3e93533d/ruff-0.14.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a497ec0c3d2c88561b6d90f9c29f5ae68221ac00d471f306fa21fa4264ce5fcd", size = 13101684, upload-time = "2025-10-31T00:25:56.253Z" }, + { url = "https://files.pythonhosted.org/packages/a5/32/e310133f8af5cd11f8cc30f52522a3ebccc5ea5bff4b492f94faceaca7a8/ruff-0.14.3-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:e231e1be58fc568950a04fbe6887c8e4b85310e7889727e2b81db205c45059eb", size = 14535000, upload-time = "2025-10-31T00:25:58.397Z" }, + { url = "https://files.pythonhosted.org/packages/a2/a1/7b0470a22158c6d8501eabc5e9b6043c99bede40fa1994cadf6b5c2a61c7/ruff-0.14.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:469e35872a09c0e45fecf48dd960bfbce056b5db2d5e6b50eca329b4f853ae20", size = 14156450, upload-time = "2025-10-31T00:26:00.889Z" }, + { url = "https://files.pythonhosted.org/packages/0a/96/24bfd9d1a7f532b560dcee1a87096332e461354d3882124219bcaff65c09/ruff-0.14.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d6bc90307c469cb9d28b7cfad90aaa600b10d67c6e22026869f585e1e8a2db0", size = 13568414, upload-time = "2025-10-31T00:26:03.291Z" }, + { url = "https://files.pythonhosted.org/packages/a7/e7/138b883f0dfe4ad5b76b58bf4ae675f4d2176ac2b24bdd81b4d966b28c61/ruff-0.14.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2f8a0bbcffcfd895df39c9a4ecd59bb80dca03dc43f7fb63e647ed176b741e", size = 13315293, upload-time = "2025-10-31T00:26:05.708Z" }, + { url = "https://files.pythonhosted.org/packages/33/f4/c09bb898be97b2eb18476b7c950df8815ef14cf956074177e9fbd40b7719/ruff-0.14.3-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:678fdd7c7d2d94851597c23ee6336d25f9930b460b55f8598e011b57c74fd8c5", size = 13539444, upload-time = "2025-10-31T00:26:08.09Z" }, + { url = "https://files.pythonhosted.org/packages/9c/aa/b30a1db25fc6128b1dd6ff0741fa4abf969ded161599d07ca7edd0739cc0/ruff-0.14.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:1ec1ac071e7e37e0221d2f2dbaf90897a988c531a8592a6a5959f0603a1ecf5e", size = 12252581, upload-time = "2025-10-31T00:26:10.297Z" }, + { url = "https://files.pythonhosted.org/packages/da/13/21096308f384d796ffe3f2960b17054110a9c3828d223ca540c2b7cc670b/ruff-0.14.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:afcdc4b5335ef440d19e7df9e8ae2ad9f749352190e96d481dc501b753f0733e", size = 12307503, upload-time = "2025-10-31T00:26:12.646Z" }, + { url = "https://files.pythonhosted.org/packages/cb/cc/a350bac23f03b7dbcde3c81b154706e80c6f16b06ff1ce28ed07dc7b07b0/ruff-0.14.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:7bfc42f81862749a7136267a343990f865e71fe2f99cf8d2958f684d23ce3dfa", size = 12675457, upload-time = "2025-10-31T00:26:15.044Z" }, + { url = "https://files.pythonhosted.org/packages/cb/76/46346029fa2f2078826bc88ef7167e8c198e58fe3126636e52f77488cbba/ruff-0.14.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a65e448cfd7e9c59fae8cf37f9221585d3354febaad9a07f29158af1528e165f", size = 13403980, upload-time = "2025-10-31T00:26:17.81Z" }, + { url = "https://files.pythonhosted.org/packages/9f/a4/35f1ef68c4e7b236d4a5204e3669efdeefaef21f0ff6a456792b3d8be438/ruff-0.14.3-py3-none-win32.whl", hash = "sha256:f3d91857d023ba93e14ed2d462ab62c3428f9bbf2b4fbac50a03ca66d31991f7", size = 12500045, upload-time = "2025-10-31T00:26:20.503Z" }, + { url = "https://files.pythonhosted.org/packages/03/15/51960ae340823c9859fb60c63301d977308735403e2134e17d1d2858c7fb/ruff-0.14.3-py3-none-win_amd64.whl", hash = "sha256:d7b7006ac0756306db212fd37116cce2bd307e1e109375e1c6c106002df0ae5f", size = 13594005, upload-time = "2025-10-31T00:26:22.533Z" }, + { url = "https://files.pythonhosted.org/packages/b7/73/4de6579bac8e979fca0a77e54dec1f1e011a0d268165eb8a9bc0982a6564/ruff-0.14.3-py3-none-win_arm64.whl", hash = "sha256:26eb477ede6d399d898791d01961e16b86f02bc2486d0d1a7a9bb2379d055dc1", size = 12590017, upload-time = "2025-10-31T00:26:24.52Z" }, +] + +[[package]] +name = "rumdl" +version = "0.3.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c6/ec/826c3e86e97a20266da6f5d265ecc4228e692f32071b4eca733afd81860e/rumdl-0.3.2.tar.gz", hash = "sha256:cfabc5bc98b4379496b7feeb9caf7f51888081e5a3bb4b1972cb629bcffb65a1", size = 6006614, upload-time = "2026-10-28T15:30:14.655Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/54/e16c69b61222798221111f28b63f4c8d97d5a72c89ddd0d244bd6198f3d0/rumdl-0.3.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:e42076d20f798f2a006671ce9b647f238466957d5363c057afc6a46e8960f0db", size = 8287795, upload-time = "2026-10-28T15:30:04.261Z" }, + { url = "https://files.pythonhosted.org/packages/5a/6e/9189a61f3b8a9b1c5e6d508b2958023b111f2b8291f96b8b677e03d6d2b7/rumdl-0.3.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:8cbd42b21ae554bce0452d9fc366b3c20585102ee024033e191dd7c24d34ea67", size = 7706426, upload-time = "2026-10-28T15:29:56.576Z" }, + { url = "https://files.pythonhosted.org/packages/f2/8e/a0ab280bffbd951544d6e64fe41cb30087cec7eb3754ebc80f6b1778f517/rumdl-0.3.2-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:f2013176864b10984a5c385727db804ff9823b7516fde5786a9e5da654b341e8", size = 8005951, upload-time = "2026-10-28T15:29:59.269Z" }, + { url = "https://files.pythonhosted.org/packages/be/b8/4d1596cbdb92d3df69e23e3e232a866e1bb02ac441d63d8e90f860c42c24/rumdl-0.3.2-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:621ec2bee5195f4ce017ee4fc6fa62c23fe4ade1250e4fa3dcfb51992fef72b3", size = 8577137, upload-time = "2026-10-28T15:32:45.675Z" }, + { url = "https://files.pythonhosted.org/packages/33/7b/370f9c7d170e1256a71b853c04c88a51e07e849fede2b43a560822f499e4/rumdl-0.3.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:ee9dc51af07de10d7cda83c4c3af4cf47307678104911dac8340983df187d331", size = 7972059, upload-time = "2026-10-28T15:30:01.563Z" }, + { url = "https://files.pythonhosted.org/packages/45/f6/4350b53b764926f5bd8db429d36296822d0e2adac3cd8c68a7fab65b2d98/rumdl-0.3.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:aa198a9d4352459602832b85c6c7a487278df901f98c6b5e9322345ac427d082", size = 8568077, upload-time = "2026-10-28T15:30:12.01Z" }, + { url = "https://files.pythonhosted.org/packages/99/0a/c36054b97d3e497827d89e0c2767c805af8db7edc5bc0521560c1e7a836d/rumdl-0.3.2-py3-none-win_amd64.whl", hash = "sha256:50295352d5ee1b284ed982b06c6bbe73e01522223872e3b620f80ab3bb5e69cb", size = 8700946, upload-time = "2026-10-28T15:30:06.701Z" }, +] + [[package]] name = "s3fs" version = "2026.9.0" @@ -5883,6 +6165,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/60/45/c7b5c3168458db837e8ceab06dc77824e18202679d0463f0e8f002143a97/tinycss2-1.5.1-py3-none-any.whl", hash = "sha256:3415ba0f5839c062696996998176c4a3751d18b7edaaeeb658c9ce21ec150661", size = 28404, upload-time = "2025-11-23T10:29:08.676Z" }, ] +[[package]] +name = "tokenize-rt" +version = "6.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/69/ed/8f07e893132d5051d86a553e749d5c89b2a4776eb3a579b72ed61f8559ca/tokenize_rt-6.2.0.tar.gz", hash = "sha256:8439c042b330c553fdbe1758e4a05c0ed460dbbbb24a606f11f0dee75da4cad6", size = 5476, upload-time = "2025-05-23T23:48:00.035Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/f0/3fe8c6e69135a845f4106f2ff8b6805638d4e85c264e70114e8126689587/tokenize_rt-6.2.0-py2.py3-none-any.whl", hash = "sha256:a152bf4f249c847a66497a4a95f63376ed68ac6abf092a2f7cfb29d044ecff44", size = 6004, upload-time = "2025-05-23T23:47:58.812Z" }, +] + [[package]] name = "tomli" version = "2.4.1" @@ -6042,6 +6333,45 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6d/1c/e6e91852b72566b1ab9aca504712857dfb89324c12092a5bd61d42d34427/ty-0.0.84-py3-none-win_arm64.whl", hash = "sha256:6c9d3718c4a0d318bc9c92af9b2aa85c3294e7dff54cf1d69ad79568f56516da", size = 13656453, upload-time = "2026-09-24T13:16:21.51Z" }, ] +[[package]] +name = "types-cachetools" +version = "7.0.0.20260713" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/34/64/66d7efdb36ecf6826aca5415e59fe2df96e97d24157147e53acfbe8dda11/types_cachetools-7.0.0.20260713.tar.gz", hash = "sha256:f1acf079e9c66a81e096a897ef0b261a82117cf856834e37b4bd0c9a116a076a", size = 10199, upload-time = "2026-07-13T05:22:21.845Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/c7/d3525c9dbdc1be7786bad46655ef051b6e7993f656d304719ec40079c91c/types_cachetools-7.0.0.20260713-py3-none-any.whl", hash = "sha256:6db9bcc7a3840d39e91c04117d85a9d0937eacc9d14d12a873e2b01a2d24a71d", size = 9615, upload-time = "2026-07-13T05:22:20.76Z" }, +] + +[[package]] +name = "types-pytest-lazy-fixture" +version = "0.6.3.20260518" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/65/46/a79ab84cdace98870440968006c86eec5cab22b64f805e1770d8b19bf8d8/types_pytest_lazy_fixture-0.6.3.20260518.tar.gz", hash = "sha256:5e4c259c3c0eddbbf7719d81109050f4e9dcf9f650291a6ff3de42b964734acd", size = 7472, upload-time = "2026-05-18T06:05:45.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/3e/bfcb69336e4a1c69b87f634a3ddebeb6c77f55932631407d191d8b394128/types_pytest_lazy_fixture-0.6.3.20260518-py3-none-any.whl", hash = "sha256:5fc2bc417c2804145ddfcff91bd3f472c8cfbe79b84a2a3abae661ef45408ec1", size = 7599, upload-time = "2026-05-18T06:05:44.471Z" }, +] + +[[package]] +name = "types-pytz" +version = "2026.5.0.20261006" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/47/b493f47f2dd0971db06459439ebdeb114cf95029188268099b331bbc4727/types_pytz-2026.5.0.20261006.tar.gz", hash = "sha256:1a522c2ec03aad8d4baaf97105958019ad51704b1e471c882d1c6ccea3e5e64b", size = 11073, upload-time = "2026-10-06T08:15:12.87Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/38/e375026fb4ff74fed6bdb84d7d336571d0603b6b9675751579cec84cfc9b/types_pytz-2026.5.0.20261006-py3-none-any.whl", hash = "sha256:9e4a893b362a8eed4e10a348c80603ade65bdb3819419e589364afafe2ea08b1", size = 10168, upload-time = "2026-10-06T08:15:11.985Z" }, +] + +[[package]] +name = "types-requests" +version = "2.33.0.20261006" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/57/15/9b7e2e2e7c87d01366185b198b3febc6bc0c973f2bf21a62da4ab7d3495e/types_requests-2.33.0.20261006.tar.gz", hash = "sha256:0652999e9306aea345f40732d58fa49a7f6cade6a0d74d92119c5c8d82eddaf0", size = 25316, upload-time = "2026-10-06T08:15:57.782Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/72/b82789207b3d360ce9f5a4372c790c52cc3dd928aeb7d4faeec652b651b2/types_requests-2.33.0.20261006-py3-none-any.whl", hash = "sha256:26cc8146505cab33cda9737991929e4144c559bebe05078ccc6998f27c4ca2c1", size = 21445, upload-time = "2026-10-06T08:15:56.658Z" }, +] + [[package]] name = "typing-extensions" version = "4.16.0" From 708729c55578214bc8ae8221d4e2b49031e19dc7 Mon Sep 17 00:00:00 2001 From: Orest Tokovenko Date: Fri, 30 Oct 2026 10:39:31 -0700 Subject: [PATCH 2/8] dev: read .gitignore only when it exists The sdist carries no .gitignore, and release verification loads the noxfile from an unpacked sdist, so the read must tolerate its absence. --- MANIFEST.in | 1 - noxfile.py | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 56f8a7e0f9..bbfc8e4dd5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -25,7 +25,6 @@ exclude pyiceberg/avro/decoder_fast.c recursive-include tests * # Include development files -include .gitignore include justfile include noxfile.py include uv.lock diff --git a/noxfile.py b/noxfile.py index 95136e7395..c6a8655691 100644 --- a/noxfile.py +++ b/noxfile.py @@ -41,7 +41,10 @@ SKIP_DIRS = (".git", ".nox", ".venv", "build", "dist", "vendor") COMPOSE = ("docker", "compose", "-f", "dev/docker-compose-integration.yml") -IGNORE = GitIgnoreSpec.from_lines(Path(".gitignore").read_text(encoding="utf-8").splitlines()) +# The sdist has no .gitignore, and release verification loads this file from an unpacked sdist. +IGNORE = GitIgnoreSpec.from_lines( + Path(".gitignore").read_text(encoding="utf-8").splitlines() if Path(".gitignore").exists() else [] +) class Env(StrEnum): From ce5378b2a7a75d36f6e92fb56dec6e889c86640a Mon Sep 17 00:00:00 2001 From: Orest Tokovenko Date: Fri, 30 Oct 2026 10:48:49 -0700 Subject: [PATCH 3/8] ci: call nox sessions directly instead of going through just The justfile is a convenience for local runs. CI has no reason to install a task runner to reach the sessions it already names. --- .github/workflows/python-ci-docs.yml | 2 +- .github/workflows/python-integration.yml | 18 +++++++++--------- .github/workflows/python-release-docs.yml | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/python-ci-docs.yml b/.github/workflows/python-ci-docs.yml index 358d42cb74..b3baa311e1 100644 --- a/.github/workflows/python-ci-docs.yml +++ b/.github/workflows/python-ci-docs.yml @@ -45,4 +45,4 @@ jobs: - name: Install UV uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 - name: Build docs - run: uvx --from rust-just just docs-build + run: uv run nox -s docs-build diff --git a/.github/workflows/python-integration.yml b/.github/workflows/python-integration.yml index 4a1d526833..531e104c72 100644 --- a/.github/workflows/python-integration.yml +++ b/.github/workflows/python-integration.yml @@ -48,9 +48,9 @@ jobs: - name: Install system dependencies run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos - name: Install - run: uvx --from rust-just just install + run: uv run nox -s install - name: Run integration tests with coverage - run: COVERAGE=1 uvx --from rust-just just test-integration + run: COVERAGE=1 uv run nox -s integration - name: Show debug logs if: ${{ failure() }} run: docker compose -f dev/docker-compose-integration.yml logs @@ -78,9 +78,9 @@ jobs: - name: Install system dependencies run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos - name: Install - run: uvx --from rust-just just install + run: uv run nox -s install - name: Run s3 integration tests with coverage - run: COVERAGE=1 uvx --from rust-just just test-s3 + run: COVERAGE=1 uv run nox -s s3 - name: Show debug logs if: ${{ failure() }} run: docker compose -f dev/docker-compose.yml logs @@ -108,9 +108,9 @@ jobs: - name: Install system dependencies run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos - name: Install - run: uvx --from rust-just just install + run: uv run nox -s install - name: Run adls integration tests with coverage - run: COVERAGE=1 uvx --from rust-just just test-adls + run: COVERAGE=1 uv run nox -s adls - name: Show debug logs if: ${{ failure() }} run: docker compose -f dev/docker-compose-azurite.yml logs @@ -138,9 +138,9 @@ jobs: - name: Install system dependencies run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos - name: Install - run: uvx --from rust-just just install + run: uv run nox -s install - name: Run gcs integration tests with coverage - run: COVERAGE=1 uvx --from rust-just just test-gcs + run: COVERAGE=1 uv run nox -s gcs - name: Show debug logs if: ${{ failure() }} run: docker compose -f dev/docker-compose-gcs-server.yml logs @@ -173,7 +173,7 @@ jobs: pattern: coverage-* merge-multiple: true - name: Generate coverage report (75%) # Coverage threshold should only increase over time — never decrease it! - run: COVERAGE_FAIL_UNDER=75 uvx --from rust-just just coverage-report + run: COVERAGE_FAIL_UNDER=75 uv run nox -s coverage python-integration-required: if: ${{ always() }} diff --git a/.github/workflows/python-release-docs.yml b/.github/workflows/python-release-docs.yml index 236d58fef6..a4fd17cacf 100644 --- a/.github/workflows/python-release-docs.yml +++ b/.github/workflows/python-release-docs.yml @@ -44,7 +44,7 @@ jobs: - name: Install UV uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 - name: Build docs - run: uvx --from rust-just just docs-build + run: uv run nox -s docs-build - name: Copy working-directory: ./mkdocs run: mv ./site /tmp/site From f0f1f02b53175ab3b8baea8fadfa6467d4fb18d8 Mon Sep 17 00:00:00 2001 From: Orest Tokovenko Date: Fri, 30 Oct 2026 10:58:52 -0700 Subject: [PATCH 4/8] ci: drop the install step that every following step repeats `uv run` syncs the project environment on demand, and the test and coverage sessions build their own environments from uv.lock, so the explicit install only synced the extras a second time. --- .github/workflows/python-ci.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 401dae881d..f26464d942 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -47,8 +47,6 @@ jobs: run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos - name: Check uv.lock is up to date run: uv lock --check - - name: Install - run: uv run nox -s install - name: Run linters run: uv run nox # The Python version matrix lives in noxfile.py, so nox runs every supported @@ -72,8 +70,6 @@ jobs: uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 with: enable-cache: true - - name: Install - run: uv run nox -s install - name: Run unit tests run: uv run nox -s tests -p 3.12 From 53971f0ba1d6affdd7e0ccfbb0ded5d1b1ef8ec3 Mon Sep 17 00:00:00 2001 From: Orest Tokovenko Date: Fri, 30 Oct 2026 14:40:26 -0700 Subject: [PATCH 5/8] dev: read the env switches without an enum wrapper The wrapper named every member after the string it held, and only two of the six reached a call site, so it added a lookup without removing a literal. --- noxfile.py | 30 ++++++++---------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/noxfile.py b/noxfile.py index c6a8655691..ed4ba8d667 100644 --- a/noxfile.py +++ b/noxfile.py @@ -21,7 +21,6 @@ import shutil import sys from collections.abc import Iterator -from enum import StrEnum from pathlib import Path import nox @@ -40,6 +39,12 @@ NOTEBOOKS = ("notebooks", "tests/notebooks") SKIP_DIRS = (".git", ".nox", ".venv", "build", "dist", "vendor") COMPOSE = ("docker", "compose", "-f", "dev/docker-compose-integration.yml") +CI = "CI" in os.environ +COVERAGE = os.environ.get("COVERAGE") == "1" +COVERAGE_FAIL_UNDER = os.environ.get("COVERAGE_FAIL_UNDER", "85") +KEEP_COMPOSE = os.environ.get("KEEP_COMPOSE") == "1" +COVERAGE_ARGS = ["-m", "coverage", "run", "--parallel-mode", "--source=pyiceberg"] if COVERAGE else [] + # The sdist has no .gitignore, and release verification loads this file from an unpacked sdist. IGNORE = GitIgnoreSpec.from_lines( @@ -47,25 +52,6 @@ ) -class Env(StrEnum): - """Environment variables that change how the sessions run.""" - - CI = "CI" - COVERAGE = "COVERAGE" - COVERAGE_FAIL_UNDER = "COVERAGE_FAIL_UNDER" - GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT = "GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT" - KEEP_COMPOSE = "KEEP_COMPOSE" - UV_PROJECT_ENVIRONMENT = "UV_PROJECT_ENVIRONMENT" - - -# The environment is fixed for the whole run, so the switches are read once at import -CI = Env.CI in os.environ -COVERAGE = os.environ.get(Env.COVERAGE) == "1" -COVERAGE_FAIL_UNDER = os.environ.get(Env.COVERAGE_FAIL_UNDER, "85") -KEEP_COMPOSE = os.environ.get(Env.KEEP_COMPOSE) == "1" -COVERAGE_ARGS = ["-m", "coverage", "run", "--parallel-mode", "--source=pyiceberg"] if COVERAGE else [] - - def _ignored(path: Path) -> bool: """Return whether .gitignore excludes the path.""" relative = path.as_posix() @@ -285,7 +271,7 @@ def tests(session: nox.Session) -> None: sync = ["uv", "sync", "--locked", "--all-extras", *_unsupported_extras()] if not compiled: sync += ["--reinstall-package", "pyiceberg"] - session.run_install(*sync, env={Env.UV_PROJECT_ENVIRONMENT: str(session.env_dir)}) + session.run_install(*sync, env={"UV_PROJECT_ENVIRONMENT": str(session.env_dir)}) extra = [] if session.posargs else ["-v", "-x"] session.run( "python", *COVERAGE_ARGS, "-m", "pytest", "tests/", "-m", "unit", "--ignore=tests/integration", *extra, *_posargs(session) @@ -342,7 +328,7 @@ def gcs(session: nox.Session) -> None: session.run("sh", "./dev/run-gcs-server.sh", external=True) # gcsfs 2026.6.0 sends HNS/Zonal bucket detection to endpoint_url, which fake-gcs-server # does not support; leaving it on makes the GCS sanity tests take close to an hour. - _pytest(session, "tests/", "-m", "gcs", "-v", "-x", env={Env.GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT: "false"}) + _pytest(session, "tests/", "-m", "gcs", "-v", "-x", env={"GCSFS_EXPERIMENTAL_ZB_HNS_SUPPORT": "false"}) @nox.session(name="test-notebook", tags=["test"], default=False, venv_backend="none") From 9b047add28282de14515992ea11f62b8f22cda6f Mon Sep 17 00:00:00 2001 From: Orest Tokovenko Date: Fri, 30 Oct 2026 14:40:26 -0700 Subject: [PATCH 6/8] dev: drop the .coverage.* ignore rule Those files are not new: the Makefile ran coverage with --parallel-mode as well, and noxfile's clean session still removes them. --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index a9f2bcc4d5..626fccca2f 100644 --- a/.gitignore +++ b/.gitignore @@ -30,7 +30,6 @@ build/ dist/ sdist/ .coverage -.coverage.* coverage.xml .pytest_cache/ From fb171d9540794a63fc9428d119a863d71961288f Mon Sep 17 00:00:00 2001 From: Orest Tokovenko Date: Fri, 30 Oct 2026 14:44:32 -0700 Subject: [PATCH 7/8] dev: let coverage read its threshold from pyproject.toml coverage already reads [tool.coverage.run] source from pyproject.toml, so the session stops repeating --source, and the 85 default moves next to it as [tool.coverage.report] fail_under. COVERAGE_FAIL_UNDER still overrides it, because the integration job accepts lower coverage than the unit suite. --- noxfile.py | 12 ++++++------ pyproject.toml | 3 +++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/noxfile.py b/noxfile.py index ed4ba8d667..2a02976b4c 100644 --- a/noxfile.py +++ b/noxfile.py @@ -41,9 +41,10 @@ COMPOSE = ("docker", "compose", "-f", "dev/docker-compose-integration.yml") CI = "CI" in os.environ COVERAGE = os.environ.get("COVERAGE") == "1" -COVERAGE_FAIL_UNDER = os.environ.get("COVERAGE_FAIL_UNDER", "85") +COVERAGE_FAIL_UNDER = os.environ.get("COVERAGE_FAIL_UNDER") KEEP_COMPOSE = os.environ.get("KEEP_COMPOSE") == "1" -COVERAGE_ARGS = ["-m", "coverage", "run", "--parallel-mode", "--source=pyiceberg"] if COVERAGE else [] +COVERAGE_ARGS = ["-m", "coverage", "run", "--parallel-mode"] if COVERAGE else [] +FAIL_UNDER_ARGS = [f"--fail-under={COVERAGE_FAIL_UNDER}"] if COVERAGE_FAIL_UNDER else [] # The sdist has no .gitignore, and release verification loads this file from an unpacked sdist. @@ -282,11 +283,10 @@ def tests(session: nox.Session) -> None: def coverage(session: nox.Session) -> None: """Combine coverage data and enforce the coverage threshold.""" session.install("--group", "coverage") - fail_under = COVERAGE_FAIL_UNDER session.run("coverage", "combine") - session.run("coverage", "report", "-m", f"--fail-under={fail_under}") - session.run("coverage", "html") - session.run("coverage", "xml") + session.run("coverage", "report", "-m", *FAIL_UNDER_ARGS) + session.run("coverage", "html", *FAIL_UNDER_ARGS) + session.run("coverage", "xml", *FAIL_UNDER_ARGS) @nox.session(tags=["test"], default=False, venv_backend="none") diff --git a/pyproject.toml b/pyproject.toml index 2ba90aa8f8..9810fec867 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -318,3 +318,6 @@ allowed-unresolved-imports = ["pyiceberg_core.datafusion", "griffe", "mkdocs_gen [tool.coverage.run] source = ['pyiceberg/'] + +[tool.coverage.report] +fail_under = 85 From 5486c9746f0a4843c84aceb410b2930ce76fc2fe Mon Sep 17 00:00:00 2001 From: Orest Tokovenko Date: Fri, 30 Oct 2026 14:50:14 -0700 Subject: [PATCH 8/8] dev: let mypy read its flags from pyproject.toml mypy finds the config file on its own, so --config-file was redundant, and --scripts-are-modules is a setting rather than an invocation choice. --- noxfile.py | 2 +- pyproject.toml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 2a02976b4c..165ee930c2 100644 --- a/noxfile.py +++ b/noxfile.py @@ -219,7 +219,7 @@ def typecheck_mypy(session: nox.Session) -> None: """ session.install("--group", "types") files = [str(path) for path in _tracked(".py", ".pyi")] - session.run("mypy", "--config-file=pyproject.toml", "--scripts-are-modules", *files, *_posargs(session)) + session.run("mypy", *files, *_posargs(session)) @nox.session(name="typecheck-ty", tags=["experimental"], allow_parallel=True, default=False) diff --git a/pyproject.toml b/pyproject.toml index 9810fec867..3dc532b9d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -296,6 +296,7 @@ filterwarnings = [ mypy_path = "python" no_implicit_optional = true namespace_packages = false +scripts_are_modules = true warn_redundant_casts = true warn_unreachable = true warn_unused_ignores = true