From a81854305c13f4f66870ab583cf6ff139b0a2be1 Mon Sep 17 00:00:00 2001 From: prrao87 Date: Sat, 7 Nov 2026 10:12:44 -0500 Subject: [PATCH 1/3] ci: install protoc for the Windows wheel from its GitHub release Chocolatey's community feed was unreachable from the runner during the v0.8.1 release run, so build-windows failed before it started compiling. Download protoc 29.3 from the protobuf release instead and check it against a pinned SHA-256. --- .github/workflows/python-publish.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 0261af4..3d41e4d 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -148,7 +148,13 @@ jobs: - uses: astral-sh/setup-uv@v4 - name: Install build dependencies - run: choco install protoc --version=29.3 -y --no-progress + # protoc comes from its GitHub release: Chocolatey's feed is intermittently unreachable from runners. + run: | + $zip = "$env:RUNNER_TEMP\protoc.zip" + Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v29.3/protoc-29.3-win64.zip -OutFile $zip + if ((Get-FileHash $zip -Algorithm SHA256).Hash -ne "57EA59E9F551AD8D71FFAA9B5CFBE0CA1F4E720972A1DB7EC2D12AB44BFF9383") { throw "protoc checksum mismatch" } + Expand-Archive $zip "$env:RUNNER_TEMP\protoc" + "$env:RUNNER_TEMP\protoc\bin" | Out-File -Append -Encoding utf8 $env:GITHUB_PATH - name: Build Windows x86-64 wheel uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad # v1 From c159d3f7a54b3c62257f3c99fcf8e70eb756d9c5 Mon Sep 17 00:00:00 2001 From: prrao87 Date: Sat, 7 Nov 2026 10:17:59 -0500 Subject: [PATCH 2/3] ci: add Linux aarch64 wheels, pin actions and audit with zizmor - Build a manylinux_2_28 aarch64 wheel natively on ubuntu-24.04-arm and test it on Python 3.12 to 3.14, like the other wheels. pylance has aarch64 wheels, so it installs without compiling. No musl wheels: pylance publishes none, so they would still compile Lance. - Pin every action to a commit SHA with its exact version, and let Dependabot update the pins weekly with a 7-day cooldown. - Add a Workflow Audit job that runs zizmor, and fix what it found: pass the release tag through an env var instead of expanding it in the script, stop persisting checkout credentials, give every workflow read-only contents permission, and restore no caches (sccache, uv) in the wheel builds. --- .github/dependabot.yml | 14 ++++ .github/workflows/docs.yml | 6 +- .github/workflows/python-publish.yml | 119 +++++++++++++++++---------- .github/workflows/python-test.yml | 19 +++-- .github/workflows/rust-test.yml | 9 +- .github/workflows/spelling.yml | 9 +- .github/workflows/style.yml | 15 +++- .github/workflows/zizmor.yml | 38 +++++++++ AGENTS.md | 1 + README.md | 2 +- RELEASE.md | 5 +- docs/getting-started.md | 2 +- docs/index.md | 4 +- python/README.md | 6 +- 14 files changed, 182 insertions(+), 67 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/zizmor.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..614d85f --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,14 @@ +# Keep the commit-SHA pins on GitHub Actions current. One grouped PR a week, and only for +# releases at least a week old, so a compromised release has time to be caught first. +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + cooldown: + default-days: 7 + groups: + github-actions: + patterns: + - "*" diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 5b81252..fabbf59 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -34,8 +34,10 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 10 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@v4 + uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0 - name: Build the site run: uv run --project docs --locked zensical build --strict diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 3d41e4d..8f1361a 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -1,5 +1,5 @@ -# Build, test and publish lance-csr wheels: Linux x86-64 (manylinux_2_28), native -# Apple Silicon (macOS arm64) and Windows x86-64, plus an sdist for every other platform. +# Build, test and publish lance-csr wheels: Linux x86-64 and aarch64 (manylinux_2_28), +# native Apple Silicon (macOS arm64) and Windows x86-64, plus an sdist for every other platform. # # Pull requests that change how wheels are built (this workflow, the wheel check, the # release build profile) build and test them without publishing. A release builds and tests @@ -45,18 +45,31 @@ env: jobs: build-linux: # Built on the host with zig as the linker, targeting glibc 2.28 (RHEL 8, Ubuntu 20.04 - # and newer); the same command produced the alpha's tested Linux wheels. - runs-on: ubuntu-24.04 + # and newer); the same command produced the alpha's tested Linux wheels. Each + # architecture builds natively on its own runner. + strategy: + fail-fast: false + matrix: + include: + - arch: x86_64 + runner: ubuntu-24.04 + - arch: aarch64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} timeout-minutes: 90 steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - - uses: astral-sh/setup-uv@v4 + - uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0 + with: + + enable-cache: false - name: Install build dependencies run: | # protoc is needed by Lance's build scripts, not by wheel users. @@ -67,18 +80,20 @@ jobs: uv tool install ziglang "$(uv tool dir)/ziglang/bin/python" -c "import os, ziglang; print(os.path.dirname(ziglang.__file__))" >> "$GITHUB_PATH" - - name: Build manylinux_2_28 x86-64 wheel - uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad # v1 + - name: Build manylinux_2_28 ${{ matrix.arch }} wheel + uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad # v1.50.1 with: command: build working-directory: python - target: x86_64-unknown-linux-gnu + target: ${{ matrix.arch }}-unknown-linux-gnu container: "off" - sccache: "true" + # No sccache: release wheels are built without restoring any cache. + sccache: "false" args: --release --locked --strip --zig --compatibility manylinux_2_28 --out dist - name: Build sdist - uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad # v1 + if: matrix.arch == 'x86_64' + uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad # v1.50.1 with: command: sdist working-directory: python @@ -90,9 +105,9 @@ jobs: uvx twine check dist/* sha256sum dist/* - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: python-dist-linux + name: python-dist-linux-${{ matrix.arch }} path: python/dist/* if-no-files-found: error @@ -103,24 +118,29 @@ jobs: env: MACOSX_DEPLOYMENT_TARGET: "11.0" steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - - uses: astral-sh/setup-uv@v4 + - uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0 + + with: + enable-cache: false - name: Install build dependencies run: brew install protobuf - name: Build Apple Silicon wheel - uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad # v1 + uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad # v1.50.1 with: command: build working-directory: python target: aarch64-apple-darwin - sccache: "true" + # No sccache: release wheels are built without restoring any cache. + sccache: "false" args: --release --locked --strip --out dist - name: Inspect wheel @@ -129,7 +149,7 @@ jobs: uvx twine check dist/*.whl shasum -a 256 dist/*.whl - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: python-dist-macos-arm64 path: python/dist/*.whl @@ -139,14 +159,18 @@ jobs: runs-on: windows-2022 timeout-minutes: 90 steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - - uses: astral-sh/setup-uv@v4 + - uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0 + with: + + enable-cache: false - name: Install build dependencies # protoc comes from its GitHub release: Chocolatey's feed is intermittently unreachable from runners. run: | @@ -157,12 +181,12 @@ jobs: "$env:RUNNER_TEMP\protoc\bin" | Out-File -Append -Encoding utf8 $env:GITHUB_PATH - name: Build Windows x86-64 wheel - uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad # v1 + uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad # v1.50.1 with: command: build working-directory: python target: x86_64-pc-windows-msvc - # sccache's Windows server dropped the connection mid-way through compiling lance. + # No sccache: release wheels are built without restoring any cache. sccache: "false" args: --release --locked --strip --out dist @@ -173,7 +197,7 @@ jobs: uvx twine check dist/*.whl sha256sum dist/*.whl - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: python-dist-windows-x64 path: python/dist/*.whl @@ -187,20 +211,24 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-24.04, macos-15, windows-2022] + os: [ubuntu-24.04, ubuntu-24.04-arm, macos-15, windows-2022] python: ["3.12", "3.13", "3.14"] runs-on: ${{ matrix.os }} timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: + persist-credentials: false sparse-checkout: | ci python/python/tests - - uses: astral-sh/setup-uv@v4 + - uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0 + + with: - - uses: actions/download-artifact@v4 + enable-cache: false + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: pattern: python-dist-* merge-multiple: true @@ -213,7 +241,7 @@ jobs: ! command -v cargo && ! command -v rustc # This platform's wheel, by path, so PyPI's copy of the same version can't stand in. case "$RUNNER_OS" in - Linux) wheel=$(ls "$PWD"/dist/*manylinux*.whl) ;; + Linux) wheel=$(ls "$PWD"/dist/*manylinux*_"$(uname -m)".whl) ;; macOS) wheel=$(ls "$PWD"/dist/*macosx*.whl) ;; Windows) wheel=$(cygpath -w "$(ls "$PWD"/dist/*win_amd64*.whl)") ;; esac @@ -238,25 +266,31 @@ jobs: id-token: write contents: read steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: + persist-credentials: false sparse-checkout: python/pyproject.toml - name: Check the release tag matches the package version if: github.event_name == 'release' + env: + TAG: ${{ github.event.release.tag_name }} run: | version=$(grep -m1 '^version = ' python/pyproject.toml | cut -d'"' -f2) - test "${{ github.event.release.tag_name }}" = "v$version" || { - echo "Release tag ${{ github.event.release.tag_name }} does not match version $version"; exit 1; } + test "$TAG" = "v$version" || { + echo "Release tag $TAG does not match version $version"; exit 1; } - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: pattern: python-dist-* merge-multiple: true path: dist - - uses: astral-sh/setup-uv@v4 + - uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0 + + with: + enable-cache: false - name: Publish to PyPI run: uv publish --trusted-publishing always dist/* @@ -274,8 +308,9 @@ jobs: id-token: write contents: read steps: - - uses: actions/checkout@v4 - + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false - name: Install build dependencies run: | sudo apt-get update @@ -285,7 +320,7 @@ jobs: - name: Package and verify the crate run: cargo package --locked -p lance-csr - - uses: rust-lang/crates-io-auth-action@v1 + - uses: rust-lang/crates-io-auth-action@3268eb413133e2960fa4ae0e963ca76380c54cd9 # v1.0.6 id: auth - name: Publish to crates.io diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index a503760..594d7cf 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -26,6 +26,9 @@ env: RUSTFLAGS: "-C debuginfo=line-tables-only" CARGO_INCREMENTAL: "0" +permissions: + contents: read + jobs: test: runs-on: ubuntu-24.04 @@ -34,13 +37,15 @@ jobs: matrix: python-version: ["3.12"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false - name: Setup Rust - uses: actions-rust-lang/setup-rust-toolchain@v1 + uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 with: # The rust-cache step below caches the bindings crate. cache: false - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: save-if: ${{ github.ref == 'refs/heads/main' }} shared-key: "lance-csr-python" @@ -56,7 +61,7 @@ jobs: sudo apt update sudo apt install -y protobuf-compiler - name: Install uv - uses: astral-sh/setup-uv@v4 + uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0 - name: Install locked dependencies working-directory: python # The package itself is built by maturin develop below, in debug mode. @@ -72,9 +77,11 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 15 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@v4 + uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0 - name: Install locked dependencies working-directory: python # Linting needs no build of the Rust extension. diff --git a/.github/workflows/rust-test.yml b/.github/workflows/rust-test.yml index 078c7d8..6b6d2fc 100644 --- a/.github/workflows/rust-test.yml +++ b/.github/workflows/rust-test.yml @@ -25,14 +25,19 @@ env: RUST_BACKTRACE: "1" CARGO_INCREMENTAL: "0" +permissions: + contents: read + jobs: test: runs-on: ubuntu-24.04 timeout-minutes: 30 steps: # rust-toolchain.toml selects the toolchain. - - uses: actions/checkout@v4 - - uses: Swatinem/rust-cache@v2 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: shared-key: "lance-csr-deps" # Pull requests restore main's cache but don't save copies that crowd it out. diff --git a/.github/workflows/spelling.yml b/.github/workflows/spelling.yml index dfc52ba..a4fc864 100644 --- a/.github/workflows/spelling.yml +++ b/.github/workflows/spelling.yml @@ -10,12 +10,17 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +permissions: + contents: read + jobs: typos: name: Spell Check runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false - name: Check spelling - uses: crate-ci/typos@v1.26.0 + uses: crate-ci/typos@6802cc60d4e7f78b9d5454f6cf3935c042d5e1e3 # v1.26.0 \ No newline at end of file diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index 518ebf0..5764b3b 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -23,13 +23,18 @@ env: CARGO_TERM_COLOR: always RUSTFLAGS: "-C debuginfo=1" +permissions: + contents: read + jobs: format: runs-on: ubuntu-24.04 timeout-minutes: 15 steps: - - uses: actions/checkout@v4 - - uses: actions-rust-lang/setup-rust-toolchain@v1 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 with: components: rustfmt # Formatting compiles nothing, so there is nothing to cache. @@ -41,8 +46,10 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 15 steps: - - uses: actions/checkout@v4 - - uses: Swatinem/rust-cache@v2 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: shared-key: "lance-csr-deps" save-if: ${{ github.ref == 'refs/heads/main' }} diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml new file mode 100644 index 0000000..979d14b --- /dev/null +++ b/.github/workflows/zizmor.yml @@ -0,0 +1,38 @@ +# Audit the workflows and Dependabot config with zizmor: unpinned actions, template +# injection, cache poisoning, persisted credentials and broad token permissions. +name: Workflow Audit +on: + push: + branches: + - main + paths: + - .github/** + pull_request: + paths: + - .github/** + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + zizmor: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + + - uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0 + with: + enable-cache: false + + - name: Run zizmor + # The token enables the online audits: impostor commits and known-vulnerable actions. + run: uvx zizmor@1.32.0 --format=github .github + env: + GH_TOKEN: ${{ github.token }} diff --git a/AGENTS.md b/AGENTS.md index f508dad..f8c7438 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,6 +18,7 @@ ## Coding Style & Naming Conventions - Format Rust with `cargo fmt --all`; keep modules and functions snake_case, types PascalCase, and reuse `snafu` error patterns. - Run `cargo clippy --workspace --all-targets -- -D warnings` to catch lint regressions, as CI does. +- Pin every GitHub Action to a full commit SHA with its exact version in a comment (`@ # v4.4.0`); Dependabot updates the pins. The Workflow Audit job runs `zizmor`; run `uvx zizmor@1.32.0 .github` before pushing workflow changes. - Use 4-space indentation in Python; maintain snake_case modules, CamelCase classes, and type-annotated public APIs. - Apply `ruff format python/` before committing; `ruff check` and `pyright` enforce import hygiene and typing. diff --git a/README.md b/README.md index 009bb8d..23121a2 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ case needs distributed graph queries or graph algorithms, that's the kind of eng uv add lance-csr ``` -Prebuilt wheels cover Linux x86-64, native Apple Silicon and Windows x86-64 (CPython 3.12+); no Rust toolchain is needed. On +Prebuilt wheels cover Linux x86-64 and aarch64, native Apple Silicon and Windows x86-64 (CPython 3.12+); no Rust toolchain is needed. On Windows you can also use lance-csr inside WSL2, where it installs the Linux wheel. From Rust, run `cargo add lance-csr` and see the [Rust guide](https://lance-csr.hdclabs.ai/rust/). To build from source, see [`python/DEVELOPMENT.md`](python/DEVELOPMENT.md). diff --git a/RELEASE.md b/RELEASE.md index 2632d61..71b2159 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -64,14 +64,15 @@ Publishing it starts `.github/workflows/python-publish.yml`: | Job | Time | What it does | | --- | ---: | --- | -| `build-linux` | about 22 min | manylinux_2_28 x86-64 wheel and the sdist | +| `build-linux` | about 22 min (x86-64), TBD (aarch64) | manylinux_2_28 x86-64 and aarch64 wheels, each built natively (`ubuntu-24.04`, `ubuntu-24.04-arm`), and the sdist | | `build-macos` | about 35 min | macOS arm64 wheel, built natively on `macos-15` | | `build-windows` | about 52 min | Windows x86-64 wheel, built natively on `windows-2022` | | `test` | under 1 min each | each wheel on Python 3.12, 3.13 and 3.14 | | `publish` | seconds | checks the tag matches `python/pyproject.toml`, then uploads through PyPI trusted publishing | | `publish-crates` | about 10 to 15 min | after `publish`: packages and verifies `lance-csr`, then uploads it through crates.io trusted publishing | -The build jobs run in parallel. Nothing reaches PyPI unless every build and test job passes, and nothing +The build jobs run in parallel and restore no caches (no sccache, no uv cache), so nothing a pull request's run left +behind can end up in a release wheel. Nothing reaches PyPI unless every build and test job passes, and nothing reaches crates.io unless PyPI publishing succeeds. The `pypi` and `crates-io` environments only accept `v*` tags. The PyPI trusted publisher is registered for workflow `python-publish.yml` and environment `pypi`, and the crates.io trusted publisher for `lance-csr` for the same workflow and environment `crates-io`, so don't diff --git a/docs/getting-started.md b/docs/getting-started.md index 3c3185e..87c31f4 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -25,7 +25,7 @@ lance-csr is published on PyPI as `lance-csr`. Use [uv](https://docs.astral.sh/u uv add lance-csr --no-build-package lance-csr ``` -Prebuilt wheels cover **Linux x86-64** (glibc 2.28+: RHEL 8, Ubuntu 20.04 and later), **Apple Silicon** +Prebuilt wheels cover **Linux x86-64 and aarch64** (glibc 2.28+: RHEL 8, Ubuntu 20.04 and later), **Apple Silicon** (macOS 11+) and **Windows x86-64**, for **CPython 3.12, 3.13 and 3.14**. Installing a wheel needs no Rust toolchain. On Windows you can also use lance-csr inside WSL2, where it installs the Linux wheel. Other platforms build from the source distribution, which needs Rust and `protoc` (see [Contributing](contributing.md)). diff --git a/docs/index.md b/docs/index.md index 973ed6a..1b3886e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -152,8 +152,8 @@ engine we're building at HDC Labs. [Reach out to us](https://hyperdimensionalcom more about that work. lance-csr is in **alpha**. The CSR (`use_csr=True`) is marked experimental: results are identical with and -without it, but the API around it may change. Prebuilt wheels cover Linux x86-64 and Apple Silicon for CPython -3.12 and newer. +without it, but the API around it may change. Prebuilt wheels cover Linux x86-64 and aarch64, Apple Silicon and Windows x86-64 for +CPython 3.12 and newer. [:fontawesome-brands-github: GitHub](https://github.com/hyperdimensionalcomputingai/lance-csr){ .md-button } [:fontawesome-brands-python: PyPI](https://pypi.org/project/lance-csr/){ .md-button } diff --git a/python/README.md b/python/README.md index 9beab6f..fe76637 100644 --- a/python/README.md +++ b/python/README.md @@ -18,7 +18,7 @@ Or run a script with it, without a project: uv run --with lance-csr python your_script.py ``` -Prebuilt wheels are published for Linux x86-64 (glibc 2.28 or newer: RHEL 8, Ubuntu 20.04 and later), +Prebuilt wheels are published for Linux x86-64 and aarch64 (glibc 2.28 or newer: RHEL 8, Ubuntu 20.04 and later), native Apple Silicon (macOS 11 or newer) and Windows x86-64, for CPython 3.12 and later. The x86-64 wheels need a CPU with AVX2 (Intel Haswell, AMD Excavator or newer). Installing from a wheel needs no Rust toolchain; to make sure uv never falls back to compiling from source, use: @@ -27,7 +27,7 @@ no Rust toolchain; to make sure uv never falls back to compiling from source, us uv add lance-csr --no-build-package lance-csr ``` -On Windows you can also use lance-csr inside WSL2, where it installs the Linux x86-64 wheel. Other +On Windows you can also use lance-csr inside WSL2, where it installs the Linux wheel. Other platforms install from the source distribution, which needs Rust and `protoc`. ## Quick Start @@ -207,7 +207,7 @@ r2 = engine.execute( ## Requirements -- Python 3.12 or newer (3.12, 3.13 and 3.14 tested); macOS arm64, Linux x86-64 or Windows x86-64 (natively or in WSL2) +- Python 3.12 or newer (3.12, 3.13 and 3.14 tested); macOS arm64, Linux x86-64 or aarch64, or Windows x86-64 (natively or in WSL2) - `pylance` 13.x for writing Lance datasets ## Contributing From 86386bb10d1ac5661f950e2f73aab0b74e21a65f Mon Sep 17 00:00:00 2001 From: prrao87 Date: Sat, 7 Nov 2026 10:39:42 -0500 Subject: [PATCH 3/3] docs: define CSR in the README and say Windows has its own wheel Explain what a CSR is, where it comes from and which graph databases use it, before the README relies on the term. Rename "Much wider Cypher" to "Broader openCypher support". Say that Windows installs the native wheel and doesn't need WSL2. --- README.md | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 23121a2..79a6828 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,12 @@ Apache DataFusion, with Python bindings. It is a fork of [lance-graph](https://g and a research test bed: it pushes the limits of CSR indexes and DataFusion on top of the Lance format, with the goal of making multi-hop path traversal fast. +A CSR (compressed sparse row) is a compact adjacency format from the sparse-matrix literature. Each node's +neighbours sit in one contiguous slice of an array, and a second array of offsets says where each slice starts, so +following a relationship is an array lookup instead of a join. Embedded graph databases such as +[Kùzu](https://github.com/kuzudb/kuzu) and [LadybugDB](https://github.com/LadybugDB/ladybug) store their +relationships this way; lance-csr builds the same structure over relationship tables stored in Lance. + **Documentation:** Visit the [lance-csr documentation site](https://lance-csr.hdclabs.ai/) for getting started, Cypher guides, the CSR index, the Python API reference, and benchmarks. @@ -47,9 +53,9 @@ case needs distributed graph queries or graph algorithms, that's the kind of eng joins, updated incrementally as the Lance datasets change. See [the CSR index](https://lance-csr.hdclabs.ai/csr/). - **A Lance-native engine** that reads each table lazily from a pinned Lance snapshot or holds it in memory, and `checkout_latest()`es to new versions. -- **Much wider Cypher**, enough to run the Graph Data Council's Interactive complex queries: `OPTIONAL MATCH`, - pattern predicates, chained `WITH`, `UNWIND`, list and pattern comprehensions, `shortestPath`, `$parameters` - and more. +- **Broader openCypher support**, enough to run the Graph Data Council's Interactive complex queries: + `OPTIONAL MATCH`, pattern predicates, chained `WITH`, `UNWIND`, list and pattern comprehensions, + `shortestPath`, `$parameters` and more. - **Correctness fixes** to behaviour inherited from lance-graph, each with a regression test, and current dependencies (Lance 13, DataFusion 54, Arrow 58). @@ -59,9 +65,11 @@ case needs distributed graph queries or graph algorithms, that's the kind of eng uv add lance-csr ``` -Prebuilt wheels cover Linux x86-64 and aarch64, native Apple Silicon and Windows x86-64 (CPython 3.12+); no Rust toolchain is needed. On -Windows you can also use lance-csr inside WSL2, where it installs the Linux wheel. From Rust, run -`cargo add lance-csr` and see the [Rust guide](https://lance-csr.hdclabs.ai/rust/). To build from source, see [`python/DEVELOPMENT.md`](python/DEVELOPMENT.md). +Prebuilt wheels cover Linux x86-64 and aarch64, native Apple Silicon and Windows x86-64 (CPython 3.12+), so no +Rust toolchain is needed. On Windows, the same command installs the native Windows wheel; you don't need WSL2, +though lance-csr works there too, with the Linux wheel. From Rust, run `cargo add lance-csr` and see the +[Rust guide](https://lance-csr.hdclabs.ai/rust/). To build from source, see +[`python/DEVELOPMENT.md`](python/DEVELOPMENT.md). ## Example