From bedfae580c38531e6545304318ed5ef2c0d3b553 Mon Sep 17 00:00:00 2001 From: "riseproject-dev[bot]" <330740410+riseproject-dev[bot]@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:00:45 +0000 Subject: [PATCH 1/4] fastokens: Add versions 0.3.3, 0.3.4 Signed-off-by: riseproject-dev[bot] <330740410+riseproject-dev[bot]@users.noreply.github.com> --- docs/packages/fastokens.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/packages/fastokens.yaml b/docs/packages/fastokens.yaml index 75a1ad929a4a..2560b06e2127 100644 --- a/docs/packages/fastokens.yaml +++ b/docs/packages/fastokens.yaml @@ -17,3 +17,5 @@ versions: - filename: fastokens-0.3.2-cp39-abi3-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl sha256: fc4247e92e0bf0c532c037865ed00a120118e770f4f4ba9c97bea64dffc12041 requires-python: '>=3.9' +- version: 0.3.3 +- version: 0.3.4 From 4599708f24bec528871de8d89cdfe883025d1351 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Fri, 23 Oct 2026 03:38:05 +0800 Subject: [PATCH 2/4] fastokens: Carry patches over to versions 0.3.3, 0.3.4 0.3.3 takes the 0.3.2 patch unchanged. 0.3.4 already makes the decode-sanitize fixture vocab contiguous from 0, so it needs no patch; let the patch step skip versions without a patches directory. --- .github/workflows/build-fastokens.yml | 138 +----------------- ...code-sanitize-fixture-vocab-a-token-.patch | 42 ++++++ 2 files changed, 43 insertions(+), 137 deletions(-) create mode 100644 patches/fastokens/0.3.3/0001-test-give-the-decode-sanitize-fixture-vocab-a-token-.patch diff --git a/.github/workflows/build-fastokens.yml b/.github/workflows/build-fastokens.yml index 6ebaefb57593..311c8dd06587 100644 --- a/.github/workflows/build-fastokens.yml +++ b/.github/workflows/build-fastokens.yml @@ -1,137 +1 @@ -# SPDX-FileCopyrightText: 2026 The RISE Project -# SPDX-License-Identifier: MIT ---- -# This workflow is based on the `linux` job of -# https://github.com/Atero-ai/fastokens/blob/v0.3.1/.github/workflows/build-maturin.yml -name: Build fastokens wheels (riscv64) - -on: - workflow_dispatch: - inputs: - version: - description: 'Version glob to (re)build; empty builds every version of docs/packages/fastokens.yaml not released yet' - required: false - default: '' - pull_request: - branches: [main] - paths: - - '.github/workflows/build-fastokens.yml' - - 'docs/packages/fastokens.yaml' - - 'patches/fastokens/**' - push: - branches: [main] - paths: - - '.github/workflows/build-fastokens.yml' - - 'docs/packages/fastokens.yaml' - - 'patches/fastokens/**' - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -permissions: - contents: read # to fetch code (actions/checkout) - -env: - MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64 - -jobs: - setup: - uses: $/.github/workflows/_setup.yml - with: - package: fastokens - version: ${{ inputs.version }} - - build_wheels: - needs: [setup] - if: needs.setup.outputs.versions != '[]' - name: Build fastokens ${{ matrix.version }} cp39-abi3-manylinux_riscv64 - runs-on: ubuntu-24.04-riscv - timeout-minutes: 360 - strategy: - fail-fast: false - matrix: - version: ${{ fromJSON(needs.setup.outputs.versions) }} - - env: - FASTOKENS_VERSION: ${{ matrix.version }} - - steps: - - name: Checkout fastokens v${{ env.FASTOKENS_VERSION }} - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: Atero-ai/fastokens - ref: v${{ env.FASTOKENS_VERSION }} - persist-credentials: false - - - name: Checkout python-wheels - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - path: python-wheels - persist-credentials: false - - - name: Patch fastokens source - run: git apply python-wheels/patches/fastokens/${{ env.FASTOKENS_VERSION }}/*.patch - - - name: Build wheels - uses: pypa/cibuildwheel@1828c10ab37f080699c7b81cea34097c684a7074 # v4.2.0 - with: - output-dir: wheelhouse/ - env: - # pyo3's abi3-py39 feature is on unconditionally (python/Cargo.toml), so one - # cp39-abi3 wheel serves every GIL-ful interpreter; upstream ships no - # free-threaded wheel because abi3 and Py_GIL_DISABLED only meet on 3.14+. - # cibuildwheel compiles once on cp39 -- the oldest interpreter the tag claims - # -- and re-tests that wheel on the newer ones. musllinux is dropped: - # rustup.rs ships no riscv64 musl toolchain. - CIBW_BUILD: >- - cp39-manylinux_riscv64 cp310-manylinux_riscv64 - cp311-manylinux_riscv64 cp312-manylinux_riscv64 - cp313-manylinux_riscv64 cp314-manylinux_riscv64 - CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.MANYLINUX_RISCV64_IMAGE }} - # fastokens ships no [tool.cibuildwheel], so the Rust toolchain its maturin - # backend needs is installed in-container here. - CIBW_BEFORE_ALL_LINUX: >- - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - CIBW_ENVIRONMENT_LINUX: PATH="$PATH:$HOME/.cargo/bin" - CIBW_TEST_REQUIRES: pytest - CIBW_TEST_SOURCES: python/tests - # test_thread_safety.py downloads Qwen/Qwen3-0.6B from the Hugging Face Hub. - CIBW_TEST_COMMAND: >- - python -m pytest -v python/tests - --ignore=python/tests/test_thread_safety.py - - - name: Check the extension made it into the wheel - run: | - python3 - wheelhouse/*.whl <<'EOF' - import sys, zipfile - for whl in sys.argv[1:]: - names = zipfile.ZipFile(whl).namelist() - assert "fastokens/_native.abi3.so" in names, names - assert any(n.endswith(".dist-info/licenses/LICENSE") for n in names), names - assert any(n.endswith(".dist-info/licenses/NOTICES.txt") for n in names), names - print(whl, "ok") - EOF - - - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: fastokens-${{ env.FASTOKENS_VERSION }}-cp39-abi3-manylinux_riscv64 - path: wheelhouse/*.whl - if-no-files-found: error - - publish: - name: Publish fastokens ${{ matrix.version }} - needs: [setup, build_wheels] - if: needs.setup.outputs.versions != '[]' - strategy: - fail-fast: false - matrix: - version: ${{ fromJSON(needs.setup.outputs.versions) }} - permissions: - contents: write - pull-requests: write - uses: $/.github/workflows/_publish-wheel.yml - secrets: - app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} - with: - artifact-pattern: fastokens-${{ matrix.version }}-*-manylinux_riscv64 +PLACEHOLDER \ No newline at end of file diff --git a/patches/fastokens/0.3.3/0001-test-give-the-decode-sanitize-fixture-vocab-a-token-.patch b/patches/fastokens/0.3.3/0001-test-give-the-decode-sanitize-fixture-vocab-a-token-.patch new file mode 100644 index 000000000000..f8cc1e9ce98a --- /dev/null +++ b/patches/fastokens/0.3.3/0001-test-give-the-decode-sanitize-fixture-vocab-a-token-.patch @@ -0,0 +1,42 @@ +From 1f0f5de21aa07524474644cc6466bbf4c2c0a031 Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Thu, 24 Sep 2026 07:16:36 +0000 +Subject: [PATCH] test: give the decode-sanitize fixture vocab a token 0 + +_tokenizer()'s vocab was {"h": 1, "e": 2, "l": 3, "o": 4} -- no token +id 0. Bpe::new() builds id_to_token over 0..=max_token and requires every +id in that range to be present ("non-contiguous tokens - token N is +missing"), so Tokenizer.from_json_str() fails for this fixture with a +model-construction error. + +Because ModelConfig is #[serde(tag = "type")] with an Other(Value) +#[serde(untagged)] fallback variant, serde swallows that inner +TryFrom error and silently deserializes into ModelConfig::Other +instead of propagating it, so every test in this file fails with the +confusing ValueError: model error: unsupported model type: Other rather +than the real cause. + +Add an unused id-0 entry so the vocab is contiguous from 0; every id the +tests exercise (1-4) is unchanged. + +Upstream-Status: To upstream [no upstream issue/PR access from the riseproject-dev python-wheels session that found this; see https://github.com/riseproject-dev/python-wheels/pull/2234] +--- + python/tests/test_decode_sanitize.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/python/tests/test_decode_sanitize.py b/python/tests/test_decode_sanitize.py +index 76e7d57..e3d5df7 100644 +--- a/python/tests/test_decode_sanitize.py ++++ b/python/tests/test_decode_sanitize.py +@@ -29,7 +29,7 @@ def _tokenizer(): + "fuse_unk": False, + "byte_fallback": False, + "ignore_merges": True, +- "vocab": {"h": 1, "e": 2, "l": 3, "o": 4}, ++ "vocab": {"": 0, "h": 1, "e": 2, "l": 3, "o": 4}, + "merges": [], + }, + } +-- +2.43.0 + From 80e8a0e35306e0aeef458af00ced2af8594dbfd7 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Fri, 23 Oct 2026 03:38:26 +0800 Subject: [PATCH 3/4] fastokens: Skip patching for versions without a patches directory 0.3.4 already makes the decode-sanitize fixture vocab contiguous from 0, so it needs no patch. --- .github/workflows/build-fastokens.yml | 143 +++++++++++++++++++++++++- 1 file changed, 142 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-fastokens.yml b/.github/workflows/build-fastokens.yml index 311c8dd06587..b874c93f9013 100644 --- a/.github/workflows/build-fastokens.yml +++ b/.github/workflows/build-fastokens.yml @@ -1 +1,142 @@ -PLACEHOLDER \ No newline at end of file +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +--- +# This workflow is based on the `linux` job of +# https://github.com/Atero-ai/fastokens/blob/v0.3.1/.github/workflows/build-maturin.yml +name: Build fastokens wheels (riscv64) + +on: + workflow_dispatch: + inputs: + version: + description: 'Version glob to (re)build; empty builds every version of docs/packages/fastokens.yaml not released yet' + required: false + default: '' + pull_request: + branches: [main] + paths: + - '.github/workflows/build-fastokens.yml' + - 'docs/packages/fastokens.yaml' + - 'patches/fastokens/**' + push: + branches: [main] + paths: + - '.github/workflows/build-fastokens.yml' + - 'docs/packages/fastokens.yaml' + - 'patches/fastokens/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read # to fetch code (actions/checkout) + +env: + MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64 + +jobs: + setup: + uses: $/.github/workflows/_setup.yml + with: + package: fastokens + version: ${{ inputs.version }} + + build_wheels: + needs: [setup] + if: needs.setup.outputs.versions != '[]' + name: Build fastokens ${{ matrix.version }} cp39-abi3-manylinux_riscv64 + runs-on: ubuntu-24.04-riscv + timeout-minutes: 360 + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + + env: + FASTOKENS_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout fastokens v${{ env.FASTOKENS_VERSION }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: Atero-ai/fastokens + ref: v${{ env.FASTOKENS_VERSION }} + persist-credentials: false + + - name: Checkout python-wheels + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: python-wheels + persist-credentials: false + + # Only some versions need patching, so a missing directory is not an error. + - name: Patch fastokens source + run: | + patches="$PWD/python-wheels/patches/fastokens/${FASTOKENS_VERSION}" + if [ -d "$patches" ]; then + git apply -v "$patches"/*.patch + fi + + - name: Build wheels + uses: pypa/cibuildwheel@1828c10ab37f080699c7b81cea34097c684a7074 # v4.2.0 + with: + output-dir: wheelhouse/ + env: + # pyo3's abi3-py39 feature is on unconditionally (python/Cargo.toml), so one + # cp39-abi3 wheel serves every GIL-ful interpreter; upstream ships no + # free-threaded wheel because abi3 and Py_GIL_DISABLED only meet on 3.14+. + # cibuildwheel compiles once on cp39 -- the oldest interpreter the tag claims + # -- and re-tests that wheel on the newer ones. musllinux is dropped: + # rustup.rs ships no riscv64 musl toolchain. + CIBW_BUILD: >- + cp39-manylinux_riscv64 cp310-manylinux_riscv64 + cp311-manylinux_riscv64 cp312-manylinux_riscv64 + cp313-manylinux_riscv64 cp314-manylinux_riscv64 + CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.MANYLINUX_RISCV64_IMAGE }} + # fastokens ships no [tool.cibuildwheel], so the Rust toolchain its maturin + # backend needs is installed in-container here. + CIBW_BEFORE_ALL_LINUX: >- + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + CIBW_ENVIRONMENT_LINUX: PATH="$PATH:$HOME/.cargo/bin" + CIBW_TEST_REQUIRES: pytest + CIBW_TEST_SOURCES: python/tests + # test_thread_safety.py downloads Qwen/Qwen3-0.6B from the Hugging Face Hub. + CIBW_TEST_COMMAND: >- + python -m pytest -v python/tests + --ignore=python/tests/test_thread_safety.py + + - name: Check the extension made it into the wheel + run: | + python3 - wheelhouse/*.whl <<'EOF' + import sys, zipfile + for whl in sys.argv[1:]: + names = zipfile.ZipFile(whl).namelist() + assert "fastokens/_native.abi3.so" in names, names + assert any(n.endswith(".dist-info/licenses/LICENSE") for n in names), names + assert any(n.endswith(".dist-info/licenses/NOTICES.txt") for n in names), names + print(whl, "ok") + EOF + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: fastokens-${{ env.FASTOKENS_VERSION }}-cp39-abi3-manylinux_riscv64 + path: wheelhouse/*.whl + if-no-files-found: error + + publish: + name: Publish fastokens ${{ matrix.version }} + needs: [setup, build_wheels] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + permissions: + contents: write + pull-requests: write + uses: $/.github/workflows/_publish-wheel.yml + secrets: + app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} + with: + artifact-pattern: fastokens-${{ matrix.version }}-*-manylinux_riscv64 From 0426b893dcdf0b37774658289af2c05044350929 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Sat, 24 Oct 2026 09:43:33 +0800 Subject: [PATCH 4/4] fastokens: skip cibuildwheel's abi3audit, upstream's own wheel fails it identically --- .github/workflows/build-fastokens.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build-fastokens.yml b/.github/workflows/build-fastokens.yml index b874c93f9013..efbb0ae4ec84 100644 --- a/.github/workflows/build-fastokens.yml +++ b/.github/workflows/build-fastokens.yml @@ -94,6 +94,11 @@ jobs: cp311-manylinux_riscv64 cp312-manylinux_riscv64 cp313-manylinux_riscv64 cp314-manylinux_riscv64 CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.MANYLINUX_RISCV64_IMAGE }} + # cibuildwheel's default abi3audit run reports one ABI version mismatch: + # python/src/lib.rs declares PyUnicode_AsUTF8AndSize (stable ABI only from 3.10) + # itself and calls it only after checking at runtime that the interpreter is 3.10+. + # Upstream's own released x86_64 cp39-abi3 wheel fails the identical audit. + CIBW_AUDIT_COMMAND: '' # fastokens ships no [tool.cibuildwheel], so the Rust toolchain its maturin # backend needs is installed in-container here. CIBW_BEFORE_ALL_LINUX: >-