From 63dbb74f2956e0c84a9e261ad5e172311b89deff Mon Sep 17 00:00:00 2001 From: Bruno Verachten Date: Wed, 15 Jul 2026 14:37:12 +0200 Subject: [PATCH 1/5] sentencepiece: build wheels for riscv64 Refs #164 Signed-off-by: Bruno Verachten --- .github/workflows/build-sentencepiece.yml | 101 ++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 .github/workflows/build-sentencepiece.yml diff --git a/.github/workflows/build-sentencepiece.yml b/.github/workflows/build-sentencepiece.yml new file mode 100644 index 0000000..bf9b575 --- /dev/null +++ b/.github/workflows/build-sentencepiece.yml @@ -0,0 +1,101 @@ +--- +name: Build sentencepiece wheels (riscv64) + +on: + workflow_dispatch: + inputs: + version: + description: 'sentencepiece version to build (git tag without leading v, e.g. 0.2.2)' + required: true + default: '0.2.2' + pull_request: + paths: + - '.github/workflows/build-sentencepiece.yml' + - 'actions/publish-to-gitlab/**' + +concurrency: + group: ${{ github.workflow }}-${{ inputs.version || '0.2.2' }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read # to fetch code (actions/checkout) + +env: + SENTENCEPIECE_VERSION: ${{ inputs.version || '0.2.2' }} + UV_EXTRA_INDEX_URL: https://pypi.riseproject.dev/simple/ + UV_INDEX_STRATEGY: unsafe-best-match + UV_ONLY_BINARY: ':all:' + +jobs: + build_wheels: + name: Build sentencepiece ${{ inputs.version || '0.2.2' }} ${{ matrix.python }}-manylinux_riscv64 + runs-on: ubuntu-24.04-riscv + strategy: + fail-fast: false + matrix: + python: ["cp312", "cp313", "cp314", "cp314t"] + + steps: + # sentencepiece's Python packaging lives in the `python/` subdirectory + # (setup.py + pyproject.toml there), so cibuildwheel is pointed at it via + # package-dir. The C++ core is vendored through a submodule, hence + # submodules: true. + - name: Checkout sentencepiece v${{ env.SENTENCEPIECE_VERSION }} + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: google/sentencepiece + ref: v${{ env.SENTENCEPIECE_VERSION }} + submodules: true + persist-credentials: false + + - name: Build wheels + uses: pypa/cibuildwheel@294735312765b09d24a2fbec22660ce817587d55 # v4.1.0 + with: + package-dir: python + env: + CIBW_BUILD: ${{ matrix.python }}-manylinux_riscv64 + CIBW_BUILD_VERBOSITY: 1 + CIBW_ENVIRONMENT: CMAKE_BUILD_PARALLEL_LEVEL=8 + # test-sources are resolved relative to the checkout root, so the + # path is prefixed with python/. gen_stubs_test imports a repo-level + # helper module that is not part of the copied test tree, and the + # manual cleanup script is not a pytest module; both are skipped. + CIBW_TEST_SOURCES: python/test + CIBW_TEST_COMMAND: >- + pytest -v python/test + --ignore=python/test/gen_stubs_test.py + --ignore=python/test/clean_sentencepiece_test_manual.py + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: sentencepiece-${{ env.SENTENCEPIECE_VERSION }}-${{ matrix.python }}-manylinux_riscv64 + path: ./wheelhouse/*.whl + if-no-files-found: error + + publish: + name: Publish sentencepiece ${{ inputs.version || '0.2.2' }} to GitLab + needs: build_wheels + # Only publish when the workflow was triggered from main with a specific + # version. Manual trigger is the only entry point, so checking the ref is + # enough to gate uploads. + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + permissions: + contents: read + + steps: + - name: Download wheels + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: sentencepiece-${{ env.SENTENCEPIECE_VERSION }}-*-manylinux_riscv64 + path: dist + merge-multiple: true + + - name: Publish to GitLab PyPI registry + uses: riseproject-dev/python-wheels/actions/publish-to-gitlab@main + with: + gitlab-username: ${{ vars.GITLAB_DEPLOY_USER }} + gitlab-token: ${{ secrets.GITLAB_DEPLOY_TOKEN }} + gitlab-project-id: ${{ vars.GITLAB_PROJECT_ID }} + files: | + dist/*.whl From 7e2fa6267ad953f1328c8bd6d4dd46477b7ac45f Mon Sep 17 00:00:00 2001 From: Bruno Verachten Date: Wed, 15 Jul 2026 14:52:45 +0200 Subject: [PATCH 2/5] sentencepiece: also build cp311 wheels Match the documented default matrix (3.11 through 3.14t); numpy's 3.12+ floor is the exception, not the rule. Signed-off-by: Bruno Verachten --- .github/workflows/build-sentencepiece.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-sentencepiece.yml b/.github/workflows/build-sentencepiece.yml index bf9b575..3c1e7a5 100644 --- a/.github/workflows/build-sentencepiece.yml +++ b/.github/workflows/build-sentencepiece.yml @@ -33,7 +33,7 @@ jobs: strategy: fail-fast: false matrix: - python: ["cp312", "cp313", "cp314", "cp314t"] + python: ["cp311", "cp312", "cp313", "cp314", "cp314t"] steps: # sentencepiece's Python packaging lives in the `python/` subdirectory From 99153dfe5ca1254929ec39e3eccd43834786ecd9 Mon Sep 17 00:00:00 2001 From: Bruno Verachten Date: Wed, 15 Jul 2026 14:59:27 +0200 Subject: [PATCH 3/5] sentencepiece: run tests closer to upstream Copy python/tools alongside python/test so gen_stubs_test can import gen_stubs, and drop the ad-hoc pytest ignores to mirror upstream's 'pytest -v {project}/test'. Signed-off-by: Bruno Verachten --- .github/workflows/build-sentencepiece.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-sentencepiece.yml b/.github/workflows/build-sentencepiece.yml index 3c1e7a5..24b2313 100644 --- a/.github/workflows/build-sentencepiece.yml +++ b/.github/workflows/build-sentencepiece.yml @@ -56,15 +56,14 @@ jobs: CIBW_BUILD: ${{ matrix.python }}-manylinux_riscv64 CIBW_BUILD_VERBOSITY: 1 CIBW_ENVIRONMENT: CMAKE_BUILD_PARALLEL_LEVEL=8 - # test-sources are resolved relative to the checkout root, so the - # path is prefixed with python/. gen_stubs_test imports a repo-level - # helper module that is not part of the copied test tree, and the - # manual cleanup script is not a pytest module; both are skipped. - CIBW_TEST_SOURCES: python/test - CIBW_TEST_COMMAND: >- - pytest -v python/test - --ignore=python/test/gen_stubs_test.py - --ignore=python/test/clean_sentencepiece_test_manual.py + # Mirror upstream's `test-command = "pytest -v {project}/test"`. + # test-sources are resolved relative to the checkout root (hence the + # python/ prefix); tools/ is copied alongside test/ because + # gen_stubs_test inserts ../tools on sys.path to import gen_stubs. + # test-requires (pytest, numpy, protobuf) come from the project's own + # [tool.cibuildwheel] table in python/pyproject.toml. + CIBW_TEST_SOURCES: python/test python/tools + CIBW_TEST_COMMAND: pytest -v python/test - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: From 56ffa0ab224dd4d1f0cc30c0f98e9f22391ec773 Mon Sep 17 00:00:00 2001 From: Bruno Verachten Date: Tue, 28 Jul 2026 15:02:53 +0200 Subject: [PATCH 4/5] sentencepiece: rebuild workflow from upstream wheel.yml Reshape the workflow after google/sentencepiece's own .github/workflows/wheel.yml instead of build-numpy.yml, and follow the current development guide. Build side: run cibuildwheel from the python/ directory the way upstream does, including the data/*.bin copy into src/sentencepiece/package_data that the tests rely on, and pin cibuildwheel to 3.4.0, the version upstream pins in requirements/cibuildwheel.txt. That makes the project's own [tool.cibuildwheel] table usable as-is, so the CIBW_TEST_SOURCES and CIBW_TEST_COMMAND overrides are no longer needed. Upstream's hash-pinned requirements files are not reused since they also pin twine's dependency chain, which would be compiled from source on the runner. Add upstream's free-threading test job, running against 3.14t rather than 3.13t, and pin the manylinux_riscv64 image so the in-container test dependencies resolve against our registry rather than building numpy from an sdist. Trim the matrix to the default 3.12 to 3.14t set, and switch publishing to the publish-wheels action with the permissions it needs, dropping the branch check that the dry-run logic now handles. Signed-off-by: Bruno Verachten --- .github/workflows/build-sentencepiece.yml | 123 +++++++++++++++------- 1 file changed, 87 insertions(+), 36 deletions(-) diff --git a/.github/workflows/build-sentencepiece.yml b/.github/workflows/build-sentencepiece.yml index 24b2313..3816d31 100644 --- a/.github/workflows/build-sentencepiece.yml +++ b/.github/workflows/build-sentencepiece.yml @@ -11,7 +11,6 @@ on: pull_request: paths: - '.github/workflows/build-sentencepiece.yml' - - 'actions/publish-to-gitlab/**' concurrency: group: ${{ github.workflow }}-${{ inputs.version || '0.2.2' }}-${{ github.head_ref || github.run_id }} @@ -25,6 +24,9 @@ env: UV_EXTRA_INDEX_URL: https://pypi.riseproject.dev/simple/ UV_INDEX_STRATEGY: unsafe-best-match UV_ONLY_BINARY: ':all:' + MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64 + # The version upstream pins in .github/workflows/requirements/cibuildwheel.txt + CIBUILDWHEEL_VERSION: '3.4.0' jobs: build_wheels: @@ -33,13 +35,9 @@ jobs: strategy: fail-fast: false matrix: - python: ["cp311", "cp312", "cp313", "cp314", "cp314t"] + python: ["cp312", "cp313", "cp314", "cp314t"] steps: - # sentencepiece's Python packaging lives in the `python/` subdirectory - # (setup.py + pyproject.toml there), so cibuildwheel is pointed at it via - # package-dir. The C++ core is vendored through a submodule, hence - # submodules: true. - name: Checkout sentencepiece v${{ env.SENTENCEPIECE_VERSION }} uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: @@ -48,53 +46,106 @@ jobs: submodules: true persist-credentials: false - - name: Build wheels - uses: pypa/cibuildwheel@294735312765b09d24a2fbec22660ce817587d55 # v4.1.0 + - name: Install Python + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: - package-dir: python + python-version: '3.12' + activate-environment: true + enable-cache: false + + # Upstream installs from requirements/base.txt and requirements/cibuildwheel.txt + # with --require-hashes. Those files also pin twine and its dependency chain + # (cffi, cryptography), which we would have to build from source here, so we + # install cibuildwheel on its own at the version upstream pins. + - name: Install cibuildwheel + run: uv pip install "cibuildwheel==${{ env.CIBUILDWHEEL_VERSION }}" + + # Copying data/*.bin into the package is upstream's step, and the tests need + # it. Build and test settings come from [tool.cibuildwheel] in + # python/pyproject.toml (test-requires pytest/numpy/protobuf, test-command + # "pytest -v {project}/test"), so only the riscv64-specific parts are set + # here. CIBW_BUILD pins one target, which is what upstream's CIBW_ARCHS_LINUX + # and CIBW_SKIP work out to for the Linux half of their matrix. + - name: Build wheels + working-directory: python + run: | + mkdir -p src/sentencepiece/package_data + cp ../data/*.bin src/sentencepiece/package_data + python -m cibuildwheel --output-dir wheelhouse env: CIBW_BUILD: ${{ matrix.python }}-manylinux_riscv64 + CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.MANYLINUX_RISCV64_IMAGE }} CIBW_BUILD_VERBOSITY: 1 - CIBW_ENVIRONMENT: CMAKE_BUILD_PARALLEL_LEVEL=8 - # Mirror upstream's `test-command = "pytest -v {project}/test"`. - # test-sources are resolved relative to the checkout root (hence the - # python/ prefix); tools/ is copied alongside test/ because - # gen_stubs_test inserts ../tools on sys.path to import gen_stubs. - # test-requires (pytest, numpy, protobuf) come from the project's own - # [tool.cibuildwheel] table in python/pyproject.toml. - CIBW_TEST_SOURCES: python/test python/tools - CIBW_TEST_COMMAND: pytest -v python/test + CIBW_ENABLE: cpython-freethreading + # PIP_EXTRA_INDEX_URL covers the test-requires install inside the + # container, where numpy has riscv64 wheels in our registry but not on + # PyPI. + CIBW_ENVIRONMENT: >- + CMAKE_BUILD_PARALLEL_LEVEL=8 + PIP_EXTRA_INDEX_URL=https://pypi.riseproject.dev/simple/ - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: sentencepiece-${{ env.SENTENCEPIECE_VERSION }}-${{ matrix.python }}-manylinux_riscv64 - path: ./wheelhouse/*.whl + path: ./python/wheelhouse/*.whl if-no-files-found: error + # Upstream runs this job against 3.13t. We use 3.14t, the freethreaded version in + # the python-wheels default matrix. + free-threading: + name: Test sentencepiece ${{ inputs.version || '0.2.2' }} free-threaded + needs: [build_wheels] + runs-on: ubuntu-24.04-riscv + steps: + - name: Checkout sentencepiece v${{ env.SENTENCEPIECE_VERSION }} tests + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: google/sentencepiece + ref: v${{ env.SENTENCEPIECE_VERSION }} + persist-credentials: false + sparse-checkout: | + python/test + python/tools + data/botchan.txt + + - name: Install Python + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + with: + python-version: '3.14t' + activate-environment: true + enable-cache: false + + - name: Download free-threaded wheel + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: sentencepiece-${{ env.SENTENCEPIECE_VERSION }}-cp314t-manylinux_riscv64 + path: python/wheelhouse + merge-multiple: true + + - name: Install sentencepiece wheel + run: uv pip install --find-links=python/wheelhouse sentencepiece + + - name: Install test dependencies + run: uv pip install "sentencepiece[test]" pytest-run-parallel --find-links=python/wheelhouse + + - name: Run free-threading tests + working-directory: python + run: pytest -v --parallel-threads 4 + publish: name: Publish sentencepiece ${{ inputs.version || '0.2.2' }} to GitLab - needs: build_wheels - # Only publish when the workflow was triggered from main with a specific - # version. Manual trigger is the only entry point, so checking the ref is - # enough to gate uploads. - if: github.ref == 'refs/heads/main' + needs: [build_wheels, free-threading] runs-on: ubuntu-latest permissions: - contents: read + contents: write + pull-requests: write steps: - - name: Download wheels - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - pattern: sentencepiece-${{ env.SENTENCEPIECE_VERSION }}-*-manylinux_riscv64 - path: dist - merge-multiple: true - - - name: Publish to GitLab PyPI registry - uses: riseproject-dev/python-wheels/actions/publish-to-gitlab@main + - name: Publish wheels and open docs PR + uses: riseproject-dev/python-wheels/actions/publish-wheels@main with: + artifact-pattern: sentencepiece-${{ env.SENTENCEPIECE_VERSION }}-*-manylinux_riscv64 gitlab-username: ${{ vars.GITLAB_DEPLOY_USER }} gitlab-token: ${{ secrets.GITLAB_DEPLOY_TOKEN }} gitlab-project-id: ${{ vars.GITLAB_PROJECT_ID }} - files: | - dist/*.whl + gh-token: ${{ secrets.GITHUB_TOKEN }} From 51770ed9dafe96d63fb86d6f5dab3ddeb15b2919 Mon Sep 17 00:00:00 2001 From: Bruno Verachten Date: Tue, 4 Aug 2026 22:08:16 +0200 Subject: [PATCH 5/5] sentencepiece: select versions with CIBW_SKIP instead of a matrix Replaces the per-version job matrix with a single cibuildwheel invocation, matching upstream's wheel.yml shape so the two are easier to compare. Version selection now lives in CIBW_SKIP, which drops cp39, cp310 and cp311. This serializes the build on one riscv64 runner rather than spreading it across four. The four jobs took 28 to 30 minutes each in parallel, so expect roughly two hours wall instead of forty minutes, in exchange for holding a single runner. Also drops CIBW_ENABLE: cpython-freethreading, which was doing nothing here. cibuildwheel 3.4.0 only gates cp313t behind it, and the build produces cp314t either way. The four per-version artifacts become one, so the free-threading job now downloads that artifact by name and lets uv resolve the cp314t wheel out of it. Signed-off-by: Bruno Verachten --- .github/workflows/build-sentencepiece.yml | 35 +++++++++++++---------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-sentencepiece.yml b/.github/workflows/build-sentencepiece.yml index 3816d31..52c2836 100644 --- a/.github/workflows/build-sentencepiece.yml +++ b/.github/workflows/build-sentencepiece.yml @@ -30,12 +30,8 @@ env: jobs: build_wheels: - name: Build sentencepiece ${{ inputs.version || '0.2.2' }} ${{ matrix.python }}-manylinux_riscv64 + name: Build sentencepiece ${{ inputs.version || '0.2.2' }} manylinux_riscv64 runs-on: ubuntu-24.04-riscv - strategy: - fail-fast: false - matrix: - python: ["cp312", "cp313", "cp314", "cp314t"] steps: - name: Checkout sentencepiece v${{ env.SENTENCEPIECE_VERSION }} @@ -64,8 +60,10 @@ jobs: # it. Build and test settings come from [tool.cibuildwheel] in # python/pyproject.toml (test-requires pytest/numpy/protobuf, test-command # "pytest -v {project}/test"), so only the riscv64-specific parts are set - # here. CIBW_BUILD pins one target, which is what upstream's CIBW_ARCHS_LINUX - # and CIBW_SKIP work out to for the Linux half of their matrix. + # here. This mirrors upstream's single cibuildwheel invocation per + # platform: CIBW_ARCHS_LINUX resolves to riscv64 on this runner, and + # CIBW_SKIP carries the version selection rather than a matrix, so the + # diff against upstream's wheel.yml stays readable. - name: Build wheels working-directory: python run: | @@ -73,10 +71,14 @@ jobs: cp ../data/*.bin src/sentencepiece/package_data python -m cibuildwheel --output-dir wheelhouse env: - CIBW_BUILD: ${{ matrix.python }}-manylinux_riscv64 + CIBW_ARCHS_LINUX: auto + # Upstream skips musllinux and win32. We skip musllinux for the same + # reason, plus cp39/cp310/cp311: those predate riscv64 being a target + # anyone ships for, and building them here would roughly double the + # runtime of an already serialized job for no consumer. + CIBW_SKIP: "cp39-* cp310-* cp311-* *-musllinux_*" CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.MANYLINUX_RISCV64_IMAGE }} CIBW_BUILD_VERBOSITY: 1 - CIBW_ENABLE: cpython-freethreading # PIP_EXTRA_INDEX_URL covers the test-requires install inside the # container, where numpy has riscv64 wheels in our registry but not on # PyPI. @@ -86,12 +88,14 @@ jobs: - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: sentencepiece-${{ env.SENTENCEPIECE_VERSION }}-${{ matrix.python }}-manylinux_riscv64 + name: sentencepiece-${{ env.SENTENCEPIECE_VERSION }}-cibw-wheels-manylinux_riscv64 path: ./python/wheelhouse/*.whl if-no-files-found: error - # Upstream runs this job against 3.13t. We use 3.14t, the freethreaded version in - # the python-wheels default matrix. + # Upstream runs this job against 3.13t. We use 3.14t, the freethreaded + # version in the python-wheels default matrix. cibuildwheel only gates + # cp313t behind CIBW_ENABLE=cpython-freethreading, so cp314t comes out of + # the build job above without any extra opt-in. free-threading: name: Test sentencepiece ${{ inputs.version || '0.2.2' }} free-threaded needs: [build_wheels] @@ -115,12 +119,13 @@ jobs: activate-environment: true enable-cache: false - - name: Download free-threaded wheel + # One artifact now holds every version the build job produced. uv picks the + # cp314t wheel out of it because that is what the interpreter above resolves. + - name: Download wheels uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - pattern: sentencepiece-${{ env.SENTENCEPIECE_VERSION }}-cp314t-manylinux_riscv64 + name: sentencepiece-${{ env.SENTENCEPIECE_VERSION }}-cibw-wheels-manylinux_riscv64 path: python/wheelhouse - merge-multiple: true - name: Install sentencepiece wheel run: uv pip install --find-links=python/wheelhouse sentencepiece