diff --git a/.github/workflows/build-hf-xet.yml b/.github/workflows/build-hf-xet.yml new file mode 100644 index 0000000..6f949c6 --- /dev/null +++ b/.github/workflows/build-hf-xet.yml @@ -0,0 +1,143 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT + +name: Build hf-xet wheels (riscv64) + +on: + workflow_dispatch: + inputs: + version: + description: 'hf-xet version to build (git tag without leading v, e.g. 1.6.0)' + required: true + default: '1.6.0' + pull_request: + paths: + - '.github/workflows/build-hf-xet.yml' + - '.github/workflows/test-hf-xet.yml' + +concurrency: + group: ${{ github.workflow }}-${{ inputs.version || '1.6.0' }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read # to fetch code (actions/checkout) + +env: + HFXET_VERSION: ${{ inputs.version || '1.6.0' }} + +jobs: + # The 3.14 wheel is abi3 compatible, so a separate build is only needed for + # 3.14t (free-threaded CPython has no stable ABI to target yet). + build_wheels: + name: Build hf-xet ${{ inputs.version || '1.6.0' }} ${{ matrix.python-version }}-riscv64 + runs-on: ubuntu-24.04-riscv + strategy: + fail-fast: false + matrix: + platform: + - target: riscv64gc-unknown-linux-gnu + arch: riscv64 + manylinux: 2_39 + python-version: + - '3.14' + - '3.14t' + + steps: + - name: Checkout hf-xet v${{ env.HFXET_VERSION }} + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: huggingface/xet-core + ref: v${{ env.HFXET_VERSION }} + persist-credentials: false + + - name: Install Python + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + with: + python-version: ${{ matrix.python-version }} + activate-environment: true + enable-cache: false + + - name: Install wheel tooling + run: | + uv pip install --upgrade pip + pip install wheel + + - uses: ./.github/actions/set-build-profile + with: + tag: v${{ env.HFXET_VERSION }} + + - name: Build wheels + uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 + with: + target: ${{ matrix.platform.target }} + args: --profile ${{ env.BUILD_PROFILE }} -i ${{ matrix.python-version }} --out dist + sccache: 'true' + manylinux: ${{ matrix.platform.manylinux }} + working-directory: hf_xet + before-script-linux: | + if command -v apt-get &> /dev/null; then + apt-get update && apt-get install libatomic-ops-dev -y + elif command -v yum &> /dev/null; then + yum install libatomic_ops-devel perl-IPC-Cmd -y + else + echo "Neither apt-get nor yum is installed. Please install a package manager." + exit 1 + fi + git config --global --add safe.directory "*" + + - name: Strip debug symbols into a separate file + if: env.IS_RELEASE == 'true' + run: | + mkdir hf_xet/dbg + mkdir dist + pushd dist + cp ../hf_xet/dist/* . + LATEST_WHEEL=$(ls -tr1 *.whl | tail -n 1) + WHEEL_NAME=$(basename $LATEST_WHEEL .whl) + WHEEL_VERSION=$(echo $LATEST_WHEEL | cut -d '-' -f 2) + SYMBOL_FILE=${WHEEL_NAME}.so.dbg + wheel unpack $LATEST_WHEEL + rm *.whl + objcopy --only-keep-debug hf_xet-${WHEEL_VERSION}/hf_xet/hf_xet.*.so hf_xet-${WHEEL_VERSION}/hf_xet/${SYMBOL_FILE} + objcopy --strip-debug hf_xet-${WHEEL_VERSION}/hf_xet/hf_xet.*.so + objcopy --add-gnu-debuglink=hf_xet-${WHEEL_VERSION}/hf_xet/${SYMBOL_FILE} hf_xet-${WHEEL_VERSION}/hf_xet/hf_xet.*.so + mv hf_xet-${WHEEL_VERSION}/hf_xet/${SYMBOL_FILE} ../hf_xet/dbg/${SYMBOL_FILE} + wheel pack hf_xet-${WHEEL_VERSION} + rm -rf hf_xet-${WHEEL_VERSION} + + - name: Copy unstripped wheels to upload directory + if: env.IS_RELEASE != 'true' + run: | + mkdir dist + cp hf_xet/dist/* dist/ + + - name: Upload debug symbols + if: env.IS_RELEASE == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: hf-xet-${{ env.HFXET_VERSION }}-${{ matrix.python-version }}-riscv64-dbg + path: hf_xet/dbg + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: hf-xet-${{ env.HFXET_VERSION }}-${{ matrix.python-version }}-riscv64 + path: dist/*.whl + if-no-files-found: error + + publish: + name: Publish hf-xet ${{ inputs.version || '1.6.0' }} to GitLab + needs: [build_wheels] + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + + steps: + - name: Publish wheels and open docs PR + uses: riseproject-dev/python-wheels/actions/publish-wheels@main + with: + artifact-pattern: hf-xet-${{ env.HFXET_VERSION }}-*-riscv64 + gitlab-username: ${{ vars.GITLAB_DEPLOY_USER }} + gitlab-token: ${{ secrets.GITLAB_DEPLOY_TOKEN }} + gitlab-project-id: ${{ vars.GITLAB_PROJECT_ID }} + gh-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test-hf-xet.yml b/.github/workflows/test-hf-xet.yml new file mode 100644 index 0000000..6145f14 --- /dev/null +++ b/.github/workflows/test-hf-xet.yml @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT + +name: Test hf-xet (riscv64) + +on: + workflow_dispatch: + inputs: + version: + description: 'hf-xet version to test (git tag without leading v, e.g. 1.6.0)' + required: true + default: '1.6.0' + pull_request: + paths: + - '.github/workflows/test-hf-xet.yml' + +concurrency: + group: ${{ github.workflow }}-${{ inputs.version || '1.6.0' }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +env: + HFXET_VERSION: ${{ inputs.version || '1.6.0' }} + +jobs: + # Mirrors ci.yml's build_and_test-linux job: build the extension and run + # hf_xet's own pytest suite against it. Builds a wheel and installs it + # rather than using upstream's `maturin develop` - see the comment on the + # Build wheel step below. hf-xet-tests.yml (huggingface_hub's xet + # integration tests) and the workspace-wide `cargo test` in ci.yml are + # dropped - the former always installs the released PyPI wheel rather than + # what's built here, and the latter exercises git_xet/xet_client code + # outside the Python package. + test: + permissions: + contents: read + name: "Test hf-xet ${{ inputs.version || '1.6.0' }} — Python 3.12 on riscv64" + runs-on: ubuntu-24.04-riscv + + steps: + - name: Checkout hf-xet v${{ env.HFXET_VERSION }} + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: huggingface/xet-core + ref: v${{ env.HFXET_VERSION }} + persist-credentials: false + + - name: Install Python + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + with: + python-version: '3.12' + activate-environment: true + enable-cache: false + + - name: Install libpython for pyo3 linking + run: | + set -eux + py_ver=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') + sudo apt-get update + if ! apt-cache show "libpython${py_ver}-dev" >/dev/null 2>&1; then + sudo apt-get install -y software-properties-common + sudo add-apt-repository -y ppa:deadsnakes/ppa + sudo apt-get update + fi + sudo apt-get install -y "libpython${py_ver}-dev" + + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 # master + with: + toolchain: 1.94.1 + + - name: Build and Test hf_xet + run: cd hf_xet && cargo test --verbose --no-fail-fast + + # `command: develop` in PyO3/maturin-action is hard-restricted to its + # non-container `hostBuild` path (innerMain() only ever sets useDocker + # for 'build'/'publish'), which downloads its maturin binary via a + # native-arch check that only recognizes 'arm64' vs. everything else as + # 'x86_64' - it fetches an x86_64 binary and fails to execute it here. + # `build` isn't restricted that way and reuses the same riscv64 + # manylinux container as build-hf-xet.yml, so build a wheel and install + # it instead of using `develop` directly. + - name: Build wheel + uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 + with: + command: build + target: riscv64gc-unknown-linux-gnu + args: -i 3.12 --out dist + sccache: 'true' + manylinux: '2_39' + working-directory: hf_xet + before-script-linux: | + if command -v apt-get &> /dev/null; then + apt-get update && apt-get install libatomic-ops-dev -y + elif command -v yum &> /dev/null; then + yum install libatomic_ops-devel perl-IPC-Cmd -y + else + echo "Neither apt-get nor yum is installed. Please install a package manager." + exit 1 + fi + git config --global --add safe.directory "*" + + - name: Install built wheel + run: uv pip install hf_xet/dist/*.whl + + - name: Python integration tests (hf_xet) + run: | + uv pip install pytest + pytest hf_xet/tests/ -v