diff --git a/.github/workflows/build-python-wheels.yml b/.github/workflows/build-python-wheels.yml index 65a5a168..8c24a75e 100644 --- a/.github/workflows/build-python-wheels.yml +++ b/.github/workflows/build-python-wheels.yml @@ -2,8 +2,8 @@ name: Python Packages on: push: - branches: [ develop ] - tags: [ "v*" ] + branches: [develop] + tags: ["v*"] # following two lines must be commented out before merging, we don't want to run this on PRs # but for testing this we run this on PRs # pull_request: @@ -12,67 +12,75 @@ on: jobs: build-wheels: name: build wheels using cibuildwheel - runs-on: ${{ format('{0}-latest', matrix.os) }} + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - include: - - os: "ubuntu" - platform: "linux" - arch: "x86_64" - - os: "macos" - arch: "universal2" - - os: "windows" - arch: "AMD64" - env: - build-sdist: ${{ matrix.os == 'ubuntu' && matrix.arch == 'x86_64' }} + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest] steps: - uses: actions/checkout@v4 - - name: setup python for sdist and cibuildwheel + - name: download dictionary for PGO + if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'ubuntu-24.04-arm' }} + run: bash fetch_dictionary.sh "20220519" "core" + - uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: cibuildwheel ${{ matrix.os }} + uses: pypa/cibuildwheel@v3.0.0 + with: + package-dir: python + output-dir: python/dist + config-file: "{package}/pyproject.toml" + + - uses: actions/upload-artifact@v4 + with: + name: artifact-${{ matrix.os }} + path: | + ./python/dist/*.whl + ./python/dist/*.tar.gz + + build-sdist: + name: build sdist + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: setup python for sdist uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: "3.11" - name: Install dependencies run: | - python -m pip install --upgrade pip - python -m pip install --upgrade setuptools setuptools-rust build cibuildwheel - - name: download dictionary for PGO - if: ${{ matrix.os == 'ubuntu' }} - run: bash fetch_dictionary.sh "20220519" "core" - - name: Add aarch64/x86 target for macos - if: ${{ matrix.os == 'macos' }} - run: rustup target add aarch64-apple-darwin x86_64-apple-darwin + python -m pip install --upgrade pip + python -m pip install --upgrade setuptools setuptools-rust build - uses: actions/cache@v4 with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - name: build sdist - if: ${{ env.build-sdist == 'true' }} working-directory: ./python run: bash build-sdist.sh - - name: cibuildwheel ${{ matrix.platform || matrix.os }} - env: - # most configuration are in python/pyproject.toml - CIBW_PLATFORM: ${{ matrix.platform || matrix.os }} - CIBW_ARCHS: ${{ matrix.arch }} - run: cibuildwheel python/ --output-dir python/dist/ - - uses: actions/upload-artifact@v4 with: - name: artifact-${{ matrix.os }} + name: artifact-sdist path: | ./python/dist/*.whl ./python/dist/*.tar.gz upload-to-pypi: # run only if all have succeeded - needs: [ build-wheels ] + needs: [build-wheels, build-sdist] # https://stackoverflow.com/questions/58475748/github-actions-how-to-check-if-current-push-has-new-tag-is-new-release if: startsWith(github.ref, 'refs/tags/v') # only for tags runs-on: ubuntu-latest diff --git a/.github/workflows/python-upload-test.yml b/.github/workflows/python-upload-test.yml index e1074b06..74113f7c 100644 --- a/.github/workflows/python-upload-test.yml +++ b/.github/workflows/python-upload-test.yml @@ -2,74 +2,86 @@ name: Python Packages Test on: push: - branches: [ pre/v* ] + branches: [pre/v*] jobs: build-wheels: name: build wheels using cibuildwheel - runs-on: ${{ format('{0}-latest', matrix.os) }} + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - include: - - os: "ubuntu" - platform: "linux" - arch: "x86_64" - - os: "macos" - arch: "universal2" - - os: "windows" - arch: "AMD64" - env: - build-sdist: ${{ matrix.os == 'ubuntu' && matrix.arch == 'x86_64' }} + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest] + + steps: + - uses: actions/checkout@v4 + - name: download dictionary for PGO + if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'ubuntu-24.04-arm' }} + run: bash fetch_dictionary.sh "20220519" "core" + - uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Modify version for TestPyPI upload + run: python ./python/modify_version_for_testpypi.py + - name: cibuildwheel ${{ matrix.os }} + uses: pypa/cibuildwheel@v3.0.0 + with: + package-dir: python + output-dir: python/dist + config-file: "{package}/pyproject.toml" + - uses: actions/upload-artifact@v4 + with: + name: artifact-${{ matrix.os }} + path: | + ./python/dist/*.whl + ./python/dist/*.tar.gz + + build-sdist: + name: build sdist + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: setup python for sdist and cibuildwheel + - name: setup python for sdist uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: "3.11" - name: Install dependencies run: | - python -m pip install --upgrade pip - python -m pip install --upgrade setuptools setuptools-rust build cibuildwheel packaging - - name: download dictionary for PGO - if: ${{ matrix.os == 'ubuntu' }} - run: bash fetch_dictionary.sh "20220519" "core" - - name: Add aarch64/x86 target for macos - if: ${{ matrix.os == 'macos' }} - run: rustup target add aarch64-apple-darwin x86_64-apple-darwin + python -m pip install --upgrade pip + python -m pip install --upgrade setuptools setuptools-rust build - uses: actions/cache@v4 with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - name: Modify version for TestPyPI upload run: python ./python/modify_version_for_testpypi.py - name: build sdist - if: ${{ env.build-sdist == 'true' }} working-directory: ./python run: bash build-sdist.sh - - name: cibuildwheel ${{ matrix.platform || matrix.os }} - env: - # most configuration are in python/pyproject.toml - CIBW_PLATFORM: ${{ matrix.platform || matrix.os }} - CIBW_ARCHS: ${{ matrix.arch }} - run: cibuildwheel python/ --output-dir python/dist/ - + - uses: actions/upload-artifact@v4 with: - name: artifact-${{ matrix.os }} + name: artifact-sdist path: | ./python/dist/*.whl ./python/dist/*.tar.gz upload-to-testpypi: # run only if all have succeeded - needs: [ build-wheels ] + needs: [build-wheels, build-sdist] runs-on: ubuntu-latest steps: - uses: actions/download-artifact@v4 @@ -88,12 +100,12 @@ jobs: verbose: true install-and-test: - needs: [ upload-to-testpypi ] + needs: [upload-to-testpypi] strategy: fail-fast: false matrix: - os: [ ubuntu-latest, windows-latest, macOS-latest ] - target: [ "3.9", "3.10", "3.11", "3.12", "3.13", "3.13t" ] + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macOS-latest] + target: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.13t"] include: - os: "ubuntu-latest" target: "sdist" @@ -127,7 +139,7 @@ jobs: # tokenizers for py3.13t is not provided yet if: ${{ matrix.target != '3.13t' }} run: python -m pip install tokenizers - + - name: Run test if: ${{ matrix.target != '3.13t' }} working-directory: ./python diff --git a/python/README.md b/python/README.md index 40a6f8c5..30027085 100644 --- a/python/README.md +++ b/python/README.md @@ -11,7 +11,7 @@ This is not a pure Python implementation, but bindings for the ## Binary wheels -We provide binary builds for macOS (10.14+), Windows and Linux only for x86_64 architecture. +We provide binary builds for macOS (10.14+), Windows and Linux x86_64/aarch64 architecture. x86 32-bit architecture is not supported and is not tested. MacOS source builds seem to work on ARM-based (Aarch64) Macs, but this architecture also is not tested and require installing Rust toolchain and Cargo. diff --git a/python/docs/source/topics/wheels.rst b/python/docs/source/topics/wheels.rst index c0cfc931..83c62507 100644 --- a/python/docs/source/topics/wheels.rst +++ b/python/docs/source/topics/wheels.rst @@ -4,8 +4,9 @@ Wheel Status We provide wheel builds for following platforms: * Linux-amd64 +* Linux-aarch64 * Windows-x86_64 -* macOS-universal2 (x86_64 + arm64) +* macOS-universal2, x86_64, arm64 Wheels for Linux are supported (we will fix bugs and problems when reported). Wheels for Windows are supported in best-effort manner (we will try to fix bugs when reported, but that can take unspecified amount of time). @@ -14,8 +15,8 @@ Wheels for macOS are **not** supported and we welcome contributions for them. Linux ------ -We build wheels only for amd64 (x86_64) architecture at the moment. -Wheels for Linux are built with `Profiled-Guided Optimizations `_ +We build wheels only for amd64 (x86_64) and arm64 (aarch64) architecture at the moment. +Wheels for Linux are built with `Profile-Guided Optimizations `_ and will probably be faster than installations which are manaully-built from the source package. Wheels are built using manylinux container and should be compatible with most of distributions. @@ -30,4 +31,4 @@ macOS The situation is similar to Windows, but these builds are tested even less. We provide universal2 builds which should work both on Intel and ARM-based Macs, -but ARM-based builds are not tested in CI and are provided on best effort basis. +but we only run tests with latest runner in CI and are provided on best effort basis. diff --git a/python/pyproject.toml b/python/pyproject.toml index 26f86b92..b2a60c8e 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -14,10 +14,12 @@ before-build = "rustup show" [tool.cibuildwheel.linux] environment = { PATH = "$HOME/.cargo/bin:$PATH", CARGO_TERM_COLOR = "always", RUSTFLAGS = "-C profile-use=/tmp/sudachi-profdata.merged -C opt-level=3" } -before-all = "curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable -y && python/generate-manylinux-profile.sh /tmp/sudachi-profdata.merged" +before-all = "curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain stable -y && python/generate-manylinux-profile.sh /tmp/sudachi-profdata.merged" [tool.cibuildwheel.macos] -environment = { PATH = "$HOME/.cargo/bin:$PATH", CARGO_TERM_COLOR = "always", MACOSX_DEPLOYMENT_TARGET = 10.12 } +archs = ["x86_64", "universal2", "arm64"] +environment = { PATH = "$HOME/.cargo/bin:$PATH", CARGO_TERM_COLOR = "always", MACOSX_DEPLOYMENT_TARGET = "10.12" } +before-all = "rustup target add aarch64-apple-darwin x86_64-apple-darwin" [tool.cibuildwheel.windows] environment = { PATH = "$UserProfile\\.cargo\\bin;$PATH" }