Skip to content

LLVM from private pypi #8938

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
alexreinking wants to merge 1 commit into
base: main
Choose a base branch
from
+66 −143
Open

LLVM from private pypi #8938

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
209 changes: 66 additions & 143 deletions .github/workflows/pip.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,3 @@
# Relevant GHA docs links:
# https://docs.github.com/en/actions/using-jobs/running-jobs-in-a-container
# https://docs.github.com/en/packages/managing-github-packages-using-github-actions-workflows/publishing-and-installing-a-package-with-github-actions#upgrading-a-workflow-that-accesses-ghcrio

name: Build PyPI package

on:
Expand All @@ -12,7 +8,7 @@ on:

env:
# TODO: detect this from repo somehow: https://github.com/halide/Halide/issues/8406
LLVM_VERSION: 21.1.1
LLVM_VERSION: 21.1.8
FLATBUFFERS_VERSION: 23.5.26
WABT_VERSION: 1.0.36

Expand All @@ -22,7 +18,6 @@ concurrency:

permissions:
contents: read # to fetch code (actions/checkout)
packages: read # to fetch packages (docker)

jobs:
build-wheels:
Expand All @@ -46,14 +41,6 @@ jobs:
MACOSX_DEPLOYMENT_TARGET: 11

steps:
- name: Login to GitHub Container Registry
if: matrix.os == 'ubuntu-latest'
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- uses: actions/checkout@v4
with:
fetch-depth: 0
Expand All @@ -62,135 +49,17 @@ jobs:
- uses: ilammy/msvc-dev-cmd@v1
- uses: lukka/[email protected]

########################################################################
# flatbuffers
########################################################################

- name: Cache flatbuffers build folder
if: matrix.os != 'ubuntu-latest'
id: cache-flatbuffers
uses: actions/cache@v4
with:
path: opt/flatbuffers
key: flatbuffers-${{ env.FLATBUFFERS_VERSION }}-${{ matrix.platform_tag }}

- uses: actions/checkout@v4
if: matrix.os != 'ubuntu-latest' && steps.cache-flatbuffers.outputs.cache-hit != 'true'
with:
path: flatbuffers-src
repository: google/flatbuffers
ref: v${{ env.FLATBUFFERS_VERSION }}

- name: Configure flatbuffers
if: matrix.os != 'ubuntu-latest' && steps.cache-flatbuffers.outputs.cache-hit != 'true'
run: >
cmake -G Ninja -S flatbuffers-src -B flatbuffers-build
"-DCMAKE_BUILD_TYPE=Release"
"-DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/opt/flatbuffers"
"-DFLATBUFFERS_BUILD_TESTS=NO"

- name: Install flatbuffers
if: matrix.os != 'ubuntu-latest' && steps.cache-flatbuffers.outputs.cache-hit != 'true'
run: |
cmake --build flatbuffers-build --target install
cmake -E rm -rf flatbuffers-src flatbuffers-build

########################################################################
# wabt
########################################################################

- name: Cache wabt build folder
if: matrix.os != 'ubuntu-latest' && matrix.os != 'windows-latest'
id: cache-wabt
uses: actions/cache@v4
with:
path: opt/wabt
key: wabt-${{ env.WABT_VERSION }}-${{ matrix.platform_tag }}

- uses: actions/checkout@v4
if: >
matrix.os != 'ubuntu-latest' && matrix.os != 'windows-latest'
&& steps.cache-wabt.outputs.cache-hit != 'true'
with:
submodules: 'true'
path: wabt-src
repository: WebAssembly/wabt
ref: ${{ env.WABT_VERSION }}

- name: Configure wabt
if: >
matrix.os != 'ubuntu-latest' && matrix.os != 'windows-latest'
&& steps.cache-wabt.outputs.cache-hit != 'true'
run: >
cmake -G Ninja -S wabt-src -B wabt-build
"-DCMAKE_BUILD_TYPE=Release"
"-DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/opt/wabt"
"-DWITH_EXCEPTIONS=ON"
"-DBUILD_TESTS=OFF"
"-DBUILD_TOOLS=OFF"
"-DBUILD_LIBWASM=OFF"
"-DUSE_INTERNAL_SHA256=ON"

- name: Install wabt
if: >
matrix.os != 'ubuntu-latest' && matrix.os != 'windows-latest'
&& steps.cache-wabt.outputs.cache-hit != 'true'
run: |
cmake --build wabt-build --target install
cmake -E rm -rf wabt-src wabt-build

########################################################################
# LLVM
########################################################################

- name: Cache LLVM build folder
if: matrix.os != 'ubuntu-latest'
id: cache-llvm
uses: actions/cache@v4
with:
path: opt/llvm
key: llvm-${{ env.LLVM_VERSION }}-${{ matrix.platform_tag }}

- uses: actions/checkout@v4
if: matrix.os != 'ubuntu-latest' && steps.cache-llvm.outputs.cache-hit != 'true'
with:
path: llvm-src
repository: llvm/llvm-project
ref: llvmorg-${{ env.LLVM_VERSION }}

- name: Configure LLVM
if: matrix.os != 'ubuntu-latest' && steps.cache-llvm.outputs.cache-hit != 'true'
run: >
cmake -G Ninja -S llvm-src/llvm -B llvm-build
"-DCMAKE_BUILD_TYPE=Release"
"-DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/opt/llvm"
"-DLLVM_ENABLE_ASSERTIONS=ON"
"-DLLVM_ENABLE_BINDINGS=OFF"
"-DLLVM_ENABLE_CURL=OFF"
"-DLLVM_ENABLE_DIA_SDK=OFF"
"-DLLVM_ENABLE_EH=ON"
"-DLLVM_ENABLE_HTTPLIB=OFF"
"-DLLVM_ENABLE_IDE=OFF"
"-DLLVM_ENABLE_LIBEDIT=OFF"
"-DLLVM_ENABLE_LIBXML2=OFF"
"-DLLVM_ENABLE_OCAMLDOC=OFF"
"-DLLVM_ENABLE_PROJECTS=clang;lld"
"-DLLVM_ENABLE_RTTI=ON"
"-DLLVM_ENABLE_RUNTIMES=compiler-rt"
"-DLLVM_ENABLE_TERMINFO=OFF"
"-DLLVM_ENABLE_WARNINGS=OFF"
"-DLLVM_ENABLE_ZLIB=OFF"
"-DLLVM_ENABLE_ZSTD=OFF"
"-DLLVM_INCLUDE_BENCHMARKS=OFF"
"-DLLVM_INCLUDE_EXAMPLES=OFF"
"-DLLVM_INCLUDE_TESTS=OFF"
"-DLLVM_TARGETS_TO_BUILD=WebAssembly;X86;AArch64;ARM;Hexagon;NVPTX;PowerPC;RISCV"

- name: Install LLVM
if: matrix.os != 'ubuntu-latest' && steps.cache-llvm.outputs.cache-hit != 'true'
if: matrix.os != 'ubuntu-latest'
shell: bash
run: |
cmake --build llvm-build --target install
cmake -E rm -rf llvm-src llvm-build
pip install "halide-llvm==${{ env.LLVM_VERSION }}" \
--extra-index-url https://pypi.halide-lang.org/simple/
echo "Halide_LLVM_ROOT=$(halide-llvm --prefix)" >> "$GITHUB_ENV"

########################################################################
# Wheels
Expand All @@ -199,12 +68,66 @@ jobs:
#- uses: mxschmitt/action-tmate@v3

- name: Build wheels
uses: pypa/cibuildwheel@v2.20.0
uses: pypa/cibuildwheel@v3.3.1
env:
CIBW_BUILD: "cp3*-${{ matrix.platform_tag }}"
CIBW_SKIP: "cp3{5,6,7,8,9}*"
# Suppress the git version tag (necessary for TestPyPI)
CIBW_ENVIRONMENT: >
CIBW_SKIP: "cp3{5,6,7,8,9}* cp314t-*"
CIBW_BEFORE_ALL_LINUX: >
/opt/python/cp312-cp312/bin/pip install cmake ninja
"halide-llvm==${{ env.LLVM_VERSION }}"
--extra-index-url https://pypi.halide-lang.org/simple/ &&
export PATH="/opt/python/cp312-cp312/bin:$PATH" &&
mkdir -p {project}/opt &&
ln -s $(halide-llvm --prefix) {project}/opt/llvm &&
git clone --depth=1 -b v${{ env.FLATBUFFERS_VERSION }}
https://github.com/google/flatbuffers /tmp/fb-src &&
cmake -G Ninja -S /tmp/fb-src -B /tmp/fb-build
-DCMAKE_BUILD_TYPE=Release
-DCMAKE_INSTALL_PREFIX={project}/opt/flatbuffers
-DFLATBUFFERS_BUILD_TESTS=NO &&
cmake --build /tmp/fb-build --target install &&
git clone --depth=1 --recurse-submodules -b ${{ env.WABT_VERSION }}
https://github.com/WebAssembly/wabt /tmp/wabt-src &&
cmake -G Ninja -S /tmp/wabt-src -B /tmp/wabt-build
-DCMAKE_BUILD_TYPE=Release
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
-DCMAKE_INSTALL_PREFIX={project}/opt/wabt
-DWITH_EXCEPTIONS=ON
-DBUILD_TESTS=OFF
-DBUILD_TOOLS=OFF
-DBUILD_LIBWASM=OFF
-DUSE_INTERNAL_SHA256=ON &&
cmake --build /tmp/wabt-build --target install
CIBW_BEFORE_ALL_MACOS: >
git clone --depth=1 -b v${{ env.FLATBUFFERS_VERSION }}
https://github.com/google/flatbuffers /tmp/fb-src &&
cmake -G Ninja -S /tmp/fb-src -B /tmp/fb-build
-DCMAKE_BUILD_TYPE=Release
-DCMAKE_INSTALL_PREFIX={project}/opt/flatbuffers
-DFLATBUFFERS_BUILD_TESTS=NO &&
cmake --build /tmp/fb-build --target install &&
git clone --depth=1 --recurse-submodules -b ${{ env.WABT_VERSION }}
https://github.com/WebAssembly/wabt /tmp/wabt-src &&
cmake -G Ninja -S /tmp/wabt-src -B /tmp/wabt-build
-DCMAKE_BUILD_TYPE=Release
-DCMAKE_INSTALL_PREFIX={project}/opt/wabt
-DWITH_EXCEPTIONS=ON
-DBUILD_TESTS=OFF
-DBUILD_TOOLS=OFF
-DBUILD_LIBWASM=OFF
-DUSE_INTERNAL_SHA256=ON &&
cmake --build /tmp/wabt-build --target install
CIBW_BEFORE_ALL_WINDOWS: >
git clone --depth=1 -b v${{ env.FLATBUFFERS_VERSION }}
https://github.com/google/flatbuffers %TEMP%\fb-src &&
cmake -G Ninja -S %TEMP%\fb-src -B %TEMP%\fb-build
-DCMAKE_BUILD_TYPE=Release
-DCMAKE_INSTALL_PREFIX={project}/opt/flatbuffers
-DFLATBUFFERS_BUILD_TESTS=NO &&
cmake --build %TEMP%\fb-build --target install
CIBW_ENVIRONMENT_LINUX: >
Halide_LLVM_ROOT=/project/opt/llvm
CMAKE_PREFIX_PATH=/project/opt
SETUPTOOLS_SCM_OVERRIDES_FOR_HALIDE='{local_scheme="no-local-version"}'
CIBW_ENVIRONMENT_MACOS: >
CMAKE_PREFIX_PATH='${{ github.workspace }}/opt'
Expand All @@ -214,7 +137,7 @@ jobs:
CMAKE_GENERATOR=Ninja
CMAKE_PREFIX_PATH='${{ github.workspace }}\opt'
SETUPTOOLS_SCM_OVERRIDES_FOR_HALIDE='{local_scheme="no-local-version"}'
CIBW_MANYLINUX_X86_64_IMAGE: "ghcr.io/halide/manylinux_2_28_x86_64-llvm:${{ env.LLVM_VERSION }}"
CIBW_BEFORE_TEST_LINUX: pip install cmake ninja
CIBW_TEST_COMMAND: >
cmake -G Ninja -S {project}/python_bindings/apps -B build -DCMAKE_BUILD_TYPE=Release &&
cmake --build build &&
Expand Down Expand Up @@ -248,7 +171,7 @@ jobs:
with:
user: upload
password: ${{ secrets.HALIDE_PYPI_PASSWORD }}
repository-url: https://pypi.halide-lang.org/legacy/
repository-url: https://pypi.halide-lang.org/

- uses: pypa/gh-action-pypi-publish@release/v1
if: github.event_name == 'release' && github.event.action == 'published'
Loading