Build wheels using general workflow

Author: jeromekelleher

.github/workflows/docker/buildwheel.sh

@@ -1,254 +1,41 @@
-name: Binary wheels
+name: Publish Python release
 
 on:
   push:
-    branches:
-      - main
-      - test
-    tags:
-      - '*'
+    branches: [test-publish]
   release:
     types: [published]
 
 jobs:
-  OSX:
-    name: Build ${{ matrix.os }} - ${{ matrix.python }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [macos-15-intel, macos-14, macos-15, macos-26]
-        python: [3.11, 3.12, 3.13, 3.14]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v6.0.2
-        with:
-            submodules: true
-      - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@v6.1.0
-        with:
-          python-version: ${{ matrix.python }}
-      - name: Install deps
-        run: |
-          brew install gsl
-          python -m pip install --upgrade pip build
-      - name: Build Wheel (arm64)
-        if: matrix.os != 'macos-15-intel'
-        run: |
-          export PLAT="arm64"
-          export _PYTHON_HOST_PLATFORM="macosx-11.0-arm64"
-          export CFLAGS+=" -arch arm64"
-          export CXXFLAGS+=" -arch arm64"
-          export ARCHFLAGS=$OLDARCHFLAGS
-          export ARCHFLAGS+=" -arch arm64"
-          export CPPFLAGS+=" -arch arm64"
-          export LDFLAGS+=" -arch arm64"
-          python -m build --wheel
-      - name: Build Wheel (x86_64)
-        if: matrix.os == 'macos-15-intel'
-        run: |
-          export PLAT="x86_64"
-          export _PYTHON_HOST_PLATFORM="macosx-11.0-x86_64"
-          export CFLAGS+=" -arch x86_64"
-          export CXXFLAGS+=" -arch x86_64"
-          export ARCHFLAGS=$OLDARCHFLAGS
-          export ARCHFLAGS+=" -arch x86_64"
-          export CPPFLAGS+=" -arch x86_64"
-          export LDFLAGS+=" -arch x86_64"
-          python -m build --wheel
-      - name: Delocate to bundle dynamic libs
-        run: |
-          python -m pip install delocate
-          delocate-wheel -v dist/*.whl
-      - name: Upload Wheels
-        uses: actions/upload-artifact@v6.0.0
-        with:
-          name: osx-wheel-${{ matrix.os }}-${{ matrix.python }}
-          path: dist
-
-  Windows:
-    name: Build Windows - ${{ matrix.python }}
-    runs-on: windows-latest
-    strategy:
-      matrix:
-        python: [3.11, 3.12, 3.13, 3.14]
-        arch: [x64]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v6.0.2
-        with:
-          submodules: true
-      - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@v6.1.0
-        with:
-          python-version: ${{ matrix.python }}
-          architecture: ${{ matrix.arch }}
-      - name: Install GSL via vcpkg
-        run: |
-          vcpkg install gsl:${{ matrix.arch }}-windows
-        shell: cmd
-      - name: Install deps
-        run: |
-          python -m pip install --upgrade pip build delvewheel
-      - name: Fix windows symlinks
-        run: |
-          if (Test-Path "lwt_interface") { Remove-Item -Recurse -Force "lwt_interface" }
-          Copy-Item -Recurse "git-submodules/tskit/python/lwt_interface" "./lwt_interface"
-        shell: powershell
-      - name: Build Wheel
-        run: |
-          # The symlink fix above confuses setuptools_scm
-          $tag = git describe --tags --exact-match HEAD 2>$null
-          if ($tag) {
-            $env:SETUPTOOLS_SCM_PRETEND_VERSION_FOR_MSPRIME = $tag
-          }
-          python -m build --wheel
-      - name: Bundle dynamic libs
-        run: |
-          $vcpkgRoot = $env:VCPKG_INSTALLATION_ROOT
-          if (-not $vcpkgRoot) { $vcpkgRoot = "C:\vcpkg" }
-          $gslBinPath = Join-Path $vcpkgRoot "installed\x64-windows\bin"
-          $env:PATH = "$gslBinPath;$env:PATH"
-          delvewheel repair dist/*.whl -w dist/ --add-path "$gslBinPath"
-        shell: powershell
-      - name: Upload Wheels
-        uses: actions/upload-artifact@v6.0.0
-        with:
-          name: win-wheel-${{ matrix.python }}
-          path: dist
-
-  manylinux:
-    runs-on: ubuntu-24.04
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v6.0.2
-        with:
-            submodules: true
-      - name: Set up Python 3.11
-        uses: actions/setup-python@v6.1.0
-        with:
-          python-version: "3.11"
-      - name: Build sdist
-        shell: bash
-        run: |
-          python -m pip install --upgrade pip build
-          python -m build --sdist
-      - name: Upload sdist
-        uses: actions/upload-artifact@v6.0.0
-        with:
-          name: sdist
-          path: dist
-      - name: Build wheels in docker
-        shell: bash
-        run: |
-          docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2014_x86_64 bash .github/workflows/docker/buildwheel.sh
-
-      - name: Upload Wheels
-        uses: actions/upload-artifact@v6.0.0
-        with:
-          name: linux-wheels
-          path: dist/wheelhouse
-
-  OSX-test:
-    name: Test ${{ matrix.os }} - ${{ matrix.python }}
-    needs: ['OSX']
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [macos-15-intel, macos-14, macos-15, macos-26]
-        python: [3.11, 3.12, 3.13, 3.14]
-    steps:
-      - name: Download wheels
-        uses: actions/download-artifact@v7.0.0
-        with:
-          name: osx-wheel-${{ matrix.os }}-${{ matrix.python }}
-      - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@v6.1.0
-        with:
-          python-version: ${{ matrix.python }}
-      - name: Install wheel and test
-        run: |
-          python -VV
-          # Install the local wheel
-          python -m pip install numpy newick>=1.3.0 tskit>=0.5.2 demes>=0.2
-          python -m pip install msprime --no-index --only-binary msprime -f .
-          python -c "import msprime"
-          msp simulate 10 tmp.trees
-          tskit info tmp.trees
-          tskit provenances tmp.trees
+  build-wheels:
+    uses: tskit-dev/.github/.github/workflows/build-wheels.yml@main
+    # with:
+    #   os-list: '["windows-2025"]' #, "ubuntu-24.04"]'
 
-  manylinux-test:
-    runs-on: ubuntu-24.04
-    needs: ['manylinux']
-    strategy:
-      matrix:
-        python: [3.11, 3.12, 3.13, 3.14]
-    steps:
-      - name: Download wheels
-        uses: actions/download-artifact@v7.0.0
-        with:
-          name: linux-wheels
-      - name: Set up Python
-        uses: actions/setup-python@v6.1.0
-        with:
-          python-version: ${{ matrix.python }}
-      - name: Install wheel and test
-        run: |
-          python -VV
-          # Install the local wheel
-          python -m pip install numpy newick demes
-          python -m pip install --no-dependencies msprime --no-index --only-binary msprime -f .
-          python -m pip install tskit
-          python -c "import msprime"
-          msp simulate 10 tmp.trees
-          tskit info tmp.trees
-          tskit provenances tmp.trees
-
-
-  Windows-test:
-    runs-on: windows-latest
-    needs: ['Windows']
-    strategy:
-      matrix:
-        python: [3.11, 3.12, 3.13, 3.14]
-    steps:
-      - name: Download wheels
-        uses: actions/download-artifact@v7.0.0
-        with:
-          name: win-wheel-${{ matrix.python }}
-      - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@v6.1.0
-        with:
-          python-version: ${{ matrix.python }}
-      - name: Install wheel and test
-        run: |
-          python -VV
-          python -m pip install numpy newick>=1.3.0 tskit>=0.5.2 demes>=0.2
-          python -m pip install msprime --no-index --only-binary msprime -f .
-          python -c "import msprime"
-          msp simulate 10 tmp.trees
-          tskit info tmp.trees
-          tskit provenances tmp.trees
-
-  PyPI_Upload:
+  publish:
     runs-on: ubuntu-24.04
     environment: release
-    needs: ['OSX-test', 'manylinux-test', 'Windows-test']
+    needs: [ 'build-wheels' ]
     permissions:
       id-token: write
     steps:
-      - name: Download all
+      - name: Download artifacts
         uses: actions/download-artifact@v7.0.0
-      - name: Move to dist
-        run: |
-          mkdir dist
-          cp */*.{whl,gz} dist/.
-          # Windows wheels have different naming pattern
-          if [ -d "win-wheel"* ]; then cp win-wheel*/*.whl dist/. || true; fi
+        with:
+          pattern: build-*
+          path: dist
+          merge-multiple: true
+
+      - name: Show artifacts
+        run: ls -lah dist
+
       - name: Publish distribution to Test PyPI
-        if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
+        if: github.event_name == 'push' && github.ref_name == 'test-publish'
         uses: pypa/gh-action-pypi-publish@v1.13.0
         with:
-          repository_url: https://test.pypi.org/legacy/
-      - name: Publish distribution to PRODUCTION PyPI
+          repository-url: https://test.pypi.org/legacy/
+          verbose: true
+
+      - name: Publish distribution to Production PyPI
         if: github.event_name == 'release'
         uses: pypa/gh-action-pypi-publish@v1.13.0

.github/workflows/docker/shared.env

@@ -66,6 +66,7 @@ include-package-data = true
 
 [tool.setuptools_scm]
 write_to = "msprime/_version.py"
+local_scheme = "no-local-version"
 
 [dependency-groups]
 test = [
@@ -105,6 +106,10 @@ lint = [
     "prek==0.3.3",
 ]
 
+wheels = [
+  "cibuildwheel",
+]
+
 dev = [
     {include-group = "docs"},
     {include-group = "lint"},
@@ -135,7 +140,6 @@ filterwarnings = [
 ]
 
 [tool.ruff]
-# Assume Python 3.11
 target-version = "py311"
 
 line-length = 89
@@ -170,3 +174,29 @@ known-third-party = [
   "numpy",
   "pytest",
 ]
+
+[tool.cibuildwheel]
+build = ["cp311-*", "cp312-*", "cp313-*", "cp314-*"]
+build-frontend = "build"
+build-verbosity = 3
+test-command = [
+  "python -c \"import msprime._msprime\"",
+  "msp --help",
+]
+
+[tool.cibuildwheel.linux]
+archs = ["x86_64"]
+before-all = ["yum -y install gsl-devel"]
+skip = ["*musllinux*"]
+
+[tool.cibuildwheel.macos]
+archs = ["x86_64", "arm64"]
+before-all = ["brew install gsl"]
+# This is because of how GSL is built by homebrew. 
+environment = { MACOSX_DEPLOYMENT_TARGET = "14.0" }
+
+[tool.cibuildwheel.windows]
+archs = ["AMD64"]
+before-all = ["vcpkg install gsl:x64-windows"]
+before-build = "pip install delvewheel"
+repair-wheel-command = "delvewheel repair --add-path C:\\vcpkg\\installed\\x64-windows\\bin -w {dest_dir} {wheel}"