Use cibuildwheel to build Linux wheels. (#1281)

* Improve robustness of tests that read data in
  from files by determining the path of the test file.

.github/workflows/wheels.yml

@@ -44,9 +44,6 @@ jobs:
   manylinux2_28:
     name: Build and test Linux wheels
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python: ["python3.8", "python3.9", "python3.10", "python3.11", "python3.12"]
     steps:
       - name: Cancel Previous Runs
         uses: styfle/cancel-workflow-action@0.12.1
@@ -59,17 +56,14 @@ jobs:
             submodules: true
             fetch-depth: 0
 
-      - name: Build wheels in docker
-        shell: bash
-        run: |
-          bash deployment/linux_wheels/run_wheel_workflow.sh ${{ matrix.python }}
-          #docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux_2_28_x86_64:2022-10-02-69a0972 bash .github/workflows/manylinux/buildwheels.sh
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v2.17.0
 
       - name: Upload Wheels
         uses: actions/upload-artifact@v4
         with:
-          name: linux-wheels-${{ matrix.python }}
-          path: dist/wheelhouse
+          name: linux-wheels
+          path: wheelhouse
 
   manylinux2_28_test:
     name: Build package from source dist
@@ -106,24 +100,40 @@ jobs:
           deactivate
           rm -rf sdist_venv
 
-          # - name: Install wheel and test
-          #   run: |
-          #     python -VV
-          #     # pip install minimal dependencies
-          #     pip install --upgrade pip
-          #     pip install wheel
-          #     pip install -r requirements/wheel_building_workflow.txt
-          #     # delete the source dir to prevent pip from mistaking it for 
-          #     # the package
-          #     rm -rf fwdpy11
-          #     # Install the local wheel
-          #     pip install fwdpy11 --no-deps --no-index --pre --only-binary fwdpy11 -f .
-          #     python -c "import fwdpy11;print(fwdpy11.__version__)"
-
+  # Test that wheels build in the docker cibuildwheel workflow
+  # can be installed on Ubuntu Linux, which is a different distro.
+  manylinux2_28_test_install_wheel:
+    name: Install the wheel on github runner
+    runs-on: ubuntu-latest
+    needs: ['manylinux2_28']
+    strategy:
+      matrix:
+        python: [3.8, 3.9, "3.10", "3.11", "3.12"]
+    steps:
+      - name: Cancel Previous Runs
+        uses: styfle/cancel-workflow-action@0.12.1
+        with:
+          access_token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Download sdist
+        uses: actions/download-artifact@v4.1.4
+        with:
+          name: linux-wheels
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+      - name: install into venv
+        run: |
+          python -m venv venv
+          source venv/bin/activate
+          python -m pip install --no-cache-dir --only-binary=fwdpy11 --pre --find-links . fwdpy11
+          python -m fwdpy11 --includes
+          python -c "import fwdpy11;print(fwdpy11.__version__)"
+          python -c "import fwdpy11;print(fwdpy11.__file__)"
+              
   upload_to_PyPI:
     name: Upload to PyPI
     runs-on: ubuntu-latest
-    needs: ['manylinux2_28', 'manylinux2_28_test']
+    needs: ['manylinux2_28', 'manylinux2_28_test', 'manylinux2_28_test_install_wheel']
     steps:
       - name: Cancel Previous Runs
         uses: styfle/cancel-workflow-action@0.12.1

deployment/linux_wheels/cibuildwheel.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# SETUP
+set -e -x
+rm -rf venv
+
+# INSTALL SYSTEM DEPENDENCIES
+
+yum -y install curl gsl-devel
+
+# INSTALL RUST
+
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | bash -s -- -y 
+source "$HOME/.cargo/env" 
+# Pin the rustc toolchain to a specific version.
+# Rust 1.64.0 will change the minimum glibc ABI
+# to a version incompatible with manylinux_2014,
+# so we need to be careful in general.
+rustup override set 1.62.1 
+# Pin cbindgen 
+cargo install --locked cbindgen@0.24.3 
+
+# Taken from msprime/#2043
+# We're running as root in the docker container so git commands issued by
+# setuptools_scm will fail without this:
+git config --global --add safe.directory /project

doc/long_vignettes/demes_event_timings.md

@@ -43,7 +43,7 @@ We can use a simple data class to record this information:
 
 ```{literalinclude} ../../tests/test_demes_event_timings.py
 :language: python
-:lines: 8-20
+:lines: 11-23
 ```
 
 There will be no genetics happening during the simulation.
@@ -53,7 +53,7 @@ generations of matings.
 
 ```{literalinclude} ../../tests/test_demes_event_timings.py
 :language: python
-:lines: 24-42
+:lines: 26-45
 ```
 
 ```{code-cell} python
@@ -91,7 +91,7 @@ times match up with their expected deme labels:
 
 ```{literalinclude} ../../tests/test_demes_event_timings.py
 :language: python
-:lines: 83-93
+:lines: 86-96
 ```
 
 ## A single pulse event
@@ -134,7 +134,7 @@ the pulse, we can make the following assertions:
 
 ```{literalinclude} ../../tests/test_demes_event_timings.py
 :language: python
-:lines: 45-61
+:lines: 48-64
 ```
 
 Let's work through the logic behind assertions in detail:
@@ -177,5 +177,5 @@ The following code asserts that these intervals are what we expect:
 
 ```{literalinclude} ../../tests/test_demes_event_timings.py
 :language: python
-:lines: 64-80
+:lines: 67-83
 ```

pyproject.toml

@@ -52,3 +52,16 @@ write_to = "fwdpy11/_version.py"
 filterwarnings = [
     "error",
 ]
+
+[tool.cibuildwheel]
+environment="PATH=$PATH:/$HOME/.cargo/bin LD_LIBRARY_PATH=fwdpy11"
+test-command = "pytest {project}/tests"
+test-requires = "pytest msprime hypothesis"
+build-frontend = "build"
+
+[tool.cibuildwheel.linux]
+build = "cp*manylinux*"
+manylinux-x86_64-image = "manylinux_2_28"
+before-all = "./deployment/linux_wheels/cibuildwheel.sh"
+archs = "x86_64"
+# skip = "pp* *musllinux*"

tests/test_binary_format_compatibility.py

@@ -2,10 +2,13 @@
 
 import fwdpy11
 
+from utils import make_path
+
 
 class TestOldBinaryFileFormats(unittest.TestCase):
     def test_reading_045(self):
-        pop = fwdpy11.DiploidPopulation.load_from_file("tests/v045.bin")
+        path_to_file = make_path("v045.bin")
+        pop = fwdpy11.DiploidPopulation.load_from_file(path_to_file)
         assert isinstance(pop, fwdpy11.DiploidPopulation) is True
         for i in pop.tables.mutations:
             self.assertEqual(

tests/test_demes2fwdpy11.py

@@ -8,6 +8,8 @@
 import numpy as np
 import pytest
 
+from utils import make_path
+
 
 def run_model_round_trip(cls):
     def _test_evolvets_roundtrip(self):
@@ -50,15 +52,17 @@ def test_burnin_inputs(self):
 class TestLoadGraph(unittest.TestCase):
     @classmethod
     def setUpClass(self):
-        self.g = demes.load("tests/test_demog.yaml")
+        self.g = demes.load(make_path("test_demog.yaml"))
         self.demog = fwdpy11.discrete_demography.from_demes(self.g, 1)
 
 
 @run_model_round_trip
 class TestLoadYAML(unittest.TestCase):
     @classmethod
     def setUpClass(self):
-        self.demog = fwdpy11.discrete_demography.from_demes("tests/test_demog.yaml", 1)
+        self.demog = fwdpy11.discrete_demography.from_demes(
+            make_path("test_demog.yaml"), 1
+        )
 
 
 @run_model_round_trip

tests/test_demes_event_timings.py

@@ -5,6 +5,8 @@
 
 import fwdpy11
 
+from utils import make_path
+
 
 @dataclass
 class Parents:
@@ -43,7 +45,7 @@ def run_model(yaml, recorder):
 
 
 def test_single_pulse():
-    yaml = "tests/demes_event_examples/single_pulse.yaml"
+    yaml = make_path("demes_event_examples/single_pulse.yaml")
     recorder = RecordParents([])
     pop = run_model(yaml, recorder)
     assert pop.generation == 110
@@ -62,7 +64,7 @@ def test_single_pulse():
 
 
 def test_burst_of_migration():
-    yaml = "tests/demes_event_examples/burst_of_migration.yaml"
+    yaml = make_path("demes_event_examples/burst_of_migration.yaml")
     recorder = RecordParents([])
     pop = run_model(yaml, recorder)
     assert pop.generation == 110
@@ -81,7 +83,7 @@ def test_burst_of_migration():
 
 
 def test_deme_existence():
-    yaml = "tests/demes_event_examples/deme_existence.yaml"
+    yaml = make_path("demes_event_examples/deme_existence.yaml")
     recorder = None
     pop = run_model(yaml, recorder)
     assert pop.generation == 150
@@ -148,20 +150,20 @@ def validate_deme_existence(yaml):
 
 
 def test_single_pulse_deme_sizes():
-    yaml = "tests/demes_event_examples/single_pulse.yaml"
+    yaml = make_path("demes_event_examples/single_pulse.yaml")
     validate_deme_existence(yaml)
 
 
 def test_burst_of_migration_deme_sizes():
-    yaml = "tests/demes_event_examples/burst_of_migration.yaml"
+    yaml = make_path("demes_event_examples/burst_of_migration.yaml")
     validate_deme_existence(yaml)
 
 
 def test_deme_existence_deme_sizes():
-    yaml = "tests/demes_event_examples/deme_existence.yaml"
+    yaml = make_path("demes_event_examples/deme_existence.yaml")
     validate_deme_existence(yaml)
 
 
 def test_generation_times_deme_sizes():
-    yaml = "tests/demes_event_examples/deme_existence_generation_time.yaml"
+    yaml = make_path("demes_event_examples/deme_existence_generation_time.yaml")
     validate_deme_existence(yaml)

tests/test_forward_demes_graph.py

@@ -4,7 +4,12 @@
 
 
 def into_graph(yaml, burnin=100, burnin_is_exact=True) -> fwdpy11.ForwardDemesGraph:
-    graph = demes.load(yaml)
+    import os
+
+    path_to_current_file = os.path.realpath(__file__)
+    current_directory = os.path.split(path_to_current_file)[0]
+    path_to_file = os.path.join(current_directory, yaml)
+    graph = demes.load(path_to_file)
     return fwdpy11.ForwardDemesGraph.from_demes(
         graph, burnin=burnin, burnin_is_exact=burnin_is_exact
     )
@@ -30,7 +35,7 @@ def validate_deme_times(demog):
 
 
 def test_single_pulse():
-    yaml = "tests/demes_event_examples/single_pulse.yaml"
+    yaml = "demes_event_examples/single_pulse.yaml"
     demog = into_graph(yaml)
     assert demog.to_forwards_time(demog.final_generation) == 0
     assert demog.to_forwards_time(-1) is None
@@ -42,7 +47,7 @@ def test_single_pulse():
 
 
 def test_burst_of_migration():
-    yaml = "tests/demes_event_examples/burst_of_migration.yaml"
+    yaml = "demes_event_examples/burst_of_migration.yaml"
     demog = into_graph(yaml)
     assert demog.to_forwards_time(demog.final_generation) == 0
     assert demog.to_forwards_time(-1) is None
@@ -52,7 +57,7 @@ def test_burst_of_migration():
 
 
 def test_deme_existence():
-    yaml = "tests/demes_event_examples/deme_existence.yaml"
+    yaml = "demes_event_examples/deme_existence.yaml"
     demog = into_graph(yaml)
     assert demog.to_forwards_time(demog.final_generation) == 0
     assert demog.to_forwards_time(-1) is None

tests/utils.py

@@ -0,0 +1,7 @@
+def make_path(yaml):
+    import os
+
+    path_to_current_file = os.path.realpath(__file__)
+    current_directory = os.path.split(path_to_current_file)[0]
+    path_to_file = os.path.join(current_directory, yaml)
+    return path_to_file