feature: add musllinux_1_2 support (#1561)

* feature: add musllinux_1_2 support

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix[test]: clean-up docker images in test_*linux*_only.py

We are reaching disk space limits (14 GB) when running tests.
Add a fixture to clean-up docker images after tests.
This fixture is applied on tests that pull a specific image for one test only in order not to take too much time pulling that image many times.

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: mayeut

bin/update_docker.py

@@ -59,6 +59,12 @@ class Image:
     Image("musllinux_1_1", "aarch64", "quay.io/pypa/musllinux_1_1_aarch64", None),
     Image("musllinux_1_1", "ppc64le", "quay.io/pypa/musllinux_1_1_ppc64le", None),
     Image("musllinux_1_1", "s390x", "quay.io/pypa/musllinux_1_1_s390x", None),
+    # musllinux_1_2 images
+    Image("musllinux_1_2", "x86_64", "quay.io/pypa/musllinux_1_2_x86_64", None),
+    Image("musllinux_1_2", "i686", "quay.io/pypa/musllinux_1_2_i686", None),
+    Image("musllinux_1_2", "aarch64", "quay.io/pypa/musllinux_1_2_aarch64", None),
+    Image("musllinux_1_2", "ppc64le", "quay.io/pypa/musllinux_1_2_ppc64le", None),
+    Image("musllinux_1_2", "s390x", "quay.io/pypa/musllinux_1_2_s390x", None),
 ]
 
 config = configparser.ConfigParser()

cibuildwheel/resources/pinned_docker_images.cfg

@@ -1,49 +1,54 @@
 [x86_64]
 manylinux1 = quay.io/pypa/manylinux1_x86_64:2023-06-25-d2e0575
 manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2022-08-05-4535177
-manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2023-07-14-55e4124
+manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2023-07-29-8793e83
 manylinux_2_24 = quay.io/pypa/manylinux_2_24_x86_64:2022-12-26-0d38463
-manylinux_2_28 = quay.io/pypa/manylinux_2_28_x86_64:2023-07-14-55e4124
-musllinux_1_1 = quay.io/pypa/musllinux_1_1_x86_64:2023-07-14-55e4124
+manylinux_2_28 = quay.io/pypa/manylinux_2_28_x86_64:2023-07-29-8793e83
+musllinux_1_1 = quay.io/pypa/musllinux_1_1_x86_64:2023-07-29-8793e83
+musllinux_1_2 = quay.io/pypa/musllinux_1_2_x86_64:2023-07-29-8793e83
 
 [i686]
 manylinux1 = quay.io/pypa/manylinux1_i686:2023-06-25-d2e0575
 manylinux2010 = quay.io/pypa/manylinux2010_i686:2022-08-05-4535177
-manylinux2014 = quay.io/pypa/manylinux2014_i686:2023-07-14-55e4124
+manylinux2014 = quay.io/pypa/manylinux2014_i686:2023-07-29-8793e83
 manylinux_2_24 = quay.io/pypa/manylinux_2_24_i686:2022-12-26-0d38463
-musllinux_1_1 = quay.io/pypa/musllinux_1_1_i686:2023-07-14-55e4124
+musllinux_1_1 = quay.io/pypa/musllinux_1_1_i686:2023-07-29-8793e83
+musllinux_1_2 = quay.io/pypa/musllinux_1_2_i686:2023-07-29-8793e83
 
 [pypy_x86_64]
 manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2022-08-05-4535177
-manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2023-07-14-55e4124
+manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2023-07-29-8793e83
 manylinux_2_24 = quay.io/pypa/manylinux_2_24_x86_64:2022-12-26-0d38463
-manylinux_2_28 = quay.io/pypa/manylinux_2_28_x86_64:2023-07-14-55e4124
+manylinux_2_28 = quay.io/pypa/manylinux_2_28_x86_64:2023-07-29-8793e83
 
 [pypy_i686]
 manylinux2010 = quay.io/pypa/manylinux2010_i686:2022-08-05-4535177
-manylinux2014 = quay.io/pypa/manylinux2014_i686:2023-07-14-55e4124
+manylinux2014 = quay.io/pypa/manylinux2014_i686:2023-07-29-8793e83
 manylinux_2_24 = quay.io/pypa/manylinux_2_24_i686:2022-12-26-0d38463
 
 [aarch64]
-manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2023-07-14-55e4124
+manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2023-07-29-8793e83
 manylinux_2_24 = quay.io/pypa/manylinux_2_24_aarch64:2022-12-26-0d38463
-manylinux_2_28 = quay.io/pypa/manylinux_2_28_aarch64:2023-07-14-55e4124
-musllinux_1_1 = quay.io/pypa/musllinux_1_1_aarch64:2023-07-14-55e4124
+manylinux_2_28 = quay.io/pypa/manylinux_2_28_aarch64:2023-07-29-8793e83
+musllinux_1_1 = quay.io/pypa/musllinux_1_1_aarch64:2023-07-29-8793e83
+musllinux_1_2 = quay.io/pypa/musllinux_1_2_aarch64:2023-07-29-8793e83
 
 [ppc64le]
-manylinux2014 = quay.io/pypa/manylinux2014_ppc64le:2023-07-14-55e4124
+manylinux2014 = quay.io/pypa/manylinux2014_ppc64le:2023-07-29-8793e83
 manylinux_2_24 = quay.io/pypa/manylinux_2_24_ppc64le:2022-12-26-0d38463
-manylinux_2_28 = quay.io/pypa/manylinux_2_28_ppc64le:2023-07-14-55e4124
-musllinux_1_1 = quay.io/pypa/musllinux_1_1_ppc64le:2023-07-14-55e4124
+manylinux_2_28 = quay.io/pypa/manylinux_2_28_ppc64le:2023-07-29-8793e83
+musllinux_1_1 = quay.io/pypa/musllinux_1_1_ppc64le:2023-07-29-8793e83
+musllinux_1_2 = quay.io/pypa/musllinux_1_2_ppc64le:2023-07-29-8793e83
 
 [s390x]
-manylinux2014 = quay.io/pypa/manylinux2014_s390x:2023-07-14-55e4124
+manylinux2014 = quay.io/pypa/manylinux2014_s390x:2023-07-29-8793e83
 manylinux_2_24 = quay.io/pypa/manylinux_2_24_s390x:2022-12-26-0d38463
-manylinux_2_28 = quay.io/pypa/manylinux_2_28_s390x:2023-07-14-55e4124
-musllinux_1_1 = quay.io/pypa/musllinux_1_1_s390x:2023-07-14-55e4124
+manylinux_2_28 = quay.io/pypa/manylinux_2_28_s390x:2023-07-29-8793e83
+musllinux_1_1 = quay.io/pypa/musllinux_1_1_s390x:2023-07-29-8793e83
+musllinux_1_2 = quay.io/pypa/musllinux_1_2_s390x:2023-07-29-8793e83
 
 [pypy_aarch64]
-manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2023-07-14-55e4124
+manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2023-07-29-8793e83
 manylinux_2_24 = quay.io/pypa/manylinux_2_24_aarch64:2022-12-26-0d38463
-manylinux_2_28 = quay.io/pypa/manylinux_2_28_aarch64:2023-07-14-55e4124
+manylinux_2_28 = quay.io/pypa/manylinux_2_28_aarch64:2023-07-29-8793e83
 

docs/options.md

@@ -981,7 +981,7 @@ Set an alternative Docker image to be used for building [manylinux / musllinux](
 For `CIBW_MANYLINUX_*_IMAGE`, the value of this option can either be set to `manylinux1`, `manylinux2010`, `manylinux2014`, `manylinux_2_24` or `manylinux_2_28` to use a pinned version of the [official manylinux images](https://github.com/pypa/manylinux). Alternatively, set these options to any other valid Docker image name. For PyPy, the `manylinux1` image is not available. For architectures other
 than x86 (x86\_64 and i686) `manylinux2014`, `manylinux_2_24` or `manylinux_2_28` must be used, because the first version of the manylinux specification that supports additional architectures is `manylinux2014`. `manylinux_2_28` is not supported for `i686` architecture.
 
-For `CIBW_MUSLLINUX_*_IMAGE`, the value of this option can either be set to `musllinux_1_1` to use a pinned version of the [official musllinux images](https://github.com/pypa/musllinux). Alternatively, set these options to any other valid Docker image name.
+For `CIBW_MUSLLINUX_*_IMAGE`, the value of this option can either be set to `musllinux_1_1` or `musllinux_1_2` to use a pinned version of the [official musllinux images](https://github.com/pypa/musllinux). Alternatively, set these options to any other valid Docker image name.
 
 If this option is blank, it will fall though to the next available definition (environment variable -> pyproject.toml -> default).
 

test/conftest.py

@@ -1,7 +1,15 @@
 from __future__ import annotations
 
+import json
+import subprocess
+from typing import Generator
+
 import pytest
 
+from cibuildwheel.util import detect_ci_provider
+
+from .utils import platform
+
 
 def pytest_addoption(parser) -> None:
     parser.addoption(
@@ -21,3 +29,29 @@ def pytest_addoption(parser) -> None:
 )
 def build_frontend_env(request) -> dict[str, str]:
     return request.param  # type: ignore[no-any-return]
+
+
+@pytest.fixture()
+def docker_cleanup() -> Generator[None, None, None]:
+    def get_images() -> set[str]:
+        images = subprocess.run(
+            ["docker", "image", "ls", "--format", "{{json .ID}}"],
+            text=True,
+            check=True,
+            stdout=subprocess.PIPE,
+        ).stdout
+        return {json.loads(image.strip()) for image in images.splitlines() if image.strip()}
+
+    if detect_ci_provider() is None or platform != "linux":
+        try:
+            yield
+        finally:
+            pass
+        return
+    images_before = get_images()
+    try:
+        yield
+    finally:
+        images_after = get_images()
+        for image in images_after - images_before:
+            subprocess.run(["docker", "rmi", image], check=False)

test/test_container_images.py

@@ -23,6 +23,7 @@
 )
 
 
+@pytest.mark.usefixtures("docker_cleanup")
 def test(tmp_path):
     if utils.platform != "linux":
         pytest.skip("the test is only relevant to the linux build")

test/test_manylinuxXXXX_only.py

@@ -53,6 +53,7 @@
     "manylinux_image",
     ["manylinux1", "manylinux2010", "manylinux2014", "manylinux_2_24", "manylinux_2_28"],
 )
+@pytest.mark.usefixtures("docker_cleanup")
 def test(manylinux_image, tmp_path):
     if utils.platform != "linux":
         pytest.skip("the container image test is only relevant to the linux build")

test/test_musllinux_X_Y_only.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+import textwrap
+
+import pytest
+
+from . import test_projects, utils
+
+project_with_manylinux_symbols = test_projects.new_c_project(
+    spam_c_top_level_add=textwrap.dedent(
+        r"""
+        #include <stdlib.h>
+
+        #if defined(__GLIBC_PREREQ)
+        #error "Must not run on a glibc linux environment"
+        #endif
+        """
+    ),
+    spam_c_function_add=textwrap.dedent(
+        r"""
+        sts = 0;
+        """
+    ),
+)
+
+
+@pytest.mark.parametrize(
+    "musllinux_image",
+    ["musllinux_1_1", "musllinux_1_2"],
+)
+@pytest.mark.usefixtures("docker_cleanup")
+def test(musllinux_image, tmp_path):
+    if utils.platform != "linux":
+        pytest.skip("the container image test is only relevant to the linux build")
+
+    project_dir = tmp_path / "project"
+    project_with_manylinux_symbols.generate(project_dir)
+
+    # build the wheels
+    add_env = {
+        "CIBW_BUILD": "*-musllinux*",
+        "CIBW_MUSLLINUX_X86_64_IMAGE": musllinux_image,
+        "CIBW_MUSLLINUX_I686_IMAGE": musllinux_image,
+        "CIBW_MUSLLINUX_AARCH64_IMAGE": musllinux_image,
+        "CIBW_MUSLLINUX_PPC64LE_IMAGE": musllinux_image,
+        "CIBW_MUSLLINUX_S390X_IMAGE": musllinux_image,
+    }
+
+    actual_wheels = utils.cibuildwheel_run(project_dir, add_env=add_env)
+    expected_wheels = utils.expected_wheels(
+        "spam",
+        "0.1.0",
+        manylinux_versions=[],
+        musllinux_versions=[musllinux_image],
+    )
+    assert set(actual_wheels) == set(expected_wheels)

test/utils.py

@@ -205,13 +205,14 @@ def expected_wheels(
             if machine_arch == "x86_64":
                 architectures.append("i686")
 
-            platform_tags = [
-                ".".join(
-                    f"{manylinux_version}_{architecture}"
-                    for manylinux_version in manylinux_versions
-                )
-                for architecture in architectures
-            ]
+            if len(manylinux_versions) > 0:
+                platform_tags = [
+                    ".".join(
+                        f"{manylinux_version}_{architecture}"
+                        for manylinux_version in manylinux_versions
+                    )
+                    for architecture in architectures
+                ]
             if len(musllinux_versions) > 0 and not python_abi_tag.startswith("pp"):
                 platform_tags.extend(
                     [

unit_test/option_prepare_test.py

@@ -101,13 +101,15 @@ def test_build_with_override_launches(monkeypatch, tmp_path):
     cibw_toml.write_text(
         """
 [tool.cibuildwheel]
-manylinux-x86_64-image = "manylinux_2_24"
+manylinux-x86_64-image = "manylinux_2_28"
+musllinux-x86_64-image = "musllinux_1_2"
 
-# Before Python 3.10, use manylinux2014
+# Before Python 3.10, use manylinux2014, musllinux_1_1
 [[tool.cibuildwheel.overrides]]
 select = "cp3?-*"
 manylinux-x86_64-image = "manylinux2014"
 manylinux-i686-image = "manylinux2014"
+musllinux-x86_64-image = "musllinux_1_1"
 
 [[tool.cibuildwheel.overrides]]
 select = "cp36-manylinux_x86_64"
@@ -122,7 +124,7 @@ def test_build_with_override_launches(monkeypatch, tmp_path):
 
     build_in_container = typing.cast(mock.Mock, linux.build_in_container)
 
-    assert build_in_container.call_count == 6
+    assert build_in_container.call_count == 7
 
     kwargs = build_in_container.call_args_list[0][1]
     assert "quay.io/pypa/manylinux2014_x86_64" in kwargs["container"]["image"]
@@ -146,7 +148,7 @@ def test_build_with_override_launches(monkeypatch, tmp_path):
     assert kwargs["options"].build_options("cp37-manylinux_x86_64").before_all == ""
 
     kwargs = build_in_container.call_args_list[2][1]
-    assert "quay.io/pypa/manylinux_2_24_x86_64" in kwargs["container"]["image"]
+    assert "quay.io/pypa/manylinux_2_28_x86_64" in kwargs["container"]["image"]
     assert kwargs["container"]["cwd"] == PurePosixPath("/project")
     assert not kwargs["container"]["simulate_32_bit"]
     identifiers = {x.identifier for x in kwargs["platform_configs"]}
@@ -169,10 +171,19 @@ def test_build_with_override_launches(monkeypatch, tmp_path):
 
     identifiers = {x.identifier for x in kwargs["platform_configs"]}
     assert identifiers == {
-        f"{x}-musllinux_x86_64" for x in ALL_IDS for x in ALL_IDS if "pp" not in x
+        f"{x}-musllinux_x86_64" for x in ALL_IDS & {"cp36", "cp37", "cp38", "cp39"} if "pp" not in x
     }
 
     kwargs = build_in_container.call_args_list[5][1]
+    assert "quay.io/pypa/musllinux_1_2_x86_64" in kwargs["container"]["image"]
+    assert kwargs["container"]["cwd"] == PurePosixPath("/project")
+    assert not kwargs["container"]["simulate_32_bit"]
+    identifiers = {x.identifier for x in kwargs["platform_configs"]}
+    assert identifiers == {
+        f"{x}-musllinux_x86_64" for x in ALL_IDS - {"cp36", "cp37", "cp38", "cp39"} if "pp" not in x
+    }
+
+    kwargs = build_in_container.call_args_list[6][1]
     assert "quay.io/pypa/musllinux_1_1_i686" in kwargs["container"]["image"]
     assert kwargs["container"]["cwd"] == PurePosixPath("/project")
     assert kwargs["container"]["simulate_32_bit"]