Merge pull request #1499 from pypa/docker-flags

Add create_args suboption to CIBW_CONTAINER_ENGINE

Author: joerick

cibuildwheel/linux.py

@@ -379,7 +379,9 @@ def build(options: Options, tmp_path: Path) -> None:  # noqa: ARG001
     try:
         # check the container engine is installed
         subprocess.run(
-            [options.globals.container_engine, "--version"], check=True, stdout=subprocess.DEVNULL
+            [options.globals.container_engine.name, "--version"],
+            check=True,
+            stdout=subprocess.DEVNULL,
         )
     except subprocess.CalledProcessError:
         print(

cibuildwheel/oci_container.py

@@ -11,15 +11,44 @@
 import typing
 import uuid
 from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
 from pathlib import Path, PurePath, PurePosixPath
 from types import TracebackType
 from typing import IO, Dict
 
 from ._compat.typing import Literal
 from .typing import PathOrStr, PopenBytes
-from .util import CIProvider, detect_ci_provider
+from .util import CIProvider, detect_ci_provider, parse_key_value_string
 
-ContainerEngine = Literal["docker", "podman"]
+ContainerEngineName = Literal["docker", "podman"]
+
+
+@dataclass(frozen=True)
+class OCIContainerEngineConfig:
+    name: ContainerEngineName
+    create_args: Sequence[str] = ()
+
+    @staticmethod
+    def from_config_string(config_string: str) -> OCIContainerEngineConfig:
+        config_dict = parse_key_value_string(config_string, ["name"])
+        name = " ".join(config_dict["name"])
+        if name not in {"docker", "podman"}:
+            msg = f"unknown container engine {name}"
+            raise ValueError(msg)
+
+        name = typing.cast(ContainerEngineName, name)
+        # some flexibility in the option name to cope with TOML conventions
+        create_args = config_dict.get("create_args") or config_dict.get("create-args") or []
+        return OCIContainerEngineConfig(name=name, create_args=create_args)
+
+    def options_summary(self) -> str | dict[str, str]:
+        if not self.create_args:
+            return self.name
+        else:
+            return {"name": self.name, "create_args": repr(self.create_args)}
+
+
+DEFAULT_ENGINE = OCIContainerEngineConfig("docker")
 
 
 class OCIContainer:
@@ -57,7 +86,7 @@ def __init__(
         image: str,
         simulate_32_bit: bool = False,
         cwd: PathOrStr | None = None,
-        engine: ContainerEngine = "docker",
+        engine: OCIContainerEngineConfig = DEFAULT_ENGINE,
     ):
         if not image:
             msg = "Must have a non-empty image to run."
@@ -84,13 +113,14 @@ def __enter__(self) -> OCIContainer:
 
         subprocess.run(
             [
-                self.engine,
+                self.engine.name,
                 "create",
                 "--env=CIBUILDWHEEL",
                 f"--name={self.name}",
                 "--interactive",
                 "--volume=/:/host",  # ignored on CircleCI
                 *network_args,
+                *self.engine.create_args,
                 self.image,
                 *shell_args,
             ],
@@ -99,7 +129,7 @@ def __enter__(self) -> OCIContainer:
 
         self.process = subprocess.Popen(
             [
-                self.engine,
+                self.engine.name,
                 "start",
                 "--attach",
                 "--interactive",
@@ -137,7 +167,7 @@ def __exit__(
         self.bash_stdin.close()
         self.bash_stdout.close()
 
-        if self.engine == "podman":
+        if self.engine.name == "podman":
             # This works around what seems to be a race condition in the podman
             # backend. The full reason is not understood. See PR #966 for a
             # discussion on possible causes and attempts to remove this line.
@@ -147,7 +177,7 @@ def __exit__(
         assert isinstance(self.name, str)
 
         subprocess.run(
-            [self.engine, "rm", "--force", "-v", self.name],
+            [self.engine.name, "rm", "--force", "-v", self.name],
             stdout=subprocess.DEVNULL,
             check=False,
         )
@@ -162,7 +192,7 @@ def copy_into(self, from_path: Path, to_path: PurePath) -> None:
         if from_path.is_dir():
             self.call(["mkdir", "-p", to_path])
             subprocess.run(
-                f"tar cf - . | {self.engine} exec -i {self.name} tar --no-same-owner -xC {shell_quote(to_path)} -f -",
+                f"tar cf - . | {self.engine.name} exec -i {self.name} tar --no-same-owner -xC {shell_quote(to_path)} -f -",
                 shell=True,
                 check=True,
                 cwd=from_path,
@@ -171,7 +201,7 @@ def copy_into(self, from_path: Path, to_path: PurePath) -> None:
             exec_process: subprocess.Popen[bytes]
             with subprocess.Popen(
                 [
-                    self.engine,
+                    self.engine.name,
                     "exec",
                     "-i",
                     str(self.name),
@@ -198,29 +228,29 @@ def copy_out(self, from_path: PurePath, to_path: Path) -> None:
         # note: we assume from_path is a dir
         to_path.mkdir(parents=True, exist_ok=True)
 
-        if self.engine == "podman":
+        if self.engine.name == "podman":
             subprocess.run(
                 [
-                    self.engine,
+                    self.engine.name,
                     "cp",
                     f"{self.name}:{from_path}/.",
                     str(to_path),
                 ],
                 check=True,
                 cwd=to_path,
             )
-        elif self.engine == "docker":
+        elif self.engine.name == "docker":
             # There is a bug in docker that prevents a simple 'cp' invocation
             # from working https://github.com/moby/moby/issues/38995
-            command = f"{self.engine} exec -i {self.name} tar -cC {shell_quote(from_path)} -f - . | tar -xf -"
+            command = f"{self.engine.name} exec -i {self.name} tar -cC {shell_quote(from_path)} -f - . | tar -xf -"
             subprocess.run(
                 command,
                 shell=True,
                 check=True,
                 cwd=to_path,
             )
         else:
-            raise KeyError(self.engine)
+            raise KeyError(self.engine.name)
 
     def glob(self, path: PurePosixPath, pattern: str) -> list[PurePosixPath]:
         glob_pattern = path.joinpath(pattern)
@@ -338,10 +368,10 @@ def environment_executor(self, command: Sequence[str], environment: dict[str, st
         return self.call(command, env=environment, capture_output=True)
 
     def debug_info(self) -> str:
-        if self.engine == "podman":
-            command = f"{self.engine} info --debug"
+        if self.engine.name == "podman":
+            command = f"{self.engine.name} info --debug"
         else:
-            command = f"{self.engine} info"
+            command = f"{self.engine.name} info"
         completed = subprocess.run(
             command,
             shell=True,

cibuildwheel/options.py

@@ -10,7 +10,6 @@
 import sys
 import textwrap
 import traceback
-import typing
 from collections.abc import Callable, Generator, Iterable, Iterator, Mapping, Set
 from pathlib import Path
 from typing import Any, Dict, List, Union
@@ -22,7 +21,7 @@
 from .architecture import Architecture
 from .environment import EnvironmentParseError, ParsedEnvironment, parse_environment
 from .logger import log
-from .oci_container import ContainerEngine
+from .oci_container import OCIContainerEngineConfig
 from .projectfiles import get_requires_python_str
 from .typing import PLATFORMS, PlatformName
 from .util import (
@@ -75,7 +74,7 @@ class GlobalOptions:
     build_selector: BuildSelector
     test_selector: TestSelector
     architectures: set[Architecture]
-    container_engine: ContainerEngine
+    container_engine: OCIContainerEngineConfig
 
 
 @dataclasses.dataclass(frozen=True)
@@ -136,8 +135,14 @@ class Override:
 
 
 class TableFmt(TypedDict):
+    # a format string, used with '.format', with `k` and `v` parameters
+    # e.g. "{k}={v}"
     item: str
+    # the string that is inserted between items
+    # e.g. " "
     sep: str
+    # a quoting function that, if supplied, is called to quote each value
+    # e.g. shlex.quote
     quote: NotRequired[Callable[[str], str]]
 
 
@@ -454,15 +459,17 @@ def globals(self) -> GlobalOptions:
         )
         test_selector = TestSelector(skip_config=test_skip)
 
-        container_engine_str = self.reader.get("container-engine")
+        container_engine_str = self.reader.get(
+            "container-engine", table={"item": "{k}:{v}", "sep": "; ", "quote": shlex.quote}
+        )
 
-        if container_engine_str not in ["docker", "podman"]:
-            msg = f"cibuildwheel: Unrecognised container_engine {container_engine_str!r}, only 'docker' and 'podman' are supported"
+        try:
+            container_engine = OCIContainerEngineConfig.from_config_string(container_engine_str)
+        except ValueError as e:
+            msg = f"cibuildwheel: Failed to parse container config. {e}"
             print(msg, file=sys.stderr)
             sys.exit(2)
 
-        container_engine = typing.cast(ContainerEngine, container_engine_str)
-
         return GlobalOptions(
             package_dir=package_dir,
             output_dir=output_dir,

cibuildwheel/util.py

@@ -13,6 +13,7 @@
 import time
 import typing
 import urllib.request
+from collections import defaultdict
 from collections.abc import Generator, Iterable, Mapping, Sequence
 from dataclasses import dataclass
 from enum import Enum
@@ -697,3 +698,40 @@ def fix_ansi_codes_for_github_actions(text: str) -> str:
                     ansi_codes.append(code)
 
     return output
+
+
+def parse_key_value_string(
+    key_value_string: str, positional_arg_names: list[str] | None = None
+) -> dict[str, list[str]]:
+    """
+    Parses a string like "docker; create_args: --some-option=value another-option"
+    """
+    if positional_arg_names is None:
+        positional_arg_names = []
+
+    shlexer = shlex.shlex(key_value_string, posix=True, punctuation_chars=";:")
+    shlexer.commenters = ""
+    parts = list(shlexer)
+    # parts now looks like
+    # ['docker', ';', 'create_args',':', '--some-option=value', 'another-option']
+
+    # split by semicolon
+    fields = [list(group) for k, group in itertools.groupby(parts, lambda x: x == ";") if not k]
+
+    result: dict[str, list[str]] = defaultdict(list)
+    for field_i, field in enumerate(fields):
+        if len(field) > 1 and field[1] == ":":
+            field_name = field[0]
+            values = field[2:]
+        else:
+            try:
+                field_name = positional_arg_names[field_i]
+            except IndexError:
+                msg = f"Failed to parse {key_value_string!r}. Too many positional arguments - expected a maximum of {len(positional_arg_names)}"
+                raise ValueError(msg) from None
+
+            values = field
+
+        result[field_name] += values
+
+    return result

docs/options.md

@@ -1048,9 +1048,12 @@ Auditwheel detects the version of the manylinux / musllinux standard in the imag
 
 
 ### `CIBW_CONTAINER_ENGINE` {: #container-engine}
-> Specify which container engine to use when building Linux wheels
+> Specify the container engine to use when building Linux wheels
 
-Options: `docker` `podman`
+Options:
+
+- `docker[;create_args: ...]`
+- `podman[;create_args: ...]`
 
 Default: `docker`
 
@@ -1059,6 +1062,12 @@ Set the container engine to use. Docker is the default, or you can switch to
 running and `docker` available on PATH. To use Podman, it needs to be
 installed and `podman` available on PATH.
 
+Arguments can be supplied to the container engine. Currently, the only option
+that's customisable is 'create_args'. Parameters to create_args are
+space-separated strings, which are passed to the container engine on the
+command line when it's creating the container. If you want to include spaces
+inside a parameter, use shell-style quoting.
+
 !!! tip
 
     While most users will stick with Docker, Podman is available in different
@@ -1073,14 +1082,22 @@ installed and `podman` available on PATH.
 !!! tab examples "Environment variables"
 
     ```yaml
+    # use podman instead of docker
     CIBW_CONTAINER_ENGINE: podman
+
+    # pass command line options to 'docker create'
+    CIBW_CONTAINER_ENGINE: "docker; create_args: --gpus all"
     ```
 
 !!! tab examples "pyproject.toml"
 
     ```toml
     [tool.cibuildwheel]
+    # use podman instead of docker
     container-engine = "podman"
+
+    # pass command line options to 'docker create'
+    container-engine = { name = "docker", create-args = ["--gpus", "all"]}
     ```
 
 

test/test_podman.py renamed to test/test_container_engine.py

@@ -7,7 +7,7 @@
 basic_project = test_projects.new_c_project()
 
 
-def test(tmp_path, capfd, request):
+def test_podman(tmp_path, capfd, request):
     if utils.platform != "linux":
         pytest.skip("the test is only relevant to the linux build")
 
@@ -38,3 +38,29 @@ def test(tmp_path, capfd, request):
     # check that stdout is bring passed-though from container correctly
     captured = capfd.readouterr()
     assert "test log statement from before-all" in captured.out
+
+
+def test_create_args(tmp_path, capfd):
+    if utils.platform != "linux":
+        pytest.skip("the test is only relevant to the linux build")
+
+    project_dir = tmp_path / "project"
+    basic_project.generate(project_dir)
+
+    # build a manylinux wheel, using create_args to set an environment variable
+    actual_wheels = utils.cibuildwheel_run(
+        project_dir,
+        add_env={
+            "CIBW_BUILD": "cp310-manylinux_*",
+            "CIBW_BEFORE_ALL": "echo TEST_CREATE_ARGS is set to $TEST_CREATE_ARGS",
+            "CIBW_CONTAINER_ENGINE": "docker; create_args: --env=TEST_CREATE_ARGS=itworks",
+        },
+    )
+
+    expected_wheels = [
+        w for w in utils.expected_wheels("spam", "0.1.0") if ("cp310-manylinux" in w)
+    ]
+    assert set(actual_wheels) == set(expected_wheels)
+
+    captured = capfd.readouterr()
+    assert "TEST_CREATE_ARGS is set to itworks" in captured.out