Implement declarative ext-modules in pyproject.toml ("experimental") (#4568)

Author: abravalheri

docs/userguide/ext_modules.rst

@@ -27,22 +27,41 @@ and all project metadata configuration in the ``pyproject.toml`` file:
    version = "0.42"
 
 To instruct setuptools to compile the ``foo.c`` file into the extension module
-``mylib.foo``, we need to add a ``setup.py`` file similar to the following:
+``mylib.foo``, we need to define an appropriate configuration in either
+``pyproject.toml`` [#pyproject.toml]_ or ``setup.py`` file ,
+similar to the following:
 
-.. code-block:: python
+.. tab:: pyproject.toml
 
-   from setuptools import Extension, setup
+    .. code-block:: toml
 
-   setup(
-       ext_modules=[
-           Extension(
-               name="mylib.foo",  # as it would be imported
-                                  # may include packages/namespaces separated by `.`
-
-               sources=["foo.c"], # all sources are compiled into a single binary file
-           ),
+       [tool.setuptools]
+       ext-modules = [
+         {name = "mylib.foo", sources = ["foo.c"]}
        ]
-   )
+
+.. tab:: setup.py
+
+    .. code-block:: python
+
+       from setuptools import Extension, setup
+
+       setup(
+           ext_modules=[
+               Extension(
+                   name="mylib.foo",
+                   sources=["foo.c"],
+               ),
+           ]
+       )
+
+The ``name`` value corresponds to how the extension module would be
+imported and may include packages/namespaces separated by ``.``.
+The ``sources`` value is a list of all source files that are compiled
+into a single binary file.
+Optionally any other parameter of :class:`setuptools.Extension` can be defined
+in the configuration file (but in the case of ``pyproject.toml`` they must be
+written using :wiki:`kebab-case` convention).
 
 .. seealso::
    You can find more information on the `Python docs about C/C++ extensions`_.
@@ -168,6 +187,16 @@ Extension API Reference
 .. autoclass:: setuptools.Extension
 
 
+----
+
+.. rubric:: Notes
+
+.. [#pyproject.toml]
+   Declarative configuration of extension modules via ``pyproject.toml`` was
+   introduced recently and is still considered experimental.
+   Therefore it might change in future versions of ``setuptools``.
+
+
 .. _Python docs about C/C++ extensions: https://docs.python.org/3/extending/extending.html
 .. _Cython: https://cython.readthedocs.io/en/stable/index.html
 .. _directory options: https://gcc.gnu.org/onlinedocs/gcc/Directory-Options.html

docs/userguide/pyproject_config.rst

@@ -88,6 +88,9 @@ file, and can be set via the ``tool.setuptools`` table:
 Key                       Value Type (TOML)           Notes
 ========================= =========================== =========================
 ``py-modules``            array                       See tip below.
+``ext-modules``           array of                    **Experimental** - Each item corresponds to a
+                          tables/inline-tables        :class:`setuptools.Extension` object and may define
+                                                      the associated parameters in :wiki:`kebab-case`.
 ``packages``              array or ``find`` directive See tip below.
 ``package-dir``           table/inline-table          Used when explicitly/manually listing ``packages``.
 ------------------------- --------------------------- -------------------------

newsfragments/4568.feature.rst

@@ -0,0 +1,2 @@
+Added support for defining ``ext-modules`` via ``pyproject.toml``
+(**EXPERIMENTAL**, may change in future releases).

setuptools/config/_apply_pyprojecttoml.py

@@ -17,10 +17,11 @@
 from inspect import cleandoc
 from itertools import chain
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Callable, Dict, Mapping, Union
+from typing import TYPE_CHECKING, Any, Callable, Dict, Mapping, TypeVar, Union
 
 from .._path import StrPath
 from ..errors import RemovedConfigError
+from ..extension import Extension
 from ..warnings import SetuptoolsWarning
 
 if TYPE_CHECKING:
@@ -35,6 +36,7 @@
 _ProjectReadmeValue: TypeAlias = Union[str, Dict[str, str]]
 _CorrespFn: TypeAlias = Callable[["Distribution", Any, StrPath], None]
 _Correspondence: TypeAlias = Union[str, _CorrespFn]
+_T = TypeVar("_T")
 
 _logger = logging.getLogger(__name__)
 
@@ -117,13 +119,14 @@ def json_compatible_key(key: str) -> str:
 
 
 def _set_config(dist: Distribution, field: str, value: Any):
+    val = _PREPROCESS.get(field, _noop)(dist, value)
     setter = getattr(dist.metadata, f"set_{field}", None)
     if setter:
-        setter(value)
+        setter(val)
     elif hasattr(dist.metadata, field) or field in SETUPTOOLS_PATCHES:
-        setattr(dist.metadata, field, value)
+        setattr(dist.metadata, field, val)
     else:
-        setattr(dist, field, value)
+        setattr(dist, field, val)
 
 
 _CONTENT_TYPES = {
@@ -218,6 +221,17 @@ def _optional_dependencies(dist: Distribution, val: dict, _root_dir):
     dist.extras_require = {**existing, **val}
 
 
+def _ext_modules(dist: Distribution, val: list[dict]) -> list[Extension]:
+    existing = dist.ext_modules or []
+    args = ({k.replace("-", "_"): v for k, v in x.items()} for x in val)
+    new = [Extension(**kw) for kw in args]
+    return [*existing, *new]
+
+
+def _noop(_dist: Distribution, val: _T) -> _T:
+    return val
+
+
 def _unify_entry_points(project_table: dict):
     project = project_table
     entry_points = project.pop("entry-points", project.pop("entry_points", {}))
@@ -376,6 +390,10 @@ def _acessor(obj):
     "license_files",
 }
 
+_PREPROCESS = {
+    "ext_modules": _ext_modules,
+}
+
 _PREVIOUSLY_DEFINED = {
     "name": _attrgetter("metadata.name"),
     "version": _attrgetter("metadata.version"),

setuptools/config/_validate_pyproject/fastjsonschema_validations.py

@@ -130,6 +130,9 @@ def read_configuration(
     asdict["tool"] = tool_table
     tool_table["setuptools"] = setuptools_table
 
+    if "ext-modules" in setuptools_table:
+        _ExperimentalConfiguration.emit(subject="[tool.setuptools.ext-modules]")
+
     with _ignore_errors(ignore_option_errors):
         # Don't complain about unrelated errors (e.g. tools not using the "tool" table)
         subset = {"project": project_table, "tool": {"setuptools": setuptools_table}}

setuptools/config/pyprojecttoml.py

@@ -158,6 +158,11 @@
       "items": {"type": "string", "format": "python-module-name-relaxed"},
       "$comment": "TODO: clarify the relationship with ``packages``"
     },
+    "ext-modules": {
+      "description": "Extension modules to be compiled by setuptools",
+      "type": "array",
+      "items": {"$ref": "#/definitions/ext-module"}
+    },
     "data-files": {
       "$$description": [
         "``dict``-like structure where each key represents a directory and",
@@ -254,6 +259,82 @@
         {"type": "string", "format": "pep561-stub-name"}
       ]
     },
+    "ext-module": {
+      "$id": "#/definitions/ext-module",
+      "title": "Extension module",
+      "description": "Parameters to construct a :class:`setuptools.Extension` object",
+      "type": "object",
+      "required": ["name", "sources"],
+      "additionalProperties": false,
+      "properties": {
+        "name": {
+          "type": "string",
+          "format": "python-module-name-relaxed"
+        },
+        "sources": {
+          "type": "array",
+          "items": {"type": "string"}
+        },
+        "include-dirs":{
+          "type": "array",
+          "items": {"type": "string"}
+        },
+        "define-macros": {
+          "type": "array",
+          "items": {
+            "type": "array",
+            "items": [
+              {"description": "macro name", "type": "string"},
+              {"description": "macro value", "oneOf": [{"type": "string"}, {"type": "null"}]}
+            ],
+            "additionalItems": false
+          }
+        },
+        "undef-macros": {
+          "type": "array",
+          "items": {"type": "string"}
+        },
+        "library-dirs": {
+          "type": "array",
+          "items": {"type": "string"}
+        },
+        "libraries": {
+          "type": "array",
+          "items": {"type": "string"}
+        },
+        "runtime-library-dirs": {
+          "type": "array",
+          "items": {"type": "string"}
+        },
+        "extra-objects": {
+          "type": "array",
+          "items": {"type": "string"}
+        },
+        "extra-compile-args": {
+          "type": "array",
+          "items": {"type": "string"}
+        },
+        "extra-link-args": {
+          "type": "array",
+          "items": {"type": "string"}
+        },
+        "export-symbols": {
+          "type": "array",
+          "items": {"type": "string"}
+        },
+        "swig-opts": {
+          "type": "array",
+          "items": {"type": "string"}
+        },
+        "depends": {
+          "type": "array",
+          "items": {"type": "string"}
+        },
+        "language": {"type": "string"},
+        "optional": {"type": "boolean"},
+        "py-limited-api": {"type": "boolean"}
+      }
+    },
     "file-directive": {
       "$id": "#/definitions/file-directive",
       "title": "'file:' directive",

setuptools/config/setuptools.schema.json

@@ -127,7 +127,7 @@ class Extension(_Extension):
     :keyword bool py_limited_api:
       opt-in flag for the usage of :doc:`Python's limited API <python:c-api/stable>`.
 
-    :raises setuptools.errors.PlatformError: if 'runtime_library_dirs' is
+    :raises setuptools.errors.PlatformError: if ``runtime_library_dirs`` is
       specified on Windows. (since v63)
     """
 

setuptools/extension.py

@@ -324,6 +324,27 @@ def test_invalid_module_name(self, tmp_path, monkeypatch, module):
             self.dist(module).py_modules
 
 
+class TestExtModules:
+    def test_pyproject_sets_attribute(self, tmp_path, monkeypatch):
+        monkeypatch.chdir(tmp_path)
+        pyproject = Path("pyproject.toml")
+        toml_config = """
+        [project]
+        name = "test"
+        version = "42.0"
+        [tool.setuptools]
+        ext-modules = [
+          {name = "my.ext", sources = ["hello.c", "world.c"]}
+        ]
+        """
+        pyproject.write_text(cleandoc(toml_config), encoding="utf-8")
+        with pytest.warns(pyprojecttoml._ExperimentalConfiguration):
+            dist = pyprojecttoml.apply_configuration(Distribution({}), pyproject)
+        assert len(dist.ext_modules) == 1
+        assert dist.ext_modules[0].name == "my.ext"
+        assert set(dist.ext_modules[0].sources) == {"hello.c", "world.c"}
+
+
 class TestDeprecatedFields:
     def test_namespace_packages(self, tmp_path):
         pyproject = tmp_path / "pyproject.toml"