Skip to content

Commit cf9351d

Browse files
committed
Automatically add files referenced by configuration to sdist (#3779)
2 parents ca3198a + 58fa95e commit cf9351d

File tree

9 files changed

+103
-22
lines changed

9 files changed

+103
-22
lines changed

changelog.d/3779.change.rst

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Files referenced by ``file:`` in ``setup.cfg`` and by ``project.readme.file``,
2+
``project.license.file`` or ``tool.setuptools.dynamic.*.file`` in
3+
``pyproject.toml`` are now automatically included in the generated sdists.

docs/userguide/declarative_config.rst

+8-5
Original file line numberDiff line numberDiff line change
@@ -169,11 +169,14 @@ Special directives:
169169
The ``file:`` directive is sandboxed and won't reach anything outside the
170170
project directory (i.e. the directory containing ``setup.cfg``/``pyproject.toml``).
171171

172-
.. attention::
173-
When using the ``file:`` directive, please make sure that all necessary
174-
files are included in the ``sdist``. You can do that via ``MANIFEST.in``
175-
or using plugins such as ``setuptools-scm``.
176-
Please have a look on :doc:`/userguide/miscellaneous` for more information.
172+
.. note::
173+
If you are using an old version of ``setuptools``, you might need to ensure
174+
that all files referenced by the ``file:`` directive are included in the ``sdist``
175+
(you can do that via ``MANIFEST.in`` or using plugins such as ``setuptools-scm``,
176+
please have a look on :doc:`/userguide/miscellaneous` for more information).
177+
178+
.. versionchanged:: 66.1.0
179+
Newer versions of ``setuptools`` will automatically add these files to the ``sdist``.
177180

178181

179182
Metadata

docs/userguide/pyproject_config.rst

+8-5
Original file line numberDiff line numberDiff line change
@@ -220,11 +220,14 @@ however please keep in mind that all non-comment lines must conform with :pep:`5
220220
(``pip``-specify syntaxes, e.g. ``-c/-r/-e`` flags, are not supported).
221221

222222

223-
.. attention::
224-
When using the ``file`` directive, please make sure that all necessary
225-
files are included in the ``sdist``. You can do that via ``MANIFEST.in``
226-
or using plugins such as ``setuptools-scm``.
227-
Please have a look on :doc:`/userguide/miscellaneous` for more information.
223+
.. note::
224+
If you are using an old version of ``setuptools``, you might need to ensure
225+
that all files referenced by the ``file`` directive are included in the ``sdist``
226+
(you can do that via ``MANIFEST.in`` or using plugins such as ``setuptools-scm``,
227+
please have a look on :doc:`/userguide/miscellaneous` for more information).
228+
229+
.. versionchanged:: 66.1.0
230+
Newer versions of ``setuptools`` will automatically add these files to the ``sdist``.
228231

229232
----
230233

setuptools/command/egg_info.py

+9-1
Original file line numberDiff line numberDiff line change
@@ -565,6 +565,7 @@ def run(self):
565565
if os.path.exists(self.template):
566566
self.read_template()
567567
self.add_license_files()
568+
self._add_referenced_files()
568569
self.prune_file_list()
569570
self.filelist.sort()
570571
self.filelist.remove_duplicates()
@@ -619,9 +620,16 @@ def add_license_files(self):
619620
license_files = self.distribution.metadata.license_files or []
620621
for lf in license_files:
621622
log.info("adding license file '%s'", lf)
622-
pass
623623
self.filelist.extend(license_files)
624624

625+
def _add_referenced_files(self):
626+
"""Add files referenced by the config (e.g. `file:` directive) to filelist"""
627+
referenced = getattr(self.distribution, '_referenced_files', [])
628+
# ^-- fallback if dist comes from distutils or is a custom class
629+
for rf in referenced:
630+
log.debug("adding file referenced by config '%s'", rf)
631+
self.filelist.extend(referenced)
632+
625633
def prune_file_list(self):
626634
build = self.get_finalized_command('build')
627635
base_dir = self.distribution.get_fullname()

setuptools/config/_apply_pyprojecttoml.py

+10-3
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
from itertools import chain
1717
from types import MappingProxyType
1818
from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set, Tuple,
19-
Type, Union)
19+
Type, Union, cast)
2020

2121
from setuptools._deprecation_warning import SetuptoolsDeprecationWarning
2222

@@ -142,22 +142,29 @@ def _long_description(dist: "Distribution", val: _DictOrStr, root_dir: _Path):
142142
from setuptools.config import expand
143143

144144
if isinstance(val, str):
145-
text = expand.read_files(val, root_dir)
145+
file: Union[str, list] = val
146+
text = expand.read_files(file, root_dir)
146147
ctype = _guess_content_type(val)
147148
else:
148-
text = val.get("text") or expand.read_files(val.get("file", []), root_dir)
149+
file = val.get("file") or []
150+
text = val.get("text") or expand.read_files(file, root_dir)
149151
ctype = val["content-type"]
150152

151153
_set_config(dist, "long_description", text)
154+
152155
if ctype:
153156
_set_config(dist, "long_description_content_type", ctype)
154157

158+
if file:
159+
dist._referenced_files.add(cast(str, file))
160+
155161

156162
def _license(dist: "Distribution", val: dict, root_dir: _Path):
157163
from setuptools.config import expand
158164

159165
if "file" in val:
160166
_set_config(dist, "license", expand.read_files([val["file"]], root_dir))
167+
dist._referenced_files.add(val["file"])
161168
else:
162169
_set_config(dist, "license", val["text"])
163170

setuptools/config/pyprojecttoml.py

+6-3
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
import warnings
99
from contextlib import contextmanager
1010
from functools import partial
11-
from typing import TYPE_CHECKING, Callable, Dict, Optional, Mapping, Union
11+
from typing import TYPE_CHECKING, Callable, Dict, Optional, Mapping, Set, Union
1212

1313
from setuptools.errors import FileError, OptionError
1414

@@ -84,8 +84,8 @@ def read_configuration(
8484
8585
:param Distribution|None: Distribution object to which the configuration refers.
8686
If not given a dummy object will be created and discarded after the
87-
configuration is read. This is used for auto-discovery of packages in the case
88-
a dynamic configuration (e.g. ``attr`` or ``cmdclass``) is expanded.
87+
configuration is read. This is used for auto-discovery of packages and in the
88+
case a dynamic configuration (e.g. ``attr`` or ``cmdclass``) is expanded.
8989
When ``expand=False`` this object is simply ignored.
9090
9191
:rtype: dict
@@ -211,6 +211,7 @@ def __init__(
211211
self.dynamic_cfg = self.setuptools_cfg.get("dynamic", {})
212212
self.ignore_option_errors = ignore_option_errors
213213
self._dist = dist
214+
self._referenced_files: Set[str] = set()
214215

215216
def _ensure_dist(self) -> "Distribution":
216217
from setuptools.dist import Distribution
@@ -241,6 +242,7 @@ def expand(self):
241242
self._expand_cmdclass(package_dir)
242243
self._expand_all_dynamic(dist, package_dir)
243244

245+
dist._referenced_files.update(self._referenced_files)
244246
return self.config
245247

246248
def _expand_packages(self):
@@ -310,6 +312,7 @@ def _expand_directive(
310312
with _ignore_errors(self.ignore_option_errors):
311313
root_dir = self.root_dir
312314
if "file" in directive:
315+
self._referenced_files.update(directive["file"])
313316
return _expand.read_files(directive["file"], root_dir)
314317
if "attr" in directive:
315318
return _expand.read_attr(directive["attr"], package_dir, root_dir)

setuptools/config/setupcfg.py

+11-4
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
from functools import partial
1313
from functools import wraps
1414
from typing import (TYPE_CHECKING, Callable, Any, Dict, Generic, Iterable, List,
15-
Optional, Tuple, TypeVar, Union)
15+
Optional, Set, Tuple, TypeVar, Union)
1616

1717
from distutils.errors import DistutilsOptionError, DistutilsFileError
1818
from setuptools.extern.packaging.requirements import Requirement, InvalidRequirement
@@ -172,6 +172,9 @@ def parse_configuration(
172172
distribution.src_root,
173173
)
174174
meta.parse()
175+
distribution._referenced_files.update(
176+
options._referenced_files, meta._referenced_files
177+
)
175178

176179
return meta, options
177180

@@ -247,6 +250,10 @@ def __init__(
247250
self.sections = sections
248251
self.set_options: List[str] = []
249252
self.ensure_discovered = ensure_discovered
253+
self._referenced_files: Set[str] = set()
254+
"""After parsing configurations, this property will enumerate
255+
all files referenced by the "file:" directive. Private API for setuptools only.
256+
"""
250257

251258
@property
252259
def parsers(self):
@@ -365,8 +372,7 @@ def parser(value):
365372

366373
return parser
367374

368-
@classmethod
369-
def _parse_file(cls, value, root_dir: _Path):
375+
def _parse_file(self, value, root_dir: _Path):
370376
"""Represents value as a string, allowing including text
371377
from nearest files using `file:` directive.
372378
@@ -388,7 +394,8 @@ def _parse_file(cls, value, root_dir: _Path):
388394
return value
389395

390396
spec = value[len(include_directive) :]
391-
filepaths = (path.strip() for path in spec.split(','))
397+
filepaths = [path.strip() for path in spec.split(',')]
398+
self._referenced_files.update(filepaths)
392399
return expand.read_files(filepaths, root_dir)
393400

394401
def _parse_attr(self, value, package_dir, root_dir: _Path):

setuptools/dist.py

+6-1
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
from glob import iglob
1818
import itertools
1919
import textwrap
20-
from typing import List, Optional, TYPE_CHECKING
20+
from typing import List, Optional, Set, TYPE_CHECKING
2121
from pathlib import Path
2222

2323
from collections import defaultdict
@@ -481,6 +481,11 @@ def __init__(self, attrs=None):
481481
},
482482
)
483483

484+
# Private API (setuptools-use only, not restricted to Distribution)
485+
# Stores files that are referenced by the configuration and need to be in the
486+
# sdist (e.g. `version = file: VERSION.txt`)
487+
self._referenced_files: Set[str] = set()
488+
484489
# Save the original dependencies before they are processed into the egg format
485490
self._orig_extras_require = {}
486491
self._orig_install_requires = []

setuptools/tests/test_sdist.py

+42
Original file line numberDiff line numberDiff line change
@@ -498,6 +498,48 @@ def test_sdist_with_latin1_encoded_filename(self):
498498
filename = filename.decode('latin-1')
499499
filename not in cmd.filelist.files
500500

501+
_EXAMPLE_DIRECTIVES = {
502+
"setup.cfg - long_description and version": """
503+
[metadata]
504+
name = testing
505+
version = file: VERSION.txt
506+
license_files = DOWHATYOUWANT
507+
long_description = file: README.rst, USAGE.rst
508+
""",
509+
"pyproject.toml - static readme/license files and dynamic version": """
510+
[project]
511+
name = "testing"
512+
readme = "USAGE.rst"
513+
license = {file = "DOWHATYOUWANT"}
514+
dynamic = ["version"]
515+
[tool.setuptools.dynamic]
516+
version = {file = ["VERSION.txt"]}
517+
"""
518+
}
519+
520+
@pytest.mark.parametrize("config", _EXAMPLE_DIRECTIVES.keys())
521+
def test_add_files_referenced_by_config_directives(self, tmp_path, config):
522+
config_file, _, _ = config.partition(" - ")
523+
config_text = self._EXAMPLE_DIRECTIVES[config]
524+
(tmp_path / 'VERSION.txt').write_text("0.42", encoding="utf-8")
525+
(tmp_path / 'README.rst').write_text("hello world!", encoding="utf-8")
526+
(tmp_path / 'USAGE.rst').write_text("hello world!", encoding="utf-8")
527+
(tmp_path / 'DOWHATYOUWANT').write_text("hello world!", encoding="utf-8")
528+
(tmp_path / config_file).write_text(config_text, encoding="utf-8")
529+
530+
dist = Distribution({"packages": []})
531+
dist.script_name = 'setup.py'
532+
dist.parse_config_files()
533+
534+
cmd = sdist(dist)
535+
cmd.ensure_finalized()
536+
with quiet():
537+
cmd.run()
538+
539+
assert 'VERSION.txt' in cmd.filelist.files
540+
assert 'USAGE.rst' in cmd.filelist.files
541+
assert 'DOWHATYOUWANT' in cmd.filelist.files
542+
501543
def test_pyproject_toml_in_sdist(self, tmpdir):
502544
"""
503545
Check if pyproject.toml is included in source distribution if present

0 commit comments

Comments
 (0)