Fix problems with missing setupcfg_examples.txt (#3239)

Author: abravalheri

MANIFEST.in

@@ -15,3 +15,4 @@ include launcher.c
 include msvc-build-launcher.cmd
 include pytest.ini
 include tox.ini
+include setuptools/tests/config/setupcfg_examples.txt

changelog.d/3233.misc.1.rst

@@ -0,0 +1 @@
+Included missing test file ``setupcfg_examples.txt`` in ``sdist``.

changelog.d/3233.misc.2.rst

@@ -0,0 +1,3 @@
+Added script that allows developers to download ``setupcfg_examples.txt`` prior to
+running tests. By caching these files it should be possible to run the test suite
+offline.

setuptools/tests/config/downloads/.gitignore

@@ -1,2 +1,4 @@
 *
 !.gitignore
+!__init__.py
+!preload.py

setuptools/tests/config/downloads/__init__.py

@@ -0,0 +1,51 @@
+import re
+from pathlib import Path
+from urllib.request import urlopen
+
+__all__ = ["DOWNLOAD_DIR", "retrieve_file", "output_file", "urls_from_file"]
+
+
+NAME_REMOVE = ("http://", "https://", "github.com/", "/raw/")
+DOWNLOAD_DIR = Path(__file__).parent
+
+
+# ----------------------------------------------------------------------
+# Please update ./preload.py accordingly when modifying this file
+# ----------------------------------------------------------------------
+
+
+def output_file(url: str, download_dir: Path = DOWNLOAD_DIR):
+    file_name = url.strip()
+    for part in NAME_REMOVE:
+        file_name = file_name.replace(part, '').strip().strip('/:').strip()
+    return Path(download_dir, re.sub(r"[^\-_\.\w\d]+", "_", file_name))
+
+
+def retrieve_file(url: str, download_dir: Path = DOWNLOAD_DIR):
+    path = output_file(url, download_dir)
+    if path.exists():
+        print(f"Skipping {url} (already exists: {path})")
+    else:
+        download_dir.mkdir(exist_ok=True, parents=True)
+        print(f"Downloading {url} to {path}")
+        download(url, path)
+    return path
+
+
+def urls_from_file(list_file: Path):
+    """``list_file`` should be a text file where each line corresponds to a URL to
+    download.
+    """
+    print(f"file: {list_file}")
+    content = list_file.read_text(encoding="utf-8")
+    return [url for url in content.splitlines() if not url.startswith("#")]
+
+
+def download(url: str, dest: Path):
+    with urlopen(url) as f:
+        data = f.read()
+
+    with open(dest, "wb") as f:
+        f.write(data)
+
+    assert Path(dest).exists()

setuptools/tests/config/downloads/preload.py

@@ -0,0 +1,18 @@
+"""This file can be used to preload files needed for testing.
+
+For example you can use::
+
+    cd setuptools/tests/config
+    python -m downloads.preload setupcfg_examples.txt
+
+to make sure the `setup.cfg` examples are downloaded before starting the tests.
+"""
+import sys
+from pathlib import Path
+
+from . import retrieve_file, urls_from_file
+
+
+if __name__ == "__main__":
+    urls = urls_from_file(Path(sys.argv[1]))
+    list(map(retrieve_file, urls))

setuptools/tests/config/test_apply_pyprojecttoml.py

@@ -1,11 +1,14 @@
 """Make sure that applying the configuration from pyproject.toml is equivalent to
 applying a similar configuration from setup.cfg
+
+To run these tests offline, please have a look on ``./downloads/preload.py``
 """
 import io
 import re
+import tarfile
 from pathlib import Path
-from urllib.request import urlopen
 from unittest.mock import Mock
+from zipfile import ZipFile
 
 import pytest
 from ini2toml.api import Translator
@@ -17,22 +20,23 @@
 from setuptools.config._apply_pyprojecttoml import _WouldIgnoreField, _some_attrgetter
 from setuptools.command.egg_info import write_requirements
 
+from .downloads import retrieve_file, urls_from_file
+
 
-EXAMPLES = (Path(__file__).parent / "setupcfg_examples.txt").read_text()
-EXAMPLE_URLS = [x for x in EXAMPLES.splitlines() if not x.startswith("#")]
-DOWNLOAD_DIR = Path(__file__).parent / "downloads"
+HERE = Path(__file__).parent
+EXAMPLES_FILE = "setupcfg_examples.txt"
 
 
 def makedist(path, **attrs):
     return Distribution({"src_root": path, **attrs})
 
 
-@pytest.mark.parametrize("url", EXAMPLE_URLS)
+@pytest.mark.parametrize("url", urls_from_file(HERE / EXAMPLES_FILE))
 @pytest.mark.filterwarnings("ignore")
 @pytest.mark.uses_network
 def test_apply_pyproject_equivalent_to_setupcfg(url, monkeypatch, tmp_path):
     monkeypatch.setattr(expand, "read_attr", Mock(return_value="0.0.1"))
-    setupcfg_example = retrieve_file(url, DOWNLOAD_DIR)
+    setupcfg_example = retrieve_file(url)
     pyproject_example = Path(tmp_path, "pyproject.toml")
     toml_config = Translator().translate(setupcfg_example.read_text(), "setup.cfg")
     pyproject_example.write_text(toml_config)
@@ -276,32 +280,19 @@ def test_optional_dependencies_dont_remove_env_markers(self, tmp_path):
         assert "bar" in reqs
 
 
-# --- Auxiliary Functions ---
-
-
-NAME_REMOVE = ("http://", "https://", "github.com/", "/raw/")
+class TestMeta:
+    def test_example_file_in_sdist(self, setuptools_sdist):
+        """Meta test to ensure tests can run from sdist"""
+        with tarfile.open(setuptools_sdist) as tar:
+            assert any(name.endswith(EXAMPLES_FILE) for name in tar.getnames())
 
+    def test_example_file_not_in_wheel(self, setuptools_wheel):
+        """Meta test to ensure auxiliary test files are not in wheel"""
+        with ZipFile(setuptools_wheel) as zipfile:
+            assert not any(name.endswith(EXAMPLES_FILE) for name in zipfile.namelist())
 
-def retrieve_file(url, download_dir):
-    file_name = url.strip()
-    for part in NAME_REMOVE:
-        file_name = file_name.replace(part, '').strip().strip('/:').strip()
-    file_name = re.sub(r"[^\-_\.\w\d]+", "_", file_name)
-    path = Path(download_dir, file_name)
-    if not path.exists():
-        download_dir.mkdir(exist_ok=True, parents=True)
-        download(url, path)
-    return path
 
-
-def download(url, dest):
-    with urlopen(url) as f:
-        data = f.read()
-
-    with open(dest, "wb") as f:
-        f.write(data)
-
-    assert Path(dest).exists()
+# --- Auxiliary Functions ---
 
 
 def core_metadata(dist) -> str: