From c06341441052e4b2410d0bf610f07e5e7a98ccb7 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 4 Jul 2023 12:46:11 +0200 Subject: [PATCH 1/6] Build: install all the latest Python "core requirements" I created a new feature flag called `INSTALL_LATEST_CORE_REQUIREMENTS` that does not depend on any other feature flag. Its goal is to always install the latest Python packages. We are trying to move away from pinning the dependencies on our side and telling users to do that on their side if they want to have reproducible builds over time. I think this is the best outcome for new projects since they will immediately use the latest versions of everything instead of old (and maybe broken) dependencies. I propose to set a particular date for this feature flag and make new projects to start building with all the latest dependencies. This will, at least, stop making the problem bigger. Later, we can decide how to communicate to old projects that we are going to install the latest requirements and if they don't want that, they should pin their dependencies. We can follow our new and shiny deprecation path we have built and tested in the last month. Related https://github.com/readthedocs/meta/discussions/8 Related https://github.com/readthedocs/readthedocs.org/issues/9562 Related https://github.com/readthedocs/readthedocs.org/issues/10402 Related https://github.com/readthedocs/readthedocs.org/issues/9081 --- .../doc_builder/python_environments.py | 61 +++++++++++++++++++ readthedocs/projects/models.py | 7 +++ 2 files changed, 68 insertions(+) diff --git a/readthedocs/doc_builder/python_environments.py b/readthedocs/doc_builder/python_environments.py index 0b85408400f..05268dcc6a0 100644 --- a/readthedocs/doc_builder/python_environments.py +++ b/readthedocs/doc_builder/python_environments.py @@ -163,6 +163,67 @@ def install_core_requirements(self): '--no-cache-dir', ] + if self.project.has_feature(Feature.INSTALL_LATEST_CORE_REQUIREMENTS): + self._install_latest_requirements(pip_install_cmd) + else: + self._install_old_requirements(pip_install_cmd) + + def _install_latest_requirements(self, pip_install_cmd): + """ + Install all the latest core requirements. + + By enabling the feature flag ``INSTALL_LATEST_CORE_REQUIREMENTS`` + projects will automatically get installed all the latest core + requirements: pip, sphinx, mock, alabaster, setuptools, etc. + + This is the new behavior and where we are moving towards. + """ + # First, upgrade pip and setuptools to their latest versions + cmd = pip_install_cmd + ["pip", "setuptools"] + self.build_env.run( + *cmd, + bin_path=self.venv_bin(), + cwd=self.checkout_path, + ) + + # Second, install all the latest core requirements + requirements = [ + "mock", + "alabaster", + "commonmark", + "recommonmark", + "setuptools", + ] + + if self.config.doctype == "mkdocs": + requirements.append("mkdocs") + else: + requirements.extend( + [ + "sphinx", + "sphinx-rtd-theme", + "readthedocs-sphinx-ext", + ] + ) + + cmd = copy.copy(pip_install_cmd) + cmd.extend(requirements) + self.build_env.run( + *cmd, + bin_path=self.venv_bin(), + cwd=self.checkout_path, + ) + + def _install_old_requirements(self, pip_install_cmd): + """ + Install old core requirements. + + There are bunch of feature flags that will be taken in consideration to + decide whether or not upgrade some of the core dependencies to their + latest versions. + + This is the old behavior and the one we want to get rid off. + """ # Install latest pip and setuptools first, # so it is used when installing the other requirements. pip_version = self.project.get_feature_value( diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index dfa145138f8..8daa7d816ee 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -1930,6 +1930,7 @@ def add_features(sender, **kwargs): DEFAULT_TO_MKDOCS_0_17_3 = 'default_to_mkdocs_0_17_3' USE_MKDOCS_LATEST = 'use_mkdocs_latest' USE_SPHINX_RTD_EXT_LATEST = 'rtd_sphinx_ext_latest' + INSTALL_LATEST_CORE_REQUIREMENTS = "install_latest_core_requirements" # Search related features DISABLE_SERVER_SIDE_SEARCH = 'disable_server_side_search' @@ -2044,6 +2045,12 @@ def add_features(sender, **kwargs): USE_SPHINX_RTD_EXT_LATEST, _("Sphinx: Use latest version of the Read the Docs Sphinx extension"), ), + ( + INSTALL_LATEST_CORE_REQUIREMENTS, + _( + "Build: Install all the latest versions of Read the Docs core requirements" + ), + ), # Search related features. ( From f028546d3e55a90431dc244054769ff6cc06cc9f Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 4 Jul 2023 12:56:15 +0200 Subject: [PATCH 2/6] Build: remove deprecated dependencies --- readthedocs/doc_builder/python_environments.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/readthedocs/doc_builder/python_environments.py b/readthedocs/doc_builder/python_environments.py index 05268dcc6a0..98860a7a72c 100644 --- a/readthedocs/doc_builder/python_environments.py +++ b/readthedocs/doc_builder/python_environments.py @@ -188,10 +188,6 @@ def _install_latest_requirements(self, pip_install_cmd): # Second, install all the latest core requirements requirements = [ - "mock", - "alabaster", - "commonmark", - "recommonmark", "setuptools", ] From 1af8d48e59a5d7552f90772348420f85935b0432 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 4 Jul 2023 13:17:29 +0200 Subject: [PATCH 3/6] Build: install latest core requirements when using Conda --- .../doc_builder/python_environments.py | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/readthedocs/doc_builder/python_environments.py b/readthedocs/doc_builder/python_environments.py index 98860a7a72c..0a56b987dfc 100644 --- a/readthedocs/doc_builder/python_environments.py +++ b/readthedocs/doc_builder/python_environments.py @@ -477,7 +477,10 @@ def _append_core_requirements(self): else: # Append conda dependencies directly to ``dependencies`` and pip # dependencies to ``dependencies.pip`` - pip_requirements, conda_requirements = self._get_core_requirements() + if self.project.has_feature(Feature.INSTALL_LATEST_CORE_REQUIREMENTS): + pip_requirements, conda_requirements = self._get_new_core_requirements() + else: + pip_requirements, conda_requirements = self._get_old_core_requirements() dependencies = environment.get('dependencies', []) pip_dependencies = {'pip': pip_requirements} @@ -515,7 +518,22 @@ def _append_core_requirements(self): 'environment file.', ) - def _get_core_requirements(self): + def _get_new_core_requirements(self): + # Use conda for requirements it packages + conda_requirements = [] + + # Install pip-only things. + pip_requirements = [] + + if self.config.doctype == "mkdocs": + pip_requirements.append("mkdocs") + else: + pip_requirements.append("readthedocs-sphinx-ext") + conda_requirements.extend(["sphinx", "sphinx_rtd_theme"]) + + return pip_requirements, conda_requirements + + def _get_old_core_requirements(self): # Use conda for requirements it packages conda_requirements = [ 'mock', @@ -544,7 +562,7 @@ def install_core_requirements(self): # create`` step. return - pip_requirements, conda_requirements = self._get_core_requirements() + pip_requirements, conda_requirements = self._get_old_core_requirements() # Install requirements via ``conda install`` command if they were # not appended to the ``environment.yml`` file. cmd = [ From b1e44d56fd589d9ab17f62e0d944f0fd0786a909 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 4 Jul 2023 13:19:16 +0200 Subject: [PATCH 4/6] Docs: update default versions documentation to match the changes --- docs/user/build-default-versions.rst | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/docs/user/build-default-versions.rst b/docs/user/build-default-versions.rst index 8a232b1f45c..b5f9f5bfc79 100644 --- a/docs/user/build-default-versions.rst +++ b/docs/user/build-default-versions.rst @@ -56,20 +56,25 @@ setuptools: All other projects use the latest version. mock: - ``1.0.1`` (could be removed in the future). + ``1.0.1``. + Projects created after August 7, 2023 won't install this dependency by default. + pillow: - ``5.4.1`` when using Python 2.7, 3.4, 3.5, 3.6, 3.7. Otherwise, its latest version - (could be removed in the future). + ``5.4.1`` when using Python 2.7, 3.4, 3.5, 3.6, 3.7. Otherwise, its latest version. + Projects created after August 7, 2023 won't install this dependency by default. alabaster: - ``0.7.x`` (could be removed in the future). + ``0.7.x``. + Projects created after August 7, 2023 won't install this dependency by default. commonmark: - ``0.8.1`` (could be removed in the future). + ``0.8.1``. + Projects created after August 7, 2023 won't install this dependency by default. recommonmark: - ``0.5.0`` (could be removed in the future). + ``0.5.0``. + Projects created after August 7, 2023 won't install this dependency by default. Conda ~~~~~ @@ -90,13 +95,16 @@ sphinx-rtd-theme: Latest version by default installed via ``conda``. mock: - Latest version by default installed via ``pip`` (could be removed in the future). + Latest version by default installed via ``pip``. + Projects created after August 7, 2023 won't install this dependency by default. pillow: - Latest version by default installed via ``pip`` (could be removed in the future). + Latest version by default installed via ``pip``. + Projects created after August 7, 2023 won't install this dependency by default. recommonmark: - Latest version by default installed via ``conda`` (could be removed in the future). + Latest version by default installed via ``conda``. + Projects created after August 7, 2023 won't install this dependency by default. Internal dependencies --------------------- From 2ce5d4198a0e2d026b6abbad68d1aecdb9798d83 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 6 Jul 2023 12:09:53 +0200 Subject: [PATCH 5/6] Feedback from review Do not install `sphinx-rtd-theme` and clarify the docstring. --- readthedocs/doc_builder/python_environments.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/readthedocs/doc_builder/python_environments.py b/readthedocs/doc_builder/python_environments.py index 0a56b987dfc..4b3759bfe48 100644 --- a/readthedocs/doc_builder/python_environments.py +++ b/readthedocs/doc_builder/python_environments.py @@ -174,7 +174,7 @@ def _install_latest_requirements(self, pip_install_cmd): By enabling the feature flag ``INSTALL_LATEST_CORE_REQUIREMENTS`` projects will automatically get installed all the latest core - requirements: pip, sphinx, mock, alabaster, setuptools, etc. + requirements: pip, setuptools, sphinx, readthedocs-sphinx-ext and mkdocs. This is the new behavior and where we are moving towards. """ @@ -187,9 +187,7 @@ def _install_latest_requirements(self, pip_install_cmd): ) # Second, install all the latest core requirements - requirements = [ - "setuptools", - ] + requirements = [] if self.config.doctype == "mkdocs": requirements.append("mkdocs") @@ -197,7 +195,6 @@ def _install_latest_requirements(self, pip_install_cmd): requirements.extend( [ "sphinx", - "sphinx-rtd-theme", "readthedocs-sphinx-ext", ] ) @@ -529,7 +526,7 @@ def _get_new_core_requirements(self): pip_requirements.append("mkdocs") else: pip_requirements.append("readthedocs-sphinx-ext") - conda_requirements.extend(["sphinx", "sphinx_rtd_theme"]) + conda_requirements.extend(["sphinx"]) return pip_requirements, conda_requirements From 8e3e87420bb9b0526162a33ec86a85f43c0b52d2 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 6 Jul 2023 12:16:38 +0200 Subject: [PATCH 6/6] Update documentation --- docs/user/build-default-versions.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/user/build-default-versions.rst b/docs/user/build-default-versions.rst index b5f9f5bfc79..c353e4c6aba 100644 --- a/docs/user/build-default-versions.rst +++ b/docs/user/build-default-versions.rst @@ -47,6 +47,7 @@ Mkdocs: sphinx-rtd-theme: Projects created before October 20, 2020 (January 21, 2021 for :doc:`/commercial/index`) use ``0.4.3``. New projects use the latest version. + Projects created after August 7, 2023 won't install this dependency by default. pip: Latest version by default. @@ -54,6 +55,7 @@ pip: setuptools: Projects using ``setup.py install`` as installation method use ``58.2.0`` or older. All other projects use the latest version. + Projects created after August 7, 2023 will always use the latest version. mock: ``1.0.1``. @@ -93,6 +95,7 @@ Sphinx: sphinx-rtd-theme: Latest version by default installed via ``conda``. + Projects created after August 7, 2023 won't install this dependency by default. mock: Latest version by default installed via ``pip``.