Build: install all the latest Python "core requirements"

docs/user/build-default-versions.rst

@@ -56,20 +56,25 @@ setuptools:
   All other projects use the latest version.
 
 mock:
-  ``1.0.1`` (could be removed in the future).
+  ``1.0.1``.
+  Projects created after August 7, 2023 won't install this dependency by default.
+
 
 pillow:
-  ``5.4.1`` when using Python 2.7, 3.4, 3.5, 3.6, 3.7. Otherwise, its latest version
-  (could be removed in the future).
+  ``5.4.1`` when using Python 2.7, 3.4, 3.5, 3.6, 3.7. Otherwise, its latest version.
+  Projects created after August 7, 2023 won't install this dependency by default.
 
 alabaster:
-  ``0.7.x`` (could be removed in the future).
+  ``0.7.x``.
+  Projects created after August 7, 2023 won't install this dependency by default.
 
 commonmark:
-  ``0.8.1`` (could be removed in the future).
+  ``0.8.1``.
+  Projects created after August 7, 2023 won't install this dependency by default.
 
 recommonmark:
-  ``0.5.0`` (could be removed in the future).
+  ``0.5.0``.
+  Projects created after August 7, 2023 won't install this dependency by default.
 
 Conda
 ~~~~~
@@ -90,13 +95,16 @@ sphinx-rtd-theme:
   Latest version by default installed via ``conda``.
 
 mock:
-  Latest version by default installed via ``pip`` (could be removed in the future).
+  Latest version by default installed via ``pip``.
+  Projects created after August 7, 2023 won't install this dependency by default.
 
 pillow:
-  Latest version by default installed via ``pip`` (could be removed in the future).
+  Latest version by default installed via ``pip``.
+  Projects created after August 7, 2023 won't install this dependency by default.
 
 recommonmark:
-  Latest version by default installed via ``conda`` (could be removed in the future).
+  Latest version by default installed via ``conda``.
+  Projects created after August 7, 2023 won't install this dependency by default.
 
 Internal dependencies
 ---------------------

readthedocs/doc_builder/python_environments.py

@@ -163,6 +163,63 @@ def install_core_requirements(self):
             '--no-cache-dir',
         ]
 
+        if self.project.has_feature(Feature.INSTALL_LATEST_CORE_REQUIREMENTS):
+            self._install_latest_requirements(pip_install_cmd)
+        else:
+            self._install_old_requirements(pip_install_cmd)
+
+    def _install_latest_requirements(self, pip_install_cmd):
+        """
+        Install all the latest core requirements.
+
+        By enabling the feature flag ``INSTALL_LATEST_CORE_REQUIREMENTS``
+        projects will automatically get installed all the latest core
+        requirements: pip, sphinx, mock, alabaster, setuptools, etc.
+
+        This is the new behavior and where we are moving towards.
+        """
+        # First, upgrade pip and setuptools to their latest versions
+        cmd = pip_install_cmd + ["pip", "setuptools"]
+        self.build_env.run(
+            *cmd,
+            bin_path=self.venv_bin(),
+            cwd=self.checkout_path,
+        )
+
+        # Second, install all the latest core requirements
+        requirements = [
+            "setuptools",
+        ]
+
+        if self.config.doctype == "mkdocs":
+            requirements.append("mkdocs")
+        else:
+            requirements.extend(
+                [
+                    "sphinx",
+                    "sphinx-rtd-theme",
+                    "readthedocs-sphinx-ext",
+                ]
+            )
+
+        cmd = copy.copy(pip_install_cmd)
+        cmd.extend(requirements)
+        self.build_env.run(
+            *cmd,
+            bin_path=self.venv_bin(),
+            cwd=self.checkout_path,
+        )
+
+    def _install_old_requirements(self, pip_install_cmd):
+        """
+        Install old core requirements.
+
+        There are bunch of feature flags that will be taken in consideration to
+        decide whether or not upgrade some of the core dependencies to their
+        latest versions.
+
+        This is the old behavior and the one we want to get rid off.
+        """
         # Install latest pip and setuptools first,
         # so it is used when installing the other requirements.
         pip_version = self.project.get_feature_value(
@@ -420,7 +477,10 @@ def _append_core_requirements(self):
         else:
             # Append conda dependencies directly to ``dependencies`` and pip
             # dependencies to ``dependencies.pip``
-            pip_requirements, conda_requirements = self._get_core_requirements()
+            if self.project.has_feature(Feature.INSTALL_LATEST_CORE_REQUIREMENTS):
+                pip_requirements, conda_requirements = self._get_new_core_requirements()
+            else:
+                pip_requirements, conda_requirements = self._get_old_core_requirements()
             dependencies = environment.get('dependencies', [])
             pip_dependencies = {'pip': pip_requirements}
 
@@ -458,7 +518,22 @@ def _append_core_requirements(self):
                     'environment file.',
                 )
 
-    def _get_core_requirements(self):
+    def _get_new_core_requirements(self):
+        # Use conda for requirements it packages
+        conda_requirements = []
+
+        # Install pip-only things.
+        pip_requirements = []
+
+        if self.config.doctype == "mkdocs":
+            pip_requirements.append("mkdocs")
+        else:
+            pip_requirements.append("readthedocs-sphinx-ext")
+            conda_requirements.extend(["sphinx", "sphinx_rtd_theme"])
+
+        return pip_requirements, conda_requirements
+
+    def _get_old_core_requirements(self):
         # Use conda for requirements it packages
         conda_requirements = [
             'mock',
@@ -487,7 +562,7 @@ def install_core_requirements(self):
             # create`` step.
             return
 
-        pip_requirements, conda_requirements = self._get_core_requirements()
+        pip_requirements, conda_requirements = self._get_old_core_requirements()
         # Install requirements via ``conda install`` command if they were
         # not appended to the ``environment.yml`` file.
         cmd = [

readthedocs/projects/models.py

@@ -1930,6 +1930,7 @@ def add_features(sender, **kwargs):
     DEFAULT_TO_MKDOCS_0_17_3 = 'default_to_mkdocs_0_17_3'
     USE_MKDOCS_LATEST = 'use_mkdocs_latest'
     USE_SPHINX_RTD_EXT_LATEST = 'rtd_sphinx_ext_latest'
+    INSTALL_LATEST_CORE_REQUIREMENTS = "install_latest_core_requirements"
 
     # Search related features
     DISABLE_SERVER_SIDE_SEARCH = 'disable_server_side_search'
@@ -2044,6 +2045,12 @@ def add_features(sender, **kwargs):
             USE_SPHINX_RTD_EXT_LATEST,
             _("Sphinx: Use latest version of the Read the Docs Sphinx extension"),
         ),
+        (
+            INSTALL_LATEST_CORE_REQUIREMENTS,
+            _(
+                "Build: Install all the latest versions of Read the Docs core requirements"
+            ),
+        ),
 
         # Search related features.
         (