From 9af26959d2c1ffbd0115f7d7b2a69e3ad18fdf8e Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 27 Jun 2024 12:36:27 +0200 Subject: [PATCH 1/5] Build: disable Sphinx manipulation Add a feature flag called `DISABLE_SPHINX_MANIPULATION` that: - don't install `readthedocs-sphinx-ext` Python package - don't append anything to the Sphinx's `conf.py` file - enable Read the Docs Addons on these versions automatically The idea behind this is start defaulting new projects to Read the Docs Addons without breaking old projects. This is a potential first step in favor of the full deprecation/removal of the Sphinx manipulation (as we already did for MkDocs). Once this happens, **building on Read the Docs will produce the exact same result than building locally**. Related https://github.com/readthedocs/addons/issues/72 --- readthedocs/doc_builder/backends/sphinx.py | 25 ++++++++++--------- readthedocs/doc_builder/director.py | 5 ++++ .../doc_builder/python_environments.py | 11 ++++---- readthedocs/projects/models.py | 9 ++++++- 4 files changed, 31 insertions(+), 19 deletions(-) diff --git a/readthedocs/doc_builder/backends/sphinx.py b/readthedocs/doc_builder/backends/sphinx.py index 5356b32feb8..b3e04a656d3 100644 --- a/readthedocs/doc_builder/backends/sphinx.py +++ b/readthedocs/doc_builder/backends/sphinx.py @@ -264,20 +264,21 @@ def append_conf(self): }, ) - # Allow symlinks, but only the ones that resolve inside the base directory. - # NOTE: if something goes wrong, - # `safe_open` raises an exception that's clearly communicated to the user. - outfile = safe_open( - self.config_file, "a", allow_symlinks=True, base_path=self.project_path - ) + if not self.project.has_feature(Feature.DISABLE_SPHINX_MANIPULATION): + # Allow symlinks, but only the ones that resolve inside the base directory. + # NOTE: if something goes wrong, + # `safe_open` raises an exception that's clearly communicated to the user. + outfile = safe_open( + self.config_file, "a", allow_symlinks=True, base_path=self.project_path + ) - # Append config to project conf file - tmpl = template_loader.get_template("doc_builder/conf.py.tmpl") - rendered = tmpl.render(self.get_config_params()) + # Append config to project conf file + tmpl = template_loader.get_template("doc_builder/conf.py.tmpl") + rendered = tmpl.render(self.get_config_params()) - with outfile: - outfile.write("\n") - outfile.write(rendered) + with outfile: + outfile.write("\n") + outfile.write(rendered) # Print the contents of conf.py in order to make the rendered # configfile visible in the build logs diff --git a/readthedocs/doc_builder/director.py b/readthedocs/doc_builder/director.py index 6fdd8202b60..5aebc60a918 100644 --- a/readthedocs/doc_builder/director.py +++ b/readthedocs/doc_builder/director.py @@ -25,6 +25,7 @@ from readthedocs.doc_builder.python_environments import Conda, Virtualenv from readthedocs.projects.constants import BUILD_COMMANDS_OUTPUT_PATH_HTML from readthedocs.projects.exceptions import RepositoryError +from readthedocs.projects.models import Feature from readthedocs.projects.signals import after_build, before_build, before_vcs from readthedocs.storage import build_tools_storage @@ -200,6 +201,10 @@ def build(self): self.run_build_job("post_build") self.store_readthedocs_build_yaml() + if self.data.project.has_feature(Feature.DISABLE_SPHINX_MANIPULATION): + # Mark this version to inject the new js client when serving it via El Proxito + self.data.version.addons = True + after_build.send( sender=self.data.version, ) diff --git a/readthedocs/doc_builder/python_environments.py b/readthedocs/doc_builder/python_environments.py index 4af234e64b3..f186fc44fcb 100644 --- a/readthedocs/doc_builder/python_environments.py +++ b/readthedocs/doc_builder/python_environments.py @@ -174,12 +174,11 @@ def _install_latest_requirements(self, pip_install_cmd): if self.config.doctype == "mkdocs": requirements.append("mkdocs") else: - requirements.extend( - [ - "sphinx", - "readthedocs-sphinx-ext", - ] - ) + requirements.append("sphinx") + + # Install ``readthedocs-sphinx-ext`` only on old projects + if not self.project.has_feature(Feature.DISABLE_SPHINX_MANIPULATION): + requirements.append("readthedocs-sphinx-ext") cmd = copy.copy(pip_install_cmd) cmd.extend(requirements) diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index ddbb0331837..43508675823 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -1896,6 +1896,7 @@ def add_features(sender, **kwargs): DONT_INSTALL_LATEST_PIP = "dont_install_latest_pip" USE_SPHINX_RTD_EXT_LATEST = "rtd_sphinx_ext_latest" INSTALL_LATEST_CORE_REQUIREMENTS = "install_latest_core_requirements" + DISABLE_SPHINX_MANIPULATION = "disable_sphinx_manipulation" # Search related features DISABLE_SERVER_SIDE_SEARCH = "disable_server_side_search" @@ -1973,6 +1974,12 @@ def add_features(sender, **kwargs): "Build: Install all the latest versions of Read the Docs core requirements" ), ), + ( + DISABLE_SPHINX_MANIPULATION, + _( + "Sphinx: Don't append `conf.py` and don't install ``readthedocs-sphinx-ext``" + ), + ), # Search related features. ( DISABLE_SERVER_SIDE_SEARCH, @@ -2033,7 +2040,7 @@ def get_feature_display(self): Because the field is not a ChoiceField here, we need to manually implement this behavior. """ - return dict(self.FEATURES).get(self.feature_id, self.feature_id) + return str(dict(self.FEATURES).get(self.feature_id, self.feature_id)) class EnvironmentVariable(TimeStampedModel, models.Model): From 8c5aa57dbe5e71da93acdcc376400079aef5e2fb Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 1 Jul 2024 16:47:04 +0200 Subject: [PATCH 2/5] Build: add `READTHEDOCS_REPOSITORY_PATH` environment variable This is a useful variable that we will require during the deprecation of the Sphinx context manipulation. Besides, it seems a useful variable that we should expose to users anyways so they can use it as relative to calculate other useful paths. --- docs/user/reference/environment-variables.rst | 6 ++++++ readthedocs/doc_builder/director.py | 3 +++ 2 files changed, 9 insertions(+) diff --git a/docs/user/reference/environment-variables.rst b/docs/user/reference/environment-variables.rst index 786d7c7a911..e648f1def62 100644 --- a/docs/user/reference/environment-variables.rst +++ b/docs/user/reference/environment-variables.rst @@ -81,6 +81,12 @@ All :doc:`build processes ` have the following environment variables au :Example: ``https://docs.readthedocs.io/ja/stable/`` :Example: ``https://example--17.org.readthedocs.build/fr/17/`` +.. envvar:: READTHEDOCS_REPOSITORY_PATH + + Path where the repository was cloned. + + :Example: ``/home/docs/checkouts/readthedocs.org/user_builds/test-builds/checkouts/latest`` + .. envvar:: READTHEDOCS_GIT_CLONE_URL URL for the remote source repository, from which the documentation is cloned. diff --git a/readthedocs/doc_builder/director.py b/readthedocs/doc_builder/director.py index 5aebc60a918..c0d9515a093 100644 --- a/readthedocs/doc_builder/director.py +++ b/readthedocs/doc_builder/director.py @@ -650,6 +650,9 @@ def get_rtd_env_vars(self): "READTHEDOCS_VERSION_NAME": self.data.version.verbose_name, "READTHEDOCS_PROJECT": self.data.project.slug, "READTHEDOCS_LANGUAGE": self.data.project.language, + "READTHEDOCS_REPOSITORY_PATH": self.data.project.checkout_path( + self.data.version.slug + ), "READTHEDOCS_OUTPUT": os.path.join( self.data.project.checkout_path(self.data.version.slug), "_readthedocs/" ), From fcf9a1d5d643668627f1b051458e4fcfeb4c1b2f Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 1 Jul 2024 17:48:51 +0200 Subject: [PATCH 3/5] Build: add `READTHEDOCS_PRODUCTION_DOMAIN` as environment variable --- readthedocs/doc_builder/director.py | 1 + readthedocs/projects/tests/test_build_tasks.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/readthedocs/doc_builder/director.py b/readthedocs/doc_builder/director.py index c0d9515a093..efae6cb1a7c 100644 --- a/readthedocs/doc_builder/director.py +++ b/readthedocs/doc_builder/director.py @@ -662,6 +662,7 @@ def get_rtd_env_vars(self): # "READTHEDOCS_GIT_HTML_URL": self.data.project.remote_repository.html_url, "READTHEDOCS_GIT_IDENTIFIER": self.data.version.identifier, "READTHEDOCS_GIT_COMMIT_HASH": self.data.build["commit"], + "READTHEDOCS_PRODUCTION_DOMAIN": settings.PRODUCTION_DOMAIN, } return env diff --git a/readthedocs/projects/tests/test_build_tasks.py b/readthedocs/projects/tests/test_build_tasks.py index cff11d8dd49..78e7cde8402 100644 --- a/readthedocs/projects/tests/test_build_tasks.py +++ b/readthedocs/projects/tests/test_build_tasks.py @@ -361,12 +361,16 @@ def test_get_env_vars(self, load_yaml_config, build_environment, config, externa "READTHEDOCS_VERSION_NAME": self.version.verbose_name, "READTHEDOCS_PROJECT": self.project.slug, "READTHEDOCS_LANGUAGE": self.project.language, + "READTHEDOCS_OUTPUT": os.path.join( + self.project.checkout_path(self.version.slug), + ), "READTHEDOCS_OUTPUT": os.path.join( self.project.checkout_path(self.version.slug), "_readthedocs/" ), "READTHEDOCS_GIT_CLONE_URL": self.project.repo, "READTHEDOCS_GIT_IDENTIFIER": self.version.identifier, "READTHEDOCS_GIT_COMMIT_HASH": self.build.commit, + "READTHEDOCS_PRODUCTION_DOMAIN": settings.PRODUCTION_DOMAIN, } self._trigger_update_docs_task() From 54bc8d0d0856309a92a95b91a70c783190766d21 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 3 Jul 2024 12:33:44 +0200 Subject: [PATCH 4/5] Document `READTHEDOCS_PRODUCTION_DOMAIN` env var --- docs/user/reference/environment-variables.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/user/reference/environment-variables.rst b/docs/user/reference/environment-variables.rst index e648f1def62..1cbf30bbd2a 100644 --- a/docs/user/reference/environment-variables.rst +++ b/docs/user/reference/environment-variables.rst @@ -9,6 +9,17 @@ All :doc:`build processes ` have the following environment variables au :Default: ``True`` +.. envvar:: READTHEDOCS_PRODUCTION_DOMAIN + + Domain where Read the Docs application/dashboard and API are running. + + :Example: ``readthedocs.org`` + :Example: ``readthedocs.com`` + :Example: ``app.readthedocs.org`` + :Example: ``app.readthedocs.com`` + :Example: ``devthedocs.org`` + :Example: ``devthedocs.com`` + .. envvar:: READTHEDOCS_PROJECT The :term:`slug` of the project being built. For example, ``my-example-project``. From c0bb53c0c81e725285e73af5a82bd3a5d8aee024 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 3 Jul 2024 12:35:43 +0200 Subject: [PATCH 5/5] Update tests --- readthedocs/projects/tests/test_build_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/projects/tests/test_build_tasks.py b/readthedocs/projects/tests/test_build_tasks.py index 78e7cde8402..be300a1faea 100644 --- a/readthedocs/projects/tests/test_build_tasks.py +++ b/readthedocs/projects/tests/test_build_tasks.py @@ -361,7 +361,7 @@ def test_get_env_vars(self, load_yaml_config, build_environment, config, externa "READTHEDOCS_VERSION_NAME": self.version.verbose_name, "READTHEDOCS_PROJECT": self.project.slug, "READTHEDOCS_LANGUAGE": self.project.language, - "READTHEDOCS_OUTPUT": os.path.join( + "READTHEDOCS_REPOSITORY_PATH": os.path.join( self.project.checkout_path(self.version.slug), ), "READTHEDOCS_OUTPUT": os.path.join(