Feature flag: remove DONT_CREATE_INDEX (#10471)

humitos · ericholscher · web-flow · commit d86371191e43 · 2023-06-28T19:06:03.000+02:00
* Feature flag: remove `DONT_CREATE_INDEX`

Remove the usage of `DONT_CREATE_INDEX` feature flag that only 1 project is
currently using.

Besides, this commit removes the auto-creation of the `index.html` file when
building with MkDocs or Sphinx. We will be checking at "Uploading" status if
there is no `index.html` file and we will fail the build in that case.

* Build: fail the build if HTML output does not contain an `index.html`

Fail the build when there is no `index.html` at its root directory.

* Tests: remove mock for `create_index`

* Update readthedocs/doc_builder/exceptions.py

Co-authored-by: Eric Holscher &lt;25510+ericholscher@users.noreply.github.com&gt;

* Do not fail the build for now

Only collect data of projects without index.html or README.html at the root.
Once we have that data will be contacting those projects to let them know this
is deprecated and will make the builds to fail.

* Keep the generation of `conf.py` in its place

We cannot remove this yet.

---------

Co-authored-by: Eric Holscher &lt;25510+ericholscher@users.noreply.github.com&gt;
diff --git a/docs/user/feature-flags.rst b/docs/user/feature-flags.rst
@@ -16,11 +16,3 @@ Available flags
 
 Makes Read the Docs to install all the requirements at once on ``conda create`` step.
 This helps users to pin dependencies on conda and to improve build time.
-
-``DONT_CREATE_INDEX``: Do not create index.md or README.rst if the project does not have one.
-
-When Read the Docs detects that your project doesn't have an ``index.md`` or ``README.rst``,
-it auto-generate one for you with instructions about how to proceed.
-
-In case you are using a static HTML page as index or an generated index from code,
-this behavior could be a problem. With this feature flag you can disable that.
diff --git a/readthedocs/doc_builder/backends/mkdocs.py b/readthedocs/doc_builder/backends/mkdocs.py
@@ -138,7 +138,7 @@ def load_yaml_config(self):
             raise MkDocsYAMLParseError(
                 'Your mkdocs.yml could not be loaded, '
                 'possibly due to a syntax error{note}'.format(note=note),
-            )
+            ) from exc
 
     def append_conf(self):
         """
@@ -156,7 +156,6 @@ def append_conf(self):
                 MkDocsYAMLParseError.INVALID_DOCS_DIR_CONFIG,
             )
 
-        self.create_index(extension='md')
         user_config['docs_dir'] = docs_dir
 
         # MkDocs <=0.17.x doesn't support absolute paths,
@@ -215,7 +214,9 @@ def append_conf(self):
             docs_dir=os.path.relpath(docs_path, self.project_path),
             mkdocs_config=user_config,
         )
-        with open(os.path.join(docs_path, 'readthedocs-data.js'), 'w') as f:
+        with open(
+            os.path.join(docs_path, "readthedocs-data.js"), "w", encoding="utf-8"
+        ) as f:
             f.write(rtd_data)
 
         # Use Read the Docs' analytics setup rather than mkdocs'
@@ -230,10 +231,11 @@ def append_conf(self):
                 user_config['theme'] = self.DEFAULT_THEME_NAME
 
         # Write the modified mkdocs configuration
-        yaml_dump_safely(
-            user_config,
-            open(self.yaml_file, 'w'),
-        )
+        with open(self.yaml_file, "w", encoding="utf-8") as f:
+            yaml_dump_safely(
+                user_config,
+                f,
+            )
 
         # Write the mkdocs.yml to the build logs
         self.run(
@@ -369,10 +371,10 @@ class SafeLoader(yaml.SafeLoader):  # pylint: disable=too-many-ancestors
     Issue https://github.com/readthedocs/readthedocs.org/issues/7461
     """
 
-    def ignore_unknown(self, node):  # pylint: disable=no-self-use, unused-argument
+    def ignore_unknown(self, node):  # pylint: disable=unused-argument
         return None
 
-    def construct_python_name(self, suffix, node):  # pylint: disable=no-self-use, unused-argument
+    def construct_python_name(self, suffix, node):  # pylint: disable=unused-argument
         return ProxyPythonName(suffix)
 
 
diff --git a/readthedocs/doc_builder/backends/sphinx.py b/readthedocs/doc_builder/backends/sphinx.py
@@ -276,9 +276,13 @@ def append_conf(self):
 
         The default content is rendered from ``doc_builder/conf.py.tmpl``.
         """
-        if self.config_file is None:
-            master_doc = self.create_index(extension='rst')
-            self._write_config(master_doc=master_doc)
+
+        # Generate a `conf.py` from a template
+        #
+        # TODO: we should remove this feature at some point to move forward
+        # with the idea of remove magic from the builders.
+        if not self.config_file:
+            self._write_config()
 
         try:
             self.config_file = (
diff --git a/readthedocs/doc_builder/base.py b/readthedocs/doc_builder/base.py
@@ -5,8 +5,6 @@
 
 import structlog
 
-from readthedocs.core.utils.filesystem import safe_open
-from readthedocs.projects.models import Feature
 
 log = structlog.get_logger(__name__)
 
@@ -15,8 +13,8 @@ def restoring_chdir(fn):
     # XXX:dc: This would be better off in a neutral module
     @wraps(fn)
     def decorator(*args, **kw):
+        path = os.getcwd()
         try:
-            path = os.getcwd()
             return fn(*args, **kw)
         finally:
             os.chdir(path)
@@ -62,44 +60,6 @@ def docs_dir(self):
 
         return self.project_path
 
-    def create_index(self, extension='md', **__):
-        """Create an index file if it needs it."""
-        docs_dir = self.docs_dir()
-
-        index_filename = os.path.join(
-            docs_dir,
-            'index.{ext}'.format(ext=extension),
-        )
-        if not os.path.exists(index_filename):
-            readme_filename = os.path.join(
-                docs_dir,
-                'README.{ext}'.format(ext=extension),
-            )
-            if os.path.exists(readme_filename):
-                return 'README'
-
-            if not self.project.has_feature(Feature.DONT_CREATE_INDEX):
-                index_text = """
-
-Welcome to Read the Docs
-------------------------
-
-This is an autogenerated index file.
-
-Please create an ``index.{ext}`` or ``README.{ext}`` file with your own content
-under the root (or ``/docs``) directory in your repository.
-
-If you want to use another markup, choose a different builder in your settings.
-Check out our `Getting Started Guide
-<https://docs.readthedocs.io/en/latest/getting_started.html>`_ to become more
-familiar with Read the Docs.
-                """
-
-                with safe_open(index_filename, "w+") as index_file:
-                    index_file.write(index_text.format(dir=docs_dir, ext=extension))
-
-        return 'index'
-
     def run(self, *args, **kwargs):
         """Proxy run to build environment."""
         return self.build_env.run(*args, **kwargs)
diff --git a/readthedocs/doc_builder/exceptions.py b/readthedocs/doc_builder/exceptions.py
@@ -52,6 +52,10 @@ class BuildUserError(BuildBaseException):
         "and it is not currently supported. "
         'Please, remove all the files but the "{artifact_type}" you want to upload.'
     )
+    BUILD_OUTPUT_HTML_NO_INDEX_FILE = gettext_noop(
+        "Your documentation did not generate an 'index.html' at its root directory. "
+        "This is required for documentation serving at the root URL for this version."
+    )
     BUILD_OUTPUT_OLD_DIRECTORY_USED = gettext_noop(
         "Some files were detected in an unsupported output path, '_build/html'. "
         "Ensure your project is configured to use the output path "
diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py
@@ -1937,7 +1937,6 @@ def add_features(sender, **kwargs):
     INDEX_FROM_HTML_FILES = 'index_from_html_files'
 
     # Build related features
-    DONT_CREATE_INDEX = "dont_create_index"
     HOSTING_INTEGRATIONS = "hosting_integrations"
     NO_CONFIG_FILE_DEPRECATED = "no_config_file"
 
@@ -2061,13 +2060,6 @@ def add_features(sender, **kwargs):
                 "sources"
             ),
         ),
-
-        (
-            DONT_CREATE_INDEX,
-            _(
-                "Sphinx: Do not create index.md or README.rst if the project does not have one."
-            ),
-        ),
         (
             HOSTING_INTEGRATIONS,
             _(
diff --git a/readthedocs/projects/tasks/builds.py b/readthedocs/projects/tasks/builds.py
@@ -543,6 +543,7 @@ def get_valid_artifact_types(self):
          - it exists
          - it is a directory
          - does not contains more than 1 files (only PDF, HTMLZip, ePUB)
+         - it contains an "index.html" file at its root directory (only HTML)
 
         TODO: remove the limitation of only 1 file.
         Add support for multiple PDF files in the output directory and
@@ -554,6 +555,28 @@ def get_valid_artifact_types(self):
                 version=self.data.version.slug,
                 type_=artifact_type,
             )
+
+            if artifact_type == "html":
+                index_html_filepath = os.path.join(artifact_directory, "index.html")
+                readme_html_filepath = os.path.join(artifact_directory, "README.html")
+                if not os.path.exists(index_html_filepath) and not os.path.exists(
+                    readme_html_filepath
+                ):
+                    log.warning(
+                        "Failing the build. "
+                        "HTML output does not contain an 'index.html' at its root directory.",
+                        index_html=index_html_filepath,
+                        readme_html=readme_html_filepath,
+                    )
+                    # TODO: uncomment this line to fail the build once we have
+                    # communicated with projects without an index.html or
+                    # README.html
+                    #
+                    # NOTE: we want to deprecate serving README.html as an
+                    # index.html file as well.
+                    #
+                    # raise BuildUserError(BuildUserError.BUILD_OUTPUT_HTML_NO_INDEX_FILE)
+
             if not os.path.exists(artifact_directory):
                 # There is no output directory.
                 # Skip this format.
diff --git a/readthedocs/rtd_tests/tests/test_doc_builder.py b/readthedocs/rtd_tests/tests/test_doc_builder.py
@@ -166,14 +166,17 @@ def test_html_context_only_has_public_versions(
         self.assertEqual(versions, {'v1', 'v2'})
 
     @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.docs_dir')
-    @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.create_index')
     @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.get_config_params')
     @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.run')
     @patch('readthedocs.builds.models.Version.get_conf_py_path')
     @patch('readthedocs.projects.models.Project.checkout_path')
     def test_create_conf_py(
-            self, checkout_path, get_conf_py_path, _,
-            get_config_params, create_index, docs_dir,
+        self,
+        checkout_path,
+        get_conf_py_path,
+        _,
+        get_config_params,
+        docs_dir,
     ):
         """
         Test for a project without ``conf.py`` file.
@@ -189,7 +192,6 @@ def test_create_conf_py(
         tmp_dir = tempfile.mkdtemp()
         checkout_path.return_value = tmp_dir
         docs_dir.return_value = tmp_dir
-        create_index.return_value = 'README.rst'
         get_config_params.return_value = {}
         get_conf_py_path.side_effect = ProjectConfigurationError
         python_env = Virtualenv(
@@ -224,14 +226,17 @@ def test_create_conf_py(
             )
 
     @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.docs_dir')
-    @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.create_index')
     @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.get_config_params')
     @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.run')
     @patch('readthedocs.builds.models.Version.get_conf_py_path')
     @patch('readthedocs.projects.models.Project.checkout_path')
     def test_multiple_conf_py(
-            self, checkout_path, get_conf_py_path, _, get_config_params,
-            create_index, docs_dir,
+        self,
+        checkout_path,
+        get_conf_py_path,
+        _,
+        get_config_params,
+        docs_dir,
     ):
         """
         Test for a project with multiple ``conf.py`` files.
@@ -245,7 +250,6 @@ def test_multiple_conf_py(
         tmp_docs_dir.join('test').mkdir().join('conf.py').write('')
         docs_dir.return_value = str(tmp_docs_dir)
         checkout_path.return_value = str(tmp_docs_dir)
-        create_index.return_value = 'README.rst'
         get_config_params.return_value = {}
         get_conf_py_path.side_effect = ProjectConfigurationError
         python_env = Virtualenv(
