Feature flag: remove DONT_CREATE_INDEX

docs/user/feature-flags.rst

@@ -16,11 +16,3 @@ Available flags
 
 Makes Read the Docs to install all the requirements at once on ``conda create`` step.
 This helps users to pin dependencies on conda and to improve build time.
-
-``DONT_CREATE_INDEX``: Do not create index.md or README.rst if the project does not have one.
-
-When Read the Docs detects that your project doesn't have an ``index.md`` or ``README.rst``,
-it auto-generate one for you with instructions about how to proceed.
-
-In case you are using a static HTML page as index or an generated index from code,
-this behavior could be a problem. With this feature flag you can disable that.

readthedocs/doc_builder/backends/mkdocs.py

@@ -138,7 +138,7 @@ def load_yaml_config(self):
             raise MkDocsYAMLParseError(
                 'Your mkdocs.yml could not be loaded, '
                 'possibly due to a syntax error{note}'.format(note=note),
-            )
+            ) from exc
 
     def append_conf(self):
         """
@@ -156,7 +156,6 @@ def append_conf(self):
                 MkDocsYAMLParseError.INVALID_DOCS_DIR_CONFIG,
             )
 
-        self.create_index(extension='md')
         user_config['docs_dir'] = docs_dir
 
         # MkDocs <=0.17.x doesn't support absolute paths,
@@ -215,7 +214,9 @@ def append_conf(self):
             docs_dir=os.path.relpath(docs_path, self.project_path),
             mkdocs_config=user_config,
         )
-        with open(os.path.join(docs_path, 'readthedocs-data.js'), 'w') as f:
+        with open(
+            os.path.join(docs_path, "readthedocs-data.js"), "w", encoding="utf-8"
+        ) as f:
             f.write(rtd_data)
 
         # Use Read the Docs' analytics setup rather than mkdocs'
@@ -230,10 +231,11 @@ def append_conf(self):
                 user_config['theme'] = self.DEFAULT_THEME_NAME
 
         # Write the modified mkdocs configuration
-        yaml_dump_safely(
-            user_config,
-            open(self.yaml_file, 'w'),
-        )
+        with open(self.yaml_file, "w", encoding="utf-8") as f:
+            yaml_dump_safely(
+                user_config,
+                f,
+            )
 
         # Write the mkdocs.yml to the build logs
         self.run(
@@ -369,10 +371,10 @@ class SafeLoader(yaml.SafeLoader):  # pylint: disable=too-many-ancestors
     Issue https://github.com/readthedocs/readthedocs.org/issues/7461
     """
 
-    def ignore_unknown(self, node):  # pylint: disable=no-self-use, unused-argument
+    def ignore_unknown(self, node):  # pylint: disable=unused-argument
         return None
 
-    def construct_python_name(self, suffix, node):  # pylint: disable=no-self-use, unused-argument
+    def construct_python_name(self, suffix, node):  # pylint: disable=unused-argument
         return ProxyPythonName(suffix)
 
 

readthedocs/doc_builder/backends/sphinx.py

@@ -276,9 +276,13 @@ def append_conf(self):
 
         The default content is rendered from ``doc_builder/conf.py.tmpl``.
         """
-        if self.config_file is None:
-            master_doc = self.create_index(extension='rst')
-            self._write_config(master_doc=master_doc)
+
+        # Generate a `conf.py` from a template
+        #
+        # TODO: we should remove this feature at some point to move forward
+        # with the idea of remove magic from the builders.
+        if not self.config_file:
+            self._write_config()
 
         try:
             self.config_file = (

readthedocs/doc_builder/base.py

@@ -5,8 +5,6 @@
 
 import structlog
 
-from readthedocs.core.utils.filesystem import safe_open
-from readthedocs.projects.models import Feature
 
 log = structlog.get_logger(__name__)
 
@@ -15,8 +13,8 @@ def restoring_chdir(fn):
     # XXX:dc: This would be better off in a neutral module
     @wraps(fn)
     def decorator(*args, **kw):
+        path = os.getcwd()
         try:
-            path = os.getcwd()
             return fn(*args, **kw)
         finally:
             os.chdir(path)
@@ -62,44 +60,6 @@ def docs_dir(self):
 
         return self.project_path
 
-    def create_index(self, extension='md', **__):
-        """Create an index file if it needs it."""
-        docs_dir = self.docs_dir()
-
-        index_filename = os.path.join(
-            docs_dir,
-            'index.{ext}'.format(ext=extension),
-        )
-        if not os.path.exists(index_filename):
-            readme_filename = os.path.join(
-                docs_dir,
-                'README.{ext}'.format(ext=extension),
-            )
-            if os.path.exists(readme_filename):
-                return 'README'
-
-            if not self.project.has_feature(Feature.DONT_CREATE_INDEX):
-                index_text = """
-
-Welcome to Read the Docs
-------------------------
-
-This is an autogenerated index file.
-
-Please create an ``index.{ext}`` or ``README.{ext}`` file with your own content
-under the root (or ``/docs``) directory in your repository.
-
-If you want to use another markup, choose a different builder in your settings.
-Check out our `Getting Started Guide
-<https://docs.readthedocs.io/en/latest/getting_started.html>`_ to become more
-familiar with Read the Docs.
-                """
-
-                with safe_open(index_filename, "w+") as index_file:
-                    index_file.write(index_text.format(dir=docs_dir, ext=extension))
-
-        return 'index'
-
     def run(self, *args, **kwargs):
         """Proxy run to build environment."""
         return self.build_env.run(*args, **kwargs)

readthedocs/doc_builder/exceptions.py

@@ -52,6 +52,10 @@ class BuildUserError(BuildBaseException):
         "and it is not currently supported. "
         'Please, remove all the files but the "{artifact_type}" you want to upload.'
     )
+    BUILD_OUTPUT_HTML_NO_INDEX_FILE = gettext_noop(
+        "Your documentation did not generate an 'index.html' at its root directory. "
+        "This is required for documentation serving at the root URL for this version."
+    )
     BUILD_OUTPUT_OLD_DIRECTORY_USED = gettext_noop(
         "Some files were detected in an unsupported output path, '_build/html'. "
         "Ensure your project is configured to use the output path "

readthedocs/projects/models.py

@@ -1937,7 +1937,6 @@ def add_features(sender, **kwargs):
     INDEX_FROM_HTML_FILES = 'index_from_html_files'
 
     # Build related features
-    DONT_CREATE_INDEX = "dont_create_index"
     HOSTING_INTEGRATIONS = "hosting_integrations"
     NO_CONFIG_FILE_DEPRECATED = "no_config_file"
 
@@ -2061,13 +2060,6 @@ def add_features(sender, **kwargs):
                 "sources"
             ),
         ),
-
-        (
-            DONT_CREATE_INDEX,
-            _(
-                "Sphinx: Do not create index.md or README.rst if the project does not have one."
-            ),
-        ),
         (
             HOSTING_INTEGRATIONS,
             _(

readthedocs/projects/tasks/builds.py

@@ -543,6 +543,7 @@ def get_valid_artifact_types(self):
          - it exists
          - it is a directory
          - does not contains more than 1 files (only PDF, HTMLZip, ePUB)
+         - it contains an "index.html" file at its root directory (only HTML)
 
         TODO: remove the limitation of only 1 file.
         Add support for multiple PDF files in the output directory and
@@ -554,6 +555,28 @@ def get_valid_artifact_types(self):
                 version=self.data.version.slug,
                 type_=artifact_type,
             )
+
+            if artifact_type == "html":
+                index_html_filepath = os.path.join(artifact_directory, "index.html")
+                readme_html_filepath = os.path.join(artifact_directory, "README.html")
+                if not os.path.exists(index_html_filepath) and not os.path.exists(
+                    readme_html_filepath
+                ):
+                    log.warning(
+                        "Failing the build. "
+                        "HTML output does not contain an 'index.html' at its root directory.",
+                        index_html=index_html_filepath,
+                        readme_html=readme_html_filepath,
+                    )
+                    # TODO: uncomment this line to fail the build once we have
+                    # communicated with projects without an index.html or
+                    # README.html
+                    #
+                    # NOTE: we want to deprecate serving README.html as an
+                    # index.html file as well.
+                    #
+                    # raise BuildUserError(BuildUserError.BUILD_OUTPUT_HTML_NO_INDEX_FILE)
+
             if not os.path.exists(artifact_directory):
                 # There is no output directory.
                 # Skip this format.

readthedocs/rtd_tests/tests/test_doc_builder.py

@@ -166,14 +166,17 @@ def test_html_context_only_has_public_versions(
         self.assertEqual(versions, {'v1', 'v2'})
 
     @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.docs_dir')
-    @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.create_index')
     @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.get_config_params')
     @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.run')
     @patch('readthedocs.builds.models.Version.get_conf_py_path')
     @patch('readthedocs.projects.models.Project.checkout_path')
     def test_create_conf_py(
-            self, checkout_path, get_conf_py_path, _,
-            get_config_params, create_index, docs_dir,
+        self,
+        checkout_path,
+        get_conf_py_path,
+        _,
+        get_config_params,
+        docs_dir,
     ):
         """
         Test for a project without ``conf.py`` file.
@@ -189,7 +192,6 @@ def test_create_conf_py(
         tmp_dir = tempfile.mkdtemp()
         checkout_path.return_value = tmp_dir
         docs_dir.return_value = tmp_dir
-        create_index.return_value = 'README.rst'
         get_config_params.return_value = {}
         get_conf_py_path.side_effect = ProjectConfigurationError
         python_env = Virtualenv(
@@ -224,14 +226,17 @@ def test_create_conf_py(
             )
 
     @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.docs_dir')
-    @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.create_index')
     @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.get_config_params')
     @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.run')
     @patch('readthedocs.builds.models.Version.get_conf_py_path')
     @patch('readthedocs.projects.models.Project.checkout_path')
     def test_multiple_conf_py(
-            self, checkout_path, get_conf_py_path, _, get_config_params,
-            create_index, docs_dir,
+        self,
+        checkout_path,
+        get_conf_py_path,
+        _,
+        get_config_params,
+        docs_dir,
     ):
         """
         Test for a project with multiple ``conf.py`` files.
@@ -245,7 +250,6 @@ def test_multiple_conf_py(
         tmp_docs_dir.join('test').mkdir().join('conf.py').write('')
         docs_dir.return_value = str(tmp_docs_dir)
         checkout_path.return_value = str(tmp_docs_dir)
-        create_index.return_value = 'README.rst'
         get_config_params.return_value = {}
         get_conf_py_path.side_effect = ProjectConfigurationError
         python_env = Virtualenv(