Merge pull request #5246 from rtfd/hotfix-exclude-files

Remove excluding files on search.

Author: ericholscher

readthedocs/config/config.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # pylint: disable=too-many-lines
 
 """Build configuration for rtd."""
@@ -36,7 +37,6 @@
     validate_string,
 )
 
-
 __all__ = (
     'ALL',
     'load',
@@ -64,14 +64,16 @@
 INVALID_KEYS_COMBINATION = 'invalid-keys-combination'
 INVALID_KEY = 'invalid-key'
 
-DOCKER_DEFAULT_IMAGE = getattr(settings, 'DOCKER_DEFAULT_IMAGE', 'readthedocs/build')
+DOCKER_DEFAULT_IMAGE = getattr(
+    settings, 'DOCKER_DEFAULT_IMAGE', 'readthedocs/build'
+)
 DOCKER_DEFAULT_VERSION = getattr(settings, 'DOCKER_DEFAULT_VERSION', '2.0')
 # These map to corresponding settings in the .org,
 # so they haven't been renamed.
 DOCKER_IMAGE = getattr(
     settings,
     'DOCKER_IMAGE',
-    '{}:{}'.format(DOCKER_DEFAULT_IMAGE, DOCKER_DEFAULT_VERSION)
+    '{}:{}'.format(DOCKER_DEFAULT_IMAGE, DOCKER_DEFAULT_VERSION),
 )
 DOCKER_IMAGE_SETTINGS = getattr(settings, 'DOCKER_IMAGE_SETTINGS', {})
 
@@ -243,9 +245,7 @@ def python_full_version(self):
             # Get the highest version of the major series version if user only
             # gave us a version of '2', or '3'
             ver = max(
-                v
-                for v in self.get_valid_python_versions()
-                if v < ver + 1
+                v for v in self.get_valid_python_versions() if v < ver + 1
             )
         return ver
 
@@ -275,7 +275,6 @@ def get_valid_python_versions_for_image(self, build_image):
         Returns supported versions for the ``DOCKER_DEFAULT_VERSION`` if not
         ``build_image`` found.
         """
-
         if build_image not in DOCKER_IMAGE_SETTINGS:
             build_image = '{}:{}'.format(
                 DOCKER_DEFAULT_IMAGE,
@@ -320,7 +319,9 @@ def get_valid_python_versions(self):
         except (KeyError, TypeError):
             versions = set()
             for _, options in DOCKER_IMAGE_SETTINGS.items():
-                versions = versions.union(options['python']['supported_versions'])
+                versions = versions.union(
+                    options['python']['supported_versions']
+                )
             return versions
 
     def get_valid_formats(self):  # noqa
@@ -510,7 +511,8 @@ def validate_requirements_file(self):
             return None
         with self.catch_validation_error('requirements_file'):
             requirements_file = validate_file(
-                requirements_file, self.base_path
+                requirements_file,
+                self.base_path,
             )
         return requirements_file
 
@@ -546,23 +548,23 @@ def python(self):
         python_install.append(
             PythonInstallRequirements(
                 requirements=requirements,
-            )
+            ),
         )
         if python['install_with_pip']:
             python_install.append(
                 PythonInstall(
                     path=self.base_path,
                     method=PIP,
                     extra_requirements=python['extra_requirements'],
-                )
+                ),
             )
         elif python['install_with_setup']:
             python_install.append(
                 PythonInstall(
                     path=self.base_path,
                     method=SETUPTOOLS,
                     extra_requirements=[],
-                )
+                ),
             )
 
         return Python(
@@ -785,30 +787,30 @@ def validate_python_install(self, index):
             with self.catch_validation_error(requirements_key):
                 requirements = validate_file(
                     self.pop_config(requirements_key),
-                    self.base_path
+                    self.base_path,
                 )
                 python_install['requirements'] = requirements
         elif 'path' in raw_install:
             path_key = key + '.path'
             with self.catch_validation_error(path_key):
                 path = validate_directory(
                     self.pop_config(path_key),
-                    self.base_path
+                    self.base_path,
                 )
                 python_install['path'] = path
 
             method_key = key + '.method'
             with self.catch_validation_error(method_key):
                 method = validate_choice(
                     self.pop_config(method_key, PIP),
-                    self.valid_install_method
+                    self.valid_install_method,
                 )
                 python_install['method'] = method
 
             extra_req_key = key + '.extra_requirements'
             with self.catch_validation_error(extra_req_key):
                 extra_requirements = validate_list(
-                    self.pop_config(extra_req_key, [])
+                    self.pop_config(extra_req_key, []),
                 )
                 if extra_requirements and python_install['method'] != PIP:
                     self.error(
@@ -1060,13 +1062,9 @@ def python(self):
         python = self._config['python']
         for install in python['install']:
             if 'requirements' in install:
-                python_install.append(
-                    PythonInstallRequirements(**install)
-                )
+                python_install.append(PythonInstallRequirements(**install),)
             elif 'path' in install:
-                python_install.append(
-                    PythonInstall(**install)
-                )
+                python_install.append(PythonInstall(**install),)
         return Python(
             version=python['version'],
             install=python_install,

readthedocs/search/documents.py

@@ -1,9 +1,11 @@
+# -*- coding: utf-8 -*-
 import logging
 
 from django.conf import settings
 from django_elasticsearch_dsl import DocType, Index, fields
 
-from readthedocs.projects.models import Project, HTMLFile
+from readthedocs.projects.models import HTMLFile, Project
+
 
 project_conf = settings.ES_INDEXES['project']
 project_index = Index(project_conf['name'])
@@ -13,7 +15,6 @@
 page_index = Index(page_conf['name'])
 page_index.settings(**page_conf['settings'])
 
-
 log = logging.getLogger(__name__)
 
 
@@ -22,10 +23,12 @@ class ProjectDocument(DocType):
 
     # Metadata
     url = fields.TextField(attr='get_absolute_url')
-    users = fields.NestedField(properties={
-        'username': fields.TextField(),
-        'id': fields.IntegerField(),
-    })
+    users = fields.NestedField(
+        properties={
+            'username': fields.TextField(),
+            'id': fields.IntegerField(),
+        }
+    )
     language = fields.KeywordField()
 
     class Meta(object):
@@ -67,7 +70,8 @@ class Meta(object):
 
     @classmethod
     def faceted_search(
-        cls, query, user, projects_list=None, versions_list=None, filter_by_user=True
+            cls, query, user, projects_list=None, versions_list=None,
+            filter_by_user=True
     ):
         from readthedocs.search.faceted_search import PageSearch
         kwargs = {
@@ -87,22 +91,26 @@ def faceted_search(
         return PageSearch(**kwargs)
 
     def get_queryset(self):
-        """Overwrite default queryset to filter certain files to index"""
+        """Overwrite default queryset to filter certain files to index."""
         queryset = super(PageDocument, self).get_queryset()
 
-        # Exclude some files to not index
-        excluded_files = [
-            'search.html',
-            'genindex.html',
-            'py-modindex.html',
-            'search/index.html',
-            'genindex/index.html',
-            'py-modindex/index.html',
-        ]
-
         # Do not index files that belong to non sphinx project
         # Also do not index certain files
-        queryset = queryset.filter(project__documentation_type__contains='sphinx')
-        for ending in excluded_files:
-            queryset = queryset.exclude(path__endswith=ending)
+        queryset = queryset.filter(
+            project__documentation_type__contains='sphinx'
+        )
+
+        # TODO: Make this smarter
+        # This was causing issues excluding some valid user documentation pages
+        # excluded_files = [
+        #     'search.html',
+        #     'genindex.html',
+        #     'py-modindex.html',
+        #     'search/index.html',
+        #     'genindex/index.html',
+        #     'py-modindex/index.html',
+        # ]
+        # for ending in excluded_files:
+        #     queryset = queryset.exclude(path=ending)
+
         return queryset