Search: move api.py into a module (#9616)

Author: stsewd

readthedocs/api/v2/proxied_urls.py

@@ -9,7 +9,7 @@
 
 from readthedocs.analytics.proxied_api import AnalyticsView
 from readthedocs.api.v2.views.proxied import ProxiedEmbedAPI, ProxiedFooterHTML
-from readthedocs.search.proxied_api import ProxiedPageSearchAPIView
+from readthedocs.search.api.v2.views import ProxiedPageSearchAPIView
 
 api_footer_urls = [
     re_path(r'footer_html/', ProxiedFooterHTML.as_view(), name='footer_html'),

readthedocs/search/api/__init__.py

@@ -0,0 +1,102 @@
+from collections import namedtuple
+from math import ceil
+
+from django.utils.translation import gettext as _
+from rest_framework.exceptions import NotFound
+from rest_framework.pagination import PageNumberPagination
+
+
+class PaginatorPage:
+
+    """
+    Mimics the result from a paginator.
+
+    By using this class, we avoid having to override a lot of methods
+    of `PageNumberPagination` to make it work with the ES DSL object.
+    """
+
+    def __init__(self, page_number, total_pages, count):
+        self.number = page_number
+        Paginator = namedtuple("Paginator", ["num_pages", "count"])
+        self.paginator = Paginator(total_pages, count)
+
+    def has_next(self):
+        return self.number < self.paginator.num_pages
+
+    def has_previous(self):
+        return self.number > 1
+
+    def next_page_number(self):
+        return self.number + 1
+
+    def previous_page_number(self):
+        return self.number - 1
+
+
+class SearchPagination(PageNumberPagination):
+
+    """Paginator for the results of PageSearch."""
+
+    page_size = 50
+    page_size_query_param = "page_size"
+    max_page_size = 100
+
+    def _get_page_number(self, number):
+        try:
+            if isinstance(number, float) and not number.is_integer():
+                raise ValueError
+            number = int(number)
+        except (TypeError, ValueError):
+            number = -1
+        return number
+
+    def paginate_queryset(self, queryset, request, view=None):
+        """
+        Override to get the paginated result from the ES queryset.
+
+        This makes use of our custom paginator and slicing support from the ES DSL object,
+        instead of the one used by django's ORM.
+
+        Mostly inspired by https://github.com/encode/django-rest-framework/blob/acbd9d8222e763c7f9c7dc2de23c430c702e06d4/rest_framework/pagination.py#L191  # noqa
+        """
+        # Needed for other methods of this class.
+        self.request = request
+
+        page_size = self.get_page_size(request)
+        page_number = request.query_params.get(self.page_query_param, 1)
+
+        original_page_number = page_number
+        page_number = self._get_page_number(page_number)
+
+        if page_number <= 0:
+            msg = self.invalid_page_message.format(
+                page_number=original_page_number,
+                message=_("Invalid page"),
+            )
+            raise NotFound(msg)
+
+        start = (page_number - 1) * page_size
+        end = page_number * page_size
+
+        result = []
+        total_count = 0
+        total_pages = 1
+
+        if queryset:
+            result = queryset[start:end].execute()
+            total_count = result.hits.total["value"]
+            hits = max(1, total_count)
+            total_pages = ceil(hits / page_size)
+
+        if total_pages > 1 and self.template is not None:
+            # The browsable API should display pagination controls.
+            self.display_page_controls = True
+
+        # Needed for other methods of this class.
+        self.page = PaginatorPage(
+            page_number=page_number,
+            total_pages=total_pages,
+            count=total_count,
+        )
+
+        return result

readthedocs/search/api/pagination.py

@@ -18,8 +18,8 @@
 from readthedocs.projects.models import Project
 
 # Structures used for storing cached data of a version mostly.
-ProjectData = namedtuple('ProjectData', ['version', 'alias'])
-VersionData = namedtuple('VersionData', ['slug', 'docs_url'])
+ProjectData = namedtuple("ProjectData", ["version", "alias"])
+VersionData = namedtuple("VersionData", ["slug", "docs_url"])
 
 
 class ProjectHighlightSerializer(serializers.Serializer):
@@ -29,31 +29,31 @@ class ProjectHighlightSerializer(serializers.Serializer):
     description = serializers.SerializerMethodField()
 
     def get_name(self, obj):
-        return list(getattr(obj, 'name', []))
+        return list(getattr(obj, "name", []))
 
     def get_slug(self, obj):
-        return list(getattr(obj, 'slug', []))
+        return list(getattr(obj, "slug", []))
 
     def get_description(self, obj):
-        return list(getattr(obj, 'description', []))
+        return list(getattr(obj, "description", []))
 
 
 class ProjectSearchSerializer(serializers.Serializer):
 
-    type = serializers.CharField(default='project', source=None, read_only=True)
+    type = serializers.CharField(default="project", source=None, read_only=True)
     name = serializers.CharField()
     slug = serializers.CharField()
-    link = serializers.CharField(source='url')
+    link = serializers.CharField(source="url")
     description = serializers.CharField()
-    highlights = ProjectHighlightSerializer(source='meta.highlight', default=dict)
+    highlights = ProjectHighlightSerializer(source="meta.highlight", default=dict)
 
 
 class PageHighlightSerializer(serializers.Serializer):
 
     title = serializers.SerializerMethodField()
 
     def get_title(self, obj):
-        return list(getattr(obj, 'title', []))
+        return list(getattr(obj, "title", []))
 
 
 class PageSearchSerializer(serializers.Serializer):
@@ -66,14 +66,14 @@ class PageSearchSerializer(serializers.Serializer):
     It's a dictionary mapping the project slug to a ProjectData object.
     """
 
-    type = serializers.CharField(default='page', source=None, read_only=True)
+    type = serializers.CharField(default="page", source=None, read_only=True)
     project = serializers.CharField()
     project_alias = serializers.SerializerMethodField()
     version = serializers.CharField()
     title = serializers.CharField()
     path = serializers.SerializerMethodField()
     domain = serializers.SerializerMethodField()
-    highlights = PageHighlightSerializer(source='meta.highlight', default=dict)
+    highlights = PageHighlightSerializer(source="meta.highlight", default=dict)
     blocks = serializers.SerializerMethodField()
 
     def _get_project_data(self, obj):
@@ -85,16 +85,18 @@ def _get_project_data(self, obj):
         If the result is fetched from the database,
         it's cached into ``projects_data``.
         """
-        project_data = self.context.get('projects_data', {}).get(obj.project)
+        project_data = self.context.get("projects_data", {}).get(obj.project)
         if project_data:
             return project_data
 
         project = Project.objects.filter(slug=obj.project).first()
         if project:
             docs_url = project.get_docs_url(version_slug=obj.version)
-            project_alias = project.superprojects.values_list('alias', flat=True).first()
+            project_alias = project.superprojects.values_list(
+                "alias", flat=True
+            ).first()
 
-            projects_data = self.context.setdefault('projects_data', {})
+            projects_data = self.context.setdefault("projects_data", {})
             version_data = VersionData(
                 slug=obj.version,
                 docs_url=docs_url,
@@ -116,7 +118,7 @@ def get_domain(self, obj):
         full_path = self._get_full_path(obj)
         if full_path:
             parsed = urlparse(full_path)
-            return f'{parsed.scheme}://{parsed.netloc}'
+            return f"{parsed.scheme}://{parsed.netloc}"
         return None
 
     def get_path(self, obj):
@@ -136,16 +138,16 @@ def _get_full_path(self, obj):
             # and always end it with / so it goes directly to proxito.
             # For a generic doctype we just strip the index.html part if it exists.
             if obj.doctype in {SPHINX_HTMLDIR, MKDOCS, GENERIC}:
-                path = re.sub('(^|/)index.html$', '/', path)
+                path = re.sub("(^|/)index.html$", "/", path)
 
-            return docs_url.rstrip('/') + '/' + path.lstrip('/')
+            return docs_url.rstrip("/") + "/" + path.lstrip("/")
         return None
 
     def get_blocks(self, obj):
         """Combine and sort inner results (domains and sections)."""
         serializers = {
-            'domain': DomainSearchSerializer,
-            'section': SectionSearchSerializer,
+            "domain": DomainSearchSerializer,
+            "section": SectionSearchSerializer,
         }
 
         inner_hits = obj.meta.inner_hits
@@ -154,19 +156,16 @@ def get_blocks(self, obj):
 
         # Make them identifiable before merging them
         for s in sections:
-            s.type = 'section'
+            s.type = "section"
         for d in domains:
-            d.type = 'domain'
+            d.type = "domain"
 
         sorted_results = sorted(
             itertools.chain(sections, domains),
-            key=attrgetter('meta.score'),
+            key=attrgetter("meta.score"),
             reverse=True,
         )
-        sorted_results = [
-            serializers[hit.type](hit).data
-            for hit in sorted_results
-        ]
+        sorted_results = [serializers[hit.type](hit).data for hit in sorted_results]
         return sorted_results
 
 
@@ -176,20 +175,20 @@ class DomainHighlightSerializer(serializers.Serializer):
     content = serializers.SerializerMethodField()
 
     def get_name(self, obj):
-        return list(getattr(obj, 'domains.name', []))
+        return list(getattr(obj, "domains.name", []))
 
     def get_content(self, obj):
-        return list(getattr(obj, 'domains.docstrings', []))
+        return list(getattr(obj, "domains.docstrings", []))
 
 
 class DomainSearchSerializer(serializers.Serializer):
 
-    type = serializers.CharField(default='domain', source=None, read_only=True)
-    role = serializers.CharField(source='role_name')
+    type = serializers.CharField(default="domain", source=None, read_only=True)
+    role = serializers.CharField(source="role_name")
     name = serializers.CharField()
-    id = serializers.CharField(source='anchor')
-    content = serializers.CharField(source='docstrings')
-    highlights = DomainHighlightSerializer(source='meta.highlight', default=dict)
+    id = serializers.CharField(source="anchor")
+    content = serializers.CharField(source="docstrings")
+    highlights = DomainHighlightSerializer(source="meta.highlight", default=dict)
 
 
 class SectionHighlightSerializer(serializers.Serializer):
@@ -198,16 +197,16 @@ class SectionHighlightSerializer(serializers.Serializer):
     content = serializers.SerializerMethodField()
 
     def get_title(self, obj):
-        return list(getattr(obj, 'sections.title', []))
+        return list(getattr(obj, "sections.title", []))
 
     def get_content(self, obj):
-        return list(getattr(obj, 'sections.content', []))
+        return list(getattr(obj, "sections.content", []))
 
 
 class SectionSearchSerializer(serializers.Serializer):
 
-    type = serializers.CharField(default='section', source=None, read_only=True)
+    type = serializers.CharField(default="section", source=None, read_only=True)
     id = serializers.CharField()
     title = serializers.CharField()
     content = serializers.CharField()
-    highlights = SectionHighlightSerializer(source='meta.highlight', default=dict)
+    highlights = SectionHighlightSerializer(source="meta.highlight", default=dict)

readthedocs/search/api/v2/__init__.py

@@ -0,0 +1,7 @@
+from django.urls import path
+
+from readthedocs.search.api.v2.views import PageSearchAPIView
+
+urlpatterns = [
+    path("", PageSearchAPIView.as_view(), name="search_api"),
+]