From 3c4c74d6024bbdc399dafb1034670cc03b70ed12 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 7 Apr 2020 11:36:22 -0500 Subject: [PATCH 1/2] Make dashboard faster for projects with a lot of subprojects This is related to https://github.com/readthedocs/readthedocs.org/issues/6455 We list all subprojects on the detail view too. It doesn't throw a 5xx, but it's slow https://readthedocs.org/projects/circuitpython/ --- readthedocs/projects/views/mixins.py | 42 ++++++++++++++++++- readthedocs/projects/views/private.py | 37 +++------------- readthedocs/projects/views/public.py | 17 ++++---- .../templates/core/project_detail_right.html | 6 +-- 4 files changed, 58 insertions(+), 44 deletions(-) diff --git a/readthedocs/projects/views/mixins.py b/readthedocs/projects/views/mixins.py index c65ec3f01bc..46b9c8ff8ee 100644 --- a/readthedocs/projects/views/mixins.py +++ b/readthedocs/projects/views/mixins.py @@ -1,10 +1,10 @@ -# -*- coding: utf-8 -*- - """Mixin classes for project views.""" +from urllib.parse import urlparse from celery import chain from django.shortcuts import get_object_or_404 +from readthedocs.core.resolver import resolve, resolve_path from readthedocs.core.utils import prepare_build from readthedocs.projects.models import Project from readthedocs.projects.signals import project_import @@ -49,6 +49,44 @@ def get_context_data(self, **kwargs): return context +class ProjectRelationListMixin: + + """Injects ``subprojects_and_urls`` into the context.""" + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context['subprojects_and_urls'] = self._get_subprojects_and_urls() + return context + + def _get_subprojects_and_urls(self): + """ + Get a tuple of subprojects and its absolute URls. + + All subprojects share the domain from the parent, + so instead of resolving the domain and path for each subproject, + we resolve only the path of each one. + """ + subprojects_and_urls = [] + + project = self.get_project() + main_domain = resolve(project) + parsed_main_domain = urlparse(main_domain) + + subprojects = project.subprojects.select_related('child') + for subproject in subprojects: + subproject_path = resolve_path(subproject.child) + parsed_subproject_domain = parsed_main_domain._replace( + path=subproject_path, + ) + subprojects_and_urls.append( + ( + subproject, + parsed_subproject_domain.geturl(), + ) + ) + return subprojects_and_urls + + class ProjectImportMixin: """Helpers to import a Project.""" diff --git a/readthedocs/projects/views/private.py b/readthedocs/projects/views/private.py index 8bd39ce8425..b730e3c5b80 100644 --- a/readthedocs/projects/views/private.py +++ b/readthedocs/projects/views/private.py @@ -2,7 +2,6 @@ import csv import logging -from urllib.parse import urlparse from allauth.socialaccount.models import SocialAccount from django.conf import settings @@ -45,7 +44,6 @@ LoginRequiredMixin, PrivateViewMixin, ) -from readthedocs.core.resolver import resolve, resolve_path from readthedocs.core.utils import broadcast, trigger_build from readthedocs.core.utils.extend import SettingsOverrideObject from readthedocs.integrations.models import HttpExchange, Integration @@ -81,7 +79,10 @@ from readthedocs.projects.notifications import EmailConfirmNotification from readthedocs.projects.utils import Echo from readthedocs.projects.views.base import ProjectAdminMixin, ProjectSpamMixin -from readthedocs.projects.views.mixins import ProjectImportMixin +from readthedocs.projects.views.mixins import ( + ProjectImportMixin, + ProjectRelationListMixin, +) from readthedocs.search.models import SearchQuery from ..tasks import retry_domain_verification @@ -459,41 +460,13 @@ def get_success_url(self): return reverse('projects_subprojects', args=[self.get_project().slug]) -class ProjectRelationshipList(ProjectRelationshipMixin, ListView): +class ProjectRelationshipList(ProjectRelationListMixin, ProjectRelationshipMixin, ListView): def get_context_data(self, **kwargs): ctx = super().get_context_data(**kwargs) ctx['superproject'] = self.project.superprojects.first() - ctx['subprojects_and_urls'] = self.get_subprojects_and_urls() return ctx - def get_subprojects_and_urls(self): - """ - Get a tuple of subprojects and its absolute URls. - - All subprojects share the domain from the parent, - so instead of resolving the domain and path for each subproject, - we resolve only the path of each one. - """ - subprojects_and_urls = [] - - main_domain = resolve(self.project) - parsed_main_domain = urlparse(main_domain) - - subprojects = self.object_list.select_related('child') - for subproject in subprojects: - subproject_path = resolve_path(subproject.child) - parsed_subproject_domain = parsed_main_domain._replace( - path=subproject_path, - ) - subprojects_and_urls.append( - ( - subproject, - parsed_subproject_domain.geturl(), - ) - ) - return subprojects_and_urls - class ProjectRelationshipCreate(ProjectRelationshipMixin, CreateView): diff --git a/readthedocs/projects/views/public.py b/readthedocs/projects/views/public.py index a388f65a774..4d796523bb3 100644 --- a/readthedocs/projects/views/public.py +++ b/readthedocs/projects/views/public.py @@ -6,8 +6,8 @@ import mimetypes import operator import os -from urllib.parse import urlparse from collections import OrderedDict +from urllib.parse import urlparse import requests from django.conf import settings @@ -16,13 +16,13 @@ from django.core.files.storage import get_storage_class from django.db.models import prefetch_related_objects from django.http import HttpResponse -from django.shortcuts import get_object_or_404, render, redirect +from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse +from django.utils.crypto import constant_time_compare +from django.utils.encoding import force_bytes from django.views import View from django.views.decorators.cache import never_cache from django.views.generic import DetailView, ListView -from django.utils.crypto import constant_time_compare -from django.utils.encoding import force_bytes from taggit.models import Tag from readthedocs.analytics.tasks import analytics_event @@ -33,12 +33,12 @@ from readthedocs.core.utils.extend import SettingsOverrideObject from readthedocs.projects.models import Project from readthedocs.projects.templatetags.projects_tags import sort_version_aware +from readthedocs.projects.views.mixins import ProjectRelationListMixin from readthedocs.proxito.views.mixins import ServeDocsMixin from readthedocs.proxito.views.utils import _get_project_data_from_request -from .base import ProjectOnboardMixin from ..constants import PRIVATE - +from .base import ProjectOnboardMixin log = logging.getLogger(__name__) search_log = logging.getLogger(__name__ + '.search') @@ -81,7 +81,7 @@ def project_redirect(request, invalid_project_slug): )) -class ProjectDetailView(BuildTriggerMixin, ProjectOnboardMixin, DetailView): +class ProjectDetailView(ProjectRelationListMixin, BuildTriggerMixin, ProjectOnboardMixin, DetailView): """Display project onboard steps.""" @@ -91,6 +91,9 @@ class ProjectDetailView(BuildTriggerMixin, ProjectOnboardMixin, DetailView): def get_queryset(self): return Project.objects.protected(self.request.user) + def get_project(self): + return self.get_object() + def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) diff --git a/readthedocs/templates/core/project_detail_right.html b/readthedocs/templates/core/project_detail_right.html index 55ef2f4a172..5b9395df09b 100644 --- a/readthedocs/templates/core/project_detail_right.html +++ b/readthedocs/templates/core/project_detail_right.html @@ -128,11 +128,11 @@

{% trans "Translations" %}

{% endblock %} {% block subprojects %} - {% if project.subprojects.exists %} + {% if subprojects_and_urls %}

{% trans "Sub Projects" %}

{% endif %} From b8976a00b761adcf1787d6e3c1c052a7fc3057ee Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 8 Apr 2020 09:26:30 -0500 Subject: [PATCH 2/2] Linter --- readthedocs/projects/views/public.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/readthedocs/projects/views/public.py b/readthedocs/projects/views/public.py index 4d796523bb3..9fdbdaa9041 100644 --- a/readthedocs/projects/views/public.py +++ b/readthedocs/projects/views/public.py @@ -81,7 +81,12 @@ def project_redirect(request, invalid_project_slug): )) -class ProjectDetailView(ProjectRelationListMixin, BuildTriggerMixin, ProjectOnboardMixin, DetailView): +class ProjectDetailView( + ProjectRelationListMixin, + BuildTriggerMixin, + ProjectOnboardMixin, + DetailView +): """Display project onboard steps."""