Merge pull request #5451 from rtfd/davidfischer/optimize-repo-queries

Optimize the repos API query

Author: ericholscher

readthedocs/oauth/models.py

@@ -179,19 +179,23 @@ def clone_fuzzy_url(self):
     def matches(self, user):
         """Projects that exist with repository URL already."""
         # Support Git scheme GitHub url format that may exist in database
-        fuzzy_url = self.clone_url.replace('git://', '').replace('.git', '')
-        projects = (Project
-                    .objects
-                    .public(user)
-                    .filter(Q(repo=self.clone_url) |
-                            Q(repo__iendswith=fuzzy_url) |
-                            Q(repo__iendswith=fuzzy_url + '.git')))  # yapf: disable
+        truncated_url = self.clone_url.replace('.git', '')
+        http_url = self.clone_url.replace('git://', 'https://').replace('.git', '')
+
+        projects = Project.objects.public(user).filter(
+            Q(repo=self.clone_url) |
+            Q(repo=truncated_url) |
+            Q(repo=truncated_url + '.git') |
+            Q(repo=http_url) |
+            Q(repo=http_url + '.git')
+        ).values('slug')
+
         return [{
-            'id': project.slug,
+            'id': project['slug'],
             'url': reverse(
                 'projects_detail',
                 kwargs={
-                    'project_slug': project.slug,
+                    'project_slug': project['slug'],
                 },
             ),
         } for project in projects]

readthedocs/projects/migrations/0041_index-repo-field.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-03-13 17:03
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import readthedocs.projects.validators
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('projects', '0040_increase_path_max_length'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='project',
+            name='repo',
+            field=models.CharField(db_index=True, help_text='Hosted documentation repository URL', max_length=255, validators=[readthedocs.projects.validators.RepositoryURLValidator()], verbose_name='Repository URL'),
+        ),
+    ]

readthedocs/projects/models.py

@@ -116,6 +116,7 @@ class Project(models.Model):
         max_length=255,
         validators=[validate_repository_url],
         help_text=_('Hosted documentation repository URL'),
+        db_index=True,
     )
     repo_type = models.CharField(
         _('Repository type'),

readthedocs/restapi/serializers.py

@@ -173,6 +173,8 @@ class RemoteRepositorySerializer(serializers.ModelSerializer):
     """Remote service repository serializer."""
 
     organization = RemoteOrganizationSerializer()
+
+    # This field does create an additional query per object returned
     matches = serializers.SerializerMethodField()
 
     class Meta:

readthedocs/restapi/views/model_views.py

@@ -337,6 +337,10 @@ def get_queryset(self):
                 service.adapter.provider_id for service in registry
             ],
         )
+
+        # optimizes for the RemoteOrganizationSerializer
+        query = query.select_related('organization')
+
         return query