Merge pull request #7675 from saadmk11/update-oauth-services

Author: humitos

readthedocs/api/v2/serializers.py

@@ -186,6 +186,7 @@ class RemoteRepositorySerializer(serializers.ModelSerializer):
 
     class Meta:
         model = RemoteRepository
+        # TODO: Remove json field after it is removed from RemoteRepository
         exclude = ('json', 'users')
 
     def get_matches(self, obj):

readthedocs/api/v2/views/model_views.py

@@ -397,10 +397,10 @@ def get_queryset(self):
             )
 
         query = query.filter(
-            account__provider__in=[
+            remote_repository_relations__account__provider__in=[
                 service.adapter.provider_id for service in registry
             ],
-        )
+        ).distinct()
 
         # optimizes for the RemoteOrganizationSerializer
         query = query.select_related('organization')

readthedocs/oauth/admin.py

@@ -4,7 +4,11 @@
 
 from django.contrib import admin
 
-from .models import RemoteOrganization, RemoteRepository
+from .models import (
+    RemoteOrganization,
+    RemoteRepository,
+    RemoteRepositoryRelation,
+)
 
 
 class RemoteRepositoryAdmin(admin.ModelAdmin):
@@ -22,4 +26,5 @@ class RemoteOrganizationAdmin(admin.ModelAdmin):
 
 
 admin.site.register(RemoteRepository, RemoteRepositoryAdmin)
+admin.site.register(RemoteRepositoryRelation)
 admin.site.register(RemoteOrganization, RemoteOrganizationAdmin)

readthedocs/oauth/migrations/0011_add_remote_relation_model.py renamed to readthedocs/oauth/migrations/0012_add_remote_relation_model.py

@@ -12,7 +12,7 @@ class Migration(migrations.Migration):
 
     dependencies = [
         migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-        ('oauth', '0010_index_full_name'),
+        ('oauth', '0011_add_default_branch'),
     ]
 
     operations = [
@@ -40,15 +40,15 @@ class Migration(migrations.Migration):
                             'user',
                             models.ForeignKey(
                                 on_delete=django.db.models.deletion.CASCADE,
-                                related_name='remote_relations',
+                                related_name='remote_repository_relations',
                                 to=settings.AUTH_USER_MODEL
                             ),
                         ),
                         (
                             'remoterepository',
                             models.ForeignKey(
                                 on_delete=django.db.models.deletion.CASCADE,
-                                related_name='remote_relations',
+                                related_name='remote_repository_relations',
                                 to='oauth.RemoteRepository'
                             ),
                         ),
@@ -64,10 +64,6 @@ class Migration(migrations.Migration):
                         verbose_name='Users'
                     ),
                 ),
-                migrations.AlterUniqueTogether(
-                    name='remoterepositoryrelation',
-                    unique_together={('remoterepository', 'user')},
-                ),
             ],
         ),
         migrations.AddField(
@@ -76,12 +72,16 @@ class Migration(migrations.Migration):
             field=models.ForeignKey(
                 blank=True,
                 null=True,
-                on_delete=django.db.models.deletion.SET_NULL,
-                related_name='remote_relations',
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name='remote_repository_relations',
                 to='socialaccount.SocialAccount',
                 verbose_name='Connected account'
             ),
         ),
+        migrations.AlterUniqueTogether(
+            name='remoterepositoryrelation',
+            unique_together={('remoterepository', 'account')},
+        ),
         migrations.AddField(
             model_name='remoterepositoryrelation',
             name='admin',

readthedocs/oauth/migrations/0012_data_migration_for_remote_relation_model.py renamed to readthedocs/oauth/migrations/0013_data_migration_for_remote_relation_model.py

@@ -10,10 +10,10 @@
 log = logging.getLogger(__name__)
 
 
-def move_data_to_remote_relations(apps, schema_editor):
+def move_data_to_remote_repository_relations(apps, schema_editor):
     RemoteRepositoryRelation = apps.get_model('oauth', 'RemoteRepositoryRelation')
 
-    def remote_relations_generator(relations, batch_size):
+    def remote_repository_relations_generator(relations, batch_size):
         for relation in relations.iterator(chunk_size=batch_size):
             relation.account_id = relation.remoterepository.account_id
             relation.admin = relation.remoterepository.admin
@@ -42,14 +42,15 @@ def remote_relations_generator(relations, batch_size):
     )
 
     batch_size = 1000
-    remote_relations = remote_relations_generator(
+
+    remote_repository_relations = remote_repository_relations_generator(
         relations_queryset, batch_size
     )
 
     # Follows Example from django docs
     # https://docs.djangoproject.com/en/2.2/ref/models/querysets/#bulk-create
     while True:
-        batch = list(islice(remote_relations, batch_size))
+        batch = list(islice(remote_repository_relations, batch_size))
 
         if not batch:
             break
@@ -68,9 +69,9 @@ def remote_relations_generator(relations, batch_size):
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('oauth', '0011_add_remote_relation_model'),
+        ('oauth', '0012_add_remote_relation_model'),
     ]
 
     operations = [
-        migrations.RunPython(move_data_to_remote_relations),
+        migrations.RunPython(move_data_to_remote_repository_relations),
     ]

readthedocs/oauth/models.py

@@ -180,15 +180,6 @@ class Meta:
     def __str__(self):
         return 'Remote repository: {}'.format(self.html_url)
 
-    def get_serialized(self, key=None, default=None):
-        try:
-            data = json.loads(self.json)
-            if key is not None:
-                return data.get(key, default)
-            return data
-        except ValueError:
-            pass
-
     @property
     def clone_fuzzy_url(self):
         """Try to match against several permutations of project URL."""
@@ -221,24 +212,33 @@ def matches(self, user):
 class RemoteRepositoryRelation(TimeStampedModel):
     remoterepository = models.ForeignKey(
         RemoteRepository,
-        related_name='remote_relations',
+        related_name='remote_repository_relations',
         on_delete=models.CASCADE
     )
     user = models.ForeignKey(
         User,
-        related_name='remote_relations',
+        related_name='remote_repository_relations',
         on_delete=models.CASCADE
     )
     account = models.ForeignKey(
         SocialAccount,
         verbose_name=_('Connected account'),
-        related_name='remote_relations',
+        related_name='remote_repository_relations',
         null=True,
         blank=True,
-        on_delete=models.SET_NULL,
+        on_delete=models.CASCADE,
     )
     admin = models.BooleanField(_('Has admin privilege'), default=False)
     json = JSONField(_('Serialized API response'))
 
     class Meta:
-        unique_together = (('remoterepository', 'user'),)
+        unique_together = (('remoterepository', 'account'),)
+
+    def get_serialized(self, key=None, default=None):
+        try:
+            data = self.json
+            if key is not None:
+                return data.get(key, default)
+            return data
+        except ValueError:
+            pass

readthedocs/oauth/services/base.py

@@ -202,10 +202,8 @@ def sync(self):
         all_remote_repositories = remote_repositories + remote_repositories_organizations
         repository_full_names = [r.full_name for r in all_remote_repositories if r is not None]
         (
-            self.user.oauth_repositories
-            .exclude(
-                Q(full_name__in=repository_full_names) | Q(project__isnull=False)
-            )
+            self.user.remote_repository_relations
+            .exclude(remoterepository__full_name__in=repository_full_names)
             .filter(account=self.account)
             .delete()
         )

readthedocs/oauth/services/bitbucket.py

@@ -14,7 +14,11 @@
 from readthedocs.builds import utils as build_utils
 from readthedocs.integrations.models import Integration
 
-from ..models import RemoteOrganization, RemoteRepository
+from ..models import (
+    RemoteOrganization,
+    RemoteRepository,
+    RemoteRepositoryRelation,
+)
 from .base import Service, SyncServiceError
 
 
@@ -57,16 +61,18 @@ def sync_repositories(self):
             resp = self.paginate(
                 'https://bitbucket.org/api/2.0/repositories/?role=admin',
             )
-            admin_repos = (
-                RemoteRepository.objects.filter(
-                    users=self.user,
-                    full_name__in=[r['full_name'] for r in resp],
+            admin_repo_relations = (
+                RemoteRepositoryRelation.objects.filter(
+                    user=self.user,
                     account=self.account,
+                    remoterepository__full_name__in=[
+                        r['full_name'] for r in resp
+                    ]
                 )
             )
-            for repo in admin_repos:
-                repo.admin = True
-                repo.save()
+            for remote_relation in admin_repo_relations:
+                remote_relation.admin = True
+                remote_relation.save()
         except (TypeError, ValueError):
             pass
 
@@ -120,13 +126,18 @@ def create_repository(self, fields, privacy=None, organization=None):
         """
         privacy = privacy or settings.DEFAULT_PRIVACY_LEVEL
         if any([
-                (privacy == 'private'),
-                (fields['is_private'] is False and privacy == 'public'),
+            (privacy == 'private'),
+            (fields['is_private'] is False and privacy == 'public'),
         ]):
             repo, _ = RemoteRepository.objects.get_or_create(
-                full_name=fields['full_name'],
-                account=self.account,
+                full_name=fields['full_name']
+            )
+            remote_relation, _ = RemoteRepositoryRelation.objects.get_or_create(
+                remoterepository=repo,
+                user=self.user,
+                account=self.account
             )
+
             if repo.organization and repo.organization != organization:
                 log.debug(
                     'Not importing %s because mismatched orgs',
@@ -135,7 +146,6 @@ def create_repository(self, fields, privacy=None, organization=None):
                 return None
 
             repo.organization = organization
-            repo.users.add(self.user)
             repo.name = fields['name']
             repo.description = fields['description']
             repo.private = fields['is_private']
@@ -163,8 +173,12 @@ def create_repository(self, fields, privacy=None, organization=None):
             if not repo.avatar_url:
                 repo.avatar_url = self.default_user_avatar_url
 
-            repo.json = json.dumps(fields)
             repo.save()
+
+            remote_relation.account = self.account
+            remote_relation.json = fields
+            remote_relation.save()
+
             return repo
 
         log.debug(

readthedocs/oauth/services/github.py

@@ -8,7 +8,6 @@
 from allauth.socialaccount.providers.github.views import GitHubOAuth2Adapter
 
 from django.conf import settings
-from django.db.models import Q
 from django.urls import reverse
 from requests.exceptions import RequestException
 
@@ -20,7 +19,11 @@
 )
 from readthedocs.integrations.models import Integration
 
-from ..models import RemoteOrganization, RemoteRepository
+from ..models import (
+    RemoteOrganization,
+    RemoteRepository,
+    RemoteRepositoryRelation,
+)
 from .base import Service, SyncServiceError
 
 log = logging.getLogger(__name__)
@@ -97,21 +100,19 @@ def create_repository(self, fields, privacy=None, organization=None):
         """
         privacy = privacy or settings.DEFAULT_PRIVACY_LEVEL
         if any([
-                (privacy == 'private'),
-                (fields['private'] is False and privacy == 'public'),
+            (privacy == 'private'),
+            (fields['private'] is False and privacy == 'public'),
         ]):
-            try:
-                repo = RemoteRepository.objects.get(
-                    full_name=fields['full_name'],
-                    users=self.user,
-                    account=self.account,
-                )
-            except RemoteRepository.DoesNotExist:
-                repo = RemoteRepository.objects.create(
-                    full_name=fields['full_name'],
-                    account=self.account,
-                )
-                repo.users.add(self.user)
+
+            repo, _ = RemoteRepository.objects.get_or_create(
+                full_name=fields['full_name']
+            )
+            remote_relation, _ = RemoteRepositoryRelation.objects.get_or_create(
+                remoterepository=repo,
+                user=self.user,
+                account=self.account
+            )
+
             if repo.organization and repo.organization != organization:
                 log.debug(
                     'Not importing %s because mismatched orgs',
@@ -125,19 +126,24 @@ def create_repository(self, fields, privacy=None, organization=None):
             repo.ssh_url = fields['ssh_url']
             repo.html_url = fields['html_url']
             repo.private = fields['private']
+            repo.vcs = 'git'
+            repo.avatar_url = fields.get('owner', {}).get('avatar_url')
+            repo.default_branch = fields.get('default_branch')
+
             if repo.private:
                 repo.clone_url = fields['ssh_url']
             else:
                 repo.clone_url = fields['clone_url']
-            repo.admin = fields.get('permissions', {}).get('admin', False)
-            repo.vcs = 'git'
-            repo.default_branch = fields.get('default_branch')
-            repo.account = self.account
-            repo.avatar_url = fields.get('owner', {}).get('avatar_url')
+
             if not repo.avatar_url:
                 repo.avatar_url = self.default_user_avatar_url
-            repo.json = json.dumps(fields)
+
             repo.save()
+
+            remote_relation.json = fields
+            remote_relation.admin = fields.get('permissions', {}).get('admin', False)
+            remote_relation.save()
+
             return repo
 
         log.debug(