Update OAuth App to Use The RemoteRepositoryRelation Model

readthedocs/api/v2/serializers.py

@@ -186,6 +186,7 @@ class RemoteRepositorySerializer(serializers.ModelSerializer):
 
     class Meta:
         model = RemoteRepository
+        # TODO: Remove json field after it is removed from RemoteRepository
         exclude = ('json', 'users')
 
     def get_matches(self, obj):

readthedocs/api/v2/views/model_views.py

@@ -397,10 +397,10 @@ def get_queryset(self):
             )
 
         query = query.filter(
-            account__provider__in=[
+            remote_repository_relations__account__provider__in=[
                 service.adapter.provider_id for service in registry
             ],
-        )
+        ).distinct()
 
         # optimizes for the RemoteOrganizationSerializer
         query = query.select_related('organization')

readthedocs/oauth/admin.py

@@ -4,7 +4,11 @@
 
 from django.contrib import admin
 
-from .models import RemoteOrganization, RemoteRepository
+from .models import (
+    RemoteOrganization,
+    RemoteRepository,
+    RemoteRepositoryRelation,
+)
 
 
 class RemoteRepositoryAdmin(admin.ModelAdmin):
@@ -22,4 +26,5 @@ class RemoteOrganizationAdmin(admin.ModelAdmin):
 
 
 admin.site.register(RemoteRepository, RemoteRepositoryAdmin)
+admin.site.register(RemoteRepositoryRelation)
 admin.site.register(RemoteOrganization, RemoteOrganizationAdmin)

readthedocs/oauth/migrations/0011_add_remote_relation_model.py

@@ -0,0 +1,120 @@
+# Generated by Django 2.2.16 on 2020-10-10 14:55
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+import django_extensions.db.fields
+import jsonfield.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('oauth', '0010_index_full_name'),
+    ]
+
+    operations = [
+        migrations.SeparateDatabaseAndState(
+            database_operations=[
+                migrations.RunSQL(
+                    sql='ALTER TABLE oauth_remoterepository_users RENAME TO oauth_remoterepositoryrelation',
+                    reverse_sql='ALTER TABLE oauth_remoterepositoryrelation RENAME TO oauth_remoterepository_users',
+                ),
+            ],
+            state_operations=[
+                migrations.CreateModel(
+                    name='RemoteRepositoryRelation',
+                    fields=[
+                        (
+                            'id',
+                            models.AutoField(
+                                auto_created=True,
+                                primary_key=True,
+                                serialize=False,
+                                verbose_name='ID',
+                            ),
+                        ),
+                        (
+                            'user',
+                            models.ForeignKey(
+                                on_delete=django.db.models.deletion.CASCADE,
+                                related_name='remote_repository_relations',
+                                to=settings.AUTH_USER_MODEL
+                            ),
+                        ),
+                        (
+                            'remoterepository',
+                            models.ForeignKey(
+                                on_delete=django.db.models.deletion.CASCADE,
+                                related_name='remote_repository_relations',
+                                to='oauth.RemoteRepository'
+                            ),
+                        ),
+                    ],
+                ),
+                migrations.AlterField(
+                    model_name='remoterepository',
+                    name='users',
+                    field=models.ManyToManyField(
+                        related_name='oauth_repositories',
+                        through='oauth.RemoteRepositoryRelation',
+                        to=settings.AUTH_USER_MODEL,
+                        verbose_name='Users'
+                    ),
+                ),
+                migrations.AlterUniqueTogether(
+                    name='remoterepositoryrelation',
+                    unique_together={('remoterepository', 'user')},
+                ),
+            ],
+        ),
+        migrations.AddField(
+            model_name='remoterepositoryrelation',
+            name='account',
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                related_name='remote_repository_relations',
+                to='socialaccount.SocialAccount',
+                verbose_name='Connected account'
+            ),
+        ),
+        migrations.AddField(
+            model_name='remoterepositoryrelation',
+            name='admin',
+            field=models.BooleanField(
+                default=False,
+                verbose_name='Has admin privilege'
+            ),
+        ),
+        migrations.AddField(
+            model_name='remoterepositoryrelation',
+            name='json',
+            field=jsonfield.fields.JSONField(
+                default=dict,
+                verbose_name='Serialized API response'
+            ),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='remoterepositoryrelation',
+            name='created',
+            field=django_extensions.db.fields.CreationDateTimeField(
+                auto_now_add=True,
+                default=django.utils.timezone.now,
+                verbose_name='created',
+            ),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='remoterepositoryrelation',
+            name='modified',
+            field=django_extensions.db.fields.ModificationDateTimeField(
+                auto_now=True,
+                verbose_name='modified'
+            ),
+        ),
+    ]

readthedocs/oauth/migrations/0012_data_migration_for_remote_relation_model.py

@@ -0,0 +1,76 @@
+# Generated by Django 2.2.16 on 2020-10-10 15:00
+
+from itertools import islice
+import json
+import logging
+
+from django.db import migrations
+from django.utils import timezone
+
+log = logging.getLogger(__name__)
+
+
+def move_data_to_remote_repository_relations(apps, schema_editor):
+    RemoteRepositoryRelation = apps.get_model('oauth', 'RemoteRepositoryRelation')
+
+    def remote_repository_relations_generator(relations, batch_size):
+        for relation in relations.iterator(chunk_size=batch_size):
+            relation.account_id = relation.remoterepository.account_id
+            relation.admin = relation.remoterepository.admin
+            relation.created = relation.remoterepository.pub_date
+            relation.modified = relation.remoterepository.modified_date
+
+            try:
+                relation.json = json.loads(relation.remoterepository.json)
+            except json.decoder.JSONDecodeError:
+                log.warning(
+                    'Could not migrate json data for remote_repository=%s',
+                    relation.remoterepository_id
+                )
+
+            yield relation
+
+    relations_queryset = RemoteRepositoryRelation.objects.filter(
+        user__last_login__gte=timezone.now() - timezone.timedelta(days=30)
+    ).select_related(
+        'remoterepository'
+    ).only(
+        'account_id', 'admin', 'created',
+        'modified', 'json', 'remoterepository__account_id',
+        'remoterepository__admin', 'remoterepository__pub_date',
+        'remoterepository__json', 'remoterepository__modified_date'
+    )
+
+    batch_size = 1000
+    remote_repository_relations = remote_repository_relations_generator(
+        relations_queryset, batch_size
+    )
+
+    # Follows Example from django docs
+    # https://docs.djangoproject.com/en/2.2/ref/models/querysets/#bulk-create
+    while True:
+        batch = list(islice(remote_repository_relations, batch_size))
+
+        if not batch:
+            break
+
+        RemoteRepositoryRelation.objects.bulk_update(
+            batch,
+            [
+                'account_id',
+                'admin', 'json',
+                'created', 'modified',
+            ],
+            batch_size
+        )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('oauth', '0011_add_remote_relation_model'),
+    ]
+
+    operations = [
+        migrations.RunPython(move_data_to_remote_repository_relations),
+    ]

readthedocs/oauth/models.py

@@ -4,14 +4,17 @@
 
 import json
 
-from allauth.socialaccount.models import SocialAccount
 from django.contrib.auth.models import User
 from django.core.validators import URLValidator
 from django.db import models
 from django.db.models import Q
 from django.urls import reverse
 from django.utils.translation import ugettext_lazy as _
 
+from allauth.socialaccount.models import SocialAccount
+from django_extensions.db.models import TimeStampedModel
+from jsonfield import JSONField
+
 from readthedocs.projects.constants import REPO_CHOICES
 from readthedocs.projects.models import Project
 
@@ -90,6 +93,7 @@ class RemoteRepository(models.Model):
         User,
         verbose_name=_('Users'),
         related_name='oauth_repositories',
+        through='RemoteRepositoryRelation'
     )
     account = models.ForeignKey(
         SocialAccount,
@@ -170,15 +174,6 @@ class Meta:
     def __str__(self):
         return 'Remote repository: {}'.format(self.html_url)
 
-    def get_serialized(self, key=None, default=None):
-        try:
-            data = json.loads(self.json)
-            if key is not None:
-                return data.get(key, default)
-            return data
-        except ValueError:
-            pass
-
     @property
     def clone_fuzzy_url(self):
         """Try to match against several permutations of project URL."""
@@ -206,3 +201,38 @@ def matches(self, user):
                 },
             ),
         } for project in projects]
+
+
+class RemoteRepositoryRelation(TimeStampedModel):
+    remoterepository = models.ForeignKey(
+        RemoteRepository,
+        related_name='remote_repository_relations',
+        on_delete=models.CASCADE
+    )
+    user = models.ForeignKey(
+        User,
+        related_name='remote_repository_relations',
+        on_delete=models.CASCADE
+    )
+    account = models.ForeignKey(
+        SocialAccount,
+        verbose_name=_('Connected account'),
+        related_name='remote_repository_relations',
+        null=True,
+        blank=True,
+        on_delete=models.SET_NULL,
+    )
+    admin = models.BooleanField(_('Has admin privilege'), default=False)
+    json = JSONField(_('Serialized API response'))
+
+    class Meta:
+        unique_together = (('remoterepository', 'user'),)
+
+    def get_serialized(self, key=None, default=None):
+        try:
+            data = self.json
+            if key is not None:
+                return data.get(key, default)
+            return data
+        except ValueError:
+            pass

readthedocs/oauth/services/base.py

@@ -202,9 +202,10 @@ def sync(self):
         all_remote_repositories = remote_repositories + remote_repositories_organizations
         repository_full_names = [r.full_name for r in all_remote_repositories if r is not None]
         (
-            self.user.oauth_repositories
+            self.user.remote_repository_relations
             .exclude(
-                Q(full_name__in=repository_full_names) | Q(project__isnull=False)
+                Q(remoterepository__full_name__in=repository_full_names) |
+                Q(remoterepository__project__isnull=False)
             )
             .filter(account=self.account)
             .delete()