From 307a1cb029fb3f31ba19b555bc57b4655f6378af Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Mon, 5 Oct 2020 22:49:35 +0600 Subject: [PATCH 1/8] Add Initial Modeling with Through Model and Data Migration for RemoteRepository Model --- .../0011_add_remote_relation_model.py | 48 ++++++++++++++++++ ...012_add_fields_to_remote_relation_model.py | 49 +++++++++++++++++++ ...ata_migration_for_remote_relation_model.py | 49 +++++++++++++++++++ ...move_field_from_remote_repository_model.py | 29 +++++++++++ readthedocs/oauth/models.py | 47 +++++++++++++----- 5 files changed, 209 insertions(+), 13 deletions(-) create mode 100644 readthedocs/oauth/migrations/0011_add_remote_relation_model.py create mode 100644 readthedocs/oauth/migrations/0012_add_fields_to_remote_relation_model.py create mode 100644 readthedocs/oauth/migrations/0013_data_migration_for_remote_relation_model.py create mode 100644 readthedocs/oauth/migrations/0014_remove_field_from_remote_repository_model.py diff --git a/readthedocs/oauth/migrations/0011_add_remote_relation_model.py b/readthedocs/oauth/migrations/0011_add_remote_relation_model.py new file mode 100644 index 00000000000..f2241be8062 --- /dev/null +++ b/readthedocs/oauth/migrations/0011_add_remote_relation_model.py @@ -0,0 +1,48 @@ +# Generated by Django 2.2.16 on 2020-10-05 06:10 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('oauth', '0010_index_full_name'), + ] + + operations = [ + migrations.SeparateDatabaseAndState( + state_operations=[ + migrations.CreateModel( + name='RemoteRelation', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ], + options={ + 'db_table': 'oauth_remoterepository_users', + }, + ), + migrations.AlterField( + model_name='remoterepository', + name='users', + field=models.ManyToManyField(related_name='oauth_repositories', through='oauth.RemoteRelation', to=settings.AUTH_USER_MODEL, verbose_name='Users'), + ), + migrations.AddField( + model_name='remoterelation', + name='remoterepository', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='remote_relations', to='oauth.RemoteRepository'), + ), + migrations.AddField( + model_name='remoterelation', + name='user', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='remote_relations', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterUniqueTogether( + name='remoterelation', + unique_together={('remoterepository', 'user')}, + ), + ] + ) + ] diff --git a/readthedocs/oauth/migrations/0012_add_fields_to_remote_relation_model.py b/readthedocs/oauth/migrations/0012_add_fields_to_remote_relation_model.py new file mode 100644 index 00000000000..144830c3cf4 --- /dev/null +++ b/readthedocs/oauth/migrations/0012_add_fields_to_remote_relation_model.py @@ -0,0 +1,49 @@ +# Generated by Django 2.2.16 on 2020-10-05 06:13 + +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import jsonfield.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('socialaccount', '0003_extra_data_default_dict'), + ('oauth', '0011_add_remote_relation_model'), + ] + + operations = [ + migrations.AddField( + model_name='remoterelation', + name='account', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='remote_relations', to='socialaccount.SocialAccount', verbose_name='Connected account'), + ), + migrations.AddField( + model_name='remoterelation', + name='active', + field=models.BooleanField(default=False, verbose_name='Active'), + ), + migrations.AddField( + model_name='remoterelation', + name='admin', + field=models.BooleanField(default=False, verbose_name='Has admin privilege'), + ), + migrations.AddField( + model_name='remoterelation', + name='json', + field=jsonfield.fields.JSONField(default=dict, verbose_name='Serialized API response'), + preserve_default=False, + ), + migrations.AddField( + model_name='remoterelation', + name='modified_date', + field=models.DateTimeField(auto_now=True, verbose_name='Modified date'), + ), + migrations.AddField( + model_name='remoterelation', + name='pub_date', + field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now, verbose_name='Publication date'), + preserve_default=False, + ), + ] diff --git a/readthedocs/oauth/migrations/0013_data_migration_for_remote_relation_model.py b/readthedocs/oauth/migrations/0013_data_migration_for_remote_relation_model.py new file mode 100644 index 00000000000..193df09338f --- /dev/null +++ b/readthedocs/oauth/migrations/0013_data_migration_for_remote_relation_model.py @@ -0,0 +1,49 @@ +# Generated by Django 2.2.16 on 2020-10-05 06:15 + +import json +from itertools import islice + +from django.db import migrations + + +def move_data_to_remote_relations(apps, schema_editor): + RemoteRelation = apps.get_model('oauth', 'RemoteRelation') + + def remote_relations_generator(relations): + for relation in relations: + relation.account = relation.remoterepository.account + relation.active = relation.remoterepository.active + relation.admin = relation.remoterepository.admin + relation.pub_date = relation.remoterepository.pub_date + try: + relation.json = json.loads(relation.remoterepository.json) + except json.decoder.JSONDecodeError: + pass + + yield relation + + relations_queryset = RemoteRelation.objects.all().select_related('remoterepository') + remote_relations = remote_relations_generator(relations_queryset) + batch_size = 5000 + + while True: + batch = list(islice(remote_relations, batch_size)) + + if not batch: + break + RemoteRelation.objects.bulk_update( + batch, + ['account', 'active', 'admin', 'pub_date', 'json'], + batch_size + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ('oauth', '0012_add_fields_to_remote_relation_model'), + ] + + operations = [ + migrations.RunPython(move_data_to_remote_relations), + ] diff --git a/readthedocs/oauth/migrations/0014_remove_field_from_remote_repository_model.py b/readthedocs/oauth/migrations/0014_remove_field_from_remote_repository_model.py new file mode 100644 index 00000000000..0e9082cbe97 --- /dev/null +++ b/readthedocs/oauth/migrations/0014_remove_field_from_remote_repository_model.py @@ -0,0 +1,29 @@ +# Generated by Django 2.2.16 on 2020-10-05 06:18 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('oauth', '0013_data_migration_for_remote_relation_model'), + ] + + operations = [ + migrations.RemoveField( + model_name='remoterepository', + name='account', + ), + migrations.RemoveField( + model_name='remoterepository', + name='active', + ), + migrations.RemoveField( + model_name='remoterepository', + name='admin', + ), + migrations.RemoveField( + model_name='remoterepository', + name='json', + ), + ] diff --git a/readthedocs/oauth/models.py b/readthedocs/oauth/models.py index 027e7e5d634..72387b3fd79 100644 --- a/readthedocs/oauth/models.py +++ b/readthedocs/oauth/models.py @@ -11,6 +11,7 @@ from django.db.models import Q from django.urls import reverse from django.utils.translation import ugettext_lazy as _ +from jsonfield import JSONField from readthedocs.projects.constants import REPO_CHOICES from readthedocs.projects.models import Project @@ -90,14 +91,7 @@ class RemoteRepository(models.Model): User, verbose_name=_('Users'), related_name='oauth_repositories', - ) - account = models.ForeignKey( - SocialAccount, - verbose_name=_('Connected account'), - related_name='remote_repositories', - null=True, - blank=True, - on_delete=models.CASCADE, + through='RemoteRelation' ) organization = models.ForeignKey( RemoteOrganization, @@ -107,8 +101,6 @@ class RemoteRepository(models.Model): blank=True, on_delete=models.CASCADE, ) - active = models.BooleanField(_('Active'), default=False) - project = models.OneToOneField( Project, on_delete=models.SET_NULL, @@ -151,7 +143,6 @@ class RemoteRepository(models.Model): html_url = models.URLField(_('HTML URL'), null=True, blank=True) private = models.BooleanField(_('Private repository'), default=False) - admin = models.BooleanField(_('Has admin privilege'), default=False) vcs = models.CharField( _('vcs'), max_length=200, @@ -159,8 +150,6 @@ class RemoteRepository(models.Model): choices=REPO_CHOICES, ) - json = models.TextField(_('Serialized API response')) - objects = RemoteRepositoryQuerySet.as_manager() class Meta: @@ -206,3 +195,35 @@ def matches(self, user): }, ), } for project in projects] + + +class RemoteRelation(models.Model): + remoterepository = models.ForeignKey( + RemoteRepository, + related_name='remote_relations', + on_delete=models.CASCADE + ) + user = models.ForeignKey( + User, + related_name='remote_relations', + on_delete=models.CASCADE + ) + account = models.ForeignKey( + SocialAccount, + verbose_name=_('Connected account'), + related_name='remote_relations', + null=True, + blank=True, + on_delete=models.SET_NULL, + ) + active = models.BooleanField(_('Active'), default=False) + admin = models.BooleanField(_('Has admin privilege'), default=False) + json = JSONField(_('Serialized API response')) + + pub_date = models.DateTimeField(_('Publication date'), auto_now_add=True) + modified_date = models.DateTimeField(_('Modified date'), auto_now=True) + + class Meta: + # Use the existing auto generated table for ManyToMany relations + db_table = 'oauth_remoterepository_users' + unique_together = (('remoterepository', 'user'),) From 7bd1a5fea6638d24713d732e53732436f4e6cec8 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 7 Oct 2020 10:50:26 +0600 Subject: [PATCH 2/8] Improve data migration performance --- .../0013_data_migration_for_remote_relation_model.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/readthedocs/oauth/migrations/0013_data_migration_for_remote_relation_model.py b/readthedocs/oauth/migrations/0013_data_migration_for_remote_relation_model.py index 193df09338f..d16bca87924 100644 --- a/readthedocs/oauth/migrations/0013_data_migration_for_remote_relation_model.py +++ b/readthedocs/oauth/migrations/0013_data_migration_for_remote_relation_model.py @@ -9,9 +9,9 @@ def move_data_to_remote_relations(apps, schema_editor): RemoteRelation = apps.get_model('oauth', 'RemoteRelation') - def remote_relations_generator(relations): - for relation in relations: - relation.account = relation.remoterepository.account + def remote_relations_generator(relations, batch_size): + for relation in relations.iterator(chunk_size=batch_size): + relation.account_id = relation.remoterepository.account_id relation.active = relation.remoterepository.active relation.admin = relation.remoterepository.admin relation.pub_date = relation.remoterepository.pub_date @@ -22,9 +22,9 @@ def remote_relations_generator(relations): yield relation - relations_queryset = RemoteRelation.objects.all().select_related('remoterepository') - remote_relations = remote_relations_generator(relations_queryset) batch_size = 5000 + relations_queryset = RemoteRelation.objects.all().select_related('remoterepository') + remote_relations = remote_relations_generator(relations_queryset, batch_size) while True: batch = list(islice(remote_relations, batch_size)) @@ -33,7 +33,7 @@ def remote_relations_generator(relations): break RemoteRelation.objects.bulk_update( batch, - ['account', 'active', 'admin', 'pub_date', 'json'], + ['account_id', 'active', 'admin', 'pub_date', 'json'], batch_size ) From 9476a36db1d182b6860b646a66a5ddaaf310ef75 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Sat, 10 Oct 2020 20:27:50 +0600 Subject: [PATCH 3/8] Logging, performance optimization for data migration --- ...ata_migration_for_remote_relation_model.py | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/readthedocs/oauth/migrations/0013_data_migration_for_remote_relation_model.py b/readthedocs/oauth/migrations/0013_data_migration_for_remote_relation_model.py index d16bca87924..05928a1b36b 100644 --- a/readthedocs/oauth/migrations/0013_data_migration_for_remote_relation_model.py +++ b/readthedocs/oauth/migrations/0013_data_migration_for_remote_relation_model.py @@ -1,11 +1,15 @@ # Generated by Django 2.2.16 on 2020-10-05 06:15 -import json from itertools import islice +import json +import logging from django.db import migrations +log = logging.getLogger(__name__) + + def move_data_to_remote_relations(apps, schema_editor): RemoteRelation = apps.get_model('oauth', 'RemoteRelation') @@ -18,19 +22,35 @@ def remote_relations_generator(relations, batch_size): try: relation.json = json.loads(relation.remoterepository.json) except json.decoder.JSONDecodeError: - pass + log.warning( + 'Could not migrate json data for remote_repository=%s', + relation.remoterepository_id + ) yield relation + relations_queryset = RemoteRelation.objects.all().select_related( + 'remoterepository' + ).only( + 'account_id', 'active', 'admin', + 'pub_date', 'json', 'remoterepository__account_id', + 'remoterepository__active', 'remoterepository__admin', + 'remoterepository__pub_date', 'remoterepository__json' + ) + batch_size = 5000 - relations_queryset = RemoteRelation.objects.all().select_related('remoterepository') - remote_relations = remote_relations_generator(relations_queryset, batch_size) + remote_relations = remote_relations_generator( + relations_queryset, batch_size + ) + # Follows Example from django docs + # https://docs.djangoproject.com/en/2.2/ref/models/querysets/#bulk-create while True: batch = list(islice(remote_relations, batch_size)) if not batch: break + RemoteRelation.objects.bulk_update( batch, ['account_id', 'active', 'admin', 'pub_date', 'json'], From 1bd03c77d4d744b03484979a3b7e751b110622d6 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Sat, 10 Oct 2020 21:45:31 +0600 Subject: [PATCH 4/8] Use TimeStampedModel model and follow django docs for migrating through model --- .../0011_add_remote_relation_model.py | 116 +++++++++++++++--- ...012_add_fields_to_remote_relation_model.py | 49 -------- ...ta_migration_for_remote_relation_model.py} | 21 ++-- ...ove_field_from_remote_repository_model.py} | 4 +- readthedocs/oauth/models.py | 11 +- 5 files changed, 118 insertions(+), 83 deletions(-) delete mode 100644 readthedocs/oauth/migrations/0012_add_fields_to_remote_relation_model.py rename readthedocs/oauth/migrations/{0013_data_migration_for_remote_relation_model.py => 0012_data_migration_for_remote_relation_model.py} (73%) rename readthedocs/oauth/migrations/{0014_remove_field_from_remote_repository_model.py => 0013_remove_field_from_remote_repository_model.py} (83%) diff --git a/readthedocs/oauth/migrations/0011_add_remote_relation_model.py b/readthedocs/oauth/migrations/0011_add_remote_relation_model.py index f2241be8062..562bbb4fc11 100644 --- a/readthedocs/oauth/migrations/0011_add_remote_relation_model.py +++ b/readthedocs/oauth/migrations/0011_add_remote_relation_model.py @@ -1,9 +1,12 @@ -# Generated by Django 2.2.16 on 2020-10-05 06:10 +# Generated by Django 2.2.16 on 2020-10-10 14:55 from django.conf import settings from django.db import migrations, models import django.db.models.deletion +import django_extensions.db.fields +import jsonfield.fields + class Migration(migrations.Migration): @@ -14,35 +17,112 @@ class Migration(migrations.Migration): operations = [ migrations.SeparateDatabaseAndState( + database_operations=[ + migrations.RunSQL( + sql='ALTER TABLE oauth_remoterepository_users RENAME TO oauth_remoterelation', + reverse_sql='ALTER TABLE oauth_remoterelation RENAME TO oauth_remoterepository_users', + ), + ], state_operations=[ migrations.CreateModel( name='RemoteRelation', fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ( + 'id', + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name='ID', + ), + ), + ( + 'user', + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name='remote_relations', + to=settings.AUTH_USER_MODEL + ), + ), + ( + 'remoterepository', + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name='remote_relations', + to='oauth.RemoteRepository' + ), + ), ], - options={ - 'db_table': 'oauth_remoterepository_users', - }, ), migrations.AlterField( model_name='remoterepository', name='users', - field=models.ManyToManyField(related_name='oauth_repositories', through='oauth.RemoteRelation', to=settings.AUTH_USER_MODEL, verbose_name='Users'), - ), - migrations.AddField( - model_name='remoterelation', - name='remoterepository', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='remote_relations', to='oauth.RemoteRepository'), - ), - migrations.AddField( - model_name='remoterelation', - name='user', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='remote_relations', to=settings.AUTH_USER_MODEL), + field=models.ManyToManyField( + related_name='oauth_repositories', + through='oauth.RemoteRelation', + to=settings.AUTH_USER_MODEL, + verbose_name='Users' + ), ), migrations.AlterUniqueTogether( name='remoterelation', unique_together={('remoterepository', 'user')}, ), - ] - ) + ], + ), + migrations.AddField( + model_name='remoterelation', + name='account', + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name='remote_relations', + to='socialaccount.SocialAccount', + verbose_name='Connected account' + ), + ), + migrations.AddField( + model_name='remoterelation', + name='active', + field=models.BooleanField( + default=False, + verbose_name='Active' + ), + ), + migrations.AddField( + model_name='remoterelation', + name='admin', + field=models.BooleanField( + default=False, + verbose_name='Has admin privilege' + ), + ), + migrations.AddField( + model_name='remoterelation', + name='json', + field=jsonfield.fields.JSONField( + default=dict, + verbose_name='Serialized API response' + ), + preserve_default=False, + ), + migrations.AddField( + model_name='remoterelation', + name='created', + field=django_extensions.db.fields.CreationDateTimeField( + auto_now_add=True, + default=django.utils.timezone.now, + verbose_name='created', + ), + preserve_default=False, + ), + migrations.AddField( + model_name='remoterelation', + name='modified', + field=django_extensions.db.fields.ModificationDateTimeField( + auto_now=True, + verbose_name='modified' + ), + ), ] diff --git a/readthedocs/oauth/migrations/0012_add_fields_to_remote_relation_model.py b/readthedocs/oauth/migrations/0012_add_fields_to_remote_relation_model.py deleted file mode 100644 index 144830c3cf4..00000000000 --- a/readthedocs/oauth/migrations/0012_add_fields_to_remote_relation_model.py +++ /dev/null @@ -1,49 +0,0 @@ -# Generated by Django 2.2.16 on 2020-10-05 06:13 - -from django.db import migrations, models -import django.db.models.deletion -import django.utils.timezone -import jsonfield.fields - - -class Migration(migrations.Migration): - - dependencies = [ - ('socialaccount', '0003_extra_data_default_dict'), - ('oauth', '0011_add_remote_relation_model'), - ] - - operations = [ - migrations.AddField( - model_name='remoterelation', - name='account', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='remote_relations', to='socialaccount.SocialAccount', verbose_name='Connected account'), - ), - migrations.AddField( - model_name='remoterelation', - name='active', - field=models.BooleanField(default=False, verbose_name='Active'), - ), - migrations.AddField( - model_name='remoterelation', - name='admin', - field=models.BooleanField(default=False, verbose_name='Has admin privilege'), - ), - migrations.AddField( - model_name='remoterelation', - name='json', - field=jsonfield.fields.JSONField(default=dict, verbose_name='Serialized API response'), - preserve_default=False, - ), - migrations.AddField( - model_name='remoterelation', - name='modified_date', - field=models.DateTimeField(auto_now=True, verbose_name='Modified date'), - ), - migrations.AddField( - model_name='remoterelation', - name='pub_date', - field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now, verbose_name='Publication date'), - preserve_default=False, - ), - ] diff --git a/readthedocs/oauth/migrations/0013_data_migration_for_remote_relation_model.py b/readthedocs/oauth/migrations/0012_data_migration_for_remote_relation_model.py similarity index 73% rename from readthedocs/oauth/migrations/0013_data_migration_for_remote_relation_model.py rename to readthedocs/oauth/migrations/0012_data_migration_for_remote_relation_model.py index 05928a1b36b..c9f3f547171 100644 --- a/readthedocs/oauth/migrations/0013_data_migration_for_remote_relation_model.py +++ b/readthedocs/oauth/migrations/0012_data_migration_for_remote_relation_model.py @@ -1,4 +1,4 @@ -# Generated by Django 2.2.16 on 2020-10-05 06:15 +# Generated by Django 2.2.16 on 2020-10-10 15:00 from itertools import islice import json @@ -18,7 +18,9 @@ def remote_relations_generator(relations, batch_size): relation.account_id = relation.remoterepository.account_id relation.active = relation.remoterepository.active relation.admin = relation.remoterepository.admin - relation.pub_date = relation.remoterepository.pub_date + relation.created = relation.remoterepository.pub_date + relation.modified = relation.remoterepository.modified_date + try: relation.json = json.loads(relation.remoterepository.json) except json.decoder.JSONDecodeError: @@ -32,10 +34,11 @@ def remote_relations_generator(relations, batch_size): relations_queryset = RemoteRelation.objects.all().select_related( 'remoterepository' ).only( - 'account_id', 'active', 'admin', - 'pub_date', 'json', 'remoterepository__account_id', + 'account_id', 'active', 'admin', 'created', + 'modified', 'json', 'remoterepository__account_id', 'remoterepository__active', 'remoterepository__admin', - 'remoterepository__pub_date', 'remoterepository__json' + 'remoterepository__pub_date', 'remoterepository__json', + 'remoterepository__modified_date' ) batch_size = 5000 @@ -53,7 +56,11 @@ def remote_relations_generator(relations, batch_size): RemoteRelation.objects.bulk_update( batch, - ['account_id', 'active', 'admin', 'pub_date', 'json'], + [ + 'account_id', 'active', + 'admin', 'json', + 'created', 'modified', + ], batch_size ) @@ -61,7 +68,7 @@ def remote_relations_generator(relations, batch_size): class Migration(migrations.Migration): dependencies = [ - ('oauth', '0012_add_fields_to_remote_relation_model'), + ('oauth', '0011_add_remote_relation_model'), ] operations = [ diff --git a/readthedocs/oauth/migrations/0014_remove_field_from_remote_repository_model.py b/readthedocs/oauth/migrations/0013_remove_field_from_remote_repository_model.py similarity index 83% rename from readthedocs/oauth/migrations/0014_remove_field_from_remote_repository_model.py rename to readthedocs/oauth/migrations/0013_remove_field_from_remote_repository_model.py index 0e9082cbe97..59c61a717b8 100644 --- a/readthedocs/oauth/migrations/0014_remove_field_from_remote_repository_model.py +++ b/readthedocs/oauth/migrations/0013_remove_field_from_remote_repository_model.py @@ -1,4 +1,4 @@ -# Generated by Django 2.2.16 on 2020-10-05 06:18 +# Generated by Django 2.2.16 on 2020-10-10 15:21 from django.db import migrations @@ -6,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ - ('oauth', '0013_data_migration_for_remote_relation_model'), + ('oauth', '0012_data_migration_for_remote_relation_model'), ] operations = [ diff --git a/readthedocs/oauth/models.py b/readthedocs/oauth/models.py index 72387b3fd79..9c9e72b5827 100644 --- a/readthedocs/oauth/models.py +++ b/readthedocs/oauth/models.py @@ -4,13 +4,15 @@ import json -from allauth.socialaccount.models import SocialAccount from django.contrib.auth.models import User from django.core.validators import URLValidator from django.db import models from django.db.models import Q from django.urls import reverse from django.utils.translation import ugettext_lazy as _ + +from allauth.socialaccount.models import SocialAccount +from django_extensions.db.models import TimeStampedModel from jsonfield import JSONField from readthedocs.projects.constants import REPO_CHOICES @@ -197,7 +199,7 @@ def matches(self, user): } for project in projects] -class RemoteRelation(models.Model): +class RemoteRelation(TimeStampedModel): remoterepository = models.ForeignKey( RemoteRepository, related_name='remote_relations', @@ -220,10 +222,5 @@ class RemoteRelation(models.Model): admin = models.BooleanField(_('Has admin privilege'), default=False) json = JSONField(_('Serialized API response')) - pub_date = models.DateTimeField(_('Publication date'), auto_now_add=True) - modified_date = models.DateTimeField(_('Modified date'), auto_now=True) - class Meta: - # Use the existing auto generated table for ManyToMany relations - db_table = 'oauth_remoterepository_users' unique_together = (('remoterepository', 'user'),) From c3592eb32c0fe3d2a0256bd5a117b377568ba254 Mon Sep 17 00:00:00 2001 From: Maksudul Haque Date: Sat, 14 Nov 2020 14:54:36 +0600 Subject: [PATCH 5/8] Updated data migrations to only migrate RemoteRepositories of recently loggedin users --- ...ata_migration_for_remote_relation_model.py | 8 +++-- ...move_field_from_remote_repository_model.py | 29 ------------------- readthedocs/oauth/models.py | 5 ++++ 3 files changed, 10 insertions(+), 32 deletions(-) delete mode 100644 readthedocs/oauth/migrations/0013_remove_field_from_remote_repository_model.py diff --git a/readthedocs/oauth/migrations/0012_data_migration_for_remote_relation_model.py b/readthedocs/oauth/migrations/0012_data_migration_for_remote_relation_model.py index c9f3f547171..840b02150cc 100644 --- a/readthedocs/oauth/migrations/0012_data_migration_for_remote_relation_model.py +++ b/readthedocs/oauth/migrations/0012_data_migration_for_remote_relation_model.py @@ -5,7 +5,7 @@ import logging from django.db import migrations - +from django.utils import timezone log = logging.getLogger(__name__) @@ -31,7 +31,9 @@ def remote_relations_generator(relations, batch_size): yield relation - relations_queryset = RemoteRelation.objects.all().select_related( + relations_queryset = RemoteRelation.objects.filter( + user__last_login__gte=timezone.now() - timezone.timedelta(days=30) + ).select_related( 'remoterepository' ).only( 'account_id', 'active', 'admin', 'created', @@ -41,7 +43,7 @@ def remote_relations_generator(relations, batch_size): 'remoterepository__modified_date' ) - batch_size = 5000 + batch_size = 1000 remote_relations = remote_relations_generator( relations_queryset, batch_size ) diff --git a/readthedocs/oauth/migrations/0013_remove_field_from_remote_repository_model.py b/readthedocs/oauth/migrations/0013_remove_field_from_remote_repository_model.py deleted file mode 100644 index 59c61a717b8..00000000000 --- a/readthedocs/oauth/migrations/0013_remove_field_from_remote_repository_model.py +++ /dev/null @@ -1,29 +0,0 @@ -# Generated by Django 2.2.16 on 2020-10-10 15:21 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('oauth', '0012_data_migration_for_remote_relation_model'), - ] - - operations = [ - migrations.RemoveField( - model_name='remoterepository', - name='account', - ), - migrations.RemoveField( - model_name='remoterepository', - name='active', - ), - migrations.RemoveField( - model_name='remoterepository', - name='admin', - ), - migrations.RemoveField( - model_name='remoterepository', - name='json', - ), - ] diff --git a/readthedocs/oauth/models.py b/readthedocs/oauth/models.py index 9c9e72b5827..6a8790fe8b6 100644 --- a/readthedocs/oauth/models.py +++ b/readthedocs/oauth/models.py @@ -103,6 +103,8 @@ class RemoteRepository(models.Model): blank=True, on_delete=models.CASCADE, ) + active = models.BooleanField(_('Active'), default=False) + project = models.OneToOneField( Project, on_delete=models.SET_NULL, @@ -145,6 +147,7 @@ class RemoteRepository(models.Model): html_url = models.URLField(_('HTML URL'), null=True, blank=True) private = models.BooleanField(_('Private repository'), default=False) + admin = models.BooleanField(_('Has admin privilege'), default=False) vcs = models.CharField( _('vcs'), max_length=200, @@ -152,6 +155,8 @@ class RemoteRepository(models.Model): choices=REPO_CHOICES, ) + json = models.TextField(_('Serialized API response')) + objects = RemoteRepositoryQuerySet.as_manager() class Meta: From c6699fb25ba9c3cb391adce77e66f2ce237b7a5f Mon Sep 17 00:00:00 2001 From: Maksudul Haque Date: Sat, 14 Nov 2020 15:00:24 +0600 Subject: [PATCH 6/8] Do not remove fields from RemoteRepository Model --- readthedocs/oauth/models.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/readthedocs/oauth/models.py b/readthedocs/oauth/models.py index 6a8790fe8b6..74dae34c0a6 100644 --- a/readthedocs/oauth/models.py +++ b/readthedocs/oauth/models.py @@ -95,6 +95,14 @@ class RemoteRepository(models.Model): related_name='oauth_repositories', through='RemoteRelation' ) + account = models.ForeignKey( + SocialAccount, + verbose_name=_('Connected account'), + related_name='remote_repositories', + null=True, + blank=True, + on_delete=models.CASCADE, + ) organization = models.ForeignKey( RemoteOrganization, verbose_name=_('Organization'), From bf5c6b1213816a572b8e626c7ee95c839fa5c0e8 Mon Sep 17 00:00:00 2001 From: Maksudul Haque Date: Mon, 16 Nov 2020 21:30:09 +0600 Subject: [PATCH 7/8] Do not Migrate Active Field --- .../oauth/migrations/0011_add_remote_relation_model.py | 8 -------- .../0012_data_migration_for_remote_relation_model.py | 10 ++++------ readthedocs/oauth/models.py | 1 - 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/readthedocs/oauth/migrations/0011_add_remote_relation_model.py b/readthedocs/oauth/migrations/0011_add_remote_relation_model.py index 562bbb4fc11..4660e4d4333 100644 --- a/readthedocs/oauth/migrations/0011_add_remote_relation_model.py +++ b/readthedocs/oauth/migrations/0011_add_remote_relation_model.py @@ -82,14 +82,6 @@ class Migration(migrations.Migration): verbose_name='Connected account' ), ), - migrations.AddField( - model_name='remoterelation', - name='active', - field=models.BooleanField( - default=False, - verbose_name='Active' - ), - ), migrations.AddField( model_name='remoterelation', name='admin', diff --git a/readthedocs/oauth/migrations/0012_data_migration_for_remote_relation_model.py b/readthedocs/oauth/migrations/0012_data_migration_for_remote_relation_model.py index 840b02150cc..028aa36654c 100644 --- a/readthedocs/oauth/migrations/0012_data_migration_for_remote_relation_model.py +++ b/readthedocs/oauth/migrations/0012_data_migration_for_remote_relation_model.py @@ -16,7 +16,6 @@ def move_data_to_remote_relations(apps, schema_editor): def remote_relations_generator(relations, batch_size): for relation in relations.iterator(chunk_size=batch_size): relation.account_id = relation.remoterepository.account_id - relation.active = relation.remoterepository.active relation.admin = relation.remoterepository.admin relation.created = relation.remoterepository.pub_date relation.modified = relation.remoterepository.modified_date @@ -36,11 +35,10 @@ def remote_relations_generator(relations, batch_size): ).select_related( 'remoterepository' ).only( - 'account_id', 'active', 'admin', 'created', + 'account_id', 'admin', 'created', 'modified', 'json', 'remoterepository__account_id', - 'remoterepository__active', 'remoterepository__admin', - 'remoterepository__pub_date', 'remoterepository__json', - 'remoterepository__modified_date' + 'remoterepository__admin', 'remoterepository__pub_date', + 'remoterepository__json', 'remoterepository__modified_date' ) batch_size = 1000 @@ -59,7 +57,7 @@ def remote_relations_generator(relations, batch_size): RemoteRelation.objects.bulk_update( batch, [ - 'account_id', 'active', + 'account_id', 'admin', 'json', 'created', 'modified', ], diff --git a/readthedocs/oauth/models.py b/readthedocs/oauth/models.py index 74dae34c0a6..d429eebbf0d 100644 --- a/readthedocs/oauth/models.py +++ b/readthedocs/oauth/models.py @@ -231,7 +231,6 @@ class RemoteRelation(TimeStampedModel): blank=True, on_delete=models.SET_NULL, ) - active = models.BooleanField(_('Active'), default=False) admin = models.BooleanField(_('Has admin privilege'), default=False) json = JSONField(_('Serialized API response')) From d90ecadbee8bf4cea73e3319166841b06253306b Mon Sep 17 00:00:00 2001 From: Maksudul Haque Date: Mon, 16 Nov 2020 22:41:19 +0600 Subject: [PATCH 8/8] Rename RemoteRelation model to RemoteRepositoryRelation --- .../0011_add_remote_relation_model.py | 20 +++++++++---------- ...ata_migration_for_remote_relation_model.py | 6 +++--- readthedocs/oauth/models.py | 4 ++-- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/readthedocs/oauth/migrations/0011_add_remote_relation_model.py b/readthedocs/oauth/migrations/0011_add_remote_relation_model.py index 4660e4d4333..10a4ccd620d 100644 --- a/readthedocs/oauth/migrations/0011_add_remote_relation_model.py +++ b/readthedocs/oauth/migrations/0011_add_remote_relation_model.py @@ -19,13 +19,13 @@ class Migration(migrations.Migration): migrations.SeparateDatabaseAndState( database_operations=[ migrations.RunSQL( - sql='ALTER TABLE oauth_remoterepository_users RENAME TO oauth_remoterelation', - reverse_sql='ALTER TABLE oauth_remoterelation RENAME TO oauth_remoterepository_users', + sql='ALTER TABLE oauth_remoterepository_users RENAME TO oauth_remoterepositoryrelation', + reverse_sql='ALTER TABLE oauth_remoterepositoryrelation RENAME TO oauth_remoterepository_users', ), ], state_operations=[ migrations.CreateModel( - name='RemoteRelation', + name='RemoteRepositoryRelation', fields=[ ( 'id', @@ -59,19 +59,19 @@ class Migration(migrations.Migration): name='users', field=models.ManyToManyField( related_name='oauth_repositories', - through='oauth.RemoteRelation', + through='oauth.RemoteRepositoryRelation', to=settings.AUTH_USER_MODEL, verbose_name='Users' ), ), migrations.AlterUniqueTogether( - name='remoterelation', + name='remoterepositoryrelation', unique_together={('remoterepository', 'user')}, ), ], ), migrations.AddField( - model_name='remoterelation', + model_name='remoterepositoryrelation', name='account', field=models.ForeignKey( blank=True, @@ -83,7 +83,7 @@ class Migration(migrations.Migration): ), ), migrations.AddField( - model_name='remoterelation', + model_name='remoterepositoryrelation', name='admin', field=models.BooleanField( default=False, @@ -91,7 +91,7 @@ class Migration(migrations.Migration): ), ), migrations.AddField( - model_name='remoterelation', + model_name='remoterepositoryrelation', name='json', field=jsonfield.fields.JSONField( default=dict, @@ -100,7 +100,7 @@ class Migration(migrations.Migration): preserve_default=False, ), migrations.AddField( - model_name='remoterelation', + model_name='remoterepositoryrelation', name='created', field=django_extensions.db.fields.CreationDateTimeField( auto_now_add=True, @@ -110,7 +110,7 @@ class Migration(migrations.Migration): preserve_default=False, ), migrations.AddField( - model_name='remoterelation', + model_name='remoterepositoryrelation', name='modified', field=django_extensions.db.fields.ModificationDateTimeField( auto_now=True, diff --git a/readthedocs/oauth/migrations/0012_data_migration_for_remote_relation_model.py b/readthedocs/oauth/migrations/0012_data_migration_for_remote_relation_model.py index 028aa36654c..ba69fca4f58 100644 --- a/readthedocs/oauth/migrations/0012_data_migration_for_remote_relation_model.py +++ b/readthedocs/oauth/migrations/0012_data_migration_for_remote_relation_model.py @@ -11,7 +11,7 @@ def move_data_to_remote_relations(apps, schema_editor): - RemoteRelation = apps.get_model('oauth', 'RemoteRelation') + RemoteRepositoryRelation = apps.get_model('oauth', 'RemoteRepositoryRelation') def remote_relations_generator(relations, batch_size): for relation in relations.iterator(chunk_size=batch_size): @@ -30,7 +30,7 @@ def remote_relations_generator(relations, batch_size): yield relation - relations_queryset = RemoteRelation.objects.filter( + relations_queryset = RemoteRepositoryRelation.objects.filter( user__last_login__gte=timezone.now() - timezone.timedelta(days=30) ).select_related( 'remoterepository' @@ -54,7 +54,7 @@ def remote_relations_generator(relations, batch_size): if not batch: break - RemoteRelation.objects.bulk_update( + RemoteRepositoryRelation.objects.bulk_update( batch, [ 'account_id', diff --git a/readthedocs/oauth/models.py b/readthedocs/oauth/models.py index d429eebbf0d..6f459a87306 100644 --- a/readthedocs/oauth/models.py +++ b/readthedocs/oauth/models.py @@ -93,7 +93,7 @@ class RemoteRepository(models.Model): User, verbose_name=_('Users'), related_name='oauth_repositories', - through='RemoteRelation' + through='RemoteRepositoryRelation' ) account = models.ForeignKey( SocialAccount, @@ -212,7 +212,7 @@ def matches(self, user): } for project in projects] -class RemoteRelation(TimeStampedModel): +class RemoteRepositoryRelation(TimeStampedModel): remoterepository = models.ForeignKey( RemoteRepository, related_name='remote_relations',