diff --git a/readthedocs/api/v2/serializers.py b/readthedocs/api/v2/serializers.py index 5aaca3f42c1..349979179a4 100644 --- a/readthedocs/api/v2/serializers.py +++ b/readthedocs/api/v2/serializers.py @@ -177,7 +177,7 @@ class RemoteOrganizationSerializer(serializers.ModelSerializer): class Meta: model = RemoteOrganization - exclude = ('json', 'email', 'users') + exclude = ('email', 'users',) class RemoteRepositorySerializer(serializers.ModelSerializer): @@ -188,16 +188,30 @@ class RemoteRepositorySerializer(serializers.ModelSerializer): # This field does create an additional query per object returned matches = serializers.SerializerMethodField() + admin = serializers.SerializerMethodField('is_admin') class Meta: model = RemoteRepository - exclude = ('json', 'users') + exclude = ('users',) def get_matches(self, obj): request = self.context['request'] if request.user is not None and request.user.is_authenticated: return obj.matches(request.user) + def is_admin(self, obj): + request = self.context['request'] + + # Use annotated value from RemoteRepositoryViewSet queryset + if hasattr(obj, 'admin'): + return obj.admin + + if request.user and request.user.is_authenticated: + return obj.remote_repository_relations.filter( + user=request.user, admin=True + ).exists() + return False + class ProviderSerializer(serializers.Serializer): diff --git a/readthedocs/api/v2/views/model_views.py b/readthedocs/api/v2/views/model_views.py index a077a60c551..8c3f6bd796d 100644 --- a/readthedocs/api/v2/views/model_views.py +++ b/readthedocs/api/v2/views/model_views.py @@ -5,6 +5,7 @@ from allauth.socialaccount.models import SocialAccount from django.conf import settings +from django.db.models import BooleanField, Case, Value, When from django.shortcuts import get_object_or_404 from django.template.loader import render_to_string from rest_framework import decorators, permissions, status, viewsets @@ -20,7 +21,11 @@ from readthedocs.projects.models import Domain, Project from readthedocs.storage import build_commands_storage -from ..permissions import APIPermission, APIRestrictedPermission, IsOwner +from ..permissions import ( + APIPermission, + APIRestrictedPermission, + IsOwner, +) from ..serializers import ( BuildAdminSerializer, BuildCommandSerializer, @@ -298,10 +303,10 @@ class RemoteOrganizationViewSet(viewsets.ReadOnlyModelViewSet): def get_queryset(self): return ( self.model.objects.api(self.request.user).filter( - account__provider__in=[ + remote_organization_relations__account__provider__in=[ service.adapter.provider_id for service in registry - ], - ) + ] + ).distinct() ) @@ -313,7 +318,21 @@ class RemoteRepositoryViewSet(viewsets.ReadOnlyModelViewSet): pagination_class = RemoteProjectPagination def get_queryset(self): - query = self.model.objects.api(self.request.user) + if not self.request.user.is_authenticated: + return self.model.objects.none() + + # TODO: Optimize this query after deployment + query = self.model.objects.api(self.request.user).annotate( + admin=Case( + When( + remote_repository_relations__user=self.request.user, + remote_repository_relations__admin=True, + then=Value(True) + ), + default=Value(False), + output_field=BooleanField() + ) + ) full_name = self.request.query_params.get('full_name') if full_name is not None: query = query.filter(full_name__icontains=full_name) @@ -324,18 +343,20 @@ def get_queryset(self): own = self.request.query_params.get('own', None) if own is not None: query = query.filter( - account__provider=own, + remote_repository_relations__account__provider=own, organization=None, ) query = query.filter( - account__provider__in=[ + remote_repository_relations__account__provider__in=[ service.adapter.provider_id for service in registry ], - ) + ).distinct() # optimizes for the RemoteOrganizationSerializer - query = query.select_related('organization') + query = query.select_related('organization').order_by( + 'organization__name', 'full_name' + ) return query diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index 7a48ee64db2..dc5e6f13fd2 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -66,7 +66,6 @@ ) from readthedocs.builds.version_slug import VersionSlugField from readthedocs.config import LATEST_CONFIGURATION_VERSION -from readthedocs.core.utils import broadcast from readthedocs.projects.constants import ( BITBUCKET_COMMIT_URL, BITBUCKET_URL, diff --git a/readthedocs/core/signals.py b/readthedocs/core/signals.py index e3ebf23cb65..cce4b64f01b 100644 --- a/readthedocs/core/signals.py +++ b/readthedocs/core/signals.py @@ -86,7 +86,7 @@ def decide_if_cors(sender, request, **kwargs): # pylint: disable=unused-argumen @receiver(pre_delete, sender=settings.AUTH_USER_MODEL) -def delete_projects_and_organizations(sender, instance, *args, **kwargs): +def delete_projects(sender, instance, *args, **kwargs): # Here we count the owner list from the projects that the user own # Then exclude the projects where there are more than one owner # Add annotate before filter @@ -98,16 +98,7 @@ def delete_projects_and_organizations(sender, instance, *args, **kwargs): ).exclude(num_users__gt=1) ) - # Here we count the users list from the organization that the user belong - # Then exclude the organizations where there are more than one user - oauth_organizations = ( - RemoteOrganization.objects.annotate(num_users=Count('users') - ).filter(users=instance.id - ).exclude(num_users__gt=1) - ) - projects.delete() - oauth_organizations.delete() signals.check_request_enabled.connect(decide_if_cors) diff --git a/readthedocs/core/tests/test_signals.py b/readthedocs/core/tests/test_signals.py index 40dccdaa3a7..64d98ae353f 100644 --- a/readthedocs/core/tests/test_signals.py +++ b/readthedocs/core/tests/test_signals.py @@ -1,52 +1,50 @@ # -*- coding: utf-8 -*- import django_dynamic_fixture import pytest + from django.contrib.auth.models import User -from readthedocs.oauth.models import RemoteOrganization from readthedocs.projects.models import Project @pytest.mark.django_db class TestProjectOrganizationSignal: - @pytest.mark.parametrize('model_class', [Project, RemoteOrganization]) - def test_project_organization_get_deleted_upon_user_delete(self, model_class): - """If the user has Project or RemoteOrganization where he is the only - user, upon deleting his account, the Project or RemoteOrganization + def test_project_get_deleted_upon_user_delete(self): + """If the user has Project where he is the only + user, upon deleting his account, the Project should also get deleted.""" - obj = django_dynamic_fixture.get(model_class) + project = django_dynamic_fixture.get(Project) user1 = django_dynamic_fixture.get(User) - obj.users.add(user1) + project.users.add(user1) - obj.refresh_from_db() - assert obj.users.all().count() == 1 + project.refresh_from_db() + assert project.users.all().count() == 1 # Delete the user user1.delete() # The object should not exist - obj = model_class.objects.all().filter(id=obj.id) - assert not obj.exists() + project = Project.objects.all().filter(id=project.id) + assert not project.exists() - @pytest.mark.parametrize('model_class', [Project, RemoteOrganization]) - def test_multiple_users_project_organization_not_delete(self, model_class): - """Check Project or RemoteOrganization which have multiple users do not + def test_multiple_users_project_not_delete(self): + """Check Project which have multiple users do not get deleted when any of the user delete his account.""" - obj = django_dynamic_fixture.get(model_class) + project = django_dynamic_fixture.get(Project) user1 = django_dynamic_fixture.get(User) user2 = django_dynamic_fixture.get(User) - obj.users.add(user1, user2) + project.users.add(user1, user2) - obj.refresh_from_db() - assert obj.users.all().count() > 1 + project.refresh_from_db() + assert project.users.all().count() > 1 # Delete 1 user of the project user1.delete() # The project should still exist and it should have 1 user - obj.refresh_from_db() - assert obj.id - obj_users = obj.users.all() + project.refresh_from_db() + assert project.id + obj_users = project.users.all() assert len(obj_users) == 1 assert user2 in obj_users diff --git a/readthedocs/oauth/admin.py b/readthedocs/oauth/admin.py index eeee459e6ec..d6e43f34740 100644 --- a/readthedocs/oauth/admin.py +++ b/readthedocs/oauth/admin.py @@ -4,22 +4,38 @@ from django.contrib import admin -from .models import RemoteOrganization, RemoteRepository +from .models import ( + RemoteOrganization, + RemoteOrganizationRelation, + RemoteRepository, + RemoteRepositoryRelation, +) class RemoteRepositoryAdmin(admin.ModelAdmin): """Admin configuration for the RemoteRepository model.""" - raw_id_fields = ('users',) + raw_id_fields = ('project', 'organization',) -class RemoteOrganizationAdmin(admin.ModelAdmin): +class RemoteRepositoryRelationAdmin(admin.ModelAdmin): - """Admin configuration for the RemoteOrganization model.""" + """Admin configuration for the RemoteRepositoryRelation model.""" - raw_id_fields = ('users',) + raw_id_fields = ('account', 'remote_repository', 'user',) + list_select_related = ('remote_repository', 'user',) + + +class RemoteOrganizationRelationAdmin(admin.ModelAdmin): + + """Admin configuration for the RemoteOrganizationRelation model.""" + + raw_id_fields = ('account', 'remote_organization', 'user',) + list_select_related = ('remote_organization', 'user',) admin.site.register(RemoteRepository, RemoteRepositoryAdmin) -admin.site.register(RemoteOrganization, RemoteOrganizationAdmin) +admin.site.register(RemoteRepositoryRelation, RemoteRepositoryRelationAdmin) +admin.site.register(RemoteOrganization) +admin.site.register(RemoteOrganizationRelation, RemoteOrganizationRelationAdmin) diff --git a/readthedocs/oauth/constants.py b/readthedocs/oauth/constants.py new file mode 100644 index 00000000000..5496ec0c9bf --- /dev/null +++ b/readthedocs/oauth/constants.py @@ -0,0 +1,9 @@ +GITHUB = 'github' +GITLAB = 'gitlab' +BITBUCKET = 'bitbucket' + +VCS_PROVIDER_CHOICES = ( + (GITHUB, 'GitHub'), + (GITLAB, 'GitLab'), + (BITBUCKET, 'Bitbucket'), +) diff --git a/readthedocs/oauth/management/commands/load_project_remote_repo_relation.py b/readthedocs/oauth/management/commands/load_project_remote_repo_relation.py new file mode 100644 index 00000000000..194cf308a1c --- /dev/null +++ b/readthedocs/oauth/management/commands/load_project_remote_repo_relation.py @@ -0,0 +1,60 @@ +import json + +from django.core.management.base import BaseCommand + +from readthedocs.oauth.models import RemoteRepository + + +class Command(BaseCommand): + help = "Load Project and RemoteRepository Relationship from JSON file" + + def add_arguments(self, parser): + # File path of the json file containing relationship data + parser.add_argument( + '--file', + required=True, + nargs=1, + type=str, + help='File path of the json file containing relationship data.', + ) + + def handle(self, *args, **options): + file = options.get('file')[0] + + try: + # Load data from the json file + with open(file, 'r') as f: + data = json.load(f) + except Exception as e: + self.stdout.write( + self.style.ERROR( + f'Exception occurred while trying to load the file "{file}". ' + f'Exception: {e}.' + ) + ) + return + + for item in data: + try: + update_count = RemoteRepository.objects.filter( + remote_id=item['remote_id'] + ).update(project_id=item['project_id']) + + if update_count < 1: + self.stdout.write( + self.style.ERROR( + f"Could not update {item['slug']}'s " + f"relationship with {item['html_url']}, " + f"remote_id {item['remote_id']}, " + f"username: {item['username']}." + ) + ) + + except Exception as e: + self.stdout.write( + self.style.ERROR( + f"Exception occurred while trying to update {item['slug']}'s " + f"relationship with {item['html_url']}, " + f"username: {item['username']}, Exception: {e}." + ) + ) diff --git a/readthedocs/oauth/management/commands/reconnect_remoterepositories.py b/readthedocs/oauth/management/commands/reconnect_remoterepositories.py index d83efd7b1ac..2ee80cedfd1 100644 --- a/readthedocs/oauth/management/commands/reconnect_remoterepositories.py +++ b/readthedocs/oauth/management/commands/reconnect_remoterepositories.py @@ -55,7 +55,7 @@ def _connect_repositories(self, organization, no_dry_run, only_owners): Q(ssh_url__in=Subquery(organization.projects.values('repo'))) | Q(clone_url__in=Subquery(organization.projects.values('repo'))) ) - for remote in RemoteRepository.objects.filter(remote_query).order_by('pub_date'): + for remote in RemoteRepository.objects.filter(remote_query).order_by('created'): admin = json.loads(remote.json).get('permissions', {}).get('admin') if only_owners and remote.users.first() not in organization.owners.all(): diff --git a/readthedocs/oauth/management/commands/sync_vcs_data.py b/readthedocs/oauth/management/commands/sync_vcs_data.py new file mode 100644 index 00000000000..4cfb36ac1bf --- /dev/null +++ b/readthedocs/oauth/management/commands/sync_vcs_data.py @@ -0,0 +1,143 @@ +import datetime +import json + +from django.utils import timezone +from django.contrib.auth.models import User +from django.core.management.base import BaseCommand + +from readthedocs.oauth.tasks import sync_remote_repositories + + +class Command(BaseCommand): + help = "Sync OAuth RemoteRepository and RemoteOrganization" + + def add_arguments(self, parser): + parser.add_argument( + '--queue', + type=str, + default='resync-oauth', + help='Celery queue name.', + ) + parser.add_argument( + '--users', + nargs='+', + type=str, + default=[], + help='Re-sync VCS provider data for specific users only.', + ) + parser.add_argument( + '--logged-in-days-ago', + type=int, + default=0, + help='Re-sync users logged in in the last days.', + ) + parser.add_argument( + '--skip-revoked-users', + action='store_true', + default=False, + help='Skip users who revoked our access token (pulled down from Sentry).', + ) + parser.add_argument( + '--skip-users', + nargs='+', + type=str, + default=[], + help='Skip re-sync VCS provider data for specific users.', + ) + parser.add_argument( + '--max-users', + type=int, + default=100, + help='Maximum number of users that should be synced.', + ) + parser.add_argument( + '--force', + action='store_true', + default=False, + help='Force re-sync VCS provider data even if the users are already synced.', + ) + parser.add_argument( + '--dry-run', + action='store_true', + default=False, + help='Do not trigger tasks for VCS provider re-sync.', + ) + + def handle(self, *args, **options): + queue = options.get('queue') + logged_in_days_ago = options.get('logged_in_days_ago') + skip_revoked_users = options.get('skip_revoked_users') + sync_users = options.get('users') + skip_users = options.get('skip_users') + max_users = options.get('max_users') + force_sync = options.get('force') + dry_run = options.get('dry_run') + + # Filter users who have social accounts connected to their RTD account + users = User.objects.filter( + socialaccount__isnull=False + ).distinct() + + if logged_in_days_ago > 0: + users = users.filter( + last_login__gte=timezone.now() - datetime.timedelta(days=logged_in_days_ago), + ) + + if not force_sync: + users = users.filter( + remote_repository_relations__isnull=True + ).distinct() + + self.stdout.write( + self.style.SUCCESS( + f'Total {users.count()} user(s) can be synced' + ) + ) + + if sync_users: + users = users.filter(username__in=sync_users) + + if skip_users: + users = users.exclude(username__in=skip_users) + + revoked_users = [] + if skip_revoked_users: + # `revoked-users.json` was created by a script pullig down data from Sentry + # https://gist.github.com/humitos/aba1a004abeb3552fd8ef9a741f5dce1 + revoked_users = json.load(open('revoked-users.json', 'r')) + users = users.exclude(username__in=revoked_users) + self.stdout.write( + self.style.WARNING( + f'Excluding {len(revoked_users)} revoked users.' + ) + ) + + if sync_users or skip_users or revoked_users: + self.stdout.write( + self.style.SUCCESS( + f'Found {users.count()} user(s) with the given parameters' + ) + ) + + # Don't trigger VCS provider re-sync tasks if --dry-run is provided + if dry_run: + self.stdout.write( + self.style.WARNING( + 'No VCS provider re-sync task was triggered. ' + 'Run it without --dry-run to trigger the re-sync tasks.' + ) + ) + else: + users_to_sync = users.values_list('id', flat=True)[:max_users] + + self.stdout.write( + self.style.SUCCESS( + f'Triggering VCS provider re-sync task(s) for {len(users_to_sync)} user(s)' + ) + ) + + for user_id in users_to_sync: + # Trigger Sync Remote Repository Tasks for users + sync_remote_repositories.apply_async( + args=[user_id], queue=queue + ) diff --git a/readthedocs/oauth/migrations/0012_create_new_table_for_remote_organization_normalization.py b/readthedocs/oauth/migrations/0012_create_new_table_for_remote_organization_normalization.py new file mode 100644 index 00000000000..233473fd929 --- /dev/null +++ b/readthedocs/oauth/migrations/0012_create_new_table_for_remote_organization_normalization.py @@ -0,0 +1,60 @@ +# Generated by Django 2.2.17 on 2020-12-23 10:58 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import django_extensions.db.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('socialaccount', '0003_extra_data_default_dict'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('oauth', '0011_add_default_branch'), + ] + + operations = [ + migrations.CreateModel( + name='RemoteOrganization', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')), + ('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')), + ('slug', models.CharField(max_length=255, verbose_name='Slug')), + ('name', models.CharField(blank=True, max_length=255, null=True, verbose_name='Name')), + ('email', models.EmailField(blank=True, max_length=255, null=True, verbose_name='Email')), + ('avatar_url', models.URLField(blank=True, null=True, verbose_name='Avatar image URL')), + ('url', models.URLField(blank=True, null=True, verbose_name='URL to organization page')), + ('remote_id', models.CharField(db_index=True, max_length=128)), + ('vcs_provider', models.CharField(choices=[('github', 'GitHub'), ('gitlab', 'GitLab'), ('bitbucket', 'Bitbucket')], max_length=32, verbose_name='VCS provider')), + ], + options={ + 'db_table': 'oauth_remoteorganization_2020', + 'ordering': ['name'], + }, + ), + migrations.CreateModel( + name='RemoteOrganizationRelation', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')), + ('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='remote_organization_relations', to='socialaccount.SocialAccount', verbose_name='Connected account')), + ('remote_organization', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='remote_organization_relations', to='oauth.RemoteOrganization')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='remote_organization_relations', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'unique_together': {('remote_organization', 'account')}, + }, + ), + migrations.AddField( + model_name='remoteorganization', + name='users', + field=models.ManyToManyField(related_name='oauth_organizations', through='oauth.RemoteOrganizationRelation', to=settings.AUTH_USER_MODEL, verbose_name='Users'), + ), + migrations.AlterUniqueTogether( + name='remoteorganization', + unique_together={('remote_id', 'vcs_provider')}, + ), + ] diff --git a/readthedocs/oauth/migrations/0013_create_new_table_for_remote_repository_normalization.py b/readthedocs/oauth/migrations/0013_create_new_table_for_remote_repository_normalization.py new file mode 100644 index 00000000000..f2dd10d39d7 --- /dev/null +++ b/readthedocs/oauth/migrations/0013_create_new_table_for_remote_repository_normalization.py @@ -0,0 +1,70 @@ +# Generated by Django 2.2.17 on 2020-12-21 18:16 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import django_extensions.db.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('socialaccount', '0003_extra_data_default_dict'), + ('projects', '0067_change_max_length_feature_id'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('oauth', '0012_create_new_table_for_remote_organization_normalization'), + ] + + operations = [ + migrations.CreateModel( + name='RemoteRepository', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')), + ('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')), + ('name', models.CharField(max_length=255, verbose_name='Name')), + ('full_name', models.CharField(db_index=True, max_length=255, verbose_name='Full Name')), + ('description', models.TextField(blank=True, help_text='Description of the project', null=True, verbose_name='Description')), + ('avatar_url', models.URLField(blank=True, null=True, verbose_name='Owner avatar image URL')), + ('ssh_url', models.URLField(blank=True, max_length=512, validators=[django.core.validators.URLValidator(schemes=['ssh'])], verbose_name='SSH URL')), + ('clone_url', models.URLField(blank=True, max_length=512, validators=[django.core.validators.URLValidator(schemes=['http', 'https', 'ssh', 'git', 'svn'])], verbose_name='Repository clone URL')), + ('html_url', models.URLField(blank=True, null=True, verbose_name='HTML URL')), + ('private', models.BooleanField(default=False, verbose_name='Private repository')), + ('vcs', models.CharField(blank=True, choices=[('git', 'Git'), ('svn', 'Subversion'), ('hg', 'Mercurial'), ('bzr', 'Bazaar')], max_length=200, verbose_name='vcs')), + ('default_branch', models.CharField(blank=True, max_length=150, null=True, verbose_name='Default branch of the repository')), + ('remote_id', models.CharField(db_index=True, max_length=128)), + ('vcs_provider', models.CharField(choices=[('github', 'GitHub'), ('gitlab', 'GitLab'), ('bitbucket', 'Bitbucket')], max_length=32, verbose_name='VCS provider')), + ('organization', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='repositories', to='oauth.RemoteOrganization', verbose_name='Organization')), + ('project', models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='remote_repository', to='projects.Project')), + ], + options={ + 'verbose_name_plural': 'remote repositories', + 'db_table': 'oauth_remoterepository_2020', + 'ordering': ['full_name'], + }, + ), + migrations.CreateModel( + name='RemoteRepositoryRelation', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')), + ('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')), + ('admin', models.BooleanField(default=False, verbose_name='Has admin privilege')), + ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='remote_repository_relations', to='socialaccount.SocialAccount', verbose_name='Connected account')), + ('remote_repository', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='remote_repository_relations', to='oauth.RemoteRepository')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='remote_repository_relations', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'unique_together': {('remote_repository', 'account')}, + }, + ), + migrations.AddField( + model_name='remoterepository', + name='users', + field=models.ManyToManyField(related_name='oauth_repositories', through='oauth.RemoteRepositoryRelation', to=settings.AUTH_USER_MODEL, verbose_name='Users'), + ), + migrations.AlterUniqueTogether( + name='remoterepository', + unique_together={('remote_id', 'vcs_provider')}, + ), + ] diff --git a/readthedocs/oauth/models.py b/readthedocs/oauth/models.py index 2e2374944cf..4deb794baeb 100644 --- a/readthedocs/oauth/models.py +++ b/readthedocs/oauth/models.py @@ -2,9 +2,6 @@ """OAuth service models.""" -import json - -from allauth.socialaccount.models import SocialAccount from django.contrib.auth.models import User from django.core.validators import URLValidator from django.db import models @@ -12,13 +9,17 @@ from django.urls import reverse from django.utils.translation import ugettext_lazy as _ +from allauth.socialaccount.models import SocialAccount +from django_extensions.db.models import TimeStampedModel + from readthedocs.projects.constants import REPO_CHOICES from readthedocs.projects.models import Project +from .constants import VCS_PROVIDER_CHOICES from .querysets import RemoteOrganizationQuerySet, RemoteRepositoryQuerySet -class RemoteOrganization(models.Model): +class RemoteOrganization(TimeStampedModel): """ Organization from remote service. @@ -26,25 +27,12 @@ class RemoteOrganization(models.Model): This encapsulates both Github and Bitbucket """ - # Auto fields - pub_date = models.DateTimeField(_('Publication date'), auto_now_add=True) - modified_date = models.DateTimeField(_('Modified date'), auto_now=True) - users = models.ManyToManyField( User, verbose_name=_('Users'), related_name='oauth_organizations', + through='RemoteOrganizationRelation' ) - account = models.ForeignKey( - SocialAccount, - verbose_name=_('Connected account'), - related_name='remote_organizations', - null=True, - blank=True, - on_delete=models.CASCADE, - ) - active = models.BooleanField(_('Active'), default=False) - slug = models.CharField(_('Slug'), max_length=255) name = models.CharField(_('Name'), max_length=255, null=True, blank=True) email = models.EmailField(_('Email'), max_length=255, null=True, blank=True) @@ -55,25 +43,65 @@ class RemoteOrganization(models.Model): null=True, blank=True, ) - - json = models.TextField(_('Serialized API response')) + # VCS provider organization id + remote_id = models.CharField( + db_index=True, + max_length=128 + ) + vcs_provider = models.CharField( + _('VCS provider'), + choices=VCS_PROVIDER_CHOICES, + max_length=32 + ) objects = RemoteOrganizationQuerySet.as_manager() + class Meta: + ordering = ['name'] + unique_together = ('remote_id', 'vcs_provider',) + db_table = 'oauth_remoteorganization_2020' + def __str__(self): return 'Remote organization: {name}'.format(name=self.slug) - def get_serialized(self, key=None, default=None): - try: - data = json.loads(self.json) - if key is not None: - return data.get(key, default) - return data - except ValueError: - pass + def get_remote_organization_relation(self, user, social_account): + """Return RemoteOrganizationRelation object for the remote organization.""" + remote_organization_relation, _ = ( + RemoteOrganizationRelation.objects.get_or_create( + remote_organization=self, + user=user, + account=social_account + ) + ) + return remote_organization_relation -class RemoteRepository(models.Model): +class RemoteOrganizationRelation(TimeStampedModel): + remote_organization = models.ForeignKey( + RemoteOrganization, + related_name='remote_organization_relations', + on_delete=models.CASCADE + ) + user = models.ForeignKey( + User, + related_name='remote_organization_relations', + on_delete=models.CASCADE + ) + account = models.ForeignKey( + SocialAccount, + verbose_name=_('Connected account'), + related_name='remote_organization_relations', + on_delete=models.CASCADE + ) + + class Meta: + unique_together = ('remote_organization', 'account',) + + def __str__(self): + return f'{self.user.username} <-> {self.remote_organization.name}' + + +class RemoteRepository(TimeStampedModel): """ Remote importable repositories. @@ -81,23 +109,12 @@ class RemoteRepository(models.Model): This models Github and Bitbucket importable repositories """ - # Auto fields - pub_date = models.DateTimeField(_('Publication date'), auto_now_add=True) - modified_date = models.DateTimeField(_('Modified date'), auto_now=True) - # This should now be a OneToOne users = models.ManyToManyField( User, verbose_name=_('Users'), related_name='oauth_repositories', - ) - account = models.ForeignKey( - SocialAccount, - verbose_name=_('Connected account'), - related_name='remote_repositories', - null=True, - blank=True, - on_delete=models.CASCADE, + through='RemoteRepositoryRelation' ) organization = models.ForeignKey( RemoteOrganization, @@ -107,8 +124,6 @@ class RemoteRepository(models.Model): blank=True, on_delete=models.CASCADE, ) - active = models.BooleanField(_('Active'), default=False) - project = models.OneToOneField( Project, on_delete=models.SET_NULL, @@ -151,7 +166,6 @@ class RemoteRepository(models.Model): html_url = models.URLField(_('HTML URL'), null=True, blank=True) private = models.BooleanField(_('Private repository'), default=False) - admin = models.BooleanField(_('Has admin privilege'), default=False) vcs = models.CharField( _('vcs'), max_length=200, @@ -164,27 +178,28 @@ class RemoteRepository(models.Model): null=True, blank=True, ) - - json = models.TextField(_('Serialized API response')) + # VCS provider repository id + remote_id = models.CharField( + db_index=True, + max_length=128 + ) + vcs_provider = models.CharField( + _('VCS provider'), + choices=VCS_PROVIDER_CHOICES, + max_length=32 + ) objects = RemoteRepositoryQuerySet.as_manager() class Meta: - ordering = ['organization__name', 'name'] + ordering = ['full_name'] verbose_name_plural = 'remote repositories' + unique_together = ('remote_id', 'vcs_provider',) + db_table = 'oauth_remoterepository_2020' def __str__(self): return 'Remote repository: {}'.format(self.html_url) - def get_serialized(self, key=None, default=None): - try: - data = json.loads(self.json) - if key is not None: - return data.get(key, default) - return data - except ValueError: - pass - @property def clone_fuzzy_url(self): """Try to match against several permutations of project URL.""" @@ -212,3 +227,40 @@ def matches(self, user): }, ), } for project in projects] + + def get_remote_repository_relation(self, user, social_account): + """Return RemoteRepositoryRelation object for the remote repository.""" + remote_repository_relation, _ = ( + RemoteRepositoryRelation.objects.get_or_create( + remote_repository=self, + user=user, + account=social_account + ) + ) + return remote_repository_relation + + +class RemoteRepositoryRelation(TimeStampedModel): + remote_repository = models.ForeignKey( + RemoteRepository, + related_name='remote_repository_relations', + on_delete=models.CASCADE + ) + user = models.ForeignKey( + User, + related_name='remote_repository_relations', + on_delete=models.CASCADE + ) + account = models.ForeignKey( + SocialAccount, + verbose_name=_('Connected account'), + related_name='remote_repository_relations', + on_delete=models.CASCADE + ) + admin = models.BooleanField(_('Has admin privilege'), default=False) + + class Meta: + unique_together = ('remote_repository', 'account',) + + def __str__(self): + return f'{self.user.username} <-> {self.remote_repository.full_name}' diff --git a/readthedocs/oauth/services/base.py b/readthedocs/oauth/services/base.py index d9d5237b990..6ff606b0fae 100644 --- a/readthedocs/oauth/services/base.py +++ b/readthedocs/oauth/services/base.py @@ -6,12 +6,16 @@ from allauth.socialaccount.models import SocialAccount from allauth.socialaccount.providers import registry from django.conf import settings -from django.db.models import Q from django.utils import timezone from oauthlib.oauth2.rfc6749.errors import InvalidClientIdError from requests.exceptions import RequestException from requests_oauthlib import OAuth2Session +from readthedocs.oauth.models import ( + RemoteOrganizationRelation, + RemoteRepositoryRelation, +) + log = logging.getLogger(__name__) @@ -34,6 +38,7 @@ class Service: adapter = None url_pattern = None + vcs_provider_slug = None default_user_avatar_url = settings.OAUTH_AVATAR_USER_DEFAULT_URL default_org_avatar_url = settings.OAUTH_AVATAR_ORG_DEFAULT_URL @@ -200,21 +205,25 @@ def sync(self): # Delete RemoteRepository where the user doesn't have access anymore # (skip RemoteRepository tied to a Project on this user) all_remote_repositories = remote_repositories + remote_repositories_organizations - repository_full_names = [r.full_name for r in all_remote_repositories if r is not None] + repository_remote_ids = [r.remote_id for r in all_remote_repositories if r is not None] ( - self.user.oauth_repositories + self.user.remote_repository_relations .exclude( - Q(full_name__in=repository_full_names) | Q(project__isnull=False) + remote_repository__remote_id__in=repository_remote_ids, + remote_repository__vcs_provider=self.vcs_provider_slug ) .filter(account=self.account) .delete() ) # Delete RemoteOrganization where the user doesn't have access anymore - organization_slugs = [o.slug for o in remote_organizations if o is not None] + organization_remote_ids = [o.remote_id for o in remote_organizations if o is not None] ( - self.user.oauth_organizations - .exclude(slug__in=organization_slugs) + self.user.remote_organization_relations + .exclude( + remote_organization__remote_id__in=organization_remote_ids, + remote_organization__vcs_provider=self.vcs_provider_slug + ) .filter(account=self.account) .delete() ) diff --git a/readthedocs/oauth/services/bitbucket.py b/readthedocs/oauth/services/bitbucket.py index 5edc0af4abd..119f386fc29 100644 --- a/readthedocs/oauth/services/bitbucket.py +++ b/readthedocs/oauth/services/bitbucket.py @@ -14,7 +14,12 @@ from readthedocs.builds import utils as build_utils from readthedocs.integrations.models import Integration -from ..models import RemoteOrganization, RemoteRepository +from ..constants import BITBUCKET +from ..models import ( + RemoteOrganization, + RemoteRepository, + RemoteRepositoryRelation, +) from .base import Service, SyncServiceError @@ -29,6 +34,7 @@ class BitbucketService(Service): # TODO replace this with a less naive check url_pattern = re.compile(r'bitbucket.org') https_url_pattern = re.compile(r'^https:\/\/[^@]+@bitbucket.org/') + vcs_provider_slug = BITBUCKET def sync_repositories(self): """Sync repositories from Bitbucket API.""" @@ -57,16 +63,19 @@ def sync_repositories(self): resp = self.paginate( 'https://bitbucket.org/api/2.0/repositories/?role=admin', ) - admin_repos = ( - RemoteRepository.objects.filter( - users=self.user, - full_name__in=[r['full_name'] for r in resp], + admin_repo_relations = ( + RemoteRepositoryRelation.objects.filter( + user=self.user, account=self.account, + remote_repository__vcs_provider=self.vcs_provider_slug, + remote_repository__remote_id__in=[ + r['uuid'] for r in resp + ] ) ) - for repo in admin_repos: - repo.admin = True - repo.save() + for remote_repository_relation in admin_repo_relations: + remote_repository_relation.admin = True + remote_repository_relation.save() except (TypeError, ValueError): pass @@ -120,13 +129,17 @@ def create_repository(self, fields, privacy=None, organization=None): """ privacy = privacy or settings.DEFAULT_PRIVACY_LEVEL if any([ - (privacy == 'private'), - (fields['is_private'] is False and privacy == 'public'), + (privacy == 'private'), + (fields['is_private'] is False and privacy == 'public'), ]): repo, _ = RemoteRepository.objects.get_or_create( - full_name=fields['full_name'], - account=self.account, + remote_id=fields['uuid'], + vcs_provider=self.vcs_provider_slug + ) + repo.get_remote_repository_relation( + self.user, self.account ) + if repo.organization and repo.organization != organization: log.debug( 'Not importing %s because mismatched orgs', @@ -135,8 +148,8 @@ def create_repository(self, fields, privacy=None, organization=None): return None repo.organization = organization - repo.users.add(self.user) repo.name = fields['name'] + repo.full_name = fields['full_name'] repo.description = fields['description'] repo.private = fields['is_private'] @@ -157,15 +170,14 @@ def create_repository(self, fields, privacy=None, organization=None): repo.vcs = fields['scm'] mainbranch = fields.get('mainbranch') or {} repo.default_branch = mainbranch.get('name') - repo.account = self.account avatar_url = fields['links']['avatar']['href'] or '' repo.avatar_url = re.sub(r'\/16\/$', r'/32/', avatar_url) if not repo.avatar_url: repo.avatar_url = self.default_user_avatar_url - repo.json = json.dumps(fields) repo.save() + return repo log.debug( @@ -181,19 +193,25 @@ def create_organization(self, fields): :rtype: RemoteOrganization """ organization, _ = RemoteOrganization.objects.get_or_create( - slug=fields.get('username'), - account=self.account, + remote_id=fields['uuid'], + vcs_provider=self.vcs_provider_slug + ) + organization.get_remote_organization_relation( + self.user, self.account ) + + organization.slug = fields.get('username') organization.name = fields.get('display_name') organization.email = fields.get('email') organization.avatar_url = fields['links']['avatar']['href'] + if not organization.avatar_url: organization.avatar_url = self.default_org_avatar_url + organization.url = fields['links']['html']['href'] - organization.json = json.dumps(fields) - organization.account = self.account - organization.users.add(self.user) + organization.save() + return organization def get_next_url_to_paginate(self, response): diff --git a/readthedocs/oauth/services/github.py b/readthedocs/oauth/services/github.py index ee8c7e78543..33a15c6bea8 100644 --- a/readthedocs/oauth/services/github.py +++ b/readthedocs/oauth/services/github.py @@ -8,7 +8,6 @@ from allauth.socialaccount.providers.github.views import GitHubOAuth2Adapter from django.conf import settings -from django.db.models import Q from django.urls import reverse from requests.exceptions import RequestException @@ -20,7 +19,11 @@ ) from readthedocs.integrations.models import Integration -from ..models import RemoteOrganization, RemoteRepository +from ..constants import GITHUB +from ..models import ( + RemoteOrganization, + RemoteRepository, +) from .base import Service, SyncServiceError log = logging.getLogger(__name__) @@ -33,6 +36,7 @@ class GitHubService(Service): adapter = GitHubOAuth2Adapter # TODO replace this with a less naive check url_pattern = re.compile(r'github\.com') + vcs_provider_slug = GITHUB def sync_repositories(self): """Sync repositories from GitHub API.""" @@ -97,21 +101,18 @@ def create_repository(self, fields, privacy=None, organization=None): """ privacy = privacy or settings.DEFAULT_PRIVACY_LEVEL if any([ - (privacy == 'private'), - (fields['private'] is False and privacy == 'public'), + (privacy == 'private'), + (fields['private'] is False and privacy == 'public'), ]): - try: - repo = RemoteRepository.objects.get( - full_name=fields['full_name'], - users=self.user, - account=self.account, - ) - except RemoteRepository.DoesNotExist: - repo = RemoteRepository.objects.create( - full_name=fields['full_name'], - account=self.account, - ) - repo.users.add(self.user) + + repo, _ = RemoteRepository.objects.get_or_create( + remote_id=fields['id'], + vcs_provider=self.vcs_provider_slug + ) + remote_repository_relation = repo.get_remote_repository_relation( + self.user, self.account + ) + if repo.organization and repo.organization != organization: log.debug( 'Not importing %s because mismatched orgs', @@ -121,23 +122,28 @@ def create_repository(self, fields, privacy=None, organization=None): repo.organization = organization repo.name = fields['name'] + repo.full_name = fields['full_name'] repo.description = fields['description'] repo.ssh_url = fields['ssh_url'] repo.html_url = fields['html_url'] repo.private = fields['private'] + repo.vcs = 'git' + repo.avatar_url = fields.get('owner', {}).get('avatar_url') + repo.default_branch = fields.get('default_branch') + if repo.private: repo.clone_url = fields['ssh_url'] else: repo.clone_url = fields['clone_url'] - repo.admin = fields.get('permissions', {}).get('admin', False) - repo.vcs = 'git' - repo.default_branch = fields.get('default_branch') - repo.account = self.account - repo.avatar_url = fields.get('owner', {}).get('avatar_url') + if not repo.avatar_url: repo.avatar_url = self.default_user_avatar_url - repo.json = json.dumps(fields) + repo.save() + + remote_repository_relation.admin = fields.get('permissions', {}).get('admin', False) + remote_repository_relation.save() + return repo log.debug( @@ -152,27 +158,26 @@ def create_organization(self, fields): :param fields: dictionary response of data from API :rtype: RemoteOrganization """ - try: - organization = RemoteOrganization.objects.get( - slug=fields.get('login'), - users=self.user, - account=self.account, - ) - except RemoteOrganization.DoesNotExist: - organization = RemoteOrganization.objects.create( - slug=fields.get('login'), - account=self.account, - ) - organization.users.add(self.user) + organization, _ = RemoteOrganization.objects.get_or_create( + remote_id=fields['id'], + vcs_provider=self.vcs_provider_slug + ) + organization.get_remote_organization_relation( + self.user, self.account + ) + organization.url = fields.get('html_url') + # fields['login'] contains GitHub Organization slug + organization.slug = fields.get('login') organization.name = fields.get('name') organization.email = fields.get('email') organization.avatar_url = fields.get('avatar_url') + if not organization.avatar_url: organization.avatar_url = self.default_org_avatar_url - organization.json = json.dumps(fields) - organization.account = self.account + organization.save() + return organization def get_next_url_to_paginate(self, response): diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index 68a0b42953c..6213c876968 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -3,7 +3,7 @@ import json import logging import re -from urllib.parse import urljoin, urlparse +from urllib.parse import urlparse from allauth.socialaccount.providers.gitlab.views import GitLabOAuth2Adapter from django.conf import settings @@ -18,7 +18,11 @@ from readthedocs.integrations.models import Integration from readthedocs.projects.models import Project -from ..models import RemoteOrganization, RemoteRepository +from ..constants import GITLAB +from ..models import ( + RemoteOrganization, + RemoteRepository, +) from .base import Service, SyncServiceError log = logging.getLogger(__name__) @@ -45,12 +49,14 @@ class GitLabService(Service): PERMISSION_MAINTAINER = 40 PERMISSION_OWNER = 50 + vcs_provider_slug = GITLAB + def _get_repo_id(self, project): # The ID or URL-encoded path of the project # https://docs.gitlab.com/ce/api/README.html#namespaced-path-encoding try: - repo_id = json.loads(project.remote_repository.json).get('id') - except Exception: + repo_id = project.remote_repository.remote_id + except Project.remote_repository.RelatedObjectDoesNotExist: # Handle "Manual Import" when there is no RemoteRepository # associated with the project. It only works with gitlab.com at the # moment (doesn't support custom gitlab installations) @@ -188,18 +194,13 @@ def create_repository(self, fields, privacy=None, organization=None): privacy = privacy or settings.DEFAULT_PRIVACY_LEVEL repo_is_public = fields['visibility'] == 'public' if privacy == 'private' or (repo_is_public and privacy == 'public'): - try: - repo = RemoteRepository.objects.get( - full_name=fields['name_with_namespace'], - users=self.user, - account=self.account, - ) - except RemoteRepository.DoesNotExist: - repo = RemoteRepository.objects.create( - full_name=fields['name_with_namespace'], - account=self.account, - ) - repo.users.add(self.user) + repo, _ = RemoteRepository.objects.get_or_create( + remote_id=fields['id'], + vcs_provider=self.vcs_provider_slug + ) + remote_repository_relation = repo.get_remote_repository_relation( + self.user, self.account + ) if repo.organization and repo.organization != organization: log.debug( @@ -210,45 +211,50 @@ def create_repository(self, fields, privacy=None, organization=None): repo.organization = organization repo.name = fields['name'] + repo.full_name = fields['path_with_namespace'] repo.description = fields['description'] repo.ssh_url = fields['ssh_url_to_repo'] repo.html_url = fields['web_url'] + repo.vcs = 'git' repo.private = not repo_is_public + repo.default_branch = fields.get('default_branch') + + owner = fields.get('owner') or {} + repo.avatar_url = ( + fields.get('avatar_url') or owner.get('avatar_url') + ) + + if not repo.avatar_url: + repo.avatar_url = self.default_user_avatar_url + if repo.private: repo.clone_url = repo.ssh_url else: repo.clone_url = fields['http_url_to_repo'] + repo.save() + project_access_level = group_access_level = self.PERMISSION_NO_ACCESS + project_access = fields.get('permissions', {}).get('project_access', {}) if project_access: project_access_level = project_access.get('access_level', self.PERMISSION_NO_ACCESS) + group_access = fields.get('permissions', {}).get('group_access', {}) if group_access: group_access_level = group_access.get('access_level', self.PERMISSION_NO_ACCESS) - repo.admin = any([ + + remote_repository_relation.admin = any([ project_access_level in (self.PERMISSION_MAINTAINER, self.PERMISSION_OWNER), group_access_level in (self.PERMISSION_MAINTAINER, self.PERMISSION_OWNER), ]) + remote_repository_relation.save() - repo.vcs = 'git' - repo.default_branch = fields.get('default_branch') - repo.account = self.account - - owner = fields.get('owner') or {} - repo.avatar_url = ( - fields.get('avatar_url') or owner.get('avatar_url') - ) - if not repo.avatar_url: - repo.avatar_url = self.default_user_avatar_url - - repo.json = json.dumps(fields) - repo.save() return repo log.info( 'Not importing %s because mismatched type: visibility=%s', - fields['name_with_namespace'], + fields['path_with_namespace'], fields['visibility'], ) @@ -259,30 +265,27 @@ def create_organization(self, fields): :param fields: dictionary response of data from API :rtype: RemoteOrganization """ - try: - organization = RemoteOrganization.objects.get( - slug=fields.get('path'), - users=self.user, - account=self.account, - ) - except RemoteOrganization.DoesNotExist: - organization = RemoteOrganization.objects.create( - slug=fields.get('path'), - account=self.account, - ) - organization.users.add(self.user) + organization, _ = RemoteOrganization.objects.get_or_create( + remote_id=fields['id'], + vcs_provider=self.vcs_provider_slug + ) + organization.get_remote_organization_relation( + self.user, self.account + ) organization.name = fields.get('name') - organization.account = self.account + organization.slug = fields.get('path') organization.url = '{url}/{path}'.format( url=self.adapter.provider_base_url, path=fields.get('path'), ) organization.avatar_url = fields.get('avatar_url') + if not organization.avatar_url: organization.avatar_url = self.default_user_avatar_url - organization.json = json.dumps(fields) + organization.save() + return organization def get_webhook_data(self, repo_id, project, integration): diff --git a/readthedocs/oauth/utils.py b/readthedocs/oauth/utils.py index babb766488b..8a3029e8c59 100644 --- a/readthedocs/oauth/utils.py +++ b/readthedocs/oauth/utils.py @@ -31,9 +31,20 @@ def update_webhook(project, integration, request=None): updated = False try: - account = project.remote_repository.account - service = service_cls(request.user, account) - updated, __ = service.update_webhook(project, integration) + remote_repository_relations = ( + project.remote_repository.remote_repository_relations.filter( + account__isnull=False, + user=request.user + ).select_related('account') + ) + + for relation in remote_repository_relations: + service = service_cls(request.user, relation.account) + updated, __ = service.update_webhook(project, integration) + + if updated: + break + except Project.remote_repository.RelatedObjectDoesNotExist: # The project was imported manually and doesn't have a RemoteRepository # attached. We do brute force over all the accounts registered for this diff --git a/readthedocs/projects/static-src/projects/js/import.js b/readthedocs/projects/static-src/projects/js/import.js index 1307f713421..fe47a956169 100644 --- a/readthedocs/projects/static-src/projects/js/import.js +++ b/readthedocs/projects/static-src/projects/js/import.js @@ -119,7 +119,6 @@ function Project(instance, view) { } }); self.private = ko.observable(instance.private); - self.active = ko.observable(instance.active); self.admin = ko.observable(instance.admin); self.is_locked = ko.computed(function () { if (view.has_sso_enabled) { diff --git a/readthedocs/projects/static/projects/js/import.js b/readthedocs/projects/static/projects/js/import.js index 5ec0acd7ca5..5766ae61731 100644 --- a/readthedocs/projects/static/projects/js/import.js +++ b/readthedocs/projects/static/projects/js/import.js @@ -1 +1 @@ -require=function o(s,i,u){function l(r,e){if(!i[r]){if(!s[r]){var t="function"==typeof require&&require;if(!e&&t)return t(r,!0);if(c)return c(r,!0);var n=new Error("Cannot find module '"+r+"'");throw n.code="MODULE_NOT_FOUND",n}var a=i[r]={exports:{}};s[r][0].call(a.exports,function(e){return l(s[r][1][e]||e)},a,a.exports,o,s,i,u)}return i[r].exports}for(var c="function"==typeof require&&require,e=0;e").attr("href",e).get(0);return Object.keys(r).map(function(e){t.search&&(t.search+="&"),t.search+=e+"="+r[e]}),t.href}function c(e,r){var t=this;t.id=i.observable(e.id),t.name=i.observable(e.name),t.slug=i.observable(e.slug),t.active=i.observable(e.active),t.avatar_url=i.observable(l(e.avatar_url,{size:32})),t.display_name=i.computed(function(){return t.name()||t.slug()}),t.filter_id=i.computed(function(){return t.id()}),t.filter_type="org",t.filtered=i.computed(function(){var e=r.filter_by();return e.id&&e.id!==t.filter_id()||e.type&&e.type!==t.filter_type})}function p(e,r){var t=this;t.id=i.observable(e.id),t.username=i.observable(e.username),t.active=i.observable(e.active),t.avatar_url=i.observable(l(e.avatar_url,{size:32})),t.provider=i.observable(e.provider),t.display_name=i.computed(function(){return t.username()}),t.filter_id=i.computed(function(){return t.provider().id}),t.filter_type="own",t.filtered=i.computed(function(){var e=r.filter_by();return e.id&&e.id!==t.filter_id()||e.type&&e.type!==t.filter_type})}function a(e,a){var o=this;o.id=i.observable(e.id),o.name=i.observable(e.name),o.full_name=i.observable(e.full_name),o.description=i.observable(e.description),o.vcs=i.observable(e.vcs),o.default_branch=i.observable(e.default_branch),o.organization=i.observable(e.organization),o.html_url=i.observable(e.html_url),o.clone_url=i.observable(e.clone_url),o.ssh_url=i.observable(e.ssh_url),o.matches=i.observable(e.matches),o.match=i.computed(function(){var e=o.matches();if(e&&0");n.attr("action",a.urls.projects_import).attr("method","POST").hide(),Object.keys(t).map(function(e){var r=u("").attr("type","hidden").attr("name",e).attr("value",t[e]);n.append(r)});var e=u("").attr("type","hidden").attr("name","csrfmiddlewaretoken").attr("value",a.csrf_token);n.append(e);var r=u("").attr("type","submit");n.append(r),u("body").append(n),n.submit()}}function o(e,r){var s=this;s.config=r||{},s.urls=r.urls||{},s.csrf_token=r.csrf_token||"",s.has_sso_enabled=r.has_sso_enabled||!1,s.error=i.observable(null),s.is_syncing=i.observable(!1),s.is_ready=i.observable(!1),s.page_current=i.observable(null),s.page_next=i.observable(null),s.page_previous=i.observable(null),s.filter_by=i.observable({id:null,type:null}),s.accounts_raw=i.observableArray(),s.organizations_raw=i.observableArray(),s.filters=i.computed(function(){var e,r=[],t=s.accounts_raw(),n=s.organizations_raw();for(e in t){var a=new p(t[e],s);r.push(a)}for(e in n){var o=new c(n[e],s);r.push(o)}return r}),s.projects=i.observableArray(),i.computed(function(){var e=s.filter_by(),r=s.page_current()||s.urls["remoterepository-list"];s.page_current()||("org"===e.type&&(r=l(s.urls["remoterepository-list"],{org:e.id})),"own"===e.type&&(r=l(s.urls["remoterepository-list"],{own:e.id}))),s.error(null),u.getJSON(r).done(function(e){var r,t=[];for(r in s.page_next(e.next),s.page_previous(e.previous),e.results){var n=new a(e.results[r],s);t.push(n)}s.projects(t)}).fail(function(e){var r=e.responseJSON.detail||e.statusText;s.error({message:r})}).always(function(){s.is_ready(!0)})}).extend({deferred:!0}),s.get_organizations=function(){u.getJSON(s.urls["remoteorganization-list"]).done(function(e){s.organizations_raw(e.results)}).fail(function(e){var r=e.responseJSON.detail||e.statusText;s.error({message:r})})},s.get_accounts=function(){u.getJSON(s.urls["remoteaccount-list"]).done(function(e){s.accounts_raw(e.results)}).fail(function(e){var r=e.responseJSON.detail||e.statusText;s.error({message:r})})},s.sync_projects=function(){var e=s.urls.api_sync_remote_repositories;s.error(null),s.is_syncing(!0),n.trigger_task({url:e,token:s.csrf_token}).then(function(e){s.get_organizations(),s.get_accounts(),s.filter_by.valueHasMutated()}).fail(function(e){s.error(e)}).always(function(){s.is_syncing(!1)})},s.has_projects=i.computed(function(){return 0").attr("href",e).get(0);return Object.keys(r).map(function(e){t.search&&(t.search+="&"),t.search+=e+"="+r[e]}),t.href}function c(e,r){var t=this;t.id=i.observable(e.id),t.name=i.observable(e.name),t.slug=i.observable(e.slug),t.active=i.observable(e.active),t.avatar_url=i.observable(l(e.avatar_url,{size:32})),t.display_name=i.computed(function(){return t.name()||t.slug()}),t.filter_id=i.computed(function(){return t.id()}),t.filter_type="org",t.filtered=i.computed(function(){var e=r.filter_by();return e.id&&e.id!==t.filter_id()||e.type&&e.type!==t.filter_type})}function p(e,r){var t=this;t.id=i.observable(e.id),t.username=i.observable(e.username),t.active=i.observable(e.active),t.avatar_url=i.observable(l(e.avatar_url,{size:32})),t.provider=i.observable(e.provider),t.display_name=i.computed(function(){return t.username()}),t.filter_id=i.computed(function(){return t.provider().id}),t.filter_type="own",t.filtered=i.computed(function(){var e=r.filter_by();return e.id&&e.id!==t.filter_id()||e.type&&e.type!==t.filter_type})}function a(e,a){var o=this;o.id=i.observable(e.id),o.name=i.observable(e.name),o.full_name=i.observable(e.full_name),o.description=i.observable(e.description),o.vcs=i.observable(e.vcs),o.default_branch=i.observable(e.default_branch),o.organization=i.observable(e.organization),o.html_url=i.observable(e.html_url),o.clone_url=i.observable(e.clone_url),o.ssh_url=i.observable(e.ssh_url),o.matches=i.observable(e.matches),o.match=i.computed(function(){var e=o.matches();if(e&&0");n.attr("action",a.urls.projects_import).attr("method","POST").hide(),Object.keys(t).map(function(e){var r=u("").attr("type","hidden").attr("name",e).attr("value",t[e]);n.append(r)});var e=u("").attr("type","hidden").attr("name","csrfmiddlewaretoken").attr("value",a.csrf_token);n.append(e);var r=u("").attr("type","submit");n.append(r),u("body").append(n),n.submit()}}function o(e,r){var s=this;s.config=r||{},s.urls=r.urls||{},s.csrf_token=r.csrf_token||"",s.has_sso_enabled=r.has_sso_enabled||!1,s.error=i.observable(null),s.is_syncing=i.observable(!1),s.is_ready=i.observable(!1),s.page_current=i.observable(null),s.page_next=i.observable(null),s.page_previous=i.observable(null),s.filter_by=i.observable({id:null,type:null}),s.accounts_raw=i.observableArray(),s.organizations_raw=i.observableArray(),s.filters=i.computed(function(){var e,r=[],t=s.accounts_raw(),n=s.organizations_raw();for(e in t){var a=new p(t[e],s);r.push(a)}for(e in n){var o=new c(n[e],s);r.push(o)}return r}),s.projects=i.observableArray(),i.computed(function(){var e=s.filter_by(),r=s.page_current()||s.urls["remoterepository-list"];s.page_current()||("org"===e.type&&(r=l(s.urls["remoterepository-list"],{org:e.id})),"own"===e.type&&(r=l(s.urls["remoterepository-list"],{own:e.id}))),s.error(null),u.getJSON(r).done(function(e){var r,t=[];for(r in s.page_next(e.next),s.page_previous(e.previous),e.results){var n=new a(e.results[r],s);t.push(n)}s.projects(t)}).fail(function(e){var r=e.responseJSON.detail||e.statusText;s.error({message:r})}).always(function(){s.is_ready(!0)})}).extend({deferred:!0}),s.get_organizations=function(){u.getJSON(s.urls["remoteorganization-list"]).done(function(e){s.organizations_raw(e.results)}).fail(function(e){var r=e.responseJSON.detail||e.statusText;s.error({message:r})})},s.get_accounts=function(){u.getJSON(s.urls["remoteaccount-list"]).done(function(e){s.accounts_raw(e.results)}).fail(function(e){var r=e.responseJSON.detail||e.statusText;s.error({message:r})})},s.sync_projects=function(){var e=s.urls.api_sync_remote_repositories;s.error(null),s.is_syncing(!0),n.trigger_task({url:e,token:s.csrf_token}).then(function(e){s.get_organizations(),s.get_accounts(),s.filter_by.valueHasMutated()}).fail(function(e){s.error(e)}).always(function(){s.is_syncing(!1)})},s.has_projects=i.computed(function(){return 0