diff --git a/readthedocs/oauth/services/base.py b/readthedocs/oauth/services/base.py index e093ae24408..043b3a30a93 100644 --- a/readthedocs/oauth/services/base.py +++ b/readthedocs/oauth/services/base.py @@ -6,6 +6,7 @@ from allauth.socialaccount.models import SocialAccount from allauth.socialaccount.providers import registry from django.conf import settings +from django.db.models import Q from django.utils import timezone from oauthlib.oauth2.rfc6749.errors import InvalidClientIdError from requests.exceptions import RequestException @@ -181,11 +182,30 @@ def paginate(self, url, **kwargs): url, debug_data, ) - return [] + + return [] def sync(self): - """Sync repositories and organizations.""" - raise NotImplementedError + """ + Sync repositories (RemoteRepository) and organizations (RemoteOrganization). + + - creates a new RemoteRepository/Organization per new repository + - updates fields for existing RemoteRepository/Organization + - deletes old RemoteRepository/Organization that are not present for this user + """ + repos = self.sync_repositories() + organizations, organization_repos = self.sync_organizations() + + # Delete RemoteRepository where the user doesn't have access anymore + # (skip RemoteRepository tied to a Project on this user) + repository_full_names = self.get_repository_full_names(repos + organization_repos) + self.user.oauth_repositories.exclude( + Q(full_name__in=repository_full_names) | Q(project__isnull=False) + ).delete() + + # Delete RemoteOrganization where the user doesn't have access anymore + organization_names = self.get_organization_names(organizations) + self.user.oauth_organizations.exclude(name__in=organization_names).delete() def create_repository(self, fields, privacy=None, organization=None): """ diff --git a/readthedocs/oauth/services/bitbucket.py b/readthedocs/oauth/services/bitbucket.py index 144b5e4508c..65f8a81f9b6 100644 --- a/readthedocs/oauth/services/bitbucket.py +++ b/readthedocs/oauth/services/bitbucket.py @@ -30,13 +30,10 @@ class BitbucketService(Service): url_pattern = re.compile(r'bitbucket.org') https_url_pattern = re.compile(r'^https:\/\/[^@]+@bitbucket.org/') - def sync(self): - """Sync repositories and teams from Bitbucket API.""" - self.sync_repositories() - self.sync_teams() - def sync_repositories(self): """Sync repositories from Bitbucket API.""" + repos = [] + # Get user repos try: repos = self.paginate( @@ -44,6 +41,7 @@ def sync_repositories(self): ) for repo in repos: self.create_repository(repo) + except (TypeError, ValueError): log.warning('Error syncing Bitbucket repositories') raise SyncServiceError( @@ -58,21 +56,26 @@ def sync_repositories(self): resp = self.paginate( 'https://bitbucket.org/api/2.0/repositories/?role=admin', ) - repos = ( + admin_repos = ( RemoteRepository.objects.filter( users=self.user, full_name__in=[r['full_name'] for r in resp], account=self.account, ) ) - for repo in repos: + for repo in admin_repos: repo.admin = True repo.save() except (TypeError, ValueError): pass - def sync_teams(self): - """Sync Bitbucket teams and team repositories.""" + return repos + + def sync_organizations(self): + """Sync Bitbucket teams (our RemoteOrganization) and team repositories.""" + teams = [] + repositories = [] + try: teams = self.paginate( 'https://api.bitbucket.org/2.0/teams/?role=member', @@ -80,8 +83,13 @@ def sync_teams(self): for team in teams: org = self.create_organization(team) repos = self.paginate(team['links']['repositories']['href']) + + # Add organization's repositories to the result + repositories.extend(repos) + for repo in repos: self.create_repository(repo, organization=org) + except ValueError: log.warning('Error syncing Bitbucket organizations') raise SyncServiceError( @@ -89,6 +97,8 @@ def sync_teams(self): 'try reconnecting your account', ) + return teams, repositories + def create_repository(self, fields, privacy=None, organization=None): """ Update or create a repository from Bitbucket API response. @@ -180,6 +190,12 @@ def create_organization(self, fields): organization.save() return organization + def get_repository_full_names(self, repositories): + return {repository.get('full_name') for repository in repositories} + + def get_organization_names(self, organizations): + return {organization.get('display_name') for organization in organizations} + def get_next_url_to_paginate(self, response): return response.json().get('next') diff --git a/readthedocs/oauth/services/github.py b/readthedocs/oauth/services/github.py index 21bbae86e76..0e62f9dba92 100644 --- a/readthedocs/oauth/services/github.py +++ b/readthedocs/oauth/services/github.py @@ -6,6 +6,7 @@ from allauth.socialaccount.models import SocialToken from allauth.socialaccount.providers.github.views import GitHubOAuth2Adapter + from django.conf import settings from django.db.models import Q from django.urls import reverse @@ -33,34 +34,27 @@ class GitHubService(Service): # TODO replace this with a less naive check url_pattern = re.compile(r'github\.com') - def sync(self): - """Sync repositories and organizations.""" - repos = self.sync_repositories() - organization_repos = self.sync_organizations() - - # Delete RemoteRepository where the user doesn't have access anymore - # (skip RemoteRepository tied to a Project on this user) - full_names = {repo.get('full_name') for repo in repos + organization_repos} - self.user.oauth_repositories.exclude( - Q(full_name__in=full_names) | Q(project__isnull=False) - ).delete() - def sync_repositories(self): """Sync repositories from GitHub API.""" - repos = self.paginate('https://api.github.com/user/repos?per_page=100') + repos = [] + try: + repos = self.paginate('https://api.github.com/user/repos?per_page=100') for repo in repos: self.create_repository(repo) - return repos except (TypeError, ValueError): log.warning('Error syncing GitHub repositories') raise SyncServiceError( 'Could not sync your GitHub repositories, ' 'try reconnecting your account' ) + return repos def sync_organizations(self): """Sync organizations from GitHub API.""" + orgs = [] + repositories = [] + try: orgs = self.paginate('https://api.github.com/user/orgs') for org in orgs: @@ -71,9 +65,13 @@ def sync_organizations(self): org_repos = self.paginate( '{org_url}/repos'.format(org_url=org['url']), ) + + # Add all the repositories for this organization to the result + repositories.extend(org_repos) + for repo in org_repos: self.create_repository(repo, organization=org_obj) - return org_repos + except (TypeError, ValueError): log.warning('Error syncing GitHub organizations') raise SyncServiceError( @@ -81,6 +79,8 @@ def sync_organizations(self): 'try reconnecting your account' ) + return orgs, repositories + def create_repository(self, fields, privacy=None, organization=None): """ Update or create a repository from GitHub API response. @@ -170,6 +170,12 @@ def create_organization(self, fields): organization.save() return organization + def get_repository_full_names(self, repositories): + return {repository.get('full_name') for repository in repositories} + + def get_organization_names(self, organizations): + return {organization.get('name') for organization in organizations} + def get_next_url_to_paginate(self, response): return response.links.get('next', {}).get('url') diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index 5f3dba7368b..806e7dc69f6 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -66,26 +66,18 @@ def get_next_url_to_paginate(self, response): def get_paginated_results(self, response): return response.json() - def sync(self): - """ - Sync repositories and organizations from GitLab API. - - See: https://docs.gitlab.com/ce/api/projects.html - """ - self.sync_repositories() - self.sync_organizations() - def sync_repositories(self): - repos = self.paginate( - '{url}/api/v4/projects'.format(url=self.adapter.provider_base_url), - per_page=100, - archived=False, - order_by='path', - sort='asc', - membership=True, - ) - + repos = [] try: + repos = self.paginate( + '{url}/api/v4/projects'.format(url=self.adapter.provider_base_url), + per_page=100, + archived=False, + order_by='path', + sort='asc', + membership=True, + ) + for repo in repos: self.create_repository(repo) except (TypeError, ValueError): @@ -95,16 +87,20 @@ def sync_repositories(self): 'try reconnecting your account' ) + return repos + def sync_organizations(self): - orgs = self.paginate( - '{url}/api/v4/groups'.format(url=self.adapter.provider_base_url), - per_page=100, - all_available=False, - order_by='path', - sort='asc', - ) + orgs = [] + repositories = [] try: + orgs = self.paginate( + '{url}/api/v4/groups'.format(url=self.adapter.provider_base_url), + per_page=100, + all_available=False, + order_by='path', + sort='asc', + ) for org in orgs: org_obj = self.create_organization(org) org_repos = self.paginate( @@ -117,6 +113,10 @@ def sync_organizations(self): order_by='path', sort='asc', ) + + # Add organization's repositories to the result + repositories.extend(org_repos) + for repo in org_repos: self.create_repository(repo, organization=org_obj) except (TypeError, ValueError): @@ -126,6 +126,8 @@ def sync_organizations(self): 'try reconnecting your account' ) + return orgs, repositories + def is_owned_by(self, owner_id): return self.account.extra_data['id'] == owner_id @@ -230,6 +232,12 @@ def create_organization(self, fields): organization.save() return organization + def get_repository_full_names(self, repositories): + return {repository.get('name_with_namespace') for repository in repositories} + + def get_organization_names(self, organizations): + return {organization.get('name') for organization in organizations} + def get_webhook_data(self, repo_id, project, integration): """ Get webhook JSON data to post to the API.