Add basic GitLab repo sync support

Author: Daniel Widerin

readthedocs/oauth/constants.py

@@ -4,9 +4,11 @@
 
 
 OAUTH_SOURCE_GITHUB = 'github'
+OAUTH_SOURCE_GITLAB = 'gitlab'
 OAUTH_SOURCE_BITBUCKET = 'bitbucket'
 
 OAUTH_SOURCE = (
     (OAUTH_SOURCE_GITHUB, _('GitHub')),
+    (OAUTH_SOURCE_GITLAB, _('GitLab')),
     (OAUTH_SOURCE_BITBUCKET, _('Bitbucket')),
 )

readthedocs/oauth/managers.py

@@ -7,7 +7,7 @@
 
 from readthedocs.privacy.loader import RelatedUserManager
 
-from .constants import OAUTH_SOURCE_GITHUB, OAUTH_SOURCE_BITBUCKET
+from .constants import OAUTH_SOURCE_GITHUB, OAUTH_SOURCE_GITLAB, OAUTH_SOURCE_BITBUCKET
 
 
 logger = logging.getLogger(__name__)
@@ -90,6 +90,39 @@ def create_from_bitbucket_api(self, api_json, user, organization=None,
             logger.debug('Not importing %s because mismatched type' %
                          api_json['name'])
 
+    def create_from_gitlab_api(self, api_json, user, organization=None,
+                               privacy=DEFAULT_PRIVACY_LEVEL):
+        logger.info('Trying GitLab: %s' % api_json['name_with_namespace'])
+        if (api_json['public'] is False and privacy == 'private' or
+                api_json['public'] is True and privacy == 'public'):
+            project, created = self.get_or_create(
+                full_name=api_json['name_with_namespace'])
+            if project.organization and project.organization != organization:
+                logger.debug('Not importing %s because mismatched orgs' %
+                             api_json['name'])
+                return None
+            else:
+                project.organization = organization
+            project.users.add(user)
+            project.name = api_json['name']
+            project.description = api_json['description']
+            project.clone_url = api_json['http_url_to_repo']
+            project.ssh_url = api_json['ssh_url_to_repo']
+            project.html_url = api_json['web_url']
+            project.vcs = 'git'
+            project.private = False
+            project.source = OAUTH_SOURCE_GITLAB
+            if api_json['avatar_url']:
+                project.avatar_url = api_json['avatar_url']
+            else:
+                project.avatar_url = None
+            project.json = json.dumps(api_json)
+            project.save()
+            return project
+        else:
+            logger.debug('Not importing %s because mismatched type' %
+                         api_json['name'])
+
 
 class RemoteOrganizationManager(RelatedUserManager):
 

readthedocs/oauth/tasks.py

@@ -6,6 +6,7 @@
 from readthedocs.core.utils.tasks import user_id_matches
 from .utils import import_bitbucket
 from .utils import import_github
+from .utils import import_gitlab
 
 
 @permission_check(user_id_matches)
@@ -17,6 +18,7 @@ def run_public(self, user_id):
         user = User.objects.get(pk=user_id)
         import_github(user, sync=True)
         import_bitbucket(user, sync=True)
+        import_gitlab(user, sync=True)
 
 
 sync_remote_repositories = celery_app.tasks[SyncRemoteRepositories.name]

readthedocs/oauth/utils.py

@@ -9,6 +9,8 @@
 from allauth.socialaccount.models import SocialToken, SocialAccount
 from allauth.socialaccount.providers.bitbucket.provider import BitbucketProvider
 from allauth.socialaccount.providers.github.provider import GitHubProvider
+from allauth.socialaccount.providers.gitlab.provider import GitLabProvider
+from allauth.socialaccount.providers.gitlab.views import GitLabOAuth2Adapter
 
 from readthedocs.builds import utils as build_utils
 from readthedocs.restapi.client import api
@@ -42,6 +44,14 @@ def get_oauth_session(user, provider):
             resource_owner_key=token.token,
             resource_owner_secret=token.token_secret
         )
+    elif provider == GitLabProvider.id:
+        session = OAuth2Session(
+            client_id=token.app.client_id,
+            token={
+                'access_token': str(token.token),
+                'token_type': 'bearer'
+            }
+        )
 
     return session or None
 
@@ -217,3 +227,59 @@ def import_bitbucket(user, sync):
                             'try reconnecting your account')
 
     return session is not None
+
+###
+# GitLab
+###
+
+def gitlab_paginate(session, url):
+    """Combines results from GitLab pagination
+
+    :param session: requests client instance
+    :param url: start url to get the data from.
+
+    """
+    result = []
+    while url:
+        r = session.get(url)
+        result.extend([r.json()])
+        if 'next' in r.links:
+            url = r.links['next']['url']
+        else:
+            url = None
+    return result
+
+
+def process_gitlab_json(user, json):
+    try:
+        for page in json:
+            for repo in page:
+                if repo.get('archived'):
+                    continue
+                RemoteRepository.objects.create_from_gitlab_api(repo,
+                                                                user=user)
+    except TypeError, e:
+        print e
+
+
+def import_gitlab(user, sync):
+    """Import from GitLab"""
+    session = get_oauth_session(user, provider=GitLabProvider.id)
+    try:
+        social_account = user.socialaccount_set.get(provider=GitLabProvider.id)
+    except SocialAccount.DoesNotExist:
+        pass
+    if sync and session:
+        # Get user repos
+        try:
+            owner_resp = gitlab_paginate(
+                session, '{0}/api/v3/projects'.format(
+                    GitLabOAuth2Adapter.provider_base_url
+                )
+            )
+            process_gitlab_json(user, owner_resp)
+        except (TypeError, ValueError):
+            raise Exception('Could not sync your GitLab repositories, '
+                            'try reconnecting your account')
+
+    return session is not None