Merge pull request #4876 from stsewd/sync-version-when-creating-branch-tag

Sync versions when creating/deleting versions

Author: ericholscher

readthedocs/core/views/hooks.py

@@ -1,21 +1,26 @@
 """Views pertaining to builds."""
 
-from __future__ import absolute_import
+from __future__ import (
+    absolute_import,
+    division,
+    print_function,
+    unicode_literals,
+)
+
 import json
+import logging
 import re
 
 from django.http import HttpResponse, HttpResponseNotFound
 from django.shortcuts import redirect
 from django.views.decorators.csrf import csrf_exempt
 
-from readthedocs.core.utils import trigger_build
 from readthedocs.builds.constants import LATEST
+from readthedocs.core.utils import trigger_build
 from readthedocs.projects import constants
-from readthedocs.projects.models import Project, Feature
+from readthedocs.projects.models import Feature, Project
 from readthedocs.projects.tasks import sync_repository_task
 
-import logging
-
 log = logging.getLogger(__name__)
 
 
@@ -75,6 +80,35 @@ def build_branches(project, branch_list):
     return (to_build, not_building)
 
 
+def sync_versions(project):
+    """
+    Sync the versions of a repo using its latest version.
+
+    This doesn't register a new build,
+    but clones the repo and syncs the versions.
+    Due that `sync_repository_task` is bound to a version,
+    we always pass the default version.
+
+    :returns: The version slug that was used to trigger the clone.
+    :rtype: str
+    """
+    try:
+        version_identifier = project.get_default_branch()
+        version = (
+            project.versions
+            .filter(identifier=version_identifier)
+            .first()
+        )
+        if not version:
+            log.info('Unable to sync from %s version', version_identifier)
+            return None
+        sync_repository_task.delay(version.pk)
+        return version.slug
+    except Exception:
+        log.exception('Unknown sync versions exception')
+    return None
+
+
 def get_project_from_url(url):
     if not url:
         return Project.objects.none()

readthedocs/oauth/services/github.py

@@ -172,7 +172,7 @@ def get_webhook_data(self, project, integration):
                 ),
                 'content_type': 'json',
             },
-            'events': ['push', 'pull_request'],
+            'events': ['push', 'pull_request', 'create', 'delete'],
         })
 
     def setup_webhook(self, project):

readthedocs/restapi/views/integrations.py

@@ -1,32 +1,44 @@
 """Endpoints integrating with Github, Bitbucket, and other webhooks."""
 
-from __future__ import absolute_import
+from __future__ import (
+    absolute_import,
+    division,
+    print_function,
+    unicode_literals,
+)
+
 import json
 import logging
 import re
 
-from builtins import object
+import six
+from django.shortcuts import get_object_or_404
 from rest_framework import permissions
-from rest_framework.views import APIView
+from rest_framework.exceptions import NotFound, ParseError
 from rest_framework.renderers import JSONRenderer
 from rest_framework.response import Response
-from rest_framework.exceptions import ParseError, NotFound
-
-from django.shortcuts import get_object_or_404
+from rest_framework.views import APIView
 
-from readthedocs.core.views.hooks import build_branches
-from readthedocs.core.signals import (webhook_github, webhook_bitbucket,
-                                      webhook_gitlab)
+from readthedocs.core.signals import (
+    webhook_bitbucket,
+    webhook_github,
+    webhook_gitlab,
+)
+from readthedocs.core.views.hooks import build_branches, sync_versions
 from readthedocs.integrations.models import HttpExchange, Integration
 from readthedocs.integrations.utils import normalize_request_payload
 from readthedocs.projects.models import Project
-import six
-
 
 log = logging.getLogger(__name__)
 
+GITHUB_EVENT_HEADER = 'HTTP_X_GITHUB_EVENT'
 GITHUB_PUSH = 'push'
+GITHUB_CREATE = 'create'
+GITHUB_DELETE = 'delete'
 GITLAB_PUSH = 'push'
+GITLAB_NULL_HASH = '0' * 40
+GITLAB_TAG_PUSH = 'tag_push'
+BITBUCKET_EVENT_HEADER = 'HTTP_X_EVENT_KEY'
 BITBUCKET_PUSH = 'repo:push'
 
 
@@ -124,6 +136,14 @@ def get_response_push(self, project, branches):
                 'project': project.slug,
                 'versions': list(to_build)}
 
+    def sync_versions(self, project):
+        version = sync_versions(project)
+        return {
+            'build_triggered': False,
+            'project': project.slug,
+            'versions': [version],
+        }
+
 
 class GitHubWebhookView(WebhookMixin, APIView):
 
@@ -140,6 +160,12 @@ class GitHubWebhookView(WebhookMixin, APIView):
             "ref": "branch-name",
             ...
         }
+
+    See full payload here:
+
+    - https://developer.github.com/v3/activity/events/types/#pushevent
+    - https://developer.github.com/v3/activity/events/types/#createevent
+    - https://developer.github.com/v3/activity/events/types/#deleteevent
     """
 
     integration_type = Integration.GITHUB_WEBHOOK
@@ -154,16 +180,23 @@ def get_data(self):
 
     def handle_webhook(self):
         # Get event and trigger other webhook events
-        event = self.request.META.get('HTTP_X_GITHUB_EVENT', 'push')
-        webhook_github.send(Project, project=self.project,
-                            data=self.data, event=event)
+        event = self.request.META.get(GITHUB_EVENT_HEADER, GITHUB_PUSH)
+        webhook_github.send(
+            Project,
+            project=self.project,
+            data=self.data,
+            event=event
+        )
         # Handle push events and trigger builds
         if event == GITHUB_PUSH:
             try:
                 branches = [self._normalize_ref(self.data['ref'])]
                 return self.get_response_push(self.project, branches)
             except KeyError:
                 raise ParseError('Parameter "ref" is required')
+        if event in (GITHUB_CREATE, GITHUB_DELETE):
+            return self.sync_versions(self.project)
+        return None
 
     def _normalize_ref(self, ref):
         pattern = re.compile(r'^refs/(heads|tags)/')
@@ -180,26 +213,55 @@ class GitLabWebhookView(WebhookMixin, APIView):
     Expects the following JSON::
 
         {
+            "before": "95790bf891e76fee5e1747ab589903a6a1f80f22",
+            "after": "da1560886d4f094c3e6c9ef40349f7d38b5d27d7",
             "object_kind": "push",
             "ref": "branch-name",
             ...
         }
+
+    See full payload here:
+
+    - https://docs.gitlab.com/ce/user/project/integrations/webhooks.html#push-events
+    - https://docs.gitlab.com/ce/user/project/integrations/webhooks.html#tag-events
     """
 
     integration_type = Integration.GITLAB_WEBHOOK
 
     def handle_webhook(self):
-        # Get event and trigger other webhook events
+        """
+        Handle GitLab events for push and tag_push.
+
+        GitLab doesn't have a separate event for creation/deletion,
+        instead, it sets the before/after field to
+        0000000000000000000000000000000000000000 ('0' * 40)
+        """
         event = self.request.data.get('object_kind', GITLAB_PUSH)
-        webhook_gitlab.send(Project, project=self.project,
-                            data=self.request.data, event=event)
+        webhook_gitlab.send(
+            Project,
+            project=self.project,
+            data=self.request.data,
+            event=event
+        )
         # Handle push events and trigger builds
-        if event == GITLAB_PUSH:
+        if event in (GITLAB_PUSH, GITLAB_TAG_PUSH):
+            data = self.request.data
+            before = data['before']
+            after = data['after']
+            # Tag/branch created/deleted
+            if GITLAB_NULL_HASH in (before, after):
+                return self.sync_versions(self.project)
+            # Normal push to master
             try:
-                branches = [self.request.data['ref'].replace('refs/heads/', '')]
+                branches = [self._normalize_ref(data['ref'])]
                 return self.get_response_push(self.project, branches)
             except KeyError:
                 raise ParseError('Parameter "ref" is required')
+        return None
+
+    def _normalize_ref(self, ref):
+        pattern = re.compile(r'^refs/(heads|tags)/')
+        return pattern.sub('', ref)
 
 
 class BitbucketWebhookView(WebhookMixin, APIView):
@@ -218,31 +280,60 @@ class BitbucketWebhookView(WebhookMixin, APIView):
                         "name": "branch-name",
                         ...
                     },
+                    "old" {
+                        "name": "branch-name",
+                        ...
+                    },
                     ...
                 }],
                 ...
             },
             ...
         }
+
+    See full payload here:
+
+    - https://confluence.atlassian.com/bitbucket/event-payloads-740262817.html#EventPayloads-Push
     """
 
     integration_type = Integration.BITBUCKET_WEBHOOK
 
     def handle_webhook(self):
-        # Get event and trigger other webhook events
-        event = self.request.META.get('HTTP_X_EVENT_KEY', BITBUCKET_PUSH)
-        webhook_bitbucket.send(Project, project=self.project,
-                               data=self.request.data, event=event)
-        # Handle push events and trigger builds
+        """
+        Handle BitBucket events for push.
+
+        BitBucket doesn't have a separate event for creation/deletion,
+        instead it sets the new attribute (null if it is a deletion)
+        and the old attribute (null if it is a creation).
+        """
+        event = self.request.META.get(BITBUCKET_EVENT_HEADER, BITBUCKET_PUSH)
+        webhook_bitbucket.send(
+            Project,
+            project=self.project,
+            data=self.request.data,
+            event=event
+        )
         if event == BITBUCKET_PUSH:
             try:
-                changes = self.request.data['push']['changes']
-                branches = [change['new']['name']
-                            for change in changes
-                            if change.get('new')]
-                return self.get_response_push(self.project, branches)
+                data = self.request.data
+                changes = data['push']['changes']
+                branches = []
+                for change in changes:
+                    old = change['old']
+                    new = change['new']
+                    # Normal push to master
+                    if old is not None and new is not None:
+                        branches.append(new['name'])
+                # BitBuck returns an array of changes rather than
+                # one webhook per change. If we have at least one normal push
+                # we don't trigger the sync versions, because that
+                # will be triggered with the normal push.
+                if branches:
+                    return self.get_response_push(self.project, branches)
+                return self.sync_versions(self.project)
             except KeyError:
                 raise ParseError('Invalid request')
+        return None
 
 
 class IsAuthenticatedOrHasToken(permissions.IsAuthenticated):