Merge pull request #5823 from readthedocs/gsoc-19-pr-builder

Merge initial work from Pull Request Builder GSOC

Author: ericholscher

.gitignore

@@ -28,6 +28,7 @@ logs/*
 media/dash
 media/epub
 media/export
+media/external
 media/html
 media/htmlzip
 media/json

docs/guides/feature-flags.rst

@@ -29,3 +29,5 @@ e.g. python-reno release notes manager is known to do that
 (error message line would probably include one of old Git commit id's).
 
 ``USE_TESTING_BUILD_IMAGE``: :featureflags:`USE_TESTING_BUILD_IMAGE`
+
+``EXTERNAL_VERSION_BUILD``: :featureflags:`EXTERNAL_VERSION_BUILD`

readthedocs/api/v2/serializers.py

@@ -118,6 +118,7 @@ class BuildSerializer(serializers.ModelSerializer):
     version_slug = serializers.ReadOnlyField(source='version.slug')
     docs_url = serializers.ReadOnlyField(source='version.get_absolute_url')
     state_display = serializers.ReadOnlyField(source='get_state_display')
+    commit_url = serializers.ReadOnlyField(source='get_commit_url')
     # Jsonfield needs an explicit serializer
     # https://github.com/dmkoch/django-jsonfield/issues/188#issuecomment-300439829
     config = serializers.JSONField(required=False)

readthedocs/api/v2/views/footer_views.py

@@ -25,7 +25,7 @@ def get_version_compare_data(project, base_version=None):
     :param base_version: We assert whether or not the base_version is also the
                          highest version in the resulting "is_highest" value.
     """
-    versions_qs = Version.objects.public(project=project)
+    versions_qs = Version.internal.public(project=project)
 
     # Take preferences over tags only if the project has at least one tag
     if versions_qs.filter(type=TAG).exists():

readthedocs/api/v2/views/integrations.py

@@ -19,16 +19,27 @@
     webhook_github,
     webhook_gitlab,
 )
-from readthedocs.core.views.hooks import build_branches, sync_versions
+from readthedocs.core.views.hooks import (
+    build_branches,
+    sync_versions,
+    get_or_create_external_version,
+    delete_external_version,
+    build_external_version,
+)
 from readthedocs.integrations.models import HttpExchange, Integration
-from readthedocs.projects.models import Project
+from readthedocs.projects.models import Project, Feature
 
 
 log = logging.getLogger(__name__)
 
 GITHUB_EVENT_HEADER = 'HTTP_X_GITHUB_EVENT'
 GITHUB_SIGNATURE_HEADER = 'HTTP_X_HUB_SIGNATURE'
 GITHUB_PUSH = 'push'
+GITHUB_PULL_REQUEST = 'pull_request'
+GITHUB_PULL_REQUEST_OPENED = 'opened'
+GITHUB_PULL_REQUEST_CLOSED = 'closed'
+GITHUB_PULL_REQUEST_REOPENED = 'reopened'
+GITHUB_PULL_REQUEST_SYNC = 'synchronize'
 GITHUB_CREATE = 'create'
 GITHUB_DELETE = 'delete'
 GITLAB_TOKEN_HEADER = 'HTTP_X_GITLAB_TOKEN'
@@ -110,6 +121,10 @@ def handle_webhook(self):
         """Handle webhook payload."""
         raise NotImplementedError
 
+    def get_external_version_data(self):
+        """Get External Version data from payload."""
+        raise NotImplementedError
+
     def is_payload_valid(self):
         """Validates the webhook's payload using the integration's secret."""
         return False
@@ -183,28 +198,96 @@ def sync_versions(self, project):
             'versions': [version],
         }
 
+    def get_external_version_response(self, project):
+        """
+        Trigger builds for External versions on pull/merge request events and return API response.
+
+        Return a JSON response with the following::
+
+            {
+                "build_triggered": true,
+                "project": "project_name",
+                "versions": [verbose_name]
+            }
+
+        :param project: Project instance
+        :type project: readthedocs.projects.models.Project
+        """
+        identifier, verbose_name = self.get_external_version_data()
+        # create or get external version object using `verbose_name`.
+        external_version = get_or_create_external_version(
+            project, identifier, verbose_name
+        )
+        # returns external version verbose_name (pull/merge request number)
+        to_build = build_external_version(project, external_version)
+
+        return {
+            'build_triggered': True,
+            'project': project.slug,
+            'versions': [to_build],
+        }
+
+    def get_delete_external_version_response(self, project):
+        """
+        Delete External version on pull/merge request `closed` events and return API response.
+
+        Return a JSON response with the following::
+
+            {
+                "version_deleted": true,
+                "project": "project_name",
+                "versions": [verbose_name]
+            }
+
+        :param project: Project instance
+        :type project: Project
+        """
+        identifier, verbose_name = self.get_external_version_data()
+        # Delete external version
+        deleted_version = delete_external_version(
+            project, identifier, verbose_name
+        )
+        return {
+            'version_deleted': deleted_version is not None,
+            'project': project.slug,
+            'versions': [deleted_version],
+        }
+
 
 class GitHubWebhookView(WebhookMixin, APIView):
 
     """
     Webhook consumer for GitHub.
 
-    Accepts webhook events from GitHub, 'push' events trigger builds. Expects the
-    webhook event type will be included in HTTP header ``X-GitHub-Event``, and
-    we will have a JSON payload.
+    Accepts webhook events from GitHub, 'push' and 'pull_request' events trigger builds.
+    Expects the webhook event type will be included in HTTP header ``X-GitHub-Event``,
+    and we will have a JSON payload.
 
     Expects the following JSON::
 
-        {
-            "ref": "branch-name",
-            ...
-        }
+        For push, create, delete Events:
+            {
+                "ref": "branch-name",
+                ...
+            }
+
+        For pull_request Events:
+            {
+                "action": "opened",
+                "number": 2,
+                "pull_request": {
+                    "head": {
+                        "sha": "ec26de721c3235aad62de7213c562f8c821"
+                    }
+                }
+            }
 
     See full payload here:
 
     - https://developer.github.com/v3/activity/events/types/#pushevent
     - https://developer.github.com/v3/activity/events/types/#createevent
     - https://developer.github.com/v3/activity/events/types/#deleteevent
+    - https://developer.github.com/v3/activity/events/types/#pullrequestevent
     """
 
     integration_type = Integration.GITHUB_WEBHOOK
@@ -218,6 +301,17 @@ def get_data(self):
                 pass
         return super().get_data()
 
+    def get_external_version_data(self):
+        """Get Commit Sha and pull request number from payload."""
+        try:
+            identifier = self.data['pull_request']['head']['sha']
+            verbose_name = str(self.data['number'])
+
+            return identifier, verbose_name
+
+        except KeyError:
+            raise ParseError('Parameters "sha" and "number" are required')
+
     def is_payload_valid(self):
         """
         GitHub use a HMAC hexdigest hash to sign the payload.
@@ -256,6 +350,7 @@ def get_digest(secret, msg):
 
     def handle_webhook(self):
         # Get event and trigger other webhook events
+        action = self.data.get('action', None)
         event = self.request.META.get(GITHUB_EVENT_HEADER, GITHUB_PUSH)
         webhook_github.send(
             Project,
@@ -272,6 +367,26 @@ def handle_webhook(self):
                 raise ParseError('Parameter "ref" is required')
         if event in (GITHUB_CREATE, GITHUB_DELETE):
             return self.sync_versions(self.project)
+
+        if (
+            self.project.has_feature(Feature.EXTERNAL_VERSION_BUILD) and
+            event == GITHUB_PULL_REQUEST and action
+        ):
+            if (
+                action in
+                [
+                    GITHUB_PULL_REQUEST_OPENED,
+                    GITHUB_PULL_REQUEST_REOPENED,
+                    GITHUB_PULL_REQUEST_SYNC
+                ]
+            ):
+                # Handle opened, synchronize, reopened pull_request event.
+                return self.get_external_version_response(self.project)
+
+            if action == GITHUB_PULL_REQUEST_CLOSED:
+                # Handle closed pull_request event.
+                return self.get_delete_external_version_response(self.project)
+
         return None
 
     def _normalize_ref(self, ref):

readthedocs/api/v2/views/model_views.py

@@ -10,7 +10,7 @@
 from rest_framework.renderers import BaseRenderer, JSONRenderer
 from rest_framework.response import Response
 
-from readthedocs.builds.constants import BRANCH, TAG
+from readthedocs.builds.constants import BRANCH, TAG, INTERNAL
 from readthedocs.builds.models import Build, BuildCommandResult, Version
 from readthedocs.core.utils import trigger_build
 from readthedocs.core.utils.extend import SettingsOverrideObject
@@ -130,7 +130,7 @@ def active_versions(self, request, **kwargs):
             Project.objects.api(request.user),
             pk=kwargs['pk'],
         )
-        versions = project.versions.filter(active=True)
+        versions = project.versions(manager=INTERNAL).filter(active=True)
         return Response({
             'versions': VersionSerializer(versions, many=True).data,
         })

readthedocs/api/v3/views.py

@@ -281,7 +281,7 @@ class VersionsViewSet(APIv3Settings, NestedViewSetMixin, ProjectQuerySetMixin,
     lookup_value_regex = r'[^/]+'
 
     filterset_class = VersionFilter
-    queryset = Version.objects.all()
+    queryset = Version.internal.all()
     permit_list_expands = [
         'last_build',
         'last_build.config',
@@ -320,7 +320,7 @@ class BuildsViewSet(APIv3Settings, NestedViewSetMixin, ProjectQuerySetMixin,
     lookup_url_kwarg = 'build_pk'
     serializer_class = BuildSerializer
     filterset_class = BuildFilter
-    queryset = Build.objects.all()
+    queryset = Build.internal.all()
     permit_list_expands = [
         'config',
     ]

readthedocs/builds/constants.py

@@ -31,13 +31,21 @@
     ('dash', _('Dash')),
 )
 
+# Manager name for Internal Versions or Builds.
+# ie: Versions and Builds Excluding pull request/merge request Versions and Builds.
+INTERNAL = 'internal'
+# Manager name for External Versions or Builds.
+# ie: Only pull request/merge request Versions and Builds.
+EXTERNAL = 'external'
+
 BRANCH = 'branch'
 TAG = 'tag'
 UNKNOWN = 'unknown'
 
 VERSION_TYPES = (
     (BRANCH, _('Branch')),
     (TAG, _('Tag')),
+    (EXTERNAL, _('External')),
     (UNKNOWN, _('Unknown')),
 )
 
@@ -53,3 +61,34 @@
     LATEST,
     STABLE,
 )
+
+# General Build Statuses
+BUILD_STATUS_FAILURE = 'failed'
+BUILD_STATUS_PENDING = 'pending'
+BUILD_STATUS_SUCCESS = 'success'
+
+# GitHub Build Statuses
+GITHUB_BUILD_STATUS_FAILURE = 'failure'
+GITHUB_BUILD_STATUS_PENDING = 'pending'
+GITHUB_BUILD_STATUS_SUCCESS = 'success'
+
+# Used to select correct Build status and description to be sent to each service API
+SELECT_BUILD_STATUS = {
+    BUILD_STATUS_FAILURE: {
+        'github': GITHUB_BUILD_STATUS_FAILURE,
+        'description': 'Read the Docs build failed!',
+    },
+    BUILD_STATUS_PENDING: {
+        'github': GITHUB_BUILD_STATUS_PENDING,
+        'description': 'Read the Docs build is in progress!',
+    },
+    BUILD_STATUS_SUCCESS: {
+        'github': GITHUB_BUILD_STATUS_SUCCESS,
+        'description': 'Read the Docs build succeeded!',
+    },
+}
+
+RTD_BUILD_STATUS_API_NAME = 'continuous-documentation/read-the-docs'
+
+GITHUB_EXTERNAL_VERSION_NAME = 'Pull Request'
+GENERIC_EXTERNAL_VERSION_NAME = 'External Version'