Support for generic webhooks

Author: stsewd

readthedocs/__init__.py

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 """Read the Docs."""
 
 import os.path

readthedocs/builds/tasks.py

@@ -1,11 +1,16 @@
+import itertools
 import json
 import logging
 from datetime import datetime, timedelta
 from io import BytesIO
 
+import requests
 from celery import Task
 from django.conf import settings
+from django.urls import reverse
+from django.utils.translation import ugettext_lazy as _
 
+from readthedocs import __version__
 from readthedocs.api.v2.serializers import BuildSerializer
 from readthedocs.api.v2.utils import (
     delete_versions_from_db,
@@ -25,11 +30,11 @@
 from readthedocs.builds.models import Build, Version
 from readthedocs.builds.utils import memcache_lock
 from readthedocs.core.permissions import AdminPermission
-from readthedocs.core.utils import trigger_build
-from readthedocs.oauth.models import RemoteRepository
+from readthedocs.core.utils import send_email, trigger_build
+from readthedocs.integrations.models import HttpExchange
 from readthedocs.oauth.notifications import GitBuildStatusFailureNotification
 from readthedocs.projects.constants import GITHUB_BRAND, GITLAB_BRAND
-from readthedocs.projects.models import Project
+from readthedocs.projects.models import Project, WebHookEvent
 from readthedocs.storage import build_commands_storage
 from readthedocs.worker import app
 
@@ -453,3 +458,174 @@ def send_build_status(build_pk, commit, status, link_to_build=False):
             build.project.slug
         )
         return False
+
+
+@app.task(queue='web')
+def send_build_notifications(version_pk, build_pk, event):
+    version = Version.objects.get_object_or_log(pk=version_pk)
+    if not version or version.type == EXTERNAL:
+        return
+
+    build = Build.objects.filter(pk=build_pk).first()
+    if not build:
+        return
+
+    sender = BuildNotificationSender(
+        version=version,
+        build=build,
+        event=event,
+    )
+    sender.send()
+
+
+class BuildNotificationSender:
+
+    webhook_timeout = 2
+
+    def __init__(self, version, build, event):
+        self.version = version
+        self.build = build
+        self.project = version.project
+        self.event = event
+
+    def send(self):
+        """
+        Send email and webhook notifications for `project`.
+
+        Email notifications are only send for build:failed events.
+        Webhooks choose to what events they subscribe to.
+        """
+        if self.event == WebHookEvent.BUILD_FAILED:
+            email_addresses = (
+                self.project.emailhook_notifications.all()
+                .values_list('email', flat=True)
+            )
+            for email in email_addresses:
+                try:
+                    self.send_email(email)
+                except Exception:
+                    log.exception(
+                        'Failed to send email notification. '
+                        'email=%s project=%s version=%s build=%s',
+                        email, self.project.slug, self.version.slug, self.build.pk,
+                    )
+
+        webhooks = (
+            self.project.webhook_notifications
+            .filter(events__name=self.event)
+        )
+        for webhook in webhooks:
+            try:
+                self.send_webhook(webhook)
+            except Exception:
+                log.exception(
+                    'Failed to send webhook. webhook=%s project=%s version=%s build=%s',
+                    webhook.id, self.project.slug, self.version.slug, self.build.pk,
+                )
+
+    def send_email(self, email):
+        """Send email notifications for build failures."""
+        # We send only what we need from the Django model objects here to avoid
+        # serialization problems in the ``readthedocs.core.tasks.send_email_task``
+        context = {
+            'version': {
+                'verbose_name': self.version.verbose_name,
+            },
+            'project': {
+                'name': self.project.name,
+            },
+            'build': {
+                'pk': self.build.pk,
+                'error': self.build.error,
+            },
+            'build_url': 'https://{}{}'.format(
+                settings.PRODUCTION_DOMAIN,
+                self.build.get_absolute_url(),
+            ),
+            'unsub_url': 'https://{}{}'.format(
+                settings.PRODUCTION_DOMAIN,
+                reverse('projects_notifications', args=[self.project.slug]),
+            ),
+        }
+
+        if self.build.commit:
+            title = _('Failed: {project[name]} ({commit})').format(
+                commit=self.build.commit[:8],
+                **context,
+            )
+        else:
+            title = _('Failed: {project[name]} ({version[verbose_name]})').format(
+                **context
+            )
+
+        log.info(
+            'Sending email notification. email=%s project=%s version=%s build=%s',
+            email, self.project.slug, self.version.slug, self.build.id,
+        )
+        send_email(
+            email,
+            title,
+            template='projects/email/build_failed.txt',
+            template_html='projects/email/build_failed.html',
+            context=context,
+        )
+
+    def send_webhook(self, webhook):
+        """
+        Send webhook notification.
+
+        The payload is signed using HMAC-SHA1,
+        for users to be able to verify the authenticity of the request.
+
+        Webhooks that don't have a payload,
+        are from the old implementation, for those we keep sending the
+        old default payload.
+
+        An HttpExchange object is created for each transaction.
+        """
+        payload = webhook.get_payload(
+            version=self.version,
+            build=self.build,
+            event=self.event,
+        )
+        if not payload:
+            # Default payload from old webhooks.
+            payload = json.dumps({
+                'name': self.project.name,
+                'slug': self.project.slug,
+                'build': {
+                    'id': self.build.id,
+                    'commit': self.build.commit,
+                    'state': self.build.state,
+                    'success': self.build.success,
+                    'date': self.build.date.strftime('%Y-%m-%d %H:%M:%S'),
+                },
+            })
+
+        headers = {
+            'content-type': 'application/json',
+            'User-Agent': f'Read-the-Docs/{__version__}'
+        }
+        if webhook.secret:
+            headers['X-Hub-Signature'] = webhook.sign_payload(payload)
+
+        try:
+            log.info(
+                'Sending webhook notification. webhook=%s project=%s version=%s build=%s',
+                webhook.pk, self.project.slug, self.version.slug, self.build.pk,
+            )
+            response = requests.post(
+                webhook.url,
+                data=payload,
+                headers=headers,
+                timeout=self.webhook_timeout,
+            )
+            HttpExchange.objects.from_requests_exchange(
+                response=response,
+                related_object=webhook,
+            )
+        except Exception:
+            log.exception(
+                'Failed to POST to webhook url. webhook=%s url=%s',
+                webhook.pk, webhook.url,
+            )

readthedocs/core/utils/__init__.py

@@ -56,10 +56,10 @@ def prepare_build(
     """
     # Avoid circular import
     from readthedocs.builds.models import Build
-    from readthedocs.projects.models import Feature, Project
+    from readthedocs.builds.tasks import send_build_notifications
+    from readthedocs.projects.models import Feature, Project, WebHookEvent
     from readthedocs.projects.tasks import (
         send_external_build_status,
-        send_notifications,
         update_docs_task,
     )
 
@@ -123,8 +123,12 @@ def prepare_build(
         )
 
     if build and version.type != EXTERNAL:
-        # Send Webhook notification for build triggered.
-        send_notifications.delay(version.pk, build_pk=build.pk, email=False)
+        # Send notifications for build triggered.
+        send_build_notifications.delay(
+            version_pk=version.pk,
+            build_pk=build.pk,
+            event=WebHookEvent.BUILD_TRIGGERED,
+        )
 
     options['priority'] = CELERY_HIGH
     if project.main_language_project:

readthedocs/integrations/models.py

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 """Integration models for external services."""
 
 import json
@@ -96,6 +94,25 @@ def from_exchange(self, req, resp, related_object, payload=None):
         self.delete_limit(related_object)
         return obj
 
+    def from_requests_exchange(self, response, related_object):
+        """
+        Create an exchange object from a requests' response.
+
+        :param response: The result from calling request.post() or similar.
+        :param related_object: Object to use for generic relationship.
+        """
+        request = response.request
+        obj = self.create(
+            related_object=related_object,
+            request_headers=request.headers or {},
+            request_body=request.body or '',
+            status_code=response.status_code,
+            response_headers=response.headers,
+            response_body=response.text,
+        )
+        self.delete_limit(related_object)
+        return obj
+
     def delete_limit(self, related_object, limit=10):
         if isinstance(related_object, Integration):
             queryset = self.filter(integrations=related_object)

readthedocs/projects/admin.py

@@ -19,12 +19,13 @@
     EmailHook,
     EnvironmentVariable,
     Feature,
-    HTTPHeader,
     HTMLFile,
+    HTTPHeader,
     ImportedFile,
     Project,
     ProjectRelationship,
     WebHook,
+    WebHookEvent,
 )
 from .notifications import (
     DeprecatedBuildWebhookNotification,
@@ -410,4 +411,5 @@ class EnvironmentVariableAdmin(admin.ModelAdmin):
 admin.site.register(Feature, FeatureAdmin)
 admin.site.register(EmailHook)
 admin.site.register(WebHook)
+admin.site.register(WebHookEvent)
 admin.site.register(HTMLFile, ImportedFileAdmin)

readthedocs/projects/forms.py

@@ -1,4 +1,5 @@
 """Project forms."""
+import json
 from random import choice
 from re import fullmatch
 from urllib.parse import urlparse
@@ -444,23 +445,46 @@ def save(self):
 
 class WebHookForm(forms.ModelForm):
 
-    """Project webhook form."""
+    project = forms.CharField(widget=forms.HiddenInput(), required=False)
+
+    class Meta:
+        model = WebHook
+        fields = ['project', 'url', 'events', 'payload', 'secret']
+        widgets = {
+            'events': forms.CheckboxSelectMultiple,
+        }
 
     def __init__(self, *args, **kwargs):
         self.project = kwargs.pop('project', None)
         super().__init__(*args, **kwargs)
 
-    def save(self, commit=True):
-        self.webhook = WebHook.objects.get_or_create(
-            url=self.cleaned_data['url'],
-            project=self.project,
-        )[0]
-        self.project.webhook_notifications.add(self.webhook)
+        if self.instance and self.instance.pk:
+            # Show secret in the detail form, but as readonly.
+            self.fields['secret'].disabled = True
+        else:
+            # Don't show the secret in the creation form.
+            self.fields.pop('secret')
+            self.fields['payload'].initial = json.dumps({
+                'event': '${event}',
+                'name': '${project.name}',
+                'slug': '${project.slug}',
+                'version': '${version.slug}',
+                'commit': '${build.commit}',
+                'build': '${build.pk}',
+            }, indent=2)
+
+    def clean_project(self):
         return self.project
 
-    class Meta:
-        model = WebHook
-        fields = ['url']
+    def clean_payload(self):
+        """Check if the payload is a valid json object and format it."""
+        payload = self.cleaned_data['payload']
+        try:
+            payload = json.loads(payload)
+            payload = json.dumps(payload, indent=2)
+        except Exception:
+            raise forms.ValidationError(_('The payload must be a valid JSON object.'))
+        return payload
 
 
 class TranslationBaseForm(forms.Form):