Support for generic webhooks

Author: stsewd

readthedocs/builds/tasks.py

@@ -1,10 +1,14 @@
+import itertools
 import json
 import logging
 from datetime import datetime, timedelta
 from io import BytesIO
 
+import requests
 from celery import Task
 from django.conf import settings
+from django.urls import reverse
+from django.utils.translation import ugettext_lazy as _
 
 from readthedocs.api.v2.serializers import BuildSerializer
 from readthedocs.api.v2.utils import (
@@ -25,11 +29,11 @@
 from readthedocs.builds.models import Build, Version
 from readthedocs.builds.utils import memcache_lock
 from readthedocs.core.permissions import AdminPermission
-from readthedocs.core.utils import trigger_build
-from readthedocs.oauth.models import RemoteRepository
+from readthedocs.core.utils import send_email, trigger_build
+from readthedocs.integrations.models import HttpExchange
 from readthedocs.oauth.notifications import GitBuildStatusFailureNotification
 from readthedocs.projects.constants import GITHUB_BRAND, GITLAB_BRAND
-from readthedocs.projects.models import Project
+from readthedocs.projects.models import Project, WebHookEvent
 from readthedocs.storage import build_commands_storage
 from readthedocs.worker import app
 
@@ -453,3 +457,150 @@ def send_build_status(build_pk, commit, status, link_to_build=False):
             build.project.slug
         )
         return False
+
+
+@app.task(queue='web')
+def send_build_notifications(version_pk, build_pk, event):
+    version = Version.objects.get_object_or_log(pk=version_pk)
+
+    if not version or version.type == EXTERNAL:
+        return
+
+    build = Build.objects.get(pk=build_pk)
+
+    sender = NotificationSender(
+        version=version,
+        build=build,
+        event=event,
+    )
+    sender.send()
+
+class NotificationSender:
+
+    def __init__(self, version, build, event):
+        self.version = version
+        self.build = build
+        self.project = version.project
+        self.event = event
+
+    def send(self):
+        if self.event == WebHookEvent.BUILD_FAILED:
+            email_addresses = (
+                self.project.emailhook_notifications.all()
+                .values_list('email', flat=True)
+            )
+            for email in email_addresses:
+                try:
+                    self.send_email(email)
+                except Exception:
+                    log.exception(
+                        'Failed to send email notification. email=%s',
+                        email,
+                    )
+
+        old_webhooks = []
+        if self.event in [WebHookEvent.BUILD_FAILED, WebHookEvent.BUILD_PASSED]:
+            old_webhooks = (
+                self.project.webhook_notifications
+                .filter(events__isnull=True)
+            )
+
+        new_webhooks = (
+            self.project.webhook_notifications
+            .filter(events__name=self.event)
+        )
+        for webhook in itertools.chain(old_webhooks, new_webhooks):
+            try:
+                self.send_webhook(webhook)
+            except Exception:
+                log.exception(
+                    'Failed to send webhook. id=%s',
+                    webhook.id,
+                )
+
+    def send_email(self, email):
+        """Send email notifications for build failure."""
+        # We send only what we need from the Django model objects here to avoid
+        # serialization problems in the ``readthedocs.core.tasks.send_email_task``
+        context = {
+            'version': {
+                'verbose_name': self.version.verbose_name,
+            },
+            'project': {
+                'name': self.project.name,
+            },
+            'build': {
+                'pk': self.build.pk,
+                'error': self.build.error,
+            },
+            'build_url': 'https://{}{}'.format(
+                settings.PRODUCTION_DOMAIN,
+                self.build.get_absolute_url(),
+            ),
+            'unsub_url': 'https://{}{}'.format(
+                settings.PRODUCTION_DOMAIN,
+                reverse('projects_notifications', args=[self.project.slug]),
+            ),
+        }
+
+        if self.build.commit:
+            title = _(
+                'Failed: {project[name]} ({commit})',
+            ).format(commit=self.build.commit[:8], **context)
+        else:
+            title = _('Failed: {project[name]} ({version[verbose_name]})').format(
+                **context
+            )
+
+        log.debug(
+            'Sending email notification. project=%s version=%s build=%s',
+            self.project.slug, self.version.slug, self.build.id,
+        )
+        send_email(
+            email,
+            title,
+            template='projects/email/build_failed.txt',
+            template_html='projects/email/build_failed.html',
+            context=context,
+        )
+
+    def send_webhook(self, webhook):
+        """Send webhook notification for project webhook."""
+        payload = webhook.get_payload(
+            version=self.version,
+            build=self.build,
+            event=self.event,
+        )
+        if not payload:
+            payload = json.dumps({
+                'name': self.project.name,
+                'slug': self.project.slug,
+                'build': {
+                    'id': self.build.id,
+                    'commit': self.build.commit,
+                    'state': self.build.state,
+                    'success': self.build.success,
+                    'date': self.build.date.strftime('%Y-%m-%d %H:%M:%S'),
+                },
+            })
+
+        headers = {'content-type': 'application/json'}
+        if webhook.secret:
+            headers['X-Hub-Signature'] = webhook.sign_payload(payload)
+
+        try:
+            log.debug(
+                'Sending webhook notification. project=%s version=%s build=%s',
+                self.project.slug, self.version.slug, self.build.pk,
+            )
+            response = requests.post(
+                webhook.url,
+                data=payload,
+                headers=headers,
+            )
+            HttpExchange.objects.from_requests_exchange(
+                response=response,
+                related_object=webhook,
+            )
+        except Exception:
+            log.exception('Failed to POST on webhook url: url=%s', webhook.url)

readthedocs/core/utils/__init__.py

@@ -56,10 +56,10 @@ def prepare_build(
     """
     # Avoid circular import
     from readthedocs.builds.models import Build
-    from readthedocs.projects.models import Feature, Project
+    from readthedocs.builds.tasks import send_build_notifications
+    from readthedocs.projects.models import Feature, Project, WebHookEvent
     from readthedocs.projects.tasks import (
         send_external_build_status,
-        send_notifications,
         update_docs_task,
     )
 
@@ -123,8 +123,12 @@ def prepare_build(
         )
 
     if build and version.type != EXTERNAL:
-        # Send Webhook notification for build triggered.
-        send_notifications.delay(version.pk, build_pk=build.pk, email=False)
+        # Send notifications for build triggered.
+        send_build_notifications.delay(
+            version_pk=version.pk,
+            build_pk=build.pk,
+            event=WebHookEvent.BUILD_TRIGGERED,
+        )
 
     options['priority'] = CELERY_HIGH
     if project.main_language_project:

readthedocs/integrations/models.py

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 """Integration models for external services."""
 
 import json
@@ -96,6 +94,19 @@ def from_exchange(self, req, resp, related_object, payload=None):
         self.delete_limit(related_object)
         return obj
 
+    def from_requests_exchange(self, response, related_object):
+        request = response.request
+        obj = self.create(
+            related_object=related_object,
+            request_headers=request.headers or {},
+            request_body=request.body or '',
+            status_code=response.status_code,
+            response_headers=response.headers,
+            response_body=response.text,
+        )
+        self.delete_limit(related_object)
+        return obj
+
     def delete_limit(self, related_object, limit=10):
         if isinstance(related_object, Integration):
             queryset = self.filter(integrations=related_object)

readthedocs/projects/admin.py

@@ -19,12 +19,13 @@
     EmailHook,
     EnvironmentVariable,
     Feature,
-    HTTPHeader,
     HTMLFile,
+    HTTPHeader,
     ImportedFile,
     Project,
     ProjectRelationship,
     WebHook,
+    WebHookEvent,
 )
 from .notifications import (
     DeprecatedBuildWebhookNotification,
@@ -410,4 +411,5 @@ class EnvironmentVariableAdmin(admin.ModelAdmin):
 admin.site.register(Feature, FeatureAdmin)
 admin.site.register(EmailHook)
 admin.site.register(WebHook)
+admin.site.register(WebHookEvent)
 admin.site.register(HTMLFile, ImportedFileAdmin)

readthedocs/projects/forms.py

@@ -1,4 +1,5 @@
 """Project forms."""
+import json
 from random import choice
 from re import fullmatch
 from urllib.parse import urlparse
@@ -463,6 +464,50 @@ class Meta:
         fields = ['url']
 
 
+class GenericWebHookForm(forms.ModelForm):
+
+    project = forms.CharField(widget=forms.HiddenInput(), required=False)
+
+    class Meta:
+        model = WebHook
+        fields = ['project', 'url', 'events', 'payload', 'secret']
+        widgets = {
+            'events': forms.CheckboxSelectMultiple,
+        }
+
+    def __init__(self, *args, **kwargs):
+        self.project = kwargs.pop('project', None)
+        super().__init__(*args, **kwargs)
+
+        if self.instance and self.instance.pk:
+            # Show secret in the detail form, but as readonly.
+            self.fields['secret'].disabled = True
+        else:
+            # Don't show the secret in the creation form.
+            del self.fields['secret']
+            self.fields['payload'].initial = json.dumps({
+                'event': '${event}',
+                'name': '${project.name}',
+                'slug': '${project.slug}',
+                'version': '${version.slug}',
+                'commit': '${build.id}',
+                'commit': '${build.commit}',
+            }, indent=2)
+
+    def clean_project(self):
+        return self.project
+
+    def clean_payload(self):
+        """Check if the payload is a valid json object and format it."""
+        payload = self.cleaned_data['payload']
+        try:
+            payload = json.loads(payload)
+            payload = json.dumps(payload, indent=2)
+        except Exception:
+            raise forms.ValidationError(_('The payload must be a valid JSON object'))
+        return payload
+
+
 class TranslationBaseForm(forms.Form):
 
     """Project translation form."""

readthedocs/projects/migrations/0082_init_generic_webhooks.py

@@ -0,0 +1,53 @@
+# Generated by Django 2.2.24 on 2021-09-22 21:02
+
+from django.db import migrations, models
+import django.utils.timezone
+import django_extensions.db.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('projects', '0081_add_another_header'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='WebHookEvent',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(choices=[('build:triggered', 'Build triggered'), ('build:passed', 'Build passed'), ('build:failed', 'Build failed')], max_length=256, unique=True)),
+            ],
+        ),
+        migrations.AddField(
+            model_name='webhook',
+            name='created',
+            field=django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, default=django.utils.timezone.now, verbose_name='created'),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='webhook',
+            name='modified',
+            field=django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified'),
+        ),
+        migrations.AddField(
+            model_name='webhook',
+            name='payload',
+            field=models.TextField(blank=True, help_text='JSON payload to send to the webhook. Check the docs for available substitutions.', null=True, verbose_name='JSON payload'),
+        ),
+        migrations.AddField(
+            model_name='webhook',
+            name='secret',
+            field=models.CharField(blank=True, help_text='Secret used to sign the payload of the webhook', max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='webhook',
+            name='url',
+            field=models.URLField(help_text='URL to send the webhook to', max_length=600, verbose_name='URL'),
+        ),
+        migrations.AddField(
+            model_name='webhook',
+            name='events',
+            field=models.ManyToManyField(help_text='Events to subscribe', related_name='webhooks', to='projects.WebHookEvent'),
+        ),
+    ]