Support for generic webhooks

Author: stsewd

readthedocs/builds/tasks.py

@@ -3,8 +3,11 @@
 from datetime import datetime, timedelta
 from io import BytesIO
 
+import requests
 from celery import Task
 from django.conf import settings
+from django.urls import reverse
+from django.utils.translation import ugettext_lazy as _
 
 from readthedocs.api.v2.serializers import BuildSerializer
 from readthedocs.api.v2.utils import (
@@ -25,8 +28,7 @@
 from readthedocs.builds.models import Build, Version
 from readthedocs.builds.utils import memcache_lock
 from readthedocs.core.permissions import AdminPermission
-from readthedocs.core.utils import trigger_build
-from readthedocs.oauth.models import RemoteRepository
+from readthedocs.core.utils import send_email, trigger_build
 from readthedocs.oauth.notifications import GitBuildStatusFailureNotification
 from readthedocs.projects.constants import GITHUB_BRAND, GITLAB_BRAND
 from readthedocs.projects.models import Project
@@ -453,3 +455,111 @@ def send_build_status(build_pk, commit, status, link_to_build=False):
             build.project.slug
         )
         return False
+
+
+@app.task(queue='web')
+def send_build_notifications(version_pk, build_pk, email=False):
+    version = Version.objects.get_object_or_log(pk=version_pk)
+
+    if not version or version.type == EXTERNAL:
+        return
+
+    build = Build.objects.get(pk=build_pk)
+
+    for hook in version.project.webhook_notifications.all():
+        webhook_notification(version, build, hook.url)
+
+    if email:
+        for email_address in version.project.emailhook_notifications.all().values_list(
+            'email',
+            flat=True,
+        ):
+            email_notification(version, build, email_address)
+
+
+def email_notification(version, build, email):
+    """
+    Send email notifications for build failure.
+
+    :param version: :py:class:`Version` instance that failed
+    :param build: :py:class:`Build` instance that failed
+    :param email: Email recipient address
+    """
+    # We send only what we need from the Django model objects here to avoid
+    # serialization problems in the ``readthedocs.core.tasks.send_email_task``
+    context = {
+        'version': {
+            'verbose_name': version.verbose_name,
+        },
+        'project': {
+            'name': version.project.name,
+        },
+        'build': {
+            'pk': build.pk,
+            'error': build.error,
+        },
+        'build_url': 'https://{}{}'.format(
+            settings.PRODUCTION_DOMAIN,
+            build.get_absolute_url(),
+        ),
+        'unsub_url': 'https://{}{}'.format(
+            settings.PRODUCTION_DOMAIN,
+            reverse('projects_notifications', args=[version.project.slug]),
+        ),
+    }
+
+    if build.commit:
+        title = _(
+            'Failed: {project[name]} ({commit})',
+        ).format(commit=build.commit[:8], **context)
+    else:
+        title = _('Failed: {project[name]} ({version[verbose_name]})').format(
+            **context
+        )
+
+    log.debug(
+        'Sending email notification. project=%s version=%s build=%s',
+        version.project.slug, version.slug, build.id,
+    )
+    send_email(
+        email,
+        title,
+        template='projects/email/build_failed.txt',
+        template_html='projects/email/build_failed.html',
+        context=context,
+    )
+
+
+def webhook_notification(version, build, hook_url):
+    """
+    Send webhook notification for project webhook.
+
+    :param version: Version instance to send hook for
+    :param build: Build instance that failed
+    :param hook_url: Hook URL to send to
+    """
+    project = version.project
+
+    data = json.dumps({
+        'name': project.name,
+        'slug': project.slug,
+        'build': {
+            'id': build.id,
+            'commit': build.commit,
+            'state': build.state,
+            'success': build.success,
+            'date': build.date.strftime('%Y-%m-%d %H:%M:%S'),
+        },
+    })
+    try:
+        log.debug(
+            'Sending webhook notification. project=%s version=%s build=%s',
+            version.project.slug, version.slug, build.pk,
+        )
+        requests.post(
+            hook_url,
+            data=data,
+            headers={'content-type': 'application/json'}
+        )
+    except Exception:
+        log.exception('Failed to POST on webhook url: url=%s', hook_url)

readthedocs/projects/admin.py

@@ -19,12 +19,13 @@
     EmailHook,
     EnvironmentVariable,
     Feature,
-    HTTPHeader,
     HTMLFile,
+    HTTPHeader,
     ImportedFile,
     Project,
     ProjectRelationship,
     WebHook,
+    WebHookEvent,
 )
 from .notifications import (
     DeprecatedBuildWebhookNotification,
@@ -410,4 +411,5 @@ class EnvironmentVariableAdmin(admin.ModelAdmin):
 admin.site.register(Feature, FeatureAdmin)
 admin.site.register(EmailHook)
 admin.site.register(WebHook)
+admin.site.register(WebHookEvent)
 admin.site.register(HTMLFile, ImportedFileAdmin)

readthedocs/projects/forms.py

@@ -1,4 +1,5 @@
 """Project forms."""
+import json
 from random import choice
 from re import fullmatch
 from urllib.parse import urlparse
@@ -463,6 +464,50 @@ class Meta:
         fields = ['url']
 
 
+class GenericWebHookForm(forms.ModelForm):
+
+    project = forms.CharField(widget=forms.HiddenInput(), required=False)
+
+    class Meta:
+        model = WebHook
+        fields = ['project', 'url', 'events', 'payload', 'secret']
+        widgets = {
+            'events': forms.CheckboxSelectMultiple,
+        }
+
+    def __init__(self, *args, **kwargs):
+        self.project = kwargs.pop('project', None)
+        super().__init__(*args, **kwargs)
+
+        if self.instance and self.instance.pk:
+            # Show secret in the detail form, but as readonly.
+            self.fields['secret'].disabled = True
+        else:
+            # Don't show the secret in the creation form.
+            del self.fields['secret']
+            self.fields['payload'].initial = json.dumps({
+                'event': '${event}',
+                'name': '${project.name}',
+                'slug': '${project.slug}',
+                'version': '${version.slug}',
+                'commit': '${build.id}',
+                'commit': '${build.commit}',
+            }, indent=2)
+
+    def clean_project(self):
+        return self.project
+
+    def clean_payload(self):
+        """Check if the payload is a valid json object and format it."""
+        payload = self.cleaned_data['payload']
+        try:
+            payload = json.loads(payload)
+            payload = json.dumps(payload, indent=2)
+        except Exception:
+            raise forms.ValidationError(_('The payload must be a valid JSON object'))
+        return payload
+
+
 class TranslationBaseForm(forms.Form):
 
     """Project translation form."""

readthedocs/projects/migrations/0082_init_generic_webhooks.py

@@ -0,0 +1,53 @@
+# Generated by Django 2.2.24 on 2021-09-22 21:02
+
+from django.db import migrations, models
+import django.utils.timezone
+import django_extensions.db.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('projects', '0081_add_another_header'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='WebHookEvent',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(choices=[('build:triggered', 'Build triggered'), ('build:passed', 'Build passed'), ('build:failed', 'Build failed')], max_length=256, unique=True)),
+            ],
+        ),
+        migrations.AddField(
+            model_name='webhook',
+            name='created',
+            field=django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, default=django.utils.timezone.now, verbose_name='created'),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='webhook',
+            name='modified',
+            field=django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified'),
+        ),
+        migrations.AddField(
+            model_name='webhook',
+            name='payload',
+            field=models.TextField(blank=True, help_text='JSON payload to send to the webhook. Check the docs for available substitutions.', null=True, verbose_name='JSON payload'),
+        ),
+        migrations.AddField(
+            model_name='webhook',
+            name='secret',
+            field=models.CharField(blank=True, help_text='Secret used to sign the payload of the webhook', max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='webhook',
+            name='url',
+            field=models.URLField(help_text='URL to send the webhook to', max_length=600, verbose_name='URL'),
+        ),
+        migrations.AddField(
+            model_name='webhook',
+            name='events',
+            field=models.ManyToManyField(help_text='Events to subscribe', related_name='webhooks', to='projects.WebHookEvent'),
+        ),
+    ]

readthedocs/projects/models.py

@@ -1,5 +1,7 @@
 """Project models."""
 import fnmatch
+import hashlib
+import hmac
 import logging
 import os
 import re
@@ -10,10 +12,12 @@
 from django.conf import settings
 from django.conf.urls import include
 from django.contrib.auth.models import User
+from django.contrib.contenttypes.fields import GenericRelation
 from django.core.validators import MaxValueValidator, MinValueValidator
 from django.db import models
 from django.db.models import Prefetch
 from django.urls import re_path, reverse
+from django.utils.crypto import get_random_string
 from django.utils.functional import cached_property
 from django.utils.translation import ugettext_lazy as _
 from django.views import defaults
@@ -1465,14 +1469,103 @@ def __str__(self):
         return self.email
 
 
-class WebHook(Notification):
+class WebHookEvent(models.Model):
+
+    BUILD_TRIGGERED = 'build:triggered'
+    BUILD_PASSED = 'build:passed'
+    BUILD_FAILED = 'build:failed'
+
+    EVENTS = (
+        (BUILD_TRIGGERED, _('Build triggered')),
+        (BUILD_PASSED, _('Build passed')),
+        (BUILD_FAILED, _('Build failed')),
+    )
+
+    name = models.CharField(
+        max_length=256,
+        unique=True,
+        choices=EVENTS,
+    )
+
+    def __str__(self):
+        return self.name
+
+
+class WebHook(Notification, TimeStampedModel):
+
     url = models.URLField(
+        _('URL'),
         max_length=600,
         help_text=_('URL to send the webhook to'),
     )
+    secret = models.CharField(
+        help_text=_('Secret used to sign the payload of the webhook'),
+        max_length=255,
+        blank=True,
+        null=True,
+    )
+    events = models.ManyToManyField(
+        WebHookEvent,
+        related_name='webhooks',
+        help_text=_('Events to subscribe'),
+    )
+    payload = models.TextField(
+        _('JSON payload'),
+        help_text=_(
+            'JSON payload to send to the webhook. '
+            'Check the docs for available substitutions.',
+        ),
+        blank=True,
+        null=True,
+    )
+    exchanges = GenericRelation(
+        'integrations.HttpExchange',
+        related_query_name='webhook',
+    )
+
+    SUBSTITUTIONS = (
+        ('${event}', _('Event that triggered the request')),
+        ('${build.id}', _('Build ID')),
+        ('${build.commit}', _('Commit being built')),
+        ('${project.slug}', _('Project slug')),
+        ('${project.name}', _('Project name')),
+        ('${version.id}', _('Version ID')),
+        ('${version.slug}', _('Version slug')),
+    )
+
+    def save(self, *args, **kwargs):
+        if not self.secret:
+            self.secret = get_random_string(length=32)
+        super().save(*args, **kwargs)
+
+    def get_payload(self, event, build):
+        project = build.project
+        version = build.version
+        substitutions = {
+            '${event}': event,
+            '${build.id}': build.id,
+            '${build.commit}': build.commit,
+            '${project.slug}': project.slug,
+            '${project.name}': project.name,
+            '${version.slug}': version.slug,
+        }
+        payload = self.payload
+        # Small protection for DDoS.
+        max_substitutions = 9
+        for substitution, value in substitutions.items():
+            payload = payload.replace(substitution, value, max_substitutions)
+        return payload
+
+    def sign_payload(self, payload):
+        digest = hmac.new(
+            self.secret.encode(),
+            msg=payload.encode(),
+            digestmod=hashlib.sha1,
+        )
+        return digest.hexdigest()
 
     def __str__(self):
-        return self.url
+        return f'{self.project.slug} {self.url}'
 
 
 class Domain(TimeStampedModel, models.Model):