diff --git a/readthedocs/projects/migrations/0026_auto__add_webhook.py b/readthedocs/projects/migrations/0026_auto__add_webhook.py new file mode 100644 index 00000000000..74dd580fb2e --- /dev/null +++ b/readthedocs/projects/migrations/0026_auto__add_webhook.py @@ -0,0 +1,159 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'WebHook' + db.create_table('projects_webhook', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('project', self.gf('django.db.models.fields.related.ForeignKey')(related_name='webhook_notifications', to=orm['projects.Project'])), + ('url', self.gf('django.db.models.fields.URLField')(max_length=200, blank=True)), + )) + db.send_create_signal('projects', ['WebHook']) + # Adding model 'EmailHook' + db.create_table('projects_emailhook', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('project', self.gf('django.db.models.fields.related.ForeignKey')(related_name='emailhook_notifications', to=orm['projects.Project'])), + ('email', self.gf('django.db.models.fields.EmailField')(max_length=75)), + )) + db.send_create_signal('projects', ['EmailHook']) + + + def backwards(self, orm): + # Deleting model 'WebHook' + db.delete_table('projects_webhook') + # Deleting model 'EmailHook' + db.delete_table('projects_emailhook') + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'projects.file': { + 'Meta': {'ordering': "('denormalized_path',)", 'object_name': 'File'}, + 'content': ('django.db.models.fields.TextField', [], {}), + 'denormalized_path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'heading': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ordering': ('django.db.models.fields.PositiveSmallIntegerField', [], {'default': '1'}), + 'parent': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'children'", 'null': 'True', 'to': "orm['projects.File']"}), + 'project': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'files'", 'to': "orm['projects.Project']"}), + 'slug': ('django.db.models.fields.SlugField', [], {'max_length': '50'}), + 'status': ('django.db.models.fields.PositiveSmallIntegerField', [], {'default': '1'}) + }, + 'projects.filerevision': { + 'Meta': {'ordering': "('-revision_number',)", 'object_name': 'FileRevision'}, + 'comment': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'created_date': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'diff': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'file': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'revisions'", 'to': "orm['projects.File']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_reverted': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'revision_number': ('django.db.models.fields.IntegerField', [], {}) + }, + 'projects.importedfile': { + 'Meta': {'object_name': 'ImportedFile'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'md5': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'project': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'imported_files'", 'to': "orm['projects.Project']"}), + 'slug': ('django.db.models.fields.SlugField', [], {'max_length': '50'}) + }, + 'projects.project': { + 'Meta': {'ordering': "('slug',)", 'object_name': 'Project'}, + 'analytics_code': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'conf_py_file': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'blank': 'True'}), + 'copyright': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}), + 'crate_url': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}), + 'default_branch': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'default_version': ('django.db.models.fields.CharField', [], {'default': "'latest'", 'max_length': '255'}), + 'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'django_packages_url': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}), + 'documentation_type': ('django.db.models.fields.CharField', [], {'default': "'sphinx'", 'max_length': '20'}), + 'featured': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'modified_date': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'project_url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}), + 'pub_date': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'related_projects': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': "orm['projects.Project']", 'null': 'True', 'through': "orm['projects.ProjectRelationship']", 'blank': 'True'}), + 'repo': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}), + 'repo_type': ('django.db.models.fields.CharField', [], {'default': "'git'", 'max_length': '10'}), + 'requirements_file': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'skip': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '255'}), + 'suffix': ('django.db.models.fields.CharField', [], {'default': "'.rst'", 'max_length': '10'}), + 'theme': ('django.db.models.fields.CharField', [], {'default': "'default'", 'max_length': '20'}), + 'use_virtualenv': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'users': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'projects'", 'symmetrical': 'False', 'to': "orm['auth.User']"}), + 'version': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}) + }, + 'projects.projectrelationship': { + 'Meta': {'object_name': 'ProjectRelationship'}, + 'child': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'superprojects'", 'to': "orm['projects.Project']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'parent': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'subprojects'", 'to': "orm['projects.Project']"}) + }, + 'projects.webhook': { + 'Meta': {'object_name': 'WebHook'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'project': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'webhook_notifications'", 'to': "orm['projects.Project']"}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}) + }, + 'taggit.tag': { + 'Meta': {'object_name': 'Tag'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'slug': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '100'}) + }, + 'taggit.taggeditem': { + 'Meta': {'object_name': 'TaggedItem'}, + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'taggit_taggeditem_tagged_items'", 'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'object_id': ('django.db.models.fields.IntegerField', [], {'db_index': 'True'}), + 'tag': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'taggit_taggeditem_items'", 'to': "orm['taggit.Tag']"}) + } + } + + complete_apps = ['projects'] diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index 35be28a0dd8..4bbd3ab5a3f 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -632,3 +632,20 @@ def get_absolute_url(self): def __unicode__(self): return '%s: %s' % (self.name, self.project) + + +class Notification(models.Model): + project = models.ForeignKey(Project, + related_name='%(class)s_notifications') + + class Meta: + abstract = True + + +class EmailHook(Notification): + email = models.EmailField() + + +class WebHook(Notification): + url = models.URLField(blank=True, verify_exists=False, + help_text='URL to send the webhook to') diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index 9f90e7f5f71..3edc3d7d8c0 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -1,7 +1,6 @@ """Tasks related to projects, including fetching repository code, cleaning ``conf.py`` files, and rebuilding documentation. """ -import decimal import fnmatch import os import re @@ -13,24 +12,22 @@ from celery.decorators import task from django.db import transaction from django.conf import settings +from django.contrib.sites.models import Site +from django.core.mail import send_mail +from django.template import Context +from django.template.loader import get_template import redis +import requests from sphinx.ext import intersphinx import slumber - -from builds.models import Version, Build +from builds.models import Build, Version from doc_builder import loading as builder_loading from doc_builder.base import restoring_chdir from projects.exceptions import ProjectImportError from projects.models import ImportedFile, Project -from projects.utils import ( - DictObj, - mkversion, - purge_version, - run, - safe_write, - slugify_uniquely, - ) +from projects.utils import (mkversion, purge_version, run, safe_write, + slugify_uniquely,) from tastyapi import client as tastyapi_client from tastyapi import api from core.utils import copy_to_app_servers, run_on_app_servers @@ -40,6 +37,7 @@ log = logging.getLogger(__name__) + @task def remove_dir(path): """ @@ -58,17 +56,17 @@ def update_docs(pk, record=True, pdf=True, man=True, epub=True, version_pk=None, """ The main entry point for updating documentation. - It handles all of the logic around whether a project is imported or we created it. - Then it will build the html docs and other requested parts. - It also handles clearing the varnish cache. + It handles all of the logic around whether a project is imported or we + created it. Then it will build the html docs and other requested parts. It + also handles clearing the varnish cache. `pk` Primary key of the project to update `record` Whether or not to keep a record of the update in the database. Useful - for preventing changes visible to the end-user when running commands from - the shell, for example. + for preventing changes visible to the end-user when running commands + from the shell, for example. """ ### @@ -95,23 +93,23 @@ def new_save(*args, **kwargs): #Create or use the 'latest' branch, which is the default for a project. branch = project.default_branch or project.vcs_repo().fallback_branch try: - version_data = api.version(project.slug).get(slug='latest')['objects'][0] + version_data = (api.version(project.slug) + .get(slug='latest')['objects'][0]) del version_data['resource_uri'] - except (slumber.exceptions.HttpClientError, IndexError) as exc: - #if exc.response.status_code in [404,500]: + except (slumber.exceptions.HttpClientError, IndexError): version_data = dict( project='/api/v1/project/%s/' % project.pk, slug='latest', active=True, verbose_name='latest', identifier=branch, - ) + ) try: version_data = api.version.post(version_data) del version_data['resource_uri'] except Exception as e: log.info("Exception in creating version: %s" % e) - #raise e + version_data['project'] = project version = Version(**version_data) version.save = new_save @@ -129,14 +127,15 @@ def new_save(*args, **kwargs): version_data['identifier'] = branch to_save = True if to_save: - version_data['project'] = "/api/v1/version/%s/" % version_data['project'].pk + version_data['project'] = ("/api/v1/version/%s/" % + version_data['project'].pk) api.version(version.pk).put(version_data) if record: #Create Build Object. build = api.build.post(dict( - project= '/api/v1/project/%s/' % project.pk, - version= '/api/v1/version/%s/' % version.pk, + project='/api/v1/project/%s/' % project.pk, + version='/api/v1/version/%s/' % version.pk, type='html', state='triggered', )) @@ -164,9 +163,10 @@ def new_save(*args, **kwargs): if record: build['state'] = 'building' api.build(build['id']).put(build) - (ret, out, err) = build_docs(project=project, build=build, version=version, - pdf=pdf, man=man, epub=epub, - record=record, force=force, update_output=update_output) + (ret, out, err) = build_docs(project=project, build=build, + version=version, pdf=pdf, man=man, + epub=epub, record=record, force=force, + update_output=update_output) if not 'no targets are out of date.' in out: if ret == 0: log.info("Successful HTML Build") @@ -174,7 +174,7 @@ def new_save(*args, **kwargs): mainsite=True, cname=True) symlink_cname(version) update_intersphinx(version.pk) - send_notifications(version) + send_notifications(version, build) log.info("Purged %s" % version) else: log.warning("Failed HTML Build") @@ -227,7 +227,7 @@ def update_imported_docs(project, version): update_docs_output['checkout'] = version_repo.update() # Ensure we have a conf file (an exception is raised if not) - conf_file = project.conf_file(version.slug) + project.conf_file(version.slug) #Do Virtualenv bits: if project.use_virtualenv: @@ -281,7 +281,7 @@ def update_imported_docs(project, version): if highest and ver_obj and ver_obj > highest: log.info("Highest verison known, building docs") update_docs.delay(ver.project.pk, version_pk=ver.pk) - except Exception, e: + except Exception: log.error("Failed to create version (tag)", exc_info=True) transaction.rollback() #break here to stop updating tags when they will all fail. @@ -303,14 +303,14 @@ def update_imported_docs(project, version): verbose_name=branch.verbose_name )) log.info("New branch found: {0}".format(branch.identifier)) - except Exception, e: + except Exception: log.error("Failed to create version (branch)", exc_info=True) transaction.rollback() #break here to stop updating branches when they will all fail. break transaction.leave_transaction_management() #TODO: Kill deleted branches - except ValueError, e: + except ValueError: log.error("Error getting tags", exc_info=True) #TODO: Find a better way to handle indexing. @@ -363,7 +363,7 @@ def build_docs(project, build, version, pdf, man, epub, record, force, update_ou del version_data['project'] try: api.version(version.pk).put(version_data) - except Exception, e: + except Exception: log.error("Unable to post a new version", exc_info=True) if html_builder.changed: @@ -450,7 +450,7 @@ def update_docs_pull(record=False, pdf=False, man=False, force=False): try: update_docs( pk=project.pk, record=record, pdf=pdf, man=man, force=force) - except Exception, e: + except Exception: log.error("update_docs_pull failed", exc_info=True) @@ -479,7 +479,7 @@ def update_intersphinx(version_pk): try: object_file = version.project.find('objects.inv', version.slug)[0] - except IndexError, e: + except IndexError: print "Failed to find objects file" return None @@ -534,7 +534,54 @@ def symlink_cname(version): run_on_app_servers('ln -nsf %s %s' % (build_dir, symlink)) -def send_notifications(version): +def send_notifications(version, build): + zenircbot_notification(version.id) + for hook in version.project.webhook_notifications.all(): + webhook_notification.delay(version.project.id, build, hook.url) + emails = version.project.emailhook_notifications.all().values_list('email', + flat=True) + for email in emails: + email_notification(version.project.id, build, email) + + +@task +def email_notification(project_id, build, email): + if build['success']: + return + project = Project.objects.get(id=project_id) + build_obj = Build.objects.get(id=build['id']) + subject = ('(ReadTheDocs) Building docs for %s failed' % project.name) + template = 'projects/notification_email.txt' + context = { + 'project': project.name, + 'build_url': 'http://%s%s' % (Site.objects.get_current().domain, + build_obj.get_absolute_url()) + } + message = get_template(template).render(Context(context)) + + send_mail(subject=subject, message=message, + from_email=settings.DEFAULT_FROM_EMAIL, recipient_list=(email,)) + + +@task +def webhook_notification(project_id, build, hook_url): + project = Project.objects.get(id=project_id) + data = json.dumps({ + 'name': project.name, + 'slug': project.slug, + 'build': { + 'id': build['id'], + 'success': build['success'], + 'date': build['date'] + } + }) + log.debug('sending notification to: %s' % hook_url) + requests.post(hook_url, data=data) + + +@task +def zenircbot_notification(version_id): + version = Version.objects.get(id=version_id) message = "Build of %s successful" % version redis_obj = redis.Redis(**settings.REDIS) IRC = getattr(settings, 'IRC_CHANNEL', '#readthedocs') diff --git a/readthedocs/settings/sqlite.py b/readthedocs/settings/sqlite.py index 937ca490a24..bf8502fe6ab 100644 --- a/readthedocs/settings/sqlite.py +++ b/readthedocs/settings/sqlite.py @@ -37,6 +37,9 @@ IMPORT_EXTERNAL_DATA = False +CELERY_ALWAYS_EAGER = True +EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' + try: from local_settings import * except: diff --git a/readthedocs/templates/projects/notification_email.txt b/readthedocs/templates/projects/notification_email.txt new file mode 100644 index 00000000000..47ea735b9b1 --- /dev/null +++ b/readthedocs/templates/projects/notification_email.txt @@ -0,0 +1,10 @@ +Hello! + +Unfortunately, we have to inform you that your docs have failed to build. You +can see what went wrong at: {{ build_url }} + +If you have questions, a good place to start is the FAQ: +http://read-the-docs.readthedocs.org/en/latest/faq.html + +Thanks, +The happy ReadTheDocs servers