diff --git a/docs/webhooks.rst b/docs/webhooks.rst index 91ccdffd65d..7ba21585e3c 100644 --- a/docs/webhooks.rst +++ b/docs/webhooks.rst @@ -88,8 +88,20 @@ Parameters This endpoint accepts the following arguments during an HTTP POST: branches - The names of the branches to trigger builds for. This can either be an array - of branch name strings, or just a single branch name string. + The names of the branches to trigger builds for. + This can either be: + + - An array of branch name strings + - Just a single branch name string + - Or an array of objects containing the following:: + + { + "name": "branch-name", + "last_commit": { + "id": "hash", + "message": "Update README" + } + } Default: **latest** @@ -118,6 +130,22 @@ token, a check will determine if the token is valid and matches the given project. If instead an authenticated user is used to make this request, a check will be performed to ensure the authenticated user is an owner of the project. +Skipping a build +---------------- + +When you push new changes to your remote repository, +you can skip the build process of your docs. +This is done by adding a mark anywhere on the commit message of the last commit, +the mark can be: + +- [skip docs] +- [docs skip] +- [skip doc] +- [doc skip] + +All integrations are supported, +for the generic API you need to provide the details of the last commit. + Debugging webhooks ------------------ diff --git a/readthedocs/core/views/hooks.py b/readthedocs/core/views/hooks.py index 9a306aaea73..4d3dc751ce6 100644 --- a/readthedocs/core/views/hooks.py +++ b/readthedocs/core/views/hooks.py @@ -66,6 +66,25 @@ def _build_version(project, slug, already_built=()): return None +def _contains_skip_mark(message): + """ + Check if a commit message has a skip mark + + For example: + - [skip docs] + - [docs skip] + """ + skip_marks = [ + r'\[skip docs?\]', + r'\[docs? skip\]', + ] + for skip_mark in skip_marks: + mark = re.compile(skip_mark, re.IGNORECASE) + if mark.search(message): + return True + return False + + def build_branches(project, branch_list): """ Build the branches for a specific project. @@ -77,7 +96,20 @@ def build_branches(project, branch_list): to_build = set() not_building = set() for branch in branch_list: - versions = project.versions_from_branch_name(branch) + commit = branch.get('last_commit') + versions = project.versions_from_branch_name(branch['name']) + to_build = set() + not_building = set() + if commit and _contains_skip_mark(commit['message']): + log.info( + '(Branch Build) Skip mark found. Skip build %s', + project.slug + ) + not_building = { + version.slug + for version in versions + } + return (to_build, not_building) for version in versions: log.info("(Branch Build) Processing %s:%s", project.slug, version.slug) @@ -181,7 +213,9 @@ def github_build(request): # noqa: D205 http_search_url = http_url.replace('http://', '').replace('https://', '') ssh_url = data['repository']['ssh_url'] ssh_search_url = ssh_url.replace('git@', '').replace('.git', '') - branches = [data['ref'].replace('refs/heads/', '')] + branches = [{ + 'name': data['ref'].replace('refs/heads/', ''), + }] except (ValueError, TypeError, KeyError): log.exception('Invalid GitHub webhook payload') return HttpResponse('Invalid request', status=400) @@ -226,7 +260,9 @@ def gitlab_build(request): # noqa: D205 data = json.loads(request.body) url = data['project']['http_url'] search_url = re.sub(r'^https?://(.*?)(?:\.git|)$', '\\1', url) - branches = [data['ref'].replace('refs/heads/', '')] + branches = [{ + 'name': data['ref'].replace('refs/heads/', ''), + }] except (ValueError, TypeError, KeyError): log.exception('Invalid GitLab webhook payload') return HttpResponse('Invalid request', status=400) @@ -275,16 +311,24 @@ def bitbucket_build(request): version = 2 if request.META.get('HTTP_USER_AGENT') == 'Bitbucket-Webhooks/2.0' else 1 if version == 1: - branches = [commit.get('branch', '') - for commit in data['commits']] + branches = [ + { + 'name': commit.get('branch', ''), + } + for commit in data['commits'] + ] repository = data['repository'] search_url = 'bitbucket.org{0}'.format( repository['absolute_url'].rstrip('/') ) elif version == 2: changes = data['push']['changes'] - branches = [change['new']['name'] - for change in changes] + branches = [ + { + 'name': change['new']['name'], + } + for change in changes + ] search_url = 'bitbucket.org/{0}'.format( data['repository']['full_name'] ) diff --git a/readthedocs/restapi/views/integrations.py b/readthedocs/restapi/views/integrations.py index ddc226cf472..a0085c279fb 100644 --- a/readthedocs/restapi/views/integrations.py +++ b/readthedocs/restapi/views/integrations.py @@ -137,8 +137,16 @@ class GitHubWebhookView(WebhookMixin, APIView): { "ref": "branch-name", + "head_commit": { + "id": "sha", + "message": "Update README.md", + ... + } ... } + + See full payload here: + https://developer.github.com/v3/activity/events/types/#pushevent """ integration_type = Integration.GITHUB_WEBHOOK @@ -159,7 +167,12 @@ def handle_webhook(self): # Handle push events and trigger builds if event == GITHUB_PUSH: try: - branches = [self.data['ref'].replace('refs/heads/', '')] + # GitHub only returns one branch. + branch = { + 'name': self.data['ref'].replace('refs/heads/', ''), + 'last_commit': self.data['head_commit'], + } + branches = [branch] return self.get_response_push(self.project, branches) except KeyError: raise ParseError('Parameter "ref" is required') @@ -177,8 +190,16 @@ class GitLabWebhookView(WebhookMixin, APIView): { "object_kind": "push", "ref": "branch-name", + "commits": [{ + "id": "sha", + "message": "Update README.md", + ... + }] ... } + + See full payload here: + https://docs.gitlab.com/ce/user/project/integrations/webhooks.html#push-events """ integration_type = Integration.GITLAB_WEBHOOK @@ -191,7 +212,13 @@ def handle_webhook(self): # Handle push events and trigger builds if event == GITLAB_PUSH: try: - branches = [self.request.data['ref'].replace('refs/heads/', '')] + # GitLab only returns one branch. + branch = { + 'name': self.request.data['ref'].replace('refs/heads/', ''), + # Assuming the first element is the last commit. + 'last_commit': self.request.data['commits'][0], + } + branches = [branch] return self.get_response_push(self.project, branches) except KeyError: raise ParseError('Parameter "ref" is required') @@ -211,6 +238,11 @@ class BitbucketWebhookView(WebhookMixin, APIView): "changes": [{ "new": { "name": "branch-name", + "target": { + "hash": "sha", + "message": "Update README.md", + ... + } ... }, ... @@ -219,6 +251,9 @@ class BitbucketWebhookView(WebhookMixin, APIView): }, ... } + + See full payload here: + https://confluence.atlassian.com/bitbucket/event-payloads-740262817.html#EventPayloads-Push """ integration_type = Integration.BITBUCKET_WEBHOOK @@ -232,13 +267,34 @@ def handle_webhook(self): if event == BITBUCKET_PUSH: try: changes = self.request.data['push']['changes'] - branches = [change['new']['name'] - for change in changes - if change.get('new')] + branches = [ + { + 'name': change['new']['name'], + 'last_commit': self._normalize_commit( + change['new']['target'] + ), + } + for change in changes + if change.get('new') + ] return self.get_response_push(self.project, branches) except KeyError: raise ParseError('Invalid request') + def _normalize_commit(self, commit): + """ + All commit dicts must have this elements at least:: + + { + 'id': 'sha', + 'message': 'Update README.md', + } + """ + return { + 'id': commit['hash'], + 'message': commit['message'], + } + class IsAuthenticatedOrHasToken(permissions.IsAuthenticated): @@ -265,6 +321,19 @@ class APIWebhookView(WebhookMixin, APIView): { "branches": ["master"] } + + Or the following JSON:: + + { + "branches": [{ + "name": "branch-name", + "last_commit": { + "id": "sha", + "message": "Update README.md" + } + }] + } + """ integration_type = Integration.API_WEBHOOK @@ -300,12 +369,20 @@ def get_project(self, **kwargs): def handle_webhook(self): try: - branches = self.request.data.get( + request_branches = self.request.data.get( 'branches', [self.project.get_default_branch()] ) - if isinstance(branches, six.string_types): - branches = [branches] + if isinstance(request_branches, six.string_types): + request_branches = [request_branches] + branches = [] + for branch in request_branches: + if isinstance(branch, six.string_types): + branches.append({ + 'name': branch, + }) + else: + branches.append(branch) return self.get_response_push(self.project, branches) except TypeError: raise ParseError('Invalid request') diff --git a/readthedocs/rtd_tests/tests/test_api.py b/readthedocs/rtd_tests/tests/test_api.py index 87e328e5037..a7f01dca25e 100644 --- a/readthedocs/rtd_tests/tests/test_api.py +++ b/readthedocs/rtd_tests/tests/test_api.py @@ -376,9 +376,16 @@ def setUp(self): def test_github_webhook(self, trigger_build): """GitHub webhook API.""" client = APIClient() + payload = { + 'ref': 'master', + 'head_commit': { + 'id': '3eea78b2', + 'message': 'Update README.md', + }, + } client.post( '/api/v2/webhook/github/{0}/'.format(self.project.slug), - {'ref': 'master'}, + payload, format='json', ) trigger_build.assert_has_calls( @@ -403,12 +410,56 @@ def test_github_invalid_webhook(self, trigger_build): self.assertEqual(resp.status_code, 200) self.assertEqual(resp.data['detail'], 'Unhandled webhook event') + def test_github_skip_mark(self, trigger_build): + client = APIClient() + payload = { + 'ref': 'master', + 'head_commit': { + 'id': '3eea78b2', + 'message': 'Update .gitignore [skip docs]', + }, + } + client.post( + '/api/v2/webhook/github/{0}/'.format(self.project.slug), + payload, + format='json', + ) + trigger_build.assert_not_called() + + payload['head_commit']['message'] = '[skip doc] Update version.py' + client.post( + '/api/v2/webhook/github/{0}/'.format(self.project.slug), + payload, + format='json', + ) + trigger_build.assert_not_called() + + payload['head_commit']['message'] = ''' +Update version.py + +[skip docs] + ''' + client.post( + '/api/v2/webhook/github/{0}/'.format(self.project.slug), + payload, + format='json', + ) + trigger_build.assert_not_called() + def test_gitlab_webhook(self, trigger_build): """GitLab webhook API.""" client = APIClient() + payload = { + 'object_kind': 'push', + 'ref': 'master', + 'commits': [{ + 'id': '3eea78b2', + 'message': 'Update README.md', + }], + } client.post( '/api/v2/webhook/gitlab/{0}/'.format(self.project.slug), - {'object_kind': 'push', 'ref': 'master'}, + payload, format='json', ) trigger_build.assert_has_calls( @@ -432,20 +483,62 @@ def test_gitlab_invalid_webhook(self, trigger_build): self.assertEqual(resp.status_code, 200) self.assertEqual(resp.data['detail'], 'Unhandled webhook event') + def test_gitlab_skip_mark(self, trigger_build): + client = APIClient() + payload = { + 'object_kind': 'push', + 'ref': 'master', + 'commits': [{ + 'id': '3eea78b2', + 'message': 'Update .gitignore [skip docs]', + }], + } + client.post( + '/api/v2/webhook/gitlab/{0}/'.format(self.project.slug), + payload, + format='json', + ) + trigger_build.assert_not_called() + + payload['commits'][0]['message'] = '[skip doc] Update version.py' + client.post( + '/api/v2/webhook/gitlab/{0}/'.format(self.project.slug), + payload, + format='json', + ) + trigger_build.assert_not_called() + + payload['commits'][0]['message'] = ''' +Update version.py + +[skip docs] + ''' + client.post( + '/api/v2/webhook/gitlab/{0}/'.format(self.project.slug), + payload, + format='json', + ) + trigger_build.assert_not_called() + def test_bitbucket_webhook(self, trigger_build): """Bitbucket webhook API.""" client = APIClient() - client.post( - '/api/v2/webhook/bitbucket/{0}/'.format(self.project.slug), - { - 'push': { - 'changes': [{ - 'new': { - 'name': 'master', + payload = { + 'push': { + 'changes': [{ + 'new': { + 'name': 'master', + 'target': { + 'hash': '3eea78b2', + 'message': 'Update README.md', }, - }], - }, + }, + }], }, + } + client.post( + '/api/v2/webhook/bitbucket/{0}/'.format(self.project.slug), + payload, format='json', ) trigger_build.assert_has_calls( @@ -491,6 +584,54 @@ def test_bitbucket_invalid_webhook(self, trigger_build): self.assertEqual(resp.status_code, 200) self.assertEqual(resp.data['detail'], 'Unhandled webhook event') + def test_bitbucket_skip_mark(self, trigger_build): + client = APIClient() + payload = { + 'push': { + 'changes': [{ + 'new': { + 'name': 'master', + 'target': { + 'hash': '3eea78b2', + 'message': 'Update README.md', + }, + }, + }], + }, + } + client.post( + '/api/v2/webhook/bitbucket/{0}/'.format(self.project.slug), + payload, + format='json', + ) + trigger_build.assert_called_with( + force=True, version=mock.ANY, project=self.project + ) + # Reset mock before test for skip build + trigger_build.reset_mock() + + payload['push']['changes'][0]['new']['target']['message'] = ( + '[skip doc] Update version.py' + ) + client.post( + '/api/v2/webhook/bitbucket/{0}/'.format(self.project.slug), + payload, + format='json', + ) + trigger_build.assert_not_called() + + payload['push']['changes'][0]['new']['target']['message'] = ''' +Update version.py + +[skip docs] + ''' + client.post( + '/api/v2/webhook/bitbucket/{0}/'.format(self.project.slug), + payload, + format='json', + ) + trigger_build.assert_not_called() + def test_generic_api_fails_without_auth(self, trigger_build): client = APIClient() resp = client.post( @@ -533,6 +674,107 @@ def test_generic_api_respects_token_auth(self, trigger_build): self.assertEqual(resp.status_code, 200) self.assertFalse(resp.data['build_triggered']) + def test_generic_api_payload_string_list(self, trigger_build): + client = APIClient() + integration = Integration.objects.create( + project=self.project, + integration_type=Integration.API_WEBHOOK, + ) + resp = client.post( + '/api/v2/webhook/{0}/{1}/'.format( + self.project.slug, + integration.pk, + ), + {'token': integration.token, 'branches': ['master']}, + format='json', + ) + self.assertEqual(resp.status_code, 200) + self.assertTrue(resp.data['build_triggered']) + + def test_generic_api_payload_dict_list(self, trigger_build): + client = APIClient() + integration = Integration.objects.create( + project=self.project, + integration_type=Integration.API_WEBHOOK, + ) + resp = client.post( + '/api/v2/webhook/{0}/{1}/'.format( + self.project.slug, + integration.pk, + ), + { + 'token': integration.token, + 'branches': [{ + 'name': 'master', + 'last_commit': { + 'id': '3eea78b2', + "message": "Update README.md" + } + }] + }, + format='json', + ) + self.assertEqual(resp.status_code, 200) + self.assertTrue(resp.data['build_triggered']) + + def test_generic_api_skip_mark(self, trigger_build): + client = APIClient() + integration = Integration.objects.create( + project=self.project, + integration_type=Integration.API_WEBHOOK, + ) + payload = { + 'token': integration.token, + 'branches': [{ + 'name': 'master', + 'last_commit': { + 'id': '3eea78b2', + "message": "Update README.md" + } + }] + } + client.post( + '/api/v2/webhook/{0}/{1}/'.format( + self.project.slug, + integration.pk, + ), + payload, + format='json', + ) + trigger_build.assert_called_with( + force=True, version=mock.ANY, project=self.project + ) + # Reset mock before test for skip build + trigger_build.reset_mock() + + payload['branches'][0]['last_commit']['message'] = ( + '[skip doc] Update version.py' + ) + client.post( + '/api/v2/webhook/{0}/{1}/'.format( + self.project.slug, + integration.pk, + ), + payload, + format='json', + ) + trigger_build.assert_not_called() + + payload['branches'][0]['last_commit']['message'] = ''' +Update version.py + +[skip docs] + ''' + client.post( + '/api/v2/webhook/{0}/{1}/'.format( + self.project.slug, + integration.pk, + ), + payload, + format='json', + ) + trigger_build.assert_not_called() + def test_generic_api_respects_basic_auth(self, trigger_build): client = APIClient() user = get(User) diff --git a/readthedocs/rtd_tests/tests/test_integrations.py b/readthedocs/rtd_tests/tests/test_integrations.py index 51006b7e757..7ec50bcf1f7 100644 --- a/readthedocs/rtd_tests/tests/test_integrations.py +++ b/readthedocs/rtd_tests/tests/test_integrations.py @@ -1,16 +1,15 @@ -from __future__ import absolute_import +from __future__ import ( + absolute_import, division, print_function, unicode_literals) from builtins import range + import django_dynamic_fixture as fixture -from django.test import TestCase, RequestFactory -from django.contrib.contenttypes.models import ContentType +from django.test import TestCase +from requests.utils import quote from rest_framework.test import APIClient -from rest_framework.test import APIRequestFactory -from rest_framework.response import Response from readthedocs.integrations.models import ( - HttpExchange, Integration, GitHubWebhook -) + GitHubWebhook, HttpExchange, Integration) from readthedocs.projects.models import Project @@ -29,15 +28,24 @@ def test_exchange_json_request_body(self): integration = fixture.get(Integration, project=project, integration_type=Integration.GITHUB_WEBHOOK, provider_data='') - resp = client.post( + payload_dict = { + 'head_commit': { + 'id': '3eea78b2', + 'message': 'Update README.md', + }, + 'ref': 'exchange_json', + } + payload_json = '{"head_commit": {"id": "3eea78b2", "message": "Update README.md"}, "ref": "exchange_json"}' + + client.post( '/api/v2/webhook/github/{0}/'.format(project.slug), - {'ref': 'exchange_json'}, + payload_dict, format='json' ) exchange = HttpExchange.objects.get(integrations=integration) self.assertEqual( exchange.request_body, - '{"ref": "exchange_json"}' + payload_json ) self.assertEqual( exchange.request_headers, @@ -62,15 +70,17 @@ def test_exchange_form_request_body(self): integration = fixture.get(Integration, project=project, integration_type=Integration.GITHUB_WEBHOOK, provider_data='') - resp = client.post( + payload_decode = '{"head_commit": {"id": "3eea78b2", "message": "Update README.md"}, "ref": "exchange_form"}' + payload_encode = quote(payload_decode) + client.post( '/api/v2/webhook/github/{0}/'.format(project.slug), - 'payload=%7B%22ref%22%3A+%22exchange_form%22%7D', + 'payload={}'.format(payload_encode), content_type='application/x-www-form-urlencoded', ) exchange = HttpExchange.objects.get(integrations=integration) self.assertEqual( exchange.request_body, - '{"ref": "exchange_form"}' + payload_decode ) self.assertEqual( exchange.request_headers, diff --git a/readthedocs/rtd_tests/tests/test_post_commit_hooks.py b/readthedocs/rtd_tests/tests/test_post_commit_hooks.py index fe781e16bd8..514605b17ff 100644 --- a/readthedocs/rtd_tests/tests/test_post_commit_hooks.py +++ b/readthedocs/rtd_tests/tests/test_post_commit_hooks.py @@ -56,23 +56,23 @@ def setUp(self): "user_name": "John Smith", "user_email": "john@example.com", "project_id": 15, - "project":{ - "name":"readthedocs", - "description":"", - "web_url":"http://example.com/mike/diaspora", + "project": { + "name": "readthedocs", + "description": "", + "web_url": "http://example.com/mike/diaspora", "avatar_url": None, - "git_ssh_url":"git@github.com:rtfd/readthedocs.org.git", - "git_http_url":"http://github.com/rtfd/readthedocs.org.git", - "namespace":"Mike", - "visibility_level":0, - "path_with_namespace":"mike/diaspora", - "default_branch":"master", - "homepage":"http://example.com/mike/diaspora", - "url":"git@github.com/rtfd/readthedocs.org.git", - "ssh_url":"git@github.com/rtfd/readthedocs.org.git", - "http_url":"http://github.com/rtfd/readthedocs.org.git" + "git_ssh_url": "git@github.com:rtfd/readthedocs.org.git", + "git_http_url": "http://github.com/rtfd/readthedocs.org.git", + "namespace": "Mike", + "visibility_level": 0, + "path_with_namespace": "mike/diaspora", + "default_branch": "master", + "homepage": "http://example.com/mike/diaspora", + "url": "git@github.com/rtfd/readthedocs.org.git", + "ssh_url": "git@github.com/rtfd/readthedocs.org.git", + "http_url": "http://github.com/rtfd/readthedocs.org.git" }, - "repository":{ + "repository": { "name": "Diaspora", "url": "git@github.com:rtfd/readthedocs.org.git", "description": "", @@ -194,6 +194,24 @@ def setUp(self): "commit/11f229c6a78f5bc8cb173104a3f7a68cdb7eb15a") }, ], + "head_commit": { + "added": [], + "author": { + "email": "eric@ericholscher.com", + "name": "Eric Holscher", + "username": "ericholscher" + }, + "distinct": False, + "id": "11f229c6a78f5bc8cb173104a3f7a68cdb7eb15a", + "message": "Fix it on the front list as well.", + "modified": [ + "readthedocs/templates/core/project_list_detailed.html" + ], + "removed": [], + "timestamp": "2011-09-12T19:38:55-07:00", + "url": ("https://github.com/wraithan/readthedocs.org/" + "commit/11f229c6a78f5bc8cb173104a3f7a68cdb7eb15a") + }, "compare": ("https://github.com/wraithan/readthedocs.org/compare/" "5b4e453...5ad7573"), "created": False,