Skip to content

Allow to skip a build on commit messages #3457

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 24 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 30 additions & 2 deletions docs/webhooks.rst
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,20 @@ Parameters
This endpoint accepts the following arguments during an HTTP POST:

branches
The names of the branches to trigger builds for. This can either be an array
of branch name strings, or just a single branch name string.
The names of the branches to trigger builds for.
This can either be:

- An array of branch name strings
- Just a single branch name string
- Or an array of objects containing the following::

{
"name": "branch-name",
"last_commit": {
"id": "hash",
"message": "Update README"
}
}

Default: **latest**

Expand Down Expand Up @@ -118,6 +130,22 @@ token, a check will determine if the token is valid and matches the given
project. If instead an authenticated user is used to make this request, a check
will be performed to ensure the authenticated user is an owner of the project.

Skipping a build
----------------

When you push new changes to your remote repository,
you can skip the build process of your docs.
This is done by adding a mark anywhere on the commit message of the last commit,
the mark can be:

- [skip docs]
- [docs skip]
- [skip doc]
- [doc skip]

All integrations are supported,
for the generic API you need to provide the details of the last commit.

Debugging webhooks
------------------

Expand Down
58 changes: 51 additions & 7 deletions readthedocs/core/views/hooks.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,25 @@ def _build_version(project, slug, already_built=()):
return None


def _contains_skip_mark(message):
"""
Check if a commit message has a skip mark

For example:
- [skip docs]
- [docs skip]
"""
skip_marks = [
r'\[skip docs?\]',
r'\[docs? skip\]',
]
for skip_mark in skip_marks:
mark = re.compile(skip_mark, re.IGNORECASE)
if mark.search(message):
return True
return False


def build_branches(project, branch_list):
"""
Build the branches for a specific project.
Expand All @@ -77,7 +96,20 @@ def build_branches(project, branch_list):
to_build = set()
not_building = set()
for branch in branch_list:
versions = project.versions_from_branch_name(branch)
commit = branch.get('last_commit')
versions = project.versions_from_branch_name(branch['name'])
to_build = set()
not_building = set()
if commit and _contains_skip_mark(commit['message']):
log.info(
'(Branch Build) Skip mark found. Skip build %s',
project.slug
)
not_building = {
version.slug
for version in versions
}
return (to_build, not_building)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Here is the logic for skipping the build, not sure if this is the correct place

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So, if the last commit contains the mark we don't build any version?

I'm confused here since the next few lines there is a logic that can build some version but not some others comming in the same webhook, right? Shouldn't we apply a similar logic here also?

That to_build and not_building rely on the function _build_version. So, this logic shouldn't be in that method? (I don't know, but is has the logging integrated and returns what it needs. On the other hand, it seems that we shouldn't call that method if we already know that we need to skip the version but there are some logic to skip versions inside that function too :/

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm confused here since the next few lines there is a logic that can build some version but not some others comming in the same webhook, right? Shouldn't we apply a similar logic here also?

Of what I understand, a branch/tag/commit can be tied to various versions so there is only one commit message for all versions, this means that all versions from this branch aren't going to be built. And also is about what you mention, we already know if the version isn't going to be built.

for version in versions:
log.info("(Branch Build) Processing %s:%s",
project.slug, version.slug)
Expand Down Expand Up @@ -181,7 +213,9 @@ def github_build(request): # noqa: D205
http_search_url = http_url.replace('http://', '').replace('https://', '')
ssh_url = data['repository']['ssh_url']
ssh_search_url = ssh_url.replace('git@', '').replace('.git', '')
branches = [data['ref'].replace('refs/heads/', '')]
branches = [{
'name': data['ref'].replace('refs/heads/', ''),
}]
except (ValueError, TypeError, KeyError):
log.exception('Invalid GitHub webhook payload')
return HttpResponse('Invalid request', status=400)
Expand Down Expand Up @@ -226,7 +260,9 @@ def gitlab_build(request): # noqa: D205
data = json.loads(request.body)
url = data['project']['http_url']
search_url = re.sub(r'^https?://(.*?)(?:\.git|)$', '\\1', url)
branches = [data['ref'].replace('refs/heads/', '')]
branches = [{
'name': data['ref'].replace('refs/heads/', ''),
}]
except (ValueError, TypeError, KeyError):
log.exception('Invalid GitLab webhook payload')
return HttpResponse('Invalid request', status=400)
Expand Down Expand Up @@ -275,16 +311,24 @@ def bitbucket_build(request):

version = 2 if request.META.get('HTTP_USER_AGENT') == 'Bitbucket-Webhooks/2.0' else 1
if version == 1:
branches = [commit.get('branch', '')
for commit in data['commits']]
branches = [
{
'name': commit.get('branch', ''),
}
for commit in data['commits']
]
repository = data['repository']
search_url = 'bitbucket.org{0}'.format(
repository['absolute_url'].rstrip('/')
)
elif version == 2:
changes = data['push']['changes']
branches = [change['new']['name']
for change in changes]
branches = [
{
'name': change['new']['name'],
}
for change in changes
]
search_url = 'bitbucket.org/{0}'.format(
data['repository']['full_name']
)
Expand Down
93 changes: 85 additions & 8 deletions readthedocs/restapi/views/integrations.py
Original file line number Diff line number Diff line change
Expand Up @@ -137,8 +137,16 @@ class GitHubWebhookView(WebhookMixin, APIView):

{
"ref": "branch-name",
"head_commit": {
"id": "sha",
"message": "Update README.md",
...
}
...
}

See full payload here:
https://developer.github.com/v3/activity/events/types/#pushevent
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are we using the v3 of the gh api?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It seems, https://developer.github.com/v3/#current-version, since we are not sending the Accept header and v3 is the default.

Copy link
Member Author

@stsewd stsewd Feb 18, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

And also v4 is using GraphQL, this will break everything.

"""

integration_type = Integration.GITHUB_WEBHOOK
Expand All @@ -159,7 +167,12 @@ def handle_webhook(self):
# Handle push events and trigger builds
if event == GITHUB_PUSH:
try:
branches = [self.data['ref'].replace('refs/heads/', '')]
# GitHub only returns one branch.
branch = {
'name': self.data['ref'].replace('refs/heads/', ''),
'last_commit': self.data['head_commit'],
}
branches = [branch]
return self.get_response_push(self.project, branches)
except KeyError:
raise ParseError('Parameter "ref" is required')
Expand All @@ -177,8 +190,16 @@ class GitLabWebhookView(WebhookMixin, APIView):
{
"object_kind": "push",
"ref": "branch-name",
"commits": [{
"id": "sha",
"message": "Update README.md",
...
}]
...
}

See full payload here:
https://docs.gitlab.com/ce/user/project/integrations/webhooks.html#push-events
"""

integration_type = Integration.GITLAB_WEBHOOK
Expand All @@ -191,7 +212,13 @@ def handle_webhook(self):
# Handle push events and trigger builds
if event == GITLAB_PUSH:
try:
branches = [self.request.data['ref'].replace('refs/heads/', '')]
# GitLab only returns one branch.
branch = {
'name': self.request.data['ref'].replace('refs/heads/', ''),
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Won't this interferes with your other PR at #3546?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just when merging/rebasing, on the logic everything would work as expected :)

# Assuming the first element is the last commit.
'last_commit': self.request.data['commits'][0],
}
branches = [branch]
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Another thought here... If we already know at this point the last commit message, shouldn't skip here the call to self.get_response_push instead of going deeper in the path?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We need the (buid, not_building) from the build_branches function to make the response.

return self.get_response_push(self.project, branches)
except KeyError:
raise ParseError('Parameter "ref" is required')
Expand All @@ -211,6 +238,11 @@ class BitbucketWebhookView(WebhookMixin, APIView):
"changes": [{
"new": {
"name": "branch-name",
"target": {
"hash": "sha",
"message": "Update README.md",
...
}
...
},
...
Expand All @@ -219,6 +251,9 @@ class BitbucketWebhookView(WebhookMixin, APIView):
},
...
}

See full payload here:
https://confluence.atlassian.com/bitbucket/event-payloads-740262817.html#EventPayloads-Push
"""

integration_type = Integration.BITBUCKET_WEBHOOK
Expand All @@ -232,13 +267,34 @@ def handle_webhook(self):
if event == BITBUCKET_PUSH:
try:
changes = self.request.data['push']['changes']
branches = [change['new']['name']
for change in changes
if change.get('new')]
branches = [
{
'name': change['new']['name'],
'last_commit': self._normalize_commit(
change['new']['target']
),
}
for change in changes
if change.get('new')
]
return self.get_response_push(self.project, branches)
except KeyError:
raise ParseError('Invalid request')

def _normalize_commit(self, commit):
"""
All commit dicts must have this elements at least::

{
'id': 'sha',
'message': 'Update README.md',
}
"""
return {
'id': commit['hash'],
'message': commit['message'],
}


class IsAuthenticatedOrHasToken(permissions.IsAuthenticated):

Expand All @@ -265,6 +321,19 @@ class APIWebhookView(WebhookMixin, APIView):
{
"branches": ["master"]
}

Or the following JSON::

{
"branches": [{
"name": "branch-name",
"last_commit": {
"id": "sha",
"message": "Update README.md"
}
}]
}

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is what I proposed for the general webhook, maintaining the backwards compatibility.

"""

integration_type = Integration.API_WEBHOOK
Expand Down Expand Up @@ -300,12 +369,20 @@ def get_project(self, **kwargs):

def handle_webhook(self):
try:
branches = self.request.data.get(
request_branches = self.request.data.get(
'branches',
[self.project.get_default_branch()]
)
if isinstance(branches, six.string_types):
branches = [branches]
if isinstance(request_branches, six.string_types):
request_branches = [request_branches]
branches = []
for branch in request_branches:
if isinstance(branch, six.string_types):
branches.append({
'name': branch,
})
else:
branches.append(branch)
return self.get_response_push(self.project, branches)
except TypeError:
raise ParseError('Invalid request')
Expand Down
Loading