Skip to content

Remove old/deprecated build endpoints #5479

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jun 10, 2019
Merged

Remove old/deprecated build endpoints #5479

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 1 addition & 15 deletions readthedocs/core/urls/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,11 @@
# -*- coding: utf-8 -*-

"""URL configuration for core app."""

from __future__ import absolute_import
from django.conf.urls import url

from readthedocs.constants import pattern_opts
from readthedocs.core import views
from readthedocs.core.views import hooks, serve
from readthedocs.core.views import serve
from readthedocs.projects.feeds import LatestProjectsFeed, NewProjectsFeed

docs_urls = [
Expand Down Expand Up @@ -43,18 +41,6 @@
]

core_urls = [
# Hooks
url(r'^github', hooks.github_build, name='github_build'),
url(r'^gitlab', hooks.gitlab_build, name='gitlab_build'),
url(r'^bitbucket', hooks.bitbucket_build, name='bitbucket_build'),
url(
(
r'^build/'
r'(?P<project_id_or_slug>{project_slug})'.format(**pattern_opts)
),
hooks.generic_build,
name='generic_build',
),
# Random other stuff
url(
r'^random/(?P<project_slug>{project_slug})'.format(**pattern_opts),
Expand Down
4 changes: 1 addition & 3 deletions readthedocs/core/views/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
# -*- coding: utf-8 -*-

"""
Core views, including the main homepage,

Expand Down Expand Up @@ -87,7 +85,7 @@ def wipe_version(request, project_slug, version_slug):
if request.method == 'POST':
wipe_version_via_slugs(
version_slug=version_slug,
project_slug=project_slug
project_slug=project_slug,
)
return redirect('project_version_list', project_slug)
return render(
Expand Down
298 changes: 0 additions & 298 deletions readthedocs/core/views/hooks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,31 +1,14 @@
"""Views pertaining to builds."""

import json
import logging
import re

from django.http import HttpResponse, HttpResponseNotFound
from django.shortcuts import redirect
from django.views.decorators.csrf import csrf_exempt

from readthedocs.builds.constants import LATEST
from readthedocs.core.utils import trigger_build
from readthedocs.projects import constants
from readthedocs.projects.models import Feature, Project
from readthedocs.projects.tasks import sync_repository_task


log = logging.getLogger(__name__)


class NoProjectException(Exception):
pass


def _allow_deprecated_webhook(project):
return project.has_feature(Feature.ALLOW_DEPRECATED_WEBHOOKS)


def _build_version(project, slug, already_built=()):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can move this to another file later I guess

"""
Where we actually trigger builds for a project and slug.
Expand Down Expand Up @@ -105,284 +88,3 @@ def sync_versions(project):
except Exception:
log.exception('Unknown sync versions exception')
return None


def get_project_from_url(url):
if not url:
return Project.objects.none()
projects = (
Project.objects.filter(repo__iendswith=url) |
Project.objects.filter(repo__iendswith=url + '.git')
)
return projects


def log_info(project, msg):
log.info(
constants.LOG_TEMPLATE,
{
'project': project,
'version': '',
'msg': msg,
}
)


def _build_url(url, projects, branches):
"""
Map a URL onto specific projects to build that are linked to that URL.

Check each of the ``branches`` to see if they are active and should be
built.
"""
ret = ''
all_built = {}
all_not_building = {}

# This endpoint doesn't require authorization, we shouldn't allow builds to
# be triggered from this any longer. Deprecation plan is to selectively
# allow access to this endpoint for now.
if not any(_allow_deprecated_webhook(project) for project in projects):
return HttpResponse('This API endpoint is deprecated', status=403)

for project in projects:
(built, not_building) = build_branches(project, branches)
if not built:
# Call sync_repository_task to update tag/branch info
version = project.versions.get(slug=LATEST)
sync_repository_task.delay(version.pk)
msg = '(URL Build) Syncing versions for %s' % project.slug
log.info(msg)
all_built[project.slug] = built
all_not_building[project.slug] = not_building

for project_slug, built in list(all_built.items()):
if built:
msg = '(URL Build) Build Started: {} [{}]'.format(
url,
' '.join(built),
)
log_info(project_slug, msg=msg)
ret += msg

for project_slug, not_building in list(all_not_building.items()):
if not_building:
msg = '(URL Build) Not Building: {} [{}]'.format(
url,
' '.join(not_building),
)
log_info(project_slug, msg=msg)
ret += msg

if not ret:
ret = '(URL Build) No known branches were pushed to.'

return HttpResponse(ret)


@csrf_exempt
def github_build(request): # noqa: D205
"""
GitHub webhook consumer.

.. warning:: **DEPRECATED**
Use :py:class:`readthedocs.api.v2.views.integrations.GitHubWebhookView`
instead of this view function

This will search for projects matching either a stripped down HTTP or SSH
URL. The search is error prone, use the API v2 webhook for new webhooks.

Old webhooks may not have specified the content type to POST with, and
therefore can use ``application/x-www-form-urlencoded`` to pass the JSON
payload. More information on the API docs here:
https://developer.github.com/webhooks/creating/#content-type
"""
if request.method == 'POST':
try:
if request.META['CONTENT_TYPE'] == 'application/x-www-form-urlencoded':
data = json.loads(request.POST.get('payload'))
else:
data = json.loads(request.body)
http_url = data['repository']['url']
http_search_url = http_url.replace('http://', '').replace('https://', '')
ssh_url = data['repository']['ssh_url']
ssh_search_url = ssh_url.replace('git@', '').replace('.git', '')
branches = [data['ref'].replace('refs/heads/', '')]
except (ValueError, TypeError, KeyError):
log.exception('Invalid GitHub webhook payload')
return HttpResponse('Invalid request', status=400)
try:
repo_projects = get_project_from_url(http_search_url)
if repo_projects:
log.info(
'GitHub webhook search: url=%s branches=%s',
http_search_url,
branches,
)
ssh_projects = get_project_from_url(ssh_search_url)
if ssh_projects:
log.info(
'GitHub webhook search: url=%s branches=%s',
ssh_search_url,
branches,
)
projects = repo_projects | ssh_projects
return _build_url(http_search_url, projects, branches)
except NoProjectException:
log.exception('Project match not found: url=%s', http_search_url)
return HttpResponseNotFound('Project not found')
else:
return HttpResponse('Method not allowed, POST is required', status=405)


@csrf_exempt
def gitlab_build(request): # noqa: D205
"""
GitLab webhook consumer.

.. warning:: **DEPRECATED**
Use :py:class:`readthedocs.api.v2.views.integrations.GitLabWebhookView`
instead of this view function

Search project repository URLs using the site URL from GitLab webhook payload.
This search is error-prone, use the API v2 webhook view for new webhooks.
"""
if request.method == 'POST':
try:
data = json.loads(request.body)
url = data['project']['http_url']
search_url = re.sub(r'^https?://(.*?)(?:\.git|)$', '\\1', url)
branches = [data['ref'].replace('refs/heads/', '')]
except (ValueError, TypeError, KeyError):
log.exception('Invalid GitLab webhook payload')
return HttpResponse('Invalid request', status=400)
log.info(
'GitLab webhook search: url=%s branches=%s',
search_url,
branches,
)
projects = get_project_from_url(search_url)
if projects:
return _build_url(search_url, projects, branches)

log.info('Project match not found: url=%s', search_url)
return HttpResponseNotFound('Project match not found')
return HttpResponse('Method not allowed, POST is required', status=405)


@csrf_exempt
def bitbucket_build(request):
"""
Consume webhooks from multiple versions of Bitbucket's API.

.. warning:: **DEPRECATED**
Use :py:class:`readthedocs.api.v2.views.integrations.BitbucketWebhookView`
instead of this view function

New webhooks are set up with v2, but v1 webhooks will still point to this
endpoint. There are also "services" that point here and submit
``application/x-www-form-urlencoded`` data.

API v1
https://confluence.atlassian.com/bitbucket/events-resources-296095220.html

API v2
https://confluence.atlassian.com/bitbucket/event-payloads-740262817.html#EventPayloads-Push

Services
https://confluence.atlassian.com/bitbucket/post-service-management-223216518.html
"""
if request.method == 'POST':
try:
if request.META['CONTENT_TYPE'] == 'application/x-www-form-urlencoded':
data = json.loads(request.POST.get('payload'))
else:
data = json.loads(request.body)

version = 2 if request.META.get('HTTP_USER_AGENT') == 'Bitbucket-Webhooks/2.0' else 1 # yapf: disabled # noqa
if version == 1:
branches = [
commit.get('branch', '') for commit in data['commits']
]
repository = data['repository']
if not repository['absolute_url']:
return HttpResponse('Invalid request', status=400)
search_url = 'bitbucket.org{}'.format(
repository['absolute_url'].rstrip('/'),
)
elif version == 2:
changes = data['push']['changes']
branches = [change['new']['name'] for change in changes]
if not data['repository']['full_name']:
return HttpResponse('Invalid request', status=400)
search_url = 'bitbucket.org/{}'.format(
data['repository']['full_name'],
)
except (TypeError, ValueError, KeyError):
log.exception('Invalid Bitbucket webhook payload')
return HttpResponse('Invalid request', status=400)

log.info(
'Bitbucket webhook search: url=%s branches=%s',
search_url,
branches,
)
log.debug('Bitbucket webhook payload:\n\n%s\n\n', data)

projects = get_project_from_url(search_url)
if projects and branches:
return _build_url(search_url, projects, branches)

if not branches:
log.info(
'Commit/branch not found url=%s branches=%s',
search_url,
branches,
)
return HttpResponseNotFound('Commit/branch not found')

log.info('Project match not found: url=%s', search_url)
return HttpResponseNotFound('Project match not found')
return HttpResponse('Method not allowed, POST is required', status=405)


@csrf_exempt
def generic_build(request, project_id_or_slug=None):
"""
Generic webhook build endpoint.

.. warning:: **DEPRECATED**

Use :py:class:`readthedocs.api.v2.views.integrations.GenericWebhookView`
instead of this view function
"""
try:
project = Project.objects.get(pk=project_id_or_slug)
# Allow slugs too
except (Project.DoesNotExist, ValueError):
try:
project = Project.objects.get(slug=project_id_or_slug)
except (Project.DoesNotExist, ValueError):
log.exception(
'(Incoming Generic Build) Repo not found: %s',
project_id_or_slug,
)
return HttpResponseNotFound(
'Repo not found: %s' % project_id_or_slug,
)
# This endpoint doesn't require authorization, we shouldn't allow builds to
# be triggered from this any longer. Deprecation plan is to selectively
# allow access to this endpoint for now.
if not _allow_deprecated_webhook(project):
return HttpResponse('This API endpoint is deprecated', status=403)
if request.method == 'POST':
slug = request.POST.get('version_slug', project.default_version)
log.info(
'(Incoming Generic Build) %s [%s]',
project.slug,
slug,
)
_build_version(project, slug)
else:
return HttpResponse('You must POST to this resource.')
return redirect('builds_project_list', project.slug)
Loading