Merge remote-tracking branch 'origin/master' into humitos/new-docker-release

Author: ericholscher

.travis.yml

@@ -9,6 +9,8 @@ jobs:
       env: TOXENV=py36,codecov ES_VERSION=6.2.4 ES_DOWNLOAD_URL=https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}.tar.gz
     - python: 3.6
       env: TOXENV=docs
+    - python: 3.6
+      env: TOXENV=docs-linkcheck
     - python: 3.6
       env: TOXENV=docs-lint
     - python: 3.6
@@ -18,6 +20,9 @@ jobs:
       env: TOXENV=eslint NODE_VERSION=10.17.0
     - python: 3.6
       env: TOXENV=migrations
+
+  allow_failures:
+    - env: TOXENV=docs-linkcheck
 cache:
   directories:
     - ~/.cache/pip

CHANGELOG.rst

@@ -918,7 +918,7 @@ Version 3.2.2
 * `@pyup-bot <https://github.com/pyup-bot>`__: Pin pytest-cov to latest version 2.6.1 (`#5276 <https://github.com/readthedocs/readthedocs.org/pull/5276>`__)
 * `@pyup-bot <https://github.com/pyup-bot>`__: Pin pillow to latest version 5.4.1 (`#5275 <https://github.com/readthedocs/readthedocs.org/pull/5275>`__)
 * `@pyup-bot <https://github.com/pyup-bot>`__: Update elasticsearch to 6.3.1 (`#5274 <https://github.com/readthedocs/readthedocs.org/pull/5274>`__)
-* `@discdiver <https://github.com/discdiver>`__: clarify github integration needs https:// prepended (`#5273 <https://github.com/readthedocs/readthedocs.org/pull/5273>`__)
+* `@discdiver <https://github.com/discdiver>`__: clarify github integration needs ``https://`` prepended (`#5273 <https://github.com/readthedocs/readthedocs.org/pull/5273>`__)
 * `@humitos <https://github.com/humitos>`__: Setup and configure pyup.io (`#5272 <https://github.com/readthedocs/readthedocs.org/pull/5272>`__)
 * `@humitos <https://github.com/humitos>`__: Update all Python dependencies (`#5269 <https://github.com/readthedocs/readthedocs.org/pull/5269>`__)
 * `@davidfischer <https://github.com/davidfischer>`__: Add note about security issue (`#5263 <https://github.com/readthedocs/readthedocs.org/pull/5263>`__)

dockerfiles/Dockerfile

@@ -34,6 +34,9 @@ RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
 
 RUN pip3 install --no-cache-dir --upgrade pip
 
+# Ensure that ``python`` is in the PATH so that ``./manage.py`` works
+RUN ln -s /usr/bin/python3 /usr/bin/python
+
 WORKDIR /tmp
 
 RUN curl -O https://raw.githubusercontent.com/readthedocs/readthedocs.org/master/requirements/pip.txt

docs/conf.py

@@ -103,6 +103,18 @@ def get_version():
 <p>Try using the search box or go to the homepage.</p>
 ''',
 }
+linkcheck_ignore = [
+    r'http://127\.0\.0\.1',
+    r'http://localhost',
+    r'https://yourproject\.readthedocs\.io',
+    r'https?://docs\.example\.com',
+    r'https://foo\.readthedocs\.io/projects',
+    r'https://github\.com.+?#L\d+',
+    r'https://github\.com/readthedocs/readthedocs\.org/issues',
+    r'https://github\.com/readthedocs/readthedocs\.org/pull',
+    r'https://docs\.readthedocs\.io/\?rtd_search',
+    r'https://readthedocs\.org/search',
+]
 
 
 def setup(app):

readthedocs/api/v2/templates/restapi/footer.html

@@ -61,16 +61,17 @@
 
       {% block readthedocs %}
       <dl>
+        {# We hardcode the URLS because we don't have access to the URLCONF of the main app from proxito #}
         <!-- These are kept as relative links for internal installs that are http -->
         <dt>{% trans "On Read the Docs" %}</dt>
         <dd>
-          <a href="//{{ settings.PRODUCTION_DOMAIN }}{% url 'projects_detail' project.slug %}">{% trans "Project Home" %}</a>
+          <a href="//{{ settings.PRODUCTION_DOMAIN }}/projects/{{ project.slug }}/">{% trans "Project Home" %}</a>
         </dd>
         <dd>
-          <a href="//{{ settings.PRODUCTION_DOMAIN }}{% url 'builds_project_list' project.slug %}">{% trans "Builds" %}</a>
+          <a href="//{{ settings.PRODUCTION_DOMAIN }}/projects/{{ project.slug }}/builds/">{% trans "Builds" %}</a>
         </dd>
         <dd>
-          <a href="//{{ settings.PRODUCTION_DOMAIN }}{% url 'project_downloads' project.slug %}">{% trans "Downloads" %}</a>
+          <a href="//{{ settings.PRODUCTION_DOMAIN }}/projects/{{ project.slug }}/downloads/">{% trans "Downloads" %}</a>
         </dd>
       </dl>
       {% endblock %}
@@ -116,7 +117,8 @@
         <dt>{% trans "Search" %}</dt>
         <dd>
           <div style="padding: 6px;">
-            <form id="flyout-search-form" class="wy-form" target="_blank" action="//{{ settings.PRODUCTION_DOMAIN }}{% url 'elastic_project_search' project.slug %}" method="get">
+            {# We hardcode the URLS because we don't have access to the URLCONF of the main app from proxito #}
+            <form id="flyout-search-form" class="wy-form" target="_blank" action="//{{ settings.PRODUCTION_DOMAIN }}/projects/{{ project.slug }}/search/" method="get">
               <input type="text" name="q" placeholder="{% trans "Search docs" %}">
               </form>
           </div>

readthedocs/api/v2/views/integrations.py

@@ -202,7 +202,7 @@ def sync_versions(self, project):
             'build_triggered': False,
             'project': project.slug,
             'versions': [version],
-            'versions_synced': True,
+            'versions_synced': version is not None,
         }
 
     def get_external_version_response(self, project):

readthedocs/core/views/hooks.py

@@ -4,6 +4,7 @@
 
 from readthedocs.builds.constants import EXTERNAL
 from readthedocs.core.utils import trigger_build
+from readthedocs.projects.models import Project
 from readthedocs.projects.tasks import sync_repository_task
 
 
@@ -73,8 +74,16 @@ def sync_versions(project):
     we always pass the default version.
 
     :returns: The version slug that was used to trigger the clone.
-    :rtype: str
+    :rtype: str or ``None`` if failed
     """
+
+    if not Project.objects.is_active(project):
+        log.warning(
+            'Sync not triggered because Project is not active: project=%s',
+            project.slug,
+        )
+        return None
+
     try:
         version_identifier = project.get_default_branch()
         version = (
@@ -85,7 +94,16 @@ def sync_versions(project):
         if not version:
             log.info('Unable to sync from %s version', version_identifier)
             return None
-        sync_repository_task.delay(version.pk)
+
+        options = {}
+        if project.build_queue:
+            # respect the queue for this project
+            options['queue'] = project.build_queue
+
+        sync_repository_task.apply_async(
+            (version.pk,),
+            **options,
+        )
         return version.slug
     except Exception:
         log.exception('Unknown sync versions exception')

readthedocs/projects/admin.py

@@ -2,6 +2,7 @@
 
 from django.contrib import admin, messages
 from django.contrib.admin.actions import delete_selected
+from django.forms import BaseInlineFormSet
 from django.utils.translation import ugettext_lazy as _
 
 from readthedocs.builds.models import Version
@@ -53,12 +54,27 @@ class ProjectRelationshipInline(admin.TabularInline):
     raw_id_fields = ('child',)
 
 
+class VersionInlineFormSet(BaseInlineFormSet):
+
+    """Limit the number of versions displayed in the inline."""
+
+    LIMIT = 200
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.queryset = self.queryset[:self.LIMIT]
+
+
 class VersionInline(admin.TabularInline):
 
     """Version inline relationship view for :py:class:`ProjectAdmin`."""
 
+    formset = VersionInlineFormSet
     model = Version
 
+    def get_queryset(self, request):
+        return super().get_queryset(request).select_related("project")
+
 
 class RedirectInline(admin.TabularInline):
 

readthedocs/projects/tasks.py

@@ -154,7 +154,7 @@ def sync_versions(self, version_repo):
         """
         Update tags/branches hitting the API.
 
-        It may trigger a new build to the stable version when hittig the
+        It may trigger a new build to the stable version when hitting the
         ``sync_versions`` endpoint.
         """
         version_post_data = {'repo': version_repo.repo_url}
@@ -1798,7 +1798,11 @@ def webhook_notification(version, build, hook_url):
         }
     )
     try:
-        requests.post(hook_url, data=data)
+        requests.post(
+            hook_url,
+            data=data,
+            headers={'content-type': 'application/json'}
+        )
     except Exception:
         log.exception('Failed to POST on webhook url: url=%s', hook_url)
 

readthedocs/rtd_tests/tests/test_api.py

@@ -773,7 +773,10 @@ class IntegrationsTests(TestCase):
     fixtures = ['eric.json', 'test_data.json']
 
     def setUp(self):
-        self.project = get(Project)
+        self.project = get(
+            Project,
+            build_queue=None,
+        )
         self.feature_flag = get(
             Feature,
             projects=[self.project],
@@ -847,6 +850,31 @@ def test_webhook_skipped_project(self, trigger_build):
         self.assertEqual(response.status_code, status.HTTP_406_NOT_ACCEPTABLE)
         self.assertFalse(trigger_build.called)
 
+    @mock.patch('readthedocs.core.views.hooks.sync_repository_task')
+    def test_sync_repository_custom_project_queue(self, sync_repository_task, trigger_build):
+        client = APIClient()
+        self.project.build_queue = 'specific-build-queue'
+        self.project.save()
+
+        headers = {GITHUB_EVENT_HEADER: GITHUB_CREATE}
+        resp = client.post(
+            '/api/v2/webhook/github/{}/'.format(self.project.slug),
+            self.github_payload,
+            format='json',
+            **headers,
+        )
+        self.assertEqual(resp.status_code, status.HTTP_200_OK)
+        self.assertFalse(resp.data['build_triggered'])
+        self.assertEqual(resp.data['project'], self.project.slug)
+        self.assertEqual(resp.data['versions'], [LATEST])
+        self.assertTrue(resp.data['versions_synced'])
+        trigger_build.assert_not_called()
+        latest_version = self.project.versions.get(slug=LATEST)
+        sync_repository_task.apply_async.assert_called_with(
+            (latest_version.pk,),
+            queue='specific-build-queue',
+        )
+
     def test_github_webhook_for_branches(self, trigger_build):
         """GitHub webhook API."""
         client = APIClient()
@@ -927,7 +955,7 @@ def test_github_webhook_no_build_on_delete(self, sync_repository_task, trigger_b
         self.assertEqual(resp.data['versions'], [LATEST])
         trigger_build.assert_not_called()
         latest_version = self.project.versions.get(slug=LATEST)
-        sync_repository_task.delay.assert_called_with(latest_version.pk)
+        sync_repository_task.apply_async.assert_called_with((latest_version.pk,))
 
     @mock.patch('readthedocs.core.views.hooks.sync_repository_task')
     def test_github_create_event(self, sync_repository_task, trigger_build):
@@ -946,7 +974,7 @@ def test_github_create_event(self, sync_repository_task, trigger_build):
         self.assertEqual(resp.data['versions'], [LATEST])
         trigger_build.assert_not_called()
         latest_version = self.project.versions.get(slug=LATEST)
-        sync_repository_task.delay.assert_called_with(latest_version.pk)
+        sync_repository_task.apply_async.assert_called_with((latest_version.pk,))
 
     @mock.patch('readthedocs.core.utils.trigger_build')
     def test_github_pull_request_opened_event(self, trigger_build, core_trigger_build):
@@ -1191,7 +1219,7 @@ def test_github_delete_event(self, sync_repository_task, trigger_build):
         self.assertEqual(resp.data['versions'], [LATEST])
         trigger_build.assert_not_called()
         latest_version = self.project.versions.get(slug=LATEST)
-        sync_repository_task.delay.assert_called_with(latest_version.pk)
+        sync_repository_task.apply_async.assert_called_with((latest_version.pk,))
 
     def test_github_parse_ref(self, trigger_build):
         wh = GitHubWebhookView()
@@ -1399,7 +1427,7 @@ def test_gitlab_push_hook_creation(
         self.assertEqual(resp.data['versions'], [LATEST])
         trigger_build.assert_not_called()
         latest_version = self.project.versions.get(slug=LATEST)
-        sync_repository_task.delay.assert_called_with(latest_version.pk)
+        sync_repository_task.apply_async.assert_called_with((latest_version.pk,))
 
     @mock.patch('readthedocs.core.views.hooks.sync_repository_task')
     def test_gitlab_push_hook_deletion(
@@ -1421,7 +1449,7 @@ def test_gitlab_push_hook_deletion(
         self.assertEqual(resp.data['versions'], [LATEST])
         trigger_build.assert_not_called()
         latest_version = self.project.versions.get(slug=LATEST)
-        sync_repository_task.delay.assert_called_with(latest_version.pk)
+        sync_repository_task.apply_async.assert_called_with((latest_version.pk,))
 
     @mock.patch('readthedocs.core.views.hooks.sync_repository_task')
     def test_gitlab_tag_push_hook_creation(
@@ -1444,7 +1472,7 @@ def test_gitlab_tag_push_hook_creation(
         self.assertEqual(resp.data['versions'], [LATEST])
         trigger_build.assert_not_called()
         latest_version = self.project.versions.get(slug=LATEST)
-        sync_repository_task.delay.assert_called_with(latest_version.pk)
+        sync_repository_task.apply_async.assert_called_with((latest_version.pk,))
 
     @mock.patch('readthedocs.core.views.hooks.sync_repository_task')
     def test_gitlab_tag_push_hook_deletion(
@@ -1467,7 +1495,7 @@ def test_gitlab_tag_push_hook_deletion(
         self.assertEqual(resp.data['versions'], [LATEST])
         trigger_build.assert_not_called()
         latest_version = self.project.versions.get(slug=LATEST)
-        sync_repository_task.delay.assert_called_with(latest_version.pk)
+        sync_repository_task.apply_async.assert_called_with((latest_version.pk,))
 
     def test_gitlab_invalid_webhook(self, trigger_build):
         """GitLab webhook unhandled event."""
@@ -1919,7 +1947,7 @@ def test_bitbucket_push_hook_creation(
         self.assertEqual(resp.data['versions'], [LATEST])
         trigger_build.assert_not_called()
         latest_version = self.project.versions.get(slug=LATEST)
-        sync_repository_task.delay.assert_called_with(latest_version.pk)
+        sync_repository_task.apply_async.assert_called_with((latest_version.pk,))
 
     @mock.patch('readthedocs.core.views.hooks.sync_repository_task')
     def test_bitbucket_push_hook_deletion(
@@ -1938,7 +1966,7 @@ def test_bitbucket_push_hook_deletion(
         self.assertEqual(resp.data['versions'], [LATEST])
         trigger_build.assert_not_called()
         latest_version = self.project.versions.get(slug=LATEST)
-        sync_repository_task.delay.assert_called_with(latest_version.pk)
+        sync_repository_task.apply_async.assert_called_with((latest_version.pk,))
 
     def test_bitbucket_invalid_webhook(self, trigger_build):
         """Bitbucket webhook unhandled event."""

readthedocs/rtd_tests/tests/test_build_notifications.py

@@ -1,6 +1,8 @@
 # -*- coding: utf-8 -*-
 """Notifications sent after build is completed."""
 
+import json
+
 import django_dynamic_fixture as fixture
 from django.core import mail
 from django.test import TestCase
@@ -38,6 +40,28 @@ def test_send_webhook_notification(self):
 
         self.assertEqual(len(mail.outbox), 0)
 
+    def test_send_webhook_notification_has_content_type_header(self):
+        hook = fixture.get(WebHook, project=self.project)
+        data = json.dumps({
+            'name': self.project.name,
+            'slug': self.project.slug,
+            'build': {
+                'id': self.build.id,
+                'commit': self.build.commit,
+                'state': self.build.state,
+                'success': self.build.success,
+                'date': self.build.date.strftime('%Y-%m-%d %H:%M:%S'),
+            },
+        })
+        with patch('readthedocs.projects.tasks.requests.post') as mock:
+            mock.return_value = None
+            send_notifications(self.version.pk, self.build.pk)
+            mock.assert_called_once_with(
+                hook.url,
+                data=data,
+                headers={'content-type': 'application/json'}
+            )
+
     def test_send_email_notification(self):
         fixture.get(EmailHook, project=self.project)
         send_notifications(self.version.pk, self.build.pk, email=True)