Merge branch 'master' into dont-trigger-two-syncs

Author: stsewd

.travis.yml

@@ -9,6 +9,8 @@ jobs:
       env: TOXENV=py36,codecov ES_VERSION=6.2.4 ES_DOWNLOAD_URL=https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}.tar.gz
     - python: 3.6
       env: TOXENV=docs
+    - python: 3.6
+      env: TOXENV=docs-linkcheck
     - python: 3.6
       env: TOXENV=docs-lint
     - python: 3.6
@@ -18,6 +20,9 @@ jobs:
       env: TOXENV=eslint NODE_VERSION=10.17.0
     - python: 3.6
       env: TOXENV=migrations
+
+  allow_failures:
+    - env: TOXENV=docs-linkcheck
 cache:
   directories:
     - ~/.cache/pip

CHANGELOG.rst

@@ -918,7 +918,7 @@ Version 3.2.2
 * `@pyup-bot <https://github.com/pyup-bot>`__: Pin pytest-cov to latest version 2.6.1 (`#5276 <https://github.com/readthedocs/readthedocs.org/pull/5276>`__)
 * `@pyup-bot <https://github.com/pyup-bot>`__: Pin pillow to latest version 5.4.1 (`#5275 <https://github.com/readthedocs/readthedocs.org/pull/5275>`__)
 * `@pyup-bot <https://github.com/pyup-bot>`__: Update elasticsearch to 6.3.1 (`#5274 <https://github.com/readthedocs/readthedocs.org/pull/5274>`__)
-* `@discdiver <https://github.com/discdiver>`__: clarify github integration needs https:// prepended (`#5273 <https://github.com/readthedocs/readthedocs.org/pull/5273>`__)
+* `@discdiver <https://github.com/discdiver>`__: clarify github integration needs ``https://`` prepended (`#5273 <https://github.com/readthedocs/readthedocs.org/pull/5273>`__)
 * `@humitos <https://github.com/humitos>`__: Setup and configure pyup.io (`#5272 <https://github.com/readthedocs/readthedocs.org/pull/5272>`__)
 * `@humitos <https://github.com/humitos>`__: Update all Python dependencies (`#5269 <https://github.com/readthedocs/readthedocs.org/pull/5269>`__)
 * `@davidfischer <https://github.com/davidfischer>`__: Add note about security issue (`#5263 <https://github.com/readthedocs/readthedocs.org/pull/5263>`__)

dockerfiles/Dockerfile

@@ -34,6 +34,9 @@ RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
 
 RUN pip3 install --no-cache-dir --upgrade pip
 
+# Ensure that ``python`` is in the PATH so that ``./manage.py`` works
+RUN ln -s /usr/bin/python3 /usr/bin/python
+
 WORKDIR /tmp
 
 RUN curl -O https://raw.githubusercontent.com/readthedocs/readthedocs.org/master/requirements/pip.txt

docs/conf.py

@@ -103,6 +103,18 @@ def get_version():
 <p>Try using the search box or go to the homepage.</p>
 ''',
 }
+linkcheck_ignore = [
+    r'http://127\.0\.0\.1',
+    r'http://localhost',
+    r'https://yourproject\.readthedocs\.io',
+    r'https?://docs\.example\.com',
+    r'https://foo\.readthedocs\.io/projects',
+    r'https://github\.com.+?#L\d+',
+    r'https://github\.com/readthedocs/readthedocs\.org/issues',
+    r'https://github\.com/readthedocs/readthedocs\.org/pull',
+    r'https://docs\.readthedocs\.io/\?rtd_search',
+    r'https://readthedocs\.org/search',
+]
 
 
 def setup(app):

readthedocs/projects/admin.py

@@ -2,6 +2,7 @@
 
 from django.contrib import admin, messages
 from django.contrib.admin.actions import delete_selected
+from django.forms import BaseInlineFormSet
 from django.utils.translation import ugettext_lazy as _
 
 from readthedocs.builds.models import Version
@@ -53,12 +54,27 @@ class ProjectRelationshipInline(admin.TabularInline):
     raw_id_fields = ('child',)
 
 
+class VersionInlineFormSet(BaseInlineFormSet):
+
+    """Limit the number of versions displayed in the inline."""
+
+    LIMIT = 200
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.queryset = self.queryset[:self.LIMIT]
+
+
 class VersionInline(admin.TabularInline):
 
     """Version inline relationship view for :py:class:`ProjectAdmin`."""
 
+    formset = VersionInlineFormSet
     model = Version
 
+    def get_queryset(self, request):
+        return super().get_queryset(request).select_related("project")
+
 
 class RedirectInline(admin.TabularInline):
 

readthedocs/settings/base.py

@@ -312,6 +312,8 @@ def USE_PROMOS(self):  # noqa
     CELERYD_TASK_TIME_LIMIT = 60 * 60  # 60 minutes
     CELERY_SEND_TASK_ERROR_EMAILS = False
     CELERYD_HIJACK_ROOT_LOGGER = False
+    # This stops us from pre-fetching a task that then sits around on the builder
+    CELERY_ACKS_LATE = True
     # Don't queue a bunch of tasks in the workers
     CELERYD_PREFETCH_MULTIPLIER = 1
     CELERY_CREATE_MISSING_QUEUES = True

readthedocs/settings/docker_compose.py

@@ -123,3 +123,7 @@ def DATABASES(self):  # noqa
     AZURE_EMULATED_MODE = True
     AZURE_CUSTOM_DOMAIN = 'storage:10000'
     AZURE_SSL = False
+
+    # Remove the checks on the number of fields being submitted
+    # This limit is mostly hit on large forms in the Django admin
+    DATA_UPLOAD_MAX_NUMBER_FIELDS = None

readthedocs/vcs_support/backends/git.py

@@ -203,16 +203,28 @@ def clone(self):
     def tags(self):
         versions = []
         repo = git.Repo(self.working_dir)
+
+        # Build a cache of tag -> commit
+        # GitPython is not very optimized for reading large numbers of tags
+        ref_cache = {}  # 'ref/tags/<tag>' -> hexsha
+        for hexsha, ref in git.TagReference._iter_packed_refs(repo):
+            ref_cache[ref] = hexsha
+
         for tag in repo.tags:
-            try:
-                versions.append(VCSVersion(self, str(tag.commit), str(tag)))
-            except ValueError:
-                # ValueError: Cannot resolve commit as tag TAGNAME points to a
-                # blob object - use the `.object` property instead to access it
-                # This is not a real tag for us, so we skip it
-                # https://github.com/rtfd/readthedocs.org/issues/4440
-                log.warning('Git tag skipped: %s', tag, exc_info=True)
-                continue
+            if tag.path in ref_cache:
+                hexsha = ref_cache[tag.path]
+            else:
+                try:
+                    hexsha = str(tag.commit)
+                except ValueError:
+                    # ValueError: Cannot resolve commit as tag TAGNAME points to a
+                    # blob object - use the `.object` property instead to access it
+                    # This is not a real tag for us, so we skip it
+                    # https://github.com/rtfd/readthedocs.org/issues/4440
+                    log.warning('Git tag skipped: %s', tag, exc_info=True)
+                    continue
+
+            versions.append(VCSVersion(self, hexsha, str(tag)))
         return versions
 
     @property

requirements/pip.txt

@@ -3,7 +3,7 @@ pip==19.3.1
 appdirs==1.4.3
 virtualenv==16.7.7
 
-django==1.11.27  # pyup: <1.12
+django==1.11.28  # pyup: <1.12
 django-extensions==2.2.5
 django_polymorphic==2.1.2
 
@@ -42,7 +42,7 @@ celery==4.1.1  # pyup: ignore
 
 django-allauth==0.40.0
 
-GitPython==3.0.4
+GitPython==3.0.8
 
 # Search
 elasticsearch==6.4.0  # pyup: <7.0.0

tox.ini

@@ -27,6 +27,12 @@ changedir = {toxinidir}/docs
 commands =
     sphinx-build -W --keep-going -b html -d {envtmpdir}/doctrees . {envtmpdir}/html
 
+[testenv:docs-linkcheck]
+description = Check for broken links in the docs
+changedir = {toxinidir}/docs
+commands =
+    sphinx-build -W --keep-going -q -b linkcheck -d {envtmpdir}/doctrees . {envtmpdir}/linkcheck
+
 [testenv:migrations]
 description = check for missing migrations
 changedir = {toxinidir}