Merge tag '11.7.0' into rel

Author: humitos

CHANGELOG.rst

@@ -1,3 +1,29 @@
+Version 11.7.0
+--------------
+
+:Date: August 27, 2024
+
+* `@github-actions[bot] <https://github.com/github-actions[bot]>`__: Dependencies: all packages updated via pip-tools (`#11566 <https://github.com/readthedocs/readthedocs.org/pull/11566>`__)
+* `@kurtmckee <https://github.com/kurtmckee>`__: Fix a minor typo (`#11565 <https://github.com/readthedocs/readthedocs.org/pull/11565>`__)
+* `@agjohnson <https://github.com/agjohnson>`__: Use clone_url for remote repository field (`#11564 <https://github.com/readthedocs/readthedocs.org/pull/11564>`__)
+* `@stsewd <https://github.com/stsewd>`__: Tasks: delete closed PRs every 30 minutes (`#11563 <https://github.com/readthedocs/readthedocs.org/pull/11563>`__)
+* `@stsewd <https://github.com/stsewd>`__: Settings: define threshold setting for removing projects from ES (`#11560 <https://github.com/readthedocs/readthedocs.org/pull/11560>`__)
+* `@humitos <https://github.com/humitos>`__: Add project: improve way to get the `basics` form (`#11558 <https://github.com/readthedocs/readthedocs.org/pull/11558>`__)
+* `@stsewd <https://github.com/stsewd>`__: Search: add function to re-index a project (`#11555 <https://github.com/readthedocs/readthedocs.org/pull/11555>`__)
+* `@humitos <https://github.com/humitos>`__: Release 11.6.0 (`#11554 <https://github.com/readthedocs/readthedocs.org/pull/11554>`__)
+* `@humitos <https://github.com/humitos>`__: Docs: make URLs not linkable (`#11553 <https://github.com/readthedocs/readthedocs.org/pull/11553>`__)
+* `@stsewd <https://github.com/stsewd>`__: Update common (`#11552 <https://github.com/readthedocs/readthedocs.org/pull/11552>`__)
+* `@humitos <https://github.com/humitos>`__: Docs: make translation section more generic (`#11549 <https://github.com/readthedocs/readthedocs.org/pull/11549>`__)
+* `@mgeier <https://github.com/mgeier>`__: Update instructions for Jupyter notebook galleries (`#11545 <https://github.com/readthedocs/readthedocs.org/pull/11545>`__)
+* `@stsewd <https://github.com/stsewd>`__: Allauth: 2FA (`#11524 <https://github.com/readthedocs/readthedocs.org/pull/11524>`__)
+* `@ericholscher <https://github.com/ericholscher>`__: Release 10.21.0 (`#11185 <https://github.com/readthedocs/readthedocs.org/pull/11185>`__)
+* `@humitos <https://github.com/humitos>`__: Build: show the YAML config file before validating it (`#11175 <https://github.com/readthedocs/readthedocs.org/pull/11175>`__)
+* `@humitos <https://github.com/humitos>`__: Release 10.18.0 (`#11096 <https://github.com/readthedocs/readthedocs.org/pull/11096>`__)
+* `@github-actions[bot] <https://github.com/github-actions[bot]>`__: Dependencies: all packages updated via pip-tools (`#11090 <https://github.com/readthedocs/readthedocs.org/pull/11090>`__)
+* `@stsewd <https://github.com/stsewd>`__: Build: use version slug for get_version_slug (`#11085 <https://github.com/readthedocs/readthedocs.org/pull/11085>`__)
+* `@stsewd <https://github.com/stsewd>`__: Integrations: Don't allow webhooks without a secret (`#11083 <https://github.com/readthedocs/readthedocs.org/pull/11083>`__)
+* `@humitos <https://github.com/humitos>`__: Addons: sorting algorithm for versions customizable on flyout (`#11069 <https://github.com/readthedocs/readthedocs.org/pull/11069>`__)
+
 Version 11.6.0
 --------------
 

docs/conf.py

@@ -76,7 +76,7 @@
 
 master_doc = "index"
 copyright = "Read the Docs, Inc & contributors"
-version = "11.6.0"
+version = "11.7.0"
 release = version
 exclude_patterns = ["_build", "shared", "_includes"]
 default_role = "obj"

docs/user/config-file/v2.rst

@@ -776,7 +776,7 @@ Patterns can include one or more of the following special characters:
         - search.html
         - search/index.html
         - 404.html
-        - 404/index.html'
+        - 404/index.html
 
 .. note::
 

docs/user/subprojects.rst

@@ -15,8 +15,8 @@ This is useful for:
 
 For a main project ``example-project``, a subproject ``example-project-plugin`` can be made available as follows:
 
-* Main project: https://example-project.readthedocs.io/en/latest/
-* Subproject: https://example-project.readthedocs.io/projects/plugin/en/latest/
+* Main project: ``https://example-project.readthedocs.io/en/latest/``
+* Subproject: ``https://example-project.readthedocs.io/projects/plugin/en/latest/``
 
 .. seealso::
 
@@ -38,8 +38,8 @@ If the example project ``example-project`` was set up with a custom domain,
 ``docs.example.com``, the URLs for projects ``example-project`` and ``example-project-plugin`` with alias ``plugin`` would
 respectively be at:
 
-* ``example-project``: https://docs.example.com/en/latest/
-* ``example-project-plugin``: https://docs.example.com/projects/plugin/en/latest/
+* ``example-project``: ``https://docs.example.com/en/latest/``
+* ``example-project-plugin``: ``https://docs.example.com/projects/plugin/en/latest/``
 
 Using aliases
 -------------

package.json

@@ -1,6 +1,6 @@
 {
   "name": "readthedocs",
-  "version": "11.6.0",
+  "version": "11.7.0",
   "description": "Read the Docs build dependencies",
   "author": "Read the Docs, Inc <[email protected]>",
   "scripts": {

readthedocs/__init__.py

@@ -1,4 +1,4 @@
 """Read the Docs."""
 
 
-__version__ = "11.6.0"
+__version__ = "11.7.0"

readthedocs/builds/tasks.py

@@ -251,7 +251,7 @@ def delete_closed_external_versions(limit=200, days=30 * 3):
     queryset = Version.external.filter(
         state=EXTERNAL_VERSION_STATE_CLOSED,
         modified__lte=days_ago,
-    )[:limit]
+    ).order_by("modified")[:limit]
     for version in queryset:
         try:
             last_build = version.last_build

readthedocs/projects/forms.py

@@ -90,7 +90,7 @@ def _get_remote_repository_choices(self):
         if current_remote_repo and current_remote_repo not in queryset:
             options.append((current_remote_repo.pk, str(current_remote_repo)))
 
-        options.extend((repo.pk, str(repo)) for repo in queryset)
+        options.extend((repo.pk, repo.clone_url) for repo in queryset)
         return options
 
     def save(self, commit=True):

readthedocs/projects/tasks/search.py

@@ -2,7 +2,7 @@
 
 import structlog
 
-from readthedocs.builds.constants import BUILD_STATE_FINISHED
+from readthedocs.builds.constants import BUILD_STATE_FINISHED, INTERNAL
 from readthedocs.builds.models import Build, Version
 from readthedocs.projects.models import HTMLFile, Project
 from readthedocs.projects.signals import files_changed
@@ -95,7 +95,7 @@ def reindex_version(version_id, search_index_name=None):
 
     build_config = latest_successful_build.config or {}
     search_config = build_config.get("search", {})
-    search_ranking = search_config.get("ranking", [])
+    search_ranking = search_config.get("ranking", {})
     search_ignore = search_config.get("ignore", [])
 
     try:
@@ -109,6 +109,31 @@ def reindex_version(version_id, search_index_name=None):
         log.exception("Failed during creation of new files")
 
 
+@app.task(queue="reindex")
+def index_project(project_slug, skip_if_exists=False):
+    """
+    Index all active versions of the project.
+
+    If ``skip_if_exists`` is True, we first check if
+    the project has at least one version indexed,
+    and skip the re-indexing if it does.
+    """
+    log.bind(project_slug=project_slug)
+    project = Project.objects.filter(slug=project_slug).first()
+    if not project:
+        log.debug("Project doesn't exist.")
+        return
+
+    if skip_if_exists:
+        if PageDocument().search().filter("term", project=project.slug).count():
+            log.debug("Skipping search indexing. Project is already indexed.")
+            return
+
+    versions = project.versions(manager=INTERNAL).filter(active=True, built=True)
+    for version in versions:
+        reindex_version(version_id=version.id)
+
+
 @app.task(queue="web")
 def remove_search_indexes(project_slug, version_slug=None):
     """Wrapper around ``remove_indexed_files`` to make it a task."""

readthedocs/projects/views/private.py

@@ -357,7 +357,15 @@ def done(self, form_list, **kwargs):
         other side effects for now, by signalling a save without commit. Then,
         finish by added the members to the project and saving.
         """
-        basics_form = form_list[0]
+
+        # We need to find the "basics" for here by iterating the list of bounded instance forms
+        # because community and business have different steps -- it's not always the first one.
+        basics_form = None
+        for form in form_list:
+            if isinstance(form, self.form_list.get("basics")):
+                basics_form = form
+                break
+
         # Save the basics form to create the project instance, then alter
         # attributes directly from other forms
         project = basics_form.save()

readthedocs/search/tests/test_utils.py

@@ -1,10 +1,14 @@
 import time
+from unittest import mock
 
 import pytest
 from django.urls import reverse
 
 from readthedocs.builds.constants import LATEST, STABLE
+from readthedocs.projects.models import Project
+from readthedocs.projects.tasks.search import index_project
 from readthedocs.search import utils
+from readthedocs.search.documents import PageDocument
 from readthedocs.search.tests.utils import get_search_query_from_project_file
 
 
@@ -66,3 +70,25 @@ def test_remove_only_one_version_index(self, api_client, all_projects):
         for project in ["pipeline", "docs"]:
             for version in [LATEST, STABLE]:
                 assert self.has_results(api_client, project, version)
+
+    @mock.patch("readthedocs.projects.tasks.search.reindex_version")
+    def test_index_project(self, reindex_version, all_projects):
+        project_slug = "kuma"
+        project = Project.objects.get(slug=project_slug)
+        # Create builds for all versions
+        for version in project.versions.all():
+            version.builds.create(
+                project=project,
+                state="finished",
+                success=True,
+            )
+
+        assert PageDocument().search().filter("term", project=project_slug).count()
+
+        # Re-index is skipped, since the project is already indexed.
+        index_project(project_slug=project_slug, skip_if_exists=True)
+        reindex_version.assert_not_called()
+
+        # Re-index all versions of the project.
+        index_project(project_slug=project_slug, skip_if_exists=False)
+        assert reindex_version.call_count == project.versions.count()

readthedocs/settings/base.py

@@ -287,6 +287,7 @@ def INSTALLED_APPS(self):  # noqa
             "allauth.socialaccount.providers.github",
             "allauth.socialaccount.providers.gitlab",
             "allauth.socialaccount.providers.bitbucket_oauth2",
+            "allauth.mfa",
             "cacheops",
         ]
         if ext:
@@ -545,11 +546,11 @@ def TEMPLATES(self):
                 "delete": True,
             },
         },
-        "every-three-hours-delete-inactive-external-versions": {
+        "every-30m-delete-inactive-external-versions": {
             "task": "readthedocs.builds.tasks.delete_closed_external_versions",
             # Increase the frequency because we have 255k closed versions and they keep growing.
             # It's better to increase this frequency than the `limit=` of the task.
-            "schedule": crontab(minute=0, hour="*/3"),
+            "schedule": crontab(minute="*/30", hour="*"),
             "options": {"queue": "web"},
         },
         "every-day-resync-remote-repositories": {
@@ -661,7 +662,7 @@ def DOCKER_LIMITS(self):
         )
         return limits
 
-    # All auth
+    # Allauth
     ACCOUNT_ADAPTER = "readthedocs.core.adapters.AccountAdapter"
     ACCOUNT_EMAIL_REQUIRED = True
 
@@ -1010,6 +1011,7 @@ def SOCIALACCOUNT_PROVIDERS(self):
     RTD_SPAM_THRESHOLD_DENY_ON_ROBOTS = 200
     RTD_SPAM_THRESHOLD_DONT_SHOW_DASHBOARD = 300
     RTD_SPAM_THRESHOLD_DONT_SERVE_DOCS = 500
+    RTD_SPAM_THRESHOLD_REMOVE_FROM_SEARCH_INDEX = 500
     RTD_SPAM_THRESHOLD_DELETE_PROJECT = 1000
     RTD_SPAM_MAX_SCORE = 9999
 

requirements/deploy.txt

@@ -23,11 +23,11 @@ billiard==3.6.4.0
     # via
     #   -r requirements/pip.txt
     #   celery
-boto3==1.34.158
+boto3==1.35.5
     # via
     #   -r requirements/pip.txt
     #   django-storages
-botocore==1.34.158
+botocore==1.35.5
     # via
     #   -r requirements/pip.txt
     #   boto3
@@ -76,13 +76,14 @@ colorama==0.4.6
     # via
     #   -r requirements/pip.txt
     #   bumpver
-cron-descriptor==1.4.3
+cron-descriptor==1.4.5
     # via
     #   -r requirements/pip.txt
     #   django-celery-beat
 cryptography==43.0.0
     # via
     #   -r requirements/pip.txt
+    #   fido2
     #   pyjwt
 cssselect==1.2.0
     # via
@@ -120,15 +121,15 @@ django==4.2.15
     #   django-timezone-field
     #   djangorestframework
     #   jsonfield
-django-allauth[saml,socialaccount]==64.0.0
+django-allauth[mfa,saml,socialaccount]==64.1.0
     # via -r requirements/pip.txt
 django-annoying==0.10.7
     # via -r requirements/pip.txt
 django-autoslug==1.9.9
     # via -r requirements/pip.txt
 django-cacheops==7.0.2
     # via -r requirements/pip.txt
-django-celery-beat==2.6.0
+django-celery-beat==2.7.0
     # via -r requirements/pip.txt
 django-cors-headers==4.4.0
     # via -r requirements/pip.txt
@@ -189,22 +190,26 @@ drf-extensions==0.7.1
     # via -r requirements/pip.txt
 drf-flex-fields==1.0.2
     # via -r requirements/pip.txt
-elastic-transport==8.13.1
+elastic-transport==8.15.0
     # via
     #   -r requirements/pip.txt
     #   elasticsearch
-elasticsearch==8.14.0
+elasticsearch==8.15.0
     # via
     #   -r requirements/pip.txt
     #   elasticsearch-dsl
-elasticsearch-dsl==8.14.0
+elasticsearch-dsl==8.15.1
     # via
     #   -r requirements/pip.txt
     #   django-elasticsearch-dsl
 exceptiongroup==1.2.2
     # via ipython
 executing==2.0.1
     # via stack-data
+fido2==1.1.3
+    # via
+    #   -r requirements/pip.txt
+    #   django-allauth
 filelock==3.15.4
     # via
     #   -r requirements/pip.txt
@@ -215,7 +220,7 @@ funcy==2.0
     #   django-cacheops
 gunicorn==23.0.0
     # via -r requirements/pip.txt
-idna==3.7
+idna==3.8
     # via
     #   -r requirements/pip.txt
     #   requests
@@ -254,7 +259,7 @@ lxml==5.3.0
     #   pyquery
     #   python3-saml
     #   xmlsec
-markdown==3.6
+markdown==3.7
     # via -r requirements/pip.txt
 matplotlib-inline==0.1.7
     # via ipython
@@ -312,6 +317,10 @@ pyjwt[crypto]==2.9.0
     # via
     #   -r requirements/pip.txt
     #   django-allauth
+pypng==0.20220715.0
+    # via
+    #   -r requirements/pip.txt
+    #   qrcode
 pyquery==2.0.0
     # via -r requirements/pip.txt
 python-crontab==3.2.0
@@ -334,6 +343,10 @@ pytz==2024.1
     #   celery
 pyyaml==6.0.2
     # via -r requirements/pip.txt
+qrcode==7.4.2
+    # via
+    #   -r requirements/pip.txt
+    #   django-allauth
 redis==5.0.8
     # via
     #   -r requirements/pip.txt
@@ -363,7 +376,7 @@ s3transfer==0.10.2
     #   boto3
 selectolax==0.3.21
     # via -r requirements/pip.txt
-sentry-sdk==2.12.0
+sentry-sdk==2.13.0
     # via structlog-sentry
 six==1.16.0
     # via
@@ -414,6 +427,7 @@ typing-extensions==4.12.2
     #   ipython
     #   psycopg
     #   psycopg-pool
+    #   qrcode
 tzdata==2024.1
     # via
     #   -r requirements/pip.txt