Continue refactor

- Split into tags and branches data
- Update with latest changes from sync_versions
- Make it backwards compatible

Author: stsewd

readthedocs/api/v2/utils.py

@@ -165,14 +165,16 @@ def _set_or_create_version(project, slug, version_id, verbose_name, type_):
     return version, False
 
 
-def _get_deleted_versions_qs(project, version_data):
+def _get_deleted_versions_qs(project, tags_data, branches_data):
     # We use verbose_name for tags
     # because several tags can point to the same identifier.
     versions_tags = [
-        version['verbose_name'] for version in version_data.get('tags', [])
+        version['verbose_name']
+        for version in tags_data
     ]
     versions_branches = [
-        version['identifier'] for version in version_data.get('branches', [])
+        version['identifier']
+        for version in branches_data
     ]
 
     to_delete_qs = (
@@ -192,14 +194,18 @@ def _get_deleted_versions_qs(project, version_data):
     return to_delete_qs
 
 
-def delete_versions_from_db(project, version_data):
+def delete_versions_from_db(project, tags_data, branches_data):
     """
     Delete all versions not in the current repo.
 
     :returns: The slug of the deleted versions from the database.
     """
     to_delete_qs = (
-        _get_deleted_versions_qs(project, version_data)
+        _get_deleted_versions_qs(
+            project=project,
+            tags_data=tags_data,
+            branches_data=branches_data,
+        )
         .exclude(active=True)
     )
     deleted_versions = set(to_delete_qs.values_list('slug', flat=True))
@@ -213,10 +219,14 @@ def delete_versions_from_db(project, version_data):
     return deleted_versions
 
 
-def get_deleted_active_versions(project, version_data):
+def get_deleted_active_versions(project, tags_data, branches_data):
     """Return the slug of active versions that were deleted from the repository."""
     to_delete_qs = (
-        _get_deleted_versions_qs(project, version_data)
+        _get_deleted_versions_qs(
+            project=project,
+            tags_data=tags_data,
+            branches_data=branches_data,
+        )
         .filter(active=True)
     )
     return set(to_delete_qs.values_list('slug', flat=True))

readthedocs/api/v2/views/model_views.py

@@ -6,35 +6,21 @@
 from allauth.socialaccount.models import SocialAccount
 from django.conf import settings
 from django.core.files.storage import get_storage_class
-from django.db.models import Q
 from django.shortcuts import get_object_or_404
 from django.template.loader import render_to_string
 from rest_framework import decorators, permissions, status, viewsets
 from rest_framework.parsers import JSONParser, MultiPartParser
 from rest_framework.renderers import BaseRenderer, JSONRenderer
 from rest_framework.response import Response
 
-from readthedocs.builds.constants import (
-    BRANCH,
-    BUILD_STATE_FINISHED,
-    BUILD_STATE_TRIGGERED,
-    INTERNAL,
-    TAG,
-)
+from readthedocs.builds.constants import INTERNAL
 from readthedocs.builds.models import Build, BuildCommandResult, Version
-from readthedocs.core.utils import trigger_build
-from readthedocs.core.utils.extend import SettingsOverrideObject
+from readthedocs.builds.tasks import sync_versions_task
 from readthedocs.oauth.models import RemoteOrganization, RemoteRepository
 from readthedocs.oauth.services import GitHubService, registry
-from readthedocs.projects.models import Domain, EmailHook, Project
-from readthedocs.projects.version_handling import determine_stable_version
-
-from ..permissions import (
-    APIPermission,
-    APIRestrictedPermission,
-    IsOwner,
-    RelatedProjectIsOwner,
-)
+from readthedocs.projects.models import Domain, Project
+
+from ..permissions import APIPermission, APIRestrictedPermission, IsOwner
 from ..serializers import (
     BuildAdminSerializer,
     BuildCommandSerializer,
@@ -52,10 +38,6 @@
     ProjectPagination,
     RemoteOrganizationPagination,
     RemoteProjectPagination,
-    delete_versions_from_db,
-    get_deleted_active_versions,
-    run_automation_rules,
-    sync_versions_to_db,
 )
 
 log = logging.getLogger(__name__)
@@ -188,47 +170,26 @@ def sync_versions(self, request, **kwargs):  # noqa
         Version data in the repo is synced with what we have in the database.
 
         :returns: the identifiers for the versions that have been deleted.
-        """
-        # TODO: Delete this code. It has been moved to ``readthedocs.builds.tasks.sync_versions_task``.
-        # This is here just to migrate our existing code
 
-        #################
-        # WARNING: Do not edit or update this code
-        #################
+        .. note::
+           
+           This endpoint is deprecated in favor of `sync_versions_task`.
         """
         project = get_object_or_404(
             Project.objects.api(request.user),
             pk=kwargs['pk'],
         )
 
-        # If the currently highest non-prerelease version is active, then make
-        # the new latest version active as well.
-        current_stable = project.get_original_stable_version()
-        if current_stable is not None:
-            activate_new_stable = current_stable.active
-        else:
-            activate_new_stable = False
+        added_versions = []
+        deleted_versions = []
 
         try:
-            # Update All Versions
             data = request.data
-            added_versions = set()
-            if 'tags' in data:
-                ret_set = sync_versions_to_db(
-                    project=project,
-                    versions=data['tags'],
-                    type=TAG,
-                )
-                added_versions.update(ret_set)
-            if 'branches' in data:
-                ret_set = sync_versions_to_db(
-                    project=project,
-                    versions=data['branches'],
-                    type=BRANCH,
-                )
-                added_versions.update(ret_set)
-            deleted_versions = delete_versions_from_db(project, data)
-            deleted_active_versions = get_deleted_active_versions(project, data)
+            added_versions, deleted_versions = sync_versions_task(
+                project_pk=project.pk,
+                tags_data=data.get('tags', []),
+                branches_data=data.get('branches', []),
+            )
         except Exception as e:
             log.exception('Sync Versions Error')
             return Response(
@@ -238,19 +199,10 @@ def sync_versions(self, request, **kwargs):  # noqa
                 status=status.HTTP_400_BAD_REQUEST,
             )
 
-        try:
-            # The order of added_versions isn't deterministic.
-            # We don't track the commit time or any other metadata.
-            # We usually have one version added per webhook.
-            run_automation_rules(project, added_versions, deleted_active_versions)
-        except Exception:
-            # Don't interrupt the request if something goes wrong
-            # in the automation rules.
-            log.exception(
-                'Failed to execute automation rules for [%s]: %s',
-                project.slug, added_versions
-            )
-        """
+        return Response({
+            'added_versions': added_versions,
+            'deleted_versions': deleted_versions,
+        })
 
 
 class VersionViewSet(UserSelectViewSet):

readthedocs/builds/tasks.py

@@ -6,25 +6,19 @@
 from celery import Task
 from django.conf import settings
 from django.core.files.storage import get_storage_class
-from django.shortcuts import get_object_or_404
 
 from readthedocs.api.v2.serializers import BuildSerializer
 from readthedocs.api.v2.utils import (
-    ProjectPagination,
-    RemoteOrganizationPagination,
-    RemoteProjectPagination,
     delete_versions_from_db,
+    get_deleted_active_versions,
     run_automation_rules,
     sync_versions_to_db,
 )
 from readthedocs.builds.constants import (
     BRANCH,
-    BUILD_STATE_FINISHED,
-    BUILD_STATE_TRIGGERED,
     BUILD_STATUS_FAILURE,
     BUILD_STATUS_PENDING,
     BUILD_STATUS_SUCCESS,
-    INTERNAL,
     MAX_BUILD_COMMAND_SIZE,
     TAG,
 )
@@ -33,7 +27,6 @@
 from readthedocs.core.utils import trigger_build
 from readthedocs.projects.models import Project
 from readthedocs.projects.tasks import send_build_status
-from readthedocs.projects.version_handling import determine_stable_version
 from readthedocs.worker import app
 
 log = logging.getLogger(__name__)
@@ -188,55 +181,103 @@ def archive_builds_task(days=14, limit=200, include_cold=False, delete=False):
                 log.exception('Cold Storage save failure')
 
 
+def delete_inactive_external_versions(limit=200, days=30 * 3):
+    """
+    Delete external versions that have been marked as inactive after ``days``.
+
+    The commit status is updated to link to the build page, as the docs are removed.
+    """
+    days_ago = datetime.now() - timedelta(days=days)
+    queryset = Version.external.filter(
+        active=False,
+        modified__lte=days_ago,
+    )[:limit]
+    for version in queryset:
+        try:
+            last_build = version.last_build
+            if last_build:
+                status = BUILD_STATUS_PENDING
+                if last_build.finished:
+                    status = BUILD_STATUS_SUCCESS if last_build.success else BUILD_STATUS_FAILURE
+                send_build_status(
+                    build_pk=last_build.pk,
+                    commit=last_build.commit,
+                    status=status,
+                    link_to_build=True,
+                )
+        except Exception:
+            log.exception(
+                "Failed to send status: project=%s version=%s",
+                version.project.slug, version.slug,
+            )
+        else:
+            log.info(
+                "Removing external version. project=%s version=%s",
+                version.project.slug, version.slug,
+            )
+            version.delete()
+
+
 @app.task(
     max_retries=1,
     default_retry_delay=60,
     queue='web'
 )
-def sync_versions_task(project_pk, version_data, **kwargs):
+def sync_versions_task(project_pk, tags_data, branches_data, **kwargs):
     """
     Sync the version data in the repo (on the build server).
 
     Version data in the repo is synced with what we have in the database.
 
+    :param version_data: a dictionary of branches and tags from the project.
     :returns: the identifiers for the versions that have been deleted.
     """
     project = Project.objects.get(pk=project_pk)
 
     # If the currently highest non-prerelease version is active, then make
     # the new latest version active as well.
-    old_highest_version = determine_stable_version(project.versions.all())
-    if old_highest_version is not None:
-        activate_new_stable = old_highest_version.active
+    current_stable = project.get_original_stable_version()
+    if current_stable is not None:
+        activate_new_stable = current_stable.active
     else:
         activate_new_stable = False
 
     try:
         # Update All Versions
         added_versions = set()
-        if 'tags' in version_data:
+        if tags_data:
             ret_set = sync_versions_to_db(
                 project=project,
-                versions=version_data['tags'],
+                versions=tags_data,
                 type=TAG,
             )
             added_versions.update(ret_set)
-        if 'branches' in version_data:
+        if branches_data:
             ret_set = sync_versions_to_db(
                 project=project,
-                versions=version_data['branches'],
+                versions=branches_data,
                 type=BRANCH,
             )
             added_versions.update(ret_set)
-        deleted_versions = delete_versions_from_db(project, version_data)
+        deleted_versions = delete_versions_from_db(
+            project=project,
+            tags_data=tags_data,
+            branches_data=branches_data,
+        )
+        deleted_active_versions = get_deleted_active_versions(
+            project=project,
+            tags_data=tags_data,
+            branches_data=branches_data,
+        )
     except Exception:
         log.exception('Sync Versions Error')
+        return [], []
 
     try:
         # The order of added_versions isn't deterministic.
         # We don't track the commit time or any other metadata.
         # We usually have one version added per webhook.
-        run_automation_rules(project, added_versions)
+        run_automation_rules(project, added_versions, deleted_active_versions)
     except Exception:
         # Don't interrupt the request if something goes wrong
         # in the automation rules.
@@ -268,41 +309,4 @@ def sync_versions_task(project_pk, version_data, **kwargs):
             promoted_version.save()
             trigger_build(project=project, version=promoted_version)
 
-    return (list(added_versions), list(deleted_versions))
-
-
-def delete_inactive_external_versions(limit=200, days=30 * 3):
-    """
-    Delete external versions that have been marked as inactive after ``days``.
-
-    The commit status is updated to link to the build page, as the docs are removed.
-    """
-    days_ago = datetime.now() - timedelta(days=days)
-    queryset = Version.external.filter(
-        active=False,
-        modified__lte=days_ago,
-    )[:limit]
-    for version in queryset:
-        try:
-            last_build = version.last_build
-            if last_build:
-                status = BUILD_STATUS_PENDING
-                if last_build.finished:
-                    status = BUILD_STATUS_SUCCESS if last_build.success else BUILD_STATUS_FAILURE
-                send_build_status(
-                    build_pk=last_build.pk,
-                    commit=last_build.commit,
-                    status=status,
-                    link_to_build=True,
-                )
-        except Exception:
-            log.exception(
-                "Failed to send status: project=%s version=%s",
-                version.project.slug, version.slug,
-            )
-        else:
-            log.info(
-                "Removing external version. project=%s version=%s",
-                version.project.slug, version.slug,
-            )
-            version.delete()
+    return list(added_versions), list(deleted_versions)