diff --git a/readthedocs/api/v3/filters.py b/readthedocs/api/v3/filters.py index cbe9695dc89..9768aca8c3c 100644 --- a/readthedocs/api/v3/filters.py +++ b/readthedocs/api/v3/filters.py @@ -34,6 +34,7 @@ class Meta: fields = [ "verbose_name", "privacy_level", + "hidden", "active", "built", "uploaded", diff --git a/readthedocs/api/v3/proxied_urls.py b/readthedocs/api/v3/proxied_urls.py index 28a3ee75b76..09d5d77a782 100644 --- a/readthedocs/api/v3/proxied_urls.py +++ b/readthedocs/api/v3/proxied_urls.py @@ -10,9 +10,12 @@ from readthedocs.api.v3.proxied_views import ProxiedEmbedAPI from readthedocs.search.api.v3.views import ProxiedSearchAPI +from .urls import router + api_proxied_urls = [ path("embed/", ProxiedEmbedAPI.as_view(), name="embed_api_v3"), path("search/", ProxiedSearchAPI.as_view(), name="search_api_v3"), ] urlpatterns = api_proxied_urls +urlpatterns += router.urls diff --git a/readthedocs/api/v3/serializers.py b/readthedocs/api/v3/serializers.py index b271cf255fd..b932081bd0c 100644 --- a/readthedocs/api/v3/serializers.py +++ b/readthedocs/api/v3/serializers.py @@ -46,9 +46,16 @@ class Meta: class BaseLinksSerializer(serializers.Serializer): - def _absolute_url(self, path): + def _absolute_url(self, path, dashboard=False): scheme = "http" if settings.DEBUG else "https" - domain = settings.PRODUCTION_DOMAIN + request = self.context.get("request") + + if dashboard: + domain = settings.PRODUCTION_DOMAIN + elif request: + domain = request.get_host() + else: + domain = settings.PUBLIC_DOMAIN return urllib.parse.urlunparse((scheme, domain, path, "", "", "")) @@ -128,7 +135,7 @@ class BuildURLsSerializer(BaseLinksSerializer, serializers.Serializer): def get_project(self, obj): path = reverse("projects_detail", kwargs={"project_slug": obj.project.slug}) - return self._absolute_url(path) + return self._absolute_url(path, dashboard=True) def get_version(self, obj): if obj.version: @@ -139,7 +146,7 @@ def get_version(self, obj): "version_slug": obj.version.slug, }, ) - return self._absolute_url(path) + return self._absolute_url(path, dashboard=True) return None @@ -316,7 +323,7 @@ def get_edit(self, obj): "version_slug": obj.slug, }, ) - return self._absolute_url(path) + return self._absolute_url(path, dashboard=True) class VersionURLsSerializer(BaseLinksSerializer, serializers.Serializer): @@ -358,17 +365,13 @@ class Meta: "privacy_level", ] - def __init__(self, *args, resolver=None, version_serializer=None, **kwargs): + def __init__(self, *args, resolver=None, **kwargs): super().__init__(*args, **kwargs) # Use a shared resolver to reduce the amount of DB queries while # resolving version URLs. self.resolver = kwargs.pop("resolver", Resolver()) - # Allow passing a specific serializer when initializing it. - # This is required to pass ``VersionSerializerNoLinks`` from the addons API. - self.version_serializer = version_serializer or VersionSerializer - def get_downloads(self, obj): downloads = obj.get_downloads() data = {} @@ -390,7 +393,7 @@ def get_aliases(self, obj): if obj.slug == LATEST: alias_version = obj.project.get_original_latest_version() if alias_version and alias_version.active: - return [self.version_serializer(alias_version).data] + return [VersionSerializer(alias_version).data] return [] @@ -460,19 +463,19 @@ class ProjectURLsSerializer(BaseLinksSerializer, serializers.Serializer): def get_home(self, obj): path = reverse("projects_detail", kwargs={"project_slug": obj.slug}) - return self._absolute_url(path) + return self._absolute_url(path, dashboard=True) def get_builds(self, obj): path = reverse("builds_project_list", kwargs={"project_slug": obj.slug}) - return self._absolute_url(path) + return self._absolute_url(path, dashboard=True) def get_versions(self, obj): path = reverse("project_version_list", kwargs={"project_slug": obj.slug}) - return self._absolute_url(path) + return self._absolute_url(path, dashboard=True) def get_downloads(self, obj): path = reverse("project_downloads", kwargs={"project_slug": obj.slug}) - return self._absolute_url(path) + return self._absolute_url(path, dashboard=True) def get_documentation(self, obj): version_slug = getattr(self.parent, "version_slug", None) @@ -500,6 +503,7 @@ class ProjectLinksSerializer(BaseLinksSerializer): translations = serializers.SerializerMethodField() notifications = serializers.SerializerMethodField() sync_versions = serializers.SerializerMethodField() + filetreediff = serializers.SerializerMethodField() def get__self(self, obj): path = reverse("projects-detail", kwargs={"project_slug": obj.slug}) @@ -541,6 +545,15 @@ def get_builds(self, obj): ) return self._absolute_url(path) + def get_filetreediff(self, obj): + path = reverse( + "projects-filetreediff", + kwargs={ + "project_slug": obj.slug, + }, + ) + return self._absolute_url(path) + def get_subprojects(self, obj): path = reverse( "projects-subprojects-list", diff --git a/readthedocs/api/v3/urls.py b/readthedocs/api/v3/urls.py index f20cc6705a7..1a4b5ee9c4d 100644 --- a/readthedocs/api/v3/urls.py +++ b/readthedocs/api/v3/urls.py @@ -26,6 +26,7 @@ # allows /api/v3/projects/pip/ # allows /api/v3/projects/pip/superproject/ # allows /api/v3/projects/pip/sync-versions/ +# allows /api/v3/projects/pip/filetreediff/ projects = router.register( r"projects", ProjectsViewSet, diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index 5376a081f4c..544b0c39330 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -1,7 +1,9 @@ import django_filters.rest_framework as filters +from django.conf import settings from django.contrib.auth.models import User from django.contrib.contenttypes.models import ContentType -from django.db.models import Exists, OuterRef +from django.db.models import Case, Exists, OuterRef, When +from django.shortcuts import get_object_or_404 from rest_flex_fields import is_expanded from rest_flex_fields.views import FlexFieldsMixin from rest_framework import status @@ -19,16 +21,17 @@ from rest_framework.permissions import IsAuthenticated from rest_framework.renderers import BrowsableAPIRenderer from rest_framework.response import Response -from rest_framework.throttling import AnonRateThrottle, UserRateThrottle from rest_framework.viewsets import GenericViewSet, ModelViewSet, ReadOnlyModelViewSet from rest_framework_extensions.mixins import NestedViewSetMixin from readthedocs.api.v2.permissions import ReadOnlyPermission from readthedocs.builds.constants import EXTERNAL from readthedocs.builds.models import Build, Version +from readthedocs.core.resolver import Resolver from readthedocs.core.utils import trigger_build from readthedocs.core.utils.extend import SettingsOverrideObject from readthedocs.core.views.hooks import trigger_sync_versions +from readthedocs.filetreediff import get_diff from readthedocs.notifications.models import Notification from readthedocs.oauth.models import ( RemoteOrganization, @@ -36,11 +39,23 @@ RemoteRepositoryRelation, ) from readthedocs.organizations.models import Organization, Team +from readthedocs.projects.constants import ( + ADDONS_FLYOUT_SORTING_CALVER, + ADDONS_FLYOUT_SORTING_CUSTOM_PATTERN, + ADDONS_FLYOUT_SORTING_PYTHON_PACKAGING, + ADDONS_FLYOUT_SORTING_SEMVER_READTHEDOCS_COMPATIBLE, +) from readthedocs.projects.models import ( EnvironmentVariable, Project, ProjectRelationship, ) +from readthedocs.projects.version_handling import ( + comparable_version, + sort_versions_calver, + sort_versions_custom_pattern, + sort_versions_python_packaging, +) from readthedocs.projects.views.mixins import ProjectImportMixin from readthedocs.redirects.models import Redirect @@ -110,7 +125,7 @@ class APIv3Settings: LimitOffsetPagination.default_limit = 10 renderer_classes = (AlphabeticalSortedJSONRenderer, BrowsableAPIRenderer) - throttle_classes = (UserRateThrottle, AnonRateThrottle) + # throttle_classes = (UserRateThrottle, AnonRateThrottle) filter_backends = (filters.DjangoFilterBackend,) metadata_class = SimpleMetadata @@ -139,8 +154,12 @@ class ProjectsViewSetBase( ] def get_permissions(self): + # Endpoints under DOC_PATH_PREFIX (served by El Proxito) are always read-only. + if self.request.path.startswith(f"/{settings.DOC_PATH_PREFIX}"): + permission_classes = [ReadOnlyPermission] + # Create and list are actions that act on the current user. - if self.action in ("create", "list"): + elif self.action in ("create", "list"): permission_classes = [IsAuthenticated] # Actions that change the state of the project require admin permissions on the project. elif self.action in ("update", "partial_update", "destroy", "sync_versions"): @@ -257,6 +276,134 @@ def sync_versions(self, request, project_slug): code = status.HTTP_400_BAD_REQUEST return Response(data=data, status=code) + def _get_filetreediff_response(self, *, request, project, version): + """ + Get the file tree diff response for the given version. + + This response is only enabled for external versions, + we do the comparison between the current version and the latest version. + """ + if not version.is_external: + return None + + if not project.addons.filetreediff_enabled: + return None + + base_version = ( + project.addons.options_base_version or project.get_latest_version() + ) + # TODO: check if `self._has_permission` is important after the migration here. + # if not base_version or not self._has_permission( + # request=request, version=base_version + # ): + if not base_version: + return None + + diff = get_diff(version_a=version, version_b=base_version) + if not diff: + return None + + resolver = Resolver() + return { + "outdated": diff.outdated, + "diff": { + "added": [ + { + "filename": filename, + "urls": { + "current": resolver.resolve_version( + project=project, + filename=filename, + version=version, + ), + "base": resolver.resolve_version( + project=project, + filename=filename, + version=base_version, + ), + }, + } + for filename in diff.added + ], + "deleted": [ + { + "filename": filename, + "urls": { + "current": resolver.resolve_version( + project=project, + filename=filename, + version=version, + ), + "base": resolver.resolve_version( + project=project, + filename=filename, + version=base_version, + ), + }, + } + for filename in diff.deleted + ], + "modified": [ + { + "filename": filename, + "urls": { + "current": resolver.resolve_version( + project=project, + filename=filename, + version=version, + ), + "base": resolver.resolve_version( + project=project, + filename=filename, + version=base_version, + ), + }, + } + for filename in diff.modified + ], + }, + } + + @action(detail=True, methods=["get"], url_path="filetreediff") + def filetreediff(self, request, project_slug): + project = self.get_object() + + current_version_slug = request.GET.get("current-version", None) + base_version_slug = request.GET.get("base-version", None) + + current_version = get_object_or_404( + Version.objects.public( + user=request.user, + project=project, + only_active=True, + include_hidden=True, + only_built=True, + ).filter( + slug=current_version_slug, + ) + ) + base_version = get_object_or_404( + Version.objects.public( + user=request.user, + project=project, + only_active=True, + include_hidden=True, + only_built=True, + ).filter( + slug=base_version_slug, + ) + ) + + data = ( + self._get_filetreediff_response( + request=request, + project=project, + version=current_version, + ) + or {} + ) + return Response(data=data) + class ProjectsViewSet(SettingsOverrideObject): _default_class = ProjectsViewSetBase @@ -379,7 +526,78 @@ def update(self, request, *args, **kwargs): def get_queryset(self): """Overridden to allow internal versions only.""" - return super().get_queryset().exclude(type=EXTERNAL) + queryset = super().get_queryset().exclude(type=EXTERNAL) + + sorting = self.request.GET.get("sorting") + sorted_versions_active_built_not_hidden = None + if sorting: + project = self._get_parent_project() + versions_active_built_not_hidden = queryset.select_related( + "project" + ).order_by("-slug") + sorted_versions_active_built_not_hidden = versions_active_built_not_hidden + if not project.supports_multiple_versions: + # Return only one version when the project doesn't support multiple versions. + # That version is the only one the project serves. + sorted_versions_active_built_not_hidden = ( + versions_active_built_not_hidden.filter( + slug=project.get_default_version() + ) + ) + else: + if ( + project.addons.flyout_sorting + == ADDONS_FLYOUT_SORTING_SEMVER_READTHEDOCS_COMPATIBLE + ): + sorted_versions_active_built_not_hidden = sorted( + versions_active_built_not_hidden, + key=lambda version: comparable_version( + version.verbose_name, + repo_type=project.repo_type, + ), + reverse=True, + ) + elif ( + project.addons.flyout_sorting + == ADDONS_FLYOUT_SORTING_PYTHON_PACKAGING + ): + sorted_versions_active_built_not_hidden = ( + sort_versions_python_packaging( + versions_active_built_not_hidden, + project.addons.flyout_sorting_latest_stable_at_beginning, + ) + ) + elif project.addons.flyout_sorting == ADDONS_FLYOUT_SORTING_CALVER: + sorted_versions_active_built_not_hidden = sort_versions_calver( + versions_active_built_not_hidden, + project.addons.flyout_sorting_latest_stable_at_beginning, + ) + elif ( + project.addons.flyout_sorting + == ADDONS_FLYOUT_SORTING_CUSTOM_PATTERN + ): + sorted_versions_active_built_not_hidden = ( + sort_versions_custom_pattern( + versions_active_built_not_hidden, + project.addons.flyout_sorting_custom_pattern, + project.addons.flyout_sorting_latest_stable_at_beginning, + ) + ) + + # Sort versions in the database so the queryset can continue working in the rest of the view. + # Borrowed from: http://rednafi.com/python/sort_by_a_custom_sequence_in_django/ + version_ids = [ + version.pk for version in sorted_versions_active_built_not_hidden + ] + preferred = Case( + *( + When(id=id, then=position) + for position, id in enumerate(version_ids, start=1) + ) + ) + queryset = queryset.order_by(preferred) + + return queryset class BuildsViewSet( diff --git a/readthedocs/proxito/views/hosting.py b/readthedocs/proxito/views/hosting.py index 933f359fbec..f027e0cb741 100644 --- a/readthedocs/proxito/views/hosting.py +++ b/readthedocs/proxito/views/hosting.py @@ -1,4 +1,5 @@ """Views for hosting features.""" +import urllib.parse from functools import lru_cache import packaging @@ -7,6 +8,7 @@ from django.contrib.auth.models import AnonymousUser from django.http import Http404, JsonResponse from django.shortcuts import get_object_or_404 +from django.urls import reverse from rest_framework import permissions from rest_framework.renderers import JSONRenderer from rest_framework.views import APIView @@ -23,7 +25,6 @@ from readthedocs.core.resolver import Resolver from readthedocs.core.unresolver import UnresolverError, unresolver from readthedocs.core.utils.extend import SettingsOverrideObject -from readthedocs.filetreediff import get_diff from readthedocs.projects.constants import ( ADDONS_FLYOUT_SORTING_CALVER, ADDONS_FLYOUT_SORTING_CUSTOM_PATTERN, @@ -222,54 +223,6 @@ def get(self, request, *args, **kwargs): return JsonResponse(data, json_dumps_params={"indent": 4, "sort_keys": True}) -class NoLinksMixin: - - """Mixin to remove conflicting fields from serializers.""" - - FIELDS_TO_REMOVE = ("_links",) - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - for field in self.FIELDS_TO_REMOVE: - if field in self.fields: - del self.fields[field] - - if field in self.Meta.fields: - del self.Meta.fields[self.Meta.fields.index(field)] - - -# NOTE: the following serializers are required only to remove some fields we -# can't expose yet in this API endpoint because it's running under El Proxito -# which cannot resolve URLs pointing to the APIv3 because they are not defined -# on El Proxito. -# -# See https://github.com/readthedocs/readthedocs-ops/issues/1323 -class ProjectSerializerNoLinks(NoLinksMixin, ProjectSerializer): - def __init__(self, *args, **kwargs): - resolver = kwargs.pop("resolver", Resolver()) - super().__init__( - *args, - resolver=resolver, - **kwargs, - ) - - -class VersionSerializerNoLinks(NoLinksMixin, VersionSerializer): - def __init__(self, *args, **kwargs): - resolver = kwargs.pop("resolver", Resolver()) - super().__init__( - *args, - resolver=resolver, - version_serializer=VersionSerializerNoLinks, - **kwargs, - ) - - -class BuildSerializerNoLinks(NoLinksMixin, BuildSerializer): - pass - - class AddonsResponseBase: def get( self, @@ -338,6 +291,8 @@ def _v1(self, project, version, build, filename, url, request): sorted_versions_active_built_not_hidden = Version.objects.none() user = request.user + # TODO: remvoe the sorting code from here when addons PR is deployed + # https://github.com/readthedocs/addons/pull/468 versions_active_built_not_hidden = ( self._get_versions(request, project) .select_related("project") @@ -406,36 +361,6 @@ def _v1(self, project, version, build, filename, url, request): data = { "api_version": "1", - "projects": { - "current": ProjectSerializerNoLinks( - project, - resolver=resolver, - version_slug=version.slug if version else None, - ).data, - "translations": ProjectSerializerNoLinks( - project_translations, - resolver=resolver, - version_slug=version.slug if version else None, - many=True, - ).data, - }, - "versions": { - "current": VersionSerializerNoLinks( - version, - resolver=resolver, - ).data - if version - else None, - # These are "sorted active, built, not hidden versions" - "active": VersionSerializerNoLinks( - sorted_versions_active_built_not_hidden, - resolver=resolver, - many=True, - ).data, - }, - "builds": { - "current": BuildSerializerNoLinks(build).data if build else None, - }, # TODO: consider creating one serializer per field here. # The resulting JSON will be the same, but maybe it's easier/cleaner? "domains": { @@ -529,21 +454,132 @@ def _v1(self, project, version, build, filename, url, request): }, }, "filetreediff": { - "enabled": False, + "enabled": project.addons.filetreediff_enabled, }, }, + # TODO: remove `projects`, `versions` and `builds` once we have deployed the client + # version that uses `/_/api/v3/` endpoints: + # https://github.com/readthedocs/addons/pull/468 + "projects": { + "current": ProjectSerializer( + project, + resolver=resolver, + version_slug=version.slug if version else None, + context={"request": request}, + ).data, + "translations": ProjectSerializer( + project_translations, + resolver=resolver, + version_slug=version.slug if version else None, + context={"request": request}, + many=True, + ).data, + }, + "versions": { + "current": VersionSerializer( + version, + resolver=resolver, + context={"request": request}, + ).data + if version + else None, + # These are "sorted active, built, not hidden versions" + "active": VersionSerializer( + sorted_versions_active_built_not_hidden, + resolver=resolver, + context={"request": request}, + many=True, + ).data, + }, + "builds": { + "current": BuildSerializer( + build, + context={"request": request}, + ).data + if build + else None, + }, } - if version: - response = self._get_filetreediff_response( - request=request, - project=project, - version=version, - resolver=resolver, + if project and version and build: + base_version_slug = ( + project.addons.options_base_version.slug + if project.addons.options_base_version + else LATEST ) - if response: - data["addons"]["filetreediff"].update(response) + data["readthedocs"] = { + "urls": { + "api": { + "v3": { + "projects": { + "current": reverse( + "projects-detail", + kwargs={ + "project_slug": project.slug, + }, + ), + "translations": reverse( + "projects-translations-list", + kwargs={ + "parent_lookup_main_language_project__slug": project.slug, + }, + ), + }, + "versions": { + "current": reverse( + "projects-versions-detail", + kwargs={ + "parent_lookup_project__slug": project.slug, + "version_slug": version.slug, + }, + ), + "active": reverse( + "projects-versions-list", + kwargs={ + "parent_lookup_project__slug": project.slug, + }, + ) + + "?" + # TODO: what's the best way to accept the `?sorting=` attribute in the DRF view? + + urllib.parse.urlencode( + { + "active": True, + "built": True, + "hidden": False, + "sorting": project.addons.flyout_sorting, + "limit": 50, + }, + ), + }, + "builds": { + "current": reverse( + "projects-builds-detail", + kwargs={ + "parent_lookup_project__slug": project.slug, + "build_pk": build.pk, + }, + ), + }, + # project.addons.options_base_version.slug + "filetreediff": reverse( + "projects-filetreediff", + kwargs={ + "project_slug": project.slug, + }, + ) + + "?" + + urllib.parse.urlencode( + { + "base-version": base_version_slug, + "current-version": version.slug, + } + ), + }, + }, + } + } + if version: # Show the subprojects filter on the parent project and subproject # TODO: Remove these queries and try to find a way to get this data # from the resolver, which has already done these queries. @@ -621,92 +657,6 @@ def _v1(self, project, version, build, filename, url, request): return data - def _get_filetreediff_response(self, *, request, project, version, resolver): - """ - Get the file tree diff response for the given version. - - This response is only enabled for external versions, - we do the comparison between the current version and the latest version. - """ - if not version.is_external: - return None - - if not project.addons.filetreediff_enabled: - return None - - base_version = ( - project.addons.options_base_version or project.get_latest_version() - ) - if not base_version or not self._has_permission( - request=request, version=base_version - ): - return None - - diff = get_diff(version_a=version, version_b=base_version) - if not diff: - return None - - return { - "enabled": True, - "outdated": diff.outdated, - "diff": { - "added": [ - { - "filename": filename, - "urls": { - "current": resolver.resolve_version( - project=project, - filename=filename, - version=version, - ), - "base": resolver.resolve_version( - project=project, - filename=filename, - version=base_version, - ), - }, - } - for filename in diff.added - ], - "deleted": [ - { - "filename": filename, - "urls": { - "current": resolver.resolve_version( - project=project, - filename=filename, - version=version, - ), - "base": resolver.resolve_version( - project=project, - filename=filename, - version=base_version, - ), - }, - } - for filename in diff.deleted - ], - "modified": [ - { - "filename": filename, - "urls": { - "current": resolver.resolve_version( - project=project, - filename=filename, - version=version, - ), - "base": resolver.resolve_version( - project=project, - filename=filename, - version=base_version, - ), - }, - } - for filename in diff.modified - ], - }, - } - def _v2(self, project, version, build, filename, url, user): return { "api_version": "2",