From 31b9bf289b2213ef0b6e748b0987f9487c7b09ff Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 30 May 2019 16:59:23 +0600 Subject: [PATCH 01/52] Version Type Added --- readthedocs/builds/constants.py | 2 ++ .../0008_added_pull_request_version_type.py | 25 +++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 readthedocs/builds/migrations/0008_added_pull_request_version_type.py diff --git a/readthedocs/builds/constants.py b/readthedocs/builds/constants.py index b0dc6cfba94..e2e4fc44ac3 100644 --- a/readthedocs/builds/constants.py +++ b/readthedocs/builds/constants.py @@ -33,11 +33,13 @@ BRANCH = 'branch' TAG = 'tag' +PULL_REQUEST = 'pull_request' UNKNOWN = 'unknown' VERSION_TYPES = ( (BRANCH, _('Branch')), (TAG, _('Tag')), + (PULL_REQUEST, _('Pull Request')), (UNKNOWN, _('Unknown')), ) diff --git a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py b/readthedocs/builds/migrations/0008_added_pull_request_version_type.py new file mode 100644 index 00000000000..14d8968866d --- /dev/null +++ b/readthedocs/builds/migrations/0008_added_pull_request_version_type.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.20 on 2019-05-30 10:02 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('builds', '0007_add-automation-rules'), + ] + + operations = [ + migrations.AlterField( + model_name='version', + name='type', + field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('pull_request', 'Pull Request'), ('unknown', 'Unknown')], default='unknown', max_length=20, verbose_name='Type'), + ), + migrations.AlterField( + model_name='versionautomationrule', + name='version_type', + field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('pull_request', 'Pull Request'), ('unknown', 'Unknown')], max_length=32, verbose_name='Version type'), + ), + ] From bd119944f3010a83713a3a7a80f3d49d9098e385 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 30 May 2019 23:52:36 +0600 Subject: [PATCH 02/52] Version Model Methods Updated --- readthedocs/builds/models.py | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index 1039b815326..ca44b39f252 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -40,6 +40,7 @@ BUILD_TYPES, LATEST, NON_REPOSITORY_VERSIONS, + PULL_REQUEST, STABLE, TAG, VERSION_TYPES, @@ -142,7 +143,8 @@ def vcs_url(self): """ Generate VCS (github, gitlab, bitbucket) URL for this version. - Example: https://github.com/rtfd/readthedocs.org/tree/3.4.2/. + Branch/Tag Example: https://github.com/rtfd/readthedocs.org/tree/3.4.2/. + Pull Request Example: https://github.com/rtfd/readthedocs.org/pull/9999/. """ url = '' if self.slug == STABLE: @@ -152,12 +154,24 @@ def vcs_url(self): else: slug_url = self.slug - if ('github' in self.project.repo) or ('gitlab' in self.project.repo): - url = f'/tree/{slug_url}/' + if self.type == PULL_REQUEST: + if 'github' in self.project.repo: + url = f'/pull/{slug_url}/' - if 'bitbucket' in self.project.repo: - slug_url = self.identifier - url = f'/src/{slug_url}' + if 'gitlab' in self.project.repo: + slug_url = self.identifier + url = f'/merge_requests/{slug_url}/' + + if 'bitbucket' in self.project.repo: + slug_url = self.identifier + url = f'/pull-requests/{slug_url}' + else: + if ('github' in self.project.repo) or ('gitlab' in self.project.repo): + url = f'/tree/{slug_url}/' + + if 'bitbucket' in self.project.repo: + slug_url = self.identifier + url = f'/src/{slug_url}' # TODO: improve this replacing return self.project.repo.replace('git://', 'https://').replace('.git', '') + url @@ -220,7 +234,14 @@ def commit_name(self): # the actual tag name. return self.verbose_name - # If we came that far it's not a special version nor a branch or tag. + if self.type == PULL_REQUEST: + # If this version is a Pull Request, the identifier will + # contain the actual commit hash. which we can use to + # generate url for a given file name + return self.identifier + + # If we came that far it's not a special version + # nor a branch, tag or Pull Request. # Therefore just return the identifier to make a safe guess. log.debug( 'TODO: Raise an exception here. Testing what cases it happens', From 8f3e64b70712766aa1b85dcfef5b5a75d2f10aff Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 4 Jun 2019 15:40:32 +0600 Subject: [PATCH 03/52] Internal and External Version Manager added --- readthedocs/builds/managers.py | 33 +++++++++++++++++++++++++++++++++ readthedocs/builds/models.py | 10 +++++++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/readthedocs/builds/managers.py b/readthedocs/builds/managers.py index 68cdec6efe9..991293beae2 100644 --- a/readthedocs/builds/managers.py +++ b/readthedocs/builds/managers.py @@ -19,6 +19,7 @@ STABLE, STABLE_VERBOSE_NAME, TAG, + PULL_REQUEST, ) from .querysets import VersionQuerySet @@ -85,6 +86,38 @@ def get_object_or_log(self, **kwargs): log.warning('Version not found for given kwargs. %s' % kwargs) +class InternalVersionManagerBase(VersionManagerBase): + """ + Version manager that only includes internal version. + + It will exclude PULL_REQUEST type from the queries + and only include BRANCH, TAG, UNKONWN type Versions. + """ + def get_queryset(self): + return super().get_queryset().exclude(type=PULL_REQUEST) + + +class ExternalVersionManagerBase(VersionManagerBase): + """ + Version manager that only includes external version. + + It will only include PULL_REQUEST type Versions in the queries. + """ + def get_queryset(self): + return super().get_queryset().filter(type=PULL_REQUEST) + + class VersionManager(SettingsOverrideObject): _default_class = VersionManagerBase _override_setting = 'VERSION_MANAGER' + + +class InternalVersionManager(SettingsOverrideObject): + _default_class = InternalVersionManagerBase + _override_setting = 'INTERNAL_VERSION_MANAGER' + + +class ExternalVersionManager(SettingsOverrideObject): + _default_class = ExternalVersionManagerBase + _override_setting = 'EXTERNAL_VERSION_MANAGER' + diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index ca44b39f252..bca375570c9 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -45,7 +45,11 @@ TAG, VERSION_TYPES, ) -from .managers import VersionManager +from .managers import ( + VersionManager, + InternalVersionManager, + ExternalVersionManager +) from .querysets import BuildQuerySet, RelatedBuildQuerySet, VersionQuerySet from .utils import ( get_bitbucket_username_repo, @@ -112,6 +116,10 @@ class Version(models.Model): machine = models.BooleanField(_('Machine Created'), default=False) objects = VersionManager.from_queryset(VersionQuerySet)() + # Only include BRANCH, TAG, UNKONWN type Versions. + internal = InternalVersionManager.from_queryset(VersionQuerySet)() + # Only include PULL_REQUEST type Versions. + external = ExternalVersionManager.from_queryset(VersionQuerySet)() class Meta: unique_together = [('project', 'slug')] From 66200a0b1405f9ae301c8c581324d5420b204250 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 4 Jun 2019 16:20:19 +0600 Subject: [PATCH 04/52] lint fix --- readthedocs/builds/managers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/readthedocs/builds/managers.py b/readthedocs/builds/managers.py index 991293beae2..2a8bb4840d9 100644 --- a/readthedocs/builds/managers.py +++ b/readthedocs/builds/managers.py @@ -87,22 +87,26 @@ def get_object_or_log(self, **kwargs): class InternalVersionManagerBase(VersionManagerBase): + """ Version manager that only includes internal version. It will exclude PULL_REQUEST type from the queries and only include BRANCH, TAG, UNKONWN type Versions. """ + def get_queryset(self): return super().get_queryset().exclude(type=PULL_REQUEST) class ExternalVersionManagerBase(VersionManagerBase): + """ Version manager that only includes external version. It will only include PULL_REQUEST type Versions in the queries. """ + def get_queryset(self): return super().get_queryset().filter(type=PULL_REQUEST) @@ -120,4 +124,3 @@ class InternalVersionManager(SettingsOverrideObject): class ExternalVersionManager(SettingsOverrideObject): _default_class = ExternalVersionManagerBase _override_setting = 'EXTERNAL_VERSION_MANAGER' - From 34e4134d17622291605d50118c5967f64eaf0177 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 5 Jun 2019 01:18:20 +0600 Subject: [PATCH 05/52] All Version Querysets Updated with InternalVersionManager --- readthedocs/api/v2/views/model_views.py | 2 +- readthedocs/api/v3/views.py | 2 +- readthedocs/builds/models.py | 4 +- readthedocs/builds/views.py | 2 +- .../core/management/commands/update_repos.py | 42 +++++++++++++++++++ .../management/commands/update_versions.py | 2 +- readthedocs/core/views/serve.py | 7 ++-- readthedocs/projects/admin.py | 4 +- readthedocs/projects/forms.py | 2 +- readthedocs/projects/models.py | 8 ++-- readthedocs/projects/views/public.py | 6 +-- 11 files changed, 63 insertions(+), 18 deletions(-) diff --git a/readthedocs/api/v2/views/model_views.py b/readthedocs/api/v2/views/model_views.py index af41f28ab84..5f594a032b8 100644 --- a/readthedocs/api/v2/views/model_views.py +++ b/readthedocs/api/v2/views/model_views.py @@ -130,7 +130,7 @@ def active_versions(self, request, **kwargs): Project.objects.api(request.user), pk=kwargs['pk'], ) - versions = project.versions.filter(active=True) + versions = project.versions(manager='internal').filter(active=True) return Response({ 'versions': VersionSerializer(versions, many=True).data, }) diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index 0e8b2b55a0e..57b86360e96 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -230,7 +230,7 @@ class VersionsViewSet(APIv3Settings, NestedViewSetMixin, ProjectQuerySetMixin, lookup_value_regex = r'[^/]+' filterset_class = VersionFilter - queryset = Version.objects.all() + queryset = Version.internal.all() permit_list_expands = [ 'last_build', 'last_build.config', diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index bca375570c9..bcfc5dfff5c 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -142,7 +142,9 @@ def __str__(self): @property def ref(self): if self.slug == STABLE: - stable = determine_stable_version(self.project.versions.all()) + stable = determine_stable_version( + self.project.versions(manager='internal').all() + ) if stable: return stable.slug diff --git a/readthedocs/builds/views.py b/readthedocs/builds/views.py index e8e3d458e2c..3d6e2cc0ca1 100644 --- a/readthedocs/builds/views.py +++ b/readthedocs/builds/views.py @@ -93,7 +93,7 @@ def get_context_data(self, **kwargs): context['project'] = self.project context['active_builds'] = active_builds - context['versions'] = Version.objects.public( + context['versions'] = Version.internal.public( user=self.request.user, project=self.project, ) diff --git a/readthedocs/core/management/commands/update_repos.py b/readthedocs/core/management/commands/update_repos.py index 5852592d360..0e14f9ce339 100644 --- a/readthedocs/core/management/commands/update_repos.py +++ b/readthedocs/core/management/commands/update_repos.py @@ -76,6 +76,48 @@ def handle(self, *args, **options): version.pk, build_pk=build.pk, ) + elif version == 'internal': + log.info('Updating all internal versions for %s', slug) + for version in Version.internal.filter( + project__slug=slug, + active=True, + uploaded=False, + ): + + build = Build.objects.create( + project=version.project, + version=version, + type='html', + state='triggered', + ) + + # pylint: disable=no-value-for-parameter + tasks.update_docs_task( + version.project_id, + build_pk=build.pk, + version_pk=version.pk, + ) + elif version == 'external': + log.info('Updating all external versions for %s', slug) + for version in Version.external.filter( + project__slug=slug, + active=True, + uploaded=False, + ): + + build = Build.objects.create( + project=version.project, + version=version, + type='html', + state='triggered', + ) + + # pylint: disable=no-value-for-parameter + tasks.update_docs_task( + version.project_id, + build_pk=build.pk, + version_pk=version.pk, + ) else: p = Project.all_objects.get(slug=slug) log.info('Building %s', p) diff --git a/readthedocs/core/management/commands/update_versions.py b/readthedocs/core/management/commands/update_versions.py index b0aa1f877cb..6456accc430 100644 --- a/readthedocs/core/management/commands/update_versions.py +++ b/readthedocs/core/management/commands/update_versions.py @@ -13,7 +13,7 @@ class Command(BaseCommand): help = __doc__ def handle(self, *args, **options): - for version in Version.objects.filter(active=True, built=False): + for version in Version.internal.filter(active=True, built=False): # pylint: disable=no-value-for-parameter update_docs_task( version.pk, diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index 390063f256a..61ecc9f504a 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -383,7 +383,7 @@ def changefreqs_generator(): raise Http404 sorted_versions = sort_version_aware( - Version.objects.public( + Version.internal.public( project=project, only_active=True, ), @@ -409,8 +409,9 @@ def changefreqs_generator(): if project.translations.exists(): for translation in project.translations.all(): - translation_versions = translation.versions.public( - ).values_list('slug', flat=True) + translation_versions = translation.versions( + manager='internal' + ).public().values_list('slug', flat=True) if version.slug in translation_versions: href = project.get_docs_url( version_slug=version.slug, diff --git a/readthedocs/projects/admin.py b/readthedocs/projects/admin.py index d74e8d99075..fb4aef4772d 100644 --- a/readthedocs/projects/admin.py +++ b/readthedocs/projects/admin.py @@ -239,7 +239,7 @@ def reindex_active_versions(self, request, queryset): """Reindex all active versions of the selected projects to ES.""" qs_iterator = queryset.iterator() for project in qs_iterator: - version_qs = Version.objects.filter(project=project) + version_qs = Version.internal.filter(project=project) active_versions = version_qs.filter(active=True) if not active_versions.exists(): @@ -271,7 +271,7 @@ def wipe_all_versions(self, request, queryset): """Wipe indexes of all versions of selected projects.""" qs_iterator = queryset.iterator() for project in qs_iterator: - version_qs = Version.objects.filter(project=project) + version_qs = Version.internal.filter(project=project) if not version_qs.exists(): self.message_user( request, diff --git a/readthedocs/projects/forms.py b/readthedocs/projects/forms.py index a701a49c020..65c589c8b18 100644 --- a/readthedocs/projects/forms.py +++ b/readthedocs/projects/forms.py @@ -240,7 +240,7 @@ def __init__(self, *args, **kwargs): self.helper.add_input(Submit('save', _('Save'))) default_choice = (None, '-' * 9) - versions_choices = self.instance.versions.filter( + versions_choices = self.instance.versions(manager='internal').filter( machine=False).values_list('verbose_name', flat=True) self.fields['default_branch'].widget = forms.Select( diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index 19f58f7fab0..97f9b90c8d4 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -908,7 +908,7 @@ def api_versions(self): def active_versions(self): from readthedocs.builds.models import Version - versions = Version.objects.public(project=self, only_active=True) + versions = Version.internal.public(project=self, only_active=True) return ( versions.filter(built=True, active=True) | versions.filter(active=True, uploaded=True) @@ -922,7 +922,7 @@ def ordered_active_versions(self, user=None): } if user: kwargs['user'] = user - versions = Version.objects.public(**kwargs).select_related( + versions = Version.internal.public(**kwargs).select_related( 'project', 'project__main_language_project', ).prefetch_related( @@ -949,7 +949,7 @@ def all_active_versions(self): :returns: :py:class:`Version` queryset """ - return self.versions.filter(active=True) + return self.versions(manager='internal').filter(active=True) def get_stable_version(self): return self.versions.filter(slug=STABLE).first() @@ -961,7 +961,7 @@ def update_stable_version(self): Return ``None`` if no update was made or if there is no version on the project that can be considered stable. """ - versions = self.versions.all() + versions = self.versions(manager='internal').all() new_stable = determine_stable_version(versions) if new_stable: current_stable = self.get_stable_version() diff --git a/readthedocs/projects/views/public.py b/readthedocs/projects/views/public.py index 64d82272738..ea15bfb168b 100644 --- a/readthedocs/projects/views/public.py +++ b/readthedocs/projects/views/public.py @@ -93,7 +93,7 @@ def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) project = self.get_object() - context['versions'] = Version.objects.public( + context['versions'] = Version.internal.public( user=self.request.user, project=project, ) @@ -179,7 +179,7 @@ def project_downloads(request, project_slug): Project.objects.protected(request.user), slug=project_slug, ) - versions = Version.objects.public(user=request.user, project=project) + versions = Version.internal.public(user=request.user, project=project) versions = sort_version_aware(versions) version_data = OrderedDict() for version in versions: @@ -268,7 +268,7 @@ def project_versions(request, project_slug): slug=project_slug, ) - versions = Version.objects.public( + versions = Version.internal.public( user=request.user, project=project, only_active=False, From 3a3cfdf4f6fed6736453d7368e9961c5cb9da774 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 6 Jun 2019 02:15:34 +0600 Subject: [PATCH 06/52] Manager names moved to Constants --- readthedocs/api/v2/views/model_views.py | 4 ++-- readthedocs/builds/constants.py | 7 +++++++ readthedocs/builds/models.py | 5 +++-- readthedocs/core/management/commands/update_repos.py | 5 +++-- readthedocs/core/views/serve.py | 3 ++- readthedocs/projects/forms.py | 3 ++- readthedocs/projects/models.py | 6 +++--- 7 files changed, 22 insertions(+), 11 deletions(-) diff --git a/readthedocs/api/v2/views/model_views.py b/readthedocs/api/v2/views/model_views.py index 5f594a032b8..fca91b32626 100644 --- a/readthedocs/api/v2/views/model_views.py +++ b/readthedocs/api/v2/views/model_views.py @@ -10,7 +10,7 @@ from rest_framework.renderers import BaseRenderer, JSONRenderer from rest_framework.response import Response -from readthedocs.builds.constants import BRANCH, TAG +from readthedocs.builds.constants import BRANCH, TAG, INTERNAL from readthedocs.builds.models import Build, BuildCommandResult, Version from readthedocs.core.utils import trigger_build from readthedocs.core.utils.extend import SettingsOverrideObject @@ -130,7 +130,7 @@ def active_versions(self, request, **kwargs): Project.objects.api(request.user), pk=kwargs['pk'], ) - versions = project.versions(manager='internal').filter(active=True) + versions = project.versions(manager=INTERNAL).filter(active=True) return Response({ 'versions': VersionSerializer(versions, many=True).data, }) diff --git a/readthedocs/builds/constants.py b/readthedocs/builds/constants.py index e2e4fc44ac3..e200fc5fe25 100644 --- a/readthedocs/builds/constants.py +++ b/readthedocs/builds/constants.py @@ -55,3 +55,10 @@ LATEST, STABLE, ) + +# Manager name for Internal Versions or Builds. +# ie: Versions and Builds Excluding PULL_REQUEST Type. +INTERNAL = 'internal' +# Manager name for External Versions or Builds. +# ie: Only PULL_REQUEST Type Versions and Builds. +EXTERNAL = 'external' diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index bcfc5dfff5c..d0c5f6a854e 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -32,12 +32,13 @@ from readthedocs.projects.models import APIProject, Project from readthedocs.projects.version_handling import determine_stable_version -from .constants import ( +from readthedocs.builds.constants import ( BRANCH, BUILD_STATE, BUILD_STATE_FINISHED, BUILD_STATE_TRIGGERED, BUILD_TYPES, + INTERNAL, LATEST, NON_REPOSITORY_VERSIONS, PULL_REQUEST, @@ -143,7 +144,7 @@ def __str__(self): def ref(self): if self.slug == STABLE: stable = determine_stable_version( - self.project.versions(manager='internal').all() + self.project.versions(manager=INTERNAL).all() ) if stable: return stable.slug diff --git a/readthedocs/core/management/commands/update_repos.py b/readthedocs/core/management/commands/update_repos.py index 0e14f9ce339..b15ea8c3c9c 100644 --- a/readthedocs/core/management/commands/update_repos.py +++ b/readthedocs/core/management/commands/update_repos.py @@ -10,6 +10,7 @@ from django.core.management.base import BaseCommand +from readthedocs.builds.constants import EXTERNAL, INTERNAL from readthedocs.builds.models import Build, Version from readthedocs.core.utils import trigger_build from readthedocs.projects import tasks @@ -76,7 +77,7 @@ def handle(self, *args, **options): version.pk, build_pk=build.pk, ) - elif version == 'internal': + elif version == INTERNAL: log.info('Updating all internal versions for %s', slug) for version in Version.internal.filter( project__slug=slug, @@ -97,7 +98,7 @@ def handle(self, *args, **options): build_pk=build.pk, version_pk=version.pk, ) - elif version == 'external': + elif version == EXTERNAL: log.info('Updating all external versions for %s', slug) for version in Version.external.filter( project__slug=slug, diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index 61ecc9f504a..c6374d76cd6 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -38,6 +38,7 @@ from django.views.decorators.cache import cache_page from django.views.static import serve +from readthedocs.builds.constants import INTERNAL from readthedocs.builds.models import Version from readthedocs.core.permissions import AdminPermission from readthedocs.core.resolver import resolve, resolve_path @@ -410,7 +411,7 @@ def changefreqs_generator(): if project.translations.exists(): for translation in project.translations.all(): translation_versions = translation.versions( - manager='internal' + manager=INTERNAL ).public().values_list('slug', flat=True) if version.slug in translation_versions: href = project.get_docs_url( diff --git a/readthedocs/projects/forms.py b/readthedocs/projects/forms.py index 65c589c8b18..3c86cc39e12 100644 --- a/readthedocs/projects/forms.py +++ b/readthedocs/projects/forms.py @@ -14,6 +14,7 @@ from guardian.shortcuts import assign from textclassifier.validators import ClassifierValidator +from readthedocs.builds.constants import INTERNAL from readthedocs.core.utils import slugify, trigger_build from readthedocs.core.utils.extend import SettingsOverrideObject from readthedocs.integrations.models import Integration @@ -240,7 +241,7 @@ def __init__(self, *args, **kwargs): self.helper.add_input(Submit('save', _('Save'))) default_choice = (None, '-' * 9) - versions_choices = self.instance.versions(manager='internal').filter( + versions_choices = self.instance.versions(manager=INTERNAL).filter( machine=False).values_list('verbose_name', flat=True) self.fields['default_branch'].widget = forms.Select( diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index 97f9b90c8d4..e5d6c6c442d 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -20,7 +20,7 @@ from taggit.managers import TaggableManager from readthedocs.api.v2.client import api -from readthedocs.builds.constants import LATEST, STABLE +from readthedocs.builds.constants import LATEST, STABLE, INTERNAL from readthedocs.core.resolver import resolve, resolve_domain from readthedocs.core.utils import broadcast, slugify from readthedocs.projects import constants @@ -949,7 +949,7 @@ def all_active_versions(self): :returns: :py:class:`Version` queryset """ - return self.versions(manager='internal').filter(active=True) + return self.versions(manager=INTERNAL).filter(active=True) def get_stable_version(self): return self.versions.filter(slug=STABLE).first() @@ -961,7 +961,7 @@ def update_stable_version(self): Return ``None`` if no update was made or if there is no version on the project that can be considered stable. """ - versions = self.versions(manager='internal').all() + versions = self.versions(manager=INTERNAL).all() new_stable = determine_stable_version(versions) if new_stable: current_stable = self.get_stable_version() From 3d3de29bd0b26295215ac3c0d17b8454f2dc02b5 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 6 Jun 2019 21:25:03 +0600 Subject: [PATCH 07/52] Tests added --- readthedocs/builds/constants.py | 2 +- .../rtd_tests/tests/test_doc_serving.py | 23 +++- readthedocs/rtd_tests/tests/test_managers.py | 100 ++++++++++++++++++ readthedocs/rtd_tests/tests/test_project.py | 28 ++++- .../rtd_tests/tests/test_project_forms.py | 25 ++++- .../rtd_tests/tests/test_project_views.py | 30 +++++- readthedocs/rtd_tests/tests/test_version.py | 68 ++++++++++++ 7 files changed, 269 insertions(+), 7 deletions(-) create mode 100644 readthedocs/rtd_tests/tests/test_managers.py create mode 100644 readthedocs/rtd_tests/tests/test_version.py diff --git a/readthedocs/builds/constants.py b/readthedocs/builds/constants.py index e200fc5fe25..37474eb5365 100644 --- a/readthedocs/builds/constants.py +++ b/readthedocs/builds/constants.py @@ -58,7 +58,7 @@ # Manager name for Internal Versions or Builds. # ie: Versions and Builds Excluding PULL_REQUEST Type. -INTERNAL = 'internal' +INTERNAL = 'internal' # Manager name for External Versions or Builds. # ie: Only PULL_REQUEST Type Versions and Builds. EXTERNAL = 'external' diff --git a/readthedocs/rtd_tests/tests/test_doc_serving.py b/readthedocs/rtd_tests/tests/test_doc_serving.py index 134b191235c..ad7a56753fe 100644 --- a/readthedocs/rtd_tests/tests/test_doc_serving.py +++ b/readthedocs/rtd_tests/tests/test_doc_serving.py @@ -11,6 +11,7 @@ from django.urls import reverse from mock import mock_open, patch +from readthedocs.builds.constants import PULL_REQUEST, INTERNAL from readthedocs.builds.models import Version from readthedocs.core.middleware import SubdomainMiddleware from readthedocs.core.views import server_error_404_subdomain @@ -240,6 +241,16 @@ def test_sitemap_xml(self): project=self.public, active=True ) + # This is a Pull Request Version + pr_version = fixture.get( + Version, + identifier='pr-version', + verbose_name='pr-version', + slug='pr-9999', + project=self.public, + active=True, + type=PULL_REQUEST + ) # This also creates a Version `latest` Automatically for this project translation = fixture.get( Project, @@ -259,7 +270,7 @@ def test_sitemap_xml(self): ) self.assertEqual(response.status_code, 200) self.assertEqual(response['Content-Type'], 'application/xml') - for version in self.public.versions.filter(privacy_level=constants.PUBLIC): + for version in self.public.versions(manager=INTERNAL).filter(privacy_level=constants.PUBLIC): self.assertContains( response, self.public.get_docs_url( @@ -293,3 +304,13 @@ def test_sitemap_xml(self): # hreflang should use hyphen instead of underscore # in language and country value. (zh_CN should be zh-CN) self.assertContains(response, 'zh-CN') + + # PR Versions should not be in the sitemap_xml. + self.assertNotContains( + response, + self.public.get_docs_url( + version_slug=pr_version.slug, + lang_slug=self.public.language, + private=True, + ), + ) diff --git a/readthedocs/rtd_tests/tests/test_managers.py b/readthedocs/rtd_tests/tests/test_managers.py new file mode 100644 index 00000000000..8e440e18b85 --- /dev/null +++ b/readthedocs/rtd_tests/tests/test_managers.py @@ -0,0 +1,100 @@ +from django.test import TestCase +from django_dynamic_fixture import get + +from readthedocs.builds.constants import PULL_REQUEST, BRANCH, TAG +from readthedocs.builds.models import Version +from readthedocs.projects.constants import PUBLIC, PRIVATE, PROTECTED +from readthedocs.projects.models import Project + + +class TestVersionManagerBase(TestCase): + + fixtures = ['eric', 'test_data'] + + def setUp(self): + self.client.login(username='eric', password='test') + self.pip = Project.objects.get(slug='pip') + # Create a External Version. ie: PULL_REQUEST type Version. + self.public_pr_version = get( + Version, + project=self.pip, + active=True, + type=PULL_REQUEST, + privacy_level=PUBLIC + ) + self.private_pr_version = get( + Version, + project=self.pip, + active=True, + type=PULL_REQUEST, + privacy_level=PRIVATE + ) + self.protected_pr_version = get( + Version, + project=self.pip, + active=True, + type=PULL_REQUEST, + privacy_level=PROTECTED + ) + self.internal_versions = Version.objects.exclude(type=PULL_REQUEST) + + +class TestInternalVersionManager(TestVersionManagerBase): + + """ + Queries using Internal Manager should only include Internal Versions. + + It will exclude PULL_REQUEST type Versions from the queries + and only include BRANCH, TAG, UNKONWN type Versions. + """ + + def test_internal_version_manager_with_all(self): + self.assertNotIn(self.public_pr_version, Version.internal.all()) + + def test_internal_version_manager_with_public(self): + self.assertNotIn(self.public_pr_version, Version.internal.public()) + + def test_internal_version_manager_with_protected(self): + self.assertNotIn(self.protected_pr_version, Version.internal.protected()) + + def test_internal_version_manager_with_private(self): + self.assertNotIn(self.private_pr_version, Version.internal.private()) + + def test_internal_version_manager_with_api(self): + self.assertNotIn(self.public_pr_version, Version.internal.api()) + + def test_internal_version_manager_with_for_project(self): + self.assertNotIn(self.public_pr_version, Version.internal.for_project(self.pip)) + + +class TestExternalVersionManager(TestVersionManagerBase): + + """ + Queries using External Manager should only include External Versions. + + It will only include PULL_REQUEST type Versions in the queries. + """ + + def test_external_version_manager_with_all(self): + self.assertNotIn(self.internal_versions, Version.external.all()) + self.assertIn(self.public_pr_version, Version.external.all()) + + def test_external_version_manager_with_public(self): + self.assertNotIn(self.internal_versions, Version.external.public()) + self.assertIn(self.public_pr_version, Version.external.public()) + + def test_external_version_manager_with_protected(self): + self.assertNotIn(self.internal_versions, Version.external.protected()) + self.assertIn(self.protected_pr_version, Version.external.protected()) + + def test_external_version_manager_with_private(self): + self.assertNotIn(self.internal_versions, Version.external.private()) + self.assertIn(self.private_pr_version, Version.external.private()) + + def test_external_version_manager_with_api(self): + self.assertNotIn(self.internal_versions, Version.external.api()) + self.assertIn(self.public_pr_version, Version.external.api()) + + def test_external_version_manager_with_for_project(self): + self.assertNotIn(self.internal_versions, Version.external.for_project(self.pip)) + self.assertIn(self.public_pr_version, Version.external.for_project(self.pip)) diff --git a/readthedocs/rtd_tests/tests/test_project.py b/readthedocs/rtd_tests/tests/test_project.py index 40f86350ed4..c2db080f201 100644 --- a/readthedocs/rtd_tests/tests/test_project.py +++ b/readthedocs/rtd_tests/tests/test_project.py @@ -14,8 +14,9 @@ BUILD_STATE_FINISHED, BUILD_STATE_TRIGGERED, LATEST, + PULL_REQUEST, ) -from readthedocs.builds.models import Build +from readthedocs.builds.models import Build, Version from readthedocs.projects.exceptions import ProjectConfigurationError from readthedocs.projects.models import Project from readthedocs.projects.tasks import finish_inactive_builds @@ -29,6 +30,16 @@ class ProjectMixin: def setUp(self): self.client.login(username='eric', password='test') self.pip = Project.objects.get(slug='pip') + # Create a External Version. ie: PULL_REQUEST type Version. + self.pr_version = get( + Version, + identifier='pr-version', + verbose_name='pr-version', + slug='pr-9999', + project=self.pip, + active=True, + type=PULL_REQUEST + ) class TestProject(ProjectMixin, TestCase): @@ -134,6 +145,21 @@ def test_get_storage_path(self): 'htmlzip/pip/latest/pip.zip', ) + def test_ordered_active_versions_excludes_pr_versions(self): + self.assertNotIn(self.pr_version, self.pip.ordered_active_versions()) + + def test_active_versions_excludes_pr_versions(self): + self.assertNotIn(self.pr_version, self.pip.active_versions()) + + def test_all_active_versions_excludes_pr_versions(self): + self.assertNotIn(self.pr_version, self.pip.all_active_versions()) + + def test_update_stable_version_excludes_pr_versions(self): + # Delete all versions excluding PR Versions. + self.pip.versions.exclude(type=PULL_REQUEST).delete() + # Test that PR Version is not considered for stable. + self.assertEqual(self.pip.update_stable_version(), None) + class TestProjectTranslations(ProjectMixin, TestCase): diff --git a/readthedocs/rtd_tests/tests/test_project_forms.py b/readthedocs/rtd_tests/tests/test_project_forms.py index 50d9815d051..b2e8687208a 100644 --- a/readthedocs/rtd_tests/tests/test_project_forms.py +++ b/readthedocs/rtd_tests/tests/test_project_forms.py @@ -5,7 +5,7 @@ from django_dynamic_fixture import get from textclassifier.validators import ClassifierValidator -from readthedocs.builds.constants import LATEST, STABLE +from readthedocs.builds.constants import LATEST, STABLE, PULL_REQUEST from readthedocs.builds.models import Version from readthedocs.projects.constants import ( PRIVATE, @@ -314,7 +314,7 @@ def setUp(self): verbose_name='protected', ) - def test_list_only_non_auto_generated_versions_on_default_branch(self): + def test_list_only_non_auto_generated_versions_in_default_branch_choices(self): form = ProjectAdvancedForm(instance=self.project) # This version is created automatically by the project on save latest = self.project.versions.filter(slug=LATEST) @@ -336,7 +336,7 @@ def test_list_only_non_auto_generated_versions_on_default_branch(self): 'default_branch'].widget.choices], ) - def test_list_user_created_latest_and_stable_versions_on_default_branch(self): + def test_list_user_created_latest_and_stable_versions_in_default_branch_choices(self): self.project.versions.filter(slug=LATEST).first().delete() user_created_latest_version = get( Version, @@ -383,6 +383,25 @@ def test_commit_name_not_in_default_branch_choices(self): 'default_branch'].widget.choices], ) + def test_pr_version_not_in_default_branch_choices(self): + pr_version = get( + Version, + identifier='pr-version', + verbose_name='pr-version', + slug='pr-9999', + project=self.project, + active=True, + type=PULL_REQUEST, + privacy_level=PUBLIC, + ) + form = ProjectAdvancedForm(instance=self.project) + + self.assertNotIn( + pr_version.verbose_name, + [identifier for identifier, _ in form.fields[ + 'default_branch'].widget.choices], + ) + class TestTranslationForms(TestCase): diff --git a/readthedocs/rtd_tests/tests/test_project_views.py b/readthedocs/rtd_tests/tests/test_project_views.py index 7c5b286bdd4..90578c8dceb 100644 --- a/readthedocs/rtd_tests/tests/test_project_views.py +++ b/readthedocs/rtd_tests/tests/test_project_views.py @@ -12,7 +12,7 @@ from django_dynamic_fixture import get, new from mock import patch -from readthedocs.builds.constants import LATEST +from readthedocs.builds.constants import LATEST, PULL_REQUEST from readthedocs.builds.models import Build, Version from readthedocs.oauth.models import RemoteRepository from readthedocs.projects import tasks @@ -370,12 +370,40 @@ def test_import_demo_imported_duplicate(self): class TestPublicViews(MockBuildTestCase): def setUp(self): self.pip = get(Project, slug='pip') + self.pr_version = get( + Version, + identifier='pr-version', + verbose_name='pr-version', + slug='pr-9999', + project=self.pip, + active=True, + type=PULL_REQUEST + ) def test_project_download_media(self): url = reverse('project_download_media', args=[self.pip.slug, 'pdf', LATEST]) response = self.client.get(url) self.assertEqual(response.status_code, 302) + def test_project_detail_view_only_shows_internal_versons(self): + url = reverse('projects_detail', args=[self.pip.slug]) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertNotIn(self.pr_version, response.context['versions']) + + def test_project_downloads_only_shows_internal_versons(self): + url = reverse('project_downloads', args=[self.pip.slug]) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertNotIn(self.pr_version, response.context['versions']) + + def test_project_versions_only_shows_internal_versons(self): + url = reverse('project_version_list', args=[self.pip.slug]) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertNotIn(self.pr_version, response.context['active_versions']) + self.assertNotIn(self.pr_version, response.context['inactive_versions']) + class TestPrivateViews(MockBuildTestCase): def setUp(self): diff --git a/readthedocs/rtd_tests/tests/test_version.py b/readthedocs/rtd_tests/tests/test_version.py new file mode 100644 index 00000000000..8a6f5bf3db8 --- /dev/null +++ b/readthedocs/rtd_tests/tests/test_version.py @@ -0,0 +1,68 @@ +from django.test import TestCase +from django_dynamic_fixture import get + +from readthedocs.builds.constants import PULL_REQUEST, BRANCH, TAG +from readthedocs.builds.models import Version +from readthedocs.projects.models import Project + + +class VersionMixin: + + fixtures = ['eric', 'test_data'] + + def setUp(self): + self.client.login(username='eric', password='test') + self.pip = Project.objects.get(slug='pip') + # Create a External Version. ie: PULL_REQUEST type Version. + self.pr_version = get( + Version, + identifier='9F86D081884C7D659A2FEAA0C55AD015A', + verbose_name='pr-version', + slug='9999', + project=self.pip, + active=True, + type=PULL_REQUEST + ) + self.branch_version = get( + Version, + identifier='origin/stable', + verbose_name='stable', + slug='stable', + project=self.pip, + active=True, + type=BRANCH + ) + self.tag_version = get( + Version, + identifier='origin/master', + verbose_name='latest', + slug='latest', + project=self.pip, + active=True, + type=TAG + ) + + +class TestVersionModel(VersionMixin, TestCase): + + def test_vcs_url_for_pr_version(self): + expected_url = f'https://github.com/pypa/pip/pull/{self.pr_version.slug}/' + self.assertEqual(self.pr_version.vcs_url, expected_url) + + def test_vcs_url_for_latest_version(self): + slug = self.pip.default_branch or self.pip.vcs_repo().fallback_branch + expected_url = f'https://github.com/pypa/pip/tree/{slug}/' + self.assertEqual(self.tag_version.vcs_url, expected_url) + + def test_vcs_url_for_stable_version(self): + expected_url = f'https://github.com/pypa/pip/tree/{self.branch_version.ref}/' + self.assertEqual(self.branch_version.vcs_url, expected_url) + + def test_commit_name_for_stable_version(self): + self.assertEqual(self.branch_version.commit_name, 'stable') + + def test_commit_name_for_latest_version(self): + self.assertEqual(self.tag_version.commit_name, 'master') + + def test_commit_name_for_pr_version(self): + self.assertEqual(self.pr_version.commit_name, self.pr_version.identifier) From b55983f3467061e8ef09eca46170ccc35dcac5b2 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 6 Jun 2019 21:52:31 +0600 Subject: [PATCH 08/52] Version Update, Delete Html and wipe updated to exclude PR Versions --- readthedocs/core/views/__init__.py | 2 +- readthedocs/projects/views/private.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/readthedocs/core/views/__init__.py b/readthedocs/core/views/__init__.py index 6b438866ae5..de519a13759 100644 --- a/readthedocs/core/views/__init__.py +++ b/readthedocs/core/views/__init__.py @@ -73,7 +73,7 @@ def random_page(request, project_slug=None): # pylint: disable=unused-argument def wipe_version(request, project_slug, version_slug): version = get_object_or_404( - Version, + Version.internal.all(), project__slug=project_slug, slug=version_slug, ) diff --git a/readthedocs/projects/views/private.py b/readthedocs/projects/views/private.py index 46e83d96188..044a83641e6 100644 --- a/readthedocs/projects/views/private.py +++ b/readthedocs/projects/views/private.py @@ -157,7 +157,7 @@ def project_version_detail(request, project_slug, version_slug): slug=project_slug, ) version = get_object_or_404( - Version.objects.public( + Version.internal.public( user=request.user, project=project, only_active=False, @@ -682,7 +682,7 @@ def project_version_delete_html(request, project_slug, version_slug): slug=project_slug, ) version = get_object_or_404( - Version.objects.public( + Version.internal.public( user=request.user, project=project, only_active=False, From 61872721a793861235aef8dc283cb676b959a737 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 6 Jun 2019 23:03:09 +0600 Subject: [PATCH 09/52] Updated migrations --- .../builds/migrations/0008_added_pull_request_version_type.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py b/readthedocs/builds/migrations/0008_added_pull_request_version_type.py index 14d8968866d..75a126d37b6 100644 --- a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py +++ b/readthedocs/builds/migrations/0008_added_pull_request_version_type.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11.20 on 2019-05-30 10:02 +# Generated by Django 1.11.20 on 2019-06-06 16:51 from __future__ import unicode_literals from django.db import migrations, models From 386ca07ba20f134d9eef9ef969a4c91796c7ae42 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Sun, 9 Jun 2019 02:17:05 +0600 Subject: [PATCH 10/52] footer API updated --- readthedocs/api/v2/views/footer_views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readthedocs/api/v2/views/footer_views.py b/readthedocs/api/v2/views/footer_views.py index 09709ac1bb1..ece5e09bd89 100644 --- a/readthedocs/api/v2/views/footer_views.py +++ b/readthedocs/api/v2/views/footer_views.py @@ -9,7 +9,7 @@ from rest_framework_jsonp.renderers import JSONPRenderer from readthedocs.api.v2.signals import footer_response -from readthedocs.builds.constants import LATEST, TAG +from readthedocs.builds.constants import LATEST, TAG, INTERNAL from readthedocs.builds.models import Version from readthedocs.projects.models import Project from readthedocs.projects.version_handling import ( @@ -25,7 +25,7 @@ def get_version_compare_data(project, base_version=None): :param base_version: We assert whether or not the base_version is also the highest version in the resulting "is_highest" value. """ - versions_qs = project.versions.public().filter(active=True) + versions_qs = project.versions(manager=INTERNAL).public().filter(active=True) # Take preferences over tags only if the project has at least one tag if versions_qs.filter(type=TAG).exists(): From 29348ef08c14d472aaeace74005e1e45c6f7d6d1 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 12 Jun 2019 00:45:00 +0600 Subject: [PATCH 11/52] manager tests updated --- readthedocs/rtd_tests/tests/test_managers.py | 34 +++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_managers.py b/readthedocs/rtd_tests/tests/test_managers.py index 8e440e18b85..e69abdc4dd8 100644 --- a/readthedocs/rtd_tests/tests/test_managers.py +++ b/readthedocs/rtd_tests/tests/test_managers.py @@ -1,3 +1,4 @@ +from django.contrib.auth import get_user_model from django.test import TestCase from django_dynamic_fixture import get @@ -7,12 +8,16 @@ from readthedocs.projects.models import Project +User = get_user_model() + + class TestVersionManagerBase(TestCase): - fixtures = ['eric', 'test_data'] + fixtures = ['test_data'] def setUp(self): - self.client.login(username='eric', password='test') + self.user = User.objects.create(username='test_user', password='test') + self.client.login(username='test_user', password='test') self.pip = Project.objects.get(slug='pip') # Create a External Version. ie: PULL_REQUEST type Version. self.public_pr_version = get( @@ -54,6 +59,12 @@ def test_internal_version_manager_with_all(self): def test_internal_version_manager_with_public(self): self.assertNotIn(self.public_pr_version, Version.internal.public()) + def test_internal_version_manager_with_public_with_user_and_project(self): + self.assertNotIn( + self.public_pr_version, + Version.internal.public(self.user, self.pip) + ) + def test_internal_version_manager_with_protected(self): self.assertNotIn(self.protected_pr_version, Version.internal.protected()) @@ -83,6 +94,17 @@ def test_external_version_manager_with_public(self): self.assertNotIn(self.internal_versions, Version.external.public()) self.assertIn(self.public_pr_version, Version.external.public()) + def test_external_version_manager_with_public_with_user_and_project(self): + self.assertNotIn( + self.internal_versions, + Version.external.public(self.user, self.pip) + ) + self.assertIn( + self.public_pr_version, + Version.external.public(self.user, self.pip) + ) + + def test_external_version_manager_with_protected(self): self.assertNotIn(self.internal_versions, Version.external.protected()) self.assertIn(self.protected_pr_version, Version.external.protected()) @@ -96,5 +118,9 @@ def test_external_version_manager_with_api(self): self.assertIn(self.public_pr_version, Version.external.api()) def test_external_version_manager_with_for_project(self): - self.assertNotIn(self.internal_versions, Version.external.for_project(self.pip)) - self.assertIn(self.public_pr_version, Version.external.for_project(self.pip)) + self.assertNotIn( + self.internal_versions, Version.external.for_project(self.pip) + ) + self.assertIn( + self.public_pr_version, Version.external.for_project(self.pip) + ) From 11a0583abf4b11a394fa169fc7a14f669bdea0aa Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 13 Jun 2019 18:56:06 +0600 Subject: [PATCH 12/52] _override_setting removed from new managers --- readthedocs/builds/managers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/readthedocs/builds/managers.py b/readthedocs/builds/managers.py index 2a8bb4840d9..0bc30f02807 100644 --- a/readthedocs/builds/managers.py +++ b/readthedocs/builds/managers.py @@ -118,9 +118,7 @@ class VersionManager(SettingsOverrideObject): class InternalVersionManager(SettingsOverrideObject): _default_class = InternalVersionManagerBase - _override_setting = 'INTERNAL_VERSION_MANAGER' class ExternalVersionManager(SettingsOverrideObject): _default_class = ExternalVersionManagerBase - _override_setting = 'EXTERNAL_VERSION_MANAGER' From c100c81beba9a4f9d596a216b0a36b849bfe1fad Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 13 Jun 2019 23:11:51 +0600 Subject: [PATCH 13/52] BuildTriggerMixin updated --- readthedocs/builds/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/builds/views.py b/readthedocs/builds/views.py index 3d6e2cc0ca1..3333495f6c8 100644 --- a/readthedocs/builds/views.py +++ b/readthedocs/builds/views.py @@ -57,7 +57,7 @@ def post(self, request, project_slug): version_slug = request.POST.get('version_slug') version = get_object_or_404( - Version, + Version.internal.all(), project=project, slug=version_slug, ) From c1c03bb951d8cd0baf4b19e6acb47e599a183da7 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 13 Jun 2019 23:25:28 +0600 Subject: [PATCH 14/52] More Update --- readthedocs/search/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/search/utils.py b/readthedocs/search/utils.py index 5f04f6736b7..cab67ba5ff8 100644 --- a/readthedocs/search/utils.py +++ b/readthedocs/search/utils.py @@ -28,7 +28,7 @@ def get_project_list_or_404(project_slug, user, version_slug=None): main_project = get_object_or_404(Project, slug=project_slug) subprojects = Project.objects.filter(superprojects__parent_id=main_project.id) for project in list(subprojects) + [main_project]: - version = Version.objects.public(user).filter(project__slug=project.slug, slug=version_slug) + version = Version.internal.public(user).filter(project__slug=project.slug, slug=version_slug) if version.exists(): project_list.append(version.first().project) return project_list From 5ea5f6755c4860298a9018bebf9ed857fdd6cb54 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 13 Jun 2019 23:55:32 +0600 Subject: [PATCH 15/52] lint fix --- readthedocs/rtd_tests/tests/test_managers.py | 1 - readthedocs/search/utils.py | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_managers.py b/readthedocs/rtd_tests/tests/test_managers.py index e69abdc4dd8..f830c9af549 100644 --- a/readthedocs/rtd_tests/tests/test_managers.py +++ b/readthedocs/rtd_tests/tests/test_managers.py @@ -104,7 +104,6 @@ def test_external_version_manager_with_public_with_user_and_project(self): Version.external.public(self.user, self.pip) ) - def test_external_version_manager_with_protected(self): self.assertNotIn(self.internal_versions, Version.external.protected()) self.assertIn(self.protected_pr_version, Version.external.protected()) diff --git a/readthedocs/search/utils.py b/readthedocs/search/utils.py index cab67ba5ff8..3433195bb55 100644 --- a/readthedocs/search/utils.py +++ b/readthedocs/search/utils.py @@ -28,7 +28,9 @@ def get_project_list_or_404(project_slug, user, version_slug=None): main_project = get_object_or_404(Project, slug=project_slug) subprojects = Project.objects.filter(superprojects__parent_id=main_project.id) for project in list(subprojects) + [main_project]: - version = Version.internal.public(user).filter(project__slug=project.slug, slug=version_slug) + version = Version.internal.public(user).filter( + project__slug=project.slug, slug=version_slug + ) if version.exists(): project_list.append(version.first().project) return project_list From 56493fa016f39224ba4ef3b5a2322b853aeb0261 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Fri, 14 Jun 2019 17:35:34 +0600 Subject: [PATCH 16/52] removed unused import --- readthedocs/core/views/serve.py | 1 - 1 file changed, 1 deletion(-) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index ae60011e54e..3daebf8c721 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -38,7 +38,6 @@ from django.views.decorators.cache import cache_page from django.views.static import serve -from readthedocs.builds.constants import INTERNAL from readthedocs.builds.models import Version from readthedocs.core.permissions import AdminPermission from readthedocs.core.resolver import resolve, resolve_path From 84a69753606cb04185e4c9aabed29559e9061254 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 18 Jun 2019 01:30:26 +0600 Subject: [PATCH 17/52] External version name added everywhere --- readthedocs/builds/constants.py | 17 ++++++++--------- readthedocs/builds/managers.py | 10 +++++----- .../0008_added_pull_request_version_type.py | 6 +++--- readthedocs/builds/models.py | 14 +++++++------- readthedocs/rtd_tests/tests/test_doc_serving.py | 6 +++--- readthedocs/rtd_tests/tests/test_managers.py | 16 ++++++++-------- readthedocs/rtd_tests/tests/test_project.py | 8 ++++---- .../rtd_tests/tests/test_project_forms.py | 4 ++-- .../rtd_tests/tests/test_project_views.py | 4 ++-- readthedocs/rtd_tests/tests/test_version.py | 6 +++--- 10 files changed, 45 insertions(+), 46 deletions(-) diff --git a/readthedocs/builds/constants.py b/readthedocs/builds/constants.py index 37474eb5365..bdc9a70fe79 100644 --- a/readthedocs/builds/constants.py +++ b/readthedocs/builds/constants.py @@ -31,15 +31,21 @@ ('dash', _('Dash')), ) +# Manager name for Internal Versions or Builds. +# ie: Versions and Builds Excluding pull request/merge request Versions and Builds. +INTERNAL = 'internal' +# Manager name for External Versions or Builds. +# ie: Only pull request/merge request Versions and Builds. +EXTERNAL = 'external' + BRANCH = 'branch' TAG = 'tag' -PULL_REQUEST = 'pull_request' UNKNOWN = 'unknown' VERSION_TYPES = ( (BRANCH, _('Branch')), (TAG, _('Tag')), - (PULL_REQUEST, _('Pull Request')), + (EXTERNAL, _('External')), (UNKNOWN, _('Unknown')), ) @@ -55,10 +61,3 @@ LATEST, STABLE, ) - -# Manager name for Internal Versions or Builds. -# ie: Versions and Builds Excluding PULL_REQUEST Type. -INTERNAL = 'internal' -# Manager name for External Versions or Builds. -# ie: Only PULL_REQUEST Type Versions and Builds. -EXTERNAL = 'external' diff --git a/readthedocs/builds/managers.py b/readthedocs/builds/managers.py index 0bc30f02807..06262ed3e24 100644 --- a/readthedocs/builds/managers.py +++ b/readthedocs/builds/managers.py @@ -19,7 +19,7 @@ STABLE, STABLE_VERBOSE_NAME, TAG, - PULL_REQUEST, + EXTERNAL, ) from .querysets import VersionQuerySet @@ -91,12 +91,12 @@ class InternalVersionManagerBase(VersionManagerBase): """ Version manager that only includes internal version. - It will exclude PULL_REQUEST type from the queries + It will exclude pull request/merge request versions from the queries and only include BRANCH, TAG, UNKONWN type Versions. """ def get_queryset(self): - return super().get_queryset().exclude(type=PULL_REQUEST) + return super().get_queryset().exclude(type=EXTERNAL) class ExternalVersionManagerBase(VersionManagerBase): @@ -104,11 +104,11 @@ class ExternalVersionManagerBase(VersionManagerBase): """ Version manager that only includes external version. - It will only include PULL_REQUEST type Versions in the queries. + It will only include pull request/merge request Versions in the queries. """ def get_queryset(self): - return super().get_queryset().filter(type=PULL_REQUEST) + return super().get_queryset().filter(type=EXTERNAL) class VersionManager(SettingsOverrideObject): diff --git a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py b/readthedocs/builds/migrations/0008_added_pull_request_version_type.py index 75a126d37b6..2bf9dc62d66 100644 --- a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py +++ b/readthedocs/builds/migrations/0008_added_pull_request_version_type.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11.20 on 2019-06-06 16:51 +# Generated by Django 1.11.21 on 2019-06-17 19:26 from __future__ import unicode_literals from django.db import migrations, models @@ -15,11 +15,11 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='version', name='type', - field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('pull_request', 'Pull Request'), ('unknown', 'Unknown')], default='unknown', max_length=20, verbose_name='Type'), + field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('external', 'External'), ('unknown', 'Unknown')], default='unknown', max_length=20, verbose_name='Type'), ), migrations.AlterField( model_name='versionautomationrule', name='version_type', - field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('pull_request', 'Pull Request'), ('unknown', 'Unknown')], max_length=32, verbose_name='Version type'), + field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('external', 'External'), ('unknown', 'Unknown')], max_length=32, verbose_name='Version type'), ), ] diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index d0c5f6a854e..34a3fd372a9 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -41,7 +41,7 @@ INTERNAL, LATEST, NON_REPOSITORY_VERSIONS, - PULL_REQUEST, + EXTERNAL, STABLE, TAG, VERSION_TYPES, @@ -119,7 +119,7 @@ class Version(models.Model): objects = VersionManager.from_queryset(VersionQuerySet)() # Only include BRANCH, TAG, UNKONWN type Versions. internal = InternalVersionManager.from_queryset(VersionQuerySet)() - # Only include PULL_REQUEST type Versions. + # Only include EXTERNAL type Versions. external = ExternalVersionManager.from_queryset(VersionQuerySet)() class Meta: @@ -155,7 +155,7 @@ def vcs_url(self): Generate VCS (github, gitlab, bitbucket) URL for this version. Branch/Tag Example: https://github.com/rtfd/readthedocs.org/tree/3.4.2/. - Pull Request Example: https://github.com/rtfd/readthedocs.org/pull/9999/. + Pull/merge Request Example: https://github.com/rtfd/readthedocs.org/pull/9999/. """ url = '' if self.slug == STABLE: @@ -165,7 +165,7 @@ def vcs_url(self): else: slug_url = self.slug - if self.type == PULL_REQUEST: + if self.type == EXTERNAL: if 'github' in self.project.repo: url = f'/pull/{slug_url}/' @@ -245,14 +245,14 @@ def commit_name(self): # the actual tag name. return self.verbose_name - if self.type == PULL_REQUEST: - # If this version is a Pull Request, the identifier will + if self.type == EXTERNAL: + # If this version is a EXTERNAL version, the identifier will # contain the actual commit hash. which we can use to # generate url for a given file name return self.identifier # If we came that far it's not a special version - # nor a branch, tag or Pull Request. + # nor a branch, tag or EXTERNAL version. # Therefore just return the identifier to make a safe guess. log.debug( 'TODO: Raise an exception here. Testing what cases it happens', diff --git a/readthedocs/rtd_tests/tests/test_doc_serving.py b/readthedocs/rtd_tests/tests/test_doc_serving.py index 8a5a6609b6b..9de5afccf73 100644 --- a/readthedocs/rtd_tests/tests/test_doc_serving.py +++ b/readthedocs/rtd_tests/tests/test_doc_serving.py @@ -10,7 +10,7 @@ from django.urls import reverse from mock import mock_open, patch -from readthedocs.builds.constants import LATEST, PULL_REQUEST, INTERNAL +from readthedocs.builds.constants import LATEST, EXTERNAL, INTERNAL from readthedocs.builds.models import Version from readthedocs.core.middleware import SubdomainMiddleware from readthedocs.core.views import server_error_404_subdomain @@ -240,7 +240,7 @@ def test_sitemap_xml(self): project=self.public, active=True ) - # This is a Pull Request Version + # This is a EXTERNAL Version pr_version = fixture.get( Version, identifier='pr-version', @@ -248,7 +248,7 @@ def test_sitemap_xml(self): slug='pr-9999', project=self.public, active=True, - type=PULL_REQUEST + type=EXTERNAL ) stable_version = fixture.get( Version, diff --git a/readthedocs/rtd_tests/tests/test_managers.py b/readthedocs/rtd_tests/tests/test_managers.py index f830c9af549..924544a8be8 100644 --- a/readthedocs/rtd_tests/tests/test_managers.py +++ b/readthedocs/rtd_tests/tests/test_managers.py @@ -2,7 +2,7 @@ from django.test import TestCase from django_dynamic_fixture import get -from readthedocs.builds.constants import PULL_REQUEST, BRANCH, TAG +from readthedocs.builds.constants import EXTERNAL, BRANCH, TAG from readthedocs.builds.models import Version from readthedocs.projects.constants import PUBLIC, PRIVATE, PROTECTED from readthedocs.projects.models import Project @@ -19,29 +19,29 @@ def setUp(self): self.user = User.objects.create(username='test_user', password='test') self.client.login(username='test_user', password='test') self.pip = Project.objects.get(slug='pip') - # Create a External Version. ie: PULL_REQUEST type Version. + # Create a External Version. ie: pull/merge request Version. self.public_pr_version = get( Version, project=self.pip, active=True, - type=PULL_REQUEST, + type=EXTERNAL, privacy_level=PUBLIC ) self.private_pr_version = get( Version, project=self.pip, active=True, - type=PULL_REQUEST, + type=EXTERNAL, privacy_level=PRIVATE ) self.protected_pr_version = get( Version, project=self.pip, active=True, - type=PULL_REQUEST, + type=EXTERNAL, privacy_level=PROTECTED ) - self.internal_versions = Version.objects.exclude(type=PULL_REQUEST) + self.internal_versions = Version.objects.exclude(type=EXTERNAL) class TestInternalVersionManager(TestVersionManagerBase): @@ -49,7 +49,7 @@ class TestInternalVersionManager(TestVersionManagerBase): """ Queries using Internal Manager should only include Internal Versions. - It will exclude PULL_REQUEST type Versions from the queries + It will exclude EXTERNAL type Versions from the queries and only include BRANCH, TAG, UNKONWN type Versions. """ @@ -83,7 +83,7 @@ class TestExternalVersionManager(TestVersionManagerBase): """ Queries using External Manager should only include External Versions. - It will only include PULL_REQUEST type Versions in the queries. + It will only include pull/merge request Version in the queries. """ def test_external_version_manager_with_all(self): diff --git a/readthedocs/rtd_tests/tests/test_project.py b/readthedocs/rtd_tests/tests/test_project.py index c2db080f201..5d5194d04d2 100644 --- a/readthedocs/rtd_tests/tests/test_project.py +++ b/readthedocs/rtd_tests/tests/test_project.py @@ -14,7 +14,7 @@ BUILD_STATE_FINISHED, BUILD_STATE_TRIGGERED, LATEST, - PULL_REQUEST, + EXTERNAL, ) from readthedocs.builds.models import Build, Version from readthedocs.projects.exceptions import ProjectConfigurationError @@ -30,7 +30,7 @@ class ProjectMixin: def setUp(self): self.client.login(username='eric', password='test') self.pip = Project.objects.get(slug='pip') - # Create a External Version. ie: PULL_REQUEST type Version. + # Create a External Version. ie: pull/merge request Version. self.pr_version = get( Version, identifier='pr-version', @@ -38,7 +38,7 @@ def setUp(self): slug='pr-9999', project=self.pip, active=True, - type=PULL_REQUEST + type=EXTERNAL ) @@ -156,7 +156,7 @@ def test_all_active_versions_excludes_pr_versions(self): def test_update_stable_version_excludes_pr_versions(self): # Delete all versions excluding PR Versions. - self.pip.versions.exclude(type=PULL_REQUEST).delete() + self.pip.versions.exclude(type=EXTERNAL).delete() # Test that PR Version is not considered for stable. self.assertEqual(self.pip.update_stable_version(), None) diff --git a/readthedocs/rtd_tests/tests/test_project_forms.py b/readthedocs/rtd_tests/tests/test_project_forms.py index b2e8687208a..ba5b3c88b9d 100644 --- a/readthedocs/rtd_tests/tests/test_project_forms.py +++ b/readthedocs/rtd_tests/tests/test_project_forms.py @@ -5,7 +5,7 @@ from django_dynamic_fixture import get from textclassifier.validators import ClassifierValidator -from readthedocs.builds.constants import LATEST, STABLE, PULL_REQUEST +from readthedocs.builds.constants import LATEST, STABLE, EXTERNAL from readthedocs.builds.models import Version from readthedocs.projects.constants import ( PRIVATE, @@ -391,7 +391,7 @@ def test_pr_version_not_in_default_branch_choices(self): slug='pr-9999', project=self.project, active=True, - type=PULL_REQUEST, + type=EXTERNAL, privacy_level=PUBLIC, ) form = ProjectAdvancedForm(instance=self.project) diff --git a/readthedocs/rtd_tests/tests/test_project_views.py b/readthedocs/rtd_tests/tests/test_project_views.py index 90578c8dceb..3bc948aff9b 100644 --- a/readthedocs/rtd_tests/tests/test_project_views.py +++ b/readthedocs/rtd_tests/tests/test_project_views.py @@ -12,7 +12,7 @@ from django_dynamic_fixture import get, new from mock import patch -from readthedocs.builds.constants import LATEST, PULL_REQUEST +from readthedocs.builds.constants import LATEST, EXTERNAL from readthedocs.builds.models import Build, Version from readthedocs.oauth.models import RemoteRepository from readthedocs.projects import tasks @@ -377,7 +377,7 @@ def setUp(self): slug='pr-9999', project=self.pip, active=True, - type=PULL_REQUEST + type=EXTERNAL ) def test_project_download_media(self): diff --git a/readthedocs/rtd_tests/tests/test_version.py b/readthedocs/rtd_tests/tests/test_version.py index 8a6f5bf3db8..e414ba79cc6 100644 --- a/readthedocs/rtd_tests/tests/test_version.py +++ b/readthedocs/rtd_tests/tests/test_version.py @@ -1,7 +1,7 @@ from django.test import TestCase from django_dynamic_fixture import get -from readthedocs.builds.constants import PULL_REQUEST, BRANCH, TAG +from readthedocs.builds.constants import EXTERNAL, BRANCH, TAG from readthedocs.builds.models import Version from readthedocs.projects.models import Project @@ -13,7 +13,7 @@ class VersionMixin: def setUp(self): self.client.login(username='eric', password='test') self.pip = Project.objects.get(slug='pip') - # Create a External Version. ie: PULL_REQUEST type Version. + # Create a External Version. ie: pull/merge request Version. self.pr_version = get( Version, identifier='9F86D081884C7D659A2FEAA0C55AD015A', @@ -21,7 +21,7 @@ def setUp(self): slug='9999', project=self.pip, active=True, - type=PULL_REQUEST + type=EXTERNAL ) self.branch_version = get( Version, From 05951cfc99ddee655969e8482d5d666b690b5cdf Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 18 Jun 2019 01:43:48 +0600 Subject: [PATCH 18/52] Migration name changed --- ...uest_version_type.py => 0008_added_external_version_type.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename readthedocs/builds/migrations/{0008_added_pull_request_version_type.py => 0008_added_external_version_type.py} (94%) diff --git a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py b/readthedocs/builds/migrations/0008_added_external_version_type.py similarity index 94% rename from readthedocs/builds/migrations/0008_added_pull_request_version_type.py rename to readthedocs/builds/migrations/0008_added_external_version_type.py index 2bf9dc62d66..8de9c4f504f 100644 --- a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py +++ b/readthedocs/builds/migrations/0008_added_external_version_type.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11.21 on 2019-06-17 19:26 +# Generated by Django 1.11.21 on 2019-06-17 19:43 from __future__ import unicode_literals from django.db import migrations, models From e192300e11da4b930ceeb4df10ae8d2a1682f5d7 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 23 May 2019 18:15:08 +0600 Subject: [PATCH 19/52] Sitemap sort order priorities updated --- readthedocs/core/views/serve.py | 4 +-- readthedocs/projects/version_handling.py | 10 +++--- .../rtd_tests/tests/test_doc_serving.py | 33 ++++++++++++++++++- 3 files changed, 39 insertions(+), 8 deletions(-) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index cd5ab8fdb64..b7687d0a132 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -373,14 +373,14 @@ def changefreqs_generator(): """ Generator returning ``changefreq`` needed by sitemap.xml. - It returns ``daily`` on first iteration, then ``weekly`` and then it + It returns ``weekly`` on first iteration, then ``daily`` and then it will return always ``monthly``. We are using ``monthly`` as last value because ``never`` is too aggressive. If the tag is removed and a branch is created with the same name, we will want bots to revisit this. """ - changefreqs = ['daily', 'weekly'] + changefreqs = ['weekly', 'daily'] yield from itertools.chain(changefreqs, itertools.repeat('monthly')) if project.privacy_level == constants.PRIVATE: diff --git a/readthedocs/projects/version_handling.py b/readthedocs/projects/version_handling.py index 7a730e61fd0..093bb705c68 100644 --- a/readthedocs/projects/version_handling.py +++ b/readthedocs/projects/version_handling.py @@ -43,23 +43,23 @@ def comparable_version(version_string): """ Can be used as ``key`` argument to ``sorted``. - The ``LATEST`` version shall always beat other versions in comparison. - ``STABLE`` should be listed second. If we cannot figure out the version + The ``STABLE`` version shall always beat other versions in comparison. + ``LATEST`` should be listed second. If we cannot figure out the version number then we sort it to the bottom of the list. :param version_string: version as string object (e.g. '3.10.1' or 'latest') :type version_string: str or unicode - :returns: a comparable version object (e.g. 'latest' -> Version('99999.0')) + :returns: a comparable version object (e.g. 'latest' -> Version('9999.0')) :rtype: packaging.version.Version """ comparable = parse_version_failsafe(version_string) if not comparable: if version_string == LATEST_VERBOSE_NAME: - comparable = Version('99999.0') - elif version_string == STABLE_VERBOSE_NAME: comparable = Version('9999.0') + elif version_string == STABLE_VERBOSE_NAME: + comparable = Version('99999.0') else: comparable = Version('0.01') return comparable diff --git a/readthedocs/rtd_tests/tests/test_doc_serving.py b/readthedocs/rtd_tests/tests/test_doc_serving.py index 4f6cb93990c..fed119bbab5 100644 --- a/readthedocs/rtd_tests/tests/test_doc_serving.py +++ b/readthedocs/rtd_tests/tests/test_doc_serving.py @@ -240,6 +240,15 @@ def test_sitemap_xml(self): project=self.public, active=True ) + stable_version = fixture.get( + Version, + identifier='stable', + verbose_name='stable', + slug='stable', + privacy_level=constants.PUBLIC, + project=self.public, + active=True + ) # This also creates a Version `latest` Automatically for this project translation = fixture.get( Project, @@ -269,7 +278,7 @@ def test_sitemap_xml(self): ), ) - # stable is marked as PRIVATE and should not appear here + # PRIVATE version should not appear here self.assertNotContains( response, self.public.get_docs_url( @@ -294,6 +303,28 @@ def test_sitemap_xml(self): # in language and country value. (zh_CN should be zh-CN) self.assertContains(response, 'zh-CN') + # Check if STABLE version has 'priority of 1 and changefreq of weekly. + self.assertEqual( + response.context['versions'][0]['loc'], + self.public.get_docs_url( + version_slug=stable_version.slug, + lang_slug=self.public.language, + private=False, + ),) + self.assertEqual(response.context['versions'][0]['priority'], 1) + self.assertEqual(response.context['versions'][0]['changefreq'], 'weekly') + + # Check if LATEST version has priority of 0.9 and changefreq of daily. + self.assertEqual( + response.context['versions'][1]['loc'], + self.public.get_docs_url( + version_slug='latest', + lang_slug=self.public.language, + private=False, + ),) + self.assertEqual(response.context['versions'][1]['priority'], 0.9) + self.assertEqual(response.context['versions'][1]['changefreq'], 'daily') + @override_settings( PYTHON_MEDIA=True, USE_SUBDOMAIN=False, From d5ec837aa3ea9147ab19a538d83ff219087975b1 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Sat, 25 May 2019 17:33:14 +0600 Subject: [PATCH 20/52] ordering fix --- readthedocs/core/views/serve.py | 4 ++-- readthedocs/projects/version_handling.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index b7687d0a132..f7ab665bbc5 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -355,7 +355,7 @@ def priorities_generator(): It generates values from 1 to 0.1 by decreasing in 0.1 on each iteration. After 0.1 is reached, it will keep returning 0.1. """ - priorities = [1, 0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2] + priorities = [0.9, 1, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2] yield from itertools.chain(priorities, itertools.repeat(0.1)) def hreflang_formatter(lang): @@ -380,7 +380,7 @@ def changefreqs_generator(): aggressive. If the tag is removed and a branch is created with the same name, we will want bots to revisit this. """ - changefreqs = ['weekly', 'daily'] + changefreqs = ['daily', 'weekly'] yield from itertools.chain(changefreqs, itertools.repeat('monthly')) if project.privacy_level == constants.PRIVATE: diff --git a/readthedocs/projects/version_handling.py b/readthedocs/projects/version_handling.py index 093bb705c68..7a730e61fd0 100644 --- a/readthedocs/projects/version_handling.py +++ b/readthedocs/projects/version_handling.py @@ -43,23 +43,23 @@ def comparable_version(version_string): """ Can be used as ``key`` argument to ``sorted``. - The ``STABLE`` version shall always beat other versions in comparison. - ``LATEST`` should be listed second. If we cannot figure out the version + The ``LATEST`` version shall always beat other versions in comparison. + ``STABLE`` should be listed second. If we cannot figure out the version number then we sort it to the bottom of the list. :param version_string: version as string object (e.g. '3.10.1' or 'latest') :type version_string: str or unicode - :returns: a comparable version object (e.g. 'latest' -> Version('9999.0')) + :returns: a comparable version object (e.g. 'latest' -> Version('99999.0')) :rtype: packaging.version.Version """ comparable = parse_version_failsafe(version_string) if not comparable: if version_string == LATEST_VERBOSE_NAME: - comparable = Version('9999.0') - elif version_string == STABLE_VERBOSE_NAME: comparable = Version('99999.0') + elif version_string == STABLE_VERBOSE_NAME: + comparable = Version('9999.0') else: comparable = Version('0.01') return comparable From 1f13be1861362fd4255412015b44f6629ad80001 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Sat, 25 May 2019 17:34:46 +0600 Subject: [PATCH 21/52] doc string fix --- readthedocs/core/views/serve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index f7ab665bbc5..49207c0b5bd 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -373,7 +373,7 @@ def changefreqs_generator(): """ Generator returning ``changefreq`` needed by sitemap.xml. - It returns ``weekly`` on first iteration, then ``daily`` and then it + It returns ``daily`` on first iteration, then ``weekly`` and then it will return always ``monthly``. We are using ``monthly`` as last value because ``never`` is too From e0232b3e4ea4ba37bade2fdeb294d846629f8ab1 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 29 May 2019 17:04:04 +0600 Subject: [PATCH 22/52] sort_by_priority function added to sitemap --- readthedocs/core/views/serve.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index 49207c0b5bd..80db933ab69 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -383,6 +383,14 @@ def changefreqs_generator(): changefreqs = ['daily', 'weekly'] yield from itertools.chain(changefreqs, itertools.repeat('monthly')) + def sort_by_priority(version_list): + """This will sort the versions by priority""" + return sorted( + version_list, + key=lambda version: version['priority'], + reverse=True + ) + if project.privacy_level == constants.PRIVATE: raise Http404 @@ -437,7 +445,7 @@ def changefreqs_generator(): versions.append(element) context = { - 'versions': versions, + 'versions': sort_by_priority(versions), } return render( request, From fdcfcf6b0517af063f39adf3b8fbf06929e3ed01 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 29 May 2019 17:14:19 +0600 Subject: [PATCH 23/52] lint fix --- readthedocs/core/views/serve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index 80db933ab69..9a0dd3593cc 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -384,7 +384,7 @@ def changefreqs_generator(): yield from itertools.chain(changefreqs, itertools.repeat('monthly')) def sort_by_priority(version_list): - """This will sort the versions by priority""" + """Sorts the versions by priority. i.e: 1, 0.9, 0.8...""" return sorted( version_list, key=lambda version: version['priority'], From ef1321491e1ec767e53a81600ddb3c9145968d14 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 12 Jun 2019 00:33:08 +0600 Subject: [PATCH 24/52] Sorting by swapping positions --- readthedocs/core/views/serve.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index 9a0dd3593cc..3d1a971cfd4 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -38,6 +38,7 @@ from django.views.decorators.cache import cache_page from django.views.static import serve +from readthedocs.builds.constants import LATEST, STABLE from readthedocs.builds.models import Version from readthedocs.core.permissions import AdminPermission from readthedocs.core.resolver import resolve, resolve_path @@ -355,7 +356,7 @@ def priorities_generator(): It generates values from 1 to 0.1 by decreasing in 0.1 on each iteration. After 0.1 is reached, it will keep returning 0.1. """ - priorities = [0.9, 1, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2] + priorities = [1, 0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2] yield from itertools.chain(priorities, itertools.repeat(0.1)) def hreflang_formatter(lang): @@ -373,24 +374,16 @@ def changefreqs_generator(): """ Generator returning ``changefreq`` needed by sitemap.xml. - It returns ``daily`` on first iteration, then ``weekly`` and then it + It returns ``weekly`` on first iteration, then ``daily`` and then it will return always ``monthly``. We are using ``monthly`` as last value because ``never`` is too aggressive. If the tag is removed and a branch is created with the same name, we will want bots to revisit this. """ - changefreqs = ['daily', 'weekly'] + changefreqs = ['weekly', 'daily'] yield from itertools.chain(changefreqs, itertools.repeat('monthly')) - def sort_by_priority(version_list): - """Sorts the versions by priority. i.e: 1, 0.9, 0.8...""" - return sorted( - version_list, - key=lambda version: version['priority'], - reverse=True - ) - if project.privacy_level == constants.PRIVATE: raise Http404 @@ -400,6 +393,18 @@ def sort_by_priority(version_list): only_active=True, ), ) + + # This is a hack to swap the latest version with + # stable version to get the stable version first in the sitemap. + # We want stable with priority=1 and changefreq='weekly' and + # latest with priority=0.9 and changefreq='daily' + # More details on this: https://github.com/rtfd/readthedocs.org/issues/5447 + if ( + sorted_versions[0].slug == LATEST and + sorted_versions[1].slug == STABLE + ): + sorted_versions[0], sorted_versions[1] = sorted_versions[1], sorted_versions[0] + versions = [] for version, priority, changefreq in zip( sorted_versions, @@ -445,7 +450,7 @@ def sort_by_priority(version_list): versions.append(element) context = { - 'versions': sort_by_priority(versions), + 'versions': versions, } return render( request, From 736a3377128f3ce1f773a05aafa4081a1b27f657 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 12 Jun 2019 17:30:50 +0600 Subject: [PATCH 25/52] index out of range issue fix --- readthedocs/core/views/serve.py | 1 + 1 file changed, 1 insertion(+) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index 3d1a971cfd4..237b843e0fb 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -400,6 +400,7 @@ def changefreqs_generator(): # latest with priority=0.9 and changefreq='daily' # More details on this: https://github.com/rtfd/readthedocs.org/issues/5447 if ( + len(sorted_versions) >= 2 and sorted_versions[0].slug == LATEST and sorted_versions[1].slug == STABLE ): From c18f3810a5f76ee7038e095778eb52e5b4b2d0f7 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 12 Jun 2019 21:55:35 +0200 Subject: [PATCH 26/52] Use a real SessionBase object on FooterNoSessionMiddleware This allows all Django internals method to keep working in the same way but still using an empty session. --- readthedocs/core/middleware.py | 3 ++- readthedocs/rtd_tests/tests/test_footer.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/readthedocs/core/middleware.py b/readthedocs/core/middleware.py index 60cf4a277d3..692e45a2f8d 100644 --- a/readthedocs/core/middleware.py +++ b/readthedocs/core/middleware.py @@ -1,6 +1,7 @@ import logging from django.conf import settings +from django.contrib.sessions.backends.base import SessionBase from django.contrib.sessions.middleware import SessionMiddleware from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist from django.http import Http404, HttpResponseBadRequest @@ -205,7 +206,7 @@ def process_request(self, request): settings.SESSION_COOKIE_NAME not in request.COOKIES ): # Hack request.session otherwise the Authentication middleware complains. - request.session = {} + request.session = SessionBase() # create an empty session return super().process_request(request) diff --git a/readthedocs/rtd_tests/tests/test_footer.py b/readthedocs/rtd_tests/tests/test_footer.py index 34c1fec1f92..241bc434f59 100644 --- a/readthedocs/rtd_tests/tests/test_footer.py +++ b/readthedocs/rtd_tests/tests/test_footer.py @@ -1,4 +1,5 @@ import mock +from django.contrib.sessions.backends.base import SessionBase from django.test import TestCase from rest_framework.test import APIRequestFactory, APITestCase @@ -80,7 +81,8 @@ def test_no_session_logged_out(self): # Null session here request = self.factory.get('/api/v2/footer_html/') mid.process_request(request) - self.assertEqual(request.session, {}) + self.assertIsInstance(request.session, SessionBase) + self.assertEqual(list(request.session.keys()), []) # Proper session here home_request = self.factory.get('/') From 908860a2bc4f41943982ae17097f0cc88db25e2b Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 30 May 2019 16:59:23 +0600 Subject: [PATCH 27/52] Version Type Added --- readthedocs/builds/constants.py | 2 ++ .../0008_added_pull_request_version_type.py | 25 +++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 readthedocs/builds/migrations/0008_added_pull_request_version_type.py diff --git a/readthedocs/builds/constants.py b/readthedocs/builds/constants.py index b0dc6cfba94..e2e4fc44ac3 100644 --- a/readthedocs/builds/constants.py +++ b/readthedocs/builds/constants.py @@ -33,11 +33,13 @@ BRANCH = 'branch' TAG = 'tag' +PULL_REQUEST = 'pull_request' UNKNOWN = 'unknown' VERSION_TYPES = ( (BRANCH, _('Branch')), (TAG, _('Tag')), + (PULL_REQUEST, _('Pull Request')), (UNKNOWN, _('Unknown')), ) diff --git a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py b/readthedocs/builds/migrations/0008_added_pull_request_version_type.py new file mode 100644 index 00000000000..14d8968866d --- /dev/null +++ b/readthedocs/builds/migrations/0008_added_pull_request_version_type.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.20 on 2019-05-30 10:02 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('builds', '0007_add-automation-rules'), + ] + + operations = [ + migrations.AlterField( + model_name='version', + name='type', + field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('pull_request', 'Pull Request'), ('unknown', 'Unknown')], default='unknown', max_length=20, verbose_name='Type'), + ), + migrations.AlterField( + model_name='versionautomationrule', + name='version_type', + field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('pull_request', 'Pull Request'), ('unknown', 'Unknown')], max_length=32, verbose_name='Version type'), + ), + ] From f81bcf0718fccc8bad270da517b97dfb4df7e9c9 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 30 May 2019 23:52:36 +0600 Subject: [PATCH 28/52] Version Model Methods Updated --- readthedocs/builds/models.py | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index 1039b815326..ca44b39f252 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -40,6 +40,7 @@ BUILD_TYPES, LATEST, NON_REPOSITORY_VERSIONS, + PULL_REQUEST, STABLE, TAG, VERSION_TYPES, @@ -142,7 +143,8 @@ def vcs_url(self): """ Generate VCS (github, gitlab, bitbucket) URL for this version. - Example: https://github.com/rtfd/readthedocs.org/tree/3.4.2/. + Branch/Tag Example: https://github.com/rtfd/readthedocs.org/tree/3.4.2/. + Pull Request Example: https://github.com/rtfd/readthedocs.org/pull/9999/. """ url = '' if self.slug == STABLE: @@ -152,12 +154,24 @@ def vcs_url(self): else: slug_url = self.slug - if ('github' in self.project.repo) or ('gitlab' in self.project.repo): - url = f'/tree/{slug_url}/' + if self.type == PULL_REQUEST: + if 'github' in self.project.repo: + url = f'/pull/{slug_url}/' - if 'bitbucket' in self.project.repo: - slug_url = self.identifier - url = f'/src/{slug_url}' + if 'gitlab' in self.project.repo: + slug_url = self.identifier + url = f'/merge_requests/{slug_url}/' + + if 'bitbucket' in self.project.repo: + slug_url = self.identifier + url = f'/pull-requests/{slug_url}' + else: + if ('github' in self.project.repo) or ('gitlab' in self.project.repo): + url = f'/tree/{slug_url}/' + + if 'bitbucket' in self.project.repo: + slug_url = self.identifier + url = f'/src/{slug_url}' # TODO: improve this replacing return self.project.repo.replace('git://', 'https://').replace('.git', '') + url @@ -220,7 +234,14 @@ def commit_name(self): # the actual tag name. return self.verbose_name - # If we came that far it's not a special version nor a branch or tag. + if self.type == PULL_REQUEST: + # If this version is a Pull Request, the identifier will + # contain the actual commit hash. which we can use to + # generate url for a given file name + return self.identifier + + # If we came that far it's not a special version + # nor a branch, tag or Pull Request. # Therefore just return the identifier to make a safe guess. log.debug( 'TODO: Raise an exception here. Testing what cases it happens', From 66377c2d75ad702f6ff3520bd2bdf43f7ab739da Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 4 Jun 2019 15:40:32 +0600 Subject: [PATCH 29/52] Internal and External Version Manager added --- readthedocs/builds/managers.py | 33 +++++++++++++++++++++++++++++++++ readthedocs/builds/models.py | 10 +++++++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/readthedocs/builds/managers.py b/readthedocs/builds/managers.py index 68cdec6efe9..991293beae2 100644 --- a/readthedocs/builds/managers.py +++ b/readthedocs/builds/managers.py @@ -19,6 +19,7 @@ STABLE, STABLE_VERBOSE_NAME, TAG, + PULL_REQUEST, ) from .querysets import VersionQuerySet @@ -85,6 +86,38 @@ def get_object_or_log(self, **kwargs): log.warning('Version not found for given kwargs. %s' % kwargs) +class InternalVersionManagerBase(VersionManagerBase): + """ + Version manager that only includes internal version. + + It will exclude PULL_REQUEST type from the queries + and only include BRANCH, TAG, UNKONWN type Versions. + """ + def get_queryset(self): + return super().get_queryset().exclude(type=PULL_REQUEST) + + +class ExternalVersionManagerBase(VersionManagerBase): + """ + Version manager that only includes external version. + + It will only include PULL_REQUEST type Versions in the queries. + """ + def get_queryset(self): + return super().get_queryset().filter(type=PULL_REQUEST) + + class VersionManager(SettingsOverrideObject): _default_class = VersionManagerBase _override_setting = 'VERSION_MANAGER' + + +class InternalVersionManager(SettingsOverrideObject): + _default_class = InternalVersionManagerBase + _override_setting = 'INTERNAL_VERSION_MANAGER' + + +class ExternalVersionManager(SettingsOverrideObject): + _default_class = ExternalVersionManagerBase + _override_setting = 'EXTERNAL_VERSION_MANAGER' + diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index ca44b39f252..bca375570c9 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -45,7 +45,11 @@ TAG, VERSION_TYPES, ) -from .managers import VersionManager +from .managers import ( + VersionManager, + InternalVersionManager, + ExternalVersionManager +) from .querysets import BuildQuerySet, RelatedBuildQuerySet, VersionQuerySet from .utils import ( get_bitbucket_username_repo, @@ -112,6 +116,10 @@ class Version(models.Model): machine = models.BooleanField(_('Machine Created'), default=False) objects = VersionManager.from_queryset(VersionQuerySet)() + # Only include BRANCH, TAG, UNKONWN type Versions. + internal = InternalVersionManager.from_queryset(VersionQuerySet)() + # Only include PULL_REQUEST type Versions. + external = ExternalVersionManager.from_queryset(VersionQuerySet)() class Meta: unique_together = [('project', 'slug')] From 8bdf75c69fcd72a86abb0ac5cd05d09b0b5cf97a Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 4 Jun 2019 16:20:19 +0600 Subject: [PATCH 30/52] lint fix --- readthedocs/builds/managers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/readthedocs/builds/managers.py b/readthedocs/builds/managers.py index 991293beae2..2a8bb4840d9 100644 --- a/readthedocs/builds/managers.py +++ b/readthedocs/builds/managers.py @@ -87,22 +87,26 @@ def get_object_or_log(self, **kwargs): class InternalVersionManagerBase(VersionManagerBase): + """ Version manager that only includes internal version. It will exclude PULL_REQUEST type from the queries and only include BRANCH, TAG, UNKONWN type Versions. """ + def get_queryset(self): return super().get_queryset().exclude(type=PULL_REQUEST) class ExternalVersionManagerBase(VersionManagerBase): + """ Version manager that only includes external version. It will only include PULL_REQUEST type Versions in the queries. """ + def get_queryset(self): return super().get_queryset().filter(type=PULL_REQUEST) @@ -120,4 +124,3 @@ class InternalVersionManager(SettingsOverrideObject): class ExternalVersionManager(SettingsOverrideObject): _default_class = ExternalVersionManagerBase _override_setting = 'EXTERNAL_VERSION_MANAGER' - From 21f96de12712042a7ad42ba89052b092bb3504bb Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 5 Jun 2019 01:18:20 +0600 Subject: [PATCH 31/52] All Version Querysets Updated with InternalVersionManager --- readthedocs/api/v2/views/model_views.py | 2 +- readthedocs/api/v3/views.py | 2 +- readthedocs/builds/models.py | 4 +- readthedocs/builds/views.py | 2 +- .../core/management/commands/update_repos.py | 42 +++++++++++++++++++ .../management/commands/update_versions.py | 2 +- readthedocs/core/views/serve.py | 4 +- readthedocs/projects/admin.py | 4 +- readthedocs/projects/forms.py | 2 +- readthedocs/projects/models.py | 8 ++-- readthedocs/projects/views/public.py | 6 +-- 11 files changed, 61 insertions(+), 17 deletions(-) diff --git a/readthedocs/api/v2/views/model_views.py b/readthedocs/api/v2/views/model_views.py index af41f28ab84..5f594a032b8 100644 --- a/readthedocs/api/v2/views/model_views.py +++ b/readthedocs/api/v2/views/model_views.py @@ -130,7 +130,7 @@ def active_versions(self, request, **kwargs): Project.objects.api(request.user), pk=kwargs['pk'], ) - versions = project.versions.filter(active=True) + versions = project.versions(manager='internal').filter(active=True) return Response({ 'versions': VersionSerializer(versions, many=True).data, }) diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index 0e8b2b55a0e..57b86360e96 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -230,7 +230,7 @@ class VersionsViewSet(APIv3Settings, NestedViewSetMixin, ProjectQuerySetMixin, lookup_value_regex = r'[^/]+' filterset_class = VersionFilter - queryset = Version.objects.all() + queryset = Version.internal.all() permit_list_expands = [ 'last_build', 'last_build.config', diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index bca375570c9..bcfc5dfff5c 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -142,7 +142,9 @@ def __str__(self): @property def ref(self): if self.slug == STABLE: - stable = determine_stable_version(self.project.versions.all()) + stable = determine_stable_version( + self.project.versions(manager='internal').all() + ) if stable: return stable.slug diff --git a/readthedocs/builds/views.py b/readthedocs/builds/views.py index e8e3d458e2c..3d6e2cc0ca1 100644 --- a/readthedocs/builds/views.py +++ b/readthedocs/builds/views.py @@ -93,7 +93,7 @@ def get_context_data(self, **kwargs): context['project'] = self.project context['active_builds'] = active_builds - context['versions'] = Version.objects.public( + context['versions'] = Version.internal.public( user=self.request.user, project=self.project, ) diff --git a/readthedocs/core/management/commands/update_repos.py b/readthedocs/core/management/commands/update_repos.py index 5852592d360..0e14f9ce339 100644 --- a/readthedocs/core/management/commands/update_repos.py +++ b/readthedocs/core/management/commands/update_repos.py @@ -76,6 +76,48 @@ def handle(self, *args, **options): version.pk, build_pk=build.pk, ) + elif version == 'internal': + log.info('Updating all internal versions for %s', slug) + for version in Version.internal.filter( + project__slug=slug, + active=True, + uploaded=False, + ): + + build = Build.objects.create( + project=version.project, + version=version, + type='html', + state='triggered', + ) + + # pylint: disable=no-value-for-parameter + tasks.update_docs_task( + version.project_id, + build_pk=build.pk, + version_pk=version.pk, + ) + elif version == 'external': + log.info('Updating all external versions for %s', slug) + for version in Version.external.filter( + project__slug=slug, + active=True, + uploaded=False, + ): + + build = Build.objects.create( + project=version.project, + version=version, + type='html', + state='triggered', + ) + + # pylint: disable=no-value-for-parameter + tasks.update_docs_task( + version.project_id, + build_pk=build.pk, + version_pk=version.pk, + ) else: p = Project.all_objects.get(slug=slug) log.info('Building %s', p) diff --git a/readthedocs/core/management/commands/update_versions.py b/readthedocs/core/management/commands/update_versions.py index b0aa1f877cb..6456accc430 100644 --- a/readthedocs/core/management/commands/update_versions.py +++ b/readthedocs/core/management/commands/update_versions.py @@ -13,7 +13,7 @@ class Command(BaseCommand): help = __doc__ def handle(self, *args, **options): - for version in Version.objects.filter(active=True, built=False): + for version in Version.internal.filter(active=True, built=False): # pylint: disable=no-value-for-parameter update_docs_task( version.pk, diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index 237b843e0fb..f03f958b172 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -388,7 +388,7 @@ def changefreqs_generator(): raise Http404 sorted_versions = sort_version_aware( - Version.objects.public( + Version.internal.public( project=project, only_active=True, ), @@ -428,7 +428,7 @@ def changefreqs_generator(): if project.translations.exists(): for translation in project.translations.all(): translation_versions = ( - Version.objects.public(project=translation) + Version.internal.public(project=translation) .values_list('slug', flat=True) ) if version.slug in translation_versions: diff --git a/readthedocs/projects/admin.py b/readthedocs/projects/admin.py index d74e8d99075..fb4aef4772d 100644 --- a/readthedocs/projects/admin.py +++ b/readthedocs/projects/admin.py @@ -239,7 +239,7 @@ def reindex_active_versions(self, request, queryset): """Reindex all active versions of the selected projects to ES.""" qs_iterator = queryset.iterator() for project in qs_iterator: - version_qs = Version.objects.filter(project=project) + version_qs = Version.internal.filter(project=project) active_versions = version_qs.filter(active=True) if not active_versions.exists(): @@ -271,7 +271,7 @@ def wipe_all_versions(self, request, queryset): """Wipe indexes of all versions of selected projects.""" qs_iterator = queryset.iterator() for project in qs_iterator: - version_qs = Version.objects.filter(project=project) + version_qs = Version.internal.filter(project=project) if not version_qs.exists(): self.message_user( request, diff --git a/readthedocs/projects/forms.py b/readthedocs/projects/forms.py index a701a49c020..65c589c8b18 100644 --- a/readthedocs/projects/forms.py +++ b/readthedocs/projects/forms.py @@ -240,7 +240,7 @@ def __init__(self, *args, **kwargs): self.helper.add_input(Submit('save', _('Save'))) default_choice = (None, '-' * 9) - versions_choices = self.instance.versions.filter( + versions_choices = self.instance.versions(manager='internal').filter( machine=False).values_list('verbose_name', flat=True) self.fields['default_branch'].widget = forms.Select( diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index b636b348aa8..dff87e3885f 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -908,7 +908,7 @@ def api_versions(self): def active_versions(self): from readthedocs.builds.models import Version - versions = Version.objects.public(project=self, only_active=True) + versions = Version.internal.public(project=self, only_active=True) return ( versions.filter(built=True, active=True) | versions.filter(active=True, uploaded=True) @@ -922,7 +922,7 @@ def ordered_active_versions(self, user=None): } if user: kwargs['user'] = user - versions = Version.objects.public(**kwargs).select_related( + versions = Version.internal.public(**kwargs).select_related( 'project', 'project__main_language_project', ).prefetch_related( @@ -949,7 +949,7 @@ def all_active_versions(self): :returns: :py:class:`Version` queryset """ - return self.versions.filter(active=True) + return self.versions(manager='internal').filter(active=True) def get_stable_version(self): return self.versions.filter(slug=STABLE).first() @@ -961,7 +961,7 @@ def update_stable_version(self): Return ``None`` if no update was made or if there is no version on the project that can be considered stable. """ - versions = self.versions.all() + versions = self.versions(manager='internal').all() new_stable = determine_stable_version(versions) if new_stable: current_stable = self.get_stable_version() diff --git a/readthedocs/projects/views/public.py b/readthedocs/projects/views/public.py index 64d82272738..ea15bfb168b 100644 --- a/readthedocs/projects/views/public.py +++ b/readthedocs/projects/views/public.py @@ -93,7 +93,7 @@ def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) project = self.get_object() - context['versions'] = Version.objects.public( + context['versions'] = Version.internal.public( user=self.request.user, project=project, ) @@ -179,7 +179,7 @@ def project_downloads(request, project_slug): Project.objects.protected(request.user), slug=project_slug, ) - versions = Version.objects.public(user=request.user, project=project) + versions = Version.internal.public(user=request.user, project=project) versions = sort_version_aware(versions) version_data = OrderedDict() for version in versions: @@ -268,7 +268,7 @@ def project_versions(request, project_slug): slug=project_slug, ) - versions = Version.objects.public( + versions = Version.internal.public( user=request.user, project=project, only_active=False, From 014477d7cafbb58092c0f3781fe433f9de0fa2c4 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 6 Jun 2019 02:15:34 +0600 Subject: [PATCH 32/52] Manager names moved to Constants --- readthedocs/api/v2/views/model_views.py | 4 ++-- readthedocs/builds/constants.py | 7 +++++++ readthedocs/builds/models.py | 5 +++-- readthedocs/core/management/commands/update_repos.py | 5 +++-- readthedocs/core/views/serve.py | 8 +++++++- readthedocs/projects/forms.py | 3 ++- readthedocs/projects/models.py | 6 +++--- 7 files changed, 27 insertions(+), 11 deletions(-) diff --git a/readthedocs/api/v2/views/model_views.py b/readthedocs/api/v2/views/model_views.py index 5f594a032b8..fca91b32626 100644 --- a/readthedocs/api/v2/views/model_views.py +++ b/readthedocs/api/v2/views/model_views.py @@ -10,7 +10,7 @@ from rest_framework.renderers import BaseRenderer, JSONRenderer from rest_framework.response import Response -from readthedocs.builds.constants import BRANCH, TAG +from readthedocs.builds.constants import BRANCH, TAG, INTERNAL from readthedocs.builds.models import Build, BuildCommandResult, Version from readthedocs.core.utils import trigger_build from readthedocs.core.utils.extend import SettingsOverrideObject @@ -130,7 +130,7 @@ def active_versions(self, request, **kwargs): Project.objects.api(request.user), pk=kwargs['pk'], ) - versions = project.versions(manager='internal').filter(active=True) + versions = project.versions(manager=INTERNAL).filter(active=True) return Response({ 'versions': VersionSerializer(versions, many=True).data, }) diff --git a/readthedocs/builds/constants.py b/readthedocs/builds/constants.py index e2e4fc44ac3..e200fc5fe25 100644 --- a/readthedocs/builds/constants.py +++ b/readthedocs/builds/constants.py @@ -55,3 +55,10 @@ LATEST, STABLE, ) + +# Manager name for Internal Versions or Builds. +# ie: Versions and Builds Excluding PULL_REQUEST Type. +INTERNAL = 'internal' +# Manager name for External Versions or Builds. +# ie: Only PULL_REQUEST Type Versions and Builds. +EXTERNAL = 'external' diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index bcfc5dfff5c..d0c5f6a854e 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -32,12 +32,13 @@ from readthedocs.projects.models import APIProject, Project from readthedocs.projects.version_handling import determine_stable_version -from .constants import ( +from readthedocs.builds.constants import ( BRANCH, BUILD_STATE, BUILD_STATE_FINISHED, BUILD_STATE_TRIGGERED, BUILD_TYPES, + INTERNAL, LATEST, NON_REPOSITORY_VERSIONS, PULL_REQUEST, @@ -143,7 +144,7 @@ def __str__(self): def ref(self): if self.slug == STABLE: stable = determine_stable_version( - self.project.versions(manager='internal').all() + self.project.versions(manager=INTERNAL).all() ) if stable: return stable.slug diff --git a/readthedocs/core/management/commands/update_repos.py b/readthedocs/core/management/commands/update_repos.py index 0e14f9ce339..b15ea8c3c9c 100644 --- a/readthedocs/core/management/commands/update_repos.py +++ b/readthedocs/core/management/commands/update_repos.py @@ -10,6 +10,7 @@ from django.core.management.base import BaseCommand +from readthedocs.builds.constants import EXTERNAL, INTERNAL from readthedocs.builds.models import Build, Version from readthedocs.core.utils import trigger_build from readthedocs.projects import tasks @@ -76,7 +77,7 @@ def handle(self, *args, **options): version.pk, build_pk=build.pk, ) - elif version == 'internal': + elif version == INTERNAL: log.info('Updating all internal versions for %s', slug) for version in Version.internal.filter( project__slug=slug, @@ -97,7 +98,7 @@ def handle(self, *args, **options): build_pk=build.pk, version_pk=version.pk, ) - elif version == 'external': + elif version == EXTERNAL: log.info('Updating all external versions for %s', slug) for version in Version.external.filter( project__slug=slug, diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index f03f958b172..d82e8b60443 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -38,7 +38,7 @@ from django.views.decorators.cache import cache_page from django.views.static import serve -from readthedocs.builds.constants import LATEST, STABLE +from readthedocs.builds.constants import LATEST, STABLE, INTERNAL from readthedocs.builds.models import Version from readthedocs.core.permissions import AdminPermission from readthedocs.core.resolver import resolve, resolve_path @@ -427,10 +427,16 @@ def changefreqs_generator(): if project.translations.exists(): for translation in project.translations.all(): +<<<<<<< HEAD translation_versions = ( Version.internal.public(project=translation) .values_list('slug', flat=True) ) +======= + translation_versions = translation.versions( + manager=INTERNAL + ).public().values_list('slug', flat=True) +>>>>>>> Manager names moved to Constants if version.slug in translation_versions: href = project.get_docs_url( version_slug=version.slug, diff --git a/readthedocs/projects/forms.py b/readthedocs/projects/forms.py index 65c589c8b18..3c86cc39e12 100644 --- a/readthedocs/projects/forms.py +++ b/readthedocs/projects/forms.py @@ -14,6 +14,7 @@ from guardian.shortcuts import assign from textclassifier.validators import ClassifierValidator +from readthedocs.builds.constants import INTERNAL from readthedocs.core.utils import slugify, trigger_build from readthedocs.core.utils.extend import SettingsOverrideObject from readthedocs.integrations.models import Integration @@ -240,7 +241,7 @@ def __init__(self, *args, **kwargs): self.helper.add_input(Submit('save', _('Save'))) default_choice = (None, '-' * 9) - versions_choices = self.instance.versions(manager='internal').filter( + versions_choices = self.instance.versions(manager=INTERNAL).filter( machine=False).values_list('verbose_name', flat=True) self.fields['default_branch'].widget = forms.Select( diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index dff87e3885f..a3f36feb5c8 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -20,7 +20,7 @@ from taggit.managers import TaggableManager from readthedocs.api.v2.client import api -from readthedocs.builds.constants import LATEST, STABLE +from readthedocs.builds.constants import LATEST, STABLE, INTERNAL from readthedocs.core.resolver import resolve, resolve_domain from readthedocs.core.utils import broadcast, slugify from readthedocs.projects import constants @@ -949,7 +949,7 @@ def all_active_versions(self): :returns: :py:class:`Version` queryset """ - return self.versions(manager='internal').filter(active=True) + return self.versions(manager=INTERNAL).filter(active=True) def get_stable_version(self): return self.versions.filter(slug=STABLE).first() @@ -961,7 +961,7 @@ def update_stable_version(self): Return ``None`` if no update was made or if there is no version on the project that can be considered stable. """ - versions = self.versions(manager='internal').all() + versions = self.versions(manager=INTERNAL).all() new_stable = determine_stable_version(versions) if new_stable: current_stable = self.get_stable_version() From d3f52542e96d027f282a5876e579daeec4ed81da Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 6 Jun 2019 21:25:03 +0600 Subject: [PATCH 33/52] Tests added --- readthedocs/builds/constants.py | 2 +- .../rtd_tests/tests/test_doc_serving.py | 26 ++++- readthedocs/rtd_tests/tests/test_managers.py | 100 ++++++++++++++++++ readthedocs/rtd_tests/tests/test_project.py | 28 ++++- .../rtd_tests/tests/test_project_forms.py | 25 ++++- .../rtd_tests/tests/test_project_views.py | 30 +++++- readthedocs/rtd_tests/tests/test_version.py | 68 ++++++++++++ 7 files changed, 272 insertions(+), 7 deletions(-) create mode 100644 readthedocs/rtd_tests/tests/test_managers.py create mode 100644 readthedocs/rtd_tests/tests/test_version.py diff --git a/readthedocs/builds/constants.py b/readthedocs/builds/constants.py index e200fc5fe25..37474eb5365 100644 --- a/readthedocs/builds/constants.py +++ b/readthedocs/builds/constants.py @@ -58,7 +58,7 @@ # Manager name for Internal Versions or Builds. # ie: Versions and Builds Excluding PULL_REQUEST Type. -INTERNAL = 'internal' +INTERNAL = 'internal' # Manager name for External Versions or Builds. # ie: Only PULL_REQUEST Type Versions and Builds. EXTERNAL = 'external' diff --git a/readthedocs/rtd_tests/tests/test_doc_serving.py b/readthedocs/rtd_tests/tests/test_doc_serving.py index fed119bbab5..05d1308f4c1 100644 --- a/readthedocs/rtd_tests/tests/test_doc_serving.py +++ b/readthedocs/rtd_tests/tests/test_doc_serving.py @@ -10,7 +10,11 @@ from django.urls import reverse from mock import mock_open, patch +<<<<<<< HEAD from readthedocs.builds.constants import LATEST +======= +from readthedocs.builds.constants import PULL_REQUEST, INTERNAL +>>>>>>> Tests added from readthedocs.builds.models import Version from readthedocs.core.middleware import SubdomainMiddleware from readthedocs.core.views import server_error_404_subdomain @@ -249,6 +253,16 @@ def test_sitemap_xml(self): project=self.public, active=True ) + # This is a Pull Request Version + pr_version = fixture.get( + Version, + identifier='pr-version', + verbose_name='pr-version', + slug='pr-9999', + project=self.public, + active=True, + type=PULL_REQUEST + ) # This also creates a Version `latest` Automatically for this project translation = fixture.get( Project, @@ -268,7 +282,7 @@ def test_sitemap_xml(self): ) self.assertEqual(response.status_code, 200) self.assertEqual(response['Content-Type'], 'application/xml') - for version in self.public.versions.filter(privacy_level=constants.PUBLIC): + for version in self.public.versions(manager=INTERNAL).filter(privacy_level=constants.PUBLIC): self.assertContains( response, self.public.get_docs_url( @@ -303,6 +317,16 @@ def test_sitemap_xml(self): # in language and country value. (zh_CN should be zh-CN) self.assertContains(response, 'zh-CN') + # External Versions should not be in the sitemap_xml. + self.assertNotContains( + response, + self.public.get_docs_url( + version_slug=pr_version.slug, + lang_slug=self.public.language, + private=True, + ), + ) + # Check if STABLE version has 'priority of 1 and changefreq of weekly. self.assertEqual( response.context['versions'][0]['loc'], diff --git a/readthedocs/rtd_tests/tests/test_managers.py b/readthedocs/rtd_tests/tests/test_managers.py new file mode 100644 index 00000000000..8e440e18b85 --- /dev/null +++ b/readthedocs/rtd_tests/tests/test_managers.py @@ -0,0 +1,100 @@ +from django.test import TestCase +from django_dynamic_fixture import get + +from readthedocs.builds.constants import PULL_REQUEST, BRANCH, TAG +from readthedocs.builds.models import Version +from readthedocs.projects.constants import PUBLIC, PRIVATE, PROTECTED +from readthedocs.projects.models import Project + + +class TestVersionManagerBase(TestCase): + + fixtures = ['eric', 'test_data'] + + def setUp(self): + self.client.login(username='eric', password='test') + self.pip = Project.objects.get(slug='pip') + # Create a External Version. ie: PULL_REQUEST type Version. + self.public_pr_version = get( + Version, + project=self.pip, + active=True, + type=PULL_REQUEST, + privacy_level=PUBLIC + ) + self.private_pr_version = get( + Version, + project=self.pip, + active=True, + type=PULL_REQUEST, + privacy_level=PRIVATE + ) + self.protected_pr_version = get( + Version, + project=self.pip, + active=True, + type=PULL_REQUEST, + privacy_level=PROTECTED + ) + self.internal_versions = Version.objects.exclude(type=PULL_REQUEST) + + +class TestInternalVersionManager(TestVersionManagerBase): + + """ + Queries using Internal Manager should only include Internal Versions. + + It will exclude PULL_REQUEST type Versions from the queries + and only include BRANCH, TAG, UNKONWN type Versions. + """ + + def test_internal_version_manager_with_all(self): + self.assertNotIn(self.public_pr_version, Version.internal.all()) + + def test_internal_version_manager_with_public(self): + self.assertNotIn(self.public_pr_version, Version.internal.public()) + + def test_internal_version_manager_with_protected(self): + self.assertNotIn(self.protected_pr_version, Version.internal.protected()) + + def test_internal_version_manager_with_private(self): + self.assertNotIn(self.private_pr_version, Version.internal.private()) + + def test_internal_version_manager_with_api(self): + self.assertNotIn(self.public_pr_version, Version.internal.api()) + + def test_internal_version_manager_with_for_project(self): + self.assertNotIn(self.public_pr_version, Version.internal.for_project(self.pip)) + + +class TestExternalVersionManager(TestVersionManagerBase): + + """ + Queries using External Manager should only include External Versions. + + It will only include PULL_REQUEST type Versions in the queries. + """ + + def test_external_version_manager_with_all(self): + self.assertNotIn(self.internal_versions, Version.external.all()) + self.assertIn(self.public_pr_version, Version.external.all()) + + def test_external_version_manager_with_public(self): + self.assertNotIn(self.internal_versions, Version.external.public()) + self.assertIn(self.public_pr_version, Version.external.public()) + + def test_external_version_manager_with_protected(self): + self.assertNotIn(self.internal_versions, Version.external.protected()) + self.assertIn(self.protected_pr_version, Version.external.protected()) + + def test_external_version_manager_with_private(self): + self.assertNotIn(self.internal_versions, Version.external.private()) + self.assertIn(self.private_pr_version, Version.external.private()) + + def test_external_version_manager_with_api(self): + self.assertNotIn(self.internal_versions, Version.external.api()) + self.assertIn(self.public_pr_version, Version.external.api()) + + def test_external_version_manager_with_for_project(self): + self.assertNotIn(self.internal_versions, Version.external.for_project(self.pip)) + self.assertIn(self.public_pr_version, Version.external.for_project(self.pip)) diff --git a/readthedocs/rtd_tests/tests/test_project.py b/readthedocs/rtd_tests/tests/test_project.py index 40f86350ed4..c2db080f201 100644 --- a/readthedocs/rtd_tests/tests/test_project.py +++ b/readthedocs/rtd_tests/tests/test_project.py @@ -14,8 +14,9 @@ BUILD_STATE_FINISHED, BUILD_STATE_TRIGGERED, LATEST, + PULL_REQUEST, ) -from readthedocs.builds.models import Build +from readthedocs.builds.models import Build, Version from readthedocs.projects.exceptions import ProjectConfigurationError from readthedocs.projects.models import Project from readthedocs.projects.tasks import finish_inactive_builds @@ -29,6 +30,16 @@ class ProjectMixin: def setUp(self): self.client.login(username='eric', password='test') self.pip = Project.objects.get(slug='pip') + # Create a External Version. ie: PULL_REQUEST type Version. + self.pr_version = get( + Version, + identifier='pr-version', + verbose_name='pr-version', + slug='pr-9999', + project=self.pip, + active=True, + type=PULL_REQUEST + ) class TestProject(ProjectMixin, TestCase): @@ -134,6 +145,21 @@ def test_get_storage_path(self): 'htmlzip/pip/latest/pip.zip', ) + def test_ordered_active_versions_excludes_pr_versions(self): + self.assertNotIn(self.pr_version, self.pip.ordered_active_versions()) + + def test_active_versions_excludes_pr_versions(self): + self.assertNotIn(self.pr_version, self.pip.active_versions()) + + def test_all_active_versions_excludes_pr_versions(self): + self.assertNotIn(self.pr_version, self.pip.all_active_versions()) + + def test_update_stable_version_excludes_pr_versions(self): + # Delete all versions excluding PR Versions. + self.pip.versions.exclude(type=PULL_REQUEST).delete() + # Test that PR Version is not considered for stable. + self.assertEqual(self.pip.update_stable_version(), None) + class TestProjectTranslations(ProjectMixin, TestCase): diff --git a/readthedocs/rtd_tests/tests/test_project_forms.py b/readthedocs/rtd_tests/tests/test_project_forms.py index 50d9815d051..b2e8687208a 100644 --- a/readthedocs/rtd_tests/tests/test_project_forms.py +++ b/readthedocs/rtd_tests/tests/test_project_forms.py @@ -5,7 +5,7 @@ from django_dynamic_fixture import get from textclassifier.validators import ClassifierValidator -from readthedocs.builds.constants import LATEST, STABLE +from readthedocs.builds.constants import LATEST, STABLE, PULL_REQUEST from readthedocs.builds.models import Version from readthedocs.projects.constants import ( PRIVATE, @@ -314,7 +314,7 @@ def setUp(self): verbose_name='protected', ) - def test_list_only_non_auto_generated_versions_on_default_branch(self): + def test_list_only_non_auto_generated_versions_in_default_branch_choices(self): form = ProjectAdvancedForm(instance=self.project) # This version is created automatically by the project on save latest = self.project.versions.filter(slug=LATEST) @@ -336,7 +336,7 @@ def test_list_only_non_auto_generated_versions_on_default_branch(self): 'default_branch'].widget.choices], ) - def test_list_user_created_latest_and_stable_versions_on_default_branch(self): + def test_list_user_created_latest_and_stable_versions_in_default_branch_choices(self): self.project.versions.filter(slug=LATEST).first().delete() user_created_latest_version = get( Version, @@ -383,6 +383,25 @@ def test_commit_name_not_in_default_branch_choices(self): 'default_branch'].widget.choices], ) + def test_pr_version_not_in_default_branch_choices(self): + pr_version = get( + Version, + identifier='pr-version', + verbose_name='pr-version', + slug='pr-9999', + project=self.project, + active=True, + type=PULL_REQUEST, + privacy_level=PUBLIC, + ) + form = ProjectAdvancedForm(instance=self.project) + + self.assertNotIn( + pr_version.verbose_name, + [identifier for identifier, _ in form.fields[ + 'default_branch'].widget.choices], + ) + class TestTranslationForms(TestCase): diff --git a/readthedocs/rtd_tests/tests/test_project_views.py b/readthedocs/rtd_tests/tests/test_project_views.py index 7c5b286bdd4..90578c8dceb 100644 --- a/readthedocs/rtd_tests/tests/test_project_views.py +++ b/readthedocs/rtd_tests/tests/test_project_views.py @@ -12,7 +12,7 @@ from django_dynamic_fixture import get, new from mock import patch -from readthedocs.builds.constants import LATEST +from readthedocs.builds.constants import LATEST, PULL_REQUEST from readthedocs.builds.models import Build, Version from readthedocs.oauth.models import RemoteRepository from readthedocs.projects import tasks @@ -370,12 +370,40 @@ def test_import_demo_imported_duplicate(self): class TestPublicViews(MockBuildTestCase): def setUp(self): self.pip = get(Project, slug='pip') + self.pr_version = get( + Version, + identifier='pr-version', + verbose_name='pr-version', + slug='pr-9999', + project=self.pip, + active=True, + type=PULL_REQUEST + ) def test_project_download_media(self): url = reverse('project_download_media', args=[self.pip.slug, 'pdf', LATEST]) response = self.client.get(url) self.assertEqual(response.status_code, 302) + def test_project_detail_view_only_shows_internal_versons(self): + url = reverse('projects_detail', args=[self.pip.slug]) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertNotIn(self.pr_version, response.context['versions']) + + def test_project_downloads_only_shows_internal_versons(self): + url = reverse('project_downloads', args=[self.pip.slug]) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertNotIn(self.pr_version, response.context['versions']) + + def test_project_versions_only_shows_internal_versons(self): + url = reverse('project_version_list', args=[self.pip.slug]) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertNotIn(self.pr_version, response.context['active_versions']) + self.assertNotIn(self.pr_version, response.context['inactive_versions']) + class TestPrivateViews(MockBuildTestCase): def setUp(self): diff --git a/readthedocs/rtd_tests/tests/test_version.py b/readthedocs/rtd_tests/tests/test_version.py new file mode 100644 index 00000000000..8a6f5bf3db8 --- /dev/null +++ b/readthedocs/rtd_tests/tests/test_version.py @@ -0,0 +1,68 @@ +from django.test import TestCase +from django_dynamic_fixture import get + +from readthedocs.builds.constants import PULL_REQUEST, BRANCH, TAG +from readthedocs.builds.models import Version +from readthedocs.projects.models import Project + + +class VersionMixin: + + fixtures = ['eric', 'test_data'] + + def setUp(self): + self.client.login(username='eric', password='test') + self.pip = Project.objects.get(slug='pip') + # Create a External Version. ie: PULL_REQUEST type Version. + self.pr_version = get( + Version, + identifier='9F86D081884C7D659A2FEAA0C55AD015A', + verbose_name='pr-version', + slug='9999', + project=self.pip, + active=True, + type=PULL_REQUEST + ) + self.branch_version = get( + Version, + identifier='origin/stable', + verbose_name='stable', + slug='stable', + project=self.pip, + active=True, + type=BRANCH + ) + self.tag_version = get( + Version, + identifier='origin/master', + verbose_name='latest', + slug='latest', + project=self.pip, + active=True, + type=TAG + ) + + +class TestVersionModel(VersionMixin, TestCase): + + def test_vcs_url_for_pr_version(self): + expected_url = f'https://github.com/pypa/pip/pull/{self.pr_version.slug}/' + self.assertEqual(self.pr_version.vcs_url, expected_url) + + def test_vcs_url_for_latest_version(self): + slug = self.pip.default_branch or self.pip.vcs_repo().fallback_branch + expected_url = f'https://github.com/pypa/pip/tree/{slug}/' + self.assertEqual(self.tag_version.vcs_url, expected_url) + + def test_vcs_url_for_stable_version(self): + expected_url = f'https://github.com/pypa/pip/tree/{self.branch_version.ref}/' + self.assertEqual(self.branch_version.vcs_url, expected_url) + + def test_commit_name_for_stable_version(self): + self.assertEqual(self.branch_version.commit_name, 'stable') + + def test_commit_name_for_latest_version(self): + self.assertEqual(self.tag_version.commit_name, 'master') + + def test_commit_name_for_pr_version(self): + self.assertEqual(self.pr_version.commit_name, self.pr_version.identifier) From 6bca1e42472e67da0b564c332debeaa44d599437 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 6 Jun 2019 21:52:31 +0600 Subject: [PATCH 34/52] Version Update, Delete Html and wipe updated to exclude PR Versions --- readthedocs/core/views/__init__.py | 2 +- readthedocs/projects/views/private.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/readthedocs/core/views/__init__.py b/readthedocs/core/views/__init__.py index 6b438866ae5..de519a13759 100644 --- a/readthedocs/core/views/__init__.py +++ b/readthedocs/core/views/__init__.py @@ -73,7 +73,7 @@ def random_page(request, project_slug=None): # pylint: disable=unused-argument def wipe_version(request, project_slug, version_slug): version = get_object_or_404( - Version, + Version.internal.all(), project__slug=project_slug, slug=version_slug, ) diff --git a/readthedocs/projects/views/private.py b/readthedocs/projects/views/private.py index 46e83d96188..044a83641e6 100644 --- a/readthedocs/projects/views/private.py +++ b/readthedocs/projects/views/private.py @@ -157,7 +157,7 @@ def project_version_detail(request, project_slug, version_slug): slug=project_slug, ) version = get_object_or_404( - Version.objects.public( + Version.internal.public( user=request.user, project=project, only_active=False, @@ -682,7 +682,7 @@ def project_version_delete_html(request, project_slug, version_slug): slug=project_slug, ) version = get_object_or_404( - Version.objects.public( + Version.internal.public( user=request.user, project=project, only_active=False, From 8830a187ed84a56314ddba5c497b39065419c2cb Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 6 Jun 2019 23:03:09 +0600 Subject: [PATCH 35/52] Updated migrations --- .../builds/migrations/0008_added_pull_request_version_type.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py b/readthedocs/builds/migrations/0008_added_pull_request_version_type.py index 14d8968866d..75a126d37b6 100644 --- a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py +++ b/readthedocs/builds/migrations/0008_added_pull_request_version_type.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11.20 on 2019-05-30 10:02 +# Generated by Django 1.11.20 on 2019-06-06 16:51 from __future__ import unicode_literals from django.db import migrations, models From 5ed38c4dc238122aa69d5cd06fc78a29c76e5ff4 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Sun, 9 Jun 2019 02:17:05 +0600 Subject: [PATCH 36/52] footer API updated --- readthedocs/api/v2/views/footer_views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readthedocs/api/v2/views/footer_views.py b/readthedocs/api/v2/views/footer_views.py index 04d4fe8210b..04fedeadd3f 100644 --- a/readthedocs/api/v2/views/footer_views.py +++ b/readthedocs/api/v2/views/footer_views.py @@ -9,7 +9,7 @@ from rest_framework_jsonp.renderers import JSONPRenderer from readthedocs.api.v2.signals import footer_response -from readthedocs.builds.constants import LATEST, TAG +from readthedocs.builds.constants import LATEST, TAG, INTERNAL from readthedocs.builds.models import Version from readthedocs.projects.models import Project from readthedocs.projects.version_handling import ( @@ -25,7 +25,7 @@ def get_version_compare_data(project, base_version=None): :param base_version: We assert whether or not the base_version is also the highest version in the resulting "is_highest" value. """ - versions_qs = Version.objects.public(project=project) + versions_qs = Version.internal.public(project=project) # Take preferences over tags only if the project has at least one tag if versions_qs.filter(type=TAG).exists(): From 2d8b072255d2c42949ea4309c8864a146955a6c7 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Wed, 12 Jun 2019 00:45:00 +0600 Subject: [PATCH 37/52] manager tests updated --- readthedocs/rtd_tests/tests/test_managers.py | 34 +++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_managers.py b/readthedocs/rtd_tests/tests/test_managers.py index 8e440e18b85..e69abdc4dd8 100644 --- a/readthedocs/rtd_tests/tests/test_managers.py +++ b/readthedocs/rtd_tests/tests/test_managers.py @@ -1,3 +1,4 @@ +from django.contrib.auth import get_user_model from django.test import TestCase from django_dynamic_fixture import get @@ -7,12 +8,16 @@ from readthedocs.projects.models import Project +User = get_user_model() + + class TestVersionManagerBase(TestCase): - fixtures = ['eric', 'test_data'] + fixtures = ['test_data'] def setUp(self): - self.client.login(username='eric', password='test') + self.user = User.objects.create(username='test_user', password='test') + self.client.login(username='test_user', password='test') self.pip = Project.objects.get(slug='pip') # Create a External Version. ie: PULL_REQUEST type Version. self.public_pr_version = get( @@ -54,6 +59,12 @@ def test_internal_version_manager_with_all(self): def test_internal_version_manager_with_public(self): self.assertNotIn(self.public_pr_version, Version.internal.public()) + def test_internal_version_manager_with_public_with_user_and_project(self): + self.assertNotIn( + self.public_pr_version, + Version.internal.public(self.user, self.pip) + ) + def test_internal_version_manager_with_protected(self): self.assertNotIn(self.protected_pr_version, Version.internal.protected()) @@ -83,6 +94,17 @@ def test_external_version_manager_with_public(self): self.assertNotIn(self.internal_versions, Version.external.public()) self.assertIn(self.public_pr_version, Version.external.public()) + def test_external_version_manager_with_public_with_user_and_project(self): + self.assertNotIn( + self.internal_versions, + Version.external.public(self.user, self.pip) + ) + self.assertIn( + self.public_pr_version, + Version.external.public(self.user, self.pip) + ) + + def test_external_version_manager_with_protected(self): self.assertNotIn(self.internal_versions, Version.external.protected()) self.assertIn(self.protected_pr_version, Version.external.protected()) @@ -96,5 +118,9 @@ def test_external_version_manager_with_api(self): self.assertIn(self.public_pr_version, Version.external.api()) def test_external_version_manager_with_for_project(self): - self.assertNotIn(self.internal_versions, Version.external.for_project(self.pip)) - self.assertIn(self.public_pr_version, Version.external.for_project(self.pip)) + self.assertNotIn( + self.internal_versions, Version.external.for_project(self.pip) + ) + self.assertIn( + self.public_pr_version, Version.external.for_project(self.pip) + ) From 89f3c8af6c2d28625827bc0dc36c10f8b66424d3 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 13 Jun 2019 18:56:06 +0600 Subject: [PATCH 38/52] _override_setting removed from new managers --- readthedocs/builds/managers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/readthedocs/builds/managers.py b/readthedocs/builds/managers.py index 2a8bb4840d9..0bc30f02807 100644 --- a/readthedocs/builds/managers.py +++ b/readthedocs/builds/managers.py @@ -118,9 +118,7 @@ class VersionManager(SettingsOverrideObject): class InternalVersionManager(SettingsOverrideObject): _default_class = InternalVersionManagerBase - _override_setting = 'INTERNAL_VERSION_MANAGER' class ExternalVersionManager(SettingsOverrideObject): _default_class = ExternalVersionManagerBase - _override_setting = 'EXTERNAL_VERSION_MANAGER' From 7b63973dc50f3bee61988e40387a9f47e2803f32 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 13 Jun 2019 23:11:51 +0600 Subject: [PATCH 39/52] BuildTriggerMixin updated --- readthedocs/builds/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/builds/views.py b/readthedocs/builds/views.py index 3d6e2cc0ca1..3333495f6c8 100644 --- a/readthedocs/builds/views.py +++ b/readthedocs/builds/views.py @@ -57,7 +57,7 @@ def post(self, request, project_slug): version_slug = request.POST.get('version_slug') version = get_object_or_404( - Version, + Version.internal.all(), project=project, slug=version_slug, ) From 3a2fc6dcb1ef267d5f5bd1d0db5989837b7fbfb3 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 13 Jun 2019 23:25:28 +0600 Subject: [PATCH 40/52] More Update --- readthedocs/search/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/search/utils.py b/readthedocs/search/utils.py index 5f04f6736b7..cab67ba5ff8 100644 --- a/readthedocs/search/utils.py +++ b/readthedocs/search/utils.py @@ -28,7 +28,7 @@ def get_project_list_or_404(project_slug, user, version_slug=None): main_project = get_object_or_404(Project, slug=project_slug) subprojects = Project.objects.filter(superprojects__parent_id=main_project.id) for project in list(subprojects) + [main_project]: - version = Version.objects.public(user).filter(project__slug=project.slug, slug=version_slug) + version = Version.internal.public(user).filter(project__slug=project.slug, slug=version_slug) if version.exists(): project_list.append(version.first().project) return project_list From bca152a69c201c61ab508bbd00b26b7e51cd884a Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 13 Jun 2019 23:55:32 +0600 Subject: [PATCH 41/52] lint fix --- readthedocs/rtd_tests/tests/test_managers.py | 1 - readthedocs/search/utils.py | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_managers.py b/readthedocs/rtd_tests/tests/test_managers.py index e69abdc4dd8..f830c9af549 100644 --- a/readthedocs/rtd_tests/tests/test_managers.py +++ b/readthedocs/rtd_tests/tests/test_managers.py @@ -104,7 +104,6 @@ def test_external_version_manager_with_public_with_user_and_project(self): Version.external.public(self.user, self.pip) ) - def test_external_version_manager_with_protected(self): self.assertNotIn(self.internal_versions, Version.external.protected()) self.assertIn(self.protected_pr_version, Version.external.protected()) diff --git a/readthedocs/search/utils.py b/readthedocs/search/utils.py index cab67ba5ff8..3433195bb55 100644 --- a/readthedocs/search/utils.py +++ b/readthedocs/search/utils.py @@ -28,7 +28,9 @@ def get_project_list_or_404(project_slug, user, version_slug=None): main_project = get_object_or_404(Project, slug=project_slug) subprojects = Project.objects.filter(superprojects__parent_id=main_project.id) for project in list(subprojects) + [main_project]: - version = Version.internal.public(user).filter(project__slug=project.slug, slug=version_slug) + version = Version.internal.public(user).filter( + project__slug=project.slug, slug=version_slug + ) if version.exists(): project_list.append(version.first().project) return project_list From 5f900529099fcd83d8f03b699a43e584638f0335 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 13 Jun 2019 14:53:57 -0500 Subject: [PATCH 42/52] Move search functions Follow up from #5798 --- readthedocs/projects/tasks.py | 2 +- readthedocs/search/signals.py | 61 -------------------------------- readthedocs/search/utils.py | 66 +++++++++++++++++++++++++++++++++-- 3 files changed, 64 insertions(+), 65 deletions(-) diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index d0ebdb826a0..7efcd5a762a 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -60,7 +60,7 @@ from readthedocs.doc_builder.loader import get_builder_class from readthedocs.doc_builder.python_environments import Conda, Virtualenv from readthedocs.projects.models import APIProject, Feature -from readthedocs.search.signals import index_new_files, remove_indexed_files +from readthedocs.search.utils import index_new_files, remove_indexed_files from readthedocs.sphinx_domains.models import SphinxDomain from readthedocs.vcs_support import utils as vcs_support_utils from readthedocs.worker import app diff --git a/readthedocs/search/signals.py b/readthedocs/search/signals.py index 494725d22a7..294d96242d0 100644 --- a/readthedocs/search/signals.py +++ b/readthedocs/search/signals.py @@ -13,67 +13,6 @@ log = logging.getLogger(__name__) -def index_new_files(model, version, build): - """Index new files from the version into the search index.""" - - if not DEDConfig.autosync_enabled(): - log.info( - 'Autosync disabled, skipping indexing into the search index for: %s:%s', - version.project.slug, - version.slug, - ) - return - - try: - document = list(registry.get_documents(models=[model]))[0] - doc_obj = document() - queryset = ( - doc_obj.get_queryset() - .filter(project=version.project, version=version, build=build) - ) - log.info( - 'Indexing new objecst into search index for: %s:%s', - version.project.slug, - version.slug, - ) - doc_obj.update(queryset.iterator()) - except Exception: - log.exception('Unable to index a subset of files. Continuing.') - - -def remove_indexed_files(model, version, build): - """ - Remove files from the version from the search index. - - This excludes files from the current build. - """ - - if not DEDConfig.autosync_enabled(): - log.info( - 'Autosync disabled, skipping removal from the search index for: %s:%s', - version.project.slug, - version.slug, - ) - return - - try: - document = list(registry.get_documents(models=[model]))[0] - log.info( - 'Deleting old files from search index for: %s:%s', - version.project.slug, - version.slug, - ) - ( - document().search() - .filter('term', project=version.project.slug) - .filter('term', version=version.slug) - .exclude('term', build=build) - .delete() - ) - except Exception: - log.exception('Unable to delete a subset of files. Continuing.') - - @receiver(post_save, sender=Project) def index_project_save(instance, *args, **kwargs): """ diff --git a/readthedocs/search/utils.py b/readthedocs/search/utils.py index 3433195bb55..6cbc0272722 100644 --- a/readthedocs/search/utils.py +++ b/readthedocs/search/utils.py @@ -1,20 +1,80 @@ -# -*- coding: utf-8 -*- - """Utilities related to reading and generating indexable search content.""" import logging from django.shortcuts import get_object_or_404 +from django_elasticsearch_dsl.apps import DEDConfig from django_elasticsearch_dsl.registries import registry from readthedocs.builds.models import Version -from readthedocs.projects.models import Project, HTMLFile +from readthedocs.projects.models import HTMLFile, Project from readthedocs.search.documents import PageDocument log = logging.getLogger(__name__) +def index_new_files(model, version, build): + """Index new files from the version into the search index.""" + + if not DEDConfig.autosync_enabled(): + log.info( + 'Autosync disabled, skipping indexing into the search index for: %s:%s', + version.project.slug, + version.slug, + ) + return + + try: + document = list(registry.get_documents(models=[model]))[0] + doc_obj = document() + queryset = ( + doc_obj.get_queryset() + .filter(project=version.project, version=version, build=build) + ) + log.info( + 'Indexing new objecst into search index for: %s:%s', + version.project.slug, + version.slug, + ) + doc_obj.update(queryset.iterator()) + except Exception: + log.exception('Unable to index a subset of files. Continuing.') + + +def remove_indexed_files(model, version, build): + """ + Remove files from the version from the search index. + + This excludes files from the current build. + """ + + if not DEDConfig.autosync_enabled(): + log.info( + 'Autosync disabled, skipping removal from the search index for: %s:%s', + version.project.slug, + version.slug, + ) + return + + try: + document = list(registry.get_documents(models=[model]))[0] + log.info( + 'Deleting old files from search index for: %s:%s', + version.project.slug, + version.slug, + ) + ( + document().search() + .filter('term', project=version.project.slug) + .filter('term', version=version.slug) + .exclude('term', build=build) + .delete() + ) + except Exception: + log.exception('Unable to delete a subset of files. Continuing.') + + # TODO: Rewrite all the views using this in Class Based View, # and move this function to a mixin def get_project_list_or_404(project_slug, user, version_slug=None): From 120eb5d38fb9ae456bd15637b1b45c471bdcd5e0 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Fri, 14 Jun 2019 17:35:34 +0600 Subject: [PATCH 43/52] removed unused import --- readthedocs/core/views/serve.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index d82e8b60443..f03f958b172 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -38,7 +38,7 @@ from django.views.decorators.cache import cache_page from django.views.static import serve -from readthedocs.builds.constants import LATEST, STABLE, INTERNAL +from readthedocs.builds.constants import LATEST, STABLE from readthedocs.builds.models import Version from readthedocs.core.permissions import AdminPermission from readthedocs.core.resolver import resolve, resolve_path @@ -427,16 +427,10 @@ def changefreqs_generator(): if project.translations.exists(): for translation in project.translations.all(): -<<<<<<< HEAD translation_versions = ( Version.internal.public(project=translation) .values_list('slug', flat=True) ) -======= - translation_versions = translation.versions( - manager=INTERNAL - ).public().values_list('slug', flat=True) ->>>>>>> Manager names moved to Constants if version.slug in translation_versions: href = project.get_docs_url( version_slug=version.slug, From ef3d06171f1f693e39dd5e51b48f678b3d7517a1 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 18 Jun 2019 01:30:26 +0600 Subject: [PATCH 44/52] External version name added everywhere --- readthedocs/builds/constants.py | 17 ++++++++--------- readthedocs/builds/managers.py | 10 +++++----- .../0008_added_pull_request_version_type.py | 6 +++--- readthedocs/builds/models.py | 14 +++++++------- readthedocs/rtd_tests/tests/test_doc_serving.py | 8 ++++++-- readthedocs/rtd_tests/tests/test_managers.py | 16 ++++++++-------- readthedocs/rtd_tests/tests/test_project.py | 8 ++++---- .../rtd_tests/tests/test_project_forms.py | 4 ++-- .../rtd_tests/tests/test_project_views.py | 4 ++-- readthedocs/rtd_tests/tests/test_version.py | 6 +++--- 10 files changed, 48 insertions(+), 45 deletions(-) diff --git a/readthedocs/builds/constants.py b/readthedocs/builds/constants.py index 37474eb5365..bdc9a70fe79 100644 --- a/readthedocs/builds/constants.py +++ b/readthedocs/builds/constants.py @@ -31,15 +31,21 @@ ('dash', _('Dash')), ) +# Manager name for Internal Versions or Builds. +# ie: Versions and Builds Excluding pull request/merge request Versions and Builds. +INTERNAL = 'internal' +# Manager name for External Versions or Builds. +# ie: Only pull request/merge request Versions and Builds. +EXTERNAL = 'external' + BRANCH = 'branch' TAG = 'tag' -PULL_REQUEST = 'pull_request' UNKNOWN = 'unknown' VERSION_TYPES = ( (BRANCH, _('Branch')), (TAG, _('Tag')), - (PULL_REQUEST, _('Pull Request')), + (EXTERNAL, _('External')), (UNKNOWN, _('Unknown')), ) @@ -55,10 +61,3 @@ LATEST, STABLE, ) - -# Manager name for Internal Versions or Builds. -# ie: Versions and Builds Excluding PULL_REQUEST Type. -INTERNAL = 'internal' -# Manager name for External Versions or Builds. -# ie: Only PULL_REQUEST Type Versions and Builds. -EXTERNAL = 'external' diff --git a/readthedocs/builds/managers.py b/readthedocs/builds/managers.py index 0bc30f02807..06262ed3e24 100644 --- a/readthedocs/builds/managers.py +++ b/readthedocs/builds/managers.py @@ -19,7 +19,7 @@ STABLE, STABLE_VERBOSE_NAME, TAG, - PULL_REQUEST, + EXTERNAL, ) from .querysets import VersionQuerySet @@ -91,12 +91,12 @@ class InternalVersionManagerBase(VersionManagerBase): """ Version manager that only includes internal version. - It will exclude PULL_REQUEST type from the queries + It will exclude pull request/merge request versions from the queries and only include BRANCH, TAG, UNKONWN type Versions. """ def get_queryset(self): - return super().get_queryset().exclude(type=PULL_REQUEST) + return super().get_queryset().exclude(type=EXTERNAL) class ExternalVersionManagerBase(VersionManagerBase): @@ -104,11 +104,11 @@ class ExternalVersionManagerBase(VersionManagerBase): """ Version manager that only includes external version. - It will only include PULL_REQUEST type Versions in the queries. + It will only include pull request/merge request Versions in the queries. """ def get_queryset(self): - return super().get_queryset().filter(type=PULL_REQUEST) + return super().get_queryset().filter(type=EXTERNAL) class VersionManager(SettingsOverrideObject): diff --git a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py b/readthedocs/builds/migrations/0008_added_pull_request_version_type.py index 75a126d37b6..2bf9dc62d66 100644 --- a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py +++ b/readthedocs/builds/migrations/0008_added_pull_request_version_type.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11.20 on 2019-06-06 16:51 +# Generated by Django 1.11.21 on 2019-06-17 19:26 from __future__ import unicode_literals from django.db import migrations, models @@ -15,11 +15,11 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='version', name='type', - field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('pull_request', 'Pull Request'), ('unknown', 'Unknown')], default='unknown', max_length=20, verbose_name='Type'), + field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('external', 'External'), ('unknown', 'Unknown')], default='unknown', max_length=20, verbose_name='Type'), ), migrations.AlterField( model_name='versionautomationrule', name='version_type', - field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('pull_request', 'Pull Request'), ('unknown', 'Unknown')], max_length=32, verbose_name='Version type'), + field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('external', 'External'), ('unknown', 'Unknown')], max_length=32, verbose_name='Version type'), ), ] diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index d0c5f6a854e..34a3fd372a9 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -41,7 +41,7 @@ INTERNAL, LATEST, NON_REPOSITORY_VERSIONS, - PULL_REQUEST, + EXTERNAL, STABLE, TAG, VERSION_TYPES, @@ -119,7 +119,7 @@ class Version(models.Model): objects = VersionManager.from_queryset(VersionQuerySet)() # Only include BRANCH, TAG, UNKONWN type Versions. internal = InternalVersionManager.from_queryset(VersionQuerySet)() - # Only include PULL_REQUEST type Versions. + # Only include EXTERNAL type Versions. external = ExternalVersionManager.from_queryset(VersionQuerySet)() class Meta: @@ -155,7 +155,7 @@ def vcs_url(self): Generate VCS (github, gitlab, bitbucket) URL for this version. Branch/Tag Example: https://github.com/rtfd/readthedocs.org/tree/3.4.2/. - Pull Request Example: https://github.com/rtfd/readthedocs.org/pull/9999/. + Pull/merge Request Example: https://github.com/rtfd/readthedocs.org/pull/9999/. """ url = '' if self.slug == STABLE: @@ -165,7 +165,7 @@ def vcs_url(self): else: slug_url = self.slug - if self.type == PULL_REQUEST: + if self.type == EXTERNAL: if 'github' in self.project.repo: url = f'/pull/{slug_url}/' @@ -245,14 +245,14 @@ def commit_name(self): # the actual tag name. return self.verbose_name - if self.type == PULL_REQUEST: - # If this version is a Pull Request, the identifier will + if self.type == EXTERNAL: + # If this version is a EXTERNAL version, the identifier will # contain the actual commit hash. which we can use to # generate url for a given file name return self.identifier # If we came that far it's not a special version - # nor a branch, tag or Pull Request. + # nor a branch, tag or EXTERNAL version. # Therefore just return the identifier to make a safe guess. log.debug( 'TODO: Raise an exception here. Testing what cases it happens', diff --git a/readthedocs/rtd_tests/tests/test_doc_serving.py b/readthedocs/rtd_tests/tests/test_doc_serving.py index 05d1308f4c1..5bfbbf13f4c 100644 --- a/readthedocs/rtd_tests/tests/test_doc_serving.py +++ b/readthedocs/rtd_tests/tests/test_doc_serving.py @@ -10,11 +10,15 @@ from django.urls import reverse from mock import mock_open, patch +<<<<<<< HEAD <<<<<<< HEAD from readthedocs.builds.constants import LATEST ======= from readthedocs.builds.constants import PULL_REQUEST, INTERNAL >>>>>>> Tests added +======= +from readthedocs.builds.constants import LATEST, EXTERNAL, INTERNAL +>>>>>>> External version name added everywhere from readthedocs.builds.models import Version from readthedocs.core.middleware import SubdomainMiddleware from readthedocs.core.views import server_error_404_subdomain @@ -253,7 +257,7 @@ def test_sitemap_xml(self): project=self.public, active=True ) - # This is a Pull Request Version + # This is a EXTERNAL Version pr_version = fixture.get( Version, identifier='pr-version', @@ -261,7 +265,7 @@ def test_sitemap_xml(self): slug='pr-9999', project=self.public, active=True, - type=PULL_REQUEST + type=EXTERNAL ) # This also creates a Version `latest` Automatically for this project translation = fixture.get( diff --git a/readthedocs/rtd_tests/tests/test_managers.py b/readthedocs/rtd_tests/tests/test_managers.py index f830c9af549..924544a8be8 100644 --- a/readthedocs/rtd_tests/tests/test_managers.py +++ b/readthedocs/rtd_tests/tests/test_managers.py @@ -2,7 +2,7 @@ from django.test import TestCase from django_dynamic_fixture import get -from readthedocs.builds.constants import PULL_REQUEST, BRANCH, TAG +from readthedocs.builds.constants import EXTERNAL, BRANCH, TAG from readthedocs.builds.models import Version from readthedocs.projects.constants import PUBLIC, PRIVATE, PROTECTED from readthedocs.projects.models import Project @@ -19,29 +19,29 @@ def setUp(self): self.user = User.objects.create(username='test_user', password='test') self.client.login(username='test_user', password='test') self.pip = Project.objects.get(slug='pip') - # Create a External Version. ie: PULL_REQUEST type Version. + # Create a External Version. ie: pull/merge request Version. self.public_pr_version = get( Version, project=self.pip, active=True, - type=PULL_REQUEST, + type=EXTERNAL, privacy_level=PUBLIC ) self.private_pr_version = get( Version, project=self.pip, active=True, - type=PULL_REQUEST, + type=EXTERNAL, privacy_level=PRIVATE ) self.protected_pr_version = get( Version, project=self.pip, active=True, - type=PULL_REQUEST, + type=EXTERNAL, privacy_level=PROTECTED ) - self.internal_versions = Version.objects.exclude(type=PULL_REQUEST) + self.internal_versions = Version.objects.exclude(type=EXTERNAL) class TestInternalVersionManager(TestVersionManagerBase): @@ -49,7 +49,7 @@ class TestInternalVersionManager(TestVersionManagerBase): """ Queries using Internal Manager should only include Internal Versions. - It will exclude PULL_REQUEST type Versions from the queries + It will exclude EXTERNAL type Versions from the queries and only include BRANCH, TAG, UNKONWN type Versions. """ @@ -83,7 +83,7 @@ class TestExternalVersionManager(TestVersionManagerBase): """ Queries using External Manager should only include External Versions. - It will only include PULL_REQUEST type Versions in the queries. + It will only include pull/merge request Version in the queries. """ def test_external_version_manager_with_all(self): diff --git a/readthedocs/rtd_tests/tests/test_project.py b/readthedocs/rtd_tests/tests/test_project.py index c2db080f201..5d5194d04d2 100644 --- a/readthedocs/rtd_tests/tests/test_project.py +++ b/readthedocs/rtd_tests/tests/test_project.py @@ -14,7 +14,7 @@ BUILD_STATE_FINISHED, BUILD_STATE_TRIGGERED, LATEST, - PULL_REQUEST, + EXTERNAL, ) from readthedocs.builds.models import Build, Version from readthedocs.projects.exceptions import ProjectConfigurationError @@ -30,7 +30,7 @@ class ProjectMixin: def setUp(self): self.client.login(username='eric', password='test') self.pip = Project.objects.get(slug='pip') - # Create a External Version. ie: PULL_REQUEST type Version. + # Create a External Version. ie: pull/merge request Version. self.pr_version = get( Version, identifier='pr-version', @@ -38,7 +38,7 @@ def setUp(self): slug='pr-9999', project=self.pip, active=True, - type=PULL_REQUEST + type=EXTERNAL ) @@ -156,7 +156,7 @@ def test_all_active_versions_excludes_pr_versions(self): def test_update_stable_version_excludes_pr_versions(self): # Delete all versions excluding PR Versions. - self.pip.versions.exclude(type=PULL_REQUEST).delete() + self.pip.versions.exclude(type=EXTERNAL).delete() # Test that PR Version is not considered for stable. self.assertEqual(self.pip.update_stable_version(), None) diff --git a/readthedocs/rtd_tests/tests/test_project_forms.py b/readthedocs/rtd_tests/tests/test_project_forms.py index b2e8687208a..ba5b3c88b9d 100644 --- a/readthedocs/rtd_tests/tests/test_project_forms.py +++ b/readthedocs/rtd_tests/tests/test_project_forms.py @@ -5,7 +5,7 @@ from django_dynamic_fixture import get from textclassifier.validators import ClassifierValidator -from readthedocs.builds.constants import LATEST, STABLE, PULL_REQUEST +from readthedocs.builds.constants import LATEST, STABLE, EXTERNAL from readthedocs.builds.models import Version from readthedocs.projects.constants import ( PRIVATE, @@ -391,7 +391,7 @@ def test_pr_version_not_in_default_branch_choices(self): slug='pr-9999', project=self.project, active=True, - type=PULL_REQUEST, + type=EXTERNAL, privacy_level=PUBLIC, ) form = ProjectAdvancedForm(instance=self.project) diff --git a/readthedocs/rtd_tests/tests/test_project_views.py b/readthedocs/rtd_tests/tests/test_project_views.py index 90578c8dceb..3bc948aff9b 100644 --- a/readthedocs/rtd_tests/tests/test_project_views.py +++ b/readthedocs/rtd_tests/tests/test_project_views.py @@ -12,7 +12,7 @@ from django_dynamic_fixture import get, new from mock import patch -from readthedocs.builds.constants import LATEST, PULL_REQUEST +from readthedocs.builds.constants import LATEST, EXTERNAL from readthedocs.builds.models import Build, Version from readthedocs.oauth.models import RemoteRepository from readthedocs.projects import tasks @@ -377,7 +377,7 @@ def setUp(self): slug='pr-9999', project=self.pip, active=True, - type=PULL_REQUEST + type=EXTERNAL ) def test_project_download_media(self): diff --git a/readthedocs/rtd_tests/tests/test_version.py b/readthedocs/rtd_tests/tests/test_version.py index 8a6f5bf3db8..e414ba79cc6 100644 --- a/readthedocs/rtd_tests/tests/test_version.py +++ b/readthedocs/rtd_tests/tests/test_version.py @@ -1,7 +1,7 @@ from django.test import TestCase from django_dynamic_fixture import get -from readthedocs.builds.constants import PULL_REQUEST, BRANCH, TAG +from readthedocs.builds.constants import EXTERNAL, BRANCH, TAG from readthedocs.builds.models import Version from readthedocs.projects.models import Project @@ -13,7 +13,7 @@ class VersionMixin: def setUp(self): self.client.login(username='eric', password='test') self.pip = Project.objects.get(slug='pip') - # Create a External Version. ie: PULL_REQUEST type Version. + # Create a External Version. ie: pull/merge request Version. self.pr_version = get( Version, identifier='9F86D081884C7D659A2FEAA0C55AD015A', @@ -21,7 +21,7 @@ def setUp(self): slug='9999', project=self.pip, active=True, - type=PULL_REQUEST + type=EXTERNAL ) self.branch_version = get( Version, From 5b33bbd71a257fdce5ff49897cb8845e74707df6 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 18 Jun 2019 01:43:48 +0600 Subject: [PATCH 45/52] Migration name changed --- ...uest_version_type.py => 0008_added_external_version_type.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename readthedocs/builds/migrations/{0008_added_pull_request_version_type.py => 0008_added_external_version_type.py} (94%) diff --git a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py b/readthedocs/builds/migrations/0008_added_external_version_type.py similarity index 94% rename from readthedocs/builds/migrations/0008_added_pull_request_version_type.py rename to readthedocs/builds/migrations/0008_added_external_version_type.py index 2bf9dc62d66..8de9c4f504f 100644 --- a/readthedocs/builds/migrations/0008_added_pull_request_version_type.py +++ b/readthedocs/builds/migrations/0008_added_external_version_type.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11.21 on 2019-06-17 19:26 +# Generated by Django 1.11.21 on 2019-06-17 19:43 from __future__ import unicode_literals from django.db import migrations, models From 518074335bcc722d4342c8d6ac9d34a68d215b1b Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Sat, 8 Jun 2019 21:10:38 +0600 Subject: [PATCH 46/52] Build Managers added --- readthedocs/builds/managers.py | 60 ++++++++++++++++++++++++++++++++- readthedocs/builds/models.py | 26 +++++++++----- readthedocs/builds/querysets.py | 1 - 3 files changed, 77 insertions(+), 10 deletions(-) diff --git a/readthedocs/builds/managers.py b/readthedocs/builds/managers.py index 06262ed3e24..78e274b5008 100644 --- a/readthedocs/builds/managers.py +++ b/readthedocs/builds/managers.py @@ -21,7 +21,7 @@ TAG, EXTERNAL, ) -from .querysets import VersionQuerySet +from .querysets import VersionQuerySet, BuildQuerySet log = logging.getLogger(__name__) @@ -122,3 +122,61 @@ class InternalVersionManager(SettingsOverrideObject): class ExternalVersionManager(SettingsOverrideObject): _default_class = ExternalVersionManagerBase + + +class BuildManagerBase(models.Manager): + + """ + Build manager for manager only queries. + + For creating different Managers. + """ + + @classmethod + def from_queryset(cls, queryset_class, class_name=None): + # This is overridden because :py:meth:`models.Manager.from_queryset` + # uses `inspect` to retrieve the class methods, and the proxy class has + # no direct members. + queryset_class = get_override_class( + BuildQuerySet, + BuildQuerySet._default_class, # pylint: disable=protected-access + ) + return super().from_queryset(queryset_class, class_name) + + +class InternalBuildManagerBase(BuildManagerBase): + + """ + Build manager that only includes internal version builds. + + It will exclude PULL_REQUEST type Version builds from the queries + and only include BRANCH, TAG, UNKONWN type Version builds. + """ + + def get_queryset(self): + return super().get_queryset().exclude(version__type=PULL_REQUEST) + + +class ExternalBuildManagerBase(BuildManagerBase): + + """ + Build manager that only includes external version builds. + + It will only include PULL_REQUEST type Versions builds in the queries. + """ + + def get_queryset(self): + return super().get_queryset().filter(version__type=PULL_REQUEST) + + +class BuildManager(SettingsOverrideObject): + _default_class = BuildManagerBase + _override_setting = 'BUILD_MANAGER' + + +class InternalBuildManager(SettingsOverrideObject): + _default_class = InternalBuildManagerBase + + +class ExternalBuildManager(SettingsOverrideObject): + _default_class = ExternalBuildManagerBase diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index 34a3fd372a9..ba176acfb65 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -46,18 +46,25 @@ TAG, VERSION_TYPES, ) -from .managers import ( +from readthedocs.builds.managers import ( VersionManager, InternalVersionManager, - ExternalVersionManager + ExternalVersionManager, + BuildManager, + InternalBuildManager, + ExternalBuildManager, ) -from .querysets import BuildQuerySet, RelatedBuildQuerySet, VersionQuerySet -from .utils import ( +from readthedocs.builds.querysets import ( + BuildQuerySet, + RelatedBuildQuerySet, + VersionQuerySet, +) +from readthedocs.builds.utils import ( get_bitbucket_username_repo, get_github_username_repo, get_gitlab_username_repo, ) -from .version_slug import VersionSlugField +from readthedocs.builds.version_slug import VersionSlugField log = logging.getLogger(__name__) @@ -627,9 +634,12 @@ class Build(models.Model): help_text='Build steps stored outside the database.', ) - # Manager - - objects = BuildQuerySet.as_manager() + # Managers + objects = BuildManager.from_queryset(BuildQuerySet)() + # Only include BRANCH, TAG, UNKONWN type Version builds. + internal = InternalBuildManager.from_queryset(BuildQuerySet)() + # Only include PULL_REQUEST type Version builds. + external = ExternalBuildManager.from_queryset(BuildQuerySet)() CONFIG_KEY = '__config' diff --git a/readthedocs/builds/querysets.py b/readthedocs/builds/querysets.py index 6b4c9132b1a..d6e4c1bda2c 100644 --- a/readthedocs/builds/querysets.py +++ b/readthedocs/builds/querysets.py @@ -116,7 +116,6 @@ def api(self, user=None, detail=True): class BuildQuerySet(SettingsOverrideObject): _default_class = BuildQuerySetBase - _override_setting = 'BUILD_MANAGER' class RelatedBuildQuerySetBase(models.QuerySet): From 8301679c94976297f7a4cb3253a2f91ab9a05cc1 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Thu, 13 Jun 2019 22:59:45 +0600 Subject: [PATCH 47/52] Managers used in all the places --- readthedocs/api/v3/views.py | 2 +- readthedocs/builds/models.py | 4 ++-- readthedocs/builds/views.py | 15 +++++++++++++++ readthedocs/core/templatetags/privacy_tags.py | 2 +- readthedocs/projects/models.py | 2 +- readthedocs/projects/querysets.py | 4 ++-- 6 files changed, 22 insertions(+), 7 deletions(-) diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index 57b86360e96..e3ddb9daa93 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -269,7 +269,7 @@ class BuildsViewSet(APIv3Settings, NestedViewSetMixin, ProjectQuerySetMixin, lookup_url_kwarg = 'build_pk' serializer_class = BuildSerializer filterset_class = BuildFilter - queryset = Build.objects.all() + queryset = Build.internal.all() permit_list_expands = [ 'config', ] diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index ba176acfb65..c77681c50a0 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -207,7 +207,7 @@ def config(self): :rtype: dict """ last_build = ( - self.builds.filter( + self.builds(manager=INTERNAL).filter( state='finished', success=True, ).order_by('-date').first() @@ -662,7 +662,7 @@ def previous(self): date = self.date or timezone.now() if self.project is not None and self.version is not None: return ( - Build.objects.filter( + Build.internal.filter( project=self.project, version=self.version, date__lt=date, diff --git a/readthedocs/builds/views.py b/readthedocs/builds/views.py index 3333495f6c8..479d0797458 100644 --- a/readthedocs/builds/views.py +++ b/readthedocs/builds/views.py @@ -84,6 +84,21 @@ def post(self, request, project_slug): class BuildList(BuildBase, BuildTriggerMixin, ListView): + def get_queryset(self): + # this is used to include only internal version + # builds in the build list page + self.project_slug = self.kwargs.get('project_slug', None) + self.project = get_object_or_404( + Project.objects.protected(self.request.user), + slug=self.project_slug, + ) + queryset = Build.internal.public( + user=self.request.user, + project=self.project, + ).select_related('project', 'version') + + return queryset + def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) diff --git a/readthedocs/core/templatetags/privacy_tags.py b/readthedocs/core/templatetags/privacy_tags.py index 115bb9eadd2..e4ad684a8a0 100644 --- a/readthedocs/core/templatetags/privacy_tags.py +++ b/readthedocs/core/templatetags/privacy_tags.py @@ -26,7 +26,7 @@ def get_public_projects(context, user): viewer=context['request'].user, ).prefetch_latest_build().annotate( _good_build=Exists( - Build.objects.filter(success=True, project=OuterRef('pk'))) + Build.internal.filter(success=True, project=OuterRef('pk'))) ) context['public_projects'] = projects return '' diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index a3f36feb5c8..2e782c7296a 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -768,7 +768,7 @@ def has_good_build(self): # Used for Database optimization. if hasattr(self, '_good_build'): return self._good_build - return self.builds.filter(success=True).exists() + return self.builds(manager=INTERNAL).filter(success=True).exists() @property def has_versions(self): diff --git a/readthedocs/projects/querysets.py b/readthedocs/projects/querysets.py index 7ea7249ae0d..ff4221fd590 100644 --- a/readthedocs/projects/querysets.py +++ b/readthedocs/projects/querysets.py @@ -85,13 +85,13 @@ def prefetch_latest_build(self): # Prefetch the latest build for each project. subquery = Subquery( - Build.objects.filter( + Build.internal.filter( project=OuterRef('project_id') ).order_by('-date').values_list('id', flat=True)[:1] ) latest_build = Prefetch( 'builds', - Build.objects.filter(pk__in=subquery), + Build.internal.filter(pk__in=subquery), to_attr=self.model.LATEST_BUILD_CACHE, ) return self.prefetch_related(latest_build) From 7b75ee817a8c525f8ee3f0507134a0b370dedfc4 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Fri, 14 Jun 2019 05:00:06 +0600 Subject: [PATCH 48/52] build manager tests added --- readthedocs/rtd_tests/tests/test_managers.py | 90 +++++++++++++++++++- 1 file changed, 89 insertions(+), 1 deletion(-) diff --git a/readthedocs/rtd_tests/tests/test_managers.py b/readthedocs/rtd_tests/tests/test_managers.py index 924544a8be8..0d649dfac1e 100644 --- a/readthedocs/rtd_tests/tests/test_managers.py +++ b/readthedocs/rtd_tests/tests/test_managers.py @@ -3,7 +3,7 @@ from django_dynamic_fixture import get from readthedocs.builds.constants import EXTERNAL, BRANCH, TAG -from readthedocs.builds.models import Version +from readthedocs.builds.models import Version, Build from readthedocs.projects.constants import PUBLIC, PRIVATE, PROTECTED from readthedocs.projects.models import Project @@ -123,3 +123,91 @@ def test_external_version_manager_with_for_project(self): self.assertIn( self.public_pr_version, Version.external.for_project(self.pip) ) + + +class TestBuildManagerBase(TestCase): + + fixtures = ['test_data'] + + def setUp(self): + self.user = User.objects.create(username='test_user', password='test') + self.client.login(username='test_user', password='test') + self.pip = Project.objects.get(slug='pip') + print(self.pip.versions.all()) + # Create a External Version and build. ie: PULL_REQUEST type Version. + self.pr_version = get( + Version, + project=self.pip, + active=True, + type=PULL_REQUEST, + privacy_level=PUBLIC + ) + self.pr_version_build = get( + Build, + project=self.pip, + version=self.pr_version + ) + # Create a Internal Version build. + self.internal_version_build = get( + Build, + project=self.pip, + version=self.pip.versions.get(slug='0.8') + ) + + self.internal_builds = Build.objects.exclude(version__type=PULL_REQUEST) + + +class TestInternalBuildManager(TestBuildManagerBase): + + """ + Queries using Internal Manager should only include Internal Version builds. + + It will exclude PULL_REQUEST type Version builds from the queries + and only include BRANCH, TAG, UNKONWN type Versions. + """ + + def test_internal_build_manager_with_all(self): + self.assertNotIn(self.pr_version_build, Build.internal.all()) + + def test_internal_build_manager_with_public(self): + self.assertNotIn(self.pr_version_build, Build.internal.public()) + + def test_internal_build_manager_with_public_with_user_and_project(self): + self.assertNotIn( + self.pr_version_build, + Build.internal.public(self.user, self.pip) + ) + + def test_internal_build_manager_with_api(self): + self.assertNotIn(self.pr_version_build, Build.internal.api()) + + +class TestExternalBuildManager(TestBuildManagerBase): + + """ + Queries using External Manager should only include External Version builds. + + It will only include PULL_REQUEST type Version builds in the queries. + """ + + def test_external_build_manager_with_all(self): + self.assertNotIn(self.internal_builds, Build.external.all()) + self.assertIn(self.pr_version_build, Build.external.all()) + + def test_external_build_manager_with_public(self): + self.assertNotIn(self.internal_builds, Build.external.public()) + self.assertIn(self.pr_version_build, Build.external.public()) + + def test_external_build_manager_with_public_with_user_and_project(self): + self.assertNotIn( + self.internal_builds, + Build.external.public(self.user, self.pip) + ) + self.assertIn( + self.pr_version_build, + Build.external.public(self.user, self.pip) + ) + + def test_external_build_manager_with_api(self): + self.assertNotIn(self.internal_builds, Build.external.api()) + self.assertIn(self.pr_version_build, Build.external.api()) From 8c7b02b2c15cfb71dee0b4c7b926de6db9b18eba Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Fri, 14 Jun 2019 05:06:12 +0600 Subject: [PATCH 49/52] More Tests added --- readthedocs/builds/models.py | 2 +- readthedocs/rtd_tests/tests/test_project.py | 6 +++++ readthedocs/rtd_tests/tests/test_views.py | 25 +++++++++++++++++++-- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index c77681c50a0..7e7e827f91e 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -662,7 +662,7 @@ def previous(self): date = self.date or timezone.now() if self.project is not None and self.version is not None: return ( - Build.internal.filter( + Build.objects.filter( project=self.project, version=self.version, date__lt=date, diff --git a/readthedocs/rtd_tests/tests/test_project.py b/readthedocs/rtd_tests/tests/test_project.py index 5d5194d04d2..d9cb1cdca15 100644 --- a/readthedocs/rtd_tests/tests/test_project.py +++ b/readthedocs/rtd_tests/tests/test_project.py @@ -160,6 +160,12 @@ def test_update_stable_version_excludes_pr_versions(self): # Test that PR Version is not considered for stable. self.assertEqual(self.pip.update_stable_version(), None) + def test_has_good_build_excludes_pr_versions(self): + # Delete all versions excluding PR Versions. + self.pip.versions.exclude(type=PULL_REQUEST).delete() + # Test that PR Version is not considered for has_good_build. + self.assertFalse(self.pip.has_good_build) + class TestProjectTranslations(ProjectMixin, TestCase): diff --git a/readthedocs/rtd_tests/tests/test_views.py b/readthedocs/rtd_tests/tests/test_views.py index 25e7b60a756..bfbf07ce243 100644 --- a/readthedocs/rtd_tests/tests/test_views.py +++ b/readthedocs/rtd_tests/tests/test_views.py @@ -7,8 +7,8 @@ from django.urls import reverse from django_dynamic_fixture import get, new -from readthedocs.builds.constants import LATEST -from readthedocs.builds.models import Build +from readthedocs.builds.constants import LATEST, PULL_REQUEST +from readthedocs.builds.models import Build, Version from readthedocs.core.permissions import AdminPermission from readthedocs.projects.forms import UpdateProjectForm from readthedocs.projects.models import HTMLFile, Project @@ -266,6 +266,7 @@ class BuildViewTests(TestCase): def setUp(self): self.client.login(username='eric', password='test') + self.pip = Project.objects.get(slug='pip') @mock.patch('readthedocs.projects.tasks.update_docs_task') def test_build_redirect(self, mock): @@ -276,3 +277,23 @@ def test_build_redirect(self, mock): r._headers['location'][1], '/projects/pip/builds/%s/' % build.pk, ) + + def test_build_list_does_not_include_pr_versions(self): + pr_version = get( + Version, + project = self.pip, + active = True, + type = PULL_REQUEST, + ) + pr_version_build = get( + Build, + project = self.pip, + version = pr_version + ) + response = self.client.get( + reverse('builds_project_list', args=[self.pip.slug]), + ) + self.assertEqual(response.status_code, 200) + + self.assertNotIn(pr_version_build, response.context['build_qs']) + self.assertNotIn(pr_version_build, response.context['active_builds']) From 5d6c750fc2580c5e4b73755e1abbd5c4bb17e61f Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 18 Jun 2019 01:55:10 +0600 Subject: [PATCH 50/52] External version name added --- readthedocs/builds/managers.py | 8 ++++---- readthedocs/builds/models.py | 2 +- readthedocs/rtd_tests/tests/test_managers.py | 10 +++++----- readthedocs/rtd_tests/tests/test_project.py | 2 +- readthedocs/rtd_tests/tests/test_views.py | 4 ++-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/readthedocs/builds/managers.py b/readthedocs/builds/managers.py index 78e274b5008..d80852478f2 100644 --- a/readthedocs/builds/managers.py +++ b/readthedocs/builds/managers.py @@ -149,12 +149,12 @@ class InternalBuildManagerBase(BuildManagerBase): """ Build manager that only includes internal version builds. - It will exclude PULL_REQUEST type Version builds from the queries + It will exclude pull request/merge request version builds from the queries and only include BRANCH, TAG, UNKONWN type Version builds. """ def get_queryset(self): - return super().get_queryset().exclude(version__type=PULL_REQUEST) + return super().get_queryset().exclude(version__type=EXTERNAL) class ExternalBuildManagerBase(BuildManagerBase): @@ -162,11 +162,11 @@ class ExternalBuildManagerBase(BuildManagerBase): """ Build manager that only includes external version builds. - It will only include PULL_REQUEST type Versions builds in the queries. + It will only include pull request/merge request version builds in the queries. """ def get_queryset(self): - return super().get_queryset().filter(version__type=PULL_REQUEST) + return super().get_queryset().filter(version__type=EXTERNAL) class BuildManager(SettingsOverrideObject): diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index 7e7e827f91e..75abdcfda24 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -638,7 +638,7 @@ class Build(models.Model): objects = BuildManager.from_queryset(BuildQuerySet)() # Only include BRANCH, TAG, UNKONWN type Version builds. internal = InternalBuildManager.from_queryset(BuildQuerySet)() - # Only include PULL_REQUEST type Version builds. + # Only include EXTERNAL type Version builds. external = ExternalBuildManager.from_queryset(BuildQuerySet)() CONFIG_KEY = '__config' diff --git a/readthedocs/rtd_tests/tests/test_managers.py b/readthedocs/rtd_tests/tests/test_managers.py index 0d649dfac1e..50aca0f8150 100644 --- a/readthedocs/rtd_tests/tests/test_managers.py +++ b/readthedocs/rtd_tests/tests/test_managers.py @@ -134,12 +134,12 @@ def setUp(self): self.client.login(username='test_user', password='test') self.pip = Project.objects.get(slug='pip') print(self.pip.versions.all()) - # Create a External Version and build. ie: PULL_REQUEST type Version. + # Create a External Version and build. ie: pull/merge request Version. self.pr_version = get( Version, project=self.pip, active=True, - type=PULL_REQUEST, + type=EXTERNAL, privacy_level=PUBLIC ) self.pr_version_build = get( @@ -154,7 +154,7 @@ def setUp(self): version=self.pip.versions.get(slug='0.8') ) - self.internal_builds = Build.objects.exclude(version__type=PULL_REQUEST) + self.internal_builds = Build.objects.exclude(version__type=EXTERNAL) class TestInternalBuildManager(TestBuildManagerBase): @@ -162,7 +162,7 @@ class TestInternalBuildManager(TestBuildManagerBase): """ Queries using Internal Manager should only include Internal Version builds. - It will exclude PULL_REQUEST type Version builds from the queries + It will exclude pull/merge request Version builds from the queries and only include BRANCH, TAG, UNKONWN type Versions. """ @@ -187,7 +187,7 @@ class TestExternalBuildManager(TestBuildManagerBase): """ Queries using External Manager should only include External Version builds. - It will only include PULL_REQUEST type Version builds in the queries. + It will only include pull/merge request Version builds in the queries. """ def test_external_build_manager_with_all(self): diff --git a/readthedocs/rtd_tests/tests/test_project.py b/readthedocs/rtd_tests/tests/test_project.py index d9cb1cdca15..af5fb5d508a 100644 --- a/readthedocs/rtd_tests/tests/test_project.py +++ b/readthedocs/rtd_tests/tests/test_project.py @@ -162,7 +162,7 @@ def test_update_stable_version_excludes_pr_versions(self): def test_has_good_build_excludes_pr_versions(self): # Delete all versions excluding PR Versions. - self.pip.versions.exclude(type=PULL_REQUEST).delete() + self.pip.versions.exclude(type=EXTERNAL).delete() # Test that PR Version is not considered for has_good_build. self.assertFalse(self.pip.has_good_build) diff --git a/readthedocs/rtd_tests/tests/test_views.py b/readthedocs/rtd_tests/tests/test_views.py index bfbf07ce243..71d69ab2e6b 100644 --- a/readthedocs/rtd_tests/tests/test_views.py +++ b/readthedocs/rtd_tests/tests/test_views.py @@ -7,7 +7,7 @@ from django.urls import reverse from django_dynamic_fixture import get, new -from readthedocs.builds.constants import LATEST, PULL_REQUEST +from readthedocs.builds.constants import LATEST, EXTERNAL from readthedocs.builds.models import Build, Version from readthedocs.core.permissions import AdminPermission from readthedocs.projects.forms import UpdateProjectForm @@ -283,7 +283,7 @@ def test_build_list_does_not_include_pr_versions(self): Version, project = self.pip, active = True, - type = PULL_REQUEST, + type = EXTERNAL, ) pr_version_build = get( Build, From 75efef807a8f8d7e340d83e8841ee9a09bae0a7c Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 18 Jun 2019 16:32:29 +0600 Subject: [PATCH 51/52] naming updated --- .../rtd_tests/tests/test_doc_serving.py | 4 ++-- readthedocs/rtd_tests/tests/test_managers.py | 22 +++++++++---------- readthedocs/rtd_tests/tests/test_project.py | 2 +- readthedocs/rtd_tests/tests/test_views.py | 12 +++++----- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_doc_serving.py b/readthedocs/rtd_tests/tests/test_doc_serving.py index 8809be15f59..379387aac5c 100644 --- a/readthedocs/rtd_tests/tests/test_doc_serving.py +++ b/readthedocs/rtd_tests/tests/test_doc_serving.py @@ -241,7 +241,7 @@ def test_sitemap_xml(self): active=True ) # This is a EXTERNAL Version - pr_version = fixture.get( + external_version = fixture.get( Version, identifier='pr-version', verbose_name='pr-version', @@ -327,7 +327,7 @@ def test_sitemap_xml(self): self.assertNotContains( response, self.public.get_docs_url( - version_slug=pr_version.slug, + version_slug=external_version.slug, lang_slug=self.public.language, private=True, ), diff --git a/readthedocs/rtd_tests/tests/test_managers.py b/readthedocs/rtd_tests/tests/test_managers.py index 2e7364e35b3..a17aee9c7a7 100644 --- a/readthedocs/rtd_tests/tests/test_managers.py +++ b/readthedocs/rtd_tests/tests/test_managers.py @@ -135,17 +135,17 @@ def setUp(self): self.pip = Project.objects.get(slug='pip') print(self.pip.versions.all()) # Create a External Version and build. ie: pull/merge request Version. - self.pr_version = get( + self.external_version = get( Version, project=self.pip, active=True, type=EXTERNAL, privacy_level=PUBLIC ) - self.pr_version_build = get( + self.external_version_build = get( Build, project=self.pip, - version=self.pr_version + version=self.external_version ) # Create a Internal Version build. self.internal_version_build = get( @@ -167,19 +167,19 @@ class TestInternalBuildManager(TestBuildManagerBase): """ def test_internal_build_manager_with_all(self): - self.assertNotIn(self.pr_version_build, Build.internal.all()) + self.assertNotIn(self.external_version_build, Build.internal.all()) def test_internal_build_manager_with_public(self): - self.assertNotIn(self.pr_version_build, Build.internal.public()) + self.assertNotIn(self.external_version_build, Build.internal.public()) def test_internal_build_manager_with_public_with_user_and_project(self): self.assertNotIn( - self.pr_version_build, + self.external_version_build, Build.internal.public(self.user, self.pip) ) def test_internal_build_manager_with_api(self): - self.assertNotIn(self.pr_version_build, Build.internal.api()) + self.assertNotIn(self.external_version_build, Build.internal.api()) class TestExternalBuildManager(TestBuildManagerBase): @@ -192,11 +192,11 @@ class TestExternalBuildManager(TestBuildManagerBase): def test_external_build_manager_with_all(self): self.assertNotIn(self.internal_builds, Build.external.all()) - self.assertIn(self.pr_version_build, Build.external.all()) + self.assertIn(self.external_version_build, Build.external.all()) def test_external_build_manager_with_public(self): self.assertNotIn(self.internal_builds, Build.external.public()) - self.assertIn(self.pr_version_build, Build.external.public()) + self.assertIn(self.external_version_build, Build.external.public()) def test_external_build_manager_with_public_with_user_and_project(self): self.assertNotIn( @@ -204,10 +204,10 @@ def test_external_build_manager_with_public_with_user_and_project(self): Build.external.public(self.user, self.pip) ) self.assertIn( - self.pr_version_build, + self.external_version_build, Build.external.public(self.user, self.pip) ) def test_external_build_manager_with_api(self): self.assertNotIn(self.internal_builds, Build.external.api()) - self.assertIn(self.pr_version_build, Build.external.api()) + self.assertIn(self.external_version_build, Build.external.api()) diff --git a/readthedocs/rtd_tests/tests/test_project.py b/readthedocs/rtd_tests/tests/test_project.py index 023db2734d2..ded11f52c53 100644 --- a/readthedocs/rtd_tests/tests/test_project.py +++ b/readthedocs/rtd_tests/tests/test_project.py @@ -160,7 +160,7 @@ def test_update_stable_version_excludes_external_versions(self): # Test that PR Version is not considered for stable. self.assertEqual(self.pip.update_stable_version(), None) - def test_has_good_build_excludes_pr_versions(self): + def test_has_good_build_excludes_external_versions(self): # Delete all versions excluding PR Versions. self.pip.versions.exclude(type=EXTERNAL).delete() # Test that PR Version is not considered for has_good_build. diff --git a/readthedocs/rtd_tests/tests/test_views.py b/readthedocs/rtd_tests/tests/test_views.py index 71d69ab2e6b..de77945b311 100644 --- a/readthedocs/rtd_tests/tests/test_views.py +++ b/readthedocs/rtd_tests/tests/test_views.py @@ -278,22 +278,22 @@ def test_build_redirect(self, mock): '/projects/pip/builds/%s/' % build.pk, ) - def test_build_list_does_not_include_pr_versions(self): - pr_version = get( + def test_build_list_does_not_include_external_versions(self): + external_version = get( Version, project = self.pip, active = True, type = EXTERNAL, ) - pr_version_build = get( + external_version_build = get( Build, project = self.pip, - version = pr_version + version = external_version ) response = self.client.get( reverse('builds_project_list', args=[self.pip.slug]), ) self.assertEqual(response.status_code, 200) - self.assertNotIn(pr_version_build, response.context['build_qs']) - self.assertNotIn(pr_version_build, response.context['active_builds']) + self.assertNotIn(external_version_build, response.context['build_qs']) + self.assertNotIn(external_version_build, response.context['active_builds']) From 51188e7e7be8c4147f11247d7dbf278b65d24ef5 Mon Sep 17 00:00:00 2001 From: saadmk11 Date: Tue, 18 Jun 2019 18:33:57 +0600 Subject: [PATCH 52/52] fix --- readthedocs/rtd_tests/tests/test_doc_serving.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_doc_serving.py b/readthedocs/rtd_tests/tests/test_doc_serving.py index 379387aac5c..734ed3338b1 100644 --- a/readthedocs/rtd_tests/tests/test_doc_serving.py +++ b/readthedocs/rtd_tests/tests/test_doc_serving.py @@ -240,16 +240,6 @@ def test_sitemap_xml(self): project=self.public, active=True ) - # This is a EXTERNAL Version - external_version = fixture.get( - Version, - identifier='pr-version', - verbose_name='pr-version', - slug='pr-9999', - project=self.public, - active=True, - type=EXTERNAL - ) stable_version = fixture.get( Version, identifier='stable',