Version Model Update for Pull Request Builder

readthedocs/api/v2/views/footer_views.py

@@ -9,7 +9,7 @@
 from rest_framework_jsonp.renderers import JSONPRenderer
 
 from readthedocs.api.v2.signals import footer_response
-from readthedocs.builds.constants import LATEST, TAG
+from readthedocs.builds.constants import LATEST, TAG, INTERNAL
 from readthedocs.builds.models import Version
 from readthedocs.projects.models import Project
 from readthedocs.projects.version_handling import (
@@ -25,7 +25,7 @@ def get_version_compare_data(project, base_version=None):
     :param base_version: We assert whether or not the base_version is also the
                          highest version in the resulting "is_highest" value.
     """
-    versions_qs = Version.objects.public(project=project)
+    versions_qs = Version.internal.public(project=project)
 
     # Take preferences over tags only if the project has at least one tag
     if versions_qs.filter(type=TAG).exists():

readthedocs/api/v2/views/model_views.py

@@ -10,7 +10,7 @@
 from rest_framework.renderers import BaseRenderer, JSONRenderer
 from rest_framework.response import Response
 
-from readthedocs.builds.constants import BRANCH, TAG
+from readthedocs.builds.constants import BRANCH, TAG, INTERNAL
 from readthedocs.builds.models import Build, BuildCommandResult, Version
 from readthedocs.core.utils import trigger_build
 from readthedocs.core.utils.extend import SettingsOverrideObject
@@ -130,7 +130,7 @@ def active_versions(self, request, **kwargs):
             Project.objects.api(request.user),
             pk=kwargs['pk'],
         )
-        versions = project.versions.filter(active=True)
+        versions = project.versions(manager=INTERNAL).filter(active=True)
         return Response({
             'versions': VersionSerializer(versions, many=True).data,
         })

readthedocs/api/v3/views.py

@@ -230,7 +230,7 @@ class VersionsViewSet(APIv3Settings, NestedViewSetMixin, ProjectQuerySetMixin,
     lookup_value_regex = r'[^/]+'
 
     filterset_class = VersionFilter
-    queryset = Version.objects.all()
+    queryset = Version.internal.all()
     permit_list_expands = [
         'last_build',
         'last_build.config',

readthedocs/builds/constants.py

@@ -31,13 +31,21 @@
     ('dash', _('Dash')),
 )
 
+# Manager name for Internal Versions or Builds.
+# ie: Versions and Builds Excluding pull request/merge request Versions and Builds.
+INTERNAL = 'internal'
+# Manager name for External Versions or Builds.
+# ie: Only pull request/merge request Versions and Builds.
+EXTERNAL = 'external'
+
 BRANCH = 'branch'
 TAG = 'tag'
 UNKNOWN = 'unknown'
 
 VERSION_TYPES = (
     (BRANCH, _('Branch')),
     (TAG, _('Tag')),
+    (EXTERNAL, _('External')),
     (UNKNOWN, _('Unknown')),
 )
 

readthedocs/builds/managers.py

@@ -19,6 +19,7 @@
     STABLE,
     STABLE_VERBOSE_NAME,
     TAG,
+    EXTERNAL,
 )
 from .querysets import VersionQuerySet
 
@@ -85,6 +86,39 @@ def get_object_or_log(self, **kwargs):
             log.warning('Version not found for given kwargs. %s' % kwargs)
 
 
+class InternalVersionManagerBase(VersionManagerBase):
+
+    """
+    Version manager that only includes internal version.
+
+    It will exclude pull request/merge request versions from the queries
+    and only include BRANCH, TAG, UNKONWN type Versions.
+    """
+
+    def get_queryset(self):
+        return super().get_queryset().exclude(type=EXTERNAL)
+
+
+class ExternalVersionManagerBase(VersionManagerBase):
+
+    """
+    Version manager that only includes external version.
+
+    It will only include pull request/merge request Versions in the queries.
+    """
+
+    def get_queryset(self):
+        return super().get_queryset().filter(type=EXTERNAL)
+
+
 class VersionManager(SettingsOverrideObject):
     _default_class = VersionManagerBase
     _override_setting = 'VERSION_MANAGER'
+
+
+class InternalVersionManager(SettingsOverrideObject):
+    _default_class = InternalVersionManagerBase
+
+
+class ExternalVersionManager(SettingsOverrideObject):
+    _default_class = ExternalVersionManagerBase

readthedocs/builds/migrations/0008_added_external_version_type.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.21 on 2019-06-17 19:43
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('builds', '0007_add-automation-rules'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='version',
+            name='type',
+            field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('external', 'External'), ('unknown', 'Unknown')], default='unknown', max_length=20, verbose_name='Type'),
+        ),
+        migrations.AlterField(
+            model_name='versionautomationrule',
+            name='version_type',
+            field=models.CharField(choices=[('branch', 'Branch'), ('tag', 'Tag'), ('external', 'External'), ('unknown', 'Unknown')], max_length=32, verbose_name='Version type'),
+        ),
+    ]

readthedocs/builds/models.py

@@ -32,19 +32,25 @@
 from readthedocs.projects.models import APIProject, Project
 from readthedocs.projects.version_handling import determine_stable_version
 
-from .constants import (
+from readthedocs.builds.constants import (
     BRANCH,
     BUILD_STATE,
     BUILD_STATE_FINISHED,
     BUILD_STATE_TRIGGERED,
     BUILD_TYPES,
+    INTERNAL,
     LATEST,
     NON_REPOSITORY_VERSIONS,
+    EXTERNAL,
     STABLE,
     TAG,
     VERSION_TYPES,
 )
-from .managers import VersionManager
+from .managers import (
+    VersionManager,
+    InternalVersionManager,
+    ExternalVersionManager
+)
 from .querysets import BuildQuerySet, RelatedBuildQuerySet, VersionQuerySet
 from .utils import (
     get_bitbucket_username_repo,
@@ -111,6 +117,10 @@ class Version(models.Model):
     machine = models.BooleanField(_('Machine Created'), default=False)
 
     objects = VersionManager.from_queryset(VersionQuerySet)()
+    # Only include BRANCH, TAG, UNKONWN type Versions.
+    internal = InternalVersionManager.from_queryset(VersionQuerySet)()
+    # Only include EXTERNAL type Versions.
+    external = ExternalVersionManager.from_queryset(VersionQuerySet)()
 
     class Meta:
         unique_together = [('project', 'slug')]
@@ -133,7 +143,9 @@ def __str__(self):
     @property
     def ref(self):
         if self.slug == STABLE:
-            stable = determine_stable_version(self.project.versions.all())
+            stable = determine_stable_version(
+                self.project.versions(manager=INTERNAL).all()
+            )
             if stable:
                 return stable.slug
 
@@ -142,7 +154,8 @@ def vcs_url(self):
         """
         Generate VCS (github, gitlab, bitbucket) URL for this version.
 
-        Example: https://github.com/rtfd/readthedocs.org/tree/3.4.2/.
+        Branch/Tag Example: https://github.com/rtfd/readthedocs.org/tree/3.4.2/.
+        Pull/merge Request Example: https://github.com/rtfd/readthedocs.org/pull/9999/.
         """
         url = ''
         if self.slug == STABLE:
@@ -152,12 +165,25 @@ def vcs_url(self):
         else:
             slug_url = self.slug
 
-        if ('github' in self.project.repo) or ('gitlab' in self.project.repo):
-            url = f'/tree/{slug_url}/'
+        if self.type == EXTERNAL:
+            if 'github' in self.project.repo:
+                slug_url = self.verbose_name
+                url = f'/pull/{slug_url}/'
 
-        if 'bitbucket' in self.project.repo:
-            slug_url = self.identifier
-            url = f'/src/{slug_url}'
+            if 'gitlab' in self.project.repo:
+                slug_url = self.identifier
+                url = f'/merge_requests/{slug_url}/'
+
+            if 'bitbucket' in self.project.repo:
+                slug_url = self.identifier
+                url = f'/pull-requests/{slug_url}'
+        else:
+            if ('github' in self.project.repo) or ('gitlab' in self.project.repo):
+                url = f'/tree/{slug_url}/'
+
+            if 'bitbucket' in self.project.repo:
+                slug_url = self.identifier
+                url = f'/src/{slug_url}'
 
         # TODO: improve this replacing
         return self.project.repo.replace('git://', 'https://').replace('.git', '') + url
@@ -220,7 +246,14 @@ def commit_name(self):
             # the actual tag name.
             return self.verbose_name
 
-        # If we came that far it's not a special version nor a branch or tag.
+        if self.type == EXTERNAL:
+            # If this version is a EXTERNAL version, the identifier will
+            # contain the actual commit hash. which we can use to
+            # generate url for a given file name
+            return self.identifier
+
+        # If we came that far it's not a special version
+        # nor a branch, tag or EXTERNAL version.
         # Therefore just return the identifier to make a safe guess.
         log.debug(
             'TODO: Raise an exception here. Testing what cases it happens',

readthedocs/builds/views.py

@@ -57,7 +57,7 @@ def post(self, request, project_slug):
 
         version_slug = request.POST.get('version_slug')
         version = get_object_or_404(
-            Version,
+            Version.internal.all(),
             project=project,
             slug=version_slug,
         )
@@ -93,7 +93,7 @@ def get_context_data(self, **kwargs):
 
         context['project'] = self.project
         context['active_builds'] = active_builds
-        context['versions'] = Version.objects.public(
+        context['versions'] = Version.internal.public(
             user=self.request.user,
             project=self.project,
         )

readthedocs/core/management/commands/update_repos.py

@@ -10,6 +10,7 @@
 
 from django.core.management.base import BaseCommand
 
+from readthedocs.builds.constants import EXTERNAL, INTERNAL
 from readthedocs.builds.models import Build, Version
 from readthedocs.core.utils import trigger_build
 from readthedocs.projects import tasks
@@ -76,6 +77,48 @@ def handle(self, *args, **options):
                             version.pk,
                             build_pk=build.pk,
                         )
+                elif version == INTERNAL:
+                    log.info('Updating all internal versions for %s', slug)
+                    for version in Version.internal.filter(
+                            project__slug=slug,
+                            active=True,
+                            uploaded=False,
+                    ):
+
+                        build = Build.objects.create(
+                            project=version.project,
+                            version=version,
+                            type='html',
+                            state='triggered',
+                        )
+
+                        # pylint: disable=no-value-for-parameter
+                        tasks.update_docs_task(
+                            version.project_id,
+                            build_pk=build.pk,
+                            version_pk=version.pk,
+                        )
+                elif version == EXTERNAL:
+                    log.info('Updating all external versions for %s', slug)
+                    for version in Version.external.filter(
+                            project__slug=slug,
+                            active=True,
+                            uploaded=False,
+                    ):
+
+                        build = Build.objects.create(
+                            project=version.project,
+                            version=version,
+                            type='html',
+                            state='triggered',
+                        )
+
+                        # pylint: disable=no-value-for-parameter
+                        tasks.update_docs_task(
+                            version.project_id,
+                            build_pk=build.pk,
+                            version_pk=version.pk,
+                        )
                 else:
                     p = Project.all_objects.get(slug=slug)
                     log.info('Building %s', p)

readthedocs/core/management/commands/update_versions.py

@@ -13,7 +13,7 @@ class Command(BaseCommand):
     help = __doc__
 
     def handle(self, *args, **options):
-        for version in Version.objects.filter(active=True, built=False):
+        for version in Version.internal.filter(active=True, built=False):
             # pylint: disable=no-value-for-parameter
             update_docs_task(
                 version.pk,

readthedocs/core/views/__init__.py

@@ -73,7 +73,7 @@ def random_page(request, project_slug=None):  # pylint: disable=unused-argument
 
 def wipe_version(request, project_slug, version_slug):
     version = get_object_or_404(
-        Version,
+        Version.internal.all(),
         project__slug=project_slug,
         slug=version_slug,
     )

readthedocs/core/views/serve.py

@@ -404,7 +404,7 @@ def changefreqs_generator():
         raise Http404
 
     sorted_versions = sort_version_aware(
-        Version.objects.public(
+        Version.internal.public(
             project=project,
             only_active=True,
         ),
@@ -444,7 +444,7 @@ def changefreqs_generator():
         if project.translations.exists():
             for translation in project.translations.all():
                 translation_versions = (
-                    Version.objects.public(project=translation)
+                    Version.internal.public(project=translation)
                     .values_list('slug', flat=True)
                 )
                 if version.slug in translation_versions:

readthedocs/projects/admin.py

@@ -239,7 +239,7 @@ def reindex_active_versions(self, request, queryset):
         """Reindex all active versions of the selected projects to ES."""
         qs_iterator = queryset.iterator()
         for project in qs_iterator:
-            version_qs = Version.objects.filter(project=project)
+            version_qs = Version.internal.filter(project=project)
             active_versions = version_qs.filter(active=True)
 
             if not active_versions.exists():
@@ -271,7 +271,7 @@ def wipe_all_versions(self, request, queryset):
         """Wipe indexes of all versions of selected projects."""
         qs_iterator = queryset.iterator()
         for project in qs_iterator:
-            version_qs = Version.objects.filter(project=project)
+            version_qs = Version.internal.filter(project=project)
             if not version_qs.exists():
                 self.message_user(
                     request,

readthedocs/projects/forms.py

@@ -14,6 +14,7 @@
 from guardian.shortcuts import assign
 from textclassifier.validators import ClassifierValidator
 
+from readthedocs.builds.constants import INTERNAL
 from readthedocs.core.utils import slugify, trigger_build
 from readthedocs.core.utils.extend import SettingsOverrideObject
 from readthedocs.integrations.models import Integration
@@ -240,7 +241,7 @@ def __init__(self, *args, **kwargs):
         self.helper.add_input(Submit('save', _('Save')))
 
         default_choice = (None, '-' * 9)
-        versions_choices = self.instance.versions.filter(
+        versions_choices = self.instance.versions(manager=INTERNAL).filter(
             machine=False).values_list('verbose_name', flat=True)
 
         self.fields['default_branch'].widget = forms.Select(