Implement hidden state for versions

docs/api/v3.rst

@@ -394,6 +394,7 @@ Version detail
             "ref": "19.0.2",
             "built": true,
             "active": true,
+            "hidden": false,
             "type": "tag",
             "last_build": "{BUILD}",
             "downloads": {
@@ -460,7 +461,8 @@ Version update
     .. sourcecode:: json
 
         {
-            "active": true
+            "active": true,
+            "hidden": false
         }
 
     :statuscode 204: Updated successfully

docs/guides/hiding-a-version.rst

@@ -0,0 +1,20 @@
+Hide a Version and Keep its Docs Online
+=======================================
+
+If you manage a project with a lot of versions,
+the version (flyout) menu of your docs can be easily overwhelmed and hard to navigate.
+
+.. figure::  /_static/images/guides/flyout-overwhelmed.png
+   :align: center
+
+   Overwhelmed flyout menu
+
+You can deactivate the version to remove its docs,
+but removing its docs isn't always an option.
+To not list a version in the flyout menu while keeping its docs online, you can mark it as hidden.
+Go to the :guilabel:`Versions` tab of your project, click on :guilabel:`Edit` and mark the ``Hidden`` option.
+
+Users that have a link to your old version will still be able to see your docs.
+And new users can see all your versions (including hidden versions) in the versions tab of your project at ``https://readthedocs.org/projects/<your-project>/versions/``
+
+Check the docs about :ref:`versions' states <versions:States>` for more information.

docs/guides/index.rst

@@ -36,15 +36,16 @@ These guides will help you customize or tune aspects of Read the Docs.
     autobuild-docs-for-pull-requests
     build-notifications
     build-using-too-many-resources
-    technical-docs-seo-guide
     canonical
     conda
     environment-variables
     feature-flags
     google-analytics
+    hiding-a-version
     searching-with-readthedocs
     sitemaps
     specifying-dependencies
+    technical-docs-seo-guide
     wipe-environment
 
 

docs/versions.rst

@@ -58,6 +58,53 @@ they will be redirected to the **Default version**.
 This defaults to **latest**,
 but could also point to your latest released version.
 
+States
+------
+
+States define the visibility of a version across the site.
+You can change the states of a version from the :guilabel:`Versions` tab of your project.
+
+Active
+~~~~~~
+
+- **Active**
+
+  - Docs for this version are visible
+  - Builds can be triggered for this version
+
+- **Inactive**
+
+  - Docs for this version aren't visible
+  - Builds can't be triggered for this version
+
+When you deactivate a version, its docs are removed.
+
+Hidden
+~~~~~~
+
+- **Not hidden and Active**
+
+  - This version is listed on the version (flyout) menu on the docs site
+  - This version is shown in search results on the docs site
+
+- **Hidden and Active**
+
+  - This version isn't listed on the version (flyout) menu on the docs site
+  - This version isn't show in search results from another version on the docs site
+    (like on search results from a superproject)
+
+Hiding a version doesn't make it private,
+any user with a link to its docs would be able to see it.
+This is useful when:
+
+- You no longer support a version, but you don't want to remove its docs.
+- You have a work in progress version and don't want to publish its docs just yet. 
+
+.. note::
+
+   Active versions that are hidden will be listed as ``Disallow: /path/to/version/``
+   in the default `robots.txt file <https://www.robotstxt.org/>`__ created by Read the Docs.
+
 Version warning
 ---------------
 

readthedocs/api/v2/views/footer_views.py

@@ -127,6 +127,7 @@ def _get_active_versions_sorted(self):
         project = self._get_project()
         versions = project.ordered_active_versions(
             user=self.request.user,
+            include_hidden=False,
         )
         return versions
 

readthedocs/api/v3/serializers.py

@@ -220,6 +220,7 @@ class Meta:
             'ref',
             'built',
             'active',
+            'hidden',
             'type',
             'downloads',
             'urls',

readthedocs/api/v3/tests/responses/projects-detail.json

@@ -2,6 +2,7 @@
     "active_versions": [
         {
             "active": true,
+            "hidden": false,
             "built": true,
             "downloads": {
                 "epub": "https://project.readthedocs.io/_/downloads/en/v1.0/epub/",

readthedocs/api/v3/tests/responses/projects-versions-builds-list_POST.json

@@ -72,6 +72,7 @@
     "triggered": true,
     "version": {
         "active": true,
+        "hidden": false,
         "built": true,
         "downloads": {
             "epub": "https://project.readthedocs.io/_/downloads/en/v1.0/epub/",

readthedocs/api/v3/tests/responses/projects-versions-detail.json

@@ -1,5 +1,6 @@
 {
     "active": true,
+    "hidden": false,
     "built": true,
     "downloads": {
         "epub": "https://project.readthedocs.io/_/downloads/en/v1.0/epub/",

readthedocs/builds/forms.py

@@ -3,7 +3,10 @@
 import re
 import textwrap
 
+from crispy_forms.helper import FormHelper
+from crispy_forms.layout import HTML, Fieldset, Layout
 from django import forms
+from django.template.loader import render_to_string
 from django.utils.translation import ugettext_lazy as _
 
 from readthedocs.builds.constants import (
@@ -22,7 +25,36 @@ class VersionForm(HideProtectedLevelMixin, forms.ModelForm):
 
     class Meta:
         model = Version
-        fields = ['active', 'privacy_level']
+        states_fields = ['active', 'hidden']
+        privacy_fields = ['privacy_level']
+        fields = (
+            *states_fields,
+            *privacy_fields,
+        )
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        # TODO: remove when this field is no-nullable
+        self.fields['hidden'].widget = forms.CheckboxInput()
+        self.fields['hidden'].empty_value = False
+
+        self.helper = FormHelper()
+        self.helper.layout = Layout(
+            Fieldset(
+                _('States'),
+                HTML(render_to_string('projects/project_version_states_help_text.html')),
+                *self.Meta.states_fields,
+            ),
+            Fieldset(
+                _('Privacy'),
+                *self.Meta.privacy_fields,
+            ),
+            HTML(render_to_string(
+                'projects/project_version_submit.html',
+                context={'version': self.instance},
+            )),
+        )
 
     def clean_active(self):
         active = self.cleaned_data['active']

readthedocs/builds/migrations/0018_add_hidden_field_to_version.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.11 on 2020-03-18 01:47
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('builds', '0017_builds_deterministic_order_index'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='version',
+            name='hidden',
+            field=models.BooleanField(null=True, default=False, help_text='Hide this version from the version (flyout) menu and search results?', verbose_name='Hidden'),
+        ),
+    ]

readthedocs/builds/migrations/0019_migrate_protected_versions_to_hidden.py

@@ -0,0 +1,20 @@
+# Generated by Django 2.2.11 on 2020-03-18 18:27
+
+from django.db import migrations
+
+
+def forwards_func(apps, schema_editor):
+    """Migrate all protected versions to be hidden."""
+    Version = apps.get_model('builds', 'Version')
+    Version.objects.filter(privacy_level='protected').update(hidden=True)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('builds', '0018_add_hidden_field_to_version'),
+    ]
+
+    operations = [
+        migrations.RunPython(forwards_func),
+    ]

readthedocs/builds/models.py

@@ -136,6 +136,13 @@ class Version(models.Model):
         default=settings.DEFAULT_VERSION_PRIVACY_LEVEL,
         help_text=_('Level of privacy for this Version.'),
     )
+    hidden = models.BooleanField(
+        _('Hidden'),
+        # To avoid downtime during deploy, remove later.
+        null=True,
+        default=False,
+        help_text=_('Hide this version from the version (flyout) menu and search results?')
+    )
     machine = models.BooleanField(_('Machine Created'), default=False)
 
     # Whether the latest successful build for this version contains certain media types

readthedocs/builds/querysets.py

@@ -24,14 +24,16 @@ def _add_user_repos(self, queryset, user):
             queryset = user_queryset | queryset
         return queryset
 
-    def public(self, user=None, project=None, only_active=True):
+    def public(self, user=None, project=None, only_active=True, include_hidden=True):
         queryset = self.filter(privacy_level=constants.PUBLIC)
         if user:
             queryset = self._add_user_repos(queryset, user)
         if project:
             queryset = queryset.filter(project=project)
         if only_active:
             queryset = queryset.filter(active=True)
+        if not include_hidden:
+            queryset = queryset.filter(hidden=False)
         return queryset.distinct()
 
     def protected(self, user=None, project=None, only_active=True):

readthedocs/proxito/tests/test_full.py

@@ -5,6 +5,7 @@
 
 import django_dynamic_fixture as fixture
 from django.conf import settings
+from textwrap import dedent
 from django.core.cache import cache
 from django.http import HttpResponse
 from django.test.utils import override_settings
@@ -18,6 +19,8 @@
     SPHINX,
     SPHINX_HTMLDIR,
     SPHINX_SINGLEHTML,
+    PUBLIC,
+    PRIVATE,
 )
 from readthedocs.projects.models import Project, Domain
 from readthedocs.rtd_tests.storage import BuildMediaFileSystemStorageTest
@@ -285,10 +288,71 @@ def test_default_robots_txt(self, storage_exists):
             HTTP_HOST='project.readthedocs.io',
         )
         self.assertEqual(response.status_code, 200)
-        self.assertEqual(
-            response.content,
-            b'User-agent: *\nAllow: /\nSitemap: https://project.readthedocs.io/sitemap.xml\n'
+        expected = dedent(
+            """
+            User-agent: *
+
+            Disallow: # Allow everything
+
+            Sitemap: https://project.readthedocs.io/sitemap.xml
+            """
+        ).lstrip()
+        self.assertEqual(response.content.decode(), expected)
+
+    @mock.patch.object(BuildMediaFileSystemStorageTest, 'exists')
+    def test_default_robots_txt_disallow_hidden_versions(self, storage_exists):
+        storage_exists.return_value = False
+        self.project.versions.update(active=True, built=True)
+        fixture.get(
+            Version,
+            project=self.project,
+            slug='hidden',
+            active=True,
+            hidden=True,
+            privacy_level=PUBLIC,
+        )
+        fixture.get(
+            Version,
+            project=self.project,
+            slug='hidden-2',
+            active=True,
+            hidden=True,
+            privacy_level=PUBLIC,
         )
+        fixture.get(
+            Version,
+            project=self.project,
+            slug='hidden-and-inactive',
+            active=False,
+            hidden=True,
+            privacy_level=PUBLIC,
+        )
+        fixture.get(
+            Version,
+            project=self.project,
+            slug='hidden-and-private',
+            active=False,
+            hidden=True,
+            privacy_level=PRIVATE,
+        )
+
+        response = self.client.get(
+            reverse('robots_txt'),
+            HTTP_HOST='project.readthedocs.io',
+        )
+        self.assertEqual(response.status_code, 200)
+        expected = dedent(
+            """
+            User-agent: *
+
+            Disallow: /en/hidden-2/ # Hidden version
+
+            Disallow: /en/hidden/ # Hidden version
+
+            Sitemap: https://project.readthedocs.io/sitemap.xml
+            """
+        ).lstrip()
+        self.assertEqual(response.content.decode(), expected)
 
     @mock.patch.object(BuildMediaFileSystemStorageTest, 'exists')
     def test_default_robots_txt_private_version(self, storage_exists):