diff --git a/docs/_static/images/guides/flyout-overwhelmed.png b/docs/_static/images/guides/flyout-overwhelmed.png new file mode 100644 index 00000000000..dd54f090d8a Binary files /dev/null and b/docs/_static/images/guides/flyout-overwhelmed.png differ diff --git a/docs/api/v3.rst b/docs/api/v3.rst index 4eb4c0f04de..18f03735af3 100644 --- a/docs/api/v3.rst +++ b/docs/api/v3.rst @@ -394,6 +394,7 @@ Version detail "ref": "19.0.2", "built": true, "active": true, + "hidden": false, "type": "tag", "last_build": "{BUILD}", "downloads": { @@ -460,7 +461,8 @@ Version update .. sourcecode:: json { - "active": true + "active": true, + "hidden": false } :statuscode 204: Updated successfully diff --git a/docs/guides/hiding-a-version.rst b/docs/guides/hiding-a-version.rst new file mode 100644 index 00000000000..fa6403f1037 --- /dev/null +++ b/docs/guides/hiding-a-version.rst @@ -0,0 +1,20 @@ +Hide a Version and Keep its Docs Online +======================================= + +If you manage a project with a lot of versions, +the version (flyout) menu of your docs can be easily overwhelmed and hard to navigate. + +.. figure:: /_static/images/guides/flyout-overwhelmed.png + :align: center + + Overwhelmed flyout menu + +You can deactivate the version to remove its docs, +but removing its docs isn't always an option. +To not list a version in the flyout menu while keeping its docs online, you can mark it as hidden. +Go to the :guilabel:`Versions` tab of your project, click on :guilabel:`Edit` and mark the ``Hidden`` option. + +Users that have a link to your old version will still be able to see your docs. +And new users can see all your versions (including hidden versions) in the versions tab of your project at ``https://readthedocs.org/projects//versions/`` + +Check the docs about :ref:`versions' states ` for more information. diff --git a/docs/guides/index.rst b/docs/guides/index.rst index 5fa52126cda..a0e5129817a 100644 --- a/docs/guides/index.rst +++ b/docs/guides/index.rst @@ -36,15 +36,16 @@ These guides will help you customize or tune aspects of Read the Docs. autobuild-docs-for-pull-requests build-notifications build-using-too-many-resources - technical-docs-seo-guide canonical conda environment-variables feature-flags google-analytics + hiding-a-version searching-with-readthedocs sitemaps specifying-dependencies + technical-docs-seo-guide wipe-environment diff --git a/docs/versions.rst b/docs/versions.rst index bdd22c0c0a8..afe0fbf490e 100644 --- a/docs/versions.rst +++ b/docs/versions.rst @@ -58,6 +58,53 @@ they will be redirected to the **Default version**. This defaults to **latest**, but could also point to your latest released version. +States +------ + +States define the visibility of a version across the site. +You can change the states of a version from the :guilabel:`Versions` tab of your project. + +Active +~~~~~~ + +- **Active** + + - Docs for this version are visible + - Builds can be triggered for this version + +- **Inactive** + + - Docs for this version aren't visible + - Builds can't be triggered for this version + +When you deactivate a version, its docs are removed. + +Hidden +~~~~~~ + +- **Not hidden and Active** + + - This version is listed on the version (flyout) menu on the docs site + - This version is shown in search results on the docs site + +- **Hidden and Active** + + - This version isn't listed on the version (flyout) menu on the docs site + - This version isn't show in search results from another version on the docs site + (like on search results from a superproject) + +Hiding a version doesn't make it private, +any user with a link to its docs would be able to see it. +This is useful when: + +- You no longer support a version, but you don't want to remove its docs. +- You have a work in progress version and don't want to publish its docs just yet. + +.. note:: + + Active versions that are hidden will be listed as ``Disallow: /path/to/version/`` + in the default `robots.txt file `__ created by Read the Docs. + Version warning --------------- diff --git a/readthedocs/api/v2/views/footer_views.py b/readthedocs/api/v2/views/footer_views.py index b04aff45a89..644d6c7f36a 100644 --- a/readthedocs/api/v2/views/footer_views.py +++ b/readthedocs/api/v2/views/footer_views.py @@ -127,6 +127,7 @@ def _get_active_versions_sorted(self): project = self._get_project() versions = project.ordered_active_versions( user=self.request.user, + include_hidden=False, ) return versions diff --git a/readthedocs/api/v3/serializers.py b/readthedocs/api/v3/serializers.py index 99fb848bcdf..67a85f8ba67 100644 --- a/readthedocs/api/v3/serializers.py +++ b/readthedocs/api/v3/serializers.py @@ -220,6 +220,7 @@ class Meta: 'ref', 'built', 'active', + 'hidden', 'type', 'downloads', 'urls', diff --git a/readthedocs/api/v3/tests/responses/projects-detail.json b/readthedocs/api/v3/tests/responses/projects-detail.json index bf95062418c..6698bc30500 100644 --- a/readthedocs/api/v3/tests/responses/projects-detail.json +++ b/readthedocs/api/v3/tests/responses/projects-detail.json @@ -2,6 +2,7 @@ "active_versions": [ { "active": true, + "hidden": false, "built": true, "downloads": { "epub": "https://project.readthedocs.io/_/downloads/en/v1.0/epub/", diff --git a/readthedocs/api/v3/tests/responses/projects-versions-builds-list_POST.json b/readthedocs/api/v3/tests/responses/projects-versions-builds-list_POST.json index b37580b196b..ce335249df2 100644 --- a/readthedocs/api/v3/tests/responses/projects-versions-builds-list_POST.json +++ b/readthedocs/api/v3/tests/responses/projects-versions-builds-list_POST.json @@ -72,6 +72,7 @@ "triggered": true, "version": { "active": true, + "hidden": false, "built": true, "downloads": { "epub": "https://project.readthedocs.io/_/downloads/en/v1.0/epub/", diff --git a/readthedocs/api/v3/tests/responses/projects-versions-detail.json b/readthedocs/api/v3/tests/responses/projects-versions-detail.json index a2661e705d0..6c8145f6bb0 100644 --- a/readthedocs/api/v3/tests/responses/projects-versions-detail.json +++ b/readthedocs/api/v3/tests/responses/projects-versions-detail.json @@ -1,5 +1,6 @@ { "active": true, + "hidden": false, "built": true, "downloads": { "epub": "https://project.readthedocs.io/_/downloads/en/v1.0/epub/", diff --git a/readthedocs/builds/forms.py b/readthedocs/builds/forms.py index 7edb70f12c6..872b378b884 100644 --- a/readthedocs/builds/forms.py +++ b/readthedocs/builds/forms.py @@ -3,7 +3,10 @@ import re import textwrap +from crispy_forms.helper import FormHelper +from crispy_forms.layout import HTML, Fieldset, Layout from django import forms +from django.template.loader import render_to_string from django.utils.translation import ugettext_lazy as _ from readthedocs.builds.constants import ( @@ -22,7 +25,36 @@ class VersionForm(HideProtectedLevelMixin, forms.ModelForm): class Meta: model = Version - fields = ['active', 'privacy_level'] + states_fields = ['active', 'hidden'] + privacy_fields = ['privacy_level'] + fields = ( + *states_fields, + *privacy_fields, + ) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # TODO: remove when this field is no-nullable + self.fields['hidden'].widget = forms.CheckboxInput() + self.fields['hidden'].empty_value = False + + self.helper = FormHelper() + self.helper.layout = Layout( + Fieldset( + _('States'), + HTML(render_to_string('projects/project_version_states_help_text.html')), + *self.Meta.states_fields, + ), + Fieldset( + _('Privacy'), + *self.Meta.privacy_fields, + ), + HTML(render_to_string( + 'projects/project_version_submit.html', + context={'version': self.instance}, + )), + ) def clean_active(self): active = self.cleaned_data['active'] diff --git a/readthedocs/builds/migrations/0018_add_hidden_field_to_version.py b/readthedocs/builds/migrations/0018_add_hidden_field_to_version.py new file mode 100644 index 00000000000..075b5c02606 --- /dev/null +++ b/readthedocs/builds/migrations/0018_add_hidden_field_to_version.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.11 on 2020-03-18 01:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('builds', '0017_builds_deterministic_order_index'), + ] + + operations = [ + migrations.AddField( + model_name='version', + name='hidden', + field=models.BooleanField(null=True, default=False, help_text='Hide this version from the version (flyout) menu and search results?', verbose_name='Hidden'), + ), + ] diff --git a/readthedocs/builds/migrations/0019_migrate_protected_versions_to_hidden.py b/readthedocs/builds/migrations/0019_migrate_protected_versions_to_hidden.py new file mode 100644 index 00000000000..7b48926b565 --- /dev/null +++ b/readthedocs/builds/migrations/0019_migrate_protected_versions_to_hidden.py @@ -0,0 +1,20 @@ +# Generated by Django 2.2.11 on 2020-03-18 18:27 + +from django.db import migrations + + +def forwards_func(apps, schema_editor): + """Migrate all protected versions to be hidden.""" + Version = apps.get_model('builds', 'Version') + Version.objects.filter(privacy_level='protected').update(hidden=True) + + +class Migration(migrations.Migration): + + dependencies = [ + ('builds', '0018_add_hidden_field_to_version'), + ] + + operations = [ + migrations.RunPython(forwards_func), + ] diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index c36ba7c37cf..f5949f97c87 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -136,6 +136,13 @@ class Version(models.Model): default=settings.DEFAULT_VERSION_PRIVACY_LEVEL, help_text=_('Level of privacy for this Version.'), ) + hidden = models.BooleanField( + _('Hidden'), + # To avoid downtime during deploy, remove later. + null=True, + default=False, + help_text=_('Hide this version from the version (flyout) menu and search results?') + ) machine = models.BooleanField(_('Machine Created'), default=False) # Whether the latest successful build for this version contains certain media types diff --git a/readthedocs/builds/querysets.py b/readthedocs/builds/querysets.py index fd14608af86..79f2362b6b3 100644 --- a/readthedocs/builds/querysets.py +++ b/readthedocs/builds/querysets.py @@ -24,7 +24,7 @@ def _add_user_repos(self, queryset, user): queryset = user_queryset | queryset return queryset - def public(self, user=None, project=None, only_active=True): + def public(self, user=None, project=None, only_active=True, include_hidden=True): queryset = self.filter(privacy_level=constants.PUBLIC) if user: queryset = self._add_user_repos(queryset, user) @@ -32,6 +32,8 @@ def public(self, user=None, project=None, only_active=True): queryset = queryset.filter(project=project) if only_active: queryset = queryset.filter(active=True) + if not include_hidden: + queryset = queryset.filter(hidden=False) return queryset.distinct() def protected(self, user=None, project=None, only_active=True): diff --git a/readthedocs/proxito/tests/test_full.py b/readthedocs/proxito/tests/test_full.py index 22105f7649f..696f0fb9d3a 100644 --- a/readthedocs/proxito/tests/test_full.py +++ b/readthedocs/proxito/tests/test_full.py @@ -5,6 +5,7 @@ import django_dynamic_fixture as fixture from django.conf import settings +from textwrap import dedent from django.core.cache import cache from django.http import HttpResponse from django.test.utils import override_settings @@ -18,6 +19,8 @@ SPHINX, SPHINX_HTMLDIR, SPHINX_SINGLEHTML, + PUBLIC, + PRIVATE, ) from readthedocs.projects.models import Project, Domain from readthedocs.rtd_tests.storage import BuildMediaFileSystemStorageTest @@ -285,10 +288,71 @@ def test_default_robots_txt(self, storage_exists): HTTP_HOST='project.readthedocs.io', ) self.assertEqual(response.status_code, 200) - self.assertEqual( - response.content, - b'User-agent: *\nAllow: /\nSitemap: https://project.readthedocs.io/sitemap.xml\n' + expected = dedent( + """ + User-agent: * + + Disallow: # Allow everything + + Sitemap: https://project.readthedocs.io/sitemap.xml + """ + ).lstrip() + self.assertEqual(response.content.decode(), expected) + + @mock.patch.object(BuildMediaFileSystemStorageTest, 'exists') + def test_default_robots_txt_disallow_hidden_versions(self, storage_exists): + storage_exists.return_value = False + self.project.versions.update(active=True, built=True) + fixture.get( + Version, + project=self.project, + slug='hidden', + active=True, + hidden=True, + privacy_level=PUBLIC, + ) + fixture.get( + Version, + project=self.project, + slug='hidden-2', + active=True, + hidden=True, + privacy_level=PUBLIC, ) + fixture.get( + Version, + project=self.project, + slug='hidden-and-inactive', + active=False, + hidden=True, + privacy_level=PUBLIC, + ) + fixture.get( + Version, + project=self.project, + slug='hidden-and-private', + active=False, + hidden=True, + privacy_level=PRIVATE, + ) + + response = self.client.get( + reverse('robots_txt'), + HTTP_HOST='project.readthedocs.io', + ) + self.assertEqual(response.status_code, 200) + expected = dedent( + """ + User-agent: * + + Disallow: /en/hidden-2/ # Hidden version + + Disallow: /en/hidden/ # Hidden version + + Sitemap: https://project.readthedocs.io/sitemap.xml + """ + ).lstrip() + self.assertEqual(response.content.decode(), expected) @mock.patch.object(BuildMediaFileSystemStorageTest, 'exists') def test_default_robots_txt_private_version(self, storage_exists): diff --git a/readthedocs/proxito/views/serve.py b/readthedocs/proxito/views/serve.py index 00650f50f3e..dee3cdddcd7 100644 --- a/readthedocs/proxito/views/serve.py +++ b/readthedocs/proxito/views/serve.py @@ -4,6 +4,7 @@ import logging from urllib.parse import urlparse +from readthedocs.core.resolver import resolve_path from django.conf import settings from django.core.files.storage import get_storage_class from django.http import Http404, HttpResponse, HttpResponseRedirect @@ -387,11 +388,29 @@ def get(self, request, project): scheme='https', domain=project.subdomain(), ) - return HttpResponse( - 'User-agent: *\nAllow: /\nSitemap: {}\n'.format(sitemap_url), + context = { + 'sitemap_url': sitemap_url, + 'hidden_paths': self._get_hidden_paths(project), + } + return render( + request, + 'robots.txt', + context, content_type='text/plain', ) + def _get_hidden_paths(self, project): + """Get the absolute paths of the public hidden versions of `project`.""" + hidden_versions = ( + Version.internal.public(project=project) + .filter(hidden=True) + ) + hidden_paths = [ + resolve_path(project, version_slug=version.slug) + for version in hidden_versions + ] + return hidden_paths + class ServeRobotsTXT(SettingsOverrideObject): _default_class = ServeRobotsTXTBase diff --git a/readthedocs/rtd_tests/tests/test_footer.py b/readthedocs/rtd_tests/tests/test_footer.py index 771fdeecc0a..a77d4cbb1f0 100644 --- a/readthedocs/rtd_tests/tests/test_footer.py +++ b/readthedocs/rtd_tests/tests/test_footer.py @@ -197,6 +197,33 @@ def test_index_pages_sphinx_htmldir(self): self.assertNotIn('/en/latest/foo/index/bar.html', response.data['html']) self.assertNotIn('/en/latest/foo/index/bar/index.html', response.data['html']) + def test_hidden_versions(self): + hidden_version = get( + Version, + slug='2.0', + hidden=True, + privacy_level=PUBLIC, + project=self.pip, + ) + + # The hidden version doesn't appear on the footer + self.url = ( + reverse('footer_html') + + f'?project={self.pip.slug}&version={self.latest.slug}&page=index&docroot=/' + ) + response = self.render() + self.assertIn('/en/latest/', response.data['html']) + self.assertNotIn('/en/2.0/', response.data['html']) + + # We can access the hidden version, but it doesn't appear on the footer + self.url = ( + reverse('footer_html') + + f'?project={self.pip.slug}&version={hidden_version.slug}&page=index&docroot=/' + ) + response = self.render() + self.assertIn('/en/latest/', response.data['html']) + self.assertNotIn('/en/2.0/', response.data['html']) + class TestFooterHTML(BaseTestFooterHTML, TestCase): diff --git a/readthedocs/search/api.py b/readthedocs/search/api.py index d3be5a29ae3..d5e123d2863 100644 --- a/readthedocs/search/api.py +++ b/readthedocs/search/api.py @@ -167,14 +167,15 @@ def get_all_projects(self): main_version = self._get_version() main_project = self._get_project() + all_projects = [main_project] + subprojects = Project.objects.filter( superprojects__parent_id=main_project.id, ) - all_projects = [] - for project in list(subprojects) + [main_project]: + for project in subprojects: version = ( Version.internal - .public(user=self.request.user, project=project) + .public(user=self.request.user, project=project, include_hidden=False) .filter(slug=main_version.slug) .first() ) diff --git a/readthedocs/search/tests/test_api.py b/readthedocs/search/tests/test_api.py index aea84e5c782..56cb5b57ac7 100644 --- a/readthedocs/search/tests/test_api.py +++ b/readthedocs/search/tests/test_api.py @@ -282,6 +282,47 @@ def test_get_all_projects_returns_empty_results(self, api_client, project): data = resp.data['results'] assert len(data) == 0 + def test_doc_search_hidden_versions(self, api_client, all_projects): + """Test Document search return results from subprojects also""" + project = all_projects[0] + subproject = all_projects[1] + version = project.versions.all()[0] + # Add another project as subproject of the project + project.add_subproject(subproject) + + version_subproject = subproject.versions.first() + version_subproject.hidden = True + version_subproject.save() + + # Now search with subproject content but explicitly filter by the parent project + query = get_search_query_from_project_file(project_slug=subproject.slug) + search_params = { + 'q': query, + 'project': project.slug, + 'version': version.slug + } + resp = self.get_search(api_client, search_params) + assert resp.status_code == 200 + + # The version from the subproject is hidden, so isn't show on the results. + data = resp.data['results'] + assert len(data) == 0 + + # Now search on the subproject with hidden version + query = get_search_query_from_project_file(project_slug=subproject.slug) + search_params = { + 'q': query, + 'project': subproject.slug, + 'version': version_subproject.slug + } + resp = self.get_search(api_client, search_params) + assert resp.status_code == 200 + # We can still search inside the hidden version + data = resp.data['results'] + assert len(data) == 1 + first_result = data[0] + assert first_result['project'] == subproject.slug + class TestDocumentSearch(BaseTestDocumentSearch): diff --git a/readthedocs/templates/projects/project_version_detail.html b/readthedocs/templates/projects/project_version_detail.html index c9135dfaf0b..6d0b0303e60 100644 --- a/readthedocs/templates/projects/project_version_detail.html +++ b/readthedocs/templates/projects/project_version_detail.html @@ -1,5 +1,6 @@ {% extends "projects/base_project.html" %} +{% load crispy_forms_tags %} {% load i18n %} {% load privacy_tags %} @@ -32,14 +33,6 @@

Editing {{ version.slug }}

{% endif %} {% endif %} + {% crispy form %} -
- {% csrf_token %} - {{ form.as_p }} -

- - {% trans "or" %} - {% trans "wipe "%} -

-
{% endblock %} diff --git a/readthedocs/templates/projects/project_version_states_help_text.html b/readthedocs/templates/projects/project_version_states_help_text.html new file mode 100644 index 00000000000..4df2178694f --- /dev/null +++ b/readthedocs/templates/projects/project_version_states_help_text.html @@ -0,0 +1,7 @@ +{% load i18n %} + +

+ {% blocktrans trimmed with docs_link="https://docs.readthedocs.io/page/versions.html#states" %} + Learn more about states here. + {% endblocktrans %} +

diff --git a/readthedocs/templates/projects/project_version_submit.html b/readthedocs/templates/projects/project_version_submit.html new file mode 100644 index 00000000000..068645b1a7b --- /dev/null +++ b/readthedocs/templates/projects/project_version_submit.html @@ -0,0 +1,7 @@ +{% load i18n %} + +

+ + {% trans "or" %} + {% trans "wipe "%} +

diff --git a/readthedocs/templates/robots.txt b/readthedocs/templates/robots.txt new file mode 100644 index 00000000000..60e6af578e3 --- /dev/null +++ b/readthedocs/templates/robots.txt @@ -0,0 +1,7 @@ +User-agent: * +{% for path in hidden_paths %} +Disallow: {{ path }} # Hidden version +{% empty %} +Disallow: # Allow everything +{% endfor %} +Sitemap: {{ sitemap_url }}