From e9b1406cb30c6de2882b2f143ee8c71385f694b4 Mon Sep 17 00:00:00 2001 From: dojutsu-user Date: Tue, 17 Sep 2019 02:59:01 +0530 Subject: [PATCH 1/9] remove files from cloud storage --- readthedocs/core/views/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/readthedocs/core/views/__init__.py b/readthedocs/core/views/__init__.py index b40308a6152..2c523d1502e 100644 --- a/readthedocs/core/views/__init__.py +++ b/readthedocs/core/views/__init__.py @@ -10,6 +10,7 @@ from urllib.parse import urlparse from django.conf import settings +from django.core.files.storage import get_storage_class from django.http import HttpResponseRedirect, Http404, JsonResponse from django.shortcuts import render, get_object_or_404, redirect from django.views.generic import TemplateView @@ -76,6 +77,16 @@ def wipe_version(request, project_slug, version_slug): version_slug=version_slug, project_slug=project_slug, ) + + # clear cloud storage + storage = get_storage_class(settings.RTD_BUILD_MEDIA_STORAGE)() + storage_path = version.project.get_storage_path( + type_='html', + version_slug=version_slug, + include_file=False, + ) + storage.delete_directory(storage_path) + return redirect('project_version_list', project_slug) return render( request, From 9fabd5e0c2fc55f0abcf8ba38cf0317dec17dc27 Mon Sep 17 00:00:00 2001 From: dojutsu-user Date: Tue, 17 Sep 2019 03:25:39 +0530 Subject: [PATCH 2/9] add logger --- readthedocs/core/views/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/readthedocs/core/views/__init__.py b/readthedocs/core/views/__init__.py index 2c523d1502e..5f21e3eae96 100644 --- a/readthedocs/core/views/__init__.py +++ b/readthedocs/core/views/__init__.py @@ -86,6 +86,7 @@ def wipe_version(request, project_slug, version_slug): include_file=False, ) storage.delete_directory(storage_path) + log.info('Deleted cloud storage: %s' % storage_path) return redirect('project_version_list', project_slug) return render( From 393c9bfee8f20a43d59893ee90e0cf1411a6714e Mon Sep 17 00:00:00 2001 From: dojutsu-user Date: Tue, 17 Sep 2019 11:25:31 +0530 Subject: [PATCH 3/9] refactoring --- readthedocs/builds/admin.py | 4 ++-- readthedocs/core/utils/general.py | 26 ++++++++++++++++++++++++-- readthedocs/core/views/__init__.py | 16 ++-------------- 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/readthedocs/builds/admin.py b/readthedocs/builds/admin.py index 2560bfe8d41..fa4bb19e985 100644 --- a/readthedocs/builds/admin.py +++ b/readthedocs/builds/admin.py @@ -20,7 +20,7 @@ VersionAutomationRule, ) from readthedocs.core.utils import trigger_build -from readthedocs.core.utils.general import wipe_version_via_slugs +from readthedocs.core.utils.general import _wipe_version_helper from readthedocs.projects.models import HTMLFile from readthedocs.search.utils import _indexing_helper @@ -114,7 +114,7 @@ class VersionAdmin(admin.ModelAdmin): def wipe_selected_versions(self, request, queryset): """Wipes the selected versions.""" for version in queryset: - wipe_version_via_slugs( + _wipe_version_helper( version_slug=version.slug, project_slug=version.project.slug ) diff --git a/readthedocs/core/utils/general.py b/readthedocs/core/utils/general.py index 9b7c41b21a9..b363fae4073 100644 --- a/readthedocs/core/utils/general.py +++ b/readthedocs/core/utils/general.py @@ -2,6 +2,8 @@ import os +from django.conf import settings +from django.core.files.storage import get_storage_class from django.shortcuts import get_object_or_404 from readthedocs.core.utils import broadcast @@ -9,8 +11,14 @@ from readthedocs.builds.models import Version -def wipe_version_via_slugs(version_slug, project_slug): - """Wipes the given version of a given project.""" +def _wipe_version_helper(version_slug, project_slug): + """ + Wipes the given version of a project. + + It does two things: + * Clears the `checkouts`, `envs`, and `conda` direcories (if exist). + * Removes the html files from cloud storage. + """ version = get_object_or_404( Version, slug=version_slug, @@ -23,3 +31,17 @@ def wipe_version_via_slugs(version_slug, project_slug): ] for del_dir in del_dirs: broadcast(type='build', task=remove_dirs, args=[(del_dir,)]) + + _clear_html_files_from_cloud_storage(version) + + +def _clear_html_files_from_cloud_storage(version): + """Removes html files from cloud storage for a given version of a project.""" + + storage = get_storage_class(settings.RTD_BUILD_MEDIA_STORAGE)() + storage_path = version.project.get_storage_path( + type_='html', + version_slug=version.slug, + include_file=False, + ) + storage.delete_directory(storage_path) diff --git a/readthedocs/core/views/__init__.py b/readthedocs/core/views/__init__.py index 5f21e3eae96..bfc8ba9fba5 100644 --- a/readthedocs/core/views/__init__.py +++ b/readthedocs/core/views/__init__.py @@ -10,14 +10,13 @@ from urllib.parse import urlparse from django.conf import settings -from django.core.files.storage import get_storage_class from django.http import HttpResponseRedirect, Http404, JsonResponse from django.shortcuts import render, get_object_or_404, redirect from django.views.generic import TemplateView from django.views.static import serve as static_serve from readthedocs.builds.models import Version -from readthedocs.core.utils.general import wipe_version_via_slugs +from readthedocs.core.utils.general import _wipe_version_helper from readthedocs.core.resolver import resolve_path from readthedocs.core.symlink import PrivateSymlink, PublicSymlink from readthedocs.projects.constants import PRIVATE @@ -73,21 +72,10 @@ def wipe_version(request, project_slug, version_slug): raise Http404('You must own this project to wipe it.') if request.method == 'POST': - wipe_version_via_slugs( + _wipe_version_helper( version_slug=version_slug, project_slug=project_slug, ) - - # clear cloud storage - storage = get_storage_class(settings.RTD_BUILD_MEDIA_STORAGE)() - storage_path = version.project.get_storage_path( - type_='html', - version_slug=version_slug, - include_file=False, - ) - storage.delete_directory(storage_path) - log.info('Deleted cloud storage: %s' % storage_path) - return redirect('project_version_list', project_slug) return render( request, From ee05e15263a735912ee39acf12cb2db2ce731f27 Mon Sep 17 00:00:00 2001 From: dojutsu-user Date: Tue, 17 Sep 2019 11:25:40 +0530 Subject: [PATCH 4/9] update test --- readthedocs/rtd_tests/tests/test_core_utils.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_core_utils.py b/readthedocs/rtd_tests/tests/test_core_utils.py index 813acc91907..8f580dd87ee 100644 --- a/readthedocs/rtd_tests/tests/test_core_utils.py +++ b/readthedocs/rtd_tests/tests/test_core_utils.py @@ -11,7 +11,7 @@ from readthedocs.builds.constants import LATEST from readthedocs.builds.models import Version from readthedocs.core.utils import slugify, trigger_build -from readthedocs.core.utils.general import wipe_version_via_slugs +from readthedocs.core.utils.general import _wipe_version_helper from readthedocs.projects.models import Project from readthedocs.projects.tasks import remove_dirs @@ -200,8 +200,8 @@ def test_slugify(self): ) @mock.patch('readthedocs.core.utils.general.broadcast') - def test_wipe_version_via_slug(self, mock_broadcast): - wipe_version_via_slugs( + def test_wipe_version_helper(self, mock_broadcast): + _wipe_version_helper( version_slug=self.version.slug, project_slug=self.version.project.slug ) @@ -221,20 +221,20 @@ def test_wipe_version_via_slug(self, mock_broadcast): ) @mock.patch('readthedocs.core.utils.general.broadcast') - def test_wipe_version_via_slug_wrong_param(self, mock_broadcast): + def test_wipe_version_helper_wrong_param(self, mock_broadcast): self.assertFalse(Version.objects.filter(slug='wrong-slug').exists()) with self.assertRaises(Http404): - wipe_version_via_slugs( + _wipe_version_helper( version_slug='wrong-slug', project_slug=self.version.project.slug ) mock_broadcast.assert_not_called() @mock.patch('readthedocs.core.utils.general.broadcast') - def test_wipe_version_via_slugs_same_version_slug_with_diff_proj(self, mock_broadcast): + def test_wipe_version_helper_same_version_slug_with_diff_proj(self, mock_broadcast): project_2 = get(Project) version_2 = get(Version, project=project_2, slug=self.version.slug) - wipe_version_via_slugs( + _wipe_version_helper( version_slug=version_2.slug, project_slug=project_2.slug, ) From d23a05ebd337b2ed3fe6949e69de145ed9503f56 Mon Sep 17 00:00:00 2001 From: dojutsu-user Date: Tue, 17 Sep 2019 11:59:30 +0530 Subject: [PATCH 5/9] add faq --- docs/faq.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/faq.rst b/docs/faq.rst index a6dc49871fa..182a2dfe8da 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -245,6 +245,13 @@ What commit of Read the Docs is in production? We deploy readthedocs.org from the `rel` branch in our GitHub repository. You can see the latest commits that have been deployed by looking on GitHub: https://github.com/readthedocs/readthedocs.org/commits/rel +How can I remove stale results from the search? +----------------------------------------------- + +If the search results contains stale/outdated results, +you can update the search index by :doc:`wiping the build environment ` and then rebuilding your docs. + + How can I avoid search results having a deprecated version of my docs? ---------------------------------------------------------------------- From 4619ea5cc2e3262718557548e3289c6e90e84508 Mon Sep 17 00:00:00 2001 From: dojutsu-user Date: Wed, 18 Sep 2019 00:36:46 +0530 Subject: [PATCH 6/9] add improvements --- readthedocs/builds/admin.py | 4 ++-- readthedocs/core/utils/general.py | 8 ++++---- readthedocs/core/views/__init__.py | 4 ++-- .../rtd_tests/tests/test_core_utils.py | 19 ++++++++++++------- 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/readthedocs/builds/admin.py b/readthedocs/builds/admin.py index fa4bb19e985..2560bfe8d41 100644 --- a/readthedocs/builds/admin.py +++ b/readthedocs/builds/admin.py @@ -20,7 +20,7 @@ VersionAutomationRule, ) from readthedocs.core.utils import trigger_build -from readthedocs.core.utils.general import _wipe_version_helper +from readthedocs.core.utils.general import wipe_version_via_slugs from readthedocs.projects.models import HTMLFile from readthedocs.search.utils import _indexing_helper @@ -114,7 +114,7 @@ class VersionAdmin(admin.ModelAdmin): def wipe_selected_versions(self, request, queryset): """Wipes the selected versions.""" for version in queryset: - _wipe_version_helper( + wipe_version_via_slugs( version_slug=version.slug, project_slug=version.project.slug ) diff --git a/readthedocs/core/utils/general.py b/readthedocs/core/utils/general.py index b363fae4073..a13b047091a 100644 --- a/readthedocs/core/utils/general.py +++ b/readthedocs/core/utils/general.py @@ -9,9 +9,10 @@ from readthedocs.core.utils import broadcast from readthedocs.projects.tasks import remove_dirs from readthedocs.builds.models import Version +from readthedocs.projects.tasks import remove_build_storage_paths -def _wipe_version_helper(version_slug, project_slug): +def wipe_version_via_slugs(version_slug, project_slug): """ Wipes the given version of a project. @@ -36,12 +37,11 @@ def _wipe_version_helper(version_slug, project_slug): def _clear_html_files_from_cloud_storage(version): - """Removes html files from cloud storage for a given version of a project.""" + """Removes html files from media storage (cloud or local) for a given version of a project.""" - storage = get_storage_class(settings.RTD_BUILD_MEDIA_STORAGE)() storage_path = version.project.get_storage_path( type_='html', version_slug=version.slug, include_file=False, ) - storage.delete_directory(storage_path) + remove_build_storage_paths.delay([storage_path]) diff --git a/readthedocs/core/views/__init__.py b/readthedocs/core/views/__init__.py index bfc8ba9fba5..b40308a6152 100644 --- a/readthedocs/core/views/__init__.py +++ b/readthedocs/core/views/__init__.py @@ -16,7 +16,7 @@ from django.views.static import serve as static_serve from readthedocs.builds.models import Version -from readthedocs.core.utils.general import _wipe_version_helper +from readthedocs.core.utils.general import wipe_version_via_slugs from readthedocs.core.resolver import resolve_path from readthedocs.core.symlink import PrivateSymlink, PublicSymlink from readthedocs.projects.constants import PRIVATE @@ -72,7 +72,7 @@ def wipe_version(request, project_slug, version_slug): raise Http404('You must own this project to wipe it.') if request.method == 'POST': - _wipe_version_helper( + wipe_version_via_slugs( version_slug=version_slug, project_slug=project_slug, ) diff --git a/readthedocs/rtd_tests/tests/test_core_utils.py b/readthedocs/rtd_tests/tests/test_core_utils.py index 8f580dd87ee..b72c666f281 100644 --- a/readthedocs/rtd_tests/tests/test_core_utils.py +++ b/readthedocs/rtd_tests/tests/test_core_utils.py @@ -11,7 +11,7 @@ from readthedocs.builds.constants import LATEST from readthedocs.builds.models import Version from readthedocs.core.utils import slugify, trigger_build -from readthedocs.core.utils.general import _wipe_version_helper +from readthedocs.core.utils.general import wipe_version_via_slugs from readthedocs.projects.models import Project from readthedocs.projects.tasks import remove_dirs @@ -200,8 +200,12 @@ def test_slugify(self): ) @mock.patch('readthedocs.core.utils.general.broadcast') - def test_wipe_version_helper(self, mock_broadcast): - _wipe_version_helper( + @mock.patch('readthedocs.core.utils.general.remove_build_storage_paths') + @mock.patch.object(Project, 'get_storage_path') + def test_wipe_version_via_slugs(self, mock_get_storage_path, mock_remove_build_storage_paths, mock_broadcast): + mock_get_storage_path.return_value = 'test/path/' + + wipe_version_via_slugs( version_slug=self.version.slug, project_slug=self.version.project.slug ) @@ -219,22 +223,23 @@ def test_wipe_version_helper(self, mock_broadcast): ], any_order=False ) + mock_remove_build_storage_paths.delay.assert_called_once_with(['test/path/']) @mock.patch('readthedocs.core.utils.general.broadcast') - def test_wipe_version_helper_wrong_param(self, mock_broadcast): + def test_wipe_version_via_slugs_wrong_param(self, mock_broadcast): self.assertFalse(Version.objects.filter(slug='wrong-slug').exists()) with self.assertRaises(Http404): - _wipe_version_helper( + wipe_version_via_slugs( version_slug='wrong-slug', project_slug=self.version.project.slug ) mock_broadcast.assert_not_called() @mock.patch('readthedocs.core.utils.general.broadcast') - def test_wipe_version_helper_same_version_slug_with_diff_proj(self, mock_broadcast): + def test_wipe_version_via_slugs_same_version_slug_with_diff_proj(self, mock_broadcast): project_2 = get(Project) version_2 = get(Version, project=project_2, slug=self.version.slug) - _wipe_version_helper( + wipe_version_via_slugs( version_slug=version_2.slug, project_slug=project_2.slug, ) From 6e0f81f913f8209ff4242321d3b847bca37062b2 Mon Sep 17 00:00:00 2001 From: dojutsu-user Date: Wed, 18 Sep 2019 00:39:57 +0530 Subject: [PATCH 7/9] revert test name --- readthedocs/rtd_tests/tests/test_core_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/rtd_tests/tests/test_core_utils.py b/readthedocs/rtd_tests/tests/test_core_utils.py index b72c666f281..dcc53514833 100644 --- a/readthedocs/rtd_tests/tests/test_core_utils.py +++ b/readthedocs/rtd_tests/tests/test_core_utils.py @@ -226,7 +226,7 @@ def test_wipe_version_via_slugs(self, mock_get_storage_path, mock_remove_build_s mock_remove_build_storage_paths.delay.assert_called_once_with(['test/path/']) @mock.patch('readthedocs.core.utils.general.broadcast') - def test_wipe_version_via_slugs_wrong_param(self, mock_broadcast): + def test_wipe_version_via_slug_wrong_param(self, mock_broadcast): self.assertFalse(Version.objects.filter(slug='wrong-slug').exists()) with self.assertRaises(Http404): wipe_version_via_slugs( From fc8b26a1c748480b70943282f60888686de78090 Mon Sep 17 00:00:00 2001 From: dojutsu-user Date: Wed, 18 Sep 2019 11:25:42 +0530 Subject: [PATCH 8/9] typo fixes --- readthedocs/core/utils/general.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readthedocs/core/utils/general.py b/readthedocs/core/utils/general.py index a13b047091a..0735a9234f9 100644 --- a/readthedocs/core/utils/general.py +++ b/readthedocs/core/utils/general.py @@ -17,7 +17,7 @@ def wipe_version_via_slugs(version_slug, project_slug): Wipes the given version of a project. It does two things: - * Clears the `checkouts`, `envs`, and `conda` direcories (if exist). + * Clears the `checkouts`, `envs`, and `conda` directories (if exist). * Removes the html files from cloud storage. """ version = get_object_or_404( @@ -36,7 +36,7 @@ def wipe_version_via_slugs(version_slug, project_slug): _clear_html_files_from_cloud_storage(version) -def _clear_html_files_from_cloud_storage(version): +def _clear_html_files_from_media_storage(version): """Removes html files from media storage (cloud or local) for a given version of a project.""" storage_path = version.project.get_storage_path( From cbbb92e2cdc0a94586969a741451dcb33193fe22 Mon Sep 17 00:00:00 2001 From: dojutsu-user Date: Thu, 19 Sep 2019 00:46:21 +0530 Subject: [PATCH 9/9] fix func name --- readthedocs/core/utils/general.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/core/utils/general.py b/readthedocs/core/utils/general.py index 0735a9234f9..dbc4a79e735 100644 --- a/readthedocs/core/utils/general.py +++ b/readthedocs/core/utils/general.py @@ -33,7 +33,7 @@ def wipe_version_via_slugs(version_slug, project_slug): for del_dir in del_dirs: broadcast(type='build', task=remove_dirs, args=[(del_dir,)]) - _clear_html_files_from_cloud_storage(version) + _clear_html_files_from_media_storage(version) def _clear_html_files_from_media_storage(version):