Merge pull request #6876 from readthedocs/cache-robotstxt

Add 12 hour caching to our robots.txt serving

Author: ericholscher

readthedocs/proxito/tests/test_full.py

@@ -5,6 +5,7 @@
 
 import django_dynamic_fixture as fixture
 from django.conf import settings
+from django.core.cache import cache
 from django.http import HttpResponse
 from django.test.utils import override_settings
 from django.urls import reverse
@@ -236,6 +237,11 @@ def test_project_nginx_serving_unicode_filename(self):
 class TestAdditionalDocViews(BaseDocServing):
     # Test that robots.txt and sitemap.xml work
 
+    def tearDown(self):
+        super().tearDown()
+        # Cleanup cache to avoid throttling on tests
+        cache.clear()
+
     @mock.patch('readthedocs.proxito.views.serve.get_storage_class')
     def test_default_robots_txt(self, storage_mock):
         storage_mock()().exists.return_value = False
@@ -326,7 +332,8 @@ def test_directory_indexes_get_args(self):
         self.project.versions.update(active=True, built=True)
         # Confirm we've serving from storage for the `index-exists/index.html` file
         response = self.client.get(
-            reverse('proxito_404_handler', kwargs={'proxito_path': '/en/latest/index-exists?foo=bar'}),
+            reverse('proxito_404_handler', kwargs={
+                    'proxito_path': '/en/latest/index-exists?foo=bar'}),
             HTTP_HOST='project.readthedocs.io',
         )
         self.assertEqual(

readthedocs/proxito/views/serve.py

@@ -309,6 +309,7 @@ class ServeError404(SettingsOverrideObject):
 class ServeRobotsTXTBase(ServeDocsMixin, View):
 
     @method_decorator(map_project_slug)
+    @method_decorator(cache_page(60 * 60 * 12))  # 12 hours
     def get(self, request, project):
         """
         Serve custom user's defined ``/robots.txt``.
@@ -369,7 +370,7 @@ class ServeRobotsTXT(SettingsOverrideObject):
 class ServeSitemapXMLBase(View):
 
     @method_decorator(map_project_slug)
-    @method_decorator(cache_page(60 * 60 * 24 * 3))  # 3 days
+    @method_decorator(cache_page(60 * 60 * 12))  # 12 hours
     def get(self, request, project):
         """
         Generate and serve a ``sitemap.xml`` for a particular ``project``.