Merge pull request #9168 from readthedocs/serve-static-files-same-domain

Author: ericholscher

dockerfiles/nginx/proxito.conf

@@ -31,10 +31,10 @@ server {
     }
 
     # Sendfile support to serve the actual files that Proxito has specified
-    location /proxito/ {
+    location ~* "^/(proxito|proxito-static)/(.+)" {
         internal;
         # Nginx will strip the `/proxito/` and pass just the `$storage/$type/$proj/$ver/$filename`
-        proxy_pass http://storage:9000/;
+        proxy_pass http://storage:9000/$2;
 
         proxy_set_header Host storage:9000;
         proxy_set_header X-Real-IP $remote_addr;

readthedocs/api/v2/mixins.py

@@ -5,14 +5,17 @@
 log = structlog.get_logger(__name__)
 
 
-class CachedResponseMixin:
+class CDNCacheTagsMixin:
 
     """
     Add cache tags for project and version to the response of this view.
 
     The view inheriting this mixin should implement the
     `self._get_project` and `self._get_version` methods.
 
+    If `self._get_version` returns `None`,
+    only the project level tags are added.
+
     You can add an extra per-project tag by overriding the `project_cache_tag` attribute.
     """
 
@@ -37,13 +40,18 @@ def _get_cache_tags(self):
         try:
             project = self._get_project()
             version = self._get_version()
-            tags = [
-                project.slug,
-                get_cache_tag(project.slug, version.slug),
-            ]
-            if self.project_cache_tag:
-                tags.append(get_cache_tag(project.slug, self.project_cache_tag))
-            return tags
-        except Exception as e:
-            log.debug('Error while retrieving project and version for this view.')
-        return []
+        except Exception:
+            log.warning(
+                "Error while retrieving project or version for this view.",
+                exc_info=True,
+            )
+            return []
+
+        tags = []
+        if project:
+            tags.append(project.slug)
+        if project and version:
+            tags.append(get_cache_tag(project.slug, version.slug))
+        if project and self.project_cache_tag:
+            tags.append(get_cache_tag(project.slug, self.project_cache_tag))
+        return tags

readthedocs/api/v2/views/footer_views.py

@@ -12,7 +12,7 @@
 from rest_framework.views import APIView
 from rest_framework_jsonp.renderers import JSONPRenderer
 
-from readthedocs.api.v2.mixins import CachedResponseMixin
+from readthedocs.api.v2.mixins import CDNCacheTagsMixin
 from readthedocs.api.v2.permissions import IsAuthorizedToViewVersion
 from readthedocs.builds.constants import LATEST, TAG
 from readthedocs.builds.models import Version
@@ -83,7 +83,7 @@ def get_version_compare_data(project, base_version=None):
     return ret_val
 
 
-class BaseFooterHTML(CachedResponseMixin, APIView):
+class BaseFooterHTML(CDNCacheTagsMixin, APIView):
 
     """
     Render and return footer markup.

readthedocs/core/mixins.py

@@ -31,7 +31,7 @@ class ProxiedAPIMixin:
     authentication_classes = []
 
 
-class CachedView:
+class CDNCacheControlMixin:
 
     """
     Allow to cache views at the CDN level when privacy levels are enabled.

readthedocs/doc_builder/backends/mkdocs.py

@@ -27,9 +27,9 @@ def get_absolute_static_url():
     """
     static_url = settings.STATIC_URL
 
-    if not static_url.startswith('http'):
+    if not static_url.startswith("http"):
         domain = settings.PRODUCTION_DOMAIN
-        static_url = 'http://{}{}'.format(domain, static_url)
+        static_url = "http://{}{}".format(domain, static_url)
 
     return static_url
 
@@ -147,8 +147,14 @@ def append_conf(self):
         self.create_index(extension='md')
         user_config['docs_dir'] = docs_dir
 
-        # Set mkdocs config values
-        static_url = get_absolute_static_url()
+        # MkDocs <=0.17.x doesn't support absolute paths,
+        # it needs one with a full domain.
+        if self.project.has_feature(Feature.DEFAULT_TO_MKDOCS_0_17_3):
+            static_url = get_absolute_static_url()
+        else:
+            static_url = self.project.proxied_static_path
+
+        # Set mkdocs config values.
         extra_assets = {
             'extra_javascript': [
                 'readthedocs-data.js',

readthedocs/doc_builder/backends/sphinx.py

@@ -5,13 +5,13 @@
 """
 import codecs
 import itertools
-import structlog
 import os
 import shutil
 import zipfile
 from glob import glob
 from pathlib import Path
 
+import structlog
 from django.conf import settings
 from django.template import loader as template_loader
 from django.template.loader import render_to_string
@@ -169,21 +169,21 @@ def get_config_params(self):
         )
 
         data = {
-            'html_theme': 'sphinx_rtd_theme',
-            'html_theme_import': 'sphinx_rtd_theme',
-            'current_version': self.version.verbose_name,
-            'project': self.project,
-            'version': self.version,
-            'settings': settings,
-            'conf_py_path': conf_py_path,
-            'api_host': settings.PUBLIC_API_URL,
-            'commit': commit,
-            'versions': versions,
-            'downloads': downloads,
-            'subproject_urls': subproject_urls,
-            'build_url': build_url,
-            'vcs_url': vcs_url,
-
+            "html_theme": "sphinx_rtd_theme",
+            "html_theme_import": "sphinx_rtd_theme",
+            "current_version": self.version.verbose_name,
+            "project": self.project,
+            "version": self.version,
+            "settings": settings,
+            "conf_py_path": conf_py_path,
+            "api_host": settings.PUBLIC_API_URL,
+            "commit": commit,
+            "versions": versions,
+            "downloads": downloads,
+            "subproject_urls": subproject_urls,
+            "build_url": build_url,
+            "vcs_url": vcs_url,
+            "proxied_static_path": self.project.proxied_static_path,
             # GitHub
             'github_user': github_user,
             'github_repo': github_repo,

readthedocs/doc_builder/templates/doc_builder/conf.py.tmpl

@@ -81,6 +81,7 @@ context = {
     'MEDIA_URL': "{{ settings.MEDIA_URL }}",
     'STATIC_URL': "{{ settings.STATIC_URL }}",
     'PRODUCTION_DOMAIN': "{{ settings.PRODUCTION_DOMAIN }}",
+    'proxied_static_path': "{{ proxied_static_path }}",
     'versions': [{% for version in versions %}
     ("{{ version.slug }}", "/{{ version.project.language }}/{{ version.slug}}/"),{% endfor %}
     ],

readthedocs/embed/v3/views.py

@@ -2,12 +2,9 @@
 
 import re
 from urllib.parse import urlparse
-import requests
 
+import requests
 import structlog
-
-from selectolax.parser import HTMLParser
-
 from django.conf import settings
 from django.core.cache import cache
 from django.shortcuts import get_object_or_404
@@ -17,8 +14,9 @@
 from rest_framework.renderers import BrowsableAPIRenderer, JSONRenderer
 from rest_framework.response import Response
 from rest_framework.views import APIView
+from selectolax.parser import HTMLParser
 
-from readthedocs.api.v2.mixins import CachedResponseMixin
+from readthedocs.api.v2.mixins import CDNCacheTagsMixin
 from readthedocs.core.unresolver import unresolve
 from readthedocs.core.utils.extend import SettingsOverrideObject
 from readthedocs.embed.utils import clean_links
@@ -28,7 +26,7 @@
 log = structlog.get_logger(__name__)
 
 
-class EmbedAPIBase(CachedResponseMixin, APIView):
+class EmbedAPIBase(CDNCacheTagsMixin, APIView):
 
     # pylint: disable=line-too-long
     # pylint: disable=no-self-use

readthedocs/embed/views.py

@@ -4,6 +4,7 @@
 import json
 import re
 
+import structlog
 from django.shortcuts import get_object_or_404
 from django.template.defaultfilters import slugify
 from django.utils.functional import cached_property
@@ -14,15 +15,13 @@
 from rest_framework.response import Response
 from rest_framework.views import APIView
 
-import structlog
-
-from readthedocs.api.v2.mixins import CachedResponseMixin
+from readthedocs.api.v2.mixins import CDNCacheTagsMixin
 from readthedocs.api.v2.permissions import IsAuthorizedToViewVersion
 from readthedocs.builds.constants import EXTERNAL
 from readthedocs.core.resolver import resolve
 from readthedocs.core.unresolver import unresolve
 from readthedocs.core.utils.extend import SettingsOverrideObject
-from readthedocs.embed.utils import recurse_while_none, clean_links
+from readthedocs.embed.utils import clean_links, recurse_while_none
 from readthedocs.projects.models import Project
 from readthedocs.storage import build_media_storage
 
@@ -36,7 +35,7 @@ def escape_selector(selector):
     return ret
 
 
-class EmbedAPIBase(CachedResponseMixin, APIView):
+class EmbedAPIBase(CDNCacheTagsMixin, APIView):
 
     # pylint: disable=line-too-long
 

readthedocs/projects/models.py

@@ -624,6 +624,11 @@ def proxied_api_url(self):
         """
         return self.proxied_api_host.strip('/') + '/'
 
+    @property
+    def proxied_static_path(self):
+        """Path for static files hosted on the user's doc domain."""
+        return f"{self.proxied_api_host}/static/"
+
     @property
     def regex_urlconf(self):
         """

readthedocs/proxito/README.rst

@@ -42,6 +42,7 @@ What can/can't be cached?
   This view is already cached at the application level.
 - ServeSitemapXML: can be cached. It displays only public versions, for everyone.
   This view is already cached at the application level.
+- ServeStaticFiles: can be cached, all files are the same for all projects and users.
 - Embed API: can be cached for public versions.
 - Search:
   This view checks for user permissions, can't be cached.

readthedocs/proxito/tests/test_full.py

@@ -1138,6 +1138,26 @@ def test_sitemap_all_private_versions(self):
         )
         self.assertEqual(response.status_code, 404)
 
+    @override_settings(
+        STATICFILES_STORAGE="readthedocs.rtd_tests.storage.BuildMediaFileSystemStorageTest"
+    )
+    def test_serve_static_files(self):
+        resp = self.client.get(
+            reverse(
+                "proxito_static_files",
+                args=["javascript/readthedocs-doc-embed.js"],
+            ),
+            HTTP_HOST="project.readthedocs.io",
+        )
+        self.assertEqual(resp.status_code, 200)
+        self.assertEqual(
+            resp.headers["x-accel-redirect"],
+            "/proxito-static/media/javascript/readthedocs-doc-embed.js",
+        )
+        self.assertEqual(
+            resp.headers["Cache-Tag"], "project,project:rtd-staticfiles,rtd-staticfiles"
+        )
+
 
 @override_settings(
     ALLOW_PRIVATE_REPOS=True,
@@ -1187,6 +1207,13 @@ def _test_cache_control_header_project(self, expected_value, host=None):
             self.assertEqual(resp.headers['CDN-Cache-Control'], 'public', url)
             self.assertEqual(resp.headers['Cache-Tag'], 'project', url)
 
+        # Proxied static files are always cached.
+        resp = self.client.get("/_/static/file.js", secure=True, HTTP_HOST=host)
+        self.assertEqual(resp.headers["CDN-Cache-Control"], "public")
+        self.assertEqual(
+            resp.headers["Cache-Tag"], "project,project:rtd-staticfiles,rtd-staticfiles"
+        )
+
         # Slash redirect is done at the middleware level.
         # So, it doesn't take into consideration the privacy level of the
         # version, and always defaults to private.
@@ -1228,6 +1255,13 @@ def _test_cache_control_header_subproject(self, expected_value, host=None):
             self.assertEqual(resp.headers['CDN-Cache-Control'], 'public', url)
             self.assertEqual(resp.headers['Cache-Tag'], 'subproject', url)
 
+        # Proxied static files are always cached.
+        resp = self.client.get("/_/static/file.js", secure=True, HTTP_HOST=host)
+        self.assertEqual(resp.headers["CDN-Cache-Control"], "public")
+        self.assertEqual(
+            resp.headers["Cache-Tag"], "project,project:rtd-staticfiles,rtd-staticfiles"
+        )
+
         # Slash redirect is done at the middleware level.
         # So, it doesn't take into consideration the privacy level of the
         # version, and always defaults to private.

readthedocs/proxito/urls.py

@@ -46,8 +46,9 @@
     ServePageRedirect,
     ServeRobotsTXT,
     ServeSitemapXML,
+    ServeStaticFiles,
 )
-from readthedocs.proxito.views.utils import proxito_404_page_handler, fast_404
+from readthedocs.proxito.views.utils import fast_404, proxito_404_page_handler
 
 DOC_PATH_PREFIX = getattr(settings, 'DOC_PATH_PREFIX', '')
 
@@ -106,6 +107,17 @@
         ),
         include('readthedocs.api.v3.proxied_urls'),
     ),
+    # Serve static files
+    # /_/static/file.js
+    re_path(
+        r"^{DOC_PATH_PREFIX}static/"
+        r"(?P<filename>{filename_slug})$".format(
+            DOC_PATH_PREFIX=DOC_PATH_PREFIX,
+            **pattern_opts,
+        ),
+        ServeStaticFiles.as_view(),
+        name="proxito_static_files",
+    ),
 ]
 
 core_urls = [