Merge pull request #6963 from readthedocs/project-urlconf

Add ability for users to set their own URLConf

Author: ericholscher

readthedocs/api/v2/serializers.py

@@ -27,6 +27,7 @@ class Meta:
             'documentation_type',
             'users',
             'canonical_url',
+            'urlconf',
         )
 
 

readthedocs/core/resolver.py

@@ -62,6 +62,7 @@ def base_resolve_path(
             subproject_slug=None,
             subdomain=None,
             cname=None,
+            urlconf=None,
     ):
         """Resolve a with nothing smart, just filling in the blanks."""
         # Only support `/docs/project' URLs outside our normal environment. Normally
@@ -79,6 +80,31 @@ def base_resolve_path(
         else:
             url += '{language}/{version_slug}/{filename}'
 
+        # Allow users to override their own URLConf
+        # This logic could be cleaned up with a standard set of variable replacements
+        if urlconf:
+            url = urlconf
+            url = url.replace(
+                '$version',
+                '{version_slug}',
+            )
+            url = url.replace(
+                '$language',
+                '{language}',
+            )
+            url = url.replace(
+                '$filename',
+                '{filename}',
+            )
+            url = url.replace(
+                '$subproject',
+                '{subproject_slug}',
+            )
+            if '$' in url:
+                log.warning(
+                    'Unconverted variable in a resolver URLConf: url=%s', url
+                )
+
         return url.format(
             project_slug=project_slug,
             filename=filename,
@@ -97,6 +123,7 @@ def resolve_path(
             single_version=None,
             subdomain=None,
             cname=None,
+            urlconf=None,
     ):
         """Resolve a URL with a subset of fields defined."""
         version_slug = version_slug or project.get_default_version()
@@ -122,6 +149,7 @@ def resolve_path(
             subproject_slug=subproject_slug,
             cname=cname,
             subdomain=subdomain,
+            urlconf=urlconf or project.urlconf,
         )
 
     def resolve_domain(self, project):

readthedocs/core/static-src/core/js/doc-embed/rtd-data.js

@@ -59,8 +59,10 @@ function get() {
 
     $.extend(config, defaults, window.READTHEDOCS_DATA);
 
-    // Force to use new settings
-    config.proxied_api_host = '/_';
+    if (!("proxied_api_host" in config)) {
+        // Use direct proxied API host
+        config.proxied_api_host = '/_';
+    }
 
     return config;
 }

readthedocs/core/static/core/js/readthedocs-doc-embed.js

@@ -97,6 +97,7 @@ context = {
     'conf_py_path': '{{ conf_py_path }}',
     'api_host': '{{ api_host }}',
     'github_user': '{{ github_user }}',
+    'proxied_api_host': '{{ project.proxied_api_host }}',
     'github_repo': '{{ github_repo }}',
     'github_version': '{{ github_version }}',
     'display_github': {{ display_github }},

readthedocs/doc_builder/templates/doc_builder/conf.py.tmpl

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.10 on 2020-05-26 14:34
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('projects', '0050_migrate_external_builds'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='project',
+            name='urlconf',
+            field=models.CharField(default=None, help_text='Supports the following keys: $language, $version, $subproject, $filename. An example: `$language/$version/$filename`.', max_length=255, null=True, verbose_name='Documentation URL Configuration'),
+        ),
+    ]

readthedocs/projects/migrations/0051_project_urlconf_feature.py

@@ -12,17 +12,20 @@
 from django.core.files.storage import get_storage_class
 from django.db import models
 from django.db.models import Prefetch
-from django.urls import NoReverseMatch, reverse
+from django.urls import reverse, re_path
+from django.conf.urls import include
 from django.utils.functional import cached_property
 from django.utils.translation import ugettext_lazy as _
 from django_extensions.db.models import TimeStampedModel
+from django.views import defaults
 from shlex import quote
 from taggit.managers import TaggableManager
 
 from readthedocs.api.v2.client import api
 from readthedocs.builds.constants import LATEST, STABLE, INTERNAL, EXTERNAL
 from readthedocs.core.resolver import resolve, resolve_domain
 from readthedocs.core.utils import broadcast, slugify
+from readthedocs.constants import pattern_opts
 from readthedocs.doc_builder.constants import DOCKER_LIMITS
 from readthedocs.projects import constants
 from readthedocs.projects.exceptions import ProjectConfigurationError
@@ -44,6 +47,7 @@
 from readthedocs.vcs_support.backends import backend_cls
 from readthedocs.vcs_support.utils import Lock, NonBlockingLock
 
+
 from .constants import (
     MEDIA_TYPES,
     MEDIA_TYPE_PDF,
@@ -53,7 +57,6 @@
 
 
 log = logging.getLogger(__name__)
-DOC_PATH_PREFIX = getattr(settings, 'DOC_PATH_PREFIX', '')
 
 
 class ProjectRelationship(models.Model):
@@ -201,6 +204,16 @@ class Project(models.Model):
             'DirectoryHTMLBuilder">More info on sphinx builders</a>.',
         ),
     )
+    urlconf = models.CharField(
+        _('Documentation URL Configuration'),
+        max_length=255,
+        default=None,
+        null=True,
+        help_text=_(
+            'Supports the following keys: $language, $version, $subproject, $filename. '
+            'An example: `$language/$version/$filename`.'
+        ),
+    )
 
     external_builds_enabled = models.BooleanField(
         _('Build pull requests for this project'),
@@ -549,13 +562,117 @@ def get_production_media_url(self, type_, version_slug):
 
         if self.is_subproject:
             # docs.example.com/_/downloads/<alias>/<lang>/<ver>/pdf/
-            path = f'//{domain}/{DOC_PATH_PREFIX}downloads/{self.alias}/{self.language}/{version_slug}/{type_}/'  # noqa
+            path = f'//{domain}/{self.proxied_api_url}downloads/{self.alias}/{self.language}/{version_slug}/{type_}/'  # noqa
         else:
             # docs.example.com/_/downloads/<lang>/<ver>/pdf/
-            path = f'//{domain}/{DOC_PATH_PREFIX}downloads/{self.language}/{version_slug}/{type_}/'
+            path = f'//{domain}/{self.proxied_api_url}downloads/{self.language}/{version_slug}/{type_}/'  # noqa
 
         return path
 
+    @property
+    def proxied_api_host(self):
+        """
+        Used for the proxied_api_host in javascript.
+
+        This needs to start with a slash at the root of the domain,
+        and end without a slash
+        """
+        if self.urlconf:
+            # Add our proxied api host at the first place we have a $variable
+            # This supports both subpaths & normal root hosting
+            url_prefix = self.urlconf.split('$', 1)[0]
+            return '/' + url_prefix.strip('/') + '/_'
+        return '/_'
+
+    @property
+    def proxied_api_url(self):
+        """
+        Like the api_host but for use as a URL prefix.
+
+        It can't start with a /, but has to end with one.
+        """
+        return self.proxied_api_host.strip('/') + '/'
+
+    @property
+    def regex_urlconf(self):
+        """
+        Convert User's URLConf into a proper django URLConf.
+
+        This replaces the user-facing syntax with the regex syntax.
+        """
+        to_convert = self.urlconf
+
+        # We should standardize these names so we can loop over them easier
+        to_convert = to_convert.replace(
+            '$version',
+            '(?P<version_slug>{regex})'.format(regex=pattern_opts['version_slug'])
+        )
+        to_convert = to_convert.replace(
+            '$language',
+            '(?P<lang_slug>{regex})'.format(regex=pattern_opts['lang_slug'])
+        )
+        to_convert = to_convert.replace(
+            '$filename',
+            '(?P<filename>{regex})'.format(regex=pattern_opts['filename_slug'])
+        )
+        to_convert = to_convert.replace(
+            '$subproject',
+            '(?P<subproject_slug>{regex})'.format(regex=pattern_opts['project_slug'])
+        )
+
+        if '$' in to_convert:
+            log.warning(
+                'Unconverted variable in a project URLConf: project=%s to_convert=%s',
+                self, to_convert
+            )
+        return to_convert
+
+    @property
+    def proxito_urlconf(self):
+        """
+        Returns a URLConf class that is dynamically inserted via proxito.
+
+        It is used for doc serving on projects that have their own ``urlconf``.
+        """
+        from readthedocs.projects.views.public import ProjectDownloadMedia
+        from readthedocs.proxito.views.serve import ServeDocs
+        from readthedocs.proxito.views.utils import proxito_404_page_handler
+        from readthedocs.proxito.urls import core_urls
+
+        class ProxitoURLConf:
+
+            """A URLConf dynamically inserted by Proxito."""
+
+            proxied_urls = [
+                re_path(
+                    r'{proxied_api_url}api/v2/'.format(proxied_api_url=self.proxied_api_url),
+                    include('readthedocs.api.v2.proxied_urls'),
+                    name='user_proxied_api'
+                ),
+                re_path(
+                    r'{proxied_api_url}downloads/'
+                    r'(?P<lang_slug>{lang_slug})/'
+                    r'(?P<version_slug>{version_slug})/'
+                    r'(?P<type_>[-\w]+)/$'.format(
+                        proxied_api_url=self.proxied_api_url,
+                        **pattern_opts),
+                    ProjectDownloadMedia.as_view(same_domain_url=True),
+                    name='user_proxied_downloads'
+                ),
+            ]
+            docs_urls = [
+                re_path(
+                    '^{regex_urlconf}'.format(regex_urlconf=self.regex_urlconf),
+                    ServeDocs.as_view(),
+                    name='user_proxied_serve_docs'
+                ),
+            ]
+            urlpatterns = proxied_urls + core_urls + docs_urls
+            handler404 = proxito_404_page_handler
+            handler500 = defaults.server_error
+
+        return ProxitoURLConf
+
     @property
     def is_subproject(self):
         """Return whether or not this project is a subproject."""

readthedocs/projects/models.py

@@ -6,6 +6,7 @@
 Additional processing is done to get the project from the URL in the ``views.py`` as well.
 """
 import logging
+import sys
 
 from django.conf import settings
 from django.shortcuts import render
@@ -166,6 +167,28 @@ def process_request(self, request):  # noqa
         # Otherwise set the slug on the request
         request.host_project_slug = request.slug = ret
 
+        try:
+            project = Project.objects.get(slug=request.host_project_slug)
+        except Project.DoesNotExist:
+            log.exception('No host_project_slug set on project')
+            return None
+
+        # This is hacky because Django wants a module for the URLConf,
+        # instead of also accepting string
+        if project.urlconf:
+
+            # Stop Django from caching URLs
+            project_timestamp = project.modified_date.strftime("%Y%m%d.%H%M%S")
+            url_key = f'readthedocs.urls.fake.{project.slug}.{project_timestamp}'
+
+            log.info(
+                'Setting URLConf: project=%s url_key=%s urlconf=%s',
+                project, url_key, project.urlconf,
+            )
+            if url_key not in sys.modules:
+                sys.modules[url_key] = project.proxito_urlconf
+            request.urlconf = url_key
+
         return None
 
     def process_response(self, request, response):  # noqa