Dev environment: Add settings necessary for https and reverse proxy

dockerfiles/settings/proxito.py

@@ -1,3 +1,5 @@
+import os
+
 from readthedocs.settings.proxito.base import CommunityProxitoSettingsMixin
 
 from .docker_compose import DockerBaseSettings
@@ -19,5 +21,8 @@ def DEBUG_TOOLBAR_CONFIG(self):
             'SHOW_TOOLBAR_CALLBACK': lambda request: False,
         }
 
+    if os.environ.get("RTD_FORCE_HTTPS"):
+        PROXITO_DEV_DISABLE_SUSPICIOUS_HOST_CHECK = True
+
 
 ProxitoDevSettings.load_settings(__name__)

docs/dev/install.rst

@@ -114,7 +114,9 @@ save some work while typing docker compose commands. This section explains these
     * ``--init`` is used the first time this command is ran to run initial migrations, create an admin user, etc
     * ``--no-reload`` makes all celery processes and django runserver
       to use no reload and do not watch for files changes
-    * ``--ngrok`` is useful when it's required to access the local instance from outside (e.g. GitHub webhook)
+    * ``--no-django-debug`` runs all containers with ``DEBUG=False``
+    * ``--http-domain`` configures an external domain for the environment (useful for Ngrok or other https proxy)
+    * ``--https`` if using an HTTPS proxy, you may need to force the ``https://`` protocol for settings that otherwise automatically detect it as ``http://``
     * ``--ext-theme`` to use the new dashboard templates
     * ``--webpack`` to start the Webpack dev server for the new dashboard templates
 

readthedocs/core/unresolver.py

@@ -471,11 +471,12 @@ def unresolve_domain(self, domain):
                 log.info("Invalid format of external versions domain.", domain=domain)
                 raise InvalidExternalDomainError(domain=domain)
 
-        if public_domain in domain or external_domain in domain:
-            # NOTE: This can catch some possibly valid domains (docs.readthedocs.io.com)
-            # for example, but these might be phishing, so let's block them for now.
-            log.warning("Weird variation of our domain.", domain=domain)
-            raise SuspiciousHostnameError(domain=domain)
+        if not getattr(settings, "PROXITO_DEV_DISABLE_SUSPICIOUS_HOST_CHECK", False):
+            if public_domain in domain or external_domain in domain:
+                # NOTE: This can catch some possibly valid domains (docs.readthedocs.io.com)
+                # for example, but these might be phishing, so let's block them for now.
+                log.warning("Weird variation of our domain.", domain=domain)
+                raise SuspiciousHostnameError(domain=domain)
 
         # Custom domain.
         domain_object = (

readthedocs/settings/base.py

@@ -1051,6 +1051,8 @@ def DOCKER_LIMITS(self):
     RTD_SPAM_THRESHOLD_DELETE_PROJECT = 1000
     RTD_SPAM_MAX_SCORE = 9999
 
+    PROXITO_DEV_DISABLE_SUSPICIOUS_HOST_CHECK = False
+
     CACHEOPS_ENABLED = False
     CACHEOPS_TIMEOUT = 60 * 60  # seconds
     CACHEOPS_OPS = {'get', 'fetch'}

readthedocs/settings/docker_compose.py

@@ -8,6 +8,8 @@ class DockerBaseSettings(CommunityBaseSettings):
 
     """Settings for local development with Docker"""
 
+    DEBUG = bool(os.environ.get('RTD_DJANGO_DEBUG', True))
+
     DOCKER_ENABLE = True
     RTD_DOCKER_COMPOSE = True
     RTD_DOCKER_COMPOSE_VOLUME = 'community_build-user-builds'
@@ -30,6 +32,10 @@ class DockerBaseSettings(CommunityBaseSettings):
     # In the local docker environment, nginx should be trusted to set the host correctly
     USE_X_FORWARDED_HOST = True
 
+    # Assume running on forwarded https
+    if os.environ.get("RTD_FORCE_HTTPS"):
+        SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
+
     MULTIPLE_BUILD_SERVERS = ['build']
 
     # https://docs.docker.com/engine/reference/commandline/run/#add-entries-to-container-hosts-file---add-host
@@ -149,6 +155,13 @@ def DATABASES(self):  # noqa
         }
 
     ACCOUNT_EMAIL_VERIFICATION = "none"
+
+    # Assume running on forwarded https, needs a special option for socialauth because
+    # it detects HTTPS from the request automatically, and we may be running the app
+    # on :80 behind a :443 proxy.
+    if os.environ.get("RTD_FORCE_HTTPS"):
+        ACCOUNT_DEFAULT_HTTP_PROTOCOL = "https"
+
     SESSION_COOKIE_DOMAIN = None
     CACHES = {
         'default': {
@@ -193,6 +206,12 @@ def DATABASES(self):  # noqa
     AWS_S3_ENDPOINT_URL = 'http://storage:9000/'
     AWS_QUERYSTRING_AUTH = False
 
+    # Force the protocol for generated URLs to be https://, otherwise
+    # http:// is used because the storage server is running in a http
+    # container.
+    if os.environ.get("RTD_FORCE_HTTPS"):
+        S3_STATIC_STORAGE_OVERRIDE_PROTOCOL = "https"
+
     RTD_SAVE_BUILD_COMMANDS_TO_STORAGE = True
     RTD_BUILD_COMMANDS_STORAGE = 'readthedocs.storage.s3_storage.S3BuildCommandsStorage'
     BUILD_COLD_STORAGE_URL = 'http://storage:9000/builds'

readthedocs/storage/mixins.py

@@ -6,21 +6,29 @@
 class OverrideHostnameMixin:
 
     """
-    Override the hostname when outputting URLs.
+    Override the hostname or protocol when outputting URLs.
 
     This is useful for use with a CDN or when proxying outside of Blob Storage
 
     See: https://github.com/jschneier/django-storages/pull/658
     """
 
-    override_hostname = None    # Just the hostname without scheme (eg. 'assets.readthedocs.org')
+    override_hostname = (
+        None  # use the hostname without scheme (eg. 'assets.readthedocs.org')
+    )
+    override_protocol = (
+        None  # set to "http" or "https". None = inherit automatic setting.
+    )
 
     def url(self, *args, **kwargs):
         url = super().url(*args, **kwargs)
 
-        if self.override_hostname:
+        if self.override_hostname or self.override_protocol:
             parts = list(urlsplit(url))
-            parts[1] = self.override_hostname
+            if self.override_protocol:
+                parts[0] = self.override_protocol
+            if self.override_hostname:
+                parts[1] = self.override_hostname
             url = urlunsplit(parts)
 
         return url

readthedocs/storage/s3_storage.py

@@ -77,8 +77,9 @@ def __init__(self, *args, **kwargs):
 
 class S3StaticStorageMixin:
 
-    bucket_name = getattr(settings, 'S3_STATIC_STORAGE_BUCKET', None)
-    override_hostname = getattr(settings, 'S3_STATIC_STORAGE_OVERRIDE_HOSTNAME', None)
+    bucket_name = getattr(settings, "S3_STATIC_STORAGE_BUCKET", None)
+    override_hostname = getattr(settings, "S3_STATIC_STORAGE_OVERRIDE_HOSTNAME", None)
+    override_protocol = getattr(settings, "S3_STATIC_STORAGE_OVERRIDE_PROTOCOL", None)
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)