Bring Azure storage backend classes to this repository (#6433)

Bring Azure storage backend classes to this repository

Author: humitos

docker-compose.yml

@@ -4,7 +4,6 @@
 # We do not offer support on this file and configuration at this point.
 # DO NOT USE DOCKER-COMPOSE SETUP FOR PRODUCTION OR CUSTOM INSTALLATION.
 
-# GITHUB_TOKEN environment variable is required to build the images
 version: '3'
 
 volumes:
@@ -19,8 +18,6 @@ services:
     build:
       context: .
       dockerfile: ${PWD}/docker/Dockerfile
-      args:
-        GITHUB_TOKEN: ${GITHUB_TOKEN}
 
   nginx:
     image: nginx

docker/Dockerfile

@@ -1,7 +1,5 @@
 FROM ubuntu:18.04
 
-ARG GITHUB_TOKEN
-
 ENV DEBIAN_FRONTEND noninteractive
 ENV LANG C.UTF-8
 
@@ -25,18 +23,13 @@ RUN apt-get -y install \
         libjpeg-dev \
         sqlite
 
-RUN pip3 install --upgrade pip
+RUN pip3 install --no-cache-dir --upgrade pip
 
 RUN mkdir checkouts
 WORKDIR /usr/src/app/checkouts
 
 COPY requirements readthedocs.org/requirements
 
-RUN pip install --no-cache-dir -r readthedocs.org/requirements/docker.txt
-
-# We clone the -ext here only to install the requirements at this point.
-# -ext is also mounted as volume from the upper directory
-RUN git clone --single-branch --depth 1 https://${GITHUB_TOKEN}@github.com/readthedocs/readthedocs-ext
-RUN pip install --no-cache-dir -e readthedocs-ext
+RUN pip3 install --no-cache-dir -r readthedocs.org/requirements/docker.txt
 
 WORKDIR /usr/src/app/checkouts/readthedocs.org

readthedocs/settings/docker_compose.py

@@ -89,11 +89,11 @@ def DATABASES(self):  # noqa
     AZURE_OVERWRITE_FILES = True
 
     # Storage backend for build media artifacts (PDF, HTML, ePub, etc.)
-    RTD_BUILD_MEDIA_STORAGE = 'readthedocsext.storage.azure_storage.AzureBuildMediaStorage'
+    RTD_BUILD_MEDIA_STORAGE = 'readthedocs.storage.azure_storage.AzureBuildMediaStorage'
     AZURE_STATIC_STORAGE_HOSTNAME = 'community.dev.readthedocs.io'
 
     # Storage for static files (those collected with `collectstatic`)
-    STATICFILES_STORAGE = 'readthedocsext.storage.azure_storage.AzureStaticStorage'
+    STATICFILES_STORAGE = 'readthedocs.storage.azure_storage.AzureStaticStorage'
 
     STATICFILES_DIRS = [
         os.path.join(CommunityDevSettings.SITE_ROOT, 'readthedocs', 'static'),

readthedocs/storage/__init__.py

@@ -0,0 +1,50 @@
+# pylint: disable=abstract-method
+# Disable: Method 'path' is abstract in class 'Storage' but is not overridden
+
+"""Django storage classes to use with Azure Blob storage service."""
+
+from azure.common import AzureMissingResourceHttpError
+from django.conf import settings
+from django.contrib.staticfiles.storage import ManifestFilesMixin
+from storages.backends.azure_storage import AzureStorage
+
+from readthedocs.builds.storage import BuildMediaStorageMixin
+
+from .mixins import OverrideHostnameMixin
+
+
+class AzureBuildMediaStorage(BuildMediaStorageMixin, OverrideHostnameMixin, AzureStorage):
+
+    """An Azure Storage backend for build artifacts."""
+
+    azure_container = getattr(settings, 'AZURE_MEDIA_STORAGE_CONTAINER', None) or 'media'
+    override_hostname = getattr(settings, 'AZURE_MEDIA_STORAGE_HOSTNAME', None)
+
+
+class AzureBuildStorage(AzureStorage):
+
+    """An Azure Storage backend for build cold storage."""
+
+    azure_container = getattr(settings, 'AZURE_BUILD_STORAGE_CONTAINER', None) or 'builds'
+
+
+class AzureStaticStorage(OverrideHostnameMixin, ManifestFilesMixin, AzureStorage):
+
+    """
+    An Azure Storage backend for static media.
+
+    * Uses Django's ManifestFilesMixin to have unique file paths (eg. core.a6f5e2c.css)
+    """
+
+    azure_container = getattr(settings, 'AZURE_STATIC_STORAGE_CONTAINER', None) or 'static'
+    override_hostname = getattr(settings, 'AZURE_STATIC_STORAGE_HOSTNAME', None)
+
+    def read_manifest(self):
+        """Handle a workaround to make Azure work with Django on the first 'collectstatic'."""
+        try:
+            return super().read_manifest()
+        except AzureMissingResourceHttpError:
+            # Normally Django handles this transparently as long as failing
+            # to read the manifest throws an IOError. However, failing to
+            # read a missing file from Azure storage doesn't currently
+            return None

readthedocs/storage/azure_storage.py

@@ -0,0 +1,26 @@
+"""Django storage mixin classes for different storage backends (Azure, S3)."""
+
+from urllib.parse import urlsplit, urlunsplit
+
+
+class OverrideHostnameMixin:
+
+    """
+    Override the hostname when outputting URLs.
+
+    This is useful for use with a CDN or when proxying outside of Blob Storage
+
+    See: https://github.com/jschneier/django-storages/pull/658
+    """
+
+    override_hostname = None    # Just the hostname without scheme (eg. 'assets.readthedocs.org')
+
+    def url(self, *args, **kwargs):
+        url = super().url(*args, **kwargs)
+
+        if self.override_hostname:
+            parts = list(urlsplit(url))
+            parts[1] = self.override_hostname
+            url = urlunsplit(parts)
+
+        return url

readthedocs/storage/mixins.py

@@ -107,7 +107,9 @@ django-cors-middleware==1.4.0
 user-agents==2.0
 
 # Utilities used to upload build media to cloud storage
-django-storages==1.7.2
+django-storages[azure]==1.7.2
+azure-storage-blob==1.5.0
+azure-storage-common==1.4.2
 
 # Required only in development and linting
 django-debug-toolbar==2.0