diff --git a/docker-compose.yml b/docker-compose.yml index 735484e8962..b0e36620a31 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,6 @@ # We do not offer support on this file and configuration at this point. # DO NOT USE DOCKER-COMPOSE SETUP FOR PRODUCTION OR CUSTOM INSTALLATION. -# GITHUB_TOKEN environment variable is required to build the images version: '3' volumes: @@ -19,8 +18,6 @@ services: build: context: . dockerfile: ${PWD}/docker/Dockerfile - args: - GITHUB_TOKEN: ${GITHUB_TOKEN} nginx: image: nginx diff --git a/docker/Dockerfile b/docker/Dockerfile index ce4daac7d06..de324c262aa 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,5 @@ FROM ubuntu:18.04 -ARG GITHUB_TOKEN - ENV DEBIAN_FRONTEND noninteractive ENV LANG C.UTF-8 @@ -25,18 +23,13 @@ RUN apt-get -y install \ libjpeg-dev \ sqlite -RUN pip3 install --upgrade pip +RUN pip3 install --no-cache-dir --upgrade pip RUN mkdir checkouts WORKDIR /usr/src/app/checkouts COPY requirements readthedocs.org/requirements -RUN pip install --no-cache-dir -r readthedocs.org/requirements/docker.txt - -# We clone the -ext here only to install the requirements at this point. -# -ext is also mounted as volume from the upper directory -RUN git clone --single-branch --depth 1 https://${GITHUB_TOKEN}@github.com/readthedocs/readthedocs-ext -RUN pip install --no-cache-dir -e readthedocs-ext +RUN pip3 install --no-cache-dir -r readthedocs.org/requirements/docker.txt WORKDIR /usr/src/app/checkouts/readthedocs.org diff --git a/readthedocs/settings/docker_compose.py b/readthedocs/settings/docker_compose.py index b1932399d29..7920754637e 100644 --- a/readthedocs/settings/docker_compose.py +++ b/readthedocs/settings/docker_compose.py @@ -89,11 +89,11 @@ def DATABASES(self): # noqa AZURE_OVERWRITE_FILES = True # Storage backend for build media artifacts (PDF, HTML, ePub, etc.) - RTD_BUILD_MEDIA_STORAGE = 'readthedocsext.storage.azure_storage.AzureBuildMediaStorage' + RTD_BUILD_MEDIA_STORAGE = 'readthedocs.storage.azure_storage.AzureBuildMediaStorage' AZURE_STATIC_STORAGE_HOSTNAME = 'community.dev.readthedocs.io' # Storage for static files (those collected with `collectstatic`) - STATICFILES_STORAGE = 'readthedocsext.storage.azure_storage.AzureStaticStorage' + STATICFILES_STORAGE = 'readthedocs.storage.azure_storage.AzureStaticStorage' STATICFILES_DIRS = [ os.path.join(CommunityDevSettings.SITE_ROOT, 'readthedocs', 'static'), diff --git a/readthedocs/storage/__init__.py b/readthedocs/storage/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/readthedocs/storage/azure_storage.py b/readthedocs/storage/azure_storage.py new file mode 100644 index 00000000000..be239974679 --- /dev/null +++ b/readthedocs/storage/azure_storage.py @@ -0,0 +1,50 @@ +# pylint: disable=abstract-method +# Disable: Method 'path' is abstract in class 'Storage' but is not overridden + +"""Django storage classes to use with Azure Blob storage service.""" + +from azure.common import AzureMissingResourceHttpError +from django.conf import settings +from django.contrib.staticfiles.storage import ManifestFilesMixin +from storages.backends.azure_storage import AzureStorage + +from readthedocs.builds.storage import BuildMediaStorageMixin + +from .mixins import OverrideHostnameMixin + + +class AzureBuildMediaStorage(BuildMediaStorageMixin, OverrideHostnameMixin, AzureStorage): + + """An Azure Storage backend for build artifacts.""" + + azure_container = getattr(settings, 'AZURE_MEDIA_STORAGE_CONTAINER', None) or 'media' + override_hostname = getattr(settings, 'AZURE_MEDIA_STORAGE_HOSTNAME', None) + + +class AzureBuildStorage(AzureStorage): + + """An Azure Storage backend for build cold storage.""" + + azure_container = getattr(settings, 'AZURE_BUILD_STORAGE_CONTAINER', None) or 'builds' + + +class AzureStaticStorage(OverrideHostnameMixin, ManifestFilesMixin, AzureStorage): + + """ + An Azure Storage backend for static media. + + * Uses Django's ManifestFilesMixin to have unique file paths (eg. core.a6f5e2c.css) + """ + + azure_container = getattr(settings, 'AZURE_STATIC_STORAGE_CONTAINER', None) or 'static' + override_hostname = getattr(settings, 'AZURE_STATIC_STORAGE_HOSTNAME', None) + + def read_manifest(self): + """Handle a workaround to make Azure work with Django on the first 'collectstatic'.""" + try: + return super().read_manifest() + except AzureMissingResourceHttpError: + # Normally Django handles this transparently as long as failing + # to read the manifest throws an IOError. However, failing to + # read a missing file from Azure storage doesn't currently + return None diff --git a/readthedocs/storage/mixins.py b/readthedocs/storage/mixins.py new file mode 100644 index 00000000000..0d9e7259b95 --- /dev/null +++ b/readthedocs/storage/mixins.py @@ -0,0 +1,26 @@ +"""Django storage mixin classes for different storage backends (Azure, S3).""" + +from urllib.parse import urlsplit, urlunsplit + + +class OverrideHostnameMixin: + + """ + Override the hostname when outputting URLs. + + This is useful for use with a CDN or when proxying outside of Blob Storage + + See: https://github.com/jschneier/django-storages/pull/658 + """ + + override_hostname = None # Just the hostname without scheme (eg. 'assets.readthedocs.org') + + def url(self, *args, **kwargs): + url = super().url(*args, **kwargs) + + if self.override_hostname: + parts = list(urlsplit(url)) + parts[1] = self.override_hostname + url = urlunsplit(parts) + + return url diff --git a/requirements/pip.txt b/requirements/pip.txt index 7bfdb2d9897..8d43e43a5b6 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -107,7 +107,9 @@ django-cors-middleware==1.4.0 user-agents==2.0 # Utilities used to upload build media to cloud storage -django-storages==1.7.2 +django-storages[azure]==1.7.2 +azure-storage-blob==1.5.0 +azure-storage-common==1.4.2 # Required only in development and linting django-debug-toolbar==2.0