Skip to content

Commit ea52919

Browse files
committed
Bring Azure storage backend classes to this repository
This commit allow us to build and run all the Read the Docs stack without depending on readthedocs-ext repository allowing anyone to launch "docker-compose up"
1 parent 0359c86 commit ea52919

File tree

7 files changed

+75
-15
lines changed

7 files changed

+75
-15
lines changed

docker-compose.yml

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
# We do not offer support on this file and configuration at this point.
55
# DO NOT USE DOCKER-COMPOSE SETUP FOR PRODUCTION OR CUSTOM INSTALLATION.
66

7-
# GITHUB_TOKEN environment variable is required to build the images
87
version: '3'
98

109
volumes:
@@ -19,8 +18,6 @@ services:
1918
build:
2019
context: .
2120
dockerfile: ${PWD}/docker/Dockerfile
22-
args:
23-
GITHUB_TOKEN: ${GITHUB_TOKEN}
2421

2522
nginx:
2623
image: nginx

docker/Dockerfile

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
FROM ubuntu:18.04
22

3-
ARG GITHUB_TOKEN
4-
53
ENV DEBIAN_FRONTEND noninteractive
64
ENV LANG C.UTF-8
75

@@ -25,18 +23,13 @@ RUN apt-get -y install \
2523
libjpeg-dev \
2624
sqlite
2725

28-
RUN pip3 install --upgrade pip
26+
RUN pip3 install --no-cache-dir --upgrade pip
2927

3028
RUN mkdir checkouts
3129
WORKDIR /usr/src/app/checkouts
3230

3331
COPY requirements readthedocs.org/requirements
3432

35-
RUN pip install --no-cache-dir -r readthedocs.org/requirements/docker.txt
36-
37-
# We clone the -ext here only to install the requirements at this point.
38-
# -ext is also mounted as volume from the upper directory
39-
RUN git clone --single-branch --depth 1 https://${GITHUB_TOKEN}@github.com/readthedocs/readthedocs-ext
40-
RUN pip install --no-cache-dir -e readthedocs-ext
33+
RUN pip3 install --no-cache-dir -r readthedocs.org/requirements/docker.txt
4134

4235
WORKDIR /usr/src/app/checkouts/readthedocs.org

readthedocs/settings/docker_compose.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -89,11 +89,11 @@ def DATABASES(self): # noqa
8989
AZURE_OVERWRITE_FILES = True
9090

9191
# Storage backend for build media artifacts (PDF, HTML, ePub, etc.)
92-
RTD_BUILD_MEDIA_STORAGE = 'readthedocsext.storage.azure_storage.AzureBuildMediaStorage'
92+
RTD_BUILD_MEDIA_STORAGE = 'readthedocs.storage.azure_storage.AzureBuildMediaStorage'
9393
AZURE_STATIC_STORAGE_HOSTNAME = 'community.dev.readthedocs.io'
9494

9595
# Storage for static files (those collected with `collectstatic`)
96-
STATICFILES_STORAGE = 'readthedocsext.storage.azure_storage.AzureStaticStorage'
96+
STATICFILES_STORAGE = 'readthedocs.storage.azure_storage.AzureStaticStorage'
9797

9898
STATICFILES_DIRS = [
9999
os.path.join(CommunityDevSettings.SITE_ROOT, 'readthedocs', 'static'),

readthedocs/storage/__init__.py

Whitespace-only changes.

readthedocs/storage/azure_storage.py

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
from azure.common import AzureMissingResourceHttpError
2+
from django.conf import settings
3+
from django.contrib.staticfiles.storage import ManifestFilesMixin
4+
from storages.backends.azure_storage import AzureStorage
5+
6+
from readthedocs.builds.storage import BuildMediaStorageMixin
7+
8+
from .base import OverrideHostnameMixin
9+
10+
11+
class AzureBuildMediaStorage(BuildMediaStorageMixin, OverrideHostnameMixin, AzureStorage):
12+
13+
"""An Azure Storage backend for build artifacts."""
14+
15+
azure_container = getattr(settings, 'AZURE_MEDIA_STORAGE_CONTAINER', None) or 'media'
16+
override_hostname = getattr(settings, 'AZURE_MEDIA_STORAGE_HOSTNAME', None)
17+
18+
19+
class AzureBuildStorage(AzureStorage):
20+
21+
"""An Azure Storage backend for build cold storage."""
22+
23+
azure_container = getattr(settings, 'AZURE_BUILD_STORAGE_CONTAINER', None) or 'builds'
24+
25+
26+
class AzureStaticStorage(OverrideHostnameMixin, ManifestFilesMixin, AzureStorage):
27+
28+
"""
29+
An Azure Storage backend for static media.
30+
31+
* Uses Django's ManifestFilesMixin to have unique file paths (eg. core.a6f5e2c.css)
32+
"""
33+
34+
azure_container = getattr(settings, 'AZURE_STATIC_STORAGE_CONTAINER', None) or 'static'
35+
override_hostname = getattr(settings, 'AZURE_STATIC_STORAGE_HOSTNAME', None)
36+
37+
def read_manifest(self):
38+
"""Handle a workaround to make Azure work with Django on the first 'collectstatic'."""
39+
try:
40+
return super().read_manifest()
41+
except AzureMissingResourceHttpError:
42+
# Normally Django handles this transparently as long as failing
43+
# to read the manifest throws an IOError. However, failing to
44+
# read a missing file from Azure storage doesn't currently
45+
return None

readthedocs/storage/base.py

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
from urllib.parse import urlsplit, urlunsplit
2+
3+
class OverrideHostnameMixin:
4+
5+
"""
6+
Override the hostname when outputting URLs.
7+
8+
This is useful for use with a CDN or when proxying outside of Blob Storage
9+
10+
See: https://github.com/jschneier/django-storages/pull/658
11+
"""
12+
13+
override_hostname = None # Just the hostname without scheme (eg. 'assets.readthedocs.org')
14+
15+
def url(self, *args, **kwargs):
16+
url = super().url(*args, **kwargs)
17+
18+
if self.override_hostname:
19+
parts = list(urlsplit(url))
20+
parts[1] = self.override_hostname
21+
url = urlunsplit(parts)
22+
23+
return url

requirements/pip.txt

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,9 @@ django-cors-middleware==1.4.0
107107
user-agents==2.0
108108

109109
# Utilities used to upload build media to cloud storage
110-
django-storages==1.7.2
110+
django-storages[azure]==1.7.2
111+
azure-storage-blob==1.5.0
112+
azure-storage-common==1.4.2
111113

112114
# Required only in development and linting
113115
django-debug-toolbar==2.0

0 commit comments

Comments
 (0)