Merge pull request #7812 from readthedocs/humitos/docker-env-s3-storage

Use S3 (MinIO emulator) as storage backend

Author: humitos

common

@@ -70,29 +70,3 @@ services:
     environment:
       - DJANGO_SETTINGS_MODULE=readthedocs.settings.build_docker
       - CELERY_APP_NAME=readthedocs
-
-  storage:
-    # https://github.com/Azure/Azurite
-    # Pin the version of Azurite as there are sometimes backwards incompatible changes
-    image: mcr.microsoft.com/azure-storage/azurite:3.5.0
-    volumes:
-      - storage_data:/data
-    ports:
-      - "10000:10000"
-    networks:
-      readthedocs:
-    command: ["azurite-blob", "--silent", "--blobPort", "10000", "--blobHost", "0.0.0.0", "--location", "/data", "--loose"]
-
-  azure-cli:
-    image: microsoft/azure-cli
-    volumes:
-      - ${PWD}/common/dockerfiles/scripts/create-containers.sh:/usr/local/bin/create-containers.sh
-    environment:
-      - INIT=${INIT:-}
-    links:
-      - storage
-    depends_on:
-      - storage
-    networks:
-      readthedocs:
-    command: ["/usr/local/bin/create-containers.sh"]

docker-compose.override.yml

@@ -5,7 +5,7 @@ server {
 
     # TODO: serve the favicon.ico from the project/version itself instead
     location /favicon.ico {
-        proxy_pass http://storage:10000/devstoreaccount1/static/images/favicon.ico;
+        proxy_pass http://storage:9000/static/images/favicon.ico;
         break;
     }
 
@@ -34,9 +34,9 @@ server {
     location /proxito/ {
         internal;
         # Nginx will strip the `/proxito/` and pass just the `$storage/$type/$proj/$ver/$filename`
-        proxy_pass http://storage:10000/;
+        proxy_pass http://storage:9000/;
 
-        proxy_set_header Host storage:10000;
+        proxy_set_header Host storage:9000;
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_set_header X-Forwarded-Host $host;

dockerfiles/nginx/proxito.conf

@@ -5,7 +5,12 @@ server {
     server_name community.dev.readthedocs.io;
 
     location /favicon.ico {
-        proxy_pass http://storage:10000/devstoreaccount1/static/images/favicon.ico;
+        proxy_pass http://storage:9000/static/images/favicon.ico;
+        break;
+    }
+
+    location /static/ {
+        proxy_pass http://storage:9000/static/;
         break;
     }
 

dockerfiles/nginx/web.conf

@@ -2,14 +2,16 @@
 
 
 class CeleryDevSettings(DockerBaseSettings):
-    # Since we can't properly set CORS on Azurite container
-    # (see https://github.com/Azure/Azurite/issues/55#issuecomment-503380561)
+    # TODO: review this since it may not be needed with MinIO (S3). For now,
+    # this is still required, but the CORS issue may have disappeared in MinIO.
+
+    # Since we can't properly set CORS on storage container
     # trying to fetch ``objects.inv`` from celery container fails because the
     # URL is like http://community.dev.readthedocs.io/... and it should be
-    # http://storage:10000/... This setting fixes that.
+    # http://storage:9000/... This setting fixes that.
     # Once we can use CORS, we should define this setting in the
     # ``docker_compose.py`` file instead.
-    AZURE_MEDIA_STORAGE_HOSTNAME = 'storage:10000'
+    S3_MEDIA_STORAGE_OVERRIDE_HOSTNAME = 'storage:9000'
 
 
 CeleryDevSettings.load_settings(__name__)

dockerfiles/settings/celery.py

@@ -106,6 +106,8 @@ After cloning ``readthedocs.org`` repository, you need to
 
       inv docker.up  --init  # --init is only needed the first time
 
+#. go to http://localhost:9000/ (MinIO S3 storage backend), click "..." and then "Edit Policy" and give "Read Only" access on all the buckets (``static`` and ``media``).
+
 #. go to http://community.dev.readthedocs.io to access your local instance of Read the Docs.
 
 
@@ -122,8 +124,7 @@ save some work while typing docker compose commands. This section explains these
     Starts all the containers needed to run Read the Docs completely.
 
     * ``--no-search`` can be passed to disable search
-    * ``--init`` is used the first time this command is ran to run initial migrations, create an admin user,
-      setup Azurite containers, etc
+    * ``--init`` is used the first time this command is ran to run initial migrations, create an admin user, etc
     * ``--no-reload`` makes all celery processes and django runserver
       to use no reload and do not watch for files changes
 

docs/development/standards.rst

@@ -18,10 +18,12 @@ class DockerBaseSettings(CommunityDevSettings):
     PRODUCTION_DOMAIN = 'community.dev.readthedocs.io'
     PUBLIC_DOMAIN = 'community.dev.readthedocs.io'
     PUBLIC_API_URL = f'http://{PRODUCTION_DOMAIN}'
+
     SLUMBER_API_HOST = 'http://web:8000'
+
     RTD_EXTERNAL_VERSION_DOMAIN = 'org.dev.readthedocs.build'
 
-    STATIC_URL = '/devstoreaccount1/static/'
+    STATIC_URL = '/static/'
 
     # In the local docker environment, nginx should be trusted to set the host correctly
     USE_X_FORWARDED_HOST = True
@@ -64,8 +66,21 @@ def RTD_EXT_THEME_DEV_SERVER(self):
     def LOGGING(self):
         logging = super().LOGGING
         logging['loggers'].update({
-            # Disable azurite logging
-            'azure.storage.common': {
+            # Disable S3 logging
+            'boto3': {
+                'handlers': ['null'],
+                'propagate': False,
+            },
+            'botocore': {
+                'handlers': ['null'],
+                'propagate': False,
+            },
+            's3transfer': {
+                'handlers': ['null'],
+                'propagate': False,
+            },
+            # Disable Docker API logging
+            'urllib3': {
                 'handlers': ['null'],
                 'propagate': False,
             },
@@ -115,37 +130,38 @@ def show_debug_toolbar(request):
 
     EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend"
 
-    # https://github.com/Azure/Azurite/blob/master/README.md#default-storage-account
-    AZURE_ACCOUNT_NAME = 'devstoreaccount1'
-    AZURE_ACCOUNT_KEY = 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=='
-    AZURE_CONTAINER = 'static'
-    AZURE_STATIC_STORAGE_CONTAINER = AZURE_CONTAINER
-
-    # We want to replace files for the same version built
-    AZURE_OVERWRITE_FILES = True
-
-    # Storage backend for build media artifacts (PDF, HTML, ePub, etc.)
-    RTD_BUILD_MEDIA_STORAGE = 'readthedocs.storage.azure_storage.AzureBuildMediaStorage'
-    AZURE_STATIC_STORAGE_HOSTNAME = 'assets.community.dev.readthedocs.io:10000'
-
-    RTD_SAVE_BUILD_COMMANDS_TO_STORAGE = True
-    RTD_BUILD_COMMANDS_STORAGE = 'readthedocs.storage.azure_storage.AzureBuildStorage'
-
+    RTD_BUILD_MEDIA_STORAGE = 'readthedocs.storage.s3_storage.S3BuildMediaStorage'
     # Storage backend for build cached environments
-    RTD_BUILD_ENVIRONMENT_STORAGE = 'readthedocs.storage.azure_storage.AzureBuildEnvironmentStorage'
-
+    RTD_BUILD_ENVIRONMENT_STORAGE = 'readthedocs.storage.s3_storage.S3BuildEnvironmentStorage'
     # Storage for static files (those collected with `collectstatic`)
-    STATICFILES_STORAGE = 'readthedocs.storage.azure_storage.AzureStaticStorage'
+    STATICFILES_STORAGE = 'readthedocs.storage.s3_storage.S3StaticStorage'
+
+    AWS_ACCESS_KEY_ID = 'admin'
+    AWS_SECRET_ACCESS_KEY = 'password'
+    S3_MEDIA_STORAGE_BUCKET = 'media'
+    S3_BUILD_COMMANDS_STORAGE_BUCKET = 'builds'
+    S3_BUILD_ENVIRONMENT_STORAGE_BUCKET = 'envs'
+    S3_STATIC_STORAGE_BUCKET = 'static'
+    S3_STATIC_STORAGE_OVERRIDE_HOSTNAME = 'community.dev.readthedocs.io'
+    S3_MEDIA_STORAGE_OVERRIDE_HOSTNAME = 'community.dev.readthedocs.io'
+
+    AWS_AUTO_CREATE_BUCKET = True
+    AWS_DEFAULT_ACL = 'public-read'
+    AWS_BUCKET_ACL = 'public-read'
+    AWS_S3_ENCRYPTION = False
+    AWS_S3_SECURE_URLS = False
+    AWS_S3_USE_SSL = False
+    AWS_S3_ENDPOINT_URL = 'http://storage:9000/'
+    AWS_QUERYSTRING_AUTH = False
+
+    RTD_SAVE_BUILD_COMMANDS_TO_STORAGE = True
+    RTD_BUILD_COMMANDS_STORAGE = 'readthedocs.storage.s3_storage.S3BuildCommandsStorage'
+    BUILD_COLD_STORAGE_URL = 'http://storage:9000/builds'
 
     STATICFILES_DIRS = [
         os.path.join(CommunityDevSettings.SITE_ROOT, 'readthedocs', 'static'),
         os.path.join(CommunityDevSettings.SITE_ROOT, 'media'),
     ]
-    AZURE_BUILD_COMMANDS_STORAGE_CONTAINER = 'builds'
-    BUILD_COLD_STORAGE_URL = 'http://storage:10000/builds'
-    AZURE_EMULATED_MODE = True
-    AZURE_CUSTOM_DOMAIN = 'storage:10000'
-    AZURE_SSL = False
 
     # Remove the checks on the number of fields being submitted
     # This limit is mostly hit on large forms in the Django admin

readthedocs/settings/docker_compose.py

@@ -0,0 +1,90 @@
+"""
+AWS S3 Storage backends.
+
+We override the backends provided by django-storages to add some small pieces
+that we need to make our project to work as we want. For example, using
+ManifestFilesMixin for static files and OverrideHostnameMixin to make it work
+in our Docker Development environment.
+"""
+
+# Disable abstract method because we are not overriding all the methods
+# pylint: disable=abstract-method
+from django.conf import settings
+from django.contrib.staticfiles.storage import ManifestFilesMixin
+from django.core.exceptions import ImproperlyConfigured
+from storages.backends.s3boto3 import S3Boto3Storage
+
+from readthedocs.builds.storage import BuildMediaStorageMixin
+from readthedocs.storage.mixins import OverrideHostnameMixin
+
+
+class S3BuildMediaStorage(BuildMediaStorageMixin, OverrideHostnameMixin, S3Boto3Storage):
+
+    """An AWS S3 Storage backend for build artifacts."""
+
+    bucket_name = getattr(settings, 'S3_MEDIA_STORAGE_BUCKET')
+    override_hostname = getattr(settings, 'S3_MEDIA_STORAGE_OVERRIDE_HOSTNAME', None)
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        if not self.bucket_name:
+            raise ImproperlyConfigured(
+                'AWS S3 not configured correctly. '
+                'Ensure S3_MEDIA_STORAGE_BUCKET is defined.',
+            )
+
+
+class S3BuildCommandsStorage(S3Boto3Storage):
+
+    """An AWS S3 Storage backend for build commands."""
+
+    bucket_name = getattr(settings, 'S3_BUILD_COMMANDS_STORAGE_BUCKET', None)
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        if not self.bucket_name:
+            raise ImproperlyConfigured(
+                'AWS S3 not configured correctly. '
+                'Ensure S3_BUILD_COMMANDS_STORAGE_BUCKET is defined.',
+            )
+
+
+class S3StaticStorage(OverrideHostnameMixin, ManifestFilesMixin, S3Boto3Storage):
+
+    """
+    An AWS S3 Storage backend for static media.
+
+    * Uses Django's ManifestFilesMixin to have unique file paths (eg. core.a6f5e2c.css)
+    """
+
+    bucket_name = getattr(settings, 'S3_STATIC_STORAGE_BUCKET')
+    override_hostname = getattr(settings, 'S3_STATIC_STORAGE_OVERRIDE_HOSTNAME', None)
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        if not self.bucket_name:
+            raise ImproperlyConfigured(
+                'AWS S3 not configured correctly. '
+                'Ensure S3_STATIC_STORAGE_BUCKET is defined.',
+            )
+
+        self.bucket_acl = 'public-read'
+        self.default_acl = 'public-read'
+        self.querystring_auth = False
+
+
+class S3BuildEnvironmentStorage(BuildMediaStorageMixin, S3Boto3Storage):
+
+    bucket_name = getattr(settings, 'S3_BUILD_ENVIRONMENT_STORAGE_BUCKET')
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        if not self.bucket_name:
+            raise ImproperlyConfigured(
+                'AWS S3 not configured correctly. '
+                'Ensure S3_BUILD_ENVIRONMENT_STORAGE_BUCKET is defined.',
+            )

readthedocs/storage/azure_storage.py

@@ -101,11 +101,19 @@ django-cors-middleware==1.4.0  # pyup: ignore
 # User agent parsing - used for analytics purposes
 user-agents==2.2.0
 
+
 # Utilities used to upload build media to cloud storage
 # django-storages is pinned to this particular commit because it
 # supports generating URLs with other method than GET when using
-# private buckets
-git+https://github.com/jschneier/django-storages@d0f027c98a877f75615cfc42b4d51c038fa41bf6#egg=django-storages[azure]==1.9.1
+# private buckets.
+#
+# Besides, support for the corresponding AWS_BUCKET_ACL and
+# AWS_AUTO_CREATE_BUCKET settings have been removed in 1.10. We depend on this
+# in our Docker setup. We can upgrade it but we need to add a
+# `create_buckets.sh` to be called on `--init` as we used to do for Azurite
+# https://github.com/jschneier/django-storages/pull/636
+git+https://github.com/jschneier/django-storages@d0f027c98a877f75615cfc42b4d51c038fa41bf6#egg=django-storages[boto3]==1.9.1
+
 
 # Required only in development and linting
 django-debug-toolbar==3.2