Build: use new Docker images from design document

readthedocs/doc_builder/environments.py

@@ -784,6 +784,9 @@ def __init__(self, *args, **kwargs):
         # the image set by user or,
         if self.config and self.config.build.image:
             self.container_image = self.config.build.image
+        # the new Docker image structure or,
+        if self.project.has_feature(Feature.USE_NEW_DOCKER_IMAGES_STRUCTURE):
+            self.container_image = 'readthedocs/build:ubuntu20'
         # the image overridden by the project (manually set by an admin).
         if self.project.container_image:
             self.container_image = self.project.container_image

readthedocs/doc_builder/python_environments.py

@@ -8,6 +8,7 @@
 import logging
 import os
 import shutil
+import tarfile
 
 import yaml
 from django.conf import settings
@@ -22,6 +23,7 @@
 from readthedocs.doc_builder.loader import get_builder_class
 from readthedocs.projects.constants import LOG_TEMPLATE
 from readthedocs.projects.models import Feature
+from readthedocs.storage import build_languages_storage
 
 log = logging.getLogger(__name__)
 
@@ -72,6 +74,103 @@ def delete_existing_venv_dir(self):
             )
             shutil.rmtree(venv_dir)
 
+    def install_languages(self):
+        if settings.RTD_DOCKER_COMPOSE:
+            # Create a symlink for ``root`` user to use the same ``.asdf``
+            # installation than ``docs`` user. Required for local building
+            # since everything is run as ``root`` when using Local Development
+            # instance
+            cmd = [
+                'ln',
+                '-s',
+                '/home/docs/.asdf',
+                '/root/.asdf',
+            ]
+            self.build_env.run(
+                *cmd,
+            )
+
+        # TODO: do not use a Feature flag here, but check for ``build.os`` and
+        # ``build.languages`` instead
+        if self.project.has_feature(Feature.USE_NEW_DOCKER_IMAGES_STRUCTURE):
+            # TODO: iterate over ``build.languages`` and install all languages
+            # specified for this project
+            build_os = 'ubuntu20'
+            language = 'python'
+            version = '3.9.6'
+
+            # TODO: generate the correct path for the Python version
+            # language_path = f'{build_os}/{language}/2021-08-30/{version}.tar.gz'
+            language_path = f'{build_os}-{language}-{version}.tar.gz'
+            language_version_cached = build_languages_storage.exists(language_path)
+            if language_version_cached:
+                remote_fd = build_languages_storage.open(language_path, mode='rb')
+                with tarfile.open(fileobj=remote_fd) as tar:
+                    # Extract it on the shared path between host and Docker container
+                    extract_path = os.path.join(self.project.doc_path, 'languages')
+                    tar.extractall(extract_path)
+
+                    # Move the extracted content to the ``asdf`` installation
+                    cmd = [
+                        'mv',
+                        f'{extract_path}/{version}',
+                        f'/home/docs/.asdf/installs/{language}/{version}',
+                    ]
+                    self.build_env.run(
+                        *cmd,
+                    )
+            else:
+                # If the language version selected is not available from the
+                # cache we compile it at build time
+                cmd = [
+                    # TODO: make this environment variable to work
+                    # 'PYTHON_CONFIGURE_OPTS="--enable-shared"',
+                    'asdf',
+                    'install',
+                    language,
+                    version,
+                ]
+                self.build_env.run(
+                    *cmd,
+                )
+
+            # Make the language version chosen by the user the default one
+            cmd = [
+                'asdf',
+                'global',
+                language,
+                version,
+            ]
+            self.build_env.run(
+                *cmd,
+            )
+
+            # Recreate shims for this language to make the new version
+            # installed available
+            cmd = [
+                'asdf',
+                'reshim',
+                language,
+            ]
+            self.build_env.run(
+                *cmd,
+            )
+
+            if language == 'python' and not language_version_cached:
+                # Install our own requirements if the version is compiled
+                cmd = [
+                    'python',
+                    '-m'
+                    'pip',
+                    'install',
+                    '-U',
+                    'virtualenv',
+                    'setuptools',
+                ]
+                self.build_env.run(
+                    *cmd,
+                )
+
     def install_requirements(self):
         """Install all requirements from the config object."""
         for install in self.config.python.install:
@@ -317,8 +416,15 @@ def setup_base(self):
         cli_args.append(
             self.venv_path(),
         )
+
+        # TODO: make ``self.config.python_interpreter`` return the correct value
+        if self.project.has_feature(Feature.USE_NEW_DOCKER_IMAGES_STRUCTURE):
+            python_interpreter = 'python'
+        else:
+            python_interpreter = self.config.python_interpreter
+
         self.build_env.run(
-            self.config.python_interpreter,
+            python_interpreter,
             *cli_args,
             # Don't use virtualenv bin that doesn't exist yet
             bin_path=None,

readthedocs/projects/models.py

@@ -1669,6 +1669,7 @@ def add_features(sender, **kwargs):
     USE_SPHINX_BUILDERS = 'use_sphinx_builders'
     DEDUPLICATE_BUILDS = 'deduplicate_builds'
     DONT_CREATE_INDEX = 'dont_create_index'
+    USE_NEW_DOCKER_IMAGES_STRUCTURE = 'use_new_docker_images_structure'
 
     FEATURES = (
         (ALLOW_DEPRECATED_WEBHOOKS, _('Allow deprecated webhook views')),
@@ -1690,6 +1691,10 @@ def add_features(sender, **kwargs):
             USE_TESTING_BUILD_IMAGE,
             _('Use Docker image labelled as `testing` to build the docs'),
         ),
+        (
+            USE_NEW_DOCKER_IMAGES_STRUCTURE,
+            _('Use new Docker images that install languages at build time'),
+        ),
         (
             API_LARGE_DATA,
             _('Try alternative method of posting large data'),

readthedocs/projects/tasks.py

@@ -1144,6 +1144,7 @@ def update_app_instances(
 
     def setup_build(self):
         self.install_system_dependencies()
+        self.python_env.install_languages()
         self.setup_python_environment()
 
     def setup_python_environment(self):

readthedocs/settings/base.py

@@ -310,6 +310,7 @@ def USE_PROMOS(self):  # noqa
     # https://docs.readthedocs.io/page/development/settings.html#rtd-build-media-storage
     RTD_BUILD_MEDIA_STORAGE = 'readthedocs.builds.storage.BuildMediaFileSystemStorage'
     RTD_BUILD_ENVIRONMENT_STORAGE = 'readthedocs.builds.storage.BuildMediaFileSystemStorage'
+    RTD_BUILD_LANGUAGES_STORAGE = 'readthedocs.builds.storage.BuildMediaFileSystemStorage'
     RTD_BUILD_COMMANDS_STORAGE = 'readthedocs.builds.storage.BuildMediaFileSystemStorage'
 
     @property

readthedocs/settings/docker_compose.py

@@ -134,6 +134,8 @@ def show_debug_toolbar(request):
     RTD_BUILD_MEDIA_STORAGE = 'readthedocs.storage.s3_storage.S3BuildMediaStorage'
     # Storage backend for build cached environments
     RTD_BUILD_ENVIRONMENT_STORAGE = 'readthedocs.storage.s3_storage.S3BuildEnvironmentStorage'
+    # Storage backend for build languages
+    RTD_BUILD_LANGUAGES_STORAGE = 'readthedocs.storage.s3_storage.S3BuildLanguagesStorage'
     # Storage for static files (those collected with `collectstatic`)
     STATICFILES_STORAGE = 'readthedocs.storage.s3_storage.S3StaticStorage'
 
@@ -142,6 +144,7 @@ def show_debug_toolbar(request):
     S3_MEDIA_STORAGE_BUCKET = 'media'
     S3_BUILD_COMMANDS_STORAGE_BUCKET = 'builds'
     S3_BUILD_ENVIRONMENT_STORAGE_BUCKET = 'envs'
+    S3_BUILD_LANGUAGES_STORAGE_BUCKET = 'languages'
     S3_STATIC_STORAGE_BUCKET = 'static'
     S3_STATIC_STORAGE_OVERRIDE_HOSTNAME = 'community.dev.readthedocs.io'
     S3_MEDIA_STORAGE_OVERRIDE_HOSTNAME = 'community.dev.readthedocs.io'

readthedocs/storage/__init__.py

@@ -26,6 +26,12 @@ def _setup(self):
         self._wrapped = get_storage_class(settings.RTD_BUILD_COMMANDS_STORAGE)()
 
 
+class ConfiguredBuildLanguagesStorage(LazyObject):
+    def _setup(self):
+        self._wrapped = get_storage_class(settings.RTD_BUILD_LANGUAGES_STORAGE)()
+
+
 build_media_storage = ConfiguredBuildMediaStorage()
 build_environment_storage = ConfiguredBuildEnvironmentStorage()
 build_commands_storage = ConfiguredBuildCommandsStorage()
+build_languages_storage = ConfiguredBuildLanguagesStorage()

readthedocs/storage/s3_storage.py

@@ -89,3 +89,17 @@ def __init__(self, *args, **kwargs):
                 'AWS S3 not configured correctly. '
                 'Ensure S3_BUILD_ENVIRONMENT_STORAGE_BUCKET is defined.',
             )
+
+
+class S3BuildLanguagesStorage(S3PrivateBucketMixin, BuildMediaStorageMixin, S3Boto3Storage):
+
+    bucket_name = getattr(settings, 'S3_BUILD_LANGUAGES_STORAGE_BUCKET', None)
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        if not self.bucket_name:
+            raise ImproperlyConfigured(
+                'AWS S3 not configured correctly. '
+                'Ensure S3_BUILD_LANGUAGES_STORAGE_BUCKET is defined.',
+            )

requirements/docker.txt

@@ -28,3 +28,6 @@ argh==0.26.2
 
 # run tests
 tox==3.24.3
+
+# AWS utilities to use against MinIO
+awscli==1.20.34

scripts/compile_version_upload_s3.sh

@@ -0,0 +1,97 @@
+#!/bin/bash
+#
+#
+# Script to compile a languge version and upload it to the cache.
+#
+# This script automates the process to build and upload a Python/Node/Rust/Go
+# version and upload it to S3 making it available for the builders. When a
+# pre-compiled version is available in the cache, builds are faster because they
+# don't have to donwload and compile.
+#
+# LOCAL DEVELOPMENT ENVIRONMENT
+# https://docs.readthedocs.io/en/latest/development/install.html
+#
+# You can run this script from you local environment to create cached version
+# and upload them to MinIO. For this, it's required that you have the MinIO
+# instance running before executing this script command:
+#
+#   inv docker.up
+#
+#
+# PRODUCTION ENVIRONMENT
+#
+# To create a pre-compiled cached version and make it available on production,
+# the script has to be ran from a builder (build-default or build-large) and
+# it's required to set the following environment variables for an IAM user with
+# permissions on ``language`` S3's bucket:
+#
+#   AWS_ACCESS_KEY_ID
+#   AWS_SECRET_ACCESS_KEY
+#   AWS_ENDPOINT_URL
+#
+#
+# USAGE
+#
+#  ./scripts/compile_version_upload.sh python 3.9.6
+#
+#
+
+set -e
+
+# Define variables
+SLEEP=350
+OS="ubuntu20"
+LANGUAGE=$1
+VERSION=$2
+
+# Spin up a container with the Ubuntu 20.04 LTS image
+CONTAINER_ID=$(docker run --user docs --rm --detach readthedocs/build:$OS sleep $SLEEP)
+echo "Running all the commands in Docker container: $CONTAINER_ID"
+
+# Install the language version requested
+if [[ $LANGUAGE == "python" ]]
+then
+    docker exec --env PYTHON_CONFIGURE_OPTS="--enable-shared" $CONTAINER_ID asdf install $LANGUAGE $VERSION
+else
+    docker exec $CONTAINER_ID asdf install $LANGUAGE $VERSION
+fi
+
+# Set the default version and reshim
+docker exec $CONTAINER_ID asdf global $LANGUAGE $VERSION
+docker exec $CONTAINER_ID asdf reshim $LANGUAGE
+
+# Install dependencies for this version
+if [[ $LANGUAGE == "python" ]] && [[ ! $VERSION =~ (^miniconda.*|^mambaforge.*) ]]
+then
+    RTD_PIP_VERSION=21.2.4
+    RTD_SETUPTOOLS_VERSION=57.4.0
+    RTD_VIRTUALENV_VERSION=20.7.2
+
+    if [[ $VERSION == "2.7.18" ]]
+    then
+        # Pin to the latest versions supported on Python 2.7
+        RTD_PIP_VERSION=20.3.4
+        RTD_SETUPTOOLS_VERSION=44.1.1
+        RTD_VIRTUALENV_VERSION=20.7.2
+    fi
+    docker exec $CONTAINER_ID $LANGUAGE -m pip install -U pip==$RTD_PIP_VERSION setuptools==$RTD_SETUPTOOLS_VERSION virtualenv==$RTD_VIRTUALENV_VERSION
+fi
+
+# Compress it as a .tar.gz without include the full path in the compressed file
+docker exec $CONTAINER_ID tar --create --gzip --directory=/home/docs/.asdf/installs/$LANGUAGE --file=$OS-$LANGUAGE-$VERSION.tar.gz $VERSION
+
+# Copy the .tar.gz from the container to the host
+docker cp $CONTAINER_ID:/home/docs/$OS-$LANGUAGE-$VERSION.tar.gz .
+
+# Kill the container
+docker container kill $CONTAINER_ID
+
+# Upload the .tar.gz to S3
+AWS_ENDPOINT_URL="${AWS_ENDPOINT_URL:-http://localhost:9000}"
+AWS_LANGUAGES_BUCKET="${AWS_LANGUAGES_BUCKET:-languages}"
+AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-admin}" \
+AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-password}" \
+aws --endpoint-url $AWS_ENDPOINT_URL s3 cp $OS-$LANGUAGE-$VERSION.tar.gz s3://$AWS_LANGUAGES_BUCKET
+
+# Delete the .tar.gz file from the host
+rm $OS-$LANGUAGE-$VERSION.tar.gz