Build: use new Docker images from design document

readthedocs/doc_builder/environments.py

@@ -782,8 +782,11 @@ def __init__(self, *args, **kwargs):
         if self.project.has_feature(Feature.USE_TESTING_BUILD_IMAGE):
             self.container_image = 'readthedocs/build:testing'
         # the image set by user or,
-        if self.config and self.config.build.image:
+        if self.config and getattr(self.config.build, 'image', None):
             self.container_image = self.config.build.image
+        # the new Docker image structure or,
+        if self.config and getattr(self.config.build, 'os', None):
+            self.container_image = settings.RTD_DOCKER_BUILD_SETTINGS['os'][self.config.build.os]
         # the image overridden by the project (manually set by an admin).
         if self.project.container_image:
             self.container_image = self.project.container_image

readthedocs/doc_builder/python_environments.py

@@ -8,6 +8,7 @@
 import logging
 import os
 import shutil
+import tarfile
 
 import yaml
 from django.conf import settings
@@ -22,6 +23,7 @@
 from readthedocs.doc_builder.loader import get_builder_class
 from readthedocs.projects.constants import LOG_TEMPLATE
 from readthedocs.projects.models import Feature
+from readthedocs.storage import build_tools_storage
 
 log = logging.getLogger(__name__)
 
@@ -72,6 +74,121 @@ def delete_existing_venv_dir(self):
             )
             shutil.rmtree(venv_dir)
 
+    def install_build_tools(self):
+        build_os = getattr(self.config.build, 'os', None)
+        build_tools = getattr(self.config.build, 'tools', None)
+        if not build_os or not build_tools:
+            log.debug(
+                'Not installing "build.tools" because they are not defined. '
+                'build.os=%s build.tools=%s',
+                build_os,
+                build_tools,
+            )
+            return
+
+        if settings.RTD_DOCKER_COMPOSE:
+            # Create a symlink for ``root`` user to use the same ``.asdf``
+            # installation than ``docs`` user. Required for local building
+            # since everything is run as ``root`` when using Local Development
+            # instance
+            cmd = [
+                'ln',
+                '-s',
+                os.path.join(settings.RTD_DOCKER_WORKDIR, '.asdf'),
+                '/root/.asdf',
+            ]
+            self.build_env.run(
+                *cmd,
+            )
+
+        for tool, version in build_tools.items():
+            # TODO: ask config file object for the specific tool version required by asdf
+            version = self.config.settings['tools'][tool][version]
+
+            # TODO: generate the correct path for the Python version
+            # tool_path = f'{build_os}/{tool}/2021-08-30/{version}.tar.gz'
+            tool_path = f'{build_os}-{tool}-{version}.tar.gz'
+            tool_version_cached = build_tools_storage.exists(tool_path)
+            if tool_version_cached:
+                remote_fd = build_tools_storage.open(tool_path, mode='rb')
+                with tarfile.open(fileobj=remote_fd) as tar:
+                    # Extract it on the shared path between host and Docker container
+                    extract_path = os.path.join(self.project.doc_path, 'tools')
+                    tar.extractall(extract_path)
+
+                    # Move the extracted content to the ``asdf`` installation
+                    cmd = [
+                        'mv',
+                        f'{extract_path}/{version}',
+                        os.path.join(settings.RTD_DOCKER_WORKDIR, f'.asdf/installs/{tool}/{version}'),
+                    ]
+                    self.build_env.run(
+                        *cmd,
+                    )
+            else:
+                log.debug(
+                    'Cached version for tool not found. os=%s tool=%s version=% filename=%s',
+                    build_os,
+                    tool,
+                    version,
+                    tool_path,
+                )
+                # If the tool version selected is not available from the
+                # cache we compile it at build time
+                cmd = [
+                    # TODO: make this environment variable to work
+                    # 'PYTHON_CONFIGURE_OPTS="--enable-shared"',
+                    'asdf',
+                    'install',
+                    tool,
+                    version,
+                ]
+                self.build_env.run(
+                    *cmd,
+                )
+
+            # Make the tool version chosen by the user the default one
+            cmd = [
+                'asdf',
+                'global',
+                tool,
+                version,
+            ]
+            self.build_env.run(
+                *cmd,
+            )
+
+            # Recreate shims for this tool to make the new version
+            # installed available
+            cmd = [
+                'asdf',
+                'reshim',
+                tool,
+            ]
+            self.build_env.run(
+                *cmd,
+            )
+
+            if all([
+                    tool == 'python',
+                    # Do not install them if the tool version was cached
+                    not tool_version_cached,
+                    # Do not install them on conda/mamba
+                    self.config.python_interpreter == 'python',
+            ]):
+                # Install our own requirements if the version is compiled
+                cmd = [
+                    'python',
+                    '-mpip',
+                    'install',
+                    '-U',
+                    'virtualenv',
+                    'setuptools',
+                ]
+                self.build_env.run(
+                    *cmd,
+                )
+
     def install_requirements(self):
         """Install all requirements from the config object."""
         for install in self.config.python.install:
@@ -214,7 +331,7 @@ def is_obsolete(self):
         env_build_hash = env_build.get('hash', None)
 
         if isinstance(self.build_env, DockerBuildEnvironment):
-            build_image = self.config.build.image or DOCKER_IMAGE
+            build_image = self.config.docker_image
             image_hash = self.build_env.image_hash
         else:
             # e.g. LocalBuildEnvironment
@@ -269,7 +386,7 @@ def save_environment_json(self):
         }
 
         if isinstance(self.build_env, DockerBuildEnvironment):
-            build_image = self.config.build.image or DOCKER_IMAGE
+            build_image = self.config.docker_image
             data.update({
                 'build': {
                     'image': build_image,
@@ -317,6 +434,7 @@ def setup_base(self):
         cli_args.append(
             self.venv_path(),
         )
+
         self.build_env.run(
             self.config.python_interpreter,
             *cli_args,
@@ -496,6 +614,11 @@ def conda_bin_name(self):
 
         See https://github.com/QuantStack/mamba
         """
+        # Config file using ``build.tools.python``
+        if self.config.using_build_tools:
+            return self.config.python_interpreter
+
+        # Config file using ``conda``
         if self.project.has_feature(Feature.CONDA_USES_MAMBA):
             return 'mamba'
         return 'conda'
@@ -556,8 +679,12 @@ def setup_base(self):
             self._append_core_requirements()
             self._show_environment_yaml()
 
-        # TODO: remove it when ``mamba`` is installed in the Docker image
-        if self.project.has_feature(Feature.CONDA_USES_MAMBA):
+        if all([
+                # The project has CONDA_USES_MAMBA feature enabled and,
+                self.project.has_feature(Feature.CONDA_USES_MAMBA),
+                # the project is not using ``build.tools``
+                not self.config.using_build_tools,
+        ]):
             self._install_mamba()
 
         self.build_env.run(

readthedocs/projects/tasks.py

@@ -762,7 +762,10 @@ def run_build(self, record):
         # Environment used for building code, usually with Docker
         with self.build_env:
             python_env_cls = Virtualenv
-            if self.config.conda is not None:
+            if any([
+                    self.config.conda is not None,
+                    self.config.python_interpreter in ('conda', 'mamba'),
+            ]):
                 log.info(
                     LOG_TEMPLATE,
                     {
@@ -1164,6 +1167,10 @@ def setup_python_environment(self):
         else:
             self.python_env.delete_existing_build_dir()
 
+        # Install all ``build.tools`` specified by the user
+        if self.config.using_build_tools:
+            self.python_env.install_build_tools()
+
         self.python_env.setup_base()
         self.python_env.save_environment_json()
         self.python_env.install_core_requirements()

readthedocs/settings/base.py

@@ -310,6 +310,7 @@ def USE_PROMOS(self):  # noqa
     # https://docs.readthedocs.io/page/development/settings.html#rtd-build-media-storage
     RTD_BUILD_MEDIA_STORAGE = 'readthedocs.builds.storage.BuildMediaFileSystemStorage'
     RTD_BUILD_ENVIRONMENT_STORAGE = 'readthedocs.builds.storage.BuildMediaFileSystemStorage'
+    RTD_BUILD_TOOLS_STORAGE = 'readthedocs.builds.storage.BuildMediaFileSystemStorage'
     RTD_BUILD_COMMANDS_STORAGE = 'readthedocs.builds.storage.BuildMediaFileSystemStorage'
 
     @property

readthedocs/settings/docker_compose.py

@@ -134,6 +134,8 @@ def show_debug_toolbar(request):
     RTD_BUILD_MEDIA_STORAGE = 'readthedocs.storage.s3_storage.S3BuildMediaStorage'
     # Storage backend for build cached environments
     RTD_BUILD_ENVIRONMENT_STORAGE = 'readthedocs.storage.s3_storage.S3BuildEnvironmentStorage'
+    # Storage backend for build languages
+    RTD_BUILD_TOOLS_STORAGE = 'readthedocs.storage.s3_storage.S3BuildToolsStorage'
     # Storage for static files (those collected with `collectstatic`)
     STATICFILES_STORAGE = 'readthedocs.storage.s3_storage.S3StaticStorage'
 
@@ -142,6 +144,7 @@ def show_debug_toolbar(request):
     S3_MEDIA_STORAGE_BUCKET = 'media'
     S3_BUILD_COMMANDS_STORAGE_BUCKET = 'builds'
     S3_BUILD_ENVIRONMENT_STORAGE_BUCKET = 'envs'
+    S3_BUILD_TOOLS_STORAGE_BUCKET = 'build-tools'
     S3_STATIC_STORAGE_BUCKET = 'static'
     S3_STATIC_STORAGE_OVERRIDE_HOSTNAME = 'community.dev.readthedocs.io'
     S3_MEDIA_STORAGE_OVERRIDE_HOSTNAME = 'community.dev.readthedocs.io'

readthedocs/storage/__init__.py

@@ -26,6 +26,12 @@ def _setup(self):
         self._wrapped = get_storage_class(settings.RTD_BUILD_COMMANDS_STORAGE)()
 
 
+class ConfiguredBuildToolsStorage(LazyObject):
+    def _setup(self):
+        self._wrapped = get_storage_class(settings.RTD_BUILD_TOOLS_STORAGE)()
+
+
 build_media_storage = ConfiguredBuildMediaStorage()
 build_environment_storage = ConfiguredBuildEnvironmentStorage()
 build_commands_storage = ConfiguredBuildCommandsStorage()
+build_tools_storage = ConfiguredBuildToolsStorage()

readthedocs/storage/s3_storage.py

@@ -89,3 +89,17 @@ def __init__(self, *args, **kwargs):
                 'AWS S3 not configured correctly. '
                 'Ensure S3_BUILD_ENVIRONMENT_STORAGE_BUCKET is defined.',
             )
+
+
+class S3BuildToolsStorage(S3PrivateBucketMixin, BuildMediaStorageMixin, S3Boto3Storage):
+
+    bucket_name = getattr(settings, 'S3_BUILD_TOOLS_STORAGE_BUCKET', None)
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        if not self.bucket_name:
+            raise ImproperlyConfigured(
+                'AWS S3 not configured correctly. '
+                'Ensure S3_BUILD_TOOLS_STORAGE_BUCKET is defined.',
+            )

requirements/docker.txt

@@ -28,3 +28,6 @@ argh==0.26.2
 
 # run tests
 tox==3.24.4
+
+# AWS utilities to use against MinIO
+awscli==1.20.34

scripts/compile_version_upload_s3.sh

@@ -0,0 +1,97 @@
+#!/bin/bash
+#
+#
+# Script to compile a languge version and upload it to the cache.
+#
+# This script automates the process to build and upload a Python/Node/Rust/Go
+# version and upload it to S3 making it available for the builders. When a
+# pre-compiled version is available in the cache, builds are faster because they
+# don't have to donwload and compile.
+#
+# LOCAL DEVELOPMENT ENVIRONMENT
+# https://docs.readthedocs.io/en/latest/development/install.html
+#
+# You can run this script from you local environment to create cached version
+# and upload them to MinIO. For this, it's required that you have the MinIO
+# instance running before executing this script command:
+#
+#   inv docker.up
+#
+#
+# PRODUCTION ENVIRONMENT
+#
+# To create a pre-compiled cached version and make it available on production,
+# the script has to be ran from a builder (build-default or build-large) and
+# it's required to set the following environment variables for an IAM user with
+# permissions on ``build-tools`` S3's bucket:
+#
+#   AWS_ACCESS_KEY_ID
+#   AWS_SECRET_ACCESS_KEY
+#   AWS_ENDPOINT_URL
+#
+#
+# USAGE
+#
+#  ./scripts/compile_version_upload.sh python 3.9.6
+#
+#
+
+set -e
+
+# Define variables
+SLEEP=350
+OS="ubuntu-20.04"
+TOOL=$1
+VERSION=$2
+
+# Spin up a container with the Ubuntu 20.04 LTS image
+CONTAINER_ID=$(docker run --user docs --rm --detach readthedocs/build:$OS sleep $SLEEP)
+echo "Running all the commands in Docker container: $CONTAINER_ID"
+
+# Install the tool version requested
+if [[ $TOOL == "python" ]]
+then
+    docker exec --env PYTHON_CONFIGURE_OPTS="--enable-shared" $CONTAINER_ID asdf install $TOOL $VERSION
+else
+    docker exec $CONTAINER_ID asdf install $TOOL $VERSION
+fi
+
+# Set the default version and reshim
+docker exec $CONTAINER_ID asdf global $TOOL $VERSION
+docker exec $CONTAINER_ID asdf reshim $TOOL
+
+# Install dependencies for this version
+if [[ $TOOL == "python" ]] && [[ ! $VERSION =~ (^miniconda.*|^mambaforge.*) ]]
+then
+    RTD_PIP_VERSION=21.2.4
+    RTD_SETUPTOOLS_VERSION=57.4.0
+    RTD_VIRTUALENV_VERSION=20.7.2
+
+    if [[ $VERSION == "2.7.18" ]]
+    then
+        # Pin to the latest versions supported on Python 2.7
+        RTD_PIP_VERSION=20.3.4
+        RTD_SETUPTOOLS_VERSION=44.1.1
+        RTD_VIRTUALENV_VERSION=20.7.2
+    fi
+    docker exec $CONTAINER_ID $TOOL -m pip install -U pip==$RTD_PIP_VERSION setuptools==$RTD_SETUPTOOLS_VERSION virtualenv==$RTD_VIRTUALENV_VERSION
+fi
+
+# Compress it as a .tar.gz without include the full path in the compressed file
+docker exec $CONTAINER_ID tar --create --gzip --directory=/home/docs/.asdf/installs/$TOOL --file=$OS-$TOOL-$VERSION.tar.gz $VERSION
+
+# Copy the .tar.gz from the container to the host
+docker cp $CONTAINER_ID:/home/docs/$OS-$TOOL-$VERSION.tar.gz .
+
+# Kill the container
+docker container kill $CONTAINER_ID
+
+# Upload the .tar.gz to S3
+AWS_ENDPOINT_URL="${AWS_ENDPOINT_URL:-http://localhost:9000}"
+AWS_BUILD_TOOLS_BUCKET="${AWS_BUILD_TOOLS_BUCKET:-build-tools}"
+AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-admin}" \
+AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-password}" \
+aws --endpoint-url $AWS_ENDPOINT_URL s3 cp $OS-$TOOL-$VERSION.tar.gz s3://$AWS_BUILD_TOOLS_BUCKET
+
+# Delete the .tar.gz file from the host
+rm $OS-$TOOL-$VERSION.tar.gz