Build: use rclone for sync

Author: stsewd

dockerfiles/Dockerfile

@@ -30,7 +30,8 @@ RUN apt-get -y install \
         netcat \
         telnet \
         lsb-release \
-        npm
+        npm \
+        rclone
 
 # Gets the MinIO mc client used to add buckets upon initialization
 # If this client should have issues running inside this image, it is also

readthedocs/builds/storage.py

@@ -1,3 +1,4 @@
+from functools import cached_property
 from pathlib import Path
 
 import structlog
@@ -7,6 +8,7 @@
 from storages.utils import get_available_overwrite_name, safe_join
 
 from readthedocs.core.utils.filesystem import safe_open
+from readthedocs.storage.rclone import RClone
 
 log = structlog.get_logger(__name__)
 
@@ -153,6 +155,14 @@ def sync_directory(self, source, destination):
                 log.debug('Deleting file from media storage.', filepath=filepath)
                 self.delete(filepath)
 
+    @cached_property
+    def _rclone(self):
+        return RClone()
+
+    def rclone_sync(self, source, destination):
+        """Sync a directory recursively to storage using rclone sync."""
+        return self._rclone.sync(source, destination)
+
     def join(self, directory, filepath):
         return safe_join(directory, filepath)
 

readthedocs/projects/models.py

@@ -1845,6 +1845,7 @@ def add_features(sender, **kwargs):
     USE_SPHINX_BUILDERS = "use_sphinx_builders"
     CANCEL_OLD_BUILDS = "cancel_old_builds"
     DONT_CREATE_INDEX = "dont_create_index"
+    USE_RCLONE = "use_rclone"
 
     FEATURES = (
         (ALLOW_DEPRECATED_WEBHOOKS, _('Allow deprecated webhook views')),
@@ -2001,6 +2002,10 @@ def add_features(sender, **kwargs):
             DONT_CREATE_INDEX,
             _('Do not create index.md or README.rst if the project does not have one.'),
         ),
+        (
+            USE_RCLONE,
+            _("Use rclone for syncing files to the media storage."),
+        ),
     )
 
     projects = models.ManyToManyField(

readthedocs/projects/tasks/builds.py

@@ -833,7 +833,10 @@ def store_build_artifacts(
                 version_type=self.data.version.type,
             )
             try:
-                build_media_storage.sync_directory(from_path, to_path)
+                if self.data.project.has_feature(Feature.USE_RCLONE):
+                    build_media_storage.rclone_sync(from_path, to_path)
+                else:
+                    build_media_storage.sync_directory(from_path, to_path)
             except Exception:
                 # Ideally this should just be an IOError
                 # but some storage backends unfortunately throw other errors

readthedocs/storage/rclone.py

@@ -0,0 +1,163 @@
+"""
+Wrapper around the rclone command.
+
+See https://rclone.org/docs.
+"""
+
+import os
+import subprocess
+
+import structlog
+
+log = structlog.get_logger(__name__)
+
+
+class RClone:
+
+    """
+    RClone base class.
+
+    This class allows you to interact with an rclone remote without
+    a configuration file, the remote declaration and its options
+    are passed in the command itself.
+
+    This base class allows you to use the local file system as remote.
+
+    :param remote_type: You can see the full list of supported providers at
+     https://rclone.org/#providers. Defaults to use the local filesystem
+     (https://rclone.org/local/).
+    :param rclone_bin: Binary name or path to the rclone binary.
+     Defaults to ``rclone``.
+    :param default_options: Options passed to the rclone command.
+    :parm env_vars: Environment variables used when executing the rclone command.
+     Useful to pass secrets to the command, since all arguments and options will be logged.
+    """
+
+    remote_type = "local"
+    rclone_bin = "rclone"
+    default_options = [
+        #  Number of file transfers to run in parallel.
+        "--transfers=8",
+        "--verbose",
+    ]
+    env_vars = {}
+
+    def build_target(self, path):
+        """
+        Build the proper target using the current remote type.
+
+        We start the remote with `:` to create it on the fly,
+        instead of having to create a configuration file.
+        See https://rclone.org/docs/#backend-path-to-dir.
+
+        :param path: Path to the remote target.
+        """
+        return f":{self.remote_type}:{path}"
+
+    def execute(self, action, args, options=None):
+        """
+        Execute an rclone subcommand.
+
+        :param action: Name of the subcommand.
+        :param list args: List of positional arguments passed the to command.
+        :param list options: List of options passed to the command.
+        """
+        options = options or []
+        command = [
+            self.rclone_bin,
+            action,
+            *self.default_options,
+            *options,
+            "--",
+            *args,
+        ]
+        env = os.environ.copy()
+        # env = {}
+        env.update(self.env_vars)
+        log.info("Executing rclone command.", command=command)
+        log.debug("env", env=env)
+        result = subprocess.run(
+            command,
+            capture_output=True,
+            env=env,
+            # TODO: Fail or let the called decide what to do?
+            check=True,
+        )
+        log.debug(
+            "Result.",
+            stdout=result.stdout.decode(),
+            stderr=result.stderr.decode(),
+            exit_code=result.returncode,
+        )
+        return result
+
+    def sync(self, source, destination):
+        """
+        Run the `rclone sync` command.
+
+        See https://rclone.org/commands/rclone_sync/.
+
+        :params source: Local path to the source directory.
+        :params destination: Remote path to the destination directory.
+        """
+        # TODO: check if source can be a symlink.
+        return self.execute("sync", args=[source, self.build_target(destination)])
+
+
+class RCloneS3Remote(RClone):
+
+    """
+    RClone remote implementation for S3.
+
+    All secrets will be passed as environ variables.
+
+    See https://rclone.org/s3/.
+
+    :params bucket_name: Name of the S3 bucket.
+    :params access_key_id: AWS access key id.
+    :params secret_acces_key: AWS secret access key.
+    :params region: AWS region.
+    :params provider: S3 provider, defaults to ``AWS``.
+     Useful to use Minio during development.
+     See https://rclone.org/s3/#s3-provider.
+    :param acl: Canned ACL used when creating buckets and storing or copying objects.
+     See https://rclone.org/s3/#s3-acl.
+    :param endpoint: Custom S3 endpoint, useful for development.
+    """
+
+    remote_type = "s3"
+
+    def __init__(
+        self,
+        bucket_name,
+        access_key_id,
+        secret_acces_key,
+        region,
+        provider="AWS",
+        acl=None,
+        endpoint=None,
+    ):
+        super().__init__()
+
+        # When using minion, the region is set to None.
+        region = region or ""
+
+        # rclone S3 options passed as env vars.
+        # https://rclone.org/s3/#standard-options.
+        self.env_vars = {
+            "RCLONE_S3_PROVIDER": provider,
+            "RCLONE_S3_ACCESS_KEY_ID": access_key_id,
+            "RCLONE_S3_SECRET_ACCESS_KEY": secret_acces_key,
+            "RCLONE_S3_REGION": region,
+            "RCLONE_S3_LOCATION_CONSTRAINT": region,
+        }
+        if acl:
+            self.env_vars["RCLONE_S3_ACL"] = acl
+        if endpoint:
+            self.env_vars["RCLONE_S3_ENDPOINT"] = endpoint
+        self.bucket_name = bucket_name
+
+    def build_target(self, path):
+        """Overridden to prepend the bucket name to the path."""
+        path = f"{self.bucket_name}/{path}"
+        return super().build_target(path)

readthedocs/storage/s3_storage.py

@@ -9,16 +9,41 @@
 
 # Disable abstract method because we are not overriding all the methods
 # pylint: disable=abstract-method
+from functools import cached_property
+
 from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
 from storages.backends.s3boto3 import S3Boto3Storage, S3ManifestStaticStorage
 
 from readthedocs.builds.storage import BuildMediaStorageMixin
+from readthedocs.storage.rclone import RCloneS3Remote
 
 from .mixins import OverrideHostnameMixin, S3PrivateBucketMixin
 
 
-class S3BuildMediaStorage(BuildMediaStorageMixin, OverrideHostnameMixin, S3Boto3Storage):
+class S3BuildMediaStorageMixin(BuildMediaStorageMixin, S3Boto3Storage):
+
+    @cached_property
+    def _rclone(self):
+        provider = "AWS"
+        # If a custom endpoint URL is given and
+        # we are running in DEBUG mode, use minio as provider.
+        if self.endpoint_url and settings.DEBUG:
+            provider = "minio"
+
+        return RCloneS3Remote(
+            bucket_name=self.bucket_name,
+            access_key_id=self.access_key,
+            secret_acces_key=self.secret_key,
+            region=self.region_name,
+            acl=self.default_acl,
+            endpoint=self.endpoint_url,
+            provider=provider,
+        )
+
+
+# pylint: disable=too-many-ancestors
+class S3BuildMediaStorage(OverrideHostnameMixin, S3BuildMediaStorageMixin):
 
     """An AWS S3 Storage backend for build artifacts."""
 
@@ -94,7 +119,7 @@ class NoManifestS3StaticStorage(
     """
 
 
-class S3BuildEnvironmentStorage(S3PrivateBucketMixin, BuildMediaStorageMixin, S3Boto3Storage):
+class S3BuildEnvironmentStorage(S3PrivateBucketMixin, S3BuildMediaStorageMixin):
 
     bucket_name = getattr(settings, 'S3_BUILD_ENVIRONMENT_STORAGE_BUCKET', None)
 
@@ -108,7 +133,7 @@ def __init__(self, *args, **kwargs):
             )
 
 
-class S3BuildToolsStorage(S3PrivateBucketMixin, BuildMediaStorageMixin, S3Boto3Storage):
+class S3BuildToolsStorage(S3PrivateBucketMixin, S3BuildMediaStorageMixin):
 
     bucket_name = getattr(settings, 'S3_BUILD_TOOLS_STORAGE_BUCKET', None)