|
| 1 | +""" |
| 2 | +Wrapper around the rclone command. |
| 3 | +
|
| 4 | +See https://rclone.org/docs. |
| 5 | +""" |
| 6 | + |
| 7 | +import os |
| 8 | +import subprocess |
| 9 | + |
| 10 | +import structlog |
| 11 | +from django.utils._os import safe_join as safe_join_fs |
| 12 | +from storages.utils import safe_join |
| 13 | + |
| 14 | +log = structlog.get_logger(__name__) |
| 15 | + |
| 16 | + |
| 17 | +class BaseRClone: |
| 18 | + |
| 19 | + """ |
| 20 | + RClone base class. |
| 21 | +
|
| 22 | + This class allows you to interact with an rclone remote without |
| 23 | + a configuration file, the remote declaration and its options |
| 24 | + are passed in the command itself. |
| 25 | +
|
| 26 | + This base class allows you to use the local file system as remote. |
| 27 | +
|
| 28 | + :param remote_type: You can see the full list of supported providers at |
| 29 | + https://rclone.org/#providers. |
| 30 | + :param rclone_bin: Binary name or path to the rclone binary. |
| 31 | + Defaults to ``rclone``. |
| 32 | + :param default_options: Options passed to the rclone command. |
| 33 | + :parm env_vars: Environment variables used when executing the rclone command. |
| 34 | + Useful to pass secrets to the ``rclone` command, since all arguments and |
| 35 | + options will be logged. |
| 36 | + """ |
| 37 | + |
| 38 | + remote_type = None |
| 39 | + rclone_bin = "rclone" |
| 40 | + default_options = [ |
| 41 | + # Number of file transfers to run in parallel. |
| 42 | + # Default value is 4. |
| 43 | + "--transfers=8", |
| 44 | + # Skip based on checksum (if available) & size, not mod-time & size. |
| 45 | + "--checksum", |
| 46 | + "--verbose", |
| 47 | + ] |
| 48 | + env_vars = {} |
| 49 | + |
| 50 | + def _get_target_path(self, path): |
| 51 | + """ |
| 52 | + Get the final target path for the remote. |
| 53 | +
|
| 54 | + .. note:: |
| 55 | +
|
| 56 | + This doesn't include the remote type, |
| 57 | + this is just the destination path. |
| 58 | + """ |
| 59 | + raise NotImplementedError |
| 60 | + |
| 61 | + def get_target(self, path): |
| 62 | + """ |
| 63 | + Get the proper target using the current remote type. |
| 64 | +
|
| 65 | + We start the remote with `:` to create it on the fly, |
| 66 | + instead of having to create a configuration file. |
| 67 | + See https://rclone.org/docs/#backend-path-to-dir. |
| 68 | +
|
| 69 | + :param path: Path to the remote target. |
| 70 | + """ |
| 71 | + path = self._get_target_path(path) |
| 72 | + return f":{self.remote_type}:{path}" |
| 73 | + |
| 74 | + def execute(self, subcommand, args, options=None): |
| 75 | + """ |
| 76 | + Execute an rclone subcommand. |
| 77 | +
|
| 78 | + :param subcommand: Name of the subcommand. |
| 79 | + :param list args: List of positional arguments passed the to command. |
| 80 | + :param list options: List of options passed to the command. |
| 81 | + """ |
| 82 | + options = options or [] |
| 83 | + command = [ |
| 84 | + self.rclone_bin, |
| 85 | + subcommand, |
| 86 | + *self.default_options, |
| 87 | + *options, |
| 88 | + "--", |
| 89 | + *args, |
| 90 | + ] |
| 91 | + env = os.environ.copy() |
| 92 | + env.update(self.env_vars) |
| 93 | + log.info("Executing rclone command.", command=command) |
| 94 | + log.debug("Executing rclone commmad.", env=env) |
| 95 | + result = subprocess.run( |
| 96 | + command, |
| 97 | + capture_output=True, |
| 98 | + env=env, |
| 99 | + check=True, |
| 100 | + ) |
| 101 | + log.debug( |
| 102 | + "rclone execution finished.", |
| 103 | + stdout=result.stdout.decode(), |
| 104 | + stderr=result.stderr.decode(), |
| 105 | + exit_code=result.returncode, |
| 106 | + ) |
| 107 | + return result |
| 108 | + |
| 109 | + def sync(self, source, destination): |
| 110 | + """ |
| 111 | + Run the `rclone sync` command. |
| 112 | +
|
| 113 | + See https://rclone.org/commands/rclone_sync/. |
| 114 | +
|
| 115 | + :params source: Local path to the source directory. |
| 116 | + :params destination: Remote path to the destination directory. |
| 117 | + """ |
| 118 | + return self.execute("sync", args=[source, self.get_target(destination)]) |
| 119 | + |
| 120 | + |
| 121 | +class RCloneLocal(BaseRClone): |
| 122 | + |
| 123 | + """ |
| 124 | + RClone remote implementation for the local file system. |
| 125 | +
|
| 126 | + Used for local testing only. |
| 127 | +
|
| 128 | + See https://rclone.org/local/. |
| 129 | +
|
| 130 | + :param location: Root directory where the files will be stored. |
| 131 | + """ |
| 132 | + |
| 133 | + remote_type = "local" |
| 134 | + |
| 135 | + def __init__(self, location): |
| 136 | + self.location = location |
| 137 | + |
| 138 | + def _get_target_path(self, path): |
| 139 | + return safe_join_fs(self.location, path) |
| 140 | + |
| 141 | + |
| 142 | +class RCloneS3Remote(BaseRClone): |
| 143 | + |
| 144 | + """ |
| 145 | + RClone remote implementation for S3. |
| 146 | +
|
| 147 | + All secrets will be passed as environ variables to the rclone command. |
| 148 | +
|
| 149 | + See https://rclone.org/s3/. |
| 150 | +
|
| 151 | + :params bucket_name: Name of the S3 bucket. |
| 152 | + :params access_key_id: AWS access key id. |
| 153 | + :params secret_acces_key: AWS secret access key. |
| 154 | + :params region: AWS region. |
| 155 | + :params provider: S3 provider, defaults to ``AWS``. |
| 156 | + Useful to use Minio during development. |
| 157 | + See https://rclone.org/s3/#s3-provider. |
| 158 | + :param acl: Canned ACL used when creating buckets and storing or copying objects. |
| 159 | + See https://rclone.org/s3/#s3-acl. |
| 160 | + :param endpoint: Custom S3 endpoint, useful for development. |
| 161 | + """ |
| 162 | + |
| 163 | + remote_type = "s3" |
| 164 | + |
| 165 | + def __init__( |
| 166 | + self, |
| 167 | + bucket_name, |
| 168 | + access_key_id, |
| 169 | + secret_acces_key, |
| 170 | + region, |
| 171 | + provider="AWS", |
| 172 | + acl=None, |
| 173 | + endpoint=None, |
| 174 | + ): |
| 175 | + # rclone S3 options passed as env vars. |
| 176 | + # https://rclone.org/s3/#standard-options. |
| 177 | + self.env_vars = { |
| 178 | + "RCLONE_S3_PROVIDER": provider, |
| 179 | + "RCLONE_S3_ACCESS_KEY_ID": access_key_id, |
| 180 | + "RCLONE_S3_SECRET_ACCESS_KEY": secret_acces_key, |
| 181 | + "RCLONE_S3_REGION": region, |
| 182 | + "RCLONE_S3_LOCATION_CONSTRAINT": region, |
| 183 | + } |
| 184 | + if acl: |
| 185 | + self.env_vars["RCLONE_S3_ACL"] = acl |
| 186 | + if endpoint: |
| 187 | + self.env_vars["RCLONE_S3_ENDPOINT"] = endpoint |
| 188 | + self.bucket_name = bucket_name |
| 189 | + |
| 190 | + def _get_target_path(self, path): |
| 191 | + """Overridden to prepend the bucket name to the path.""" |
| 192 | + return safe_join(self.bucket_name, path) |
0 commit comments