Use tar to extract cache, write by chunks and new build state

readthedocs/builds/constants.py

@@ -5,13 +5,15 @@
 
 
 BUILD_STATE_TRIGGERED = 'triggered'
+BUILD_STATE_PULLING_CACHE = 'pulling-cache'
 BUILD_STATE_CLONING = 'cloning'
 BUILD_STATE_INSTALLING = 'installing'
 BUILD_STATE_BUILDING = 'building'
 BUILD_STATE_FINISHED = 'finished'
 
 BUILD_STATE = (
     (BUILD_STATE_TRIGGERED, _('Triggered')),
+    (BUILD_STATE_PULLING_CACHE, _('Pulling cache')),
     (BUILD_STATE_CLONING, _('Cloning')),
     (BUILD_STATE_INSTALLING, _('Installing')),
     (BUILD_STATE_BUILDING, _('Building')),

readthedocs/builds/migrations/0015_pulling_cache_build_state.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.10 on 2020-03-19 03:38
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('builds', '0014_migrate-doctype-from-project-to-version'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='build',
+            name='state',
+            field=models.CharField(choices=[('triggered', 'Triggered'), ('pulling-cache', 'Pulling cache'), ('cloning', 'Cloning'), ('installing', 'Installing'), ('building', 'Building'), ('finished', 'Finished')], default='finished', max_length=55, verbose_name='State'),
+        ),
+    ]

readthedocs/projects/tasks.py

@@ -14,6 +14,7 @@
 import socket
 import tarfile
 import tempfile
+import subprocess
 from collections import Counter, defaultdict
 
 import requests
@@ -33,6 +34,7 @@
     BUILD_STATE_CLONING,
     BUILD_STATE_FINISHED,
     BUILD_STATE_INSTALLING,
+    BUILD_STATE_PULLING_CACHE,
     BUILD_STATUS_SUCCESS,
     BUILD_STATUS_FAILURE,
     LATEST,
@@ -46,7 +48,12 @@
 from readthedocs.config import ConfigError
 from readthedocs.core.resolver import resolve_path
 from readthedocs.core.symlink import PrivateSymlink, PublicSymlink
-from readthedocs.core.utils import broadcast, safe_unlink, send_email
+from readthedocs.core.utils import (
+    broadcast,
+    safe_makedirs,
+    safe_unlink,
+    send_email,
+)
 from readthedocs.doc_builder.config import load_yaml_config
 from readthedocs.doc_builder.constants import DOCKER_LIMITS
 from readthedocs.doc_builder.environments import (
@@ -93,10 +100,12 @@ class CachedEnvironmentMixin:
 
     """Mixin that pull/push cached environment to storage."""
 
-    def pull_cached_environment(self):
+    def pull_cached_environment(self, environment):
         if not self.project.has_feature(feature_id=Feature.CACHED_ENVIRONMENT):
             return
 
+        environment.update_build(state=BUILD_STATE_PULLING_CACHE)
+
         storage = get_storage_class(settings.RTD_BUILD_ENVIRONMENT_STORAGE)()
         filename = self.version.get_storage_environment_cache_path()
 
@@ -122,10 +131,20 @@ def pull_cached_environment(self):
             _, tmp_filename = tempfile.mkstemp(suffix='.tar')
             remote_fd = storage.open(filename, mode='rb')
             with open(tmp_filename, mode='wb') as local_fd:
-                local_fd.write(remote_fd.read())
-
-            with tarfile.open(tmp_filename) as tar:
-                tar.extractall(self.version.project.doc_path)
+                for chunk in remote_fd.chunks():
+                    local_fd.write(chunk)
+
+            safe_makedirs(self.version.project.doc_path)
+            # Using ``tar`` command because ``tarfile`` has a memory leak when
+            # extracting big files (2.5Gb)
+            command = [
+                'tar',
+                'xvf',
+                tmp_filename,
+                '--directory',
+                self.version.project.doc_path,
+            ]
+            subprocess.run(command)
 
             # Cleanup the temporary file
             if os.path.exists(tmp_filename):
@@ -369,7 +388,7 @@ def run(self, version_pk):  # pylint: disable=arguments-differ
                     # repository in this step, and pushing it back will delete
                     # all the other cached things (Python packages, Sphinx,
                     # virtualenv, etc)
-                    self.pull_cached_environment()
+                    self.pull_cached_environment(environment)
                     self.sync_repo(environment)
             return True
         except RepositoryError:
@@ -591,7 +610,7 @@ def run_setup(self, record=True):
                     raise ProjectBuildsSkippedError
                 try:
                     with self.project.repo_nonblockinglock(version=self.version):
-                        self.pull_cached_environment()
+                        self.pull_cached_environment(environment)
                         self.setup_vcs(environment)
                 except vcs_support_utils.LockTimeout as e:
                     self.task.retry(exc=e, throw=False)