Skip to content

Commit 0f87bc2

Browse files
humitoshumitos
authored
Merge pull request #8453 from readthedocs/humitos/build-new-docker-image
2 parents d8651c1 + 94231b9 commit 0f87bc2

File tree

11 files changed

+431
-7
lines changed

11 files changed

+431
-7
lines changed

‎readthedocs/doc_builder/environments.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -782,8 +782,8 @@ def __init__(self, *args, **kwargs):
782782
if self.project.has_feature(Feature.USE_TESTING_BUILD_IMAGE):
783783
self.container_image = 'readthedocs/build:testing'
784784
# the image set by user or,
785-
if self.config and self.config.build.image:
786-
self.container_image = self.config.build.image
785+
if self.config and self.config.docker_image:
786+
self.container_image = self.config.docker_image
787787
# the image overridden by the project (manually set by an admin).
788788
if self.project.container_image:
789789
self.container_image = self.project.container_image

‎readthedocs/doc_builder/python_environments.py

+122-4
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import logging
99
import os
1010
import shutil
11+
import tarfile
1112

1213
import yaml
1314
from django.conf import settings
@@ -22,6 +23,7 @@
2223
from readthedocs.doc_builder.loader import get_builder_class
2324
from readthedocs.projects.constants import LOG_TEMPLATE
2425
from readthedocs.projects.models import Feature
26+
from readthedocs.storage import build_tools_storage
2527

2628
log = logging.getLogger(__name__)
2729

@@ -72,6 +74,112 @@ def delete_existing_venv_dir(self):
7274
)
7375
shutil.rmtree(venv_dir)
7476

77+
def install_build_tools(self):
78+
if settings.RTD_DOCKER_COMPOSE:
79+
# Create a symlink for ``root`` user to use the same ``.asdf``
80+
# installation than ``docs`` user. Required for local building
81+
# since everything is run as ``root`` when using Local Development
82+
# instance
83+
cmd = [
84+
'ln',
85+
'-s',
86+
os.path.join(settings.RTD_DOCKER_WORKDIR, '.asdf'),
87+
'/root/.asdf',
88+
]
89+
self.build_env.run(
90+
*cmd,
91+
)
92+
93+
for tool, version in self.config.build.tools.items():
94+
version = version.full_version # e.g. 3.9 -> 3.9.7
95+
96+
# TODO: generate the correct path for the Python version
97+
# tool_path = f'{self.config.build.os}/{tool}/2021-08-30/{version}.tar.gz'
98+
tool_path = f'{self.config.build.os}-{tool}-{version}.tar.gz'
99+
tool_version_cached = build_tools_storage.exists(tool_path)
100+
if tool_version_cached:
101+
remote_fd = build_tools_storage.open(tool_path, mode='rb')
102+
with tarfile.open(fileobj=remote_fd) as tar:
103+
# Extract it on the shared path between host and Docker container
104+
extract_path = os.path.join(self.project.doc_path, 'tools')
105+
tar.extractall(extract_path)
106+
107+
# Move the extracted content to the ``asdf`` installation
108+
cmd = [
109+
'mv',
110+
f'{extract_path}/{version}',
111+
os.path.join(
112+
settings.RTD_DOCKER_WORKDIR,
113+
f'.asdf/installs/{tool}/{version}',
114+
),
115+
]
116+
self.build_env.run(
117+
*cmd,
118+
)
119+
else:
120+
log.debug(
121+
'Cached version for tool not found. os=%s tool=%s version=% filename=%s',
122+
self.config.build.os,
123+
tool,
124+
version,
125+
tool_path,
126+
)
127+
# If the tool version selected is not available from the
128+
# cache we compile it at build time
129+
cmd = [
130+
# TODO: make this environment variable to work
131+
# 'PYTHON_CONFIGURE_OPTS="--enable-shared"',
132+
'asdf',
133+
'install',
134+
tool,
135+
version,
136+
]
137+
self.build_env.run(
138+
*cmd,
139+
)
140+
141+
# Make the tool version chosen by the user the default one
142+
cmd = [
143+
'asdf',
144+
'global',
145+
tool,
146+
version,
147+
]
148+
self.build_env.run(
149+
*cmd,
150+
)
151+
152+
# Recreate shims for this tool to make the new version
153+
# installed available
154+
cmd = [
155+
'asdf',
156+
'reshim',
157+
tool,
158+
]
159+
self.build_env.run(
160+
*cmd,
161+
)
162+
163+
if all([
164+
tool == 'python',
165+
# Do not install them if the tool version was cached
166+
not tool_version_cached,
167+
# Do not install them on conda/mamba
168+
self.config.python_interpreter == 'python',
169+
]):
170+
# Install our own requirements if the version is compiled
171+
cmd = [
172+
'python',
173+
'-mpip',
174+
'install',
175+
'-U',
176+
'virtualenv',
177+
'setuptools',
178+
]
179+
self.build_env.run(
180+
*cmd,
181+
)
182+
75183
def install_requirements(self):
76184
"""Install all requirements from the config object."""
77185
for install in self.config.python.install:
@@ -214,7 +322,7 @@ def is_obsolete(self):
214322
env_build_hash = env_build.get('hash', None)
215323

216324
if isinstance(self.build_env, DockerBuildEnvironment):
217-
build_image = self.config.build.image or DOCKER_IMAGE
325+
build_image = self.config.docker_image
218326
image_hash = self.build_env.image_hash
219327
else:
220328
# e.g. LocalBuildEnvironment
@@ -269,7 +377,7 @@ def save_environment_json(self):
269377
}
270378

271379
if isinstance(self.build_env, DockerBuildEnvironment):
272-
build_image = self.config.build.image or DOCKER_IMAGE
380+
build_image = self.config.docker_image
273381
data.update({
274382
'build': {
275383
'image': build_image,
@@ -317,6 +425,7 @@ def setup_base(self):
317425
cli_args.append(
318426
self.venv_path(),
319427
)
428+
320429
self.build_env.run(
321430
self.config.python_interpreter,
322431
*cli_args,
@@ -496,6 +605,11 @@ def conda_bin_name(self):
496605
497606
See https://github.com/QuantStack/mamba
498607
"""
608+
# Config file using ``build.tools.python``
609+
if self.config.using_build_tools:
610+
return self.config.python_interpreter
611+
612+
# Config file using ``conda``
499613
if self.project.has_feature(Feature.CONDA_USES_MAMBA):
500614
return 'mamba'
501615
return 'conda'
@@ -556,8 +670,12 @@ def setup_base(self):
556670
self._append_core_requirements()
557671
self._show_environment_yaml()
558672

559-
# TODO: remove it when ``mamba`` is installed in the Docker image
560-
if self.project.has_feature(Feature.CONDA_USES_MAMBA):
673+
if all([
674+
# The project has CONDA_USES_MAMBA feature enabled and,
675+
self.project.has_feature(Feature.CONDA_USES_MAMBA),
676+
# the project is not using ``build.tools``
677+
not self.config.using_build_tools,
678+
]):
561679
self._install_mamba()
562680

563681
self.build_env.run(

‎readthedocs/projects/tasks.py

+8-1
Original file line numberDiff line numberDiff line change
@@ -762,7 +762,10 @@ def run_build(self, record):
762762
# Environment used for building code, usually with Docker
763763
with self.build_env:
764764
python_env_cls = Virtualenv
765-
if self.config.conda is not None:
765+
if any([
766+
self.config.conda is not None,
767+
self.config.python_interpreter in ('conda', 'mamba'),
768+
]):
766769
log.info(
767770
LOG_TEMPLATE,
768771
{
@@ -1164,6 +1167,10 @@ def setup_python_environment(self):
11641167
else:
11651168
self.python_env.delete_existing_build_dir()
11661169

1170+
# Install all ``build.tools`` specified by the user
1171+
if self.config.using_build_tools:
1172+
self.python_env.install_build_tools()
1173+
11671174
self.python_env.setup_base()
11681175
self.python_env.save_environment_json()
11691176
self.python_env.install_core_requirements()

‎readthedocs/rtd_tests/tests/test_celery.py

+154
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
from unittest.mock import MagicMock, patch
77

88
from allauth.socialaccount.models import SocialAccount
9+
from django.conf import settings
910
from django.contrib.auth.models import User
1011
from django.test import TestCase
1112
from django_dynamic_fixture import get
@@ -541,3 +542,156 @@ def test_install_apt_packages(self, load_config, run):
541542
user='root:root',
542543
)
543544
)
545+
546+
@patch('readthedocs.projects.tasks.UpdateDocsTaskStep.build_docs', new=MagicMock)
547+
@patch('readthedocs.projects.tasks.UpdateDocsTaskStep.setup_vcs', new=MagicMock)
548+
@patch.object(BuildEnvironment, 'run')
549+
@patch('readthedocs.doc_builder.config.load_config')
550+
def test_build_tools(self, load_config, build_run):
551+
config = BuildConfigV2(
552+
{},
553+
{
554+
'version': 2,
555+
'build': {
556+
'os': 'ubuntu-20.04',
557+
'tools': {
558+
'python': '3.10',
559+
'nodejs': '16',
560+
'rust': '1.55',
561+
'golang': '1.17',
562+
},
563+
},
564+
},
565+
source_file='readthedocs.yml',
566+
)
567+
config.validate()
568+
load_config.return_value = config
569+
570+
version = self.project.versions.first()
571+
build = get(
572+
Build,
573+
project=self.project,
574+
version=version,
575+
)
576+
with mock_api(self.repo):
577+
result = tasks.update_docs_task.delay(
578+
version.pk,
579+
build_pk=build.pk,
580+
record=False,
581+
intersphinx=False,
582+
)
583+
self.assertTrue(result.successful())
584+
self.assertEqual(build_run.call_count, 14)
585+
586+
python_version = settings.RTD_DOCKER_BUILD_SETTINGS['tools']['python']['3.10']
587+
nodejs_version = settings.RTD_DOCKER_BUILD_SETTINGS['tools']['nodejs']['16']
588+
rust_version = settings.RTD_DOCKER_BUILD_SETTINGS['tools']['rust']['1.55']
589+
golang_version = settings.RTD_DOCKER_BUILD_SETTINGS['tools']['golang']['1.17']
590+
self.assertEqual(
591+
build_run.call_args_list,
592+
[
593+
mock.call('asdf', 'install', 'python', python_version),
594+
mock.call('asdf', 'global', 'python', python_version),
595+
mock.call('asdf', 'reshim', 'python'),
596+
mock.call('python', '-mpip', 'install', '-U', 'virtualenv', 'setuptools'),
597+
mock.call('asdf', 'install', 'nodejs', nodejs_version),
598+
mock.call('asdf', 'global', 'nodejs', nodejs_version),
599+
mock.call('asdf', 'reshim', 'nodejs'),
600+
mock.call('asdf', 'install', 'rust', rust_version),
601+
mock.call('asdf', 'global', 'rust', rust_version),
602+
mock.call('asdf', 'reshim', 'rust'),
603+
mock.call('asdf', 'install', 'golang', golang_version),
604+
mock.call('asdf', 'global', 'golang', golang_version),
605+
mock.call('asdf', 'reshim', 'golang'),
606+
mock.ANY,
607+
],
608+
)
609+
610+
@patch('readthedocs.projects.tasks.UpdateDocsTaskStep.build_docs', new=MagicMock)
611+
@patch('readthedocs.projects.tasks.UpdateDocsTaskStep.setup_vcs', new=MagicMock)
612+
@patch('readthedocs.doc_builder.python_environments.tarfile')
613+
@patch('readthedocs.doc_builder.python_environments.build_tools_storage')
614+
@patch.object(BuildEnvironment, 'run')
615+
@patch('readthedocs.doc_builder.config.load_config')
616+
def test_build_tools_cached(self, load_config, build_run, build_tools_storage, tarfile):
617+
config = BuildConfigV2(
618+
{},
619+
{
620+
'version': 2,
621+
'build': {
622+
'os': 'ubuntu-20.04',
623+
'tools': {
624+
'python': '3.10',
625+
'nodejs': '16',
626+
'rust': '1.55',
627+
'golang': '1.17',
628+
},
629+
},
630+
},
631+
source_file='readthedocs.yml',
632+
)
633+
config.validate()
634+
load_config.return_value = config
635+
636+
build_tools_storage.open.return_value = b''
637+
build_tools_storage.exists.return_value = True
638+
tarfile.open.return_value.__enter__.return_value.extract_all.return_value = None
639+
640+
version = self.project.versions.first()
641+
build = get(
642+
Build,
643+
project=self.project,
644+
version=version,
645+
)
646+
with mock_api(self.repo):
647+
result = tasks.update_docs_task.delay(
648+
version.pk,
649+
build_pk=build.pk,
650+
record=False,
651+
intersphinx=False,
652+
)
653+
self.assertTrue(result.successful())
654+
self.assertEqual(build_run.call_count, 13)
655+
656+
python_version = settings.RTD_DOCKER_BUILD_SETTINGS['tools']['python']['3.10']
657+
nodejs_version = settings.RTD_DOCKER_BUILD_SETTINGS['tools']['nodejs']['16']
658+
rust_version = settings.RTD_DOCKER_BUILD_SETTINGS['tools']['rust']['1.55']
659+
golang_version = settings.RTD_DOCKER_BUILD_SETTINGS['tools']['golang']['1.17']
660+
self.assertEqual(
661+
# NOTE: casting the first argument as `list()` shows a better diff
662+
# explaining where the problem is
663+
list(build_run.call_args_list),
664+
[
665+
mock.call(
666+
'mv',
667+
# Use mock.ANY here because path differs when ran locally
668+
# and on CircleCI
669+
mock.ANY,
670+
f'/home/docs/.asdf/installs/python/{python_version}',
671+
),
672+
mock.call('asdf', 'global', 'python', python_version),
673+
mock.call('asdf', 'reshim', 'python'),
674+
mock.call(
675+
'mv',
676+
mock.ANY,
677+
f'/home/docs/.asdf/installs/nodejs/{nodejs_version}',
678+
),
679+
mock.call('asdf', 'global', 'nodejs', nodejs_version),
680+
mock.call('asdf', 'reshim', 'nodejs'),
681+
mock.call(
682+
'mv',
683+
mock.ANY,
684+
f'/home/docs/.asdf/installs/rust/{rust_version}',
685+
),
686+
mock.call('asdf', 'global', 'rust', rust_version),
687+
mock.call('asdf', 'reshim', 'rust'),
688+
mock.call(
689+
'mv',
690+
mock.ANY,
691+
f'/home/docs/.asdf/installs/golang/{golang_version}',
692+
),
693+
mock.call('asdf', 'global', 'golang', golang_version),
694+
mock.call('asdf', 'reshim', 'golang'),
695+
mock.ANY,
696+
],
697+
)

‎readthedocs/settings/base.py

+1
Original file line numberDiff line numberDiff line change
@@ -311,6 +311,7 @@ def USE_PROMOS(self): # noqa
311311
# https://docs.readthedocs.io/page/development/settings.html#rtd-build-media-storage
312312
RTD_BUILD_MEDIA_STORAGE = 'readthedocs.builds.storage.BuildMediaFileSystemStorage'
313313
RTD_BUILD_ENVIRONMENT_STORAGE = 'readthedocs.builds.storage.BuildMediaFileSystemStorage'
314+
RTD_BUILD_TOOLS_STORAGE = 'readthedocs.builds.storage.BuildMediaFileSystemStorage'
314315
RTD_BUILD_COMMANDS_STORAGE = 'readthedocs.builds.storage.BuildMediaFileSystemStorage'
315316

316317
@property

0 commit comments

Comments
 (0)