Skip to content

Commit 2286206

Browse files
humitoshumitos
committed
Build: decouple install_build_tools from PythonEnvironment
The method `install_build_tools` is not tied to the Python environment, and it's more related to the build's director, instead. I'm moving it to the `BuildDirector` class also because it's needed there when using `build.commands` as the first step to execute to install the `build.tools` specified by the user to build their docs.
1 parent 5365087 commit 2286206

File tree

4 files changed

+141
-137
lines changed

4 files changed

+141
-137
lines changed

‎readthedocs/config/config.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -793,7 +793,7 @@ def validate_build_config_with_tools(self):
793793

794794
commands = []
795795
with self.catch_validation_error("build.commands"):
796-
commands = self.pop_config("build.commands")
796+
commands = self.pop_config("build.commands", default=[])
797797
validate_list(commands)
798798

799799
if not tools:

‎readthedocs/doc_builder/director.py

Lines changed: 137 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import os
22
import shutil
3+
import tarfile
34
from collections import defaultdict
45

56
import structlog
@@ -13,6 +14,7 @@
1314
from readthedocs.projects.exceptions import RepositoryError
1415
from readthedocs.projects.models import Feature
1516
from readthedocs.projects.signals import after_build, before_build, before_vcs
17+
from readthedocs.storage import build_tools_storage
1618

1719
log = structlog.get_logger(__name__)
1820

@@ -157,7 +159,7 @@ def setup_environment(self):
157159

158160
# Install all ``build.tools`` specified by the user
159161
if self.data.config.using_build_tools:
160-
self.language_environment.install_build_tools()
162+
self.install_build_tools()
161163

162164
self.run_build_job("pre_create_environment")
163165
self.create_environment()
@@ -354,6 +356,140 @@ def run_build_commands(self):
354356
# ignore=shutil.ignore_patterns(*self.ignore_patterns),
355357
)
356358

359+
def install_build_tools(self):
360+
"""
361+
Install all ``build.tools`` defined by the user in the config file.
362+
363+
It uses ``asdf`` behind the scenes to manage all the tools and versions
364+
of them. These tools/versions are stored in the Cloud cache and are
365+
downloaded on each build (~50 - ~100Mb).
366+
367+
If the requested tool/version is not present in the cache, it's
368+
installed via ``asdf`` on the fly.
369+
"""
370+
if settings.RTD_DOCKER_COMPOSE:
371+
# Create a symlink for ``root`` user to use the same ``.asdf``
372+
# installation as the ``docs`` user. Required for local building
373+
# since everything is run as ``root`` when using Local Development
374+
# instance
375+
cmd = [
376+
"ln",
377+
"-s",
378+
os.path.join(settings.RTD_DOCKER_WORKDIR, ".asdf"),
379+
"/root/.asdf",
380+
]
381+
self.build_environment.run(
382+
*cmd,
383+
record=False,
384+
)
385+
386+
for tool, version in self.data.config.build.tools.items():
387+
full_version = version.full_version # e.g. 3.9 -> 3.9.7
388+
389+
# TODO: generate the correct path for the Python version
390+
# see https://github.com/readthedocs/readthedocs.org/pull/8447#issuecomment-911562267
391+
# tool_path = f'{self.config.build.os}/{tool}/2021-08-30/{full_version}.tar.gz'
392+
tool_path = f"{self.data.config.build.os}-{tool}-{full_version}.tar.gz"
393+
tool_version_cached = build_tools_storage.exists(tool_path)
394+
if tool_version_cached:
395+
remote_fd = build_tools_storage.open(tool_path, mode="rb")
396+
with tarfile.open(fileobj=remote_fd) as tar:
397+
# Extract it on the shared path between host and Docker container
398+
extract_path = os.path.join(self.data.project.doc_path, "tools")
399+
tar.extractall(extract_path)
400+
401+
# Move the extracted content to the ``asdf`` installation
402+
cmd = [
403+
"mv",
404+
f"{extract_path}/{full_version}",
405+
os.path.join(
406+
settings.RTD_DOCKER_WORKDIR,
407+
f".asdf/installs/{tool}/{full_version}",
408+
),
409+
]
410+
self.build_environment.run(
411+
*cmd,
412+
record=False,
413+
)
414+
else:
415+
log.debug(
416+
"Cached version for tool not found.",
417+
os=self.data.config.build.os,
418+
tool=tool,
419+
full_version=full_version,
420+
tool_path=tool_path,
421+
)
422+
# If the tool version selected is not available from the
423+
# cache we compile it at build time
424+
cmd = [
425+
# TODO: make ``PYTHON_CONFIGURE_OPTS="--enable-shared"``
426+
# environment variable to work here. Note that
427+
# ``self.build_environment.run`` does not support passing
428+
# environment for a particular command:
429+
# https://github.com/readthedocs/readthedocs.org/blob/9d2d1a2/readthedocs/doc_builder/environments.py#L430-L431
430+
"asdf",
431+
"install",
432+
tool,
433+
full_version,
434+
]
435+
self.build_environment.run(
436+
*cmd,
437+
)
438+
439+
# Make the tool version chosen by the user the default one
440+
cmd = [
441+
"asdf",
442+
"global",
443+
tool,
444+
full_version,
445+
]
446+
self.build_environment.run(
447+
*cmd,
448+
)
449+
450+
# Recreate shims for this tool to make the new version
451+
# installed available
452+
# https://asdf-vm.com/learn-more/faq.html#newly-installed-exectable-not-running
453+
cmd = [
454+
"asdf",
455+
"reshim",
456+
tool,
457+
]
458+
self.build_environment.run(
459+
*cmd,
460+
record=False,
461+
)
462+
463+
if all(
464+
[
465+
tool == "python",
466+
# Do not install them if the tool version was cached
467+
# because these dependencies are already installed when
468+
# created with our script and uploaded to the cache's
469+
# bucket
470+
not tool_version_cached,
471+
# Do not install them on conda/mamba since they are not
472+
# needed because the environment is managed by conda/mamba
473+
# itself
474+
self.data.config.python_interpreter not in ("conda", "mamba"),
475+
]
476+
):
477+
# Install our own requirements if the version is compiled
478+
cmd = [
479+
"python",
480+
"-mpip",
481+
"install",
482+
"-U",
483+
"virtualenv",
484+
# We cap setuptools to avoid breakage of projects
485+
# relying on setup.py invokations,
486+
# see https://github.com/readthedocs/readthedocs.org/issues/8659
487+
"setuptools<58.3.0",
488+
]
489+
self.build_environment.run(
490+
*cmd,
491+
)
492+
357493
# Helpers
358494
#
359495
# TODO: move somewhere or change names to make them private or something to

‎readthedocs/doc_builder/python_environments.py

Lines changed: 0 additions & 135 deletions
Original file line numberDiff line numberDiff line change
@@ -4,19 +4,16 @@
44
import copy
55
import itertools
66
import os
7-
import tarfile
87

98
import structlog
109
import yaml
11-
from django.conf import settings
1210

1311
from readthedocs.config import PIP, SETUPTOOLS, ParseError
1412
from readthedocs.config import parse as parse_yaml
1513
from readthedocs.config.models import PythonInstall, PythonInstallRequirements
1614
from readthedocs.doc_builder.config import load_yaml_config
1715
from readthedocs.doc_builder.loader import get_builder_class
1816
from readthedocs.projects.models import Feature
19-
from readthedocs.storage import build_tools_storage
2017

2118
log = structlog.get_logger(__name__)
2219

@@ -40,138 +37,6 @@ def __init__(self, version, build_env, config=None):
4037
version_slug=self.version.slug,
4138
)
4239

43-
def install_build_tools(self):
44-
"""
45-
Install all ``build.tools`` defined by the user in the config file.
46-
47-
It uses ``asdf`` behind the scenes to manage all the tools and versions
48-
of them. These tools/versions are stored in the Cloud cache and are
49-
downloaded on each build (~50 - ~100Mb).
50-
51-
If the requested tool/version is not present in the cache, it's
52-
installed via ``asdf`` on the fly.
53-
"""
54-
if settings.RTD_DOCKER_COMPOSE:
55-
# Create a symlink for ``root`` user to use the same ``.asdf``
56-
# installation as the ``docs`` user. Required for local building
57-
# since everything is run as ``root`` when using Local Development
58-
# instance
59-
cmd = [
60-
'ln',
61-
'-s',
62-
os.path.join(settings.RTD_DOCKER_WORKDIR, '.asdf'),
63-
'/root/.asdf',
64-
]
65-
self.build_env.run(
66-
*cmd,
67-
record=False,
68-
)
69-
70-
for tool, version in self.config.build.tools.items():
71-
full_version = version.full_version # e.g. 3.9 -> 3.9.7
72-
73-
# TODO: generate the correct path for the Python version
74-
# see https://github.com/readthedocs/readthedocs.org/pull/8447#issuecomment-911562267
75-
# tool_path = f'{self.config.build.os}/{tool}/2021-08-30/{full_version}.tar.gz'
76-
tool_path = f'{self.config.build.os}-{tool}-{full_version}.tar.gz'
77-
tool_version_cached = build_tools_storage.exists(tool_path)
78-
if tool_version_cached:
79-
remote_fd = build_tools_storage.open(tool_path, mode='rb')
80-
with tarfile.open(fileobj=remote_fd) as tar:
81-
# Extract it on the shared path between host and Docker container
82-
extract_path = os.path.join(self.project.doc_path, 'tools')
83-
tar.extractall(extract_path)
84-
85-
# Move the extracted content to the ``asdf`` installation
86-
cmd = [
87-
'mv',
88-
f'{extract_path}/{full_version}',
89-
os.path.join(
90-
settings.RTD_DOCKER_WORKDIR,
91-
f'.asdf/installs/{tool}/{full_version}',
92-
),
93-
]
94-
self.build_env.run(
95-
*cmd,
96-
record=False,
97-
)
98-
else:
99-
log.debug(
100-
'Cached version for tool not found.',
101-
os=self.config.build.os,
102-
tool=tool,
103-
full_version=full_version,
104-
tool_path=tool_path,
105-
)
106-
# If the tool version selected is not available from the
107-
# cache we compile it at build time
108-
cmd = [
109-
# TODO: make ``PYTHON_CONFIGURE_OPTS="--enable-shared"``
110-
# environment variable to work here. Note that
111-
# ``self.build_env.run`` does not support passing
112-
# environment for a particular command:
113-
# https://github.com/readthedocs/readthedocs.org/blob/9d2d1a2/readthedocs/doc_builder/environments.py#L430-L431
114-
'asdf',
115-
'install',
116-
tool,
117-
full_version,
118-
]
119-
self.build_env.run(
120-
*cmd,
121-
)
122-
123-
# Make the tool version chosen by the user the default one
124-
cmd = [
125-
'asdf',
126-
'global',
127-
tool,
128-
full_version,
129-
]
130-
self.build_env.run(
131-
*cmd,
132-
)
133-
134-
# Recreate shims for this tool to make the new version
135-
# installed available
136-
# https://asdf-vm.com/learn-more/faq.html#newly-installed-exectable-not-running
137-
cmd = [
138-
'asdf',
139-
'reshim',
140-
tool,
141-
]
142-
self.build_env.run(
143-
*cmd,
144-
record=False,
145-
)
146-
147-
if all([
148-
tool == 'python',
149-
# Do not install them if the tool version was cached
150-
# because these dependencies are already installed when
151-
# created with our script and uploaded to the cache's
152-
# bucket
153-
not tool_version_cached,
154-
# Do not install them on conda/mamba since they are not
155-
# needed because the environment is managed by conda/mamba
156-
# itself
157-
self.config.python_interpreter not in ('conda', 'mamba'),
158-
]):
159-
# Install our own requirements if the version is compiled
160-
cmd = [
161-
'python',
162-
'-mpip',
163-
'install',
164-
'-U',
165-
'virtualenv',
166-
# We cap setuptools to avoid breakage of projects
167-
# relying on setup.py invokations,
168-
# see https://github.com/readthedocs/readthedocs.org/issues/8659
169-
'setuptools<58.3.0',
170-
]
171-
self.build_env.run(
172-
*cmd,
173-
)
174-
17540
def install_requirements(self):
17641
"""Install all requirements from the config object."""
17742
for install in self.config.python.install:

‎readthedocs/projects/tasks/builds.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -604,6 +604,9 @@ def execute(self):
604604
try:
605605
# NOTE: check if the build uses `build.commands` and only run those
606606
if self.data.config.build.commands:
607+
self.update_build(state=BUILD_STATE_INSTALLING)
608+
self.data.build_director.install_build_tools()
609+
607610
self.update_build(state=BUILD_STATE_BUILDING)
608611
self.data.build_director.run_build_commands()
609612

0 commit comments

Comments
 (0)