Skip to content

Commit a52fa2c

Browse files
committed
Build: implment build.jobs config file key
Allow people to use `build.jobs` to execute pre/post steps. ```yaml build: jobs: pre_install: - echo `date` - python path/to/myscript.py pre_build: - sed -i **/*.rst -e "s|{version}|v3.5.1|g" ```
1 parent 902590d commit a52fa2c

File tree

3 files changed

+84
-29
lines changed

3 files changed

+84
-29
lines changed

readthedocs/config/config.py

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
from .find import find_one
1616
from .models import (
1717
Build,
18+
BuildJobs,
1819
BuildTool,
1920
BuildWithTools,
2021
Conda,
@@ -751,6 +752,10 @@ def validate_conda(self):
751752
conda['environment'] = validate_path(environment, self.base_path)
752753
return conda
753754

755+
# NOTE: I think we should rename `BuildWithTools` to `BuildWithOs` since
756+
# `os` is the main and mandatory key that makes the diference
757+
#
758+
# NOTE: `build.jobs` can't be used without using `build.os`
754759
def validate_build_config_with_tools(self):
755760
"""
756761
Validates the build object (new format).
@@ -769,6 +774,22 @@ def validate_build_config_with_tools(self):
769774
for tool in tools.keys():
770775
validate_choice(tool, self.settings['tools'].keys())
771776

777+
jobs = {}
778+
with self.catch_validation_error("build.jobs"):
779+
# FIXME: should we use `default={}` or kept the `None` here and
780+
# shortcircuit the rest of the logic?
781+
jobs = self.pop_config("build.jobs", default={})
782+
validate_dict(jobs)
783+
# NOTE: besides validating that each key is one of the expected
784+
# ones, we could validate the value of each of them is a list of
785+
# commands. However, I don't think we should validate the "command"
786+
# looks like a real command.
787+
for job in jobs.keys():
788+
validate_choice(
789+
job,
790+
BuildJobs.__slots__,
791+
)
792+
772793
if not tools:
773794
self.error(
774795
key='build.tools',
@@ -780,6 +801,16 @@ def validate_build_config_with_tools(self):
780801
code=CONFIG_REQUIRED,
781802
)
782803

804+
build["jobs"] = {}
805+
for job in BuildJobs.__slots__:
806+
build["jobs"][job] = []
807+
808+
for job, commands in jobs.items():
809+
with self.catch_validation_error(f"build.jobs.{job}"):
810+
build["jobs"][job] = [
811+
validate_string(command) for command in validate_list(commands)
812+
]
813+
783814
build['tools'] = {}
784815
for tool, version in tools.items():
785816
with self.catch_validation_error(f'build.tools.{tool}'):
@@ -1263,7 +1294,10 @@ def build(self):
12631294
return BuildWithTools(
12641295
os=build['os'],
12651296
tools=tools,
1266-
apt_packages=build['apt_packages'],
1297+
jobs=BuildJobs(
1298+
**{job: commands for job, commands in build["jobs"].items()}
1299+
),
1300+
apt_packages=build["apt_packages"],
12671301
)
12681302
return Build(**build)
12691303

readthedocs/config/models.py

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ def __init__(self, **kwargs):
3737

3838
class BuildWithTools(Base):
3939

40-
__slots__ = ('os', 'tools', 'apt_packages')
40+
__slots__ = ("os", "tools", "jobs", "apt_packages")
4141

4242
def __init__(self, **kwargs):
4343
kwargs.setdefault('apt_packages', [])
@@ -49,6 +49,22 @@ class BuildTool(Base):
4949
__slots__ = ('version', 'full_version')
5050

5151

52+
class BuildJobs(Base):
53+
54+
__slots__ = (
55+
"pre_checkout",
56+
"post_checkout",
57+
"pre_system_dependencies",
58+
"post_system_dependencies",
59+
"pre_create_environment",
60+
"post_create_environment",
61+
"pre_install",
62+
"post_install",
63+
"pre_build",
64+
"post_build",
65+
)
66+
67+
5268
class Python(Base):
5369

5470
__slots__ = ('version', 'install', 'use_system_site_packages')

readthedocs/doc_builder/director.py

Lines changed: 32 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import os
2+
import shlex
23
from collections import defaultdict
34

45
import structlog
@@ -65,7 +66,7 @@ def setup_vcs(self):
6566
),
6667
)
6768

68-
environment = self.data.environment_class(
69+
self.vcs_environment = self.data.environment_class(
6970
project=self.data.project,
7071
version=self.data.version,
7172
build=self.data.build,
@@ -74,17 +75,17 @@ def setup_vcs(self):
7475
# ca-certificate package which is compatible with Lets Encrypt
7576
container_image=settings.RTD_DOCKER_BUILD_SETTINGS["os"]["ubuntu-20.04"],
7677
)
77-
with environment:
78+
with self.vcs_environment:
7879
before_vcs.send(
7980
sender=self.data.version,
80-
environment=environment,
81+
environment=self.vcs_environment,
8182
)
8283

8384
# Create the VCS repository where all the commands are going to be
8485
# executed for a particular VCS type
8586
self.vcs_repository = self.data.project.vcs_repo(
8687
version=self.data.version.slug,
87-
environment=environment,
88+
environment=self.vcs_environment,
8889
verbose_name=self.data.version.verbose_name,
8990
version_type=self.data.version.type,
9091
)
@@ -207,15 +208,17 @@ def checkout(self):
207208
self.vcs_repository.update_submodules(self.data.config)
208209

209210
def post_checkout(self):
210-
commands = [] # self.data.config.build.jobs.post_checkout
211+
commands = self.data.config.build.jobs.post_checkout
211212
for command in commands:
212-
self.build_environment.run(command)
213+
# TODO: we could make a helper `self.run(environment, command)`
214+
# that handles split and escape command
215+
self.vcs_environment.run(*shlex.split(command), escape_command=False)
213216

214217
# System dependencies (``build.apt_packages``)
215218
def pre_system_dependencies(self):
216-
commands = [] # self.data.config.build.jobs.pre_system_dependencies
219+
commands = self.data.config.build.jobs.pre_system_dependencies
217220
for command in commands:
218-
self.build_environment.run(command)
221+
self.build_environment.run(*shlex.split(command), escape_command=False)
219222

220223
# NOTE: `system_dependencies` should not be possible to override by the
221224
# user because it's executed as ``RTD_DOCKER_USER`` (e.g. ``root``) user.
@@ -253,58 +256,60 @@ def system_dependencies(self):
253256
)
254257

255258
def post_system_dependencies(self):
256-
pass
259+
commands = self.data.config.build.jobs.post_system_dependencies
260+
for command in commands:
261+
self.build_environment.run(*shlex.split(command), escape_command=False)
257262

258263
# Language environment
259264
def pre_create_environment(self):
260-
commands = [] # self.data.config.build.jobs.pre_create_environment
265+
commands = self.data.config.build.jobs.pre_create_environment
261266
for command in commands:
262-
self.build_environment.run(command)
267+
self.build_environment.run(*shlex.split(command), escape_command=False)
263268

264269
def create_environment(self):
265270
commands = [] # self.data.config.build.jobs.create_environment
266271
for command in commands:
267-
self.build_environment.run(command)
272+
self.build_environment.run(*shlex.split(command), escape_command=False)
268273

269274
if not commands:
270275
self.language_environment.setup_base()
271276

272277
def post_create_environment(self):
273-
commands = [] # self.data.config.build.jobs.post_create_environment
278+
commands = self.data.config.build.jobs.post_create_environment
274279
for command in commands:
275-
self.build_environment.run(command)
280+
self.build_environment.run(*shlex.split(command), escape_command=False)
276281

277282
# Install
278283
def pre_install(self):
279-
commands = [] # self.data.config.build.jobs.pre_install
284+
commands = self.data.config.build.jobs.pre_install
280285
for command in commands:
281-
self.build_environment.run(command)
286+
self.build_environment.run(*shlex.split(command), escape_command=False)
282287

283288
def install(self):
284289
commands = [] # self.data.config.build.jobs.install
285290
for command in commands:
286-
self.build_environment.run(command)
291+
self.build_environment.run(*shlex.split(command), escape_command=False)
287292

288293
if not commands:
289294
self.language_environment.install_core_requirements()
290295
self.language_environment.install_requirements()
291296

292297
def post_install(self):
293-
commands = [] # self.data.config.build.jobs.post_install
298+
commands = self.data.config.build.jobs.post_install
294299
for command in commands:
295-
self.build_environment.run(command)
300+
self.build_environment.run(*shlex.split(command), escape_command=False)
296301

297302
# Build
298303
def pre_build(self):
299-
commands = [] # self.data.config.build.jobs.pre_build
304+
commands = self.data.config.build.jobs.pre_build
300305
for command in commands:
301-
self.build_environment.run(command)
306+
self.build_environment.run(*shlex.split(command), escape_command=False)
302307

303308
def build_html(self):
304309
commands = [] # self.data.config.build.jobs.build.html
305310
if commands:
306311
for command in commands:
307-
self.build_environment.run(command)
312+
self.build_environment.run(*shlex.split(command), escape_command=False)
308313
return True
309314

310315
return self.build_docs_class(self.data.config.doctype)
@@ -316,7 +321,7 @@ def build_pdf(self):
316321
commands = [] # self.data.config.build.jobs.build.pdf
317322
if commands:
318323
for command in commands:
319-
self.build_environment.run(command)
324+
self.build_environment.run(*shlex.split(command), escape_command=False)
320325
return True
321326

322327
# Mkdocs has no pdf generation currently.
@@ -335,7 +340,7 @@ def build_htmlzip(self):
335340
commands = [] # self.data.config.build.jobs.build.htmlzip
336341
if commands:
337342
for command in commands:
338-
self.build_environment.run(command)
343+
self.build_environment.run(*shlex.split(command), escape_command=False)
339344
return True
340345

341346
# We don't generate a zip for mkdocs currently.
@@ -350,7 +355,7 @@ def build_epub(self):
350355
commands = [] # self.data.config.build.jobs.build.epub
351356
if commands:
352357
for command in commands:
353-
self.build_environment.run(command)
358+
self.build_environment.run(*shlex.split(command), escape_command=False)
354359
return True
355360

356361
# Mkdocs has no epub generation currently.
@@ -359,9 +364,9 @@ def build_epub(self):
359364
return False
360365

361366
def post_build(self):
362-
commands = [] # self.data.config.build.jobs.post_build
367+
commands = self.data.config.build.jobs.post_build
363368
for command in commands:
364-
self.build_environment.run(command)
369+
self.build_environment.run(*shlex.split(command), escape_command=False)
365370

366371
# Helpers
367372
#

0 commit comments

Comments
 (0)