diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index e2307b23f8f..68b4d0f62ca 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -6,6 +6,7 @@ import os import re from contextlib import contextmanager +from functools import lru_cache from django.conf import settings @@ -14,6 +15,8 @@ from .find import find_one from .models import ( Build, + BuildTool, + BuildWithTools, Conda, Mkdocs, Python, @@ -252,14 +255,33 @@ def pop_config(self, key, default=None, raise_ex=False): def validate(self): raise NotImplementedError() + @property + def using_build_tools(self): + return isinstance(self.build, BuildWithTools) + @property def python_interpreter(self): + if self.using_build_tools: + tool = self.build.tools.get('python') + if tool and tool.version.startswith('mamba'): + return 'mamba' + if tool and tool.version.startswith('miniconda'): + return 'conda' + if tool: + return 'python' + return None version = self.python_full_version if version.startswith('pypy'): # Allow to specify ``pypy3.5`` as Python interpreter return version return f'python{version}' + @property + def docker_image(self): + if self.using_build_tools: + return self.settings['os'][self.build.os] + return self.build.image + @property def python_full_version(self): version = self.python.version @@ -618,6 +640,7 @@ def conda(self): return None @property + @lru_cache(maxsize=1) def build(self): """The docker image used by the builders.""" return Build(**self._config['build']) @@ -671,6 +694,10 @@ class BuildConfigV2(BuildConfigBase): 'singlehtml': 'sphinx_singlehtml', } + @property + def settings(self): + return settings.RTD_DOCKER_BUILD_SETTINGS + def validate(self): """ Validates and process ``raw_config`` and ``env_config``. @@ -723,15 +750,52 @@ def validate_conda(self): conda['environment'] = validate_path(environment, self.base_path) return conda - def validate_build(self): + def validate_build_config_with_tools(self): """ - Validates the build object. + Validates the build object (new format). + + At least one element must be provided in ``build.tools``. + """ + build = {} + with self.catch_validation_error('build.os'): + build_os = self.pop_config('build.os', raise_ex=True) + build['os'] = validate_choice(build_os, self.settings['os'].keys()) + + tools = {} + with self.catch_validation_error('build.tools'): + tools = self.pop_config('build.tools') + validate_dict(tools) + for tool in tools.keys(): + validate_choice(tool, self.settings['tools'].keys()) + + if not tools: + self.error( + key='build.tools', + message=( + 'At least one tools of [{}] must be provided.'.format( + ' ,'.join(self.settings['tools'].keys()) + ) + ), + code=CONFIG_REQUIRED, + ) + + build['tools'] = {} + for tool, version in tools.items(): + with self.catch_validation_error(f'build.tools.{tool}'): + build['tools'][tool] = validate_choice( + version, + self.settings['tools'][tool].keys(), + ) + + build['apt_packages'] = self.validate_apt_packages() + return build + + def validate_old_build_config(self): + """ + Validates the build object (old format). It prioritizes the value from the default image if exists. """ - raw_build = self._raw_config.get('build', {}) - with self.catch_validation_error('build'): - validate_dict(raw_build) build = {} with self.catch_validation_error('build.image'): image = self.pop_config('build.image', self.default_build_image) @@ -748,6 +812,11 @@ def validate_build(self): if config_image: build['image'] = config_image + build['apt_packages'] = self.validate_apt_packages() + return build + + def validate_apt_packages(self): + apt_packages = [] with self.catch_validation_error('build.apt_packages'): raw_packages = self._raw_config.get('build', {}).get('apt_packages', []) validate_list(raw_packages) @@ -756,14 +825,22 @@ def validate_build(self): list_to_dict(raw_packages) ) - build['apt_packages'] = [ + apt_packages = [ self.validate_apt_package(index) for index in range(len(raw_packages)) ] if not raw_packages: self.pop_config('build.apt_packages') - return build + return apt_packages + + def validate_build(self): + raw_build = self._raw_config.get('build', {}) + with self.catch_validation_error('build'): + validate_dict(raw_build) + if 'os' in raw_build: + return self.validate_build_config_with_tools() + return self.validate_old_build_config() def validate_apt_package(self, index): """ @@ -821,24 +898,27 @@ def validate_python(self): .. note:: - ``version`` can be a string or number type. - ``extra_requirements`` needs to be used with ``install: 'pip'``. + - If the new build config is used (``build.os``), + ``python.version`` shouldn't exist. """ raw_python = self._raw_config.get('python', {}) with self.catch_validation_error('python'): validate_dict(raw_python) python = {} - with self.catch_validation_error('python.version'): - version = self.pop_config('python.version', '3') - if version == 3.1: - # Special case for ``python.version: 3.10``, - # yaml will transform this to the numeric value of `3.1`. - # Save some frustration to users. - version = '3.10' - version = str(version) - python['version'] = validate_choice( - version, - self.get_valid_python_versions(), - ) + if not self.using_build_tools: + with self.catch_validation_error('python.version'): + version = self.pop_config('python.version', '3') + if version == 3.1: + # Special case for ``python.version: 3.10``, + # yaml will transform this to the numeric value of `3.1`. + # Save some frustration to users. + version = '3.10' + version = str(version) + python['version'] = validate_choice( + version, + self.get_valid_python_versions(), + ) with self.catch_validation_error('python.install'): raw_install = self._raw_config.get('python', {}).get('install', []) @@ -1172,8 +1252,23 @@ def conda(self): return None @property + @lru_cache(maxsize=1) def build(self): - return Build(**self._config['build']) + build = self._config['build'] + if 'os' in build: + tools = { + tool: BuildTool( + version=version, + full_version=self.settings['tools'][tool][version], + ) + for tool, version in build['tools'].items() + } + return BuildWithTools( + os=build['os'], + tools=tools, + apt_packages=build['apt_packages'], + ) + return Build(**build) @property def python(self): @@ -1185,7 +1280,7 @@ def python(self): elif 'path' in install: python_install.append(PythonInstall(**install),) return Python( - version=python['version'], + version=python.get('version'), install=python_install, use_system_site_packages=python['use_system_site_packages'], ) diff --git a/readthedocs/config/models.py b/readthedocs/config/models.py index 95159940f0f..8e0ed70881a 100644 --- a/readthedocs/config/models.py +++ b/readthedocs/config/models.py @@ -35,6 +35,20 @@ def __init__(self, **kwargs): super().__init__(**kwargs) +class BuildWithTools(Base): + + __slots__ = ('os', 'tools', 'apt_packages') + + def __init__(self, **kwargs): + kwargs.setdefault('apt_packages', []) + super().__init__(**kwargs) + + +class BuildTool(Base): + + __slots__ = ('version', 'full_version') + + class Python(Base): __slots__ = ('version', 'install', 'use_system_site_packages') diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index 56d832befd6..4068c3c9ced 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -1,4 +1,5 @@ import os +from django.conf import settings import re import textwrap from collections import OrderedDict @@ -31,6 +32,8 @@ VERSION_INVALID, ) from readthedocs.config.models import ( + Build, + BuildWithTools, Conda, PythonInstall, PythonInstallRequirements, @@ -904,6 +907,8 @@ def test_build_image_over_empty_default(self, image): def test_build_image_default_value(self): build = self.get_build_config({}) build.validate() + assert not build.using_build_tools + assert isinstance(build.build, Build) assert build.build.image == 'readthedocs/build:latest' @pytest.mark.parametrize('value', [3, [], 'invalid']) @@ -920,6 +925,93 @@ def test_build_image_check_invalid_type(self, value): build.validate() assert excinfo.value.key == 'build.image' + @pytest.mark.parametrize('value', ['', None, 'latest']) + def test_new_build_config_invalid_os(self, value): + build = self.get_build_config( + { + 'build': { + 'os': value, + 'tools': {'python': '3'}, + }, + }, + ) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'build.os' + + @pytest.mark.parametrize('value', ['', None, 'python', ['python', 'nodejs'], {}, {'cobol': '99'}]) + def test_new_build_config_invalid_tools(self, value): + build = self.get_build_config( + { + 'build': { + 'os': 'ubuntu-20.04', + 'tools': value, + }, + }, + ) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'build.tools' + + def test_new_build_config_invalid_tools_version(self): + build = self.get_build_config( + { + 'build': { + 'os': 'ubuntu-20.04', + 'tools': {'python': '2.6'}, + }, + }, + ) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'build.tools.python' + + def test_new_build_config(self): + build = self.get_build_config( + { + 'build': { + 'os': 'ubuntu-20.04', + 'tools': {'python': '3.9'}, + }, + }, + ) + build.validate() + assert build.using_build_tools + assert isinstance(build.build, BuildWithTools) + assert build.build.os == 'ubuntu-20.04' + assert build.build.tools['python'].version == '3.9' + full_version = settings.RTD_DOCKER_BUILD_SETTINGS['tools']['python']['3.9'] + assert build.build.tools['python'].full_version == full_version + assert build.python_interpreter == 'python' + + def test_new_build_config_conflict_with_build_image(self): + build = self.get_build_config( + { + 'build': { + 'image': 'latest', + 'os': 'ubuntu-20.04', + 'tools': {'python': '3.9'}, + }, + }, + ) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'build.image' + + def test_new_build_config_conflict_with_build_python_version(self): + build = self.get_build_config( + { + 'build': { + 'os': 'ubuntu-20.04', + 'tools': {'python': '3.8'}, + }, + 'python': {'version': '3.8'}, + }, + ) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'python.version' + @pytest.mark.parametrize( 'value', [ @@ -2163,3 +2255,73 @@ def test_as_dict(self, tmpdir): }, } assert build.as_dict() == expected_dict + + def test_as_dict_new_build_config(self, tmpdir): + build = self.get_build_config( + { + 'version': 2, + 'formats': ['pdf'], + 'build': { + 'os': 'ubuntu-20.04', + 'tools': { + 'python': '3.9', + 'nodejs': '16', + }, + }, + 'python': { + 'install': [{ + 'requirements': 'requirements.txt', + }], + }, + }, + source_file=str(tmpdir.join('readthedocs.yml')), + ) + build.validate() + expected_dict = { + 'version': '2', + 'formats': ['pdf'], + 'python': { + 'version': None, + 'install': [{ + 'requirements': 'requirements.txt', + }], + 'use_system_site_packages': False, + }, + 'build': { + 'os': 'ubuntu-20.04', + 'tools': { + 'python': { + 'version': '3.9', + 'full_version': settings.RTD_DOCKER_BUILD_SETTINGS['tools']['python']['3.9'], + }, + 'nodejs': { + 'version': '16', + 'full_version': settings.RTD_DOCKER_BUILD_SETTINGS['tools']['nodejs']['16'], + }, + }, + 'apt_packages': [], + }, + 'conda': None, + 'sphinx': { + 'builder': 'sphinx', + 'configuration': None, + 'fail_on_warning': False, + }, + 'mkdocs': None, + 'doctype': 'sphinx', + 'submodules': { + 'include': [], + 'exclude': ALL, + 'recursive': False, + }, + 'search': { + 'ranking': {}, + 'ignore': [ + 'search.html', + 'search/index.html', + '404.html', + '404/index.html', + ], + }, + } + assert build.as_dict() == expected_dict diff --git a/readthedocs/rtd_tests/fixtures/spec/v2/schema.json b/readthedocs/rtd_tests/fixtures/spec/v2/schema.json index cb91c6b7e98..d0fe4bb8cdc 100644 --- a/readthedocs/rtd_tests/fixtures/spec/v2/schema.json +++ b/readthedocs/rtd_tests/fixtures/spec/v2/schema.json @@ -54,27 +54,97 @@ "title": "Build", "description": "Configuration for the documentation build process.", "type": "object", - "properties": { - "image": { - "title": "Image", - "description": "The build docker image to be used.", - "enum": [ - "stable", - "latest" - ], - "default": "latest" + "anyOf": [ + { + "properties": { + "image": { + "title": "Image", + "description": "The build docker image to be used.", + "enum": [ + "stable", + "latest" + ], + "default": "latest" + }, + "apt_packages": { + "title": "APT Packages", + "description": "List of packages to be installed with apt-get.", + "type": "array", + "items": { + "title": "APT Package", + "type": "string" + }, + "default": [] + } + }, + "additionalProperties": false }, - "apt_packages": { - "title": "APT Packages", - "description": "List of packages to be installed with apt-get.", - "type": "array", - "items": { - "title": "APT Package", - "type": "string" + { + "properties": { + "os": { + "title": "Operating System", + "description": "Operating system to be used in the build.", + "enum": [ + "ubuntu-20.04" + ] + }, + "tools": { + "title": "Tools", + "description": "Tools and their version to be used in the build.", + "type": "object", + "properties": { + "python": { + "enum": [ + "2.7", + "3", + "3.6", + "3.7", + "3.8", + "3.9", + "3.10", + "pypy3.7", + "miniconda3-4.7", + "mambaforge-4.10" + ] + }, + "nodejs": { + "enum": [ + "14", + "16" + ] + }, + "rust": { + "enum": [ + "1.55" + ] + }, + "golang": { + "enum": [ + "1.17" + ] + } + }, + "minProperties": 1, + "additionalProperties": false + }, + "apt_packages": { + "title": "APT Packages", + "description": "List of packages to be installed with apt-get.", + "type": "array", + "items": { + "title": "APT Package", + "type": "string" + }, + "default": [] + } }, - "default": [] + "required": [ + "os", + "tools" + ], + "additionalProperties": false } - } + ] }, "python": { "title": "Python", @@ -156,7 +226,8 @@ ] } } - } + }, + "additionalProperties": false }, "sphinx": { "title": "Sphix", @@ -184,7 +255,8 @@ "type": "boolean", "default": false } - } + }, + "additionalProperties": false }, "mkdocs": { "title": "mkdocs", @@ -202,7 +274,8 @@ "type": "boolean", "default": false } - } + }, + "additionalProperties": false }, "submodules": { "title": "Submodules", @@ -251,7 +324,8 @@ "type": "boolean", "default": false } - } + }, + "additionalProperties": false }, "search": { "title": "search", @@ -281,10 +355,12 @@ "404/index.html" ] } - } + }, + "additionalProperties": false } }, "required": [ "version" - ] + ], + "additionalProperties": false } diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index 85cf6f2cbb4..ab8e1be8883 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -520,6 +520,41 @@ def TEMPLATES(self): # Additional binds for the build container RTD_DOCKER_ADDITIONAL_BINDS = {} + # When updating this options, + # update the readthedocs/rtd_tests/fixtures/spec/v2/schema.json file as well. + RTD_DOCKER_BUILD_SETTINGS = { + # Mapping of build.os options to docker image. + 'os': { + 'ubuntu-20.04': f'{DOCKER_DEFAULT_IMAGE}:ubuntu20', + }, + # Mapping of build.tools options to specific versions. + 'tools': { + 'python': { + '2.7': '2.7.18', + '3.6': '3.6.15', + '3.7': '3.7.12', + '3.8': '3.8.12', + '3.9': '3.9.7', + '3.10': '3.10.0rc2', + 'pypy3.7': 'pypy3.7-7.3.5', + 'miniconda3-4.7': 'miniconda3-4.7.12', + 'mambaforge-4.10': 'mambaforge-4.10.1-5', + }, + 'nodejs': { + '14': '14.17.6', + '16': '16.9.1', + }, + 'rust': { + '1.55': '1.55.0', + }, + 'golang': { + '1.17': '1.17.1', + }, + }, + } + # Always point to the latest stable release. + RTD_DOCKER_BUILD_SETTINGS['tools']['python']['3'] = RTD_DOCKER_BUILD_SETTINGS['tools']['python']['3.9'] + def _get_docker_memory_limit(self): try: total_memory = int(subprocess.check_output(