From 4251717a629e28c5b1edcc69484439cfcee8abc6 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 9 Sep 2021 17:39:48 -0500 Subject: [PATCH 01/11] New config for new docker build images --- readthedocs/config/config.py | 114 ++++++++++--- readthedocs/config/models.py | 11 +- readthedocs/config/tests/test_config.py | 150 ++++++++++++++++++ .../rtd_tests/fixtures/spec/v2/schema.json | 122 +++++++++++--- 4 files changed, 350 insertions(+), 47 deletions(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index e2307b23f8f..d9b1c95d470 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -16,6 +16,7 @@ Build, Conda, Mkdocs, + OldBuild, Python, PythonInstall, PythonInstallRequirements, @@ -620,7 +621,7 @@ def conda(self): @property def build(self): """The docker image used by the builders.""" - return Build(**self._config['build']) + return OldBuild(**self._config['build']) @property def doctype(self): @@ -670,6 +671,19 @@ class BuildConfigV2(BuildConfigBase): 'dirhtml': 'sphinx_htmldir', 'singlehtml': 'sphinx_singlehtml', } + valid_os = ['ubuntu-20.04'] + valid_languages = { + 'python': [ + '2.7', + '3', '3.8', '3.9', '3.10', + 'pypy3.7', + 'miniconda3-4.7', + 'mambaforge-4.10', + ], + 'nodejs': ['14', '16'], + 'rust': ['1.55'], + 'golang': ['1.17'], + } def validate(self): """ @@ -723,15 +737,52 @@ def validate_conda(self): conda['environment'] = validate_path(environment, self.base_path) return conda - def validate_build(self): + def validate_new_build_config(self): + """ + Validates the build object (new format). + + At least one element must be provided in ``build.languages``. + """ + build = {} + with self.catch_validation_error('build.os'): + build_os = self.pop_config('build.os', raise_ex=True) + build['os'] = validate_choice(build_os, self.valid_os) + + languages = {} + with self.catch_validation_error('build.languages'): + languages = self.pop_config('build.languages') + validate_dict(languages) + for lang in languages.keys(): + validate_choice(lang, self.valid_languages.keys()) + + if not languages: + self.error( + key='build.languages', + message=( + 'At least one language of [{}] must be provided.'.format( + ' ,'.join(self.valid_languages.keys()) + ) + ), + code=CONFIG_REQUIRED, + ) + + build['languages'] = {} + for lang, version in languages.items(): + with self.catch_validation_error(f'build.languages.{lang}'): + build['languages'][lang] = validate_choice( + version, + self.valid_languages[lang], + ) + + build['apt_packages'] = self.validate_apt_packages() + return build + + def validate_old_build_config(self): """ - Validates the build object. + Validates the build object (old format). It prioritizes the value from the default image if exists. """ - raw_build = self._raw_config.get('build', {}) - with self.catch_validation_error('build'): - validate_dict(raw_build) build = {} with self.catch_validation_error('build.image'): image = self.pop_config('build.image', self.default_build_image) @@ -748,6 +799,11 @@ def validate_build(self): if config_image: build['image'] = config_image + build['apt_packages'] = self.validate_apt_packages() + return build + + def validate_apt_packages(self): + apt_packages = [] with self.catch_validation_error('build.apt_packages'): raw_packages = self._raw_config.get('build', {}).get('apt_packages', []) validate_list(raw_packages) @@ -756,14 +812,22 @@ def validate_build(self): list_to_dict(raw_packages) ) - build['apt_packages'] = [ + apt_packages = [ self.validate_apt_package(index) for index in range(len(raw_packages)) ] if not raw_packages: self.pop_config('build.apt_packages') - return build + return apt_packages + + def validate_build(self): + raw_build = self._raw_config.get('build', {}) + with self.catch_validation_error('build'): + validate_dict(raw_build) + if 'os' in raw_build: + return self.validate_new_build_config() + return self.validate_old_build_config() def validate_apt_package(self, index): """ @@ -821,24 +885,27 @@ def validate_python(self): .. note:: - ``version`` can be a string or number type. - ``extra_requirements`` needs to be used with ``install: 'pip'``. + - If the new build config is used (``build.os``), + ``python.version`` shouldn't exist. """ raw_python = self._raw_config.get('python', {}) with self.catch_validation_error('python'): validate_dict(raw_python) python = {} - with self.catch_validation_error('python.version'): - version = self.pop_config('python.version', '3') - if version == 3.1: - # Special case for ``python.version: 3.10``, - # yaml will transform this to the numeric value of `3.1`. - # Save some frustration to users. - version = '3.10' - version = str(version) - python['version'] = validate_choice( - version, - self.get_valid_python_versions(), - ) + if isinstance(self.build, OldBuild): + with self.catch_validation_error('python.version'): + version = self.pop_config('python.version', '3') + if version == 3.1: + # Special case for ``python.version: 3.10``, + # yaml will transform this to the numeric value of `3.1`. + # Save some frustration to users. + version = '3.10' + version = str(version) + python['version'] = validate_choice( + version, + self.get_valid_python_versions(), + ) with self.catch_validation_error('python.install'): raw_install = self._raw_config.get('python', {}).get('install', []) @@ -1173,7 +1240,10 @@ def conda(self): @property def build(self): - return Build(**self._config['build']) + klass = OldBuild + if 'os' in self._config['build']: + klass = Build + return klass(**self._config['build']) @property def python(self): @@ -1185,7 +1255,7 @@ def python(self): elif 'path' in install: python_install.append(PythonInstall(**install),) return Python( - version=python['version'], + version=python.get('version'), install=python_install, use_system_site_packages=python['use_system_site_packages'], ) diff --git a/readthedocs/config/models.py b/readthedocs/config/models.py index 95159940f0f..62a5da8ea27 100644 --- a/readthedocs/config/models.py +++ b/readthedocs/config/models.py @@ -26,7 +26,7 @@ def as_dict(self): } -class Build(Base): +class OldBuild(Base): __slots__ = ('image', 'apt_packages') @@ -35,6 +35,15 @@ def __init__(self, **kwargs): super().__init__(**kwargs) +class Build(Base): + + __slots__ = ('os', 'languages', 'apt_packages') + + def __init__(self, **kwargs): + kwargs.setdefault('apt_packages', []) + super().__init__(**kwargs) + + class Python(Base): __slots__ = ('version', 'install', 'use_system_site_packages') diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index 56d832befd6..c78298acc17 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -31,7 +31,9 @@ VERSION_INVALID, ) from readthedocs.config.models import ( + Build, Conda, + OldBuild, PythonInstall, PythonInstallRequirements, ) @@ -904,6 +906,7 @@ def test_build_image_over_empty_default(self, image): def test_build_image_default_value(self): build = self.get_build_config({}) build.validate() + assert isinstance(build.build, OldBuild) assert build.build.image == 'readthedocs/build:latest' @pytest.mark.parametrize('value', [3, [], 'invalid']) @@ -920,6 +923,89 @@ def test_build_image_check_invalid_type(self, value): build.validate() assert excinfo.value.key == 'build.image' + @pytest.mark.parametrize('value', ['', None, 'latest']) + def test_new_build_config_invalid_os(self, value): + build = self.get_build_config( + { + 'build': { + 'os': value, + 'languages': {'python': '3'}, + }, + }, + ) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'build.os' + + @pytest.mark.parametrize('value', ['', None, 'python', ['python', 'nodejs'], {}, {'cobol': '99'}]) + def test_new_build_config_invalid_languages(self, value): + build = self.get_build_config( + { + 'build': { + 'os': 'ubuntu-20.04', + 'languages': value, + }, + }, + ) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'build.languages' + + def test_new_build_config_invalid_languages_version(self): + build = self.get_build_config( + { + 'build': { + 'os': 'ubuntu-20.04', + 'languages': {'python': '2.6'}, + }, + }, + ) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'build.languages.python' + + def test_new_build_config(self): + build = self.get_build_config( + { + 'build': { + 'os': 'ubuntu-20.04', + 'languages': {'python': '3.9'}, + }, + }, + ) + build.validate() + assert isinstance(build.build, Build) + assert build.build.os == 'ubuntu-20.04' + assert build.build.languages == {'python': '3.9'} + + def test_new_build_config_conflict_with_build_image(self): + build = self.get_build_config( + { + 'build': { + 'image': 'latest', + 'os': 'ubuntu-20.04', + 'languages': {'python': '3.9'}, + }, + }, + ) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'build.image' + + def test_new_build_config_conflict_with_build_python_version(self): + build = self.get_build_config( + { + 'build': { + 'os': 'ubuntu-20.04', + 'languages': {'python': '3.8'}, + }, + 'python': {'version': '3.8'}, + }, + ) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'python.version' + @pytest.mark.parametrize( 'value', [ @@ -2163,3 +2249,67 @@ def test_as_dict(self, tmpdir): }, } assert build.as_dict() == expected_dict + + def test_as_dict_new_build_config(self, tmpdir): + build = self.get_build_config( + { + 'version': 2, + 'formats': ['pdf'], + 'build': { + 'os': 'ubuntu-20.04', + 'languages': { + 'python': '3.9', + 'nodejs': '16', + }, + }, + 'python': { + 'install': [{ + 'requirements': 'requirements.txt', + }], + }, + }, + source_file=str(tmpdir.join('readthedocs.yml')), + ) + build.validate() + expected_dict = { + 'version': '2', + 'formats': ['pdf'], + 'python': { + 'version': None, + 'install': [{ + 'requirements': 'requirements.txt', + }], + 'use_system_site_packages': False, + }, + 'build': { + 'os': 'ubuntu-20.04', + 'languages': { + 'python': '3.9', + 'nodejs': '16', + }, + 'apt_packages': [], + }, + 'conda': None, + 'sphinx': { + 'builder': 'sphinx', + 'configuration': None, + 'fail_on_warning': False, + }, + 'mkdocs': None, + 'doctype': 'sphinx', + 'submodules': { + 'include': [], + 'exclude': ALL, + 'recursive': False, + }, + 'search': { + 'ranking': {}, + 'ignore': [ + 'search.html', + 'search/index.html', + '404.html', + '404/index.html', + ], + }, + } + assert build.as_dict() == expected_dict diff --git a/readthedocs/rtd_tests/fixtures/spec/v2/schema.json b/readthedocs/rtd_tests/fixtures/spec/v2/schema.json index cb91c6b7e98..facecd6473f 100644 --- a/readthedocs/rtd_tests/fixtures/spec/v2/schema.json +++ b/readthedocs/rtd_tests/fixtures/spec/v2/schema.json @@ -54,27 +54,95 @@ "title": "Build", "description": "Configuration for the documentation build process.", "type": "object", - "properties": { - "image": { - "title": "Image", - "description": "The build docker image to be used.", - "enum": [ - "stable", - "latest" - ], - "default": "latest" + "anyOf": [ + { + "properties": { + "image": { + "title": "Image", + "description": "The build docker image to be used.", + "enum": [ + "stable", + "latest" + ], + "default": "latest" + }, + "apt_packages": { + "title": "APT Packages", + "description": "List of packages to be installed with apt-get.", + "type": "array", + "items": { + "title": "APT Package", + "type": "string" + }, + "default": [] + } + }, + "additionalProperties": false }, - "apt_packages": { - "title": "APT Packages", - "description": "List of packages to be installed with apt-get.", - "type": "array", - "items": { - "title": "APT Package", - "type": "string" + { + "properties": { + "os": { + "title": "Operating System", + "description": "Operating system to be used in the build.", + "enum": [ + "ubuntu-20.04" + ] + }, + "languages": { + "title": "Languages", + "description": "Languages and their version to be used in the build.", + "type": "object", + "properties": { + "python": { + "enum": [ + "2.7", + "3", + "3.8", + "3.9", + "3.10", + "pypy3.7", + "miniconda3-4.7", + "mambaforge-4.10" + ] + }, + "nodejs": { + "enum": [ + "14", + "16" + ] + }, + "rust": { + "enum": [ + "1.55" + ] + }, + "golang": { + "enum": [ + "1.17" + ] + } + }, + "minProperties": 1, + "additionalProperties": false + }, + "apt_packages": { + "title": "APT Packages", + "description": "List of packages to be installed with apt-get.", + "type": "array", + "items": { + "title": "APT Package", + "type": "string" + }, + "default": [] + } }, - "default": [] + "required": [ + "os", + "languages" + ], + "additionalProperties": false } - } + ] }, "python": { "title": "Python", @@ -156,7 +224,8 @@ ] } } - } + }, + "additionalProperties": false }, "sphinx": { "title": "Sphix", @@ -184,7 +253,8 @@ "type": "boolean", "default": false } - } + }, + "additionalProperties": false }, "mkdocs": { "title": "mkdocs", @@ -202,7 +272,8 @@ "type": "boolean", "default": false } - } + }, + "additionalProperties": false }, "submodules": { "title": "Submodules", @@ -251,7 +322,8 @@ "type": "boolean", "default": false } - } + }, + "additionalProperties": false }, "search": { "title": "search", @@ -281,10 +353,12 @@ "404/index.html" ] } - } + }, + "additionalProperties": false } }, "required": [ "version" - ] + ], + "additionalProperties": false } From 083492c5a791f60083b34e845267418e57a964c3 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 16 Sep 2021 15:08:07 -0500 Subject: [PATCH 02/11] Rename build.languages -> build.tools --- readthedocs/config/config.py | 51 ++++++++++--------------- readthedocs/config/models.py | 2 +- readthedocs/config/tests/test_config.py | 26 ++++++------- readthedocs/settings/base.py | 33 ++++++++++++++++ 4 files changed, 68 insertions(+), 44 deletions(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index d9b1c95d470..e6801d0f7fe 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -671,19 +671,10 @@ class BuildConfigV2(BuildConfigBase): 'dirhtml': 'sphinx_htmldir', 'singlehtml': 'sphinx_singlehtml', } - valid_os = ['ubuntu-20.04'] - valid_languages = { - 'python': [ - '2.7', - '3', '3.8', '3.9', '3.10', - 'pypy3.7', - 'miniconda3-4.7', - 'mambaforge-4.10', - ], - 'nodejs': ['14', '16'], - 'rust': ['1.55'], - 'golang': ['1.17'], - } + + @property + def settings(self): + return settings.RTD_DOCKER_BUILD_SETTINGS def validate(self): """ @@ -741,37 +732,37 @@ def validate_new_build_config(self): """ Validates the build object (new format). - At least one element must be provided in ``build.languages``. + At least one element must be provided in ``build.tools``. """ build = {} with self.catch_validation_error('build.os'): build_os = self.pop_config('build.os', raise_ex=True) - build['os'] = validate_choice(build_os, self.valid_os) + build['os'] = validate_choice(build_os, self.settings['os'].keys()) - languages = {} - with self.catch_validation_error('build.languages'): - languages = self.pop_config('build.languages') - validate_dict(languages) - for lang in languages.keys(): - validate_choice(lang, self.valid_languages.keys()) + tools = {} + with self.catch_validation_error('build.tools'): + tools = self.pop_config('build.tools') + validate_dict(tools) + for tool in tools.keys(): + validate_choice(tool, self.settings['tools'].keys()) - if not languages: + if not tools: self.error( - key='build.languages', + key='build.tools', message=( - 'At least one language of [{}] must be provided.'.format( - ' ,'.join(self.valid_languages.keys()) + 'At least one tools of [{}] must be provided.'.format( + ' ,'.join(self.settings['tools'].keys()) ) ), code=CONFIG_REQUIRED, ) - build['languages'] = {} - for lang, version in languages.items(): - with self.catch_validation_error(f'build.languages.{lang}'): - build['languages'][lang] = validate_choice( + build['tools'] = {} + for tool, version in tools.items(): + with self.catch_validation_error(f'build.tools.{tool}'): + build['tools'][tool] = validate_choice( version, - self.valid_languages[lang], + self.settings['tools'][tool].keys(), ) build['apt_packages'] = self.validate_apt_packages() diff --git a/readthedocs/config/models.py b/readthedocs/config/models.py index 62a5da8ea27..b2fcc37281b 100644 --- a/readthedocs/config/models.py +++ b/readthedocs/config/models.py @@ -37,7 +37,7 @@ def __init__(self, **kwargs): class Build(Base): - __slots__ = ('os', 'languages', 'apt_packages') + __slots__ = ('os', 'tools', 'apt_packages') def __init__(self, **kwargs): kwargs.setdefault('apt_packages', []) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index c78298acc17..e8646235935 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -929,7 +929,7 @@ def test_new_build_config_invalid_os(self, value): { 'build': { 'os': value, - 'languages': {'python': '3'}, + 'tools': {'python': '3'}, }, }, ) @@ -938,45 +938,45 @@ def test_new_build_config_invalid_os(self, value): assert excinfo.value.key == 'build.os' @pytest.mark.parametrize('value', ['', None, 'python', ['python', 'nodejs'], {}, {'cobol': '99'}]) - def test_new_build_config_invalid_languages(self, value): + def test_new_build_config_invalid_tools(self, value): build = self.get_build_config( { 'build': { 'os': 'ubuntu-20.04', - 'languages': value, + 'tools': value, }, }, ) with raises(InvalidConfig) as excinfo: build.validate() - assert excinfo.value.key == 'build.languages' + assert excinfo.value.key == 'build.tools' - def test_new_build_config_invalid_languages_version(self): + def test_new_build_config_invalid_tools_version(self): build = self.get_build_config( { 'build': { 'os': 'ubuntu-20.04', - 'languages': {'python': '2.6'}, + 'tools': {'python': '2.6'}, }, }, ) with raises(InvalidConfig) as excinfo: build.validate() - assert excinfo.value.key == 'build.languages.python' + assert excinfo.value.key == 'build.tools.python' def test_new_build_config(self): build = self.get_build_config( { 'build': { 'os': 'ubuntu-20.04', - 'languages': {'python': '3.9'}, + 'tools': {'python': '3.9'}, }, }, ) build.validate() assert isinstance(build.build, Build) assert build.build.os == 'ubuntu-20.04' - assert build.build.languages == {'python': '3.9'} + assert build.build.tools == {'python': '3.9'} def test_new_build_config_conflict_with_build_image(self): build = self.get_build_config( @@ -984,7 +984,7 @@ def test_new_build_config_conflict_with_build_image(self): 'build': { 'image': 'latest', 'os': 'ubuntu-20.04', - 'languages': {'python': '3.9'}, + 'tools': {'python': '3.9'}, }, }, ) @@ -997,7 +997,7 @@ def test_new_build_config_conflict_with_build_python_version(self): { 'build': { 'os': 'ubuntu-20.04', - 'languages': {'python': '3.8'}, + 'tools': {'python': '3.8'}, }, 'python': {'version': '3.8'}, }, @@ -2257,7 +2257,7 @@ def test_as_dict_new_build_config(self, tmpdir): 'formats': ['pdf'], 'build': { 'os': 'ubuntu-20.04', - 'languages': { + 'tools': { 'python': '3.9', 'nodejs': '16', }, @@ -2283,7 +2283,7 @@ def test_as_dict_new_build_config(self, tmpdir): }, 'build': { 'os': 'ubuntu-20.04', - 'languages': { + 'tools': { 'python': '3.9', 'nodejs': '16', }, diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index 85cf6f2cbb4..59af2e82261 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -520,6 +520,39 @@ def TEMPLATES(self): # Additional binds for the build container RTD_DOCKER_ADDITIONAL_BINDS = {} + RTD_DOCKER_BUILD_SETTINGS = { + # Mapping of build.os options to docker image. + 'os': { + 'ubuntu-20.04': f'{DOCKER_DEFAULT_IMAGE}:ubuntu20', + }, + # Mapping of build.tools options to specific versions. + 'tools': { + 'python': { + '2.7': '2.7.18', + '3.6': '3.6.15', + '3.7': '3.7.12', + '3.8': '3.8.12', + '3.9': '3.9.7', + '3.10': '3.10.0.rc2', + 'pypy3.7': 'pypy3.7-7.3.5', + 'miniconda3-4.7': 'miniconda3-4.7.12', + 'mambaforge-4.10': 'mambaforge-4.10.1-5', + }, + 'nodejs': { + '14': '14.17.6', + '16': '16.9.1', + }, + 'rust': { + '1.55': '1.55.0', + }, + 'golang': { + '1.17': '1.17.1', + }, + }, + } + # Always point to the latest stable release. + RTD_DOCKER_BUILD_SETTINGS['tools']['python']['3'] = RTD_DOCKER_BUILD_SETTINGS['tools']['python']['3.9'] + def _get_docker_memory_limit(self): try: total_memory = int(subprocess.check_output( From 2d11e095b3170fc9faaf38cab39f2db78a17c582 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 16 Sep 2021 15:14:22 -0500 Subject: [PATCH 03/11] Update schema --- readthedocs/rtd_tests/fixtures/spec/v2/schema.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/readthedocs/rtd_tests/fixtures/spec/v2/schema.json b/readthedocs/rtd_tests/fixtures/spec/v2/schema.json index facecd6473f..db0fa4f832a 100644 --- a/readthedocs/rtd_tests/fixtures/spec/v2/schema.json +++ b/readthedocs/rtd_tests/fixtures/spec/v2/schema.json @@ -88,9 +88,9 @@ "ubuntu-20.04" ] }, - "languages": { - "title": "Languages", - "description": "Languages and their version to be used in the build.", + "tools": { + "title": "Tools", + "description": "Tools and their version to be used in the build.", "type": "object", "properties": { "python": { @@ -138,7 +138,7 @@ }, "required": [ "os", - "languages" + "tools" ], "additionalProperties": false } From 3762ca7bdec580653c0acccc27568917e4b34406 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 16 Sep 2021 15:18:47 -0500 Subject: [PATCH 04/11] Update versions --- readthedocs/rtd_tests/fixtures/spec/v2/schema.json | 2 ++ readthedocs/settings/base.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/readthedocs/rtd_tests/fixtures/spec/v2/schema.json b/readthedocs/rtd_tests/fixtures/spec/v2/schema.json index db0fa4f832a..d0fe4bb8cdc 100644 --- a/readthedocs/rtd_tests/fixtures/spec/v2/schema.json +++ b/readthedocs/rtd_tests/fixtures/spec/v2/schema.json @@ -97,6 +97,8 @@ "enum": [ "2.7", "3", + "3.6", + "3.7", "3.8", "3.9", "3.10", diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index 59af2e82261..26f2d7abd6f 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -520,6 +520,8 @@ def TEMPLATES(self): # Additional binds for the build container RTD_DOCKER_ADDITIONAL_BINDS = {} + # When updating this options, + # update the readthedocs/rtd_tests/fixtures/spec/v2/schema.json file as well. RTD_DOCKER_BUILD_SETTINGS = { # Mapping of build.os options to docker image. 'os': { From 2bc3f8901f1758a8b0daf7a22824630b37c72697 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 20 Sep 2021 11:58:32 -0500 Subject: [PATCH 05/11] Update readthedocs/settings/base.py Co-authored-by: Manuel Kaufmann --- readthedocs/settings/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index 26f2d7abd6f..ab8e1be8883 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -535,7 +535,7 @@ def TEMPLATES(self): '3.7': '3.7.12', '3.8': '3.8.12', '3.9': '3.9.7', - '3.10': '3.10.0.rc2', + '3.10': '3.10.0rc2', 'pypy3.7': 'pypy3.7-7.3.5', 'miniconda3-4.7': 'miniconda3-4.7.12', 'mambaforge-4.10': 'mambaforge-4.10.1-5', From 2d57be8e018283a16c4b51bd65e116e459c17dbd Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 20 Sep 2021 13:44:27 -0500 Subject: [PATCH 06/11] Add helper methods --- readthedocs/config/config.py | 35 ++++++++++++++++++++----- readthedocs/config/models.py | 4 +-- readthedocs/config/tests/test_config.py | 8 +++--- 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index e6801d0f7fe..9206a99e520 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -6,6 +6,7 @@ import os import re from contextlib import contextmanager +from functools import lru_cache from django.conf import settings @@ -14,9 +15,9 @@ from .find import find_one from .models import ( Build, + BuildWithTools, Conda, Mkdocs, - OldBuild, Python, PythonInstall, PythonInstallRequirements, @@ -253,14 +254,32 @@ def pop_config(self, key, default=None, raise_ex=False): def validate(self): raise NotImplementedError() + @property + def using_build_tools(self): + return isinstance(self.build, BuildWithTools) + @property def python_interpreter(self): + if self.using_build_tools: + if self.build.tools['python'].startswith('mamba'): + return 'mamba' + if self.build.tools['python'].startswith('miniconda'): + return 'conda' + if self.build.tools['python']: + return 'python' + return None version = self.python_full_version if version.startswith('pypy'): # Allow to specify ``pypy3.5`` as Python interpreter return version return f'python{version}' + @property + def docker_image(self): + if self.using_build_tools: + return self.settings['os'][self.build.os] + return self.build.image + @property def python_full_version(self): version = self.python.version @@ -619,9 +638,10 @@ def conda(self): return None @property + @lru_cache(maxsize=1) def build(self): """The docker image used by the builders.""" - return OldBuild(**self._config['build']) + return Build(**self._config['build']) @property def doctype(self): @@ -728,7 +748,7 @@ def validate_conda(self): conda['environment'] = validate_path(environment, self.base_path) return conda - def validate_new_build_config(self): + def validate_build_config_with_tools(self): """ Validates the build object (new format). @@ -817,7 +837,7 @@ def validate_build(self): with self.catch_validation_error('build'): validate_dict(raw_build) if 'os' in raw_build: - return self.validate_new_build_config() + return self.validate_build_config_with_tools() return self.validate_old_build_config() def validate_apt_package(self, index): @@ -884,7 +904,7 @@ def validate_python(self): validate_dict(raw_python) python = {} - if isinstance(self.build, OldBuild): + if not self.using_build_tools: with self.catch_validation_error('python.version'): version = self.pop_config('python.version', '3') if version == 3.1: @@ -1230,10 +1250,11 @@ def conda(self): return None @property + @lru_cache(maxsize=1) def build(self): - klass = OldBuild + klass = Build if 'os' in self._config['build']: - klass = Build + klass = BuildWithTools return klass(**self._config['build']) @property diff --git a/readthedocs/config/models.py b/readthedocs/config/models.py index b2fcc37281b..7a724968f76 100644 --- a/readthedocs/config/models.py +++ b/readthedocs/config/models.py @@ -26,7 +26,7 @@ def as_dict(self): } -class OldBuild(Base): +class Build(Base): __slots__ = ('image', 'apt_packages') @@ -35,7 +35,7 @@ def __init__(self, **kwargs): super().__init__(**kwargs) -class Build(Base): +class BuildWithTools(Base): __slots__ = ('os', 'tools', 'apt_packages') diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index e8646235935..2dc5671d60b 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -32,8 +32,8 @@ ) from readthedocs.config.models import ( Build, + BuildWithTools, Conda, - OldBuild, PythonInstall, PythonInstallRequirements, ) @@ -906,7 +906,8 @@ def test_build_image_over_empty_default(self, image): def test_build_image_default_value(self): build = self.get_build_config({}) build.validate() - assert isinstance(build.build, OldBuild) + assert not build.using_build_tools + assert isinstance(build.build, Build) assert build.build.image == 'readthedocs/build:latest' @pytest.mark.parametrize('value', [3, [], 'invalid']) @@ -974,7 +975,8 @@ def test_new_build_config(self): }, ) build.validate() - assert isinstance(build.build, Build) + assert build.using_build_tools + assert isinstance(build.build, BuildWithTools) assert build.build.os == 'ubuntu-20.04' assert build.build.tools == {'python': '3.9'} From 7e921a353e097037b78726f2466782212410040a Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 21 Sep 2021 08:25:37 -0500 Subject: [PATCH 07/11] Use another object to allow build.tools['python'].version --- readthedocs/config/config.py | 20 ++++++++++++++++---- readthedocs/config/models.py | 5 +++++ readthedocs/config/tests/test_config.py | 7 ++++--- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index 9206a99e520..7764912d31c 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -15,6 +15,7 @@ from .find import find_one from .models import ( Build, + BuildTool, BuildWithTools, Conda, Mkdocs, @@ -1252,10 +1253,21 @@ def conda(self): @property @lru_cache(maxsize=1) def build(self): - klass = Build - if 'os' in self._config['build']: - klass = BuildWithTools - return klass(**self._config['build']) + build = self._config['build'] + if 'os' in build: + tools = { + tool: BuildTool( + version=version, + full_version=self.settings['tools'][tool][version], + ) + for tool, version in build['tools'].items() + } + return BuildWithTools( + os=build['os'], + tools=tools, + apt_packages=build['apt_packages'], + ) + return Build(**build) @property def python(self): diff --git a/readthedocs/config/models.py b/readthedocs/config/models.py index 7a724968f76..8e0ed70881a 100644 --- a/readthedocs/config/models.py +++ b/readthedocs/config/models.py @@ -44,6 +44,11 @@ def __init__(self, **kwargs): super().__init__(**kwargs) +class BuildTool(Base): + + __slots__ = ('version', 'full_version') + + class Python(Base): __slots__ = ('version', 'install', 'use_system_site_packages') diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index 2dc5671d60b..be302a05992 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -978,7 +978,8 @@ def test_new_build_config(self): assert build.using_build_tools assert isinstance(build.build, BuildWithTools) assert build.build.os == 'ubuntu-20.04' - assert build.build.tools == {'python': '3.9'} + assert build.build.tools['python'].version == '3.9' + assert build.build.tools['python'].full_version == '3.9.7' def test_new_build_config_conflict_with_build_image(self): build = self.get_build_config( @@ -2286,8 +2287,8 @@ def test_as_dict_new_build_config(self, tmpdir): 'build': { 'os': 'ubuntu-20.04', 'tools': { - 'python': '3.9', - 'nodejs': '16', + 'python': {'version': '3.9', 'full_version': '3.9.7'}, + 'nodejs': {'version': '16', 'full_version': '16.9.1'}, }, 'apt_packages': [], }, From ec486435e96cfafa63e195af82ad9d5741dac635 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 21 Sep 2021 08:31:26 -0500 Subject: [PATCH 08/11] Linter --- readthedocs/config/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index 7764912d31c..461696f1b2c 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -1253,7 +1253,7 @@ def conda(self): @property @lru_cache(maxsize=1) def build(self): - build = self._config['build'] + build = self._config['build'] if 'os' in build: tools = { tool: BuildTool( From 15dedcbca19a1cb587e4a66019190c6a893cc359 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 21 Sep 2021 08:44:24 -0500 Subject: [PATCH 09/11] Refactor tests --- readthedocs/config/tests/test_config.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index be302a05992..bbed3f1c35d 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -1,4 +1,5 @@ import os +from django.conf import settings import re import textwrap from collections import OrderedDict @@ -979,7 +980,8 @@ def test_new_build_config(self): assert isinstance(build.build, BuildWithTools) assert build.build.os == 'ubuntu-20.04' assert build.build.tools['python'].version == '3.9' - assert build.build.tools['python'].full_version == '3.9.7' + full_version = settings.RTD_DOCKER_BUILD_SETTINGS['tools']['python']['3.9'] + assert build.build.tools['python'].full_version == full_version def test_new_build_config_conflict_with_build_image(self): build = self.get_build_config( @@ -2287,8 +2289,14 @@ def test_as_dict_new_build_config(self, tmpdir): 'build': { 'os': 'ubuntu-20.04', 'tools': { - 'python': {'version': '3.9', 'full_version': '3.9.7'}, - 'nodejs': {'version': '16', 'full_version': '16.9.1'}, + 'python': { + 'version': '3.9', + 'full_version': settings.RTD_DOCKER_BUILD_SETTINGS['tools']['python']['3.9'], + }, + 'nodejs': { + 'version': '16', + 'full_version': settings.RTD_DOCKER_BUILD_SETTINGS['tools']['nodejs']['3.9'], + }, }, 'apt_packages': [], }, From e9dd4c09323feb444a5e7650646122457340a02d Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 21 Sep 2021 09:40:36 -0500 Subject: [PATCH 10/11] Fix python_interpreter --- readthedocs/config/config.py | 7 ++++--- readthedocs/config/tests/test_config.py | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index 461696f1b2c..68b4d0f62ca 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -262,11 +262,12 @@ def using_build_tools(self): @property def python_interpreter(self): if self.using_build_tools: - if self.build.tools['python'].startswith('mamba'): + tool = self.build.tools.get('python') + if tool and tool.version.startswith('mamba'): return 'mamba' - if self.build.tools['python'].startswith('miniconda'): + if tool and tool.version.startswith('miniconda'): return 'conda' - if self.build.tools['python']: + if tool: return 'python' return None version = self.python_full_version diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index bbed3f1c35d..a2b519d3872 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -982,6 +982,7 @@ def test_new_build_config(self): assert build.build.tools['python'].version == '3.9' full_version = settings.RTD_DOCKER_BUILD_SETTINGS['tools']['python']['3.9'] assert build.build.tools['python'].full_version == full_version + assert build.python_interpreter == 'python' def test_new_build_config_conflict_with_build_image(self): build = self.get_build_config( From ccf3c8bc79852741d27d3add197960661b6b283b Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 21 Sep 2021 11:19:30 -0500 Subject: [PATCH 11/11] Fix test --- readthedocs/config/tests/test_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index a2b519d3872..4068c3c9ced 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -2296,7 +2296,7 @@ def test_as_dict_new_build_config(self, tmpdir): }, 'nodejs': { 'version': '16', - 'full_version': settings.RTD_DOCKER_BUILD_SETTINGS['tools']['nodejs']['3.9'], + 'full_version': settings.RTD_DOCKER_BUILD_SETTINGS['tools']['nodejs']['16'], }, }, 'apt_packages': [],