From 0548e74d05416aa503782998e14e6d56ae844bf9 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 10 Jul 2018 14:15:01 -0500 Subject: [PATCH 01/67] Add tests for formats --- readthedocs/config/config.py | 14 ++++++- readthedocs/config/tests/test_config.py | 54 ++++++++++++++++++++++++- 2 files changed, 64 insertions(+), 4 deletions(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index 30101b58f0f..d72034a72ae 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -14,8 +14,8 @@ validate_file, validate_list, validate_string) __all__ = ( - 'load', 'BuildConfig', 'ConfigError', 'ConfigOptionNotSupportedError', - 'InvalidConfig', 'ProjectConfig' + 'load', 'BuildConfig', 'BuildConfigV2', 'ConfigError', + 'ConfigOptionNotSupportedError', 'InvalidConfig', 'ProjectConfig' ) @@ -582,6 +582,16 @@ def build_image(self): return self._config['build']['image'] +class BuildConfigV2(BuildConfigBase): + + """Version 1 of the configuration file.""" + + version = '2' + + def validate(self): + pass + + class ProjectConfig(list): """Wrapper for multiple build configs.""" diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index a8c308643e2..cc5fb150046 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -7,8 +7,8 @@ from pytest import raises from readthedocs.config import ( - BuildConfig, ConfigError, ConfigOptionNotSupportedError, InvalidConfig, - ProjectConfig, load) + BuildConfig, BuildConfigV2, ConfigError, ConfigOptionNotSupportedError, + InvalidConfig, ProjectConfig, load) from readthedocs.config.config import ( CONFIG_NOT_SUPPORTED, NAME_INVALID, NAME_REQUIRED, PYTHON_INVALID, TYPE_REQUIRED) @@ -765,3 +765,53 @@ def test_raise_config_not_supported(): build.redirects assert excinfo.value.configuration == 'redirects' assert excinfo.value.code == CONFIG_NOT_SUPPORTED + + +class TestBuildConfigV2(object): + + def get_build_config(self, config, env_config=None, + source_file='readthedocs.yml', source_position=0): + return BuildConfigV2( + env_config or {}, + config, + source_file=source_file, + source_position=source_position + ) + + def test_version(self): + build = self.get_build_config({}) + assert build.version == '2' + + def test_formats_check_valid(self): + build = get_build_config({'formats': ['htmlzip', 'pdf', 'epub']}) + build.validate() + assert build.formats == ['htmlzip', 'pdf', 'epub'] + + @pytest.mark.parametrize('value', [3, 'invalid', {'other': 'value'}]) + def test_formats_check_invalid_value(self, value): + build = get_build_config({'formats': value}) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'formats' + + def test_formats_check_invalid_type(self): + build = get_build_config({'formats': ['htmlzip', 'invalid', 'epub']}) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'formats' + + def test_formats_default_value(self, value): + build = get_build_config({}) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'formats' + + def test_formats_allow_empty(self): + build = get_build_config({'formats': []}) + build.validate() + assert build.formats == [] + + def test_formats_allow_all_keyword(self): + build = get_build_config({'formats': []}) + build.validate() + assert build.formats == ['htmlzip', 'pdf', 'epub'] From d373878afc2fdcd88f2267e0626e9c1f4aa072d6 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 10 Jul 2018 14:24:27 -0500 Subject: [PATCH 02/67] Fix tests --- readthedocs/config/tests/test_config.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index cc5fb150046..84001a8c639 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -783,35 +783,37 @@ def test_version(self): assert build.version == '2' def test_formats_check_valid(self): - build = get_build_config({'formats': ['htmlzip', 'pdf', 'epub']}) + build = self.get_build_config({'formats': ['htmlzip', 'pdf', 'epub']}) build.validate() assert build.formats == ['htmlzip', 'pdf', 'epub'] @pytest.mark.parametrize('value', [3, 'invalid', {'other': 'value'}]) def test_formats_check_invalid_value(self, value): - build = get_build_config({'formats': value}) + build = self.get_build_config({'formats': value}) with raises(InvalidConfig) as excinfo: build.validate() assert excinfo.value.key == 'formats' def test_formats_check_invalid_type(self): - build = get_build_config({'formats': ['htmlzip', 'invalid', 'epub']}) + build = self.get_build_config( + {'formats': ['htmlzip', 'invalid', 'epub']} + ) with raises(InvalidConfig) as excinfo: build.validate() assert excinfo.value.key == 'formats' def test_formats_default_value(self, value): - build = get_build_config({}) + build = self.get_build_config({}) with raises(InvalidConfig) as excinfo: build.validate() assert excinfo.value.key == 'formats' def test_formats_allow_empty(self): - build = get_build_config({'formats': []}) + build = self.get_build_config({'formats': []}) build.validate() assert build.formats == [] def test_formats_allow_all_keyword(self): - build = get_build_config({'formats': []}) + build = self.get_build_config({'formats': []}) build.validate() assert build.formats == ['htmlzip', 'pdf', 'epub'] From 1a124abc445609478fee43e85e59238f58fcf295 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 10 Jul 2018 14:59:05 -0500 Subject: [PATCH 03/67] Fix test --- readthedocs/config/tests/test_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index 84001a8c639..b49b67bff2b 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -802,7 +802,7 @@ def test_formats_check_invalid_type(self): build.validate() assert excinfo.value.key == 'formats' - def test_formats_default_value(self, value): + def test_formats_default_value(self): build = self.get_build_config({}) with raises(InvalidConfig) as excinfo: build.validate() From 09c1c5afe9f9231fe5e6968b0e42c56ff5005206 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 10 Jul 2018 14:59:52 -0500 Subject: [PATCH 04/67] Add tests for conda --- readthedocs/config/tests/test_config.py | 39 +++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index b49b67bff2b..08719497395 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -817,3 +817,42 @@ def test_formats_allow_all_keyword(self): build = self.get_build_config({'formats': []}) build.validate() assert build.formats == ['htmlzip', 'pdf', 'epub'] + + def test_conda_check_valid(self, tmpdir): + apply_fs(tmpdir, {'environment.yml': ''}) + build = self.get_build_config( + {'conda': {'environment': 'environment.yml'}}, + source_file=str(tmpdir.join('readthedocs.yml')), + ) + build.validate() + assert build.conda == ['htmlzip', 'pdf', 'epub'] + + def test_conda_check_invalid(self, tmpdir): + apply_fs(tmpdir, {'environment.yml': ''}) + build = self.get_build_config( + {'conda': {'environment': 'no_existing_environment.yml'}}, + source_file=str(tmpdir.join('readthedocs.yml')), + ) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'conda.file' + + @pytest.mark.parametrize('value', [3, [], 'invalid']) + def test_conda_check_invalid_value(self, value): + build = self.get_build_config({'conda': value}) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'conda' + + @pytest.mark.parametrize('value', [3, [], {}]) + def test_conda_check_invalid_file_value(self, value): + build = self.get_build_config({'conda': {'file': value}}) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'conda.file' + + def test_conda_check_file_required(self): + build = self.get_build_config({'conda': {'no-file': 'other'}}) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'conda.file' From c89544e76ea9e374196793d94ca24e95f8ebeb6c Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 10 Jul 2018 15:29:56 -0500 Subject: [PATCH 05/67] Add tests for build --- readthedocs/config/tests/test_config.py | 32 +++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index 08719497395..9ec5a6d74a8 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -856,3 +856,35 @@ def test_conda_check_file_required(self): with raises(InvalidConfig) as excinfo: build.validate() assert excinfo.value.key == 'conda.file' + + @pytest.mark.parametrize('value', ['latest']) + def test_build_check_valid(self, value): + build = self.get_build_config({'build': {'image': value}}) + build.validate() + assert build.build_image == 'readthedocs/build:{}'.format(value) + + @pytest.mark.parametrize('value', ['readthedocs/build:latest', 'one']) + def test_build_check_invalid(self, value): + build = self.get_build_config({'build': {'image': value}}) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'build.image' + + def test_build_default_value(self): + build = self.get_build_config({}) + build.validate() + assert build.build_image == 'readthedocs/build:latest' + + @pytest.mark.parametrize('value', [3, [], 'invalid']) + def test_build_check_invalid_type(self, value): + build = self.get_build_config({'build': value}) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'build' + + @pytest.mark.parametrize('value', [3, [], {}]) + def test_build_check_invalid_type_image(self, value): + build = self.get_build_config({'build': {'image': value}}) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'build.image' From 61a265ba25e49bdc8250017051cbdb6459ce92a9 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 10 Jul 2018 17:23:06 -0500 Subject: [PATCH 06/67] Fix test --- readthedocs/config/tests/test_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index 9ec5a6d74a8..c4e8269f9b8 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -825,7 +825,7 @@ def test_conda_check_valid(self, tmpdir): source_file=str(tmpdir.join('readthedocs.yml')), ) build.validate() - assert build.conda == ['htmlzip', 'pdf', 'epub'] + assert build.conda.file == str(tmpdir.join('environment.yml')) def test_conda_check_invalid(self, tmpdir): apply_fs(tmpdir, {'environment.yml': ''}) From e8a0e6059d2d7c83df4e316501d3f56644046d3a Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 10 Jul 2018 17:23:44 -0500 Subject: [PATCH 07/67] Use new interface --- readthedocs/config/tests/test_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index c4e8269f9b8..4e32407097d 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -861,7 +861,7 @@ def test_conda_check_file_required(self): def test_build_check_valid(self, value): build = self.get_build_config({'build': {'image': value}}) build.validate() - assert build.build_image == 'readthedocs/build:{}'.format(value) + assert build.build.image == 'readthedocs/build:{}'.format(value) @pytest.mark.parametrize('value', ['readthedocs/build:latest', 'one']) def test_build_check_invalid(self, value): @@ -873,7 +873,7 @@ def test_build_check_invalid(self, value): def test_build_default_value(self): build = self.get_build_config({}) build.validate() - assert build.build_image == 'readthedocs/build:latest' + assert build.build.image == 'readthedocs/build:latest' @pytest.mark.parametrize('value', [3, [], 'invalid']) def test_build_check_invalid_type(self, value): From e03bc694bbbc2be5a995597a74999a37275beee2 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 10 Jul 2018 17:25:31 -0500 Subject: [PATCH 08/67] Add tests for python --- readthedocs/config/tests/test_config.py | 226 ++++++++++++++++++++++++ 1 file changed, 226 insertions(+) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index 4e32407097d..3e950879719 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -888,3 +888,229 @@ def test_build_check_invalid_type_image(self, value): with raises(InvalidConfig) as excinfo: build.validate() assert excinfo.value.key == 'build.image' + + @pytest.mark.parametrize('value', [3, [], 'invalid']) + def test_python_check_invalid_types(self, value): + build = self.get_build_config({'python': value}) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'python' + + @pytest.mark.parametrize('image,versions', + [('latest', ['2', '2.7', '3', '3.5', '3.6'])]) + def test_python_version(self, image, versions): + for version in versions: + build = self.get_build_config({ + 'buil': { + 'image': image, + }, + 'python': { + 'version': version + }, + }) + build.validate() + assert build.python.version == version + + @pytest.mark.parametrize('image,versions', + [('latest', ['1', '2.8', '4', '3.8'])]) + def test_python_version_invalid(self, image, versions): + for version in versions: + build = self.get_build_config({ + 'buil': { + 'image': image, + }, + 'python': { + 'version': version + }, + }) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'python.version' + + def test_python_version_default(self): + build = self.get_build_config({}) + build.validate() + assert build.python.version == 3 + + @pytest.mark.parametrize('value', [3, [], {}]) + def test_python_version_check_invalid_types(self, value): + build = self.get_build_config({'python': {'version': value}}) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'python.version' + + def test_python_requirements_check_valid(self, tmpdir): + apply_fs(tmpdir, {'requirements.txt': ''}) + build = self.get_build_config( + {'python': {'requirements': 'requirements.txt'}} + ) + build.validate() + assert build.python.requirements == str(tmpdir.join('requirements.txt')) + + def test_python_requirements_check_invalid(self, tmpdir): + apply_fs(tmpdir, {'requirements.txt': ''}) + build = self.get_build_config( + {'python': {'requirements': 'invalid'}} + ) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'python.requirements' + + def test_python_requirements_default_value(self, tmpdir): + build = self.get_build_config({}) + build.validate() + assert build.python.requirements is None + + def test_python_requirements_allow_null(self): + build = self.get_build_config( + {'python': {'requirements': None}} + ) + build.validate() + assert build.python.requirements is None + + def test_python_requirements_allow_empty_string(self): + build = self.get_build_config( + {'python': {'requirements': ''}} + ) + build.validate() + assert build.python.requirements == '' + + @pytest.mark.parametrize('value', [3, [], {}]) + def test_python_requirements_check_invalid_types(self, value): + build = self.get_build_config( + {'python': {'requirements': value}} + ) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'python.requirements' + + def test_python_install_pip_check_valid(self): + build = self.get_build_config( + {'python': {'install': 'pip'}} + ) + build.validate() + assert build.python.install_with_pip is True + + def test_python_install_setuppy_check_valid(self): + build = self.get_build_config( + {'python': {'install': 'setup.py'}} + ) + build.validate() + assert build.python.install_with_setup is True + + @pytest.mark.parametrize('value', ['invalid', 'apt']) + def test_python_install_check_invalid(self, value): + build = self.get_build_config( + {'python': {'install': value}} + ) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'python.install' + + def test_python_install_allow_null(self): + build = self.get_build_config( + {'python': {'install': None}} + ) + build.validate() + assert build.python.install_with_pip is False + assert build.python.install_with_setup is False + + def test_python_install_default(self): + build = self.get_build_config({'python': {}}) + build.validate() + assert build.python.install_with_pip is False + assert build.python.install_with_setup is False + + @pytest.mark.parametrize('value', [2, [], {}]) + def test_python_install_check_invalid_type(self, value): + build = self.get_build_config( + {'python': {'install': value}} + ) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'python.install' + + def test_python_extra_requirements_and_pip(self): + build = self.get_build_config({ + 'python': { + 'install': 'pip', + 'extra_requirements': ['docs', 'tests'], + } + }) + build.validate() + assert build.python.extra_requirements == ['docs', 'tests'] + + def test_python_extra_requirements_not_install(self): + build = self.get_build_config({ + 'python': { + 'extra_requirements': ['docs', 'tests'], + } + }) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'python.extra_requirements' + + def test_python_extra_requirements_and_setup(self): + build = self.get_build_config({ + 'python': { + 'install': 'setup.py', + 'extra_requirements': ['docs', 'tests'], + } + }) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'python.extra_requirements' + + @pytest.mark.parametrize('value', [2, 'invalid', {}]) + def test_python_extra_requirements_check_type(self, value): + build = self.get_build_config({ + 'python': { + 'install': 'pip', + 'extra_requirements': value, + } + }) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'python.extra_requirements' + + @pytest.mark.parametrize('value', [None, []]) + def test_python_extra_requirements_allow_empty(self, value): + build = self.get_build_config({ + 'python': { + 'install': 'pip', + 'extra_requirements': value, + } + }) + build.validate() + assert build.python.extra_requirements == [] + + def test_python_extra_requirements_check_default(self): + build = self.get_build_config({}) + build.validate() + assert build.python.extra_requirements == [] + + @pytest.mark.parametrize('value', [True, False]) + def test_python_system_packages_check_valid(self, value): + build = self.get_build_config({ + 'python': { + 'system_packages': value, + } + }) + build.validate() + assert build.python.use_system_site_packages is value + + @pytest.mark.parametrize('value', [[], 'invalid', 0]) + def test_python_system_packages_check_invalid(self, value): + build = self.get_build_config({ + 'python': { + 'system_packages': value, + } + }) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'python.system_packages' + + def test_python_system_packages_check_default(self): + build = self.get_build_config({}) + build.validate() + assert build.python.use_system_site_packages is False From bd186682d2cc8627083d27de75c584eecd3e4268 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 10 Jul 2018 21:36:35 -0500 Subject: [PATCH 09/67] Add tests for sphinx --- readthedocs/config/tests/test_config.py | 63 +++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index 3e950879719..5c9738c8996 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -1114,3 +1114,66 @@ def test_python_system_packages_check_default(self): build = self.get_build_config({}) build.validate() assert build.python.use_system_site_packages is False + + @pytest.mark.parametrize('value', [[], True, 0, 'invalid']) + def test_sphinx_validate_type(self, value): + build = self.get_build_config({'sphinx': value}) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'sphinx' + + def test_sphinx_configuration_check_valid(self, tmpdir): + apply_fs(tmpdir, {'conf.py': ''}) + build = self.get_build_config( + {'sphinx': {'configuration': 'conf.py'}} + ) + build.validate() + assert build.sphinx.configuration == str(tmpdir.join('conf.py')) + + def test_sphinx_configuration_check_invalid(self, tmpdir): + apply_fs(tmpdir, {'conf.py': ''}) + build = self.get_build_config( + {'sphinx': {'configuration': 'invalid.py'}} + ) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'sphinx.configuration' + + def test_sphinx_configuration_allow_null(self): + build = self.get_build_config( + {'sphinx': {'configuration': None}} + ) + build.validate() + assert build.sphinx.configuration is None + + def test_sphinx_configuration_check_default(self): + build = self.get_build_config() + build.validate() + assert build.sphinx.configuration is None + + @pytest.mark.parametrize('value', [[], True, 0, {}]) + def test_sphinx_configuration_validate_type(self, value): + build = self.get_build_config( + {'sphinx': {'configuration': value}} + ) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'sphinx.configuration' + + @pytest.mark.parametrize('value', [True, False]) + def test_sphinx_fail_on_warning_check_valid(self, value): + build = self.get_build_config({'sphinx': {'fail_on_warning': value}}) + build.validate() + assert build.sphinx.fail_on_warning is value + + @pytest.mark.parametrize('value', [[], 'invalid', 0]) + def test_sphinx_fail_on_warning_check_invalid(self, value): + build = self.get_build_config({'sphinx': {'fail_on_warning': value}}) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'sphinx.fail_on_warning' + + def test_sphinx_fail_on_warning_check_default(self): + build = self.get_build_config({}) + build.validate() + assert build.sphinx.fail_on_warning is False From 27c3f5ceaa3f8022dbb5e8e5740e0be27a07a2b3 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 10 Jul 2018 21:47:55 -0500 Subject: [PATCH 10/67] Add tests for mkdocs --- readthedocs/config/tests/test_config.py | 63 +++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index 5c9738c8996..2182bfd3c04 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -1177,3 +1177,66 @@ def test_sphinx_fail_on_warning_check_default(self): build = self.get_build_config({}) build.validate() assert build.sphinx.fail_on_warning is False + + @pytest.mark.parametrize('value', [[], True, 0, 'invalid']) + def test_mkdocs_validate_type(self, value): + build = self.get_build_config({'mkdocs': value}) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'mkdocs' + + def test_mkdocs_configuration_check_valid(self, tmpdir): + apply_fs(tmpdir, {'mkdocs.yml': ''}) + build = self.get_build_config( + {'mkdocs': {'configuration': 'mkdocs.yml'}} + ) + build.validate() + assert build.mkdocs.configuration == str(tmpdir.join('mkdocs.yml')) + + def test_mkdocs_configuration_check_invalid(self, tmpdir): + apply_fs(tmpdir, {'mkdocs.yml': ''}) + build = self.get_build_config( + {'mkdocs': {'configuration': 'invalid.yml'}} + ) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'mkdocs.configuration' + + def test_mkdocs_configuration_allow_null(self): + build = self.get_build_config( + {'mkdocs': {'configuration': None}} + ) + build.validate() + assert build.mkdocs.configuration is None + + def test_mkdocs_configuration_check_default(self): + build = self.get_build_config() + build.validate() + assert build.mkdocs.configuration is None + + @pytest.mark.parametrize('value', [[], True, 0, {}]) + def test_mkdocs_configuration_validate_type(self, value): + build = self.get_build_config( + {'mkdocs': {'configuration': value}} + ) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'mkdocs.configuration' + + @pytest.mark.parametrize('value', [True, False]) + def test_mkdocs_fail_on_warning_check_valid(self, value): + build = self.get_build_config({'mkdocs': {'fail_on_warning': value}}) + build.validate() + assert build.mkdocs.fail_on_warning is value + + @pytest.mark.parametrize('value', [[], 'invalid', 0]) + def test_mkdocs_fail_on_warning_check_invalid(self, value): + build = self.get_build_config({'mkdocs': {'fail_on_warning': value}}) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'mkdocs.fail_on_warning' + + def test_mkdocs_fail_on_warning_check_default(self): + build = self.get_build_config({}) + build.validate() + assert build.mkdocs.fail_on_warning is False From 64ace4d0b6f9d866b06719bfad2122b663c526bc Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 10 Jul 2018 22:46:11 -0500 Subject: [PATCH 11/67] Add tests for submodules --- readthedocs/config/config.py | 3 +- readthedocs/config/tests/test_config.py | 166 +++++++++++++++++++++++- 2 files changed, 167 insertions(+), 2 deletions(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index d72034a72ae..e40684a8cf4 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -14,11 +14,12 @@ validate_file, validate_list, validate_string) __all__ = ( - 'load', 'BuildConfig', 'BuildConfigV2', 'ConfigError', + 'ALL', 'load', 'BuildConfig', 'BuildConfigV2', 'ConfigError', 'ConfigOptionNotSupportedError', 'InvalidConfig', 'ProjectConfig' ) +ALL = 'all' CONFIG_FILENAMES = ('readthedocs.yml', '.readthedocs.yml') diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index 2182bfd3c04..abb8d5fb143 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -7,7 +7,7 @@ from pytest import raises from readthedocs.config import ( - BuildConfig, BuildConfigV2, ConfigError, ConfigOptionNotSupportedError, + ALL, BuildConfig, BuildConfigV2, ConfigError, ConfigOptionNotSupportedError, InvalidConfig, ProjectConfig, load) from readthedocs.config.config import ( CONFIG_NOT_SUPPORTED, NAME_INVALID, NAME_REQUIRED, PYTHON_INVALID, @@ -1240,3 +1240,167 @@ def test_mkdocs_fail_on_warning_check_default(self): build = self.get_build_config({}) build.validate() assert build.mkdocs.fail_on_warning is False + + @pytest.mark.parametrize('value', [[], 'invalid', 0]) + def test_submodules_check_invalid_type(self, value): + build = self.get_build_config({'submodules': value}) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'submodules' + + def test_submodules_include_check_valid(self): + build = self.get_build_config({ + 'submodules': { + 'include': ['one', 'two'] + } + }) + build.validate() + assert build.submodules.include == ['one', 'two'] + assert build.submodules.exclude == [] + assert build.submodules.recursive is False + + @pytest.mark.parametrize('value', ['invalid', True, 0, {}]) + def test_submodules_include_check_invalid(self, value): + build = self.get_build_config({ + 'submodules': { + 'include': value, + } + }) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'submodules.include' + + def test_submodules_include_allows_all_keyword(self): + build = self.get_build_config({ + 'submodules': { + 'include': 'all', + } + }) + build.validate() + assert build.submodules.include == ALL + assert build.submodules.exclude == [] + assert build.submodules.recursive is False + + def test_submodules_exclude_check_valid(self): + build = self.get_build_config({ + 'submodules': { + 'exclude': ['one', 'two'] + } + }) + build.validate() + assert build.submodules.include == [] + assert build.submodules.exclude == ['one', 'two'] + assert build.submodules.recursive is False + + @pytest.mark.parametrize('value', ['invalid', True, 0, {}]) + def test_submodules_exclude_check_invalid(self, value): + build = self.get_build_config({ + 'submodules': { + 'exclude': value, + } + }) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'submodules.exclude' + + def test_submodules_exclude_allows_all_keyword(self): + build = self.get_build_config({ + 'submodules': { + 'exlude': 'all', + } + }) + build.validate() + assert build.submodules.include == [] + assert build.submodules.exclude == ALL + assert build.submodules.recursive is False + + def test_submodules_cant_exclude_and_include(self): + build = self.get_build_config({ + 'submodules': { + 'include': ['two'], + 'exlude': ['one'], + } + }) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'submodules' + + def test_submodules_can_exclude_include_be_empty(self): + build = self.get_build_config({ + 'submodules': { + 'exlude': 'all', + 'include': [], + } + }) + build.validate() + assert build.submodules.include == [] + assert build.submodules.exclude == ALL + assert build.submodules.recursive is False + + @pytest.mark.parametrize('value', [True, False]) + def test_submodules_recursive_check_valid(self, value): + build = self.get_build_config({ + 'submodules': { + 'include': ['one', 'two'], + 'recursive': value, + } + }) + build.validate() + assert build.submodules.include == ['one', 'two'] + assert build.submodules.exclude == [] + assert build.submodules.recursive is value + + @pytest.mark.parametrize('value', [[], 'invalid', 0]) + def test_submodules_recursive_check_invalid(self, value): + build = self.get_build_config({ + 'submodules': { + 'include': ['one', 'two'], + 'recursive': value, + } + }) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'submodules.recursive' + + def test_submodules_recursive_needs_include(self): + build = self.get_build_config({ + 'submodules': { + 'recursive': True, + } + }) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'submodules.recursive' + + build = self.get_build_config({ + 'submodules': { + 'include': [], + 'recursive': True, + } + }) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'submodules.recursive' + + def test_submodules_recursive_explict_default(self): + build = self.get_build_config({ + 'submodules': { + 'include': [], + 'recursive': False, + } + }) + build.validate() + assert build.submodules.include == [] + assert build.submodules.exclude == [] + assert build.submodules.recursive is False + + build = self.get_build_config({ + 'submodules': { + 'exclude': [], + 'recursive': False, + } + }) + build.validate() + assert build.submodules.include == [] + assert build.submodules.exclude == [] + assert build.submodules.recursive is False From 3bd4cb4e70077d66caffd1d8ad8fbced5cf78673 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 10 Jul 2018 22:46:27 -0500 Subject: [PATCH 12/67] Fix tests --- readthedocs/config/tests/test_config.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index abb8d5fb143..460b7468bde 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -804,9 +804,8 @@ def test_formats_check_invalid_type(self): def test_formats_default_value(self): build = self.get_build_config({}) - with raises(InvalidConfig) as excinfo: - build.validate() - assert excinfo.value.key == 'formats' + build.validate() + assert build.formats == [] def test_formats_allow_empty(self): build = self.get_build_config({'formats': []}) @@ -814,7 +813,7 @@ def test_formats_allow_empty(self): assert build.formats == [] def test_formats_allow_all_keyword(self): - build = self.get_build_config({'formats': []}) + build = self.get_build_config({'formats': 'all'}) build.validate() assert build.formats == ['htmlzip', 'pdf', 'epub'] From e7eb10beede96a03855f1699a82c8a69942a06f1 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 11 Jul 2018 12:15:13 -0500 Subject: [PATCH 13/67] Add tests for `install_project` default --- readthedocs/config/tests/test_config.py | 29 +++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index 460b7468bde..3d1c98ae854 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -989,6 +989,16 @@ def test_python_install_pip_check_valid(self): ) build.validate() assert build.python.install_with_pip is True + assert build.python.install_with_setup is False + + def test_python_install_pip_priority_over_default(self): + build = self.get_build_config( + {'python': {'install': 'pip'}}, + {'defaults': {'install_project': True}} + ) + build.validate() + assert build.python.install_with_pip is True + assert build.python.install_with_setup is False def test_python_install_setuppy_check_valid(self): build = self.get_build_config( @@ -996,6 +1006,25 @@ def test_python_install_setuppy_check_valid(self): ) build.validate() assert build.python.install_with_setup is True + assert build.python.install_with_pip is False + + def test_python_install_setuppy_respects_default(self): + build = self.get_build_config( + {}, + {'defaults': {'install_project': True}} + ) + build.validate() + assert build.python.install_with_pip is False + assert build.python.install_with_setup is True + + def test_python_install_setuppy_priority_over_default(self): + build = self.get_build_config( + {'python': {'install': 'setup.py'}}, + {'defaults': {'install_project': False}} + ) + build.validate() + assert build.python.install_with_pip is True + assert build.python.install_with_setup is False @pytest.mark.parametrize('value', ['invalid', 'apt']) def test_python_install_check_invalid(self, value): From 328246c80f6f6a399af97caf0f7da8c7de246891 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 11 Jul 2018 12:29:00 -0500 Subject: [PATCH 14/67] Add tests for formats default --- readthedocs/config/tests/test_config.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index 3d1c98ae854..8ce6db10bc7 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -807,6 +807,29 @@ def test_formats_default_value(self): build.validate() assert build.formats == [] + def test_formats_respect_default_values(self): + build = self.get_build_config( + {}, + {'defaults': {'formats': ['htmlzip']}} + ) + build.validate() + assert build.formats == ['htmlzip'] + + def test_formats_priority_over_defaults(self): + build = self.get_build_config( + {'formats': []}, + {'defaults': {'formats': ['htmlzip']}} + ) + build.validate() + assert build.formats == [] + + build = self.get_build_config( + {'formats': ['pdf']}, + {'defaults': {'formats': ['htmlzip']}} + ) + build.validate() + assert build.formats == ['pdf'] + def test_formats_allow_empty(self): build = self.get_build_config({'formats': []}) build.validate() From 2801bf06ab1a4c62da74e90e98508ea15c7aa503 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 11 Jul 2018 12:38:13 -0500 Subject: [PATCH 15/67] Add tests for use_system_packages default --- readthedocs/config/tests/test_config.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index 8ce6db10bc7..03879719521 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -1166,6 +1166,29 @@ def test_python_system_packages_check_default(self): build.validate() assert build.python.use_system_site_packages is False + def test_python_system_packages_respects_default(self): + build = self.get_build_config( + {}, + {'defaults': {'use_system_site_packages': True}} + ) + build.validate() + assert build.python.use_system_site_packages is True + + def test_python_system_packages_priority_over_default(self): + build = self.get_build_config( + {'python': {'system_packages': False}}, + {'defaults': {'use_system_site_packages': True}} + ) + build.validate() + assert build.python.use_system_site_packages is False + + build = self.get_build_config( + {'python': {'system_packages': True}}, + {'defaults': {'use_system_site_packages': False}} + ) + build.validate() + assert build.python.use_system_site_packages is True + @pytest.mark.parametrize('value', [[], True, 0, 'invalid']) def test_sphinx_validate_type(self, value): build = self.get_build_config({'sphinx': value}) From 56bc31c1b58ab0ab8410f11c02b4339542440622 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 11 Jul 2018 12:58:19 -0500 Subject: [PATCH 16/67] Add tests for requirements_file default --- readthedocs/config/tests/test_config.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index 03879719521..4bd1aa6724f 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -997,6 +997,24 @@ def test_python_requirements_allow_empty_string(self): build.validate() assert build.python.requirements == '' + def test_python_requirements_respects_default(self, tmpdir): + build = self.get_build_config( + {}, + {'defaults': {'requirements_file': 'requirements.txt'}}, + source_file=str(tmpdir.join('readthedocs.yml')), + ) + build.validate() + assert build.python.requirements == str(tmpdir.join('requirements.txt')) + + def test_python_requirements_priority_over_default(self, tmpdir): + build = self.get_build_config( + {'python': {'requirements': 'requirements.txt'}}, + {'defaults': {'requirements_file': 'requirements-default.txt'}}, + source_file=str(tmpdir.join('readthedocs.yml')), + ) + build.validate() + assert build.python.requirements == str(tmpdir.join('requirements.txt')) + @pytest.mark.parametrize('value', [3, [], {}]) def test_python_requirements_check_invalid_types(self, value): build = self.get_build_config( From ba3f86e1c785ca1c5f666151c3053ff13f26cc92 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 11 Jul 2018 13:26:20 -0500 Subject: [PATCH 17/67] Fix tests --- readthedocs/config/tests/test_config.py | 26 ++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index 4bd1aa6724f..8f470aa7fc2 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -954,6 +954,24 @@ def test_python_version_default(self): build.validate() assert build.python.version == 3 + @pytest.mark.parametrize('value', [1, 2, 3]) + def test_python_version_dont_respects_default(self, value): + build = self.get_build_config( + {}, + {'defaults': {'python_version': value}} + ) + build.validate() + assert build.python.version == 3 + + @pytest.mark.parametrize('value', [2, 3, 3.6]) + def test_python_version_priority_over_default(self, value): + build = self.get_build_config( + {'python': {'version': value}}, + {'defaults': {'python_version': 3}} + ) + build.validate() + assert build.python.version == value + @pytest.mark.parametrize('value', [3, [], {}]) def test_python_version_check_invalid_types(self, value): build = self.get_build_config({'python': {'version': value}}) @@ -964,7 +982,8 @@ def test_python_version_check_invalid_types(self, value): def test_python_requirements_check_valid(self, tmpdir): apply_fs(tmpdir, {'requirements.txt': ''}) build = self.get_build_config( - {'python': {'requirements': 'requirements.txt'}} + {'python': {'requirements': 'requirements.txt'}}, + source_file=str(tmpdir.join('readthedocs.yml')), ) build.validate() assert build.python.requirements == str(tmpdir.join('requirements.txt')) @@ -972,13 +991,14 @@ def test_python_requirements_check_valid(self, tmpdir): def test_python_requirements_check_invalid(self, tmpdir): apply_fs(tmpdir, {'requirements.txt': ''}) build = self.get_build_config( - {'python': {'requirements': 'invalid'}} + {'python': {'requirements': 'invalid'}}, + source_file=str(tmpdir.join('readthedocs.yml')), ) with raises(InvalidConfig) as excinfo: build.validate() assert excinfo.value.key == 'python.requirements' - def test_python_requirements_default_value(self, tmpdir): + def test_python_requirements_default_value(self): build = self.get_build_config({}) build.validate() assert build.python.requirements is None From 5489dbc63d113a72770847bc99a5f3777cee8da6 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 11 Jul 2018 13:41:55 -0500 Subject: [PATCH 18/67] Fix tests --- readthedocs/config/tests/test_config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index 8f470aa7fc2..c8c95deb2d9 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -954,14 +954,14 @@ def test_python_version_default(self): build.validate() assert build.python.version == 3 - @pytest.mark.parametrize('value', [1, 2, 3]) - def test_python_version_dont_respects_default(self, value): + @pytest.mark.parametrize('value', [2, 3]) + def test_python_version_respects_default(self, value): build = self.get_build_config( {}, {'defaults': {'python_version': value}} ) build.validate() - assert build.python.version == 3 + assert build.python.version == value @pytest.mark.parametrize('value', [2, 3, 3.6]) def test_python_version_priority_over_default(self, value): From 68033d04f47d351d1e6b0dc393b70157f6a18c21 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 11 Jul 2018 14:05:14 -0500 Subject: [PATCH 19/67] Tests for docker_image default --- readthedocs/config/tests/test_config.py | 28 +++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index c8c95deb2d9..aa733992ce4 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -880,19 +880,39 @@ def test_conda_check_file_required(self): assert excinfo.value.key == 'conda.file' @pytest.mark.parametrize('value', ['latest']) - def test_build_check_valid(self, value): + def test_build_image_check_valid(self, value): build = self.get_build_config({'build': {'image': value}}) build.validate() assert build.build.image == 'readthedocs/build:{}'.format(value) @pytest.mark.parametrize('value', ['readthedocs/build:latest', 'one']) - def test_build_check_invalid(self, value): + def test_build_image_check_invalid(self, value): build = self.get_build_config({'build': {'image': value}}) with raises(InvalidConfig) as excinfo: build.validate() assert excinfo.value.key == 'build.image' - def test_build_default_value(self): + @pytest.mark.parametrize( + 'image', ['latest', 'readthedocs/build:3.0', 'rtd/build:latest']) + def test_build_image_priorities_default(self, image): + build = self.get_build_config( + {'build': {'image': 'latest'}}, + {'defaults': {'build_image': image}} + ) + build.validate() + assert build.build.image == image + + @pytest.mark.parametrize( + 'image', ['', None]) + def test_build_image_over_empty_default(self, image): + build = self.get_build_config( + {'build': {'image': 'latest'}}, + {'defaults': {'build_image': image}} + ) + build.validate() + assert build.build.image == 'readthedocs/build:latest' + + def test_build_image_default_value(self): build = self.get_build_config({}) build.validate() assert build.build.image == 'readthedocs/build:latest' @@ -905,7 +925,7 @@ def test_build_check_invalid_type(self, value): assert excinfo.value.key == 'build' @pytest.mark.parametrize('value', [3, [], {}]) - def test_build_check_invalid_type_image(self, value): + def test_build_image_check_invalid_type(self, value): build = self.get_build_config({'build': {'image': value}}) with raises(InvalidConfig) as excinfo: build.validate() From 0029f433678f5d7c838b062d0af861891308c90c Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 11 Jul 2018 14:38:39 -0500 Subject: [PATCH 20/67] Allow strings and numbers in python.version --- readthedocs/config/tests/test_config.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index aa733992ce4..ede59fae1a4 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -939,11 +939,11 @@ def test_python_check_invalid_types(self, value): assert excinfo.value.key == 'python' @pytest.mark.parametrize('image,versions', - [('latest', ['2', '2.7', '3', '3.5', '3.6'])]) + [('latest', [2, 2.7, 3, 3.5, 3.6])]) def test_python_version(self, image, versions): for version in versions: build = self.get_build_config({ - 'buil': { + 'build': { 'image': image, }, 'python': { @@ -953,16 +953,28 @@ def test_python_version(self, image, versions): build.validate() assert build.python.version == version + def test_python_version_accepts_string(self): + build = self.get_build_config({ + 'build': { + 'image': 'latest', + }, + 'python': { + 'version': '3.6', + }, + }) + build.validate() + assert build.python.version == 3.6 + @pytest.mark.parametrize('image,versions', - [('latest', ['1', '2.8', '4', '3.8'])]) + [('latest', [1, 2.8, 4, 3.8])]) def test_python_version_invalid(self, image, versions): for version in versions: build = self.get_build_config({ - 'buil': { + 'build': { 'image': image, }, 'python': { - 'version': version + 'version': version, }, }) with raises(InvalidConfig) as excinfo: @@ -992,7 +1004,7 @@ def test_python_version_priority_over_default(self, value): build.validate() assert build.python.version == value - @pytest.mark.parametrize('value', [3, [], {}]) + @pytest.mark.parametrize('value', [[], {}]) def test_python_version_check_invalid_types(self, value): build = self.get_build_config({'python': {'version': value}}) with raises(InvalidConfig) as excinfo: From 9722e8b868f3651df1e37158ae9d6c0cb9eeb80a Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 11 Jul 2018 14:47:39 -0500 Subject: [PATCH 21/67] Test stable image --- readthedocs/config/tests/test_config.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index ede59fae1a4..f0dcab0a623 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -879,7 +879,7 @@ def test_conda_check_file_required(self): build.validate() assert excinfo.value.key == 'conda.file' - @pytest.mark.parametrize('value', ['latest']) + @pytest.mark.parametrize('value', ['stable', 'latest']) def test_build_image_check_valid(self, value): build = self.get_build_config({'build': {'image': value}}) build.validate() @@ -902,8 +902,7 @@ def test_build_image_priorities_default(self, image): build.validate() assert build.build.image == image - @pytest.mark.parametrize( - 'image', ['', None]) + @pytest.mark.parametrize('image', ['', None]) def test_build_image_over_empty_default(self, image): build = self.get_build_config( {'build': {'image': 'latest'}}, @@ -939,7 +938,8 @@ def test_python_check_invalid_types(self, value): assert excinfo.value.key == 'python' @pytest.mark.parametrize('image,versions', - [('latest', [2, 2.7, 3, 3.5, 3.6])]) + [('latest', [2, 2.7, 3, 3.5, 3.6]), + ('stable', [2, 2.7, 3, 3.5, 3.6])]) def test_python_version(self, image, versions): for version in versions: build = self.get_build_config({ @@ -966,7 +966,8 @@ def test_python_version_accepts_string(self): assert build.python.version == 3.6 @pytest.mark.parametrize('image,versions', - [('latest', [1, 2.8, 4, 3.8])]) + [('latest', [1, 2.8, 4, 3.8]), + ('stable', [1, 2.8, 4, 3.8])]) def test_python_version_invalid(self, image, versions): for version in versions: build = self.get_build_config({ From dc70c071b66f461c187ce0c648641c435d74f006 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 11 Jul 2018 18:09:06 -0500 Subject: [PATCH 22/67] Fix tests --- readthedocs/config/tests/test_config.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index f0dcab0a623..7a7c054c4c5 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -847,7 +847,7 @@ def test_conda_check_valid(self, tmpdir): source_file=str(tmpdir.join('readthedocs.yml')), ) build.validate() - assert build.conda.file == str(tmpdir.join('environment.yml')) + assert build.conda.environment == str(tmpdir.join('environment.yml')) def test_conda_check_invalid(self, tmpdir): apply_fs(tmpdir, {'environment.yml': ''}) @@ -857,7 +857,7 @@ def test_conda_check_invalid(self, tmpdir): ) with raises(InvalidConfig) as excinfo: build.validate() - assert excinfo.value.key == 'conda.file' + assert excinfo.value.key == 'conda.environment' @pytest.mark.parametrize('value', [3, [], 'invalid']) def test_conda_check_invalid_value(self, value): @@ -871,13 +871,13 @@ def test_conda_check_invalid_file_value(self, value): build = self.get_build_config({'conda': {'file': value}}) with raises(InvalidConfig) as excinfo: build.validate() - assert excinfo.value.key == 'conda.file' + assert excinfo.value.key == 'conda.environment' def test_conda_check_file_required(self): build = self.get_build_config({'conda': {'no-file': 'other'}}) with raises(InvalidConfig) as excinfo: build.validate() - assert excinfo.value.key == 'conda.file' + assert excinfo.value.key == 'conda.environment' @pytest.mark.parametrize('value', ['stable', 'latest']) def test_build_image_check_valid(self, value): From d04456821ffbefd8b672ffa9b11bf3fc98bbf9bc Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 11 Jul 2018 18:09:45 -0500 Subject: [PATCH 23/67] Add more validators --- readthedocs/config/validation.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/readthedocs/config/validation.py b/readthedocs/config/validation.py index b317612dd2c..dfc6549983f 100644 --- a/readthedocs/config/validation.py +++ b/readthedocs/config/validation.py @@ -8,10 +8,12 @@ INVALID_BOOL = 'invalid-bool' INVALID_CHOICE = 'invalid-choice' INVALID_LIST = 'invalid-list' +INVALID_DICT = 'invalid-dictionary' INVALID_DIRECTORY = 'invalid-directory' INVALID_FILE = 'invalid-file' INVALID_PATH = 'invalid-path' INVALID_STRING = 'invalid-string' +VALUE_NOT_FOUND = 'value-not-found' class ValidationError(Exception): @@ -23,9 +25,11 @@ class ValidationError(Exception): INVALID_CHOICE: 'expected one of ({choices}), got {value}', INVALID_DIRECTORY: '{value} is not a directory', INVALID_FILE: '{value} is not a file', + INVALID_DICT: '{value} is not a dictionary', INVALID_PATH: 'path {value} does not exist', INVALID_STRING: 'expected string', INVALID_LIST: 'expected list', + VALUE_NOT_FOUND: '{value} not found' } def __init__(self, value, code, format_kwargs=None): @@ -49,6 +53,12 @@ def validate_list(value): return list(value) +def validate_dict(value): + """Check if ``value`` is a dictionary.""" + if not isinstance(value, dict): + raise ValidationError(value, INVALID_DICT) + + def validate_choice(value, choices): """Check that ``value`` is in ``choices``.""" choices = validate_list(choices) @@ -59,6 +69,16 @@ def validate_choice(value, choices): return value +def validate_value_exists(value, container): + """Check that ``value`` exists in ``container``.""" + if value not in container: + raise ValidationError(value, VALUE_NOT_FOUND) + if isinstance(container, dict): + return container[value] + else: + return value + + def validate_bool(value): """Check that ``value`` is an boolean value.""" if value not in (0, 1, False, True): From f8ed4627cd6c9ce8503b69e797eda76f82485709 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 11 Jul 2018 18:11:48 -0500 Subject: [PATCH 24/67] Validate formats and conda --- readthedocs/config/config.py | 48 +++++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index e40684a8cf4..9b94110b6be 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -3,6 +3,7 @@ import os import re +from collections import namedtuple from contextlib import contextmanager import six @@ -10,8 +11,9 @@ from .find import find_one from .parser import ParseError, parse from .validation import ( - ValidationError, validate_bool, validate_choice, validate_directory, - validate_file, validate_list, validate_string) + ValidationError, validate_bool, validate_choice, validate_dict, + validate_directory, validate_file, validate_list, validate_string, + validate_value_exists) __all__ = ( 'ALL', 'load', 'BuildConfig', 'BuildConfigV2', 'ConfigError', @@ -111,6 +113,7 @@ def __init__(self, env_config, raw_config, source_file, source_position): self.raw_config = raw_config self.source_file = source_file self.source_position = source_position + self.base_path = os.path.dirname(self.source_file) self.defaults = self.env_config.get('defaults', {}) self._config = {} @@ -588,9 +591,48 @@ class BuildConfigV2(BuildConfigBase): """Version 1 of the configuration file.""" version = '2' + valid_formats = ['htmlzip', 'pdf', 'epub'] def validate(self): - pass + self._config['formats'] = self.validate_formats() + self._config['conda'] = self.validate_conda() + + def validate_formats(self): + formats = self.raw_config.get('formats', []) + if formats == ALL: + return self.valid_formats + with self.catch_validation_error('formats'): + validate_list(formats) + for format_ in formats: + validate_choice(format_, self.valid_formats) + return formats + + def validate_conda(self): + raw_conda = self.raw_config.get('conda') + if raw_conda is None: + return None + + with self.catch_validation_error('conda'): + validate_dict(raw_conda) + + conda = {} + with self.catch_validation_error('conda.environment'): + environment = validate_value_exists('environment', raw_conda) + conda['environment'] = validate_file(environment, self.base_path) + return conda + + @property + def formats(self): + return self._config['formats'] + + @property + def conda(self): + Conda = namedtuple('Conda', ['environment']) + if self._config['conda']: + return Conda( + self._config['conda']['environment'] + ) + return None class ProjectConfig(list): From 775f1743b6e80e1e6f712be10e74d4e49a0b24fa Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 11 Jul 2018 18:56:30 -0500 Subject: [PATCH 25/67] Implement build --- readthedocs/config/config.py | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index 9b94110b6be..695dbcfa2e9 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -592,10 +592,12 @@ class BuildConfigV2(BuildConfigBase): version = '2' valid_formats = ['htmlzip', 'pdf', 'epub'] + docker_versions = ['1.0', '2.0', '3.0', 'stable', 'latest'] def validate(self): self._config['formats'] = self.validate_formats() self._config['conda'] = self.validate_conda() + self._config['build'] = self.validate_build() def validate_formats(self): formats = self.raw_config.get('formats', []) @@ -621,6 +623,28 @@ def validate_conda(self): conda['environment'] = validate_file(environment, self.base_path) return conda + def validate_build(self): + raw_build = self.raw_config.get('build', {}) + with self.catch_validation_error('build'): + validate_dict(raw_build) + build = {} + with self.catch_validation_error('build.image'): + image = raw_build.get('image', 'latest') + build['image'] = validate_choice( + image, + self.docker_versions + ) + build['image'] = '{}:{}'.format( + DOCKER_DEFAULT_IMAGE, + build['image'] + ) + + # Allow to override specific project + config_image = self.defaults.get('build_image') + if config_image: + build['image'] = config_image + return build + @property def formats(self): return self._config['formats'] @@ -629,11 +653,14 @@ def formats(self): def conda(self): Conda = namedtuple('Conda', ['environment']) if self._config['conda']: - return Conda( - self._config['conda']['environment'] - ) + return Conda(**self._config['conda']) return None + @property + def build(self): + Build = namedtuple('Build', ['image']) + return Build(**self._config['build']) + class ProjectConfig(list): From 2317ce12ab414e04c4a4518940f1130b42ea5755 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 11 Jul 2018 20:43:55 -0500 Subject: [PATCH 26/67] Fix tests --- readthedocs/config/tests/test_config.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index 7a7c054c4c5..9483504c084 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -1051,6 +1051,7 @@ def test_python_requirements_allow_empty_string(self): assert build.python.requirements == '' def test_python_requirements_respects_default(self, tmpdir): + apply_fs(tmpdir, {'requirements.txt': ''}) build = self.get_build_config( {}, {'defaults': {'requirements_file': 'requirements.txt'}}, @@ -1060,6 +1061,7 @@ def test_python_requirements_respects_default(self, tmpdir): assert build.python.requirements == str(tmpdir.join('requirements.txt')) def test_python_requirements_priority_over_default(self, tmpdir): + apply_fs(tmpdir, {'requirements.txt': ''}) build = self.get_build_config( {'python': {'requirements': 'requirements.txt'}}, {'defaults': {'requirements_file': 'requirements-default.txt'}}, @@ -1117,8 +1119,8 @@ def test_python_install_setuppy_priority_over_default(self): {'defaults': {'install_project': False}} ) build.validate() - assert build.python.install_with_pip is True - assert build.python.install_with_setup is False + assert build.python.install_with_pip is False + assert build.python.install_with_setup is True @pytest.mark.parametrize('value', ['invalid', 'apt']) def test_python_install_check_invalid(self, value): @@ -1195,12 +1197,11 @@ def test_python_extra_requirements_check_type(self, value): build.validate() assert excinfo.value.key == 'python.extra_requirements' - @pytest.mark.parametrize('value', [None, []]) - def test_python_extra_requirements_allow_empty(self, value): + def test_python_extra_requirements_allow_empty(self): build = self.get_build_config({ 'python': { 'install': 'pip', - 'extra_requirements': value, + 'extra_requirements': [], } }) build.validate() @@ -1221,7 +1222,7 @@ def test_python_system_packages_check_valid(self, value): build.validate() assert build.python.use_system_site_packages is value - @pytest.mark.parametrize('value', [[], 'invalid', 0]) + @pytest.mark.parametrize('value', [[], 'invalid', 5]) def test_python_system_packages_check_invalid(self, value): build = self.get_build_config({ 'python': { @@ -1240,7 +1241,7 @@ def test_python_system_packages_check_default(self): def test_python_system_packages_respects_default(self): build = self.get_build_config( {}, - {'defaults': {'use_system_site_packages': True}} + {'defaults': {'use_system_packages': True}} ) build.validate() assert build.python.use_system_site_packages is True @@ -1248,14 +1249,14 @@ def test_python_system_packages_respects_default(self): def test_python_system_packages_priority_over_default(self): build = self.get_build_config( {'python': {'system_packages': False}}, - {'defaults': {'use_system_site_packages': True}} + {'defaults': {'use_system_packages': True}} ) build.validate() assert build.python.use_system_site_packages is False build = self.get_build_config( {'python': {'system_packages': True}}, - {'defaults': {'use_system_site_packages': False}} + {'defaults': {'use_system_packages': False}} ) build.validate() assert build.python.use_system_site_packages is True From 3ce8c607c0f7af8ddbc14b05926ed947afef2bdc Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 11 Jul 2018 20:44:32 -0500 Subject: [PATCH 27/67] More strict validation for lists --- readthedocs/config/validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/config/validation.py b/readthedocs/config/validation.py index dfc6549983f..143ff628055 100644 --- a/readthedocs/config/validation.py +++ b/readthedocs/config/validation.py @@ -46,7 +46,7 @@ def __init__(self, value, code, format_kwargs=None): def validate_list(value): """Check if ``value`` is an iterable.""" - if isinstance(value, str): + if isinstance(value, string_types) or isinstance(value, dict): raise ValidationError(value, INVALID_LIST) if not hasattr(value, '__iter__'): raise ValidationError(value, INVALID_LIST) From f3a4ea818852b9592eb8e140b283a6bbf0ce186e Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 11 Jul 2018 20:45:16 -0500 Subject: [PATCH 28/67] Add python validation --- readthedocs/config/config.py | 88 ++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index 695dbcfa2e9..0a651c82f8b 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -593,11 +593,14 @@ class BuildConfigV2(BuildConfigBase): version = '2' valid_formats = ['htmlzip', 'pdf', 'epub'] docker_versions = ['1.0', '2.0', '3.0', 'stable', 'latest'] + python_versions = [2, 2.7, 3, 3.5, 3.6] + install_options = ['pip', 'setup.py'] def validate(self): self._config['formats'] = self.validate_formats() self._config['conda'] = self.validate_conda() self._config['build'] = self.validate_build() + self._config['python'] = self.validate_python() def validate_formats(self): formats = self.raw_config.get('formats', []) @@ -645,6 +648,79 @@ def validate_build(self): build['image'] = config_image return build + def validate_python(self): + + raw_python = self.raw_config.get('python', {}) + with self.catch_validation_error('python'): + validate_dict(raw_python) + + python = {} + with self.catch_validation_error('python.version'): + version = self.defaults.get('python_version', 3) + version = raw_python.get('version', version) + if isinstance(version, six.string_types): + try: + version = int(version) + except ValueError: + try: + version = float(version) + except ValueError: + pass + python['version'] = validate_choice( + version, + self.get_valid_python_versions(), + ) + + with self.catch_validation_error('python.requirements'): + requirements = self.defaults.get('requirements_file') + requirements = raw_python.get('requirements', requirements) + if requirements != '' and requirements is not None: + requirements = validate_file(requirements, self.base_path) + python['requirements'] = requirements + + with self.catch_validation_error('python.install'): + install = ( + 'setup.py' if self.defaults.get('install_project') else None + ) + install = raw_python.get('install', install) + if install is not None: + validate_choice(install, self.install_options) + python['install_with_setup'] = install == 'setup.py' + python['install_with_pip'] = install == 'pip' + + with self.catch_validation_error('python.extra_requirements'): + extra_requirements = raw_python.get('extra_requirements', []) + extra_requirements = validate_list(extra_requirements) + if extra_requirements and not python['install_with_pip']: + self.error( + 'python.extra_requirements', + 'You need to install your project with pip ' + 'to use extra_requirements', + PYTHON_INVALID + ) + python['extra_requirements'] = [ + validate_string(extra) + for extra in extra_requirements + ] + + with self.catch_validation_error('python.system_packages'): + system_packages = self.defaults.get( + 'use_system_packages', False + ) + system_packages = raw_python.get( + 'system_packages', system_packages + ) + python['use_system_site_packages'] = validate_bool(system_packages) + + return python + + def get_valid_python_versions(self): + python_settings = DOCKER_IMAGE_SETTINGS.get(self.build.image, {}) + return python_settings.get( + 'supported_versions', + self.python_versions + ) + @property def formats(self): return self._config['formats'] @@ -661,6 +737,18 @@ def build(self): Build = namedtuple('Build', ['image']) return Build(**self._config['build']) + @property + def python(self): + Python = namedtuple( + 'Python', + [ + 'version', 'requirements', + 'install_with_pip', 'install_with_setup', + 'extra_requirements', 'use_system_site_packages', + ] + ) + return Python(**self._config['python']) + class ProjectConfig(list): From b6d472e2952bece9b7dfe36e7b67264018928d03 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 12 Jul 2018 12:13:57 -0500 Subject: [PATCH 29/67] Fix tests --- readthedocs/config/tests/test_config.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index 9483504c084..078fcce09d3 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -1271,7 +1271,8 @@ def test_sphinx_validate_type(self, value): def test_sphinx_configuration_check_valid(self, tmpdir): apply_fs(tmpdir, {'conf.py': ''}) build = self.get_build_config( - {'sphinx': {'configuration': 'conf.py'}} + {'sphinx': {'configuration': 'conf.py'}}, + source_file=str(tmpdir.join('readthedocs.yml')), ) build.validate() assert build.sphinx.configuration == str(tmpdir.join('conf.py')) @@ -1293,7 +1294,7 @@ def test_sphinx_configuration_allow_null(self): assert build.sphinx.configuration is None def test_sphinx_configuration_check_default(self): - build = self.get_build_config() + build = self.get_build_config({}) build.validate() assert build.sphinx.configuration is None @@ -1312,7 +1313,7 @@ def test_sphinx_fail_on_warning_check_valid(self, value): build.validate() assert build.sphinx.fail_on_warning is value - @pytest.mark.parametrize('value', [[], 'invalid', 0]) + @pytest.mark.parametrize('value', [[], 'invalid', 5]) def test_sphinx_fail_on_warning_check_invalid(self, value): build = self.get_build_config({'sphinx': {'fail_on_warning': value}}) with raises(InvalidConfig) as excinfo: From 780839aed78f103edeee6b32a69e960caf8e3bd4 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 12 Jul 2018 12:15:12 -0500 Subject: [PATCH 30/67] Implement validate_python --- readthedocs/config/config.py | 39 ++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index 0a651c82f8b..2034c530651 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -588,7 +588,7 @@ def build_image(self): class BuildConfigV2(BuildConfigBase): - """Version 1 of the configuration file.""" + """Version 2 of the configuration file.""" version = '2' valid_formats = ['htmlzip', 'pdf', 'epub'] @@ -599,8 +599,10 @@ class BuildConfigV2(BuildConfigBase): def validate(self): self._config['formats'] = self.validate_formats() self._config['conda'] = self.validate_conda() + # This should be called before validate_python self._config['build'] = self.validate_build() self._config['python'] = self.validate_python() + self._config['sphinx'] = self.validate_sphinx() def validate_formats(self): formats = self.raw_config.get('formats', []) @@ -649,7 +651,6 @@ def validate_build(self): return build def validate_python(self): - raw_python = self.raw_config.get('python', {}) with self.catch_validation_error('python'): validate_dict(raw_python) @@ -721,6 +722,33 @@ def get_valid_python_versions(self): self.python_versions ) + def validate_sphinx(self): + raw_sphinx = self.raw_config.get('sphinx', {}) + if raw_sphinx is None: + return None + + with self.catch_validation_error('sphinx'): + validate_dict(raw_sphinx) + + sphinx = {} + with self.catch_validation_error('sphinx.configuration'): + configuration = raw_sphinx.get('configuration') + if configuration is not None: + configuration = validate_file(configuration, self.base_path) + sphinx['configuration'] = configuration + + with self.catch_validation_error('sphinx.fail_on_warning'): + fail_on_warning = raw_sphinx.get('fail_on_warning', False) + sphinx['fail_on_warning'] = validate_bool(fail_on_warning) + + return sphinx + + def validate_mkdocs(self): + pass + + def validate_submodules(self): + pass + @property def formats(self): return self._config['formats'] @@ -749,6 +777,13 @@ def python(self): ) return Python(**self._config['python']) + @property + def sphinx(self): + Sphinx = namedtuple('Sphinx', ['configuration', 'fail_on_warning']) + if self._config['sphinx']: + return Sphinx(**self._config['sphinx']) + return None + class ProjectConfig(list): From ded5bda06d8d20162482fd2fa1d7dc8a25044300 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 12 Jul 2018 12:22:07 -0500 Subject: [PATCH 31/67] Fix tests --- readthedocs/config/tests/test_config.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index 078fcce09d3..53f4d594713 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -1335,7 +1335,8 @@ def test_mkdocs_validate_type(self, value): def test_mkdocs_configuration_check_valid(self, tmpdir): apply_fs(tmpdir, {'mkdocs.yml': ''}) build = self.get_build_config( - {'mkdocs': {'configuration': 'mkdocs.yml'}} + {'mkdocs': {'configuration': 'mkdocs.yml'}}, + source_file=str(tmpdir.join('readthedocs.yml')), ) build.validate() assert build.mkdocs.configuration == str(tmpdir.join('mkdocs.yml')) @@ -1343,7 +1344,8 @@ def test_mkdocs_configuration_check_valid(self, tmpdir): def test_mkdocs_configuration_check_invalid(self, tmpdir): apply_fs(tmpdir, {'mkdocs.yml': ''}) build = self.get_build_config( - {'mkdocs': {'configuration': 'invalid.yml'}} + {'mkdocs': {'configuration': 'invalid.yml'}}, + source_file=str(tmpdir.join('readthedocs.yml')), ) with raises(InvalidConfig) as excinfo: build.validate() @@ -1357,7 +1359,7 @@ def test_mkdocs_configuration_allow_null(self): assert build.mkdocs.configuration is None def test_mkdocs_configuration_check_default(self): - build = self.get_build_config() + build = self.get_build_config({}) build.validate() assert build.mkdocs.configuration is None @@ -1376,7 +1378,7 @@ def test_mkdocs_fail_on_warning_check_valid(self, value): build.validate() assert build.mkdocs.fail_on_warning is value - @pytest.mark.parametrize('value', [[], 'invalid', 0]) + @pytest.mark.parametrize('value', [[], 'invalid', 5]) def test_mkdocs_fail_on_warning_check_invalid(self, value): build = self.get_build_config({'mkdocs': {'fail_on_warning': value}}) with raises(InvalidConfig) as excinfo: From 62a092c4b82f2b4d08251e6f8bd9d283bc20f27f Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 12 Jul 2018 12:22:38 -0500 Subject: [PATCH 32/67] Implements mkdocs --- readthedocs/config/config.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index 2034c530651..83e2c94adf4 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -603,6 +603,7 @@ def validate(self): self._config['build'] = self.validate_build() self._config['python'] = self.validate_python() self._config['sphinx'] = self.validate_sphinx() + self._config['mkdocs'] = self.validate_mkdocs() def validate_formats(self): formats = self.raw_config.get('formats', []) @@ -744,7 +745,25 @@ def validate_sphinx(self): return sphinx def validate_mkdocs(self): - pass + raw_mkdocs = self.raw_config.get('mkdocs', {}) + if raw_mkdocs is None: + return None + + with self.catch_validation_error('mkdocs'): + validate_dict(raw_mkdocs) + + mkdocs = {} + with self.catch_validation_error('mkdocs.configuration'): + configuration = raw_mkdocs.get('configuration') + if configuration is not None: + configuration = validate_file(configuration, self.base_path) + mkdocs['configuration'] = configuration + + with self.catch_validation_error('mkdocs.fail_on_warning'): + fail_on_warning = raw_mkdocs.get('fail_on_warning', False) + mkdocs['fail_on_warning'] = validate_bool(fail_on_warning) + + return mkdocs def validate_submodules(self): pass @@ -784,6 +803,13 @@ def sphinx(self): return Sphinx(**self._config['sphinx']) return None + @property + def mkdocs(self): + Mkdocs = namedtuple('Mkdocs', ['configuration', 'fail_on_warning']) + if self._config['mkdocs']: + return Mkdocs(**self._config['mkdocs']) + return None + class ProjectConfig(list): From ae37362076df0aeca6b1a9abaecc48d658784bde Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 12 Jul 2018 12:52:30 -0500 Subject: [PATCH 33/67] Fix tests --- readthedocs/config/tests/test_config.py | 28 ++++--------------------- 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index 53f4d594713..59ffb5d8337 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -1455,7 +1455,7 @@ def test_submodules_exclude_check_invalid(self, value): def test_submodules_exclude_allows_all_keyword(self): build = self.get_build_config({ 'submodules': { - 'exlude': 'all', + 'exclude': 'all', } }) build.validate() @@ -1467,7 +1467,7 @@ def test_submodules_cant_exclude_and_include(self): build = self.get_build_config({ 'submodules': { 'include': ['two'], - 'exlude': ['one'], + 'exclude': ['one'], } }) with raises(InvalidConfig) as excinfo: @@ -1477,7 +1477,7 @@ def test_submodules_cant_exclude_and_include(self): def test_submodules_can_exclude_include_be_empty(self): build = self.get_build_config({ 'submodules': { - 'exlude': 'all', + 'exclude': 'all', 'include': [], } }) @@ -1499,7 +1499,7 @@ def test_submodules_recursive_check_valid(self, value): assert build.submodules.exclude == [] assert build.submodules.recursive is value - @pytest.mark.parametrize('value', [[], 'invalid', 0]) + @pytest.mark.parametrize('value', [[], 'invalid', 5]) def test_submodules_recursive_check_invalid(self, value): build = self.get_build_config({ 'submodules': { @@ -1511,26 +1511,6 @@ def test_submodules_recursive_check_invalid(self, value): build.validate() assert excinfo.value.key == 'submodules.recursive' - def test_submodules_recursive_needs_include(self): - build = self.get_build_config({ - 'submodules': { - 'recursive': True, - } - }) - with raises(InvalidConfig) as excinfo: - build.validate() - assert excinfo.value.key == 'submodules.recursive' - - build = self.get_build_config({ - 'submodules': { - 'include': [], - 'recursive': True, - } - }) - with raises(InvalidConfig) as excinfo: - build.validate() - assert excinfo.value.key == 'submodules.recursive' - def test_submodules_recursive_explict_default(self): build = self.get_build_config({ 'submodules': { From 143de8dc0672de2ee351d96185e27acd3211d18f Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 12 Jul 2018 12:53:05 -0500 Subject: [PATCH 34/67] Implement validate_submodules --- readthedocs/config/config.py | 51 ++++++++++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index 83e2c94adf4..0587c37399b 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -35,6 +35,7 @@ CONF_FILE_REQUIRED = 'conf-file-required' TYPE_REQUIRED = 'type-required' PYTHON_INVALID = 'python-invalid' +SUBMODULES_INVALID = 'submodules-invalid' DOCKER_DEFAULT_IMAGE = 'readthedocs/build' DOCKER_DEFAULT_VERSION = '2.0' @@ -604,6 +605,7 @@ def validate(self): self._config['python'] = self.validate_python() self._config['sphinx'] = self.validate_sphinx() self._config['mkdocs'] = self.validate_mkdocs() + self._config['submodules'] = self.validate_submodules() def validate_formats(self): formats = self.raw_config.get('formats', []) @@ -698,7 +700,7 @@ def validate_python(self): 'python.extra_requirements', 'You need to install your project with pip ' 'to use extra_requirements', - PYTHON_INVALID + code=PYTHON_INVALID ) python['extra_requirements'] = [ validate_string(extra) @@ -766,7 +768,45 @@ def validate_mkdocs(self): return mkdocs def validate_submodules(self): - pass + raw_submodules = self.raw_config.get('submodules', {}) + with self.catch_validation_error('submodules'): + validate_dict(raw_submodules) + + submodules = {} + with self.catch_validation_error('submodules.include'): + include = raw_submodules.get('include', []) + if include != ALL: + include = validate_list(include) + include = [ + validate_string(submodule) + for submodule in include + ] + submodules['include'] = include + + with self.catch_validation_error('submodules.exclude'): + exclude = raw_submodules.get('exclude', []) + if exclude != ALL: + exclude = validate_list(exclude) + exclude = [ + validate_string(submodule) + for submodule in exclude + ] + submodules['exclude'] = exclude + + with self.catch_validation_error('submodules'): + if submodules['exclude'] and submodules['include']: + self.error( + 'submodules', + 'You can not exclude and include submodules ' + 'at the same time', + code=SUBMODULES_INVALID + ) + + with self.catch_validation_error('submodules.recursive'): + recursive = raw_submodules.get('recursive', False) + submodules['recursive'] = validate_bool(recursive) + + return submodules @property def formats(self): @@ -810,6 +850,13 @@ def mkdocs(self): return Mkdocs(**self._config['mkdocs']) return None + @property + def submodules(self): + Submodules = namedtuple( + 'Submodules', ['include', 'exclude', 'recursive'] + ) + return Submodules(**self._config['submodules']) + class ProjectConfig(list): From c4721ec973ab81a2a92e5d24d04298eadb302999 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 12 Jul 2018 14:24:37 -0500 Subject: [PATCH 35/67] Linter --- readthedocs/config/validation.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/readthedocs/config/validation.py b/readthedocs/config/validation.py index 143ff628055..ab9164f335e 100644 --- a/readthedocs/config/validation.py +++ b/readthedocs/config/validation.py @@ -46,7 +46,7 @@ def __init__(self, value, code, format_kwargs=None): def validate_list(value): """Check if ``value`` is an iterable.""" - if isinstance(value, string_types) or isinstance(value, dict): + if isinstance(value, (dict, string_types)): raise ValidationError(value, INVALID_LIST) if not hasattr(value, '__iter__'): raise ValidationError(value, INVALID_LIST) @@ -75,8 +75,7 @@ def validate_value_exists(value, container): raise ValidationError(value, VALUE_NOT_FOUND) if isinstance(container, dict): return container[value] - else: - return value + return value def validate_bool(value): From 6a829ce30493a334413a0e671928ec11312756da Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 12 Jul 2018 15:56:27 -0500 Subject: [PATCH 36/67] Update submodule --- common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common b/common index e38599c2c5e..ed81bfc2608 160000 --- a/common +++ b/common @@ -1 +1 @@ -Subproject commit e38599c2c5e40d026d33528a61a1815c421a98be +Subproject commit ed81bfc2608fe23b82a22a55d8431a01762e2f02 From 07ad859068a51c42b5a44ea554be43d4717a9b4e Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 12 Jul 2018 15:59:29 -0500 Subject: [PATCH 37/67] Validates doc types --- readthedocs/config/config.py | 19 +++++++++++++++++++ readthedocs/config/tests/test_config.py | 16 +++++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index 0587c37399b..850e13ef711 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -36,6 +36,7 @@ TYPE_REQUIRED = 'type-required' PYTHON_INVALID = 'python-invalid' SUBMODULES_INVALID = 'submodules-invalid' +INVALID_KEYS_COMBINATION = 'invalid-keys-combination' DOCKER_DEFAULT_IMAGE = 'readthedocs/build' DOCKER_DEFAULT_VERSION = '2.0' @@ -603,6 +604,8 @@ def validate(self): # This should be called before validate_python self._config['build'] = self.validate_build() self._config['python'] = self.validate_python() + # Call this before validate sphinx and mkdocs + self.validate_doc_types() self._config['sphinx'] = self.validate_sphinx() self._config['mkdocs'] = self.validate_mkdocs() self._config['submodules'] = self.validate_submodules() @@ -725,6 +728,22 @@ def get_valid_python_versions(self): self.python_versions ) + def validate_doc_types(self): + """ + Validates that the user only have one type of documentation. + + This should be called before validating ``sphinx`` or ``mkdocs`` + to avoid innecessary validations. + """ + with self.catch_validation_error('.'): + if 'sphinx' in self.raw_config and 'mkdocs' in self.raw_config: + self.error( + '.', + 'You can not have the ``sphinx`` and ``mkdocs`` ' + 'keys at the same time', + code=INVALID_KEYS_COMBINATION + ) + def validate_sphinx(self): raw_sphinx = self.raw_config.get('sphinx', {}) if raw_sphinx is None: diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index 59ffb5d8337..50ecd82916e 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -1280,12 +1280,26 @@ def test_sphinx_configuration_check_valid(self, tmpdir): def test_sphinx_configuration_check_invalid(self, tmpdir): apply_fs(tmpdir, {'conf.py': ''}) build = self.get_build_config( - {'sphinx': {'configuration': 'invalid.py'}} + {'sphinx': {'configuration': 'invalid.py'}}, + source_file=str(tmpdir.join('readthedocs.yml')), ) with raises(InvalidConfig) as excinfo: build.validate() assert excinfo.value.key == 'sphinx.configuration' + def test_sphinx_cant_be_used_with_mkdocs(self, tmpdir): + apply_fs(tmpdir, {'conf.py': ''}) + build = self.get_build_config( + { + 'sphinx': {'configuration': 'conf.py'}, + 'mkdocs': {}, + }, + source_file=str(tmpdir.join('readthedocs.yml')), + ) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == '.' + def test_sphinx_configuration_allow_null(self): build = self.get_build_config( {'sphinx': {'configuration': None}} From 763f31ebab6cdd9b67b6572e17b3c3a9284b9054 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 12 Jul 2018 15:59:46 -0500 Subject: [PATCH 38/67] Docstrings --- readthedocs/config/config.py | 51 ++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index 850e13ef711..66b3ba93669 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -599,6 +599,7 @@ class BuildConfigV2(BuildConfigBase): install_options = ['pip', 'setup.py'] def validate(self): + """Validates and process ``raw_config`` and ``env_config``.""" self._config['formats'] = self.validate_formats() self._config['conda'] = self.validate_conda() # This should be called before validate_python @@ -611,6 +612,12 @@ def validate(self): self._config['submodules'] = self.validate_submodules() def validate_formats(self): + """ + Validates that formats contains only valid formats. + + The ``ALL`` keyword can be used to indicate that all formats + are used. + """ formats = self.raw_config.get('formats', []) if formats == ALL: return self.valid_formats @@ -621,6 +628,7 @@ def validate_formats(self): return formats def validate_conda(self): + """Validates the conda key.""" raw_conda = self.raw_config.get('conda') if raw_conda is None: return None @@ -635,6 +643,11 @@ def validate_conda(self): return conda def validate_build(self): + """ + Validates the build object. + + It prioritizes the value from the default image if exists. + """ raw_build = self.raw_config.get('build', {}) with self.catch_validation_error('build'): validate_dict(raw_build) @@ -657,6 +670,23 @@ def validate_build(self): return build def validate_python(self): + """ + Validates the python key. + + validate_build should be called before this, since it initialize the + build.image attribute. + + Fall back to the defaults of: + - ``version`` + - ``requirements`` + - ``install`` (only for setup.py method) + - ``system_packages`` + + + .. note:: + - ``version`` can be a string or number type. + - ``extra_requirements`` needs to be used with ``install: 'pip'``. + """ raw_python = self.raw_config.get('python', {}) with self.catch_validation_error('python'): validate_dict(raw_python) @@ -722,6 +752,11 @@ def validate_python(self): return python def get_valid_python_versions(self): + """ + Get the valid python versions for the current docker image. + + This should be called after ``validate_build()``. + """ python_settings = DOCKER_IMAGE_SETTINGS.get(self.build.image, {}) return python_settings.get( 'supported_versions', @@ -745,6 +780,11 @@ def validate_doc_types(self): ) def validate_sphinx(self): + """ + Validates the sphinx key. + + It makes sure we are using an existing configuration file. + """ raw_sphinx = self.raw_config.get('sphinx', {}) if raw_sphinx is None: return None @@ -766,6 +806,11 @@ def validate_sphinx(self): return sphinx def validate_mkdocs(self): + """ + Validates the mkdocs key. + + It makes sure we are using an existing configuration file. + """ raw_mkdocs = self.raw_config.get('mkdocs', {}) if raw_mkdocs is None: return None @@ -787,6 +832,12 @@ def validate_mkdocs(self): return mkdocs def validate_submodules(self): + """ + Validates the submodules key. + + - We can use the ``ALL`` keyword in include or exlude. + - We can't exlude and include submodules at the same time. + """ raw_submodules = self.raw_config.get('submodules', {}) with self.catch_validation_error('submodules'): validate_dict(raw_submodules) From 0eed2629f72bbb64a6eb18791b2c5de5b69dbc29 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 12 Jul 2018 16:03:17 -0500 Subject: [PATCH 39/67] Update submodule --- common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common b/common index ed81bfc2608..ae892294342 160000 --- a/common +++ b/common @@ -1 +1 @@ -Subproject commit ed81bfc2608fe23b82a22a55d8431a01762e2f02 +Subproject commit ae892294342da555c90f69c6594277b0c546ede3 From 588f35f6ed24a5175d63298db560bc0ea3b61900 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 12 Jul 2018 16:04:42 -0500 Subject: [PATCH 40/67] Add configurations for new docker images --- readthedocs/config/config.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index 66b3ba93669..54017c23391 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -50,6 +50,12 @@ 'readthedocs/build:2.0': { 'python': {'supported_versions': [2, 2.7, 3, 3.5]}, }, + 'readthedocs/build:3.0': { + 'python': {'supported_versions': [2, 2.7, 3, 3.5, 3.6]}, + }, + 'readthedocs/build:stable': { + 'python': {'supported_versions': [2, 2.7, 3, 3.5, 3.6]}, + }, 'readthedocs/build:latest': { 'python': {'supported_versions': [2, 2.7, 3, 3.3, 3.4, 3.5, 3.6]}, }, From 085cbd210bfe1a848c9cf48ca0d36f34f24fd73b Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 12 Jul 2018 16:08:47 -0500 Subject: [PATCH 41/67] Skip test --- readthedocs/config/tests/test_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index 50ecd82916e..83a1961cb2b 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -807,6 +807,7 @@ def test_formats_default_value(self): build.validate() assert build.formats == [] + @pytest.mark.skip('Needs a decision') def test_formats_respect_default_values(self): build = self.get_build_config( {}, From bbb439c92cf1fd39d662f726954f64bbe40aa21b Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 12 Jul 2018 16:14:21 -0500 Subject: [PATCH 42/67] Linter --- readthedocs/config/config.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index 54017c23391..fa9151969e0 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -890,19 +890,19 @@ def formats(self): @property def conda(self): - Conda = namedtuple('Conda', ['environment']) + Conda = namedtuple('Conda', ['environment']) # noqa if self._config['conda']: return Conda(**self._config['conda']) return None @property def build(self): - Build = namedtuple('Build', ['image']) + Build = namedtuple('Build', ['image']) # noqa return Build(**self._config['build']) @property def python(self): - Python = namedtuple( + Python = namedtuple( # noqa 'Python', [ 'version', 'requirements', @@ -914,21 +914,25 @@ def python(self): @property def sphinx(self): - Sphinx = namedtuple('Sphinx', ['configuration', 'fail_on_warning']) + Sphinx = namedtuple( # noqa + 'Sphinx', ['configuration', 'fail_on_warning'] + ) if self._config['sphinx']: return Sphinx(**self._config['sphinx']) return None @property def mkdocs(self): - Mkdocs = namedtuple('Mkdocs', ['configuration', 'fail_on_warning']) + Mkdocs = namedtuple( # noqa + 'Mkdocs', ['configuration', 'fail_on_warning'] + ) if self._config['mkdocs']: return Mkdocs(**self._config['mkdocs']) return None @property def submodules(self): - Submodules = namedtuple( + Submodules = namedtuple( # noqa 'Submodules', ['include', 'exclude', 'recursive'] ) return Submodules(**self._config['submodules']) From 4a891771f00a666d4a4bb0c2efcb3d06a7eed821 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 12 Jul 2018 16:44:07 -0500 Subject: [PATCH 43/67] Rename BuildConfig -> BuildConfigV1 --- readthedocs/config/config.py | 6 +- readthedocs/config/tests/test_config.py | 58 +++++++++---------- readthedocs/doc_builder/config.py | 4 +- .../tests/test_config_integration.py | 12 ++-- 4 files changed, 40 insertions(+), 40 deletions(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index fa9151969e0..1af6f8ffade 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -16,7 +16,7 @@ validate_value_exists) __all__ = ( - 'ALL', 'load', 'BuildConfig', 'BuildConfigV2', 'ConfigError', + 'ALL', 'load', 'BuildConfigV1', 'BuildConfigV2', 'ConfigError', 'ConfigOptionNotSupportedError', 'InvalidConfig', 'ProjectConfig' ) @@ -184,7 +184,7 @@ def __getattr__(self, name): raise ConfigOptionNotSupportedError(name) -class BuildConfig(BuildConfigBase): +class BuildConfigV1(BuildConfigBase): """Version 1 of the configuration file.""" @@ -975,7 +975,7 @@ def load(path, env_config): message=str(error)), code=CONFIG_SYNTAX_INVALID) for i, config in enumerate(configs): - build_config = BuildConfig( + build_config = BuildConfigV1( env_config, config, source_file=filename, diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index 83a1961cb2b..8034a01df3f 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -7,8 +7,8 @@ from pytest import raises from readthedocs.config import ( - ALL, BuildConfig, BuildConfigV2, ConfigError, ConfigOptionNotSupportedError, - InvalidConfig, ProjectConfig, load) + ALL, BuildConfigV1, BuildConfigV2, ConfigError, + ConfigOptionNotSupportedError, InvalidConfig, ProjectConfig, load) from readthedocs.config.config import ( CONFIG_NOT_SUPPORTED, NAME_INVALID, NAME_REQUIRED, PYTHON_INVALID, TYPE_REQUIRED) @@ -59,7 +59,7 @@ def get_build_config(config, env_config=None, source_file='readthedocs.yml', source_position=0): - return BuildConfig( + return BuildConfigV1( env_config or {}, config, source_file=source_file, @@ -101,7 +101,7 @@ def test_minimal_config(tmpdir): assert isinstance(config, ProjectConfig) assert len(config) == 1 build = config[0] - assert isinstance(build, BuildConfig) + assert isinstance(build, BuildConfigV1) def test_build_config_has_source_file(tmpdir): @@ -125,12 +125,12 @@ def test_build_config_has_source_position(tmpdir): def test_build_config_has_list_with_single_empty_value(tmpdir): base = str(apply_fs(tmpdir, config_with_explicit_empty_list)) build = load(base, env_config)[0] - assert isinstance(build, BuildConfig) + assert isinstance(build, BuildConfigV1) assert build.formats == [] def test_config_requires_name(): - build = BuildConfig( + build = BuildConfigV1( {'output_base': ''}, {}, source_file='readthedocs.yml', source_position=0 @@ -142,7 +142,7 @@ def test_config_requires_name(): def test_build_requires_valid_name(): - build = BuildConfig( + build = BuildConfigV1( {'output_base': ''}, {'name': 'with/slashes'}, source_file='readthedocs.yml', @@ -155,7 +155,7 @@ def test_build_requires_valid_name(): def test_config_requires_type(): - build = BuildConfig( + build = BuildConfigV1( {'output_base': ''}, {'name': 'docs'}, source_file='readthedocs.yml', source_position=0 @@ -167,7 +167,7 @@ def test_config_requires_type(): def test_build_requires_valid_type(): - build = BuildConfig( + build = BuildConfigV1( {'output_base': ''}, {'name': 'docs', 'type': 'unknown'}, source_file='readthedocs.yml', @@ -514,7 +514,7 @@ def it_uses_validate_file(tmpdir): def test_valid_build_config(): - build = BuildConfig(env_config, + build = BuildConfigV1(env_config, minimal_config, source_file='readthedocs.yml', source_position=0) @@ -534,7 +534,7 @@ def it_validates_to_abspath(tmpdir): apply_fs(tmpdir, {'configs': minimal_config, 'docs': {}}) with tmpdir.as_cwd(): source_file = str(tmpdir.join('configs', 'readthedocs.yml')) - build = BuildConfig( + build = BuildConfigV1( get_env_config(), {'base': '../docs'}, source_file=source_file, @@ -554,7 +554,7 @@ def it_uses_validate_directory(validate_directory): def it_fails_if_base_is_not_a_string(tmpdir): apply_fs(tmpdir, minimal_config) with tmpdir.as_cwd(): - build = BuildConfig( + build = BuildConfigV1( get_env_config(), {'base': 1}, source_file=str(tmpdir.join('readthedocs.yml')), @@ -566,7 +566,7 @@ def it_fails_if_base_is_not_a_string(tmpdir): def it_fails_if_base_does_not_exist(tmpdir): apply_fs(tmpdir, minimal_config) - build = BuildConfig( + build = BuildConfigV1( get_env_config(), {'base': 'docs'}, source_file=str(tmpdir.join('readthedocs.yml')), @@ -581,7 +581,7 @@ def describe_validate_build(): def it_fails_if_build_is_invalid_option(tmpdir): apply_fs(tmpdir, minimal_config) - build = BuildConfig( + build = BuildConfigV1( get_env_config(), {'build': {'image': 3.0}}, source_file=str(tmpdir.join('readthedocs.yml')), @@ -593,7 +593,7 @@ def it_fails_if_build_is_invalid_option(tmpdir): def it_fails_on_python_validation(tmpdir): apply_fs(tmpdir, minimal_config) - build = BuildConfig( + build = BuildConfigV1( {}, { 'build': {'image': 1.0}, @@ -609,7 +609,7 @@ def it_fails_on_python_validation(tmpdir): def it_works_on_python_validation(tmpdir): apply_fs(tmpdir, minimal_config) - build = BuildConfig( + build = BuildConfigV1( {}, { 'build': {'image': 'latest'}, @@ -622,7 +622,7 @@ def it_works_on_python_validation(tmpdir): def it_works(tmpdir): apply_fs(tmpdir, minimal_config) - build = BuildConfig( + build = BuildConfigV1( get_env_config(), {'build': {'image': 'latest'}}, source_file=str(tmpdir.join('readthedocs.yml')), @@ -632,7 +632,7 @@ def it_works(tmpdir): def default(tmpdir): apply_fs(tmpdir, minimal_config) - build = BuildConfig( + build = BuildConfigV1( get_env_config(), {}, source_file=str(tmpdir.join('readthedocs.yml')), @@ -647,7 +647,7 @@ def it_priorities_image_from_env_config(tmpdir, image): defaults = { 'build_image': image, } - build = BuildConfig( + build = BuildConfigV1( get_env_config({'defaults': defaults}), {'build': {'image': 'latest'}}, source_file=str(tmpdir.join('readthedocs.yml')), @@ -718,29 +718,29 @@ def test_requirements_file_respects_configuration(tmpdir): def test_build_validate_calls_all_subvalidators(tmpdir): apply_fs(tmpdir, minimal_config) - build = BuildConfig( + build = BuildConfigV1( {}, {}, source_file=str(tmpdir.join('readthedocs.yml')), source_position=0) - with patch.multiple(BuildConfig, + with patch.multiple(BuildConfigV1, validate_base=DEFAULT, validate_name=DEFAULT, validate_type=DEFAULT, validate_python=DEFAULT, validate_output_base=DEFAULT): build.validate() - BuildConfig.validate_base.assert_called_with() - BuildConfig.validate_name.assert_called_with() - BuildConfig.validate_type.assert_called_with() - BuildConfig.validate_python.assert_called_with() - BuildConfig.validate_output_base.assert_called_with() + BuildConfigV1.validate_base.assert_called_with() + BuildConfigV1.validate_name.assert_called_with() + BuildConfigV1.validate_type.assert_called_with() + BuildConfigV1.validate_python.assert_called_with() + BuildConfigV1.validate_output_base.assert_called_with() def test_validate_project_config(): - with patch.object(BuildConfig, 'validate') as build_validate: + with patch.object(BuildConfigV1, 'validate') as build_validate: project = ProjectConfig([ - BuildConfig( + BuildConfigV1( env_config, minimal_config, source_file='readthedocs.yml', @@ -753,7 +753,7 @@ def test_validate_project_config(): def test_load_calls_validate(tmpdir): apply_fs(tmpdir, minimal_config_dir) base = str(tmpdir) - with patch.object(BuildConfig, 'validate') as build_validate: + with patch.object(BuildConfigV1, 'validate') as build_validate: load(base, env_config) assert build_validate.call_count == 1 diff --git a/readthedocs/doc_builder/config.py b/readthedocs/doc_builder/config.py index 04b63a23f23..c3c2f1e5a78 100644 --- a/readthedocs/doc_builder/config.py +++ b/readthedocs/doc_builder/config.py @@ -4,7 +4,7 @@ from __future__ import ( absolute_import, division, print_function, unicode_literals) -from readthedocs.config import BuildConfig, ConfigError, InvalidConfig +from readthedocs.config import BuildConfigV1, ConfigError, InvalidConfig from readthedocs.config import load as load_config from .constants import DOCKER_IMAGE, DOCKER_IMAGE_SETTINGS @@ -65,7 +65,7 @@ def load_yaml_config(version): 'type': 'sphinx', 'name': version.slug, }) - config = BuildConfig( + config = BuildConfigV1( env_config=env_config, raw_config={}, source_file='empty', diff --git a/readthedocs/rtd_tests/tests/test_config_integration.py b/readthedocs/rtd_tests/tests/test_config_integration.py index 4d43b194567..3464f58bd7e 100644 --- a/readthedocs/rtd_tests/tests/test_config_integration.py +++ b/readthedocs/rtd_tests/tests/test_config_integration.py @@ -6,7 +6,7 @@ from django_dynamic_fixture import get from readthedocs.builds.models import Version -from readthedocs.config import BuildConfig, InvalidConfig, ProjectConfig +from readthedocs.config import BuildConfigV1, InvalidConfig, ProjectConfig from readthedocs.doc_builder.config import load_yaml_config from readthedocs.projects.models import Project @@ -14,7 +14,7 @@ def create_load(config=None): """Mock out the function of the build load function - This will create a ProjectConfig list of BuildConfig objects and validate + This will create a ProjectConfig list of BuildConfigV1 objects and validate them. The default load function iterates over files and builds up a list of objects. Instead of mocking all of this, just mock the end result. """ @@ -30,10 +30,10 @@ def inner(path=None, env_config=None): if env_config is not None: env_config_defaults.update(env_config) yaml_config = ProjectConfig([ - BuildConfig(env_config_defaults, - config, - source_file='readthedocs.yml', - source_position=0) + BuildConfigV1(env_config_defaults, + config, + source_file='readthedocs.yml', + source_position=0) ]) yaml_config.validate() return yaml_config From 0cd6680b13d7a66a55e4c23350ceef489b12bcd1 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 12 Jul 2018 21:53:47 -0500 Subject: [PATCH 44/67] Add version class selector --- readthedocs/config/config.py | 30 +++++++++++++-- readthedocs/config/tests/test_config.py | 49 +++++++++++++++++++++++-- 2 files changed, 72 insertions(+), 7 deletions(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index 1af6f8ffade..dfa72fdb34d 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -26,6 +26,7 @@ CONFIG_NOT_SUPPORTED = 'config-not-supported' +VERSION_INVALID = 'version-invalid' BASE_INVALID = 'base-invalid' BASE_NOT_A_DIR = 'base-not-a-directory' CONFIG_SYNTAX_INVALID = 'config-syntax-invalid' @@ -953,7 +954,8 @@ def load(path, env_config): Load a project configuration and the top-most build config for a given path. That is usually the root of the project, but will look deeper. - The config will be validated. + According to the version of the configuration a build object would be load + and validated, ``BuildConfigV1`` is the default. """ filename = find_one(path, CONFIG_FILENAMES) @@ -975,13 +977,35 @@ def load(path, env_config): message=str(error)), code=CONFIG_SYNTAX_INVALID) for i, config in enumerate(configs): - build_config = BuildConfigV1( + version = config.get('version', 1) + build_config = get_configuration_class(version)( env_config, config, source_file=filename, - source_position=i) + source_position=i + ) build_configs.append(build_config) project_config = ProjectConfig(build_configs) project_config.validate() return project_config + + +def get_configuration_class(version): + """ + Get the appropriate config class for ``version``. + + :type version: str or int + """ + configurations_class = { + 1: BuildConfigV1, + 2: BuildConfigV2 + } + try: + version = int(version) + return configurations_class[version] + except (KeyError, ValueError): + raise ConfigError( + 'Invalid version of the configuration file', + code=VERSION_INVALID + ) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index 8034a01df3f..e023e8ed017 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -1,6 +1,7 @@ from __future__ import division, print_function, unicode_literals import os +import textwrap import pytest from mock import DEFAULT, patch @@ -11,7 +12,7 @@ ConfigOptionNotSupportedError, InvalidConfig, ProjectConfig, load) from readthedocs.config.config import ( CONFIG_NOT_SUPPORTED, NAME_INVALID, NAME_REQUIRED, PYTHON_INVALID, - TYPE_REQUIRED) + TYPE_REQUIRED, VERSION_INVALID) from readthedocs.config.validation import ( INVALID_BOOL, INVALID_CHOICE, INVALID_LIST, INVALID_PATH, INVALID_STRING) @@ -104,6 +105,46 @@ def test_minimal_config(tmpdir): assert isinstance(build, BuildConfigV1) +def test_load_version1(tmpdir): + apply_fs(tmpdir, { + 'readthedocs.yml': textwrap.dedent(''' + version: 1 + ''') + }) + base = str(tmpdir) + config = load(base, get_env_config()) + assert isinstance(config, ProjectConfig) + assert len(config) == 1 + build = config[0] + assert isinstance(build, BuildConfigV1) + + +def test_load_version2(tmpdir): + apply_fs(tmpdir, { + 'readthedocs.yml': textwrap.dedent(''' + version: 2 + ''') + }) + base = str(tmpdir) + config = load(base, get_env_config()) + assert isinstance(config, ProjectConfig) + assert len(config) == 1 + build = config[0] + assert isinstance(build, BuildConfigV2) + + +def test_load_unknow_version(tmpdir): + apply_fs(tmpdir, { + 'readthedocs.yml': textwrap.dedent(''' + version: 9 + ''') + }) + base = str(tmpdir) + with raises(ConfigError) as excinfo: + load(base, get_env_config()) + assert excinfo.value.code == VERSION_INVALID + + def test_build_config_has_source_file(tmpdir): base = str(apply_fs(tmpdir, minimal_config_dir)) build = load(base, env_config)[0] @@ -515,9 +556,9 @@ def it_uses_validate_file(tmpdir): def test_valid_build_config(): build = BuildConfigV1(env_config, - minimal_config, - source_file='readthedocs.yml', - source_position=0) + minimal_config, + source_file='readthedocs.yml', + source_position=0) build.validate() assert build.name == 'docs' assert build.type == 'sphinx' From 586e993366739946762c96730f6553a0e8e57d1f Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 12 Jul 2018 22:18:21 -0500 Subject: [PATCH 45/67] Merge the config_path fro db --- readthedocs/config/config.py | 6 ++++- readthedocs/config/tests/test_config.py | 30 +++++++++++++++++++++++++ readthedocs/doc_builder/config.py | 1 + 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index dfa72fdb34d..7fe38a5be0a 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -801,7 +801,11 @@ def validate_sphinx(self): sphinx = {} with self.catch_validation_error('sphinx.configuration'): - configuration = raw_sphinx.get('configuration') + configuration = self.defaults.get('sphinx_configuration') + # The default value can be empty + if not configuration: + configuration = None + configuration = raw_sphinx.get('configuration', configuration) if configuration is not None: configuration = validate_file(configuration, self.base_path) sphinx['configuration'] = configuration diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index e023e8ed017..71c8d6e5377 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -1354,6 +1354,36 @@ def test_sphinx_configuration_check_default(self): build.validate() assert build.sphinx.configuration is None + def test_sphinx_configuration_respects_default(self, tmpdir): + apply_fs(tmpdir, {'conf.py': ''}) + build = self.get_build_config( + {}, + {'defaults': {'sphinx_configuration': 'conf.py'}}, + source_file=str(tmpdir.join('readthedocs.yml')), + ) + build.validate() + assert build.sphinx.configuration == str(tmpdir.join('conf.py')) + + def test_sphinx_configuration_defautl_can_be_empty(self, tmpdir): + apply_fs(tmpdir, {'conf.py': ''}) + build = self.get_build_config( + {}, + {'defaults': {'sphinx_configuration': ''}}, + source_file=str(tmpdir.join('readthedocs.yml')), + ) + build.validate() + assert build.sphinx.configuration is None + + def test_sphinx_configuration_priorities_over_default(self, tmpdir): + apply_fs(tmpdir, {'conf.py': '', 'conf-default.py': ''}) + build = self.get_build_config( + {'sphinx': {'configuration': 'conf.py'}}, + {'defaults': {'sphinx_configuration': 'conf-defaul.py'}}, + source_file=str(tmpdir.join('readthedocs.yml')), + ) + build.validate() + assert build.sphinx.configuration == str(tmpdir.join('conf.py')) + @pytest.mark.parametrize('value', [[], True, 0, {}]) def test_sphinx_configuration_validate_type(self, value): build = self.get_build_config( diff --git a/readthedocs/doc_builder/config.py b/readthedocs/doc_builder/config.py index c3c2f1e5a78..efefc8e6d2b 100644 --- a/readthedocs/doc_builder/config.py +++ b/readthedocs/doc_builder/config.py @@ -36,6 +36,7 @@ def load_yaml_config(version): 'use_system_packages': project.use_system_packages, 'requirements_file': project.requirements_file, 'python_version': python_version, + 'sphinx_configuration': version.get_conf_py_path(), 'build_image': project.container_image, } } From 012117e5c2c1e7e201d707d8d1820361d1cf5791 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 12 Jul 2018 22:57:25 -0500 Subject: [PATCH 46/67] Check for exception --- readthedocs/config/tests/test_config.py | 4 ++-- readthedocs/doc_builder/config.py | 8 +++++++- readthedocs/rtd_tests/tests/test_config_integration.py | 1 + 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index 71c8d6e5377..678c79376d2 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -1364,11 +1364,11 @@ def test_sphinx_configuration_respects_default(self, tmpdir): build.validate() assert build.sphinx.configuration == str(tmpdir.join('conf.py')) - def test_sphinx_configuration_defautl_can_be_empty(self, tmpdir): + def test_sphinx_configuration_defautl_can_be_none(self, tmpdir): apply_fs(tmpdir, {'conf.py': ''}) build = self.get_build_config( {}, - {'defaults': {'sphinx_configuration': ''}}, + {'defaults': {'sphinx_configuration': None}}, source_file=str(tmpdir.join('readthedocs.yml')), ) build.validate() diff --git a/readthedocs/doc_builder/config.py b/readthedocs/doc_builder/config.py index efefc8e6d2b..5431a682a13 100644 --- a/readthedocs/doc_builder/config.py +++ b/readthedocs/doc_builder/config.py @@ -6,6 +6,7 @@ from readthedocs.config import BuildConfigV1, ConfigError, InvalidConfig from readthedocs.config import load as load_config +from readthedocs.projects.models import ProjectConfigurationError from .constants import DOCKER_IMAGE, DOCKER_IMAGE_SETTINGS @@ -26,6 +27,11 @@ def load_yaml_config(version): img_name = project.container_image or DOCKER_IMAGE python_version = 3 if project.python_interpreter == 'python3' else 2 + try: + sphinx_configuration = version.get_conf_py_path() + except ProjectConfigurationError: + sphinx_configuration = None + env_config = { 'build': { 'image': img_name, @@ -36,7 +42,7 @@ def load_yaml_config(version): 'use_system_packages': project.use_system_packages, 'requirements_file': project.requirements_file, 'python_version': python_version, - 'sphinx_configuration': version.get_conf_py_path(), + 'sphinx_configuration': sphinx_configuration, 'build_image': project.container_image, } } diff --git a/readthedocs/rtd_tests/tests/test_config_integration.py b/readthedocs/rtd_tests/tests/test_config_integration.py index 3464f58bd7e..a23b75e8207 100644 --- a/readthedocs/rtd_tests/tests/test_config_integration.py +++ b/readthedocs/rtd_tests/tests/test_config_integration.py @@ -68,6 +68,7 @@ def test_python_supported_versions_default_image_1_0(self, load_config): 'use_system_packages': self.project.use_system_packages, 'requirements_file': self.project.requirements_file, 'python_version': 2, + 'sphinx_configuration': mock.ANY, 'build_image': 'readthedocs/build:1.0', }, }), From ac5c2687bc1df06271acfdbdde2b3d630465e8b5 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Fri, 13 Jul 2018 16:50:36 -0500 Subject: [PATCH 47/67] Refactor --- readthedocs/config/config.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index 7fe38a5be0a..1b1f1fb0d44 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -601,9 +601,9 @@ class BuildConfigV2(BuildConfigBase): version = '2' valid_formats = ['htmlzip', 'pdf', 'epub'] - docker_versions = ['1.0', '2.0', '3.0', 'stable', 'latest'] + valid_build_images = ['1.0', '2.0', '3.0', 'stable', 'latest'] python_versions = [2, 2.7, 3, 3.5, 3.6] - install_options = ['pip', 'setup.py'] + valid_install_options = ['pip', 'setup.py'] def validate(self): """Validates and process ``raw_config`` and ``env_config``.""" @@ -663,7 +663,7 @@ def validate_build(self): image = raw_build.get('image', 'latest') build['image'] = validate_choice( image, - self.docker_versions + self.valid_build_images ) build['image'] = '{}:{}'.format( DOCKER_DEFAULT_IMAGE, @@ -728,7 +728,7 @@ def validate_python(self): ) install = raw_python.get('install', install) if install is not None: - validate_choice(install, self.install_options) + validate_choice(install, self.valid_install_options) python['install_with_setup'] = install == 'setup.py' python['install_with_pip'] = install == 'pip' From 35b30c24f202800b2ea823ead175af9b2164d28c Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Fri, 13 Jul 2018 16:58:47 -0500 Subject: [PATCH 48/67] Update python supported versions --- readthedocs/config/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index 1b1f1fb0d44..0089472c7c2 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -52,10 +52,10 @@ 'python': {'supported_versions': [2, 2.7, 3, 3.5]}, }, 'readthedocs/build:3.0': { - 'python': {'supported_versions': [2, 2.7, 3, 3.5, 3.6]}, + 'python': {'supported_versions': [2, 2.7, 3, 3.3, 3.4, 3.5, 3.6]}, }, 'readthedocs/build:stable': { - 'python': {'supported_versions': [2, 2.7, 3, 3.5, 3.6]}, + 'python': {'supported_versions': [2, 2.7, 3, 3.3, 3.4, 3.5, 3.6]}, }, 'readthedocs/build:latest': { 'python': {'supported_versions': [2, 2.7, 3, 3.3, 3.4, 3.5, 3.6]}, From e662c6350a702df6fe976f29a86da938e5a00a2c Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Fri, 13 Jul 2018 16:59:13 -0500 Subject: [PATCH 49/67] Remove valid_python_versions --- readthedocs/config/config.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index 0089472c7c2..ffd80dc388d 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -602,7 +602,7 @@ class BuildConfigV2(BuildConfigBase): version = '2' valid_formats = ['htmlzip', 'pdf', 'epub'] valid_build_images = ['1.0', '2.0', '3.0', 'stable', 'latest'] - python_versions = [2, 2.7, 3, 3.5, 3.6] + default_build_image = 'latest' valid_install_options = ['pip', 'setup.py'] def validate(self): @@ -660,7 +660,7 @@ def validate_build(self): validate_dict(raw_build) build = {} with self.catch_validation_error('build.image'): - image = raw_build.get('image', 'latest') + image = raw_build.get('image', self.default_build_image) build['image'] = validate_choice( image, self.valid_build_images @@ -764,11 +764,12 @@ def get_valid_python_versions(self): This should be called after ``validate_build()``. """ - python_settings = DOCKER_IMAGE_SETTINGS.get(self.build.image, {}) - return python_settings.get( - 'supported_versions', - self.python_versions - ) + build_image = self.build.image + if build_image not in DOCKER_IMAGE_SETTINGS: + build_image = '{}:{}'.format( + DOCKER_DEFAULT_IMAGE, self.default_build_image + ) + return DOCKER_IMAGE_SETTINGS[build_image]['supported_versions'] def validate_doc_types(self): """ From 32ed7a5a9bd9f2e43f8035e4c7d18f03a0a6128f Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Fri, 13 Jul 2018 17:04:58 -0500 Subject: [PATCH 50/67] Fix python key --- readthedocs/config/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index ffd80dc388d..05bab86dc17 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -769,7 +769,8 @@ def get_valid_python_versions(self): build_image = '{}:{}'.format( DOCKER_DEFAULT_IMAGE, self.default_build_image ) - return DOCKER_IMAGE_SETTINGS[build_image]['supported_versions'] + python = DOCKER_IMAGE_SETTINGS[build_image]['python'] + return python['supported_versions'] def validate_doc_types(self): """ From b3bf6f4888e5ccdf073a869f63f448672b7a18f6 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Fri, 13 Jul 2018 17:06:44 -0500 Subject: [PATCH 51/67] Condense block --- readthedocs/config/config.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index 05bab86dc17..b264c095318 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -661,13 +661,9 @@ def validate_build(self): build = {} with self.catch_validation_error('build.image'): image = raw_build.get('image', self.default_build_image) - build['image'] = validate_choice( - image, - self.valid_build_images - ) build['image'] = '{}:{}'.format( DOCKER_DEFAULT_IMAGE, - build['image'] + validate_choice(image, self.valid_build_images) ) # Allow to override specific project From 8e0d6a12ce9a2430b94b6ccbacee7d33a3a03bf7 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Fri, 13 Jul 2018 17:22:09 -0500 Subject: [PATCH 52/67] Formatting --- readthedocs/config/config.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index b264c095318..9fb6e796f0c 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -685,7 +685,6 @@ def validate_python(self): - ``install`` (only for setup.py method) - ``system_packages`` - .. note:: - ``version`` can be a string or number type. - ``extra_requirements`` needs to be used with ``install: 'pip'``. @@ -908,9 +907,12 @@ def python(self): Python = namedtuple( # noqa 'Python', [ - 'version', 'requirements', - 'install_with_pip', 'install_with_setup', - 'extra_requirements', 'use_system_site_packages', + 'version', + 'requirements', + 'install_with_pip', + 'install_with_setup', + 'extra_requirements', + 'use_system_site_packages', ] ) return Python(**self._config['python']) From 1d60430031fecb71ae263f1eccda87494bdd7d3a Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Fri, 13 Jul 2018 17:35:55 -0500 Subject: [PATCH 53/67] Autocommit --- readthedocs/config/config.py | 109 ++++++++++++++++++++--------------- 1 file changed, 61 insertions(+), 48 deletions(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index 9fb6e796f0c..b02a9a2c104 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + """Build configuration for rtd.""" from __future__ import division, print_function, unicode_literals @@ -16,15 +18,19 @@ validate_value_exists) __all__ = ( - 'ALL', 'load', 'BuildConfigV1', 'BuildConfigV2', 'ConfigError', - 'ConfigOptionNotSupportedError', 'InvalidConfig', 'ProjectConfig' + 'ALL', + 'load', + 'BuildConfigV1', + 'BuildConfigV2', + 'ConfigError', + 'ConfigOptionNotSupportedError', + 'InvalidConfig', + 'ProjectConfig', ) - ALL = 'all' CONFIG_FILENAMES = ('readthedocs.yml', '.readthedocs.yml') - CONFIG_NOT_SUPPORTED = 'config-not-supported' VERSION_INVALID = 'version-invalid' BASE_INVALID = 'base-invalid' @@ -102,7 +108,8 @@ def __init__(self, key, code, error_message, source_file=None, message = self.message_template.format( key=key, code=code, - error=error_message) + error=error_message, + ) super(InvalidConfig, self).__init__(message, code=code) @@ -111,8 +118,8 @@ class BuildConfigBase(object): """ Config that handles the build of one particular documentation. - You need to call ``validate`` before the config is ready to use. - Also setting the ``output_base`` is required before using it for a build. + You need to call ``validate`` before the config is ready to use. Also + setting the ``output_base`` is required before using it for a build. """ version = None @@ -131,18 +138,18 @@ def error(self, key, message, code): """Raise an error related to ``key``.""" source = '{file} [{pos}]'.format( file=self.source_file, - pos=self.source_position + pos=self.source_position, ) error_message = '{source}: {message}'.format( source=source, - message=message + message=message, ) raise InvalidConfig( key=key, code=code, error_message=error_message, source_file=self.source_file, - source_position=self.source_position + source_position=self.source_position, ) @contextmanager @@ -156,7 +163,7 @@ def catch_validation_error(self, key): code=error.code, error_message=str(error), source_file=self.source_file, - source_position=self.source_position + source_position=self.source_position, ) def validate(self): @@ -193,12 +200,14 @@ class BuildConfigV1(BuildConfigBase): BASE_NOT_A_DIR_MESSAGE = '"base" is not a directory: {base}' NAME_REQUIRED_MESSAGE = 'Missing key "name"' NAME_INVALID_MESSAGE = ( - 'Invalid name "{name}". Valid values must match {name_re}') + 'Invalid name "{name}". Valid values must match {name_re}' + ) TYPE_REQUIRED_MESSAGE = 'Missing key "type"' CONF_FILE_REQUIRED_MESSAGE = 'Missing key "conf_file"' PYTHON_INVALID_MESSAGE = '"python" section must be a mapping.' PYTHON_EXTRA_REQUIREMENTS_INVALID_MESSAGE = ( - '"python.extra_requirements" section must be a list.') + '"python.extra_requirements" section must be a list.' + ) PYTHON_SUPPORTED_VERSIONS = [2, 2.7, 3, 3.5] DOCKER_SUPPORTED_VERSIONS = ['1.0', '2.0', 'latest'] @@ -207,9 +216,7 @@ class BuildConfigV1(BuildConfigBase): def get_valid_types(self): # noqa """Get all valid types.""" - return ( - 'sphinx', - ) + return ('sphinx',) def get_valid_python_versions(self): """Get all valid python versions.""" @@ -525,7 +532,7 @@ def base(self): @property def output_base(self): - """The output base""" + """The output base.""" return self._config['output_base'] @property @@ -622,8 +629,7 @@ def validate_formats(self): """ Validates that formats contains only valid formats. - The ``ALL`` keyword can be used to indicate that all formats - are used. + The ``ALL`` keyword can be used to indicate that all formats are used. """ formats = self.raw_config.get('formats', []) if formats == ALL: @@ -663,7 +669,10 @@ def validate_build(self): image = raw_build.get('image', self.default_build_image) build['image'] = '{}:{}'.format( DOCKER_DEFAULT_IMAGE, - validate_choice(image, self.valid_build_images) + validate_choice( + image, + self.valid_build_images, + ), ) # Allow to override specific project @@ -735,19 +744,20 @@ def validate_python(self): 'python.extra_requirements', 'You need to install your project with pip ' 'to use extra_requirements', - code=PYTHON_INVALID + code=PYTHON_INVALID, ) python['extra_requirements'] = [ - validate_string(extra) - for extra in extra_requirements + validate_string(extra) for extra in extra_requirements ] with self.catch_validation_error('python.system_packages'): system_packages = self.defaults.get( - 'use_system_packages', False + 'use_system_packages', + False, ) system_packages = raw_python.get( - 'system_packages', system_packages + 'system_packages', + system_packages, ) python['use_system_site_packages'] = validate_bool(system_packages) @@ -762,7 +772,8 @@ def get_valid_python_versions(self): build_image = self.build.image if build_image not in DOCKER_IMAGE_SETTINGS: build_image = '{}:{}'.format( - DOCKER_DEFAULT_IMAGE, self.default_build_image + DOCKER_DEFAULT_IMAGE, + self.default_build_image, ) python = DOCKER_IMAGE_SETTINGS[build_image]['python'] return python['supported_versions'] @@ -771,8 +782,8 @@ def validate_doc_types(self): """ Validates that the user only have one type of documentation. - This should be called before validating ``sphinx`` or ``mkdocs`` - to avoid innecessary validations. + This should be called before validating ``sphinx`` or ``mkdocs`` to + avoid innecessary validations. """ with self.catch_validation_error('.'): if 'sphinx' in self.raw_config and 'mkdocs' in self.raw_config: @@ -780,7 +791,7 @@ def validate_doc_types(self): '.', 'You can not have the ``sphinx`` and ``mkdocs`` ' 'keys at the same time', - code=INVALID_KEYS_COMBINATION + code=INVALID_KEYS_COMBINATION, ) def validate_sphinx(self): @@ -854,20 +865,18 @@ def validate_submodules(self): with self.catch_validation_error('submodules.include'): include = raw_submodules.get('include', []) if include != ALL: - include = validate_list(include) include = [ validate_string(submodule) - for submodule in include + for submodule in validate_list(include) ] submodules['include'] = include with self.catch_validation_error('submodules.exclude'): exclude = raw_submodules.get('exclude', []) if exclude != ALL: - exclude = validate_list(exclude) exclude = [ validate_string(submodule) - for submodule in exclude + for submodule in validate_list(exclude) ] submodules['exclude'] = exclude @@ -877,7 +886,7 @@ def validate_submodules(self): 'submodules', 'You can not exclude and include submodules ' 'at the same time', - code=SUBMODULES_INVALID + code=SUBMODULES_INVALID, ) with self.catch_validation_error('submodules.recursive'): @@ -913,14 +922,14 @@ def python(self): 'install_with_setup', 'extra_requirements', 'use_system_site_packages', - ] + ], ) return Python(**self._config['python']) @property def sphinx(self): Sphinx = namedtuple( # noqa - 'Sphinx', ['configuration', 'fail_on_warning'] + 'Sphinx', ['configuration', 'fail_on_warning'], ) if self._config['sphinx']: return Sphinx(**self._config['sphinx']) @@ -929,7 +938,7 @@ def sphinx(self): @property def mkdocs(self): Mkdocs = namedtuple( # noqa - 'Mkdocs', ['configuration', 'fail_on_warning'] + 'Mkdocs', ['configuration', 'fail_on_warning'], ) if self._config['mkdocs']: return Mkdocs(**self._config['mkdocs']) @@ -938,7 +947,7 @@ def mkdocs(self): @property def submodules(self): Submodules = namedtuple( # noqa - 'Submodules', ['include', 'exclude', 'recursive'] + 'Submodules', ['include', 'exclude', 'recursive'], ) return Submodules(**self._config['submodules']) @@ -957,9 +966,9 @@ def load(path, env_config): """ Load a project configuration and the top-most build config for a given path. - That is usually the root of the project, but will look deeper. - According to the version of the configuration a build object would be load - and validated, ``BuildConfigV1`` is the default. + That is usually the root of the project, but will look deeper. According to + the version of the configuration a build object would be load and validated, + ``BuildConfigV1`` is the default. """ filename = find_one(path, CONFIG_FILENAMES) @@ -968,8 +977,10 @@ def load(path, env_config): if files: files += ' or ' files += '{!r}'.format(CONFIG_FILENAMES[-1]) - raise ConfigError('No files {} found'.format(files), - code=CONFIG_REQUIRED) + raise ConfigError( + 'No files {} found'.format(files), + code=CONFIG_REQUIRED, + ) build_configs = [] with open(filename, 'r') as configuration_file: try: @@ -978,15 +989,17 @@ def load(path, env_config): raise ConfigError( 'Parse error in {filename}: {message}'.format( filename=filename, - message=str(error)), - code=CONFIG_SYNTAX_INVALID) + message=str(error), + ), + code=CONFIG_SYNTAX_INVALID, + ) for i, config in enumerate(configs): version = config.get('version', 1) build_config = get_configuration_class(version)( env_config, config, source_file=filename, - source_position=i + source_position=i, ) build_configs.append(build_config) @@ -1003,7 +1016,7 @@ def get_configuration_class(version): """ configurations_class = { 1: BuildConfigV1, - 2: BuildConfigV2 + 2: BuildConfigV2, } try: version = int(version) @@ -1011,5 +1024,5 @@ def get_configuration_class(version): except (KeyError, ValueError): raise ConfigError( 'Invalid version of the configuration file', - code=VERSION_INVALID + code=VERSION_INVALID, ) From f29a97a0d3daeb89bfecb2596e62c309faa870aa Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Fri, 13 Jul 2018 17:39:17 -0500 Subject: [PATCH 54/67] Autocommit --- .../tests/test_config_integration.py | 58 ++++++++++++------- 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_config_integration.py b/readthedocs/rtd_tests/tests/test_config_integration.py index a23b75e8207..ae95681c450 100644 --- a/readthedocs/rtd_tests/tests/test_config_integration.py +++ b/readthedocs/rtd_tests/tests/test_config_integration.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from __future__ import ( absolute_import, division, print_function, unicode_literals) @@ -12,7 +13,8 @@ def create_load(config=None): - """Mock out the function of the build load function + """ + Mock out the function of the build load function. This will create a ProjectConfig list of BuildConfigV1 objects and validate them. The default load function iterates over files and builds up a list of @@ -30,13 +32,16 @@ def inner(path=None, env_config=None): if env_config is not None: env_config_defaults.update(env_config) yaml_config = ProjectConfig([ - BuildConfigV1(env_config_defaults, - config, - source_file='readthedocs.yml', - source_position=0) + BuildConfigV1( + env_config_defaults, + config, + source_file='readthedocs.yml', + source_position=0, + ), ]) yaml_config.validate() return yaml_config + return inner @@ -44,8 +49,12 @@ def inner(path=None, env_config=None): class LoadConfigTests(TestCase): def setUp(self): - self.project = get(Project, main_language_project=None, - install_project=False, requirements_file='urls.py') + self.project = get( + Project, + main_language_project=None, + install_project=False, + requirements_file='urls.py', + ) self.version = get(Version, project=self.project) def test_python_supported_versions_default_image_1_0(self, load_config): @@ -57,21 +66,26 @@ def test_python_supported_versions_default_image_1_0(self, load_config): config = load_yaml_config(self.version) self.assertEqual(load_config.call_count, 1) load_config.assert_has_calls([ - mock.call(path=mock.ANY, env_config={ - 'build': {'image': 'readthedocs/build:1.0'}, - 'type': 'sphinx', - 'output_base': '', - 'name': mock.ANY, - 'defaults': { - 'install_project': self.project.install_project, - 'formats': ['htmlzip', 'epub', 'pdf'], - 'use_system_packages': self.project.use_system_packages, - 'requirements_file': self.project.requirements_file, - 'python_version': 2, - 'sphinx_configuration': mock.ANY, - 'build_image': 'readthedocs/build:1.0', + mock.call( + path=mock.ANY, + env_config={ + 'build': {'image': 'readthedocs/build:1.0'}, + 'type': 'sphinx', + 'output_base': '', + 'name': mock.ANY, + 'defaults': { + 'install_project': self.project.install_project, + 'formats': ['htmlzip', + 'epub', + 'pdf'], + 'use_system_packages': self.project.use_system_packages, + 'requirements_file': self.project.requirements_file, + 'python_version': 2, + 'sphinx_configuration': mock.ANY, + 'build_image': 'readthedocs/build:1.0', + }, }, - }), + ), ]) self.assertEqual(config.python_version, 2) @@ -116,7 +130,7 @@ def test_python_set_python_version_on_project(self, load_config): def test_python_set_python_version_in_config(self, load_config): load_config.side_effect = create_load({ - 'python': {'version': 3.5} + 'python': {'version': 3.5}, }) self.project.container_image = 'readthedocs/build:2.0' self.project.save() From cbbdd781a32c37f4901a28a1b3c7929e3ee2c2c3 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Fri, 13 Jul 2018 17:48:49 -0500 Subject: [PATCH 55/67] Autocommit --- readthedocs/config/tests/test_config.py | 246 ++++++++++++------------ 1 file changed, 120 insertions(+), 126 deletions(-) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index 678c79376d2..7abbf84f2d4 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from __future__ import division, print_function, unicode_literals import os @@ -19,33 +20,29 @@ from .utils import apply_fs env_config = { - 'output_base': '/tmp' + 'output_base': '/tmp', } - minimal_config = { 'name': 'docs', 'type': 'sphinx', } - config_with_explicit_empty_list = { 'readthedocs.yml': ''' name: docs type: sphinx formats: [] -''' +''', } - minimal_config_dir = { 'readthedocs.yml': '''\ name: docs type: sphinx -''' +''', } - multiple_config_dir = { 'readthedocs.yml': ''' name: first @@ -64,7 +61,8 @@ def get_build_config(config, env_config=None, source_file='readthedocs.yml', env_config or {}, config, source_file=source_file, - source_position=source_position) + source_position=source_position, + ) def get_env_config(extra=None): @@ -158,7 +156,8 @@ def test_build_config_has_source_position(tmpdir): assert len(builds) == 2 first, second = filter( lambda b: not b.source_file.endswith('nested/readthedocs.yml'), - builds) + builds, + ) assert first.source_position == 0 assert second.source_position == 1 @@ -172,9 +171,10 @@ def test_build_config_has_list_with_single_empty_value(tmpdir): def test_config_requires_name(): build = BuildConfigV1( - {'output_base': ''}, {}, + {'output_base': ''}, + {}, source_file='readthedocs.yml', - source_position=0 + source_position=0, ) with raises(InvalidConfig) as excinfo: build.validate() @@ -187,7 +187,7 @@ def test_build_requires_valid_name(): {'output_base': ''}, {'name': 'with/slashes'}, source_file='readthedocs.yml', - source_position=0 + source_position=0, ) with raises(InvalidConfig) as excinfo: build.validate() @@ -197,9 +197,10 @@ def test_build_requires_valid_name(): def test_config_requires_type(): build = BuildConfigV1( - {'output_base': ''}, {'name': 'docs'}, + {'output_base': ''}, + {'name': 'docs'}, source_file='readthedocs.yml', - source_position=0 + source_position=0, ) with raises(InvalidConfig) as excinfo: build.validate() @@ -210,9 +211,10 @@ def test_config_requires_type(): def test_build_requires_valid_type(): build = BuildConfigV1( {'output_base': ''}, - {'name': 'docs', 'type': 'unknown'}, + {'name': 'docs', + 'type': 'unknown'}, source_file='readthedocs.yml', - source_position=0 + source_position=0, ) with raises(InvalidConfig) as excinfo: build.validate() @@ -274,7 +276,7 @@ def it_defaults_to_list(): def it_validates_is_a_list(): build = get_build_config( {'python': {'extra_requirements': 'invalid'}}, - get_env_config() + get_env_config(), ) with raises(InvalidConfig) as excinfo: build.validate() @@ -286,13 +288,14 @@ def it_uses_validate_string(validate_string): validate_string.return_value = True build = get_build_config( {'python': {'extra_requirements': ['tests']}}, - get_env_config() + get_env_config(), ) build.validate() validate_string.assert_any_call('tests') def describe_validate_use_system_site_packages(): + def it_defaults_to_false(): build = get_build_config({'python': {}}, get_env_config()) build.validate() @@ -301,7 +304,7 @@ def it_defaults_to_false(): def it_validates_value(): build = get_build_config( {'python': {'use_system_site_packages': 'invalid'}}, - get_env_config() + get_env_config(), ) with raises(InvalidConfig) as excinfo: build.validate() @@ -313,7 +316,7 @@ def it_uses_validate_bool(validate_bool): validate_bool.return_value = True build = get_build_config( {'python': {'use_system_site_packages': 'to-validate'}}, - get_env_config() + get_env_config(), ) build.validate() validate_bool.assert_any_call('to-validate') @@ -329,7 +332,7 @@ def it_defaults_to_false(): def it_validates_value(): build = get_build_config( {'python': {'setup_py_install': 'this-is-string'}}, - get_env_config() + get_env_config(), ) with raises(InvalidConfig) as excinfo: build.validate() @@ -341,7 +344,7 @@ def it_uses_validate_bool(validate_bool): validate_bool.return_value = True build = get_build_config( {'python': {'setup_py_install': 'to-validate'}}, - get_env_config() + get_env_config(), ) build.validate() validate_bool.assert_any_call('to-validate') @@ -359,7 +362,7 @@ def it_defaults_to_a_valid_version(): def it_supports_other_versions(): build = get_build_config( {'python': {'version': 3.5}}, - get_env_config() + get_env_config(), ) build.validate() assert build.python_version == 3.5 @@ -369,7 +372,7 @@ def it_supports_other_versions(): def it_validates_versions_out_of_range(): build = get_build_config( {'python': {'version': 1.0}}, - get_env_config() + get_env_config(), ) with raises(InvalidConfig) as excinfo: build.validate() @@ -379,7 +382,7 @@ def it_validates_versions_out_of_range(): def it_validates_wrong_type(): build = get_build_config( {'python': {'version': 'this-is-string'}}, - get_env_config() + get_env_config(), ) with raises(InvalidConfig) as excinfo: build.validate() @@ -389,7 +392,7 @@ def it_validates_wrong_type(): def it_validates_wrong_type_right_value(): build = get_build_config( {'python': {'version': '3.5'}}, - get_env_config() + get_env_config(), ) build.validate() assert build.python_version == 3.5 @@ -398,7 +401,7 @@ def it_validates_wrong_type_right_value(): build = get_build_config( {'python': {'version': '3'}}, - get_env_config() + get_env_config(), ) build.validate() assert build.python_version == 3 @@ -437,11 +440,11 @@ def it_validates_env_supported_versions(): @pytest.mark.parametrize('value', [2, 3]) def it_respects_default_value(value): defaults = { - 'python_version': value + 'python_version': value, } build = get_build_config( {}, - get_env_config({'defaults': defaults}) + get_env_config({'defaults': defaults}), ) build.validate() assert build.python_version == value @@ -520,7 +523,7 @@ def it_defaults_to_source_file_directory(tmpdir): 'readthedocs.yml': '', 'setup.py': '', }, - } + }, ) with tmpdir.as_cwd(): source_file = tmpdir.join('subdir', 'readthedocs.yml') @@ -528,13 +531,16 @@ def it_defaults_to_source_file_directory(tmpdir): build = get_build_config( {}, env_config=get_env_config(), - source_file=str(source_file)) + source_file=str(source_file), + ) build.validate() assert build.python['setup_py_path'] == str(setup_py) def it_validates_value(tmpdir): with tmpdir.as_cwd(): - build = get_build_config({'python': {'setup_py_path': 'this-is-string'}}) + build = get_build_config({ + 'python': {'setup_py_path': 'this-is-string'} + }) with raises(InvalidConfig) as excinfo: build.validate_python() assert excinfo.value.key == 'python.setup_py_path' @@ -547,18 +553,19 @@ def it_uses_validate_file(tmpdir): patcher = patch('readthedocs.config.config.validate_file') with patcher as validate_file: validate_file.return_value = path - build = get_build_config( - {'python': {'setup_py_path': 'setup.py'}}) + build = get_build_config({'python': {'setup_py_path': 'setup.py'}},) build.validate_python() args, kwargs = validate_file.call_args assert args[0] == 'setup.py' def test_valid_build_config(): - build = BuildConfigV1(env_config, - minimal_config, - source_file='readthedocs.yml', - source_position=0) + build = BuildConfigV1( + env_config, + minimal_config, + source_file='readthedocs.yml', + source_position=0, + ) build.validate() assert build.name == 'docs' assert build.type == 'sphinx' @@ -579,7 +586,8 @@ def it_validates_to_abspath(tmpdir): get_env_config(), {'base': '../docs'}, source_file=source_file, - source_position=0) + source_position=0, + ) build.validate() assert build.base == str(tmpdir.join('docs')) @@ -599,7 +607,8 @@ def it_fails_if_base_is_not_a_string(tmpdir): get_env_config(), {'base': 1}, source_file=str(tmpdir.join('readthedocs.yml')), - source_position=0) + source_position=0, + ) with raises(InvalidConfig) as excinfo: build.validate() assert excinfo.value.key == 'base' @@ -611,7 +620,8 @@ def it_fails_if_base_does_not_exist(tmpdir): get_env_config(), {'base': 'docs'}, source_file=str(tmpdir.join('readthedocs.yml')), - source_position=0) + source_position=0, + ) with raises(InvalidConfig) as excinfo: build.validate() assert excinfo.value.key == 'base' @@ -626,7 +636,8 @@ def it_fails_if_build_is_invalid_option(tmpdir): get_env_config(), {'build': {'image': 3.0}}, source_file=str(tmpdir.join('readthedocs.yml')), - source_position=0) + source_position=0, + ) with raises(InvalidConfig) as excinfo: build.validate() assert excinfo.value.key == 'build' @@ -641,7 +652,8 @@ def it_fails_on_python_validation(tmpdir): 'python': {'version': '3.3'}, }, source_file=str(tmpdir.join('readthedocs.yml')), - source_position=0) + source_position=0, + ) build.validate_build() with raises(InvalidConfig) as excinfo: build.validate_python() @@ -657,7 +669,8 @@ def it_works_on_python_validation(tmpdir): 'python': {'version': '3.3'}, }, source_file=str(tmpdir.join('readthedocs.yml')), - source_position=0) + source_position=0, + ) build.validate_build() build.validate_python() @@ -667,7 +680,8 @@ def it_works(tmpdir): get_env_config(), {'build': {'image': 'latest'}}, source_file=str(tmpdir.join('readthedocs.yml')), - source_position=0) + source_position=0, + ) build.validate() assert build.build_image == 'readthedocs/build:latest' @@ -677,7 +691,8 @@ def default(tmpdir): get_env_config(), {}, source_file=str(tmpdir.join('readthedocs.yml')), - source_position=0) + source_position=0, + ) build.validate() assert build.build_image == 'readthedocs/build:2.0' @@ -692,7 +707,7 @@ def it_priorities_image_from_env_config(tmpdir, image): get_env_config({'defaults': defaults}), {'build': {'image': 'latest'}}, source_file=str(tmpdir.join('readthedocs.yml')), - source_position=0 + source_position=0, ) build.validate() assert build.build_image == image @@ -707,7 +722,7 @@ def test_use_conda_default_false(): def test_use_conda_respects_config(): build = get_build_config( {'conda': {}}, - get_env_config() + get_env_config(), ) build.validate() assert build.use_conda is True @@ -718,7 +733,7 @@ def test_validates_conda_file(tmpdir): build = get_build_config( {'conda': {'file': 'environment.yml'}}, get_env_config(), - source_file=str(tmpdir.join('readthedocs.yml')) + source_file=str(tmpdir.join('readthedocs.yml')), ) build.validate() assert build.use_conda is True @@ -734,12 +749,12 @@ def test_requirements_file_empty(): def test_requirements_file_repects_default_value(tmpdir): apply_fs(tmpdir, {'myrequirements.txt': ''}) defaults = { - 'requirements_file': 'myrequirements.txt' + 'requirements_file': 'myrequirements.txt', } build = get_build_config( {}, get_env_config({'defaults': defaults}), - source_file=str(tmpdir.join('readthedocs.yml')) + source_file=str(tmpdir.join('readthedocs.yml')), ) build.validate() assert build.requirements_file == 'myrequirements.txt' @@ -750,26 +765,28 @@ def test_requirements_file_respects_configuration(tmpdir): build = get_build_config( {'requirements_file': 'requirements.txt'}, get_env_config(), - source_file=str(tmpdir.join('readthedocs.yml')) + source_file=str(tmpdir.join('readthedocs.yml')), ) build.validate() assert build.requirements_file == 'requirements.txt' - def test_build_validate_calls_all_subvalidators(tmpdir): apply_fs(tmpdir, minimal_config) build = BuildConfigV1( {}, {}, source_file=str(tmpdir.join('readthedocs.yml')), - source_position=0) - with patch.multiple(BuildConfigV1, - validate_base=DEFAULT, - validate_name=DEFAULT, - validate_type=DEFAULT, - validate_python=DEFAULT, - validate_output_base=DEFAULT): + source_position=0, + ) + with patch.multiple( + BuildConfigV1, + validate_base=DEFAULT, + validate_name=DEFAULT, + validate_type=DEFAULT, + validate_python=DEFAULT, + validate_output_base=DEFAULT, + ): build.validate() BuildConfigV1.validate_base.assert_called_with() BuildConfigV1.validate_name.assert_called_with() @@ -785,7 +802,8 @@ def test_validate_project_config(): env_config, minimal_config, source_file='readthedocs.yml', - source_position=0) + source_position=0, + ), ]) project.validate() assert build_validate.call_count == 1 @@ -816,7 +834,7 @@ def get_build_config(self, config, env_config=None, env_config or {}, config, source_file=source_file, - source_position=source_position + source_position=source_position, ) def test_version(self): @@ -852,7 +870,7 @@ def test_formats_default_value(self): def test_formats_respect_default_values(self): build = self.get_build_config( {}, - {'defaults': {'formats': ['htmlzip']}} + {'defaults': {'formats': ['htmlzip']}}, ) build.validate() assert build.formats == ['htmlzip'] @@ -860,14 +878,14 @@ def test_formats_respect_default_values(self): def test_formats_priority_over_defaults(self): build = self.get_build_config( {'formats': []}, - {'defaults': {'formats': ['htmlzip']}} + {'defaults': {'formats': ['htmlzip']}}, ) build.validate() assert build.formats == [] build = self.get_build_config( {'formats': ['pdf']}, - {'defaults': {'formats': ['htmlzip']}} + {'defaults': {'formats': ['htmlzip']}}, ) build.validate() assert build.formats == ['pdf'] @@ -939,7 +957,7 @@ def test_build_image_check_invalid(self, value): def test_build_image_priorities_default(self, image): build = self.get_build_config( {'build': {'image': 'latest'}}, - {'defaults': {'build_image': image}} + {'defaults': {'build_image': image}}, ) build.validate() assert build.build.image == image @@ -948,7 +966,7 @@ def test_build_image_priorities_default(self, image): def test_build_image_over_empty_default(self, image): build = self.get_build_config( {'build': {'image': 'latest'}}, - {'defaults': {'build_image': image}} + {'defaults': {'build_image': image}}, ) build.validate() assert build.build.image == 'readthedocs/build:latest' @@ -989,7 +1007,7 @@ def test_python_version(self, image, versions): 'image': image, }, 'python': { - 'version': version + 'version': version, }, }) build.validate() @@ -1033,7 +1051,7 @@ def test_python_version_default(self): def test_python_version_respects_default(self, value): build = self.get_build_config( {}, - {'defaults': {'python_version': value}} + {'defaults': {'python_version': value}}, ) build.validate() assert build.python.version == value @@ -1042,7 +1060,7 @@ def test_python_version_respects_default(self, value): def test_python_version_priority_over_default(self, value): build = self.get_build_config( {'python': {'version': value}}, - {'defaults': {'python_version': 3}} + {'defaults': {'python_version': 3}}, ) build.validate() assert build.python.version == value @@ -1079,16 +1097,12 @@ def test_python_requirements_default_value(self): assert build.python.requirements is None def test_python_requirements_allow_null(self): - build = self.get_build_config( - {'python': {'requirements': None}} - ) + build = self.get_build_config({'python': {'requirements': None}},) build.validate() assert build.python.requirements is None def test_python_requirements_allow_empty_string(self): - build = self.get_build_config( - {'python': {'requirements': ''}} - ) + build = self.get_build_config({'python': {'requirements': ''}},) build.validate() assert build.python.requirements == '' @@ -1114,17 +1128,13 @@ def test_python_requirements_priority_over_default(self, tmpdir): @pytest.mark.parametrize('value', [3, [], {}]) def test_python_requirements_check_invalid_types(self, value): - build = self.get_build_config( - {'python': {'requirements': value}} - ) + build = self.get_build_config({'python': {'requirements': value}},) with raises(InvalidConfig) as excinfo: build.validate() assert excinfo.value.key == 'python.requirements' def test_python_install_pip_check_valid(self): - build = self.get_build_config( - {'python': {'install': 'pip'}} - ) + build = self.get_build_config({'python': {'install': 'pip'}},) build.validate() assert build.python.install_with_pip is True assert build.python.install_with_setup is False @@ -1132,16 +1142,14 @@ def test_python_install_pip_check_valid(self): def test_python_install_pip_priority_over_default(self): build = self.get_build_config( {'python': {'install': 'pip'}}, - {'defaults': {'install_project': True}} + {'defaults': {'install_project': True}}, ) build.validate() assert build.python.install_with_pip is True assert build.python.install_with_setup is False def test_python_install_setuppy_check_valid(self): - build = self.get_build_config( - {'python': {'install': 'setup.py'}} - ) + build = self.get_build_config({'python': {'install': 'setup.py'}},) build.validate() assert build.python.install_with_setup is True assert build.python.install_with_pip is False @@ -1149,7 +1157,7 @@ def test_python_install_setuppy_check_valid(self): def test_python_install_setuppy_respects_default(self): build = self.get_build_config( {}, - {'defaults': {'install_project': True}} + {'defaults': {'install_project': True}}, ) build.validate() assert build.python.install_with_pip is False @@ -1158,7 +1166,7 @@ def test_python_install_setuppy_respects_default(self): def test_python_install_setuppy_priority_over_default(self): build = self.get_build_config( {'python': {'install': 'setup.py'}}, - {'defaults': {'install_project': False}} + {'defaults': {'install_project': False}}, ) build.validate() assert build.python.install_with_pip is False @@ -1166,17 +1174,13 @@ def test_python_install_setuppy_priority_over_default(self): @pytest.mark.parametrize('value', ['invalid', 'apt']) def test_python_install_check_invalid(self, value): - build = self.get_build_config( - {'python': {'install': value}} - ) + build = self.get_build_config({'python': {'install': value}},) with raises(InvalidConfig) as excinfo: build.validate() assert excinfo.value.key == 'python.install' def test_python_install_allow_null(self): - build = self.get_build_config( - {'python': {'install': None}} - ) + build = self.get_build_config({'python': {'install': None}},) build.validate() assert build.python.install_with_pip is False assert build.python.install_with_setup is False @@ -1189,9 +1193,7 @@ def test_python_install_default(self): @pytest.mark.parametrize('value', [2, [], {}]) def test_python_install_check_invalid_type(self, value): - build = self.get_build_config( - {'python': {'install': value}} - ) + build = self.get_build_config({'python': {'install': value}},) with raises(InvalidConfig) as excinfo: build.validate() assert excinfo.value.key == 'python.install' @@ -1233,7 +1235,7 @@ def test_python_extra_requirements_check_type(self, value): 'python': { 'install': 'pip', 'extra_requirements': value, - } + }, }) with raises(InvalidConfig) as excinfo: build.validate() @@ -1244,7 +1246,7 @@ def test_python_extra_requirements_allow_empty(self): 'python': { 'install': 'pip', 'extra_requirements': [], - } + }, }) build.validate() assert build.python.extra_requirements == [] @@ -1259,7 +1261,7 @@ def test_python_system_packages_check_valid(self, value): build = self.get_build_config({ 'python': { 'system_packages': value, - } + }, }) build.validate() assert build.python.use_system_site_packages is value @@ -1269,7 +1271,7 @@ def test_python_system_packages_check_invalid(self, value): build = self.get_build_config({ 'python': { 'system_packages': value, - } + }, }) with raises(InvalidConfig) as excinfo: build.validate() @@ -1283,7 +1285,7 @@ def test_python_system_packages_check_default(self): def test_python_system_packages_respects_default(self): build = self.get_build_config( {}, - {'defaults': {'use_system_packages': True}} + {'defaults': {'use_system_packages': True}}, ) build.validate() assert build.python.use_system_site_packages is True @@ -1291,14 +1293,14 @@ def test_python_system_packages_respects_default(self): def test_python_system_packages_priority_over_default(self): build = self.get_build_config( {'python': {'system_packages': False}}, - {'defaults': {'use_system_packages': True}} + {'defaults': {'use_system_packages': True}}, ) build.validate() assert build.python.use_system_site_packages is False build = self.get_build_config( {'python': {'system_packages': True}}, - {'defaults': {'use_system_packages': False}} + {'defaults': {'use_system_packages': False}}, ) build.validate() assert build.python.use_system_site_packages is True @@ -1343,9 +1345,7 @@ def test_sphinx_cant_be_used_with_mkdocs(self, tmpdir): assert excinfo.value.key == '.' def test_sphinx_configuration_allow_null(self): - build = self.get_build_config( - {'sphinx': {'configuration': None}} - ) + build = self.get_build_config({'sphinx': {'configuration': None}},) build.validate() assert build.sphinx.configuration is None @@ -1386,9 +1386,7 @@ def test_sphinx_configuration_priorities_over_default(self, tmpdir): @pytest.mark.parametrize('value', [[], True, 0, {}]) def test_sphinx_configuration_validate_type(self, value): - build = self.get_build_config( - {'sphinx': {'configuration': value}} - ) + build = self.get_build_config({'sphinx': {'configuration': value}},) with raises(InvalidConfig) as excinfo: build.validate() assert excinfo.value.key == 'sphinx.configuration' @@ -1438,9 +1436,7 @@ def test_mkdocs_configuration_check_invalid(self, tmpdir): assert excinfo.value.key == 'mkdocs.configuration' def test_mkdocs_configuration_allow_null(self): - build = self.get_build_config( - {'mkdocs': {'configuration': None}} - ) + build = self.get_build_config({'mkdocs': {'configuration': None}},) build.validate() assert build.mkdocs.configuration is None @@ -1451,9 +1447,7 @@ def test_mkdocs_configuration_check_default(self): @pytest.mark.parametrize('value', [[], True, 0, {}]) def test_mkdocs_configuration_validate_type(self, value): - build = self.get_build_config( - {'mkdocs': {'configuration': value}} - ) + build = self.get_build_config({'mkdocs': {'configuration': value}},) with raises(InvalidConfig) as excinfo: build.validate() assert excinfo.value.key == 'mkdocs.configuration' @@ -1499,7 +1493,7 @@ def test_submodules_include_check_invalid(self, value): build = self.get_build_config({ 'submodules': { 'include': value, - } + }, }) with raises(InvalidConfig) as excinfo: build.validate() @@ -1509,7 +1503,7 @@ def test_submodules_include_allows_all_keyword(self): build = self.get_build_config({ 'submodules': { 'include': 'all', - } + }, }) build.validate() assert build.submodules.include == ALL @@ -1532,7 +1526,7 @@ def test_submodules_exclude_check_invalid(self, value): build = self.get_build_config({ 'submodules': { 'exclude': value, - } + }, }) with raises(InvalidConfig) as excinfo: build.validate() @@ -1542,7 +1536,7 @@ def test_submodules_exclude_allows_all_keyword(self): build = self.get_build_config({ 'submodules': { 'exclude': 'all', - } + }, }) build.validate() assert build.submodules.include == [] @@ -1554,7 +1548,7 @@ def test_submodules_cant_exclude_and_include(self): 'submodules': { 'include': ['two'], 'exclude': ['one'], - } + }, }) with raises(InvalidConfig) as excinfo: build.validate() @@ -1565,7 +1559,7 @@ def test_submodules_can_exclude_include_be_empty(self): 'submodules': { 'exclude': 'all', 'include': [], - } + }, }) build.validate() assert build.submodules.include == [] @@ -1578,7 +1572,7 @@ def test_submodules_recursive_check_valid(self, value): 'submodules': { 'include': ['one', 'two'], 'recursive': value, - } + }, }) build.validate() assert build.submodules.include == ['one', 'two'] @@ -1591,7 +1585,7 @@ def test_submodules_recursive_check_invalid(self, value): 'submodules': { 'include': ['one', 'two'], 'recursive': value, - } + }, }) with raises(InvalidConfig) as excinfo: build.validate() @@ -1602,7 +1596,7 @@ def test_submodules_recursive_explict_default(self): 'submodules': { 'include': [], 'recursive': False, - } + }, }) build.validate() assert build.submodules.include == [] @@ -1613,7 +1607,7 @@ def test_submodules_recursive_explict_default(self): 'submodules': { 'exclude': [], 'recursive': False, - } + }, }) build.validate() assert build.submodules.include == [] From 50f199cba86ab4094768d545663317569d8fb521 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 16 Jul 2018 12:22:58 -0500 Subject: [PATCH 56/67] Ignore pylint warning --- readthedocs/config/config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index b02a9a2c104..cf41cf5964c 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +# pylint: disable=too-many-lines + """Build configuration for rtd.""" from __future__ import division, print_function, unicode_literals From 8be5a073a3fbd7697f58d7f8c84702c43b383850 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 16 Jul 2018 12:54:54 -0500 Subject: [PATCH 57/67] Fix tests Some defaults (vaues from the db) are ignored. --- readthedocs/config/tests/test_config.py | 31 +++++++++++++++++-------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index 7abbf84f2d4..6150f99f940 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -866,14 +866,13 @@ def test_formats_default_value(self): build.validate() assert build.formats == [] - @pytest.mark.skip('Needs a decision') - def test_formats_respect_default_values(self): + def test_formats_overrides_default_values(self): build = self.get_build_config( {}, {'defaults': {'formats': ['htmlzip']}}, ) build.validate() - assert build.formats == ['htmlzip'] + assert build.formats == [] def test_formats_priority_over_defaults(self): build = self.get_build_config( @@ -1048,13 +1047,13 @@ def test_python_version_default(self): assert build.python.version == 3 @pytest.mark.parametrize('value', [2, 3]) - def test_python_version_respects_default(self, value): + def test_python_version_overrides_default(self, value): build = self.get_build_config( {}, {'defaults': {'python_version': value}}, ) build.validate() - assert build.python.version == value + assert build.python.version == 3 @pytest.mark.parametrize('value', [2, 3, 3.6]) def test_python_version_priority_over_default(self, value): @@ -1312,6 +1311,12 @@ def test_sphinx_validate_type(self, value): build.validate() assert excinfo.value.key == 'sphinx' + def test_sphinx_is_default_doc_type(self): + build = self.get_build_config({}) + build.validate() + assert build.sphinx is not None + assert build.mkdocs is None + def test_sphinx_configuration_check_valid(self, tmpdir): apply_fs(tmpdir, {'conf.py': ''}) build = self.get_build_config( @@ -1364,7 +1369,7 @@ def test_sphinx_configuration_respects_default(self, tmpdir): build.validate() assert build.sphinx.configuration == str(tmpdir.join('conf.py')) - def test_sphinx_configuration_defautl_can_be_none(self, tmpdir): + def test_sphinx_configuration_default_can_be_none(self, tmpdir): apply_fs(tmpdir, {'conf.py': ''}) build = self.get_build_config( {}, @@ -1378,7 +1383,7 @@ def test_sphinx_configuration_priorities_over_default(self, tmpdir): apply_fs(tmpdir, {'conf.py': '', 'conf-default.py': ''}) build = self.get_build_config( {'sphinx': {'configuration': 'conf.py'}}, - {'defaults': {'sphinx_configuration': 'conf-defaul.py'}}, + {'defaults': {'sphinx_configuration': 'conf-default.py'}}, source_file=str(tmpdir.join('readthedocs.yml')), ) build.validate() @@ -1416,6 +1421,11 @@ def test_mkdocs_validate_type(self, value): build.validate() assert excinfo.value.key == 'mkdocs' + def test_mkdocs_default(self): + build = self.get_build_config({}) + build.validate() + assert build.mkdocs is None + def test_mkdocs_configuration_check_valid(self, tmpdir): apply_fs(tmpdir, {'mkdocs.yml': ''}) build = self.get_build_config( @@ -1424,6 +1434,7 @@ def test_mkdocs_configuration_check_valid(self, tmpdir): ) build.validate() assert build.mkdocs.configuration == str(tmpdir.join('mkdocs.yml')) + assert build.sphinx is None def test_mkdocs_configuration_check_invalid(self, tmpdir): apply_fs(tmpdir, {'mkdocs.yml': ''}) @@ -1441,7 +1452,7 @@ def test_mkdocs_configuration_allow_null(self): assert build.mkdocs.configuration is None def test_mkdocs_configuration_check_default(self): - build = self.get_build_config({}) + build = self.get_build_config({'mkdocs': {}}) build.validate() assert build.mkdocs.configuration is None @@ -1466,7 +1477,7 @@ def test_mkdocs_fail_on_warning_check_invalid(self, value): assert excinfo.value.key == 'mkdocs.fail_on_warning' def test_mkdocs_fail_on_warning_check_default(self): - build = self.get_build_config({}) + build = self.get_build_config({'mkdocs': {}}) build.validate() assert build.mkdocs.fail_on_warning is False @@ -1481,7 +1492,7 @@ def test_submodules_include_check_valid(self): build = self.get_build_config({ 'submodules': { 'include': ['one', 'two'] - } + }, }) build.validate() assert build.submodules.include == ['one', 'two'] From 08069c0cfeb1a2dd02aa6e89642baadb1e9b47ef Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 16 Jul 2018 12:55:55 -0500 Subject: [PATCH 58/67] Ignore some values from the db --- readthedocs/config/config.py | 78 +++++++++++++++++++++--------------- 1 file changed, 45 insertions(+), 33 deletions(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index cf41cf5964c..a3eabf06ada 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -615,7 +615,13 @@ class BuildConfigV2(BuildConfigBase): valid_install_options = ['pip', 'setup.py'] def validate(self): - """Validates and process ``raw_config`` and ``env_config``.""" + """ + Validates and process ``raw_config`` and ``env_config``. + + Sphinx is the default doc type to be built. We don't merge some values + from the database (like formats or python.version) to allow us set + default values. + """ self._config['formats'] = self.validate_formats() self._config['conda'] = self.validate_conda() # This should be called before validate_python @@ -623,8 +629,8 @@ def validate(self): self._config['python'] = self.validate_python() # Call this before validate sphinx and mkdocs self.validate_doc_types() - self._config['sphinx'] = self.validate_sphinx() self._config['mkdocs'] = self.validate_mkdocs() + self._config['sphinx'] = self.validate_sphinx() self._config['submodules'] = self.validate_submodules() def validate_formats(self): @@ -632,6 +638,7 @@ def validate_formats(self): Validates that formats contains only valid formats. The ``ALL`` keyword can be used to indicate that all formats are used. + We ignore the default values here. """ formats = self.raw_config.get('formats', []) if formats == ALL: @@ -691,7 +698,6 @@ def validate_python(self): build.image attribute. Fall back to the defaults of: - - ``version`` - ``requirements`` - ``install`` (only for setup.py method) - ``system_packages`` @@ -706,8 +712,7 @@ def validate_python(self): python = {} with self.catch_validation_error('python.version'): - version = self.defaults.get('python_version', 3) - version = raw_python.get('version', version) + version = raw_python.get('version', 3) if isinstance(version, six.string_types): try: version = int(version) @@ -796,15 +801,48 @@ def validate_doc_types(self): code=INVALID_KEYS_COMBINATION, ) + def validate_mkdocs(self): + """ + Validates the mkdocs key. + + It makes sure we are using an existing configuration file. + """ + raw_mkdocs = self.raw_config.get('mkdocs') + if raw_mkdocs is None: + return None + + with self.catch_validation_error('mkdocs'): + validate_dict(raw_mkdocs) + + mkdocs = {} + with self.catch_validation_error('mkdocs.configuration'): + configuration = raw_mkdocs.get('configuration') + if configuration is not None: + configuration = validate_file(configuration, self.base_path) + mkdocs['configuration'] = configuration + + with self.catch_validation_error('mkdocs.fail_on_warning'): + fail_on_warning = raw_mkdocs.get('fail_on_warning', False) + mkdocs['fail_on_warning'] = validate_bool(fail_on_warning) + + return mkdocs + def validate_sphinx(self): """ Validates the sphinx key. It makes sure we are using an existing configuration file. + + .. note:: + It should be called after ``validate_mkdocs``. That way + we can default to sphinx if ``mkdocs`` is not given. """ - raw_sphinx = self.raw_config.get('sphinx', {}) + raw_sphinx = self.raw_config.get('sphinx') if raw_sphinx is None: - return None + if self.mkdocs is None: + raw_sphinx = {} + else: + return None with self.catch_validation_error('sphinx'): validate_dict(raw_sphinx) @@ -826,32 +864,6 @@ def validate_sphinx(self): return sphinx - def validate_mkdocs(self): - """ - Validates the mkdocs key. - - It makes sure we are using an existing configuration file. - """ - raw_mkdocs = self.raw_config.get('mkdocs', {}) - if raw_mkdocs is None: - return None - - with self.catch_validation_error('mkdocs'): - validate_dict(raw_mkdocs) - - mkdocs = {} - with self.catch_validation_error('mkdocs.configuration'): - configuration = raw_mkdocs.get('configuration') - if configuration is not None: - configuration = validate_file(configuration, self.base_path) - mkdocs['configuration'] = configuration - - with self.catch_validation_error('mkdocs.fail_on_warning'): - fail_on_warning = raw_mkdocs.get('fail_on_warning', False) - mkdocs['fail_on_warning'] = validate_bool(fail_on_warning) - - return mkdocs - def validate_submodules(self): """ Validates the submodules key. From 2e759ceddbf7464d2a933e7e19c3ab53c3ff673b Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 16 Jul 2018 13:52:25 -0500 Subject: [PATCH 59/67] Remove unused keys from v1 This ones have a replace in v2 --- readthedocs/config/config.py | 40 ------------------- readthedocs/config/tests/test_config.py | 38 +----------------- readthedocs/doc_builder/config.py | 16 ++------ .../tests/test_config_integration.py | 10 ++--- 4 files changed, 9 insertions(+), 95 deletions(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index a3eabf06ada..2a9f9f06a30 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -42,7 +42,6 @@ NAME_REQUIRED = 'name-required' NAME_INVALID = 'name-invalid' CONF_FILE_REQUIRED = 'conf-file-required' -TYPE_REQUIRED = 'type-required' PYTHON_INVALID = 'python-invalid' SUBMODULES_INVALID = 'submodules-invalid' INVALID_KEYS_COMBINATION = 'invalid-keys-combination' @@ -204,7 +203,6 @@ class BuildConfigV1(BuildConfigBase): NAME_INVALID_MESSAGE = ( 'Invalid name "{name}". Valid values must match {name_re}' ) - TYPE_REQUIRED_MESSAGE = 'Missing key "type"' CONF_FILE_REQUIRED_MESSAGE = 'Missing key "conf_file"' PYTHON_INVALID_MESSAGE = '"python" section must be a mapping.' PYTHON_EXTRA_REQUIREMENTS_INVALID_MESSAGE = ( @@ -216,10 +214,6 @@ class BuildConfigV1(BuildConfigBase): version = '1' - def get_valid_types(self): # noqa - """Get all valid types.""" - return ('sphinx',) - def get_valid_python_versions(self): """Get all valid python versions.""" try: @@ -242,7 +236,6 @@ def validate(self): It makes sure that: - - ``type`` is set and is a valid builder - ``base`` is a valid directory and defaults to the directory of the ``readthedocs.yml`` config file if not set """ @@ -258,16 +251,12 @@ def validate(self): # TODO: this isn't used self._config['name'] = self.validate_name() # TODO: this isn't used - self._config['type'] = self.validate_type() - # TODO: this isn't used self._config['base'] = self.validate_base() self._config['python'] = self.validate_python() self._config['formats'] = self.validate_formats() self._config['conda'] = self.validate_conda() self._config['requirements_file'] = self.validate_requirements_file() - # TODO: this isn't used - self._config['conf_file'] = self.validate_conf_file() def validate_output_base(self): """Validates that ``output_base`` exists and set its absolute path.""" @@ -299,19 +288,6 @@ def validate_name(self): return name - def validate_type(self): - """Validates that type is a valid choice.""" - type_ = self.raw_config.get('type', None) - if not type_: - type_ = self.env_config.get('type', None) - if not type_: - self.error('type', self.TYPE_REQUIRED_MESSAGE, code=TYPE_REQUIRED) - - with self.catch_validation_error('type'): - validate_choice(type_, self.get_valid_types()) - - return type_ - def validate_base(self): """Validates that path is a valid directory.""" if 'base' in self.raw_config: @@ -496,17 +472,6 @@ def validate_requirements_file(self): validate_file(requirements_file, base_path) return requirements_file - def validate_conf_file(self): - """Validates the conf.py file for sphinx.""" - if 'conf_file' not in self.raw_config: - return None - - conf_file = self.raw_config['conf_file'] - base_path = os.path.dirname(self.source_file) - with self.catch_validation_error('conf_file'): - validate_file(conf_file, base_path) - return conf_file - def validate_formats(self): """Validates that formats contains only valid formats.""" formats = self.raw_config.get('formats') @@ -537,11 +502,6 @@ def output_base(self): """The output base.""" return self._config['output_base'] - @property - def type(self): - """The documentation type.""" - return self._config['type'] - @property def formats(self): """The documentation formats to be built.""" diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index 6150f99f940..48ed7f85eae 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -13,7 +13,7 @@ ConfigOptionNotSupportedError, InvalidConfig, ProjectConfig, load) from readthedocs.config.config import ( CONFIG_NOT_SUPPORTED, NAME_INVALID, NAME_REQUIRED, PYTHON_INVALID, - TYPE_REQUIRED, VERSION_INVALID) + VERSION_INVALID) from readthedocs.config.validation import ( INVALID_BOOL, INVALID_CHOICE, INVALID_LIST, INVALID_PATH, INVALID_STRING) @@ -25,13 +25,11 @@ minimal_config = { 'name': 'docs', - 'type': 'sphinx', } config_with_explicit_empty_list = { 'readthedocs.yml': ''' name: docs -type: sphinx formats: [] ''', } @@ -39,17 +37,14 @@ minimal_config_dir = { 'readthedocs.yml': '''\ name: docs -type: sphinx ''', } multiple_config_dir = { 'readthedocs.yml': ''' name: first -type: sphinx --- name: second -type: sphinx ''', 'nested': minimal_config_dir, } @@ -70,7 +65,6 @@ def get_env_config(extra=None): defaults = { 'output_base': '', 'name': 'name', - 'type': 'sphinx', } if extra is None: extra = {} @@ -195,33 +189,6 @@ def test_build_requires_valid_name(): assert excinfo.value.code == NAME_INVALID -def test_config_requires_type(): - build = BuildConfigV1( - {'output_base': ''}, - {'name': 'docs'}, - source_file='readthedocs.yml', - source_position=0, - ) - with raises(InvalidConfig) as excinfo: - build.validate() - assert excinfo.value.key == 'type' - assert excinfo.value.code == TYPE_REQUIRED - - -def test_build_requires_valid_type(): - build = BuildConfigV1( - {'output_base': ''}, - {'name': 'docs', - 'type': 'unknown'}, - source_file='readthedocs.yml', - source_position=0, - ) - with raises(InvalidConfig) as excinfo: - build.validate() - assert excinfo.value.key == 'type' - assert excinfo.value.code == INVALID_CHOICE - - def test_version(): build = get_build_config({}, get_env_config()) assert build.version == '1' @@ -568,7 +535,6 @@ def test_valid_build_config(): ) build.validate() assert build.name == 'docs' - assert build.type == 'sphinx' assert build.base assert build.python assert 'setup_py_install' in build.python @@ -783,14 +749,12 @@ def test_build_validate_calls_all_subvalidators(tmpdir): BuildConfigV1, validate_base=DEFAULT, validate_name=DEFAULT, - validate_type=DEFAULT, validate_python=DEFAULT, validate_output_base=DEFAULT, ): build.validate() BuildConfigV1.validate_base.assert_called_with() BuildConfigV1.validate_name.assert_called_with() - BuildConfigV1.validate_type.assert_called_with() BuildConfigV1.validate_python.assert_called_with() BuildConfigV1.validate_output_base.assert_called_with() diff --git a/readthedocs/doc_builder/config.py b/readthedocs/doc_builder/config.py index 5431a682a13..df5ce45001b 100644 --- a/readthedocs/doc_builder/config.py +++ b/readthedocs/doc_builder/config.py @@ -36,6 +36,8 @@ def load_yaml_config(version): 'build': { 'image': img_name, }, + 'output_base': '', + 'name': version.slug, 'defaults': { 'install_project': project.install_project, 'formats': get_default_formats(project), @@ -52,26 +54,14 @@ def load_yaml_config(version): env_config['DOCKER_IMAGE_SETTINGS'] = img_settings try: - sphinx_env_config = env_config.copy() - sphinx_env_config.update({ - 'output_base': '', - 'type': 'sphinx', - 'name': version.slug, - }) config = load_config( path=checkout_path, - env_config=sphinx_env_config, + env_config=env_config, )[0] except InvalidConfig: # This is a subclass of ConfigError, so has to come first raise except ConfigError: - # TODO: this shouldn't be hardcoded here - env_config.update({ - 'output_base': '', - 'type': 'sphinx', - 'name': version.slug, - }) config = BuildConfigV1( env_config=env_config, raw_config={}, diff --git a/readthedocs/rtd_tests/tests/test_config_integration.py b/readthedocs/rtd_tests/tests/test_config_integration.py index ae95681c450..b78ca7f7ea5 100644 --- a/readthedocs/rtd_tests/tests/test_config_integration.py +++ b/readthedocs/rtd_tests/tests/test_config_integration.py @@ -27,7 +27,6 @@ def inner(path=None, env_config=None): env_config_defaults = { 'output_base': '', 'name': '1', - 'type': 'sphinx', } if env_config is not None: env_config_defaults.update(env_config) @@ -70,14 +69,15 @@ def test_python_supported_versions_default_image_1_0(self, load_config): path=mock.ANY, env_config={ 'build': {'image': 'readthedocs/build:1.0'}, - 'type': 'sphinx', 'output_base': '', 'name': mock.ANY, 'defaults': { 'install_project': self.project.install_project, - 'formats': ['htmlzip', - 'epub', - 'pdf'], + 'formats': [ + 'htmlzip', + 'epub', + 'pdf' + ], 'use_system_packages': self.project.use_system_packages, 'requirements_file': self.project.requirements_file, 'python_version': 2, From b4fa44f19d252f639e852c5e923ea4db270d1011 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 16 Jul 2018 14:59:19 -0500 Subject: [PATCH 60/67] Add feature flag for configuration file (v2) --- readthedocs/config/config.py | 9 +++++++-- readthedocs/config/tests/test_config.py | 6 +++--- readthedocs/doc_builder/config.py | 4 +++- readthedocs/projects/models.py | 3 +++ readthedocs/rtd_tests/tests/test_config_integration.py | 1 + 5 files changed, 17 insertions(+), 6 deletions(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index 2a9f9f06a30..47a6ca50d65 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -403,7 +403,8 @@ def validate_python(self): with self.catch_validation_error( 'python.extra_requirements'): python['extra_requirements'].append( - validate_string(extra_name)) + validate_string(extra_name) + ) # Validate setup_py_install. if 'setup_py_install' in raw_python: @@ -968,7 +969,11 @@ def load(path, env_config): code=CONFIG_SYNTAX_INVALID, ) for i, config in enumerate(configs): - version = config.get('version', 1) + allow_v2 = env_config.get('allow_v2') + if allow_v2: + version = config.get('version', 1) + else: + version = 1 build_config = get_configuration_class(version)( env_config, config, diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index 48ed7f85eae..98f32429065 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -104,7 +104,7 @@ def test_load_version1(tmpdir): ''') }) base = str(tmpdir) - config = load(base, get_env_config()) + config = load(base, get_env_config({'allow_v2': True})) assert isinstance(config, ProjectConfig) assert len(config) == 1 build = config[0] @@ -118,7 +118,7 @@ def test_load_version2(tmpdir): ''') }) base = str(tmpdir) - config = load(base, get_env_config()) + config = load(base, get_env_config({'allow_v2': True})) assert isinstance(config, ProjectConfig) assert len(config) == 1 build = config[0] @@ -133,7 +133,7 @@ def test_load_unknow_version(tmpdir): }) base = str(tmpdir) with raises(ConfigError) as excinfo: - load(base, get_env_config()) + load(base, get_env_config({'allow_v2': True})) assert excinfo.value.code == VERSION_INVALID diff --git a/readthedocs/doc_builder/config.py b/readthedocs/doc_builder/config.py index df5ce45001b..1cd1b85e564 100644 --- a/readthedocs/doc_builder/config.py +++ b/readthedocs/doc_builder/config.py @@ -6,7 +6,7 @@ from readthedocs.config import BuildConfigV1, ConfigError, InvalidConfig from readthedocs.config import load as load_config -from readthedocs.projects.models import ProjectConfigurationError +from readthedocs.projects.models import Feature, ProjectConfigurationError from .constants import DOCKER_IMAGE, DOCKER_IMAGE_SETTINGS @@ -27,12 +27,14 @@ def load_yaml_config(version): img_name = project.container_image or DOCKER_IMAGE python_version = 3 if project.python_interpreter == 'python3' else 2 + allow_v2 = project.has_feature(Feature.ALLOW_V2_CONFIG_FILE) try: sphinx_configuration = version.get_conf_py_path() except ProjectConfigurationError: sphinx_configuration = None env_config = { + 'allow_v2': allow_v2, 'build': { 'image': img_name, }, diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index 1d278b1f8ba..22613ce60ff 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -1022,6 +1022,7 @@ def add_features(sender, **kwargs): SKIP_SUBMODULES = 'skip_submodules' BUILD_JSON_ARTIFACTS_WITH_HTML = 'build_json_artifacts_with_html' DONT_OVERWRITE_SPHINX_CONTEXT = 'dont_overwrite_sphinx_context' + ALLOW_V2_CONFIG_FILE = 'allow_v2_config_file' FEATURES = ( (USE_SPHINX_LATEST, _('Use latest version of Sphinx')), @@ -1033,6 +1034,8 @@ def add_features(sender, **kwargs): 'Build the json artifacts with the html build step')), (DONT_OVERWRITE_SPHINX_CONTEXT, _( 'Do not overwrite context vars in conf.py with Read the Docs context',)), + (ALLOW_V2_CONFIG_FILE, _( + 'Allow to use the v2 of the configuration file')), ) projects = models.ManyToManyField( diff --git a/readthedocs/rtd_tests/tests/test_config_integration.py b/readthedocs/rtd_tests/tests/test_config_integration.py index b78ca7f7ea5..f0426cdee77 100644 --- a/readthedocs/rtd_tests/tests/test_config_integration.py +++ b/readthedocs/rtd_tests/tests/test_config_integration.py @@ -68,6 +68,7 @@ def test_python_supported_versions_default_image_1_0(self, load_config): mock.call( path=mock.ANY, env_config={ + 'allow_v2': mock.ANY, 'build': {'image': 'readthedocs/build:1.0'}, 'output_base': '', 'name': mock.ANY, From 046693e797ef945a72231413c820ea0310a996c5 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 18 Jul 2018 14:00:30 -0500 Subject: [PATCH 61/67] Include sphinx.builder to schema --- .../rtd_tests/fixtures/spec/v2/schema.yml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml b/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml index 6570a895cd2..645f420012e 100644 --- a/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml +++ b/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml @@ -1,4 +1,8 @@ # Read the Docs configuration file +# This schema uses https://github.com/23andMe/Yamale +# for the validation. +# Default values are indicated with a comment (``Default: ...``). +# Some values are default to the project config (settings from the web panel). # The version of the spec to be use version: enum('2') @@ -40,6 +44,7 @@ conda: build: # The build docker image to be used # Default: 'latest' + # Note: it can be overriden by a project image: enum('stable', 'latest', required=False) python: @@ -48,11 +53,11 @@ python: version: enum('2', '2.7', '3', '3.3', '3.4', '3.5', '3.6', required=False) # The path to the requirements file from the root of the project - # Default: null + # Default: null | project config requirements: path(required=False) # Install the project using python setup.py install or pip - # Default: null + # Default: null | project config install: enum('pip', 'setup.py', required=False) # Extra requirements sections to install in addition to the package dependencies @@ -60,12 +65,16 @@ python: extra_requirements: list(str(), required=False) # Give the virtual environment access to the global site-packages dir - # Default: false + # Default: false | project config system_packages: bool(required=False) sphinx: + # The builder type for the sphinx documentation + # Default: 'sphinx' + builder: enum('sphinx', 'htmldir', 'singlehtml', required=False) + # The path to the conf.py file - # Default: rtd will try to find it + # Default: rtd will try to find it | project config configuration: path(required=False) # Add the -W option to sphinx-build @@ -81,7 +90,6 @@ mkdocs: # Default: false fail_on_warning: bool(required=False) - submodules: # List of submodules to be included # Default: [] From 6dbec8c8be8660a7fea3c0934559e92d98dd5004 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 18 Jul 2018 14:30:56 -0500 Subject: [PATCH 62/67] Add tests for sphinx.builder key and doctype config property --- readthedocs/config/tests/test_config.py | 32 +++++++++++++++++++ .../rtd_tests/fixtures/spec/v2/schema.yml | 1 + 2 files changed, 33 insertions(+) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index 98f32429065..0b6dee31e49 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -1280,6 +1280,37 @@ def test_sphinx_is_default_doc_type(self): build.validate() assert build.sphinx is not None assert build.mkdocs is None + assert build.doctype == 'sphinx' + + @pytest.mark.parametrize('value,expected', + [('sphinx', 'sphinx'), + ('htmldir', 'sphinx_htmldir'), + ('singlehtml', 'sphinx_singlehtml')]) + def test_sphinx_builder_check_valid(self, value, expected): + build = self.get_build_config({'sphinx': {'builer': value}}) + build.validate() + assert build.sphinx.builder == expected + assert build.doctype == expected + + @pytest.mark.parametrize('value', [[], True, 0, 'invalid']) + def test_sphinx_builder_check_invalid(self, value): + build = self.get_build_config({'sphinx': {'builer': value}}) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'sphinx.builder' + + def test_sphinx_builder_default(self): + build = self.get_build_config({}) + build.validate() + build.sphinx.builder == 'sphinx' + + def test_sphinx_builder_ignores_default(self): + build = self.get_build_config( + {}, + {'defaults': {'doctype': 'sphinx_singlehtml'}}, + ) + build.validate() + build.sphinx.builder == 'sphinx' def test_sphinx_configuration_check_valid(self, tmpdir): apply_fs(tmpdir, {'conf.py': ''}) @@ -1398,6 +1429,7 @@ def test_mkdocs_configuration_check_valid(self, tmpdir): ) build.validate() assert build.mkdocs.configuration == str(tmpdir.join('mkdocs.yml')) + assert build.doctype == 'mkdocs' assert build.sphinx is None def test_mkdocs_configuration_check_invalid(self, tmpdir): diff --git a/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml b/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml index 645f420012e..6d16e311274 100644 --- a/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml +++ b/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml @@ -21,6 +21,7 @@ build: include('build', required=False) python: include('python', required=False) # Configuration for sphinx documentation +# Default documentation type sphinx: include('sphinx', required=False) # Configuration for mkdocs documentation From 05382ebbda2ebe553d1d44dfc87847a06d2184e8 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 18 Jul 2018 14:51:04 -0500 Subject: [PATCH 63/67] Fix tests --- readthedocs/config/tests/test_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index 0b6dee31e49..ff599fc7707 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -1287,14 +1287,14 @@ def test_sphinx_is_default_doc_type(self): ('htmldir', 'sphinx_htmldir'), ('singlehtml', 'sphinx_singlehtml')]) def test_sphinx_builder_check_valid(self, value, expected): - build = self.get_build_config({'sphinx': {'builer': value}}) + build = self.get_build_config({'sphinx': {'builder': value}}) build.validate() assert build.sphinx.builder == expected assert build.doctype == expected @pytest.mark.parametrize('value', [[], True, 0, 'invalid']) def test_sphinx_builder_check_invalid(self, value): - build = self.get_build_config({'sphinx': {'builer': value}}) + build = self.get_build_config({'sphinx': {'builder': value}}) with raises(InvalidConfig) as excinfo: build.validate() assert excinfo.value.key == 'sphinx.builder' From 6a99a80cd349da49e5d3f23bc3e8c7cd58ea349c Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 18 Jul 2018 14:51:37 -0500 Subject: [PATCH 64/67] Check for sphinx.builder --- readthedocs/config/config.py | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index 47a6ca50d65..7ffe03bb910 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -574,6 +574,11 @@ class BuildConfigV2(BuildConfigBase): valid_build_images = ['1.0', '2.0', '3.0', 'stable', 'latest'] default_build_image = 'latest' valid_install_options = ['pip', 'setup.py'] + valid_sphinx_builders = { + 'sphinx': 'sphinx', + 'htmldir': 'sphinx_htmldir', + 'singlehtml': 'sphinx_singlehtml', + } def validate(self): """ @@ -809,6 +814,13 @@ def validate_sphinx(self): validate_dict(raw_sphinx) sphinx = {} + with self.catch_validation_error('sphinx.builder'): + builder = validate_choice( + raw_sphinx.get('builder', 'sphinx'), + self.valid_sphinx_builders.keys(), + ) + sphinx['builder'] = self.valid_sphinx_builders[builder] + with self.catch_validation_error('sphinx.configuration'): configuration = self.defaults.get('sphinx_configuration') # The default value can be empty @@ -904,7 +916,8 @@ def python(self): @property def sphinx(self): Sphinx = namedtuple( # noqa - 'Sphinx', ['configuration', 'fail_on_warning'], + 'Sphinx', + ['builder', 'configuration', 'fail_on_warning'], ) if self._config['sphinx']: return Sphinx(**self._config['sphinx']) @@ -913,16 +926,24 @@ def sphinx(self): @property def mkdocs(self): Mkdocs = namedtuple( # noqa - 'Mkdocs', ['configuration', 'fail_on_warning'], + 'Mkdocs', + ['configuration', 'fail_on_warning'], ) if self._config['mkdocs']: return Mkdocs(**self._config['mkdocs']) return None + @property + def doctype(self): + if self.mkdocs: + return 'mkdocs' + return self.sphinx.builder + @property def submodules(self): Submodules = namedtuple( # noqa - 'Submodules', ['include', 'exclude', 'recursive'], + 'Submodules', + ['include', 'exclude', 'recursive'], ) return Submodules(**self._config['submodules']) From 02c9bda38f3d30cf97f177b0c9b0ab137a0a2a07 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 18 Jul 2018 14:56:55 -0500 Subject: [PATCH 65/67] Update test --- readthedocs/rtd_tests/tests/test_config_integration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/readthedocs/rtd_tests/tests/test_config_integration.py b/readthedocs/rtd_tests/tests/test_config_integration.py index d3507358cd4..a1fa306c28f 100644 --- a/readthedocs/rtd_tests/tests/test_config_integration.py +++ b/readthedocs/rtd_tests/tests/test_config_integration.py @@ -84,6 +84,7 @@ def test_python_supported_versions_default_image_1_0(self, load_config): 'python_version': 2, 'sphinx_configuration': mock.ANY, 'build_image': 'readthedocs/build:1.0', + 'doctype': self.project.documentation_type, }, }, ), From 24ed1fbec049b87ba56d950fa42e8cba42a2781d Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 18 Jul 2018 15:00:51 -0500 Subject: [PATCH 66/67] Pass doctype to defaults --- readthedocs/doc_builder/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/readthedocs/doc_builder/config.py b/readthedocs/doc_builder/config.py index 1941fb3de8e..289d0c85eb5 100644 --- a/readthedocs/doc_builder/config.py +++ b/readthedocs/doc_builder/config.py @@ -50,6 +50,7 @@ def load_yaml_config(version): 'python_version': python_version, 'sphinx_configuration': sphinx_configuration, 'build_image': project.container_image, + 'doctype': project.documentation_type, } } img_settings = DOCKER_IMAGE_SETTINGS.get(img_name, None) From aeefb0c318e7d94f73fe46b4f7a59631d5be26a6 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 19 Jul 2018 20:34:03 -0500 Subject: [PATCH 67/67] Rename sphinx builder option to html --- readthedocs/config/config.py | 4 ++-- readthedocs/config/tests/test_config.py | 2 +- readthedocs/rtd_tests/fixtures/spec/v2/schema.yml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index 7ffe03bb910..10a32f5b945 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -575,7 +575,7 @@ class BuildConfigV2(BuildConfigBase): default_build_image = 'latest' valid_install_options = ['pip', 'setup.py'] valid_sphinx_builders = { - 'sphinx': 'sphinx', + 'html': 'sphinx', 'htmldir': 'sphinx_htmldir', 'singlehtml': 'sphinx_singlehtml', } @@ -816,7 +816,7 @@ def validate_sphinx(self): sphinx = {} with self.catch_validation_error('sphinx.builder'): builder = validate_choice( - raw_sphinx.get('builder', 'sphinx'), + raw_sphinx.get('builder', 'html'), self.valid_sphinx_builders.keys(), ) sphinx['builder'] = self.valid_sphinx_builders[builder] diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index ff599fc7707..ddbf59c2eac 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -1283,7 +1283,7 @@ def test_sphinx_is_default_doc_type(self): assert build.doctype == 'sphinx' @pytest.mark.parametrize('value,expected', - [('sphinx', 'sphinx'), + [('html', 'sphinx'), ('htmldir', 'sphinx_htmldir'), ('singlehtml', 'sphinx_singlehtml')]) def test_sphinx_builder_check_valid(self, value, expected): diff --git a/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml b/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml index 6d16e311274..d192f8c3b76 100644 --- a/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml +++ b/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml @@ -71,8 +71,8 @@ python: sphinx: # The builder type for the sphinx documentation - # Default: 'sphinx' - builder: enum('sphinx', 'htmldir', 'singlehtml', required=False) + # Default: 'html' + builder: enum('html', 'htmldir', 'singlehtml', required=False) # The path to the conf.py file # Default: rtd will try to find it | project config