diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index c382cd68f2f..30101b58f0f 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -14,12 +14,15 @@ validate_file, validate_list, validate_string) __all__ = ( - 'load', 'BuildConfig', 'ConfigError', 'InvalidConfig', 'ProjectConfig') + 'load', 'BuildConfig', 'ConfigError', 'ConfigOptionNotSupportedError', + 'InvalidConfig', 'ProjectConfig' +) CONFIG_FILENAMES = ('readthedocs.yml', '.readthedocs.yml') +CONFIG_NOT_SUPPORTED = 'config-not-supported' BASE_INVALID = 'base-invalid' BASE_NOT_A_DIR = 'base-not-a-directory' CONFIG_SYNTAX_INVALID = 'config-syntax-invalid' @@ -57,6 +60,21 @@ def __init__(self, message, code): super(ConfigError, self).__init__(message) +class ConfigOptionNotSupportedError(ConfigError): + + """Error for unsupported configuration options in a version.""" + + def __init__(self, configuration): + self.configuration = configuration + template = ( + 'The "{}" configuration option is not supported in this version' + ) + super(ConfigOptionNotSupportedError, self).__init__( + template.format(self.configuration), + CONFIG_NOT_SUPPORTED + ) + + class InvalidConfig(ConfigError): """Error for a specific key validation.""" @@ -76,53 +94,43 @@ def __init__(self, key, code, error_message, source_file=None, super(InvalidConfig, self).__init__(message, code=code) -class BuildConfig(dict): +class BuildConfigBase(object): """ Config that handles the build of one particular documentation. - Config keys can be accessed with a dictionary lookup:: - - >>> build_config['type'] - 'sphinx' - - You need to call ``validate`` before the config is ready to use. Also - setting the ``output_base`` is required before using it for a build. + You need to call ``validate`` before the config is ready to use. + Also setting the ``output_base`` is required before using it for a build. """ - BASE_INVALID_MESSAGE = 'Invalid value for base: {base}' - BASE_NOT_A_DIR_MESSAGE = '"base" is not a directory: {base}' - NAME_REQUIRED_MESSAGE = 'Missing key "name"' - NAME_INVALID_MESSAGE = ( - 'Invalid name "{name}". Valid values must match {name_re}') - TYPE_REQUIRED_MESSAGE = 'Missing key "type"' - CONF_FILE_REQUIRED_MESSAGE = 'Missing key "conf_file"' - PYTHON_INVALID_MESSAGE = '"python" section must be a mapping.' - PYTHON_EXTRA_REQUIREMENTS_INVALID_MESSAGE = ( - '"python.extra_requirements" section must be a list.') - - PYTHON_SUPPORTED_VERSIONS = [2, 2.7, 3, 3.5] - DOCKER_SUPPORTED_VERSIONS = ['1.0', '2.0', 'latest'] + version = None def __init__(self, env_config, raw_config, source_file, source_position): self.env_config = env_config self.raw_config = raw_config self.source_file = source_file self.source_position = source_position - super(BuildConfig, self).__init__() + self.defaults = self.env_config.get('defaults', {}) + + self._config = {} def error(self, key, message, code): """Raise an error related to ``key``.""" source = '{file} [{pos}]'.format( file=self.source_file, - pos=self.source_position) + pos=self.source_position + ) + error_message = '{source}: {message}'.format( + source=source, + message=message + ) raise InvalidConfig( key=key, code=code, - error_message='{source}: {message}'.format(source=source, - message=message), + error_message=error_message, source_file=self.source_file, - source_position=self.source_position) + source_position=self.source_position + ) @contextmanager def catch_validation_error(self, key): @@ -135,9 +143,57 @@ def catch_validation_error(self, key): code=error.code, error_message=str(error), source_file=self.source_file, - source_position=self.source_position) + source_position=self.source_position + ) + + def validate(self): + raise NotImplementedError() + + @property + def python_interpreter(self): + ver = self.python_full_version + return 'python{0}'.format(ver) + + @property + def python_full_version(self): + ver = self.python_version + if ver in [2, 3]: + # Get the highest version of the major series version if user only + # gave us a version of '2', or '3' + ver = max( + v + for v in self.get_valid_python_versions() + if v < ver + 1 + ) + return ver + + def __getattr__(self, name): + """Raise an error for unknown attributes.""" + raise ConfigOptionNotSupportedError(name) + + +class BuildConfig(BuildConfigBase): + + """Version 1 of the configuration file.""" + + BASE_INVALID_MESSAGE = 'Invalid value for base: {base}' + BASE_NOT_A_DIR_MESSAGE = '"base" is not a directory: {base}' + NAME_REQUIRED_MESSAGE = 'Missing key "name"' + NAME_INVALID_MESSAGE = ( + 'Invalid name "{name}". Valid values must match {name_re}') + TYPE_REQUIRED_MESSAGE = 'Missing key "type"' + CONF_FILE_REQUIRED_MESSAGE = 'Missing key "conf_file"' + PYTHON_INVALID_MESSAGE = '"python" section must be a mapping.' + PYTHON_EXTRA_REQUIREMENTS_INVALID_MESSAGE = ( + '"python.extra_requirements" section must be a list.') + + PYTHON_SUPPORTED_VERSIONS = [2, 2.7, 3, 3.5] + DOCKER_SUPPORTED_VERSIONS = ['1.0', '2.0', 'latest'] + + version = '1' def get_valid_types(self): # noqa + """Get all valid types.""" return ( 'sphinx', ) @@ -169,32 +225,39 @@ def validate(self): ``readthedocs.yml`` config file if not set """ # Validate env_config. - self.validate_output_base() + # TODO: this isn't used + self._config['output_base'] = self.validate_output_base() # Validate the build environment first - self.validate_build() # Must happen before `validate_python`! + # Must happen before `validate_python`! + self._config['build'] = self.validate_build() # Validate raw_config. Order matters. - self.validate_name() - self.validate_type() - self.validate_base() - self.validate_python() - self.validate_formats() - - self.validate_conda() - self.validate_requirements_file() - self.validate_conf_file() + # TODO: this isn't used + self._config['name'] = self.validate_name() + # TODO: this isn't used + self._config['type'] = self.validate_type() + # TODO: this isn't used + self._config['base'] = self.validate_base() + self._config['python'] = self.validate_python() + self._config['formats'] = self.validate_formats() + + self._config['conda'] = self.validate_conda() + self._config['requirements_file'] = self.validate_requirements_file() + # TODO: this isn't used + self._config['conf_file'] = self.validate_conf_file() def validate_output_base(self): """Validates that ``output_base`` exists and set its absolute path.""" assert 'output_base' in self.env_config, ( '"output_base" required in "env_config"') base_path = os.path.dirname(self.source_file) - self['output_base'] = os.path.abspath( + output_base = os.path.abspath( os.path.join( self.env_config.get('output_base', base_path), ) ) + return output_base def validate_name(self): """Validates that name exists.""" @@ -212,7 +275,7 @@ def validate_name(self): name_re=name_re), code=NAME_INVALID) - self['name'] = name + return name def validate_type(self): """Validates that type is a valid choice.""" @@ -225,7 +288,7 @@ def validate_type(self): with self.catch_validation_error('type'): validate_choice(type_, self.get_valid_types()) - self['type'] = type_ + return type_ def validate_base(self): """Validates that path is a valid directory.""" @@ -236,7 +299,7 @@ def validate_base(self): with self.catch_validation_error('base'): base_path = os.path.dirname(self.source_file) base = validate_directory(base, base_path) - self['base'] = base + return base def validate_build(self): """ @@ -256,7 +319,7 @@ def validate_build(self): """ # Defaults if 'build' in self.env_config: - build = self.env_config['build'] + build = self.env_config['build'].copy() else: build = {'image': DOCKER_IMAGE} @@ -286,19 +349,27 @@ def validate_build(self): self.env_config.update( self.env_config['DOCKER_IMAGE_SETTINGS'][build['image']] ) - self['build'] = build + + # Allow to override specific project + config_image = self.defaults.get('build_image') + if config_image: + build['image'] = config_image + return build def validate_python(self): """Validates the ``python`` key, set default values it's necessary.""" + install_project = self.defaults.get('install_project', False) + use_system_packages = self.defaults.get('use_system_packages', False) + version = self.defaults.get('python_version', 2) python = { - 'use_system_site_packages': False, + 'use_system_site_packages': use_system_packages, 'pip_install': False, 'extra_requirements': [], - 'setup_py_install': False, + 'setup_py_install': install_project, 'setup_py_path': os.path.join( os.path.dirname(self.source_file), 'setup.py'), - 'version': 2, + 'version': version, } if 'python' in self.raw_config: @@ -367,7 +438,7 @@ def validate_python(self): self.get_valid_python_versions(), ) - self['python'] = python + return python def validate_conda(self): """Validates the ``conda`` key.""" @@ -387,20 +458,21 @@ def validate_conda(self): conda['file'] = validate_file( raw_conda['file'], base_path) - self['conda'] = conda + return conda + return None def validate_requirements_file(self): """Validates that the requirements file exists.""" if 'requirements_file' not in self.raw_config: + requirements_file = self.defaults.get('requirements_file') + else: + requirements_file = self.raw_config['requirements_file'] + if not requirements_file: return None - - requirements_file = self.raw_config['requirements_file'] base_path = os.path.dirname(self.source_file) with self.catch_validation_error('requirements_file'): validate_file(requirements_file, base_path) - self['requirements_file'] = requirements_file - - return True + return requirements_file def validate_conf_file(self): """Validates the conf.py file for sphinx.""" @@ -411,26 +483,103 @@ def validate_conf_file(self): base_path = os.path.dirname(self.source_file) with self.catch_validation_error('conf_file'): validate_file(conf_file, base_path) - self['conf_file'] = conf_file - - return True + return conf_file def validate_formats(self): """Validates that formats contains only valid formats.""" formats = self.raw_config.get('formats') if formats is None: - return None + return self.defaults.get('formats', []) if formats == ['none']: - self['formats'] = [] - return True + return [] with self.catch_validation_error('format'): validate_list(formats) for format_ in formats: validate_choice(format_, self.get_valid_formats()) - self['formats'] = formats - return True + return formats + + @property + def name(self): + """The project name.""" + return self._config['name'] + + @property + def base(self): + """The base directory.""" + return self._config['base'] + + @property + def output_base(self): + """The output base""" + return self._config['output_base'] + + @property + def type(self): + """The documentation type.""" + return self._config['type'] + + @property + def formats(self): + """The documentation formats to be built.""" + return self._config['formats'] + + @property + def python(self): + """Python related configuration.""" + return self._config.get('python', {}) + + @property + def python_version(self): + """Python version.""" + return self._config['python']['version'] + + @property + def pip_install(self): + """True if the project should be installed using pip.""" + return self._config['python']['pip_install'] + + @property + def install_project(self): + """True if the project should be installed.""" + if self.pip_install: + return True + return self._config['python']['setup_py_install'] + + @property + def extra_requirements(self): + """Extra requirements to be installed with pip.""" + if self.pip_install: + return self._config['python']['extra_requirements'] + return [] + + @property + def use_system_site_packages(self): + """True if the project should have access to the system packages.""" + return self._config['python']['use_system_site_packages'] + + @property + def use_conda(self): + """True if the project use Conda.""" + return self._config.get('conda') is not None + + @property + def conda_file(self): + """The Conda environment file.""" + if self.use_conda: + return self._config['conda'].get('file') + return None + + @property + def requirements_file(self): + """The project requirements file.""" + return self._config['requirements_file'] + + @property + def build_image(self): + """The docker image used by the builders.""" + return self._config['build']['image'] class ProjectConfig(list): @@ -442,11 +591,6 @@ def validate(self): for build in self: build.validate() - def set_output_base(self, directory): - """Set a common ``output_base`` for each configuration build.""" - for build in self: - build['output_base'] = os.path.abspath(directory) - def load(path, env_config): """ diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index 5b1ba7ae653..a8c308643e2 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -2,13 +2,16 @@ import os +import pytest from mock import DEFAULT, patch from pytest import raises from readthedocs.config import ( - BuildConfig, ConfigError, InvalidConfig, ProjectConfig, load) + BuildConfig, ConfigError, ConfigOptionNotSupportedError, InvalidConfig, + ProjectConfig, load) from readthedocs.config.config import ( - NAME_INVALID, NAME_REQUIRED, PYTHON_INVALID, TYPE_REQUIRED) + CONFIG_NOT_SUPPORTED, NAME_INVALID, NAME_REQUIRED, PYTHON_INVALID, + TYPE_REQUIRED) from readthedocs.config.validation import ( INVALID_BOOL, INVALID_CHOICE, INVALID_LIST, INVALID_PATH, INVALID_STRING) @@ -63,6 +66,19 @@ def get_build_config(config, env_config=None, source_file='readthedocs.yml', source_position=source_position) +def get_env_config(extra=None): + """Get the minimal env_config for the configuration object.""" + defaults = { + 'output_base': '', + 'name': 'name', + 'type': 'sphinx', + } + if extra is None: + extra = {} + defaults.update(extra) + return defaults + + def test_load_no_config_file(tmpdir): base = str(tmpdir) with raises(ConfigError): @@ -110,94 +126,117 @@ def test_build_config_has_list_with_single_empty_value(tmpdir): base = str(apply_fs(tmpdir, config_with_explicit_empty_list)) build = load(base, env_config)[0] assert isinstance(build, BuildConfig) - assert build['formats'] == [] + assert build.formats == [] def test_config_requires_name(): - build = BuildConfig({}, - {}, - source_file=None, - source_position=None) + build = BuildConfig( + {'output_base': ''}, {}, + source_file='readthedocs.yml', + source_position=0 + ) with raises(InvalidConfig) as excinfo: - build.validate_name() + build.validate() assert excinfo.value.key == 'name' assert excinfo.value.code == NAME_REQUIRED def test_build_requires_valid_name(): - build = BuildConfig({}, - {'name': 'with/slashes'}, - source_file=None, - source_position=None) + build = BuildConfig( + {'output_base': ''}, + {'name': 'with/slashes'}, + source_file='readthedocs.yml', + source_position=0 + ) with raises(InvalidConfig) as excinfo: - build.validate_name() + build.validate() assert excinfo.value.key == 'name' assert excinfo.value.code == NAME_INVALID def test_config_requires_type(): - build = BuildConfig({}, - {'name': 'docs'}, - source_file=None, - source_position=None) + build = BuildConfig( + {'output_base': ''}, {'name': 'docs'}, + source_file='readthedocs.yml', + source_position=0 + ) with raises(InvalidConfig) as excinfo: - build.validate_type() + build.validate() assert excinfo.value.key == 'type' assert excinfo.value.code == TYPE_REQUIRED def test_build_requires_valid_type(): - build = BuildConfig({}, - {'type': 'unknown'}, - source_file=None, - source_position=None) + build = BuildConfig( + {'output_base': ''}, + {'name': 'docs', 'type': 'unknown'}, + source_file='readthedocs.yml', + source_position=0 + ) with raises(InvalidConfig) as excinfo: - build.validate_type() + build.validate() assert excinfo.value.key == 'type' assert excinfo.value.code == INVALID_CHOICE +def test_version(): + build = get_build_config({}, get_env_config()) + assert build.version == '1' + + def test_empty_python_section_is_valid(): - build = get_build_config({'python': {}}) - build.validate_python() - assert 'python' in build + build = get_build_config({'python': {}}, get_env_config()) + build.validate() + assert build.python def test_python_section_must_be_dict(): - build = get_build_config({'python': 123}) + build = get_build_config({'python': 123}, get_env_config()) with raises(InvalidConfig) as excinfo: - build.validate_python() + build.validate() assert excinfo.value.key == 'python' assert excinfo.value.code == PYTHON_INVALID def test_use_system_site_packages_defaults_to_false(): - build = get_build_config({'python': {}}) - build.validate_python() + build = get_build_config({'python': {}}, get_env_config()) + build.validate() # Default is False. - assert not build['python']['use_system_site_packages'] + assert not build.use_system_site_packages + + +@pytest.mark.parametrize('value', [True, False]) +def test_use_system_site_packages_repects_default_value(value): + defaults = { + 'use_system_packages': value, + } + build = get_build_config({}, get_env_config({'defaults': defaults})) + build.validate() + assert build.use_system_site_packages is value def test_python_pip_install_default(): - build = get_build_config({'python': {}}) - build.validate_python() + build = get_build_config({'python': {}}, get_env_config()) + build.validate() # Default is False. - assert build['python']['pip_install'] is False + assert build.pip_install is False def describe_validate_python_extra_requirements(): def it_defaults_to_list(): - build = get_build_config({'python': {}}) - build.validate_python() + build = get_build_config({'python': {}}, get_env_config()) + build.validate() # Default is an empty list. - assert build['python']['extra_requirements'] == [] + assert build.extra_requirements == [] def it_validates_is_a_list(): build = get_build_config( - {'python': {'extra_requirements': 'invalid'}}) + {'python': {'extra_requirements': 'invalid'}}, + get_env_config() + ) with raises(InvalidConfig) as excinfo: - build.validate_python() + build.validate() assert excinfo.value.key == 'python.extra_requirements' assert excinfo.value.code == PYTHON_INVALID @@ -205,22 +244,26 @@ def it_validates_is_a_list(): def it_uses_validate_string(validate_string): validate_string.return_value = True build = get_build_config( - {'python': {'extra_requirements': ['tests']}}) - build.validate_python() + {'python': {'extra_requirements': ['tests']}}, + get_env_config() + ) + build.validate() validate_string.assert_any_call('tests') def describe_validate_use_system_site_packages(): def it_defaults_to_false(): - build = get_build_config({'python': {}}) - build.validate_python() - assert build['python']['setup_py_install'] is False + build = get_build_config({'python': {}}, get_env_config()) + build.validate() + assert build.use_system_site_packages is False def it_validates_value(): build = get_build_config( - {'python': {'use_system_site_packages': 'invalid'}}) + {'python': {'use_system_site_packages': 'invalid'}}, + get_env_config() + ) with raises(InvalidConfig) as excinfo: - build.validate_python() + build.validate() excinfo.value.key = 'python.use_system_site_packages' excinfo.value.code = INVALID_BOOL @@ -228,22 +271,27 @@ def it_validates_value(): def it_uses_validate_bool(validate_bool): validate_bool.return_value = True build = get_build_config( - {'python': {'use_system_site_packages': 'to-validate'}}) - build.validate_python() + {'python': {'use_system_site_packages': 'to-validate'}}, + get_env_config() + ) + build.validate() validate_bool.assert_any_call('to-validate') def describe_validate_setup_py_install(): def it_defaults_to_false(): - build = get_build_config({'python': {}}) - build.validate_python() - assert build['python']['setup_py_install'] is False + build = get_build_config({'python': {}}, get_env_config()) + build.validate() + assert build.python['setup_py_install'] is False def it_validates_value(): - build = get_build_config({'python': {'setup_py_install': 'this-is-string'}}) + build = get_build_config( + {'python': {'setup_py_install': 'this-is-string'}}, + get_env_config() + ) with raises(InvalidConfig) as excinfo: - build.validate_python() + build.validate() assert excinfo.value.key == 'python.setup_py_install' assert excinfo.value.code == INVALID_BOOL @@ -251,114 +299,172 @@ def it_validates_value(): def it_uses_validate_bool(validate_bool): validate_bool.return_value = True build = get_build_config( - {'python': {'setup_py_install': 'to-validate'}}) - build.validate_python() + {'python': {'setup_py_install': 'to-validate'}}, + get_env_config() + ) + build.validate() validate_bool.assert_any_call('to-validate') def describe_validate_python_version(): def it_defaults_to_a_valid_version(): - build = get_build_config({'python': {}}) - build.validate_python() - assert build['python']['version'] is 2 + build = get_build_config({'python': {}}, get_env_config()) + build.validate() + assert build.python_version == 2 + assert build.python_interpreter == 'python2.7' + assert build.python_full_version == 2.7 def it_supports_other_versions(): - build = get_build_config({'python': {'version': 3.5}}) - build.validate_python() - assert build['python']['version'] is 3.5 + build = get_build_config( + {'python': {'version': 3.5}}, + get_env_config() + ) + build.validate() + assert build.python_version == 3.5 + assert build.python_interpreter == 'python3.5' + assert build.python_full_version == 3.5 def it_validates_versions_out_of_range(): - build = get_build_config({'python': {'version': 1.0}}) + build = get_build_config( + {'python': {'version': 1.0}}, + get_env_config() + ) with raises(InvalidConfig) as excinfo: - build.validate_python() + build.validate() assert excinfo.value.key == 'python.version' assert excinfo.value.code == INVALID_CHOICE def it_validates_wrong_type(): - build = get_build_config({'python': {'version': 'this-is-string'}}) + build = get_build_config( + {'python': {'version': 'this-is-string'}}, + get_env_config() + ) with raises(InvalidConfig) as excinfo: - build.validate_python() + build.validate() assert excinfo.value.key == 'python.version' assert excinfo.value.code == INVALID_CHOICE def it_validates_wrong_type_right_value(): - build = get_build_config({'python': {'version': '3.5'}}) - build.validate_python() - assert build['python']['version'] == 3.5 + build = get_build_config( + {'python': {'version': '3.5'}}, + get_env_config() + ) + build.validate() + assert build.python_version == 3.5 + assert build.python_interpreter == 'python3.5' + assert build.python_full_version == 3.5 - build = get_build_config({'python': {'version': '3'}}) - build.validate_python() - assert build['python']['version'] == 3 + build = get_build_config( + {'python': {'version': '3'}}, + get_env_config() + ) + build.validate() + assert build.python_version == 3 + assert build.python_interpreter == 'python3.5' + assert build.python_full_version == 3.5 def it_validates_env_supported_versions(): build = get_build_config( {'python': {'version': 3.6}}, - env_config={'python': {'supported_versions': [3.5]}} + env_config=get_env_config( + { + 'python': {'supported_versions': [3.5]}, + 'build': {'image': 'custom'}, + } + ) ) with raises(InvalidConfig) as excinfo: - build.validate_python() + build.validate() assert excinfo.value.key == 'python.version' assert excinfo.value.code == INVALID_CHOICE build = get_build_config( {'python': {'version': 3.6}}, - env_config={'python': {'supported_versions': [3.5, 3.6]}} + env_config=get_env_config( + { + 'python': {'supported_versions': [3.5, 3.6]}, + 'build': {'image': 'custom'}, + } + ) ) - build.validate_python() - assert build['python']['version'] == 3.6 + build.validate() + assert build.python_version == 3.6 + assert build.python_interpreter == 'python3.6' + assert build.python_full_version == 3.6 + + @pytest.mark.parametrize('value', [2, 3]) + def it_respects_default_value(value): + defaults = { + 'python_version': value + } + build = get_build_config( + {}, + get_env_config({'defaults': defaults}) + ) + build.validate() + assert build.python_version == value def describe_validate_formats(): - def it_defaults_to_not_being_included(): - build = get_build_config({}) - build.validate_formats() - assert 'formats' not in build + def it_defaults_to_empty(): + build = get_build_config({}, get_env_config()) + build.validate() + assert build.formats == [] def it_gets_set_correctly(): - build = get_build_config({'formats': ['pdf']}) - build.validate_formats() - assert build['formats'] == ['pdf'] + build = get_build_config({'formats': ['pdf']}, get_env_config()) + build.validate() + assert build.formats == ['pdf'] def formats_can_be_null(): - build = get_build_config({'formats': None}) - build.validate_formats() - assert 'formats' not in build + build = get_build_config({'formats': None}, get_env_config()) + build.validate() + assert build.formats == [] def formats_with_previous_none(): - build = get_build_config({'formats': ['none']}) - build.validate_formats() - assert build['formats'] == [] + build = get_build_config({'formats': ['none']}, get_env_config()) + build.validate() + assert build.formats == [] def formats_can_be_empty(): - build = get_build_config({'formats': []}) - build.validate_formats() - assert build['formats'] == [] + build = get_build_config({'formats': []}, get_env_config()) + build.validate() + assert build.formats == [] def all_valid_formats(): - build = get_build_config({'formats': ['pdf', 'htmlzip', 'epub']}) - build.validate_formats() - assert build['formats'] == ['pdf', 'htmlzip', 'epub'] + build = get_build_config( + {'formats': ['pdf', 'htmlzip', 'epub']}, + get_env_config() + ) + build.validate() + assert build.formats == ['pdf', 'htmlzip', 'epub'] def cant_have_none_as_format(): - build = get_build_config({'formats': ['htmlzip', None]}) + build = get_build_config( + {'formats': ['htmlzip', None]}, + get_env_config() + ) with raises(InvalidConfig) as excinfo: - build.validate_formats() + build.validate() assert excinfo.value.key == 'format' assert excinfo.value.code == INVALID_CHOICE def formats_have_only_allowed_values(): - build = get_build_config({'formats': ['htmlzip', 'csv']}) + build = get_build_config( + {'formats': ['htmlzip', 'csv']}, + get_env_config() + ) with raises(InvalidConfig) as excinfo: - build.validate_formats() + build.validate() assert excinfo.value.key == 'format' assert excinfo.value.code == INVALID_CHOICE def only_list_type(): - build = get_build_config({'formats': 'no-list'}) + build = get_build_config({'formats': 'no-list'}, get_env_config()) with raises(InvalidConfig) as excinfo: - build.validate_formats() + build.validate() assert excinfo.value.key == 'format' assert excinfo.value.code == INVALID_LIST @@ -366,12 +472,24 @@ def only_list_type(): def describe_validate_setup_py_path(): def it_defaults_to_source_file_directory(tmpdir): + apply_fs( + tmpdir, + { + 'subdir': { + 'readthedocs.yml': '', + 'setup.py': '', + }, + } + ) with tmpdir.as_cwd(): source_file = tmpdir.join('subdir', 'readthedocs.yml') setup_py = tmpdir.join('subdir', 'setup.py') - build = get_build_config({}, source_file=str(source_file)) - build.validate_python() - assert build['python']['setup_py_path'] == str(setup_py) + build = get_build_config( + {}, + env_config=get_env_config(), + source_file=str(source_file)) + build.validate() + assert build.python['setup_py_path'] == str(setup_py) def it_validates_value(tmpdir): with tmpdir.as_cwd(): @@ -401,13 +519,13 @@ def test_valid_build_config(): source_file='readthedocs.yml', source_position=0) build.validate() - assert build['name'] == 'docs' - assert build['type'] == 'sphinx' - assert build['base'] - assert build['python'] - assert 'setup_py_install' in build['python'] - assert 'use_system_site_packages' in build['python'] - assert build['output_base'] + assert build.name == 'docs' + assert build.type == 'sphinx' + assert build.base + assert build.python + assert 'setup_py_install' in build.python + assert 'use_system_site_packages' in build.python + assert build.output_base def describe_validate_base(): @@ -417,18 +535,18 @@ def it_validates_to_abspath(tmpdir): with tmpdir.as_cwd(): source_file = str(tmpdir.join('configs', 'readthedocs.yml')) build = BuildConfig( - {}, + get_env_config(), {'base': '../docs'}, source_file=source_file, source_position=0) - build.validate_base() - assert build['base'] == str(tmpdir.join('docs')) + build.validate() + assert build.base == str(tmpdir.join('docs')) @patch('readthedocs.config.config.validate_directory') def it_uses_validate_directory(validate_directory): validate_directory.return_value = 'path' - build = get_build_config({'base': '../my-path'}) - build.validate_base() + build = get_build_config({'base': '../my-path'}, get_env_config()) + build.validate() # Test for first argument to validate_directory args, kwargs = validate_directory.call_args assert args[0] == '../my-path' @@ -437,24 +555,24 @@ def it_fails_if_base_is_not_a_string(tmpdir): apply_fs(tmpdir, minimal_config) with tmpdir.as_cwd(): build = BuildConfig( - {}, + get_env_config(), {'base': 1}, source_file=str(tmpdir.join('readthedocs.yml')), source_position=0) with raises(InvalidConfig) as excinfo: - build.validate_base() + build.validate() assert excinfo.value.key == 'base' assert excinfo.value.code == INVALID_STRING def it_fails_if_base_does_not_exist(tmpdir): apply_fs(tmpdir, minimal_config) build = BuildConfig( - {}, + get_env_config(), {'base': 'docs'}, source_file=str(tmpdir.join('readthedocs.yml')), source_position=0) with raises(InvalidConfig) as excinfo: - build.validate_base() + build.validate() assert excinfo.value.key == 'base' assert excinfo.value.code == INVALID_PATH @@ -464,12 +582,12 @@ def describe_validate_build(): def it_fails_if_build_is_invalid_option(tmpdir): apply_fs(tmpdir, minimal_config) build = BuildConfig( - {}, + get_env_config(), {'build': {'image': 3.0}}, source_file=str(tmpdir.join('readthedocs.yml')), source_position=0) with raises(InvalidConfig) as excinfo: - build.validate_build() + build.validate() assert excinfo.value.key == 'build' assert excinfo.value.code == INVALID_CHOICE @@ -505,22 +623,97 @@ def it_works_on_python_validation(tmpdir): def it_works(tmpdir): apply_fs(tmpdir, minimal_config) build = BuildConfig( - {}, + get_env_config(), {'build': {'image': 'latest'}}, source_file=str(tmpdir.join('readthedocs.yml')), source_position=0) - build.validate_build() - assert build['build']['image'] == 'readthedocs/build:latest' + build.validate() + assert build.build_image == 'readthedocs/build:latest' def default(tmpdir): apply_fs(tmpdir, minimal_config) build = BuildConfig( - {}, + get_env_config(), {}, source_file=str(tmpdir.join('readthedocs.yml')), source_position=0) - build.validate_build() - assert build['build']['image'] == 'readthedocs/build:2.0' + build.validate() + assert build.build_image == 'readthedocs/build:2.0' + + @pytest.mark.parametrize( + 'image', ['latest', 'readthedocs/build:3.0', 'rtd/build:latest']) + def it_priorities_image_from_env_config(tmpdir, image): + apply_fs(tmpdir, minimal_config) + defaults = { + 'build_image': image, + } + build = BuildConfig( + get_env_config({'defaults': defaults}), + {'build': {'image': 'latest'}}, + source_file=str(tmpdir.join('readthedocs.yml')), + source_position=0 + ) + build.validate() + assert build.build_image == image + + +def test_use_conda_default_false(): + build = get_build_config({}, get_env_config()) + build.validate() + assert build.use_conda is False + + +def test_use_conda_respects_config(): + build = get_build_config( + {'conda': {}}, + get_env_config() + ) + build.validate() + assert build.use_conda is True + + +def test_validates_conda_file(tmpdir): + apply_fs(tmpdir, {'environment.yml': ''}) + build = get_build_config( + {'conda': {'file': 'environment.yml'}}, + get_env_config(), + source_file=str(tmpdir.join('readthedocs.yml')) + ) + build.validate() + assert build.use_conda is True + assert build.conda_file == str(tmpdir.join('environment.yml')) + + +def test_requirements_file_empty(): + build = get_build_config({}, get_env_config()) + build.validate() + assert build.requirements_file is None + + +def test_requirements_file_repects_default_value(tmpdir): + apply_fs(tmpdir, {'myrequirements.txt': ''}) + defaults = { + 'requirements_file': 'myrequirements.txt' + } + build = get_build_config( + {}, + get_env_config({'defaults': defaults}), + source_file=str(tmpdir.join('readthedocs.yml')) + ) + build.validate() + assert build.requirements_file == 'myrequirements.txt' + + +def test_requirements_file_respects_configuration(tmpdir): + apply_fs(tmpdir, {'requirements.txt': ''}) + build = get_build_config( + {'requirements_file': 'requirements.txt'}, + get_env_config(), + source_file=str(tmpdir.join('readthedocs.yml')) + ) + build.validate() + assert build.requirements_file == 'requirements.txt' + def test_build_validate_calls_all_subvalidators(tmpdir): @@ -565,20 +758,10 @@ def test_load_calls_validate(tmpdir): assert build_validate.call_count == 1 -def test_project_set_output_base(): - project = ProjectConfig([ - BuildConfig( - env_config, - minimal_config, - source_file='readthedocs.yml', - source_position=0), - BuildConfig( - env_config, - minimal_config, - source_file='readthedocs.yml', - source_position=1), - ]) - project.set_output_base('random') - for build_config in project: - assert ( - build_config['output_base'] == os.path.join(os.getcwd(), 'random')) +def test_raise_config_not_supported(): + build = get_build_config({}, get_env_config()) + build.validate() + with raises(ConfigOptionNotSupportedError) as excinfo: + build.redirects + assert excinfo.value.configuration == 'redirects' + assert excinfo.value.code == CONFIG_NOT_SUPPORTED diff --git a/readthedocs/doc_builder/config.py b/readthedocs/doc_builder/config.py index 14f8e128703..04b63a23f23 100644 --- a/readthedocs/doc_builder/config.py +++ b/readthedocs/doc_builder/config.py @@ -4,135 +4,12 @@ from __future__ import ( absolute_import, division, print_function, unicode_literals) -from builtins import filter, object - from readthedocs.config import BuildConfig, ConfigError, InvalidConfig from readthedocs.config import load as load_config from .constants import DOCKER_IMAGE, DOCKER_IMAGE_SETTINGS -class ConfigWrapper(object): - - """ - A config object that wraps the Project & YAML based configs. - - Gives precedence to YAML, falling back to project if it isn't defined. - - We only currently implement a subset of the existing YAML config. - This should be the canonical source for our usage of the YAML files, - never accessing the config object directly. - """ - - def __init__(self, version, yaml_config): - self._version = version - self._project = version.project - self._yaml_config = yaml_config - - @property - def pip_install(self): - if 'pip_install' in self._yaml_config.get('python', {}): - return self._yaml_config['python']['pip_install'] - return False - - @property - def install_project(self): - if self.pip_install: - return True - if 'setup_py_install' in self._yaml_config.get('python', {}): - return self._yaml_config['python']['setup_py_install'] - return self._project.install_project - - @property - def extra_requirements(self): - if self.pip_install and 'extra_requirements' in self._yaml_config.get( - 'python', {}): - return self._yaml_config['python']['extra_requirements'] - return [] - - @property - def python_interpreter(self): - ver = self.python_full_version - return 'python{0}'.format(ver) - - @property - def python_version(self): - # There should always be a version in the YAML config. If the config - # version is the default response of `2`, then assume we can use the - # Python.python_interpreter version to infer this value instead. - version = 2 - if 'version' in self._yaml_config.get('python', {}): - version = self._yaml_config['python']['version'] - if version == 2 and self._project.python_interpreter == 'python3': - version = 3 - return version - - @property - def python_full_version(self): - ver = self.python_version - if ver in [2, 3]: - # Get the highest version of the major series version if user only - # gave us a version of '2', or '3' - ver = max( - list( - filter( - lambda x: x < ver + 1, - self._yaml_config.get_valid_python_versions(), - ))) - return ver - - @property - def use_system_site_packages(self): - if 'use_system_site_packages' in self._yaml_config.get('python', {}): - return self._yaml_config['python']['use_system_site_packages'] - return self._project.use_system_packages - - @property - def use_conda(self): - return 'conda' in self._yaml_config - - @property - def conda_file(self): - if 'file' in self._yaml_config.get('conda', {}): - return self._yaml_config['conda']['file'] - return None - - @property - def requirements_file(self): - if 'requirements_file' in self._yaml_config: - return self._yaml_config['requirements_file'] - return self._project.requirements_file - - @property - def formats(self): - if 'formats' in self._yaml_config: - return self._yaml_config['formats'] - formats = ['htmlzip'] - if self._project.enable_epub_build: - formats += ['epub'] - if self._project.enable_pdf_build: - formats += ['pdf'] - return formats - - @property - def build_image(self): - if self._project.container_image: - # Allow us to override per-project still - return self._project.container_image - if 'build' in self._yaml_config: - return self._yaml_config['build']['image'] - return None - - # Not implemented until we figure out how to keep in sync with the webs. - # Probably needs to be version-specific as well, not project. - # @property - # def documentation_type(self): - # if 'type' in self._yaml_config: - # return self._yaml_config['type'] - # else: - # return self._project.documentation_type - - def load_yaml_config(version): """ Load a configuration from `readthedocs.yml` file. @@ -141,15 +18,25 @@ def load_yaml_config(version): parsing consistent between projects. """ checkout_path = version.project.checkout_path(version.slug) + project = version.project # Get build image to set up the python version validation. Pass in the # build image python limitations to the loaded config so that the versions # can be rejected at validation - img_name = version.project.container_image or DOCKER_IMAGE + img_name = project.container_image or DOCKER_IMAGE + python_version = 3 if project.python_interpreter == 'python3' else 2 env_config = { 'build': { 'image': img_name, + }, + 'defaults': { + 'install_project': project.install_project, + 'formats': get_default_formats(project), + 'use_system_packages': project.use_system_packages, + 'requirements_file': project.requirements_file, + 'python_version': python_version, + 'build_image': project.container_image, } } img_settings = DOCKER_IMAGE_SETTINGS.get(img_name, None) @@ -172,10 +59,27 @@ def load_yaml_config(version): # This is a subclass of ConfigError, so has to come first raise except ConfigError: + # TODO: this shouldn't be hardcoded here + env_config.update({ + 'output_base': '', + 'type': 'sphinx', + 'name': version.slug, + }) config = BuildConfig( env_config=env_config, raw_config={}, source_file='empty', source_position=0, ) - return ConfigWrapper(version=version, yaml_config=config) + config.validate() + return config + + +def get_default_formats(project): + """Get a list of the default formats for ``project``.""" + formats = ['htmlzip'] + if project.enable_epub_build: + formats += ['epub'] + if project.enable_pdf_build: + formats += ['pdf'] + return formats diff --git a/readthedocs/doc_builder/python_environments.py b/readthedocs/doc_builder/python_environments.py index e6009f8f6cc..faaee70e3d1 100644 --- a/readthedocs/doc_builder/python_environments.py +++ b/readthedocs/doc_builder/python_environments.py @@ -14,7 +14,7 @@ import six from django.conf import settings -from readthedocs.doc_builder.config import ConfigWrapper +from readthedocs.doc_builder.config import load_yaml_config from readthedocs.doc_builder.constants import DOCKER_IMAGE from readthedocs.doc_builder.environments import DockerBuildEnvironment from readthedocs.doc_builder.loader import get_builder_class @@ -35,7 +35,7 @@ def __init__(self, version, build_env, config=None): if config: self.config = config else: - self.config = ConfigWrapper(version=version, yaml_config={}) + self.config = load_yaml_config(version) # Compute here, since it's used a lot self.checkout_path = self.project.checkout_path(self.version.slug) diff --git a/readthedocs/rtd_tests/tests/test_builds.py b/readthedocs/rtd_tests/tests/test_builds.py index 8fa6a8fff68..31b4d348fd7 100644 --- a/readthedocs/rtd_tests/tests/test_builds.py +++ b/readthedocs/rtd_tests/tests/test_builds.py @@ -8,12 +8,12 @@ from django_dynamic_fixture import fixture from readthedocs.projects.models import Project -from readthedocs.doc_builder.config import ConfigWrapper +from readthedocs.doc_builder.config import load_yaml_config from readthedocs.doc_builder.environments import LocalBuildEnvironment from readthedocs.doc_builder.python_environments import Virtualenv from readthedocs.doc_builder.loader import get_builder_class from readthedocs.projects.tasks import UpdateDocsTaskStep -from readthedocs.rtd_tests.tests.test_config_wrapper import create_load +from readthedocs.rtd_tests.tests.test_config_integration import create_load from ..mocks.environment import EnvironmentMockGroup @@ -27,8 +27,10 @@ def setUp(self): def tearDown(self): self.mocks.stop() - def test_build(self): + @mock.patch('readthedocs.doc_builder.config.load_config') + def test_build(self, load_config): '''Test full build''' + load_config.side_effect = create_load() project = get(Project, slug='project-1', documentation_type='sphinx', @@ -43,7 +45,7 @@ def test_build(self): build_env = LocalBuildEnvironment(project=project, version=version, build={}) python_env = Virtualenv(version=version, build_env=build_env) - config = ConfigWrapper(version=version, yaml_config=create_load()()[0]) + config = load_yaml_config(version) task = UpdateDocsTaskStep(build_env=build_env, project=project, python_env=python_env, version=version, search=False, localmedia=False, config=config) task.build_docs() @@ -54,8 +56,10 @@ def test_build(self): self.assertRegexpMatches(cmd[0][0], r'python') self.assertRegexpMatches(cmd[0][1], r'sphinx-build') - def test_build_respects_pdf_flag(self): + @mock.patch('readthedocs.doc_builder.config.load_config') + def test_build_respects_pdf_flag(self, load_config): '''Build output format control''' + load_config.side_effect = create_load() project = get(Project, slug='project-1', documentation_type='sphinx', @@ -67,7 +71,7 @@ def test_build_respects_pdf_flag(self): build_env = LocalBuildEnvironment(project=project, version=version, build={}) python_env = Virtualenv(version=version, build_env=build_env) - config = ConfigWrapper(version=version, yaml_config=create_load()()[0]) + config = load_yaml_config(version) task = UpdateDocsTaskStep(build_env=build_env, project=project, python_env=python_env, version=version, search=False, localmedia=False, config=config) @@ -79,8 +83,10 @@ def test_build_respects_pdf_flag(self): # PDF however was disabled and therefore not built. self.assertFalse(self.mocks.epub_build.called) - def test_build_respects_epub_flag(self): + @mock.patch('readthedocs.doc_builder.config.load_config') + def test_build_respects_epub_flag(self, load_config): '''Test build with epub enabled''' + load_config.side_effect = create_load() project = get(Project, slug='project-1', documentation_type='sphinx', @@ -92,7 +98,7 @@ def test_build_respects_epub_flag(self): build_env = LocalBuildEnvironment(project=project, version=version, build={}) python_env = Virtualenv(version=version, build_env=build_env) - config = ConfigWrapper(version=version, yaml_config=create_load()()[0]) + config = load_yaml_config(version) task = UpdateDocsTaskStep(build_env=build_env, project=project, python_env=python_env, version=version, search=False, localmedia=False, config=config) task.build_docs() @@ -103,8 +109,10 @@ def test_build_respects_epub_flag(self): # PDF however was disabled and therefore not built. self.assertFalse(self.mocks.pdf_build.called) - def test_build_respects_yaml(self): + @mock.patch('readthedocs.doc_builder.config.load_config') + def test_build_respects_yaml(self, load_config): '''Test YAML build options''' + load_config.side_effect = create_load({'formats': ['epub']}) project = get(Project, slug='project-1', documentation_type='sphinx', @@ -116,9 +124,8 @@ def test_build_respects_yaml(self): build_env = LocalBuildEnvironment(project=project, version=version, build={}) python_env = Virtualenv(version=version, build_env=build_env) - config = ConfigWrapper(version=version, yaml_config=create_load({ - 'formats': ['epub'] - })()[0]) + + config = load_yaml_config(version) task = UpdateDocsTaskStep(build_env=build_env, project=project, python_env=python_env, version=version, search=False, localmedia=False, config=config) task.build_docs() @@ -129,9 +136,11 @@ def test_build_respects_yaml(self): # PDF however was disabled and therefore not built. self.assertFalse(self.mocks.pdf_build.called) - def test_build_pdf_latex_failures(self): + @mock.patch('readthedocs.doc_builder.config.load_config') + def test_build_pdf_latex_failures(self, load_config): '''Build failure if latex fails''' + load_config.side_effect = create_load() self.mocks.patches['html_build'].stop() self.mocks.patches['pdf_build'].stop() @@ -147,7 +156,7 @@ def test_build_pdf_latex_failures(self): build_env = LocalBuildEnvironment(project=project, version=version, build={}) python_env = Virtualenv(version=version, build_env=build_env) - config = ConfigWrapper(version=version, yaml_config=create_load()()[0]) + config = load_yaml_config(version) task = UpdateDocsTaskStep(build_env=build_env, project=project, python_env=python_env, version=version, search=False, localmedia=False, config=config) @@ -171,9 +180,11 @@ def test_build_pdf_latex_failures(self): self.assertEqual(self.mocks.popen.call_count, 7) self.assertTrue(build_env.failed) - def test_build_pdf_latex_not_failure(self): + @mock.patch('readthedocs.doc_builder.config.load_config') + def test_build_pdf_latex_not_failure(self, load_config): '''Test pass during PDF builds and bad latex failure status code''' + load_config.side_effect = create_load() self.mocks.patches['html_build'].stop() self.mocks.patches['pdf_build'].stop() @@ -189,7 +200,7 @@ def test_build_pdf_latex_not_failure(self): build_env = LocalBuildEnvironment(project=project, version=version, build={}) python_env = Virtualenv(version=version, build_env=build_env) - config = ConfigWrapper(version=version, yaml_config=create_load()()[0]) + config = load_yaml_config(version) task = UpdateDocsTaskStep(build_env=build_env, project=project, python_env=python_env, version=version, search=False, localmedia=False, config=config) diff --git a/readthedocs/rtd_tests/tests/test_config_wrapper.py b/readthedocs/rtd_tests/tests/test_config_integration.py similarity index 87% rename from readthedocs/rtd_tests/tests/test_config_wrapper.py rename to readthedocs/rtd_tests/tests/test_config_integration.py index a8bcddbbe27..4d43b194567 100644 --- a/readthedocs/rtd_tests/tests/test_config_wrapper.py +++ b/readthedocs/rtd_tests/tests/test_config_integration.py @@ -1,12 +1,14 @@ -from __future__ import absolute_import +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + import mock from django.test import TestCase from django_dynamic_fixture import get -from readthedocs.config import BuildConfig, ProjectConfig, InvalidConfig from readthedocs.builds.models import Version -from readthedocs.projects.models import Project +from readthedocs.config import BuildConfig, InvalidConfig, ProjectConfig from readthedocs.doc_builder.config import load_yaml_config +from readthedocs.projects.models import Project def create_load(config=None): @@ -49,6 +51,8 @@ def setUp(self): def test_python_supported_versions_default_image_1_0(self, load_config): load_config.side_effect = create_load() self.project.container_image = 'readthedocs/build:1.0' + self.project.enable_epub_build = True + self.project.enable_pdf_build = True self.project.save() config = load_yaml_config(self.version) self.assertEqual(load_config.call_count, 1) @@ -57,7 +61,15 @@ def test_python_supported_versions_default_image_1_0(self, load_config): 'build': {'image': 'readthedocs/build:1.0'}, 'type': 'sphinx', 'output_base': '', - 'name': mock.ANY + 'name': mock.ANY, + 'defaults': { + 'install_project': self.project.install_project, + 'formats': ['htmlzip', 'epub', 'pdf'], + 'use_system_packages': self.project.use_system_packages, + 'requirements_file': self.project.requirements_file, + 'python_version': 2, + 'build_image': 'readthedocs/build:1.0', + }, }), ]) self.assertEqual(config.python_version, 2) @@ -67,7 +79,7 @@ def test_python_supported_versions_image_1_0(self, load_config): self.project.container_image = 'readthedocs/build:1.0' self.project.save() config = load_yaml_config(self.version) - self.assertEqual(config._yaml_config.get_valid_python_versions(), + self.assertEqual(config.get_valid_python_versions(), [2, 2.7, 3, 3.4]) def test_python_supported_versions_image_2_0(self, load_config): @@ -75,7 +87,7 @@ def test_python_supported_versions_image_2_0(self, load_config): self.project.container_image = 'readthedocs/build:2.0' self.project.save() config = load_yaml_config(self.version) - self.assertEqual(config._yaml_config.get_valid_python_versions(), + self.assertEqual(config.get_valid_python_versions(), [2, 2.7, 3, 3.5]) def test_python_supported_versions_image_latest(self, load_config): @@ -83,7 +95,7 @@ def test_python_supported_versions_image_latest(self, load_config): self.project.container_image = 'readthedocs/build:latest' self.project.save() config = load_yaml_config(self.version) - self.assertEqual(config._yaml_config.get_valid_python_versions(), + self.assertEqual(config.get_valid_python_versions(), [2, 2.7, 3, 3.3, 3.4, 3.5, 3.6]) def test_python_default_version(self, load_config): @@ -186,6 +198,7 @@ def test_requirements_file(self, load_config): config = load_yaml_config(self.version) self.assertEqual(config.requirements_file, requirements_file) + # Respects the requirements file from the project settings load_config.side_effect = create_load() config = load_yaml_config(self.version) self.assertEqual(config.requirements_file, 'urls.py') diff --git a/readthedocs/rtd_tests/tests/test_doc_building.py b/readthedocs/rtd_tests/tests/test_doc_building.py index d255c555ae0..6ed3843aa78 100644 --- a/readthedocs/rtd_tests/tests/test_doc_building.py +++ b/readthedocs/rtd_tests/tests/test_doc_building.py @@ -8,32 +8,33 @@ from __future__ import ( absolute_import, division, print_function, unicode_literals) -import os.path import json +import os import re import tempfile import uuid -from builtins import str import mock import pytest +from builtins import str from django.test import TestCase +from django_dynamic_fixture import get from docker.errors import APIError as DockerAPIError from docker.errors import DockerException from mock import Mock, PropertyMock, mock_open, patch -from django_dynamic_fixture import get from readthedocs.builds.constants import BUILD_STATE_CLONING from readthedocs.builds.models import Version -from readthedocs.doc_builder.config import ConfigWrapper +from readthedocs.doc_builder.config import load_yaml_config from readthedocs.doc_builder.environments import ( - BuildCommand, DockerBuildCommand, DockerBuildEnvironment, LocalBuildEnvironment) + BuildCommand, DockerBuildCommand, DockerBuildEnvironment, + LocalBuildEnvironment) from readthedocs.doc_builder.exceptions import BuildEnvironmentError from readthedocs.doc_builder.python_environments import Conda, Virtualenv from readthedocs.projects.models import Project from readthedocs.rtd_tests.mocks.environment import EnvironmentMockGroup from readthedocs.rtd_tests.mocks.paths import fake_paths_lookup -from readthedocs.rtd_tests.tests.test_config_wrapper import create_load +from readthedocs.rtd_tests.tests.test_config_integration import create_load DUMMY_BUILD_ID = 123 SAMPLE_UNICODE = u'HérÉ îß sömê ünïçó∂é' @@ -1364,7 +1365,8 @@ def setUp(self): build={'id': DUMMY_BUILD_ID}, ) - def test_save_environment_json(self): + @mock.patch('readthedocs.doc_builder.config.load_config') + def test_save_environment_json(self, load_config): config_data = { 'build': { 'image': '2.0', @@ -1373,8 +1375,8 @@ def test_save_environment_json(self): 'version': 2.7, }, } - yaml_config = create_load(config_data)()[0] - config = ConfigWrapper(version=self.version, yaml_config=yaml_config) + load_config.side_effect = create_load(config_data) + config = load_yaml_config(self.version) python_env = Virtualenv( version=self.version, @@ -1400,9 +1402,10 @@ def test_save_environment_json(self): } self.assertDictEqual(json_data, expected_data) - def test_is_obsolete_without_env_json_file(self): - yaml_config = create_load()()[0] - config = ConfigWrapper(version=self.version, yaml_config=yaml_config) + @mock.patch('readthedocs.doc_builder.config.load_config') + def test_is_obsolete_without_env_json_file(self, load_config): + load_config.side_effect = create_load() + config = load_yaml_config(self.version) with patch('os.path.exists') as exists: exists.return_value = False @@ -1414,9 +1417,10 @@ def test_is_obsolete_without_env_json_file(self): self.assertFalse(python_env.is_obsolete) - def test_is_obsolete_with_invalid_env_json_file(self): - yaml_config = create_load()()[0] - config = ConfigWrapper(version=self.version, yaml_config=yaml_config) + @mock.patch('readthedocs.doc_builder.config.load_config') + def test_is_obsolete_with_invalid_env_json_file(self, load_config): + load_config.side_effect = create_load() + config = load_yaml_config(self.version) with patch('os.path.exists') as exists: exists.return_value = True @@ -1428,7 +1432,8 @@ def test_is_obsolete_with_invalid_env_json_file(self): self.assertFalse(python_env.is_obsolete) - def test_is_obsolete_with_json_different_python_version(self): + @mock.patch('readthedocs.doc_builder.config.load_config') + def test_is_obsolete_with_json_different_python_version(self, load_config): config_data = { 'build': { 'image': '2.0', @@ -1437,8 +1442,8 @@ def test_is_obsolete_with_json_different_python_version(self): 'version': 2.7, }, } - yaml_config = create_load(config_data)()[0] - config = ConfigWrapper(version=self.version, yaml_config=yaml_config) + load_config.side_effect = create_load(config_data) + config = load_yaml_config(self.version) python_env = Virtualenv( version=self.version, @@ -1450,7 +1455,8 @@ def test_is_obsolete_with_json_different_python_version(self): exists.return_value = True self.assertTrue(python_env.is_obsolete) - def test_is_obsolete_with_json_different_build_image(self): + @mock.patch('readthedocs.doc_builder.config.load_config') + def test_is_obsolete_with_json_different_build_image(self, load_config): config_data = { 'build': { 'image': 'latest', @@ -1459,8 +1465,8 @@ def test_is_obsolete_with_json_different_build_image(self): 'version': 2.7, }, } - yaml_config = create_load(config_data)()[0] - config = ConfigWrapper(version=self.version, yaml_config=yaml_config) + load_config.side_effect = create_load(config_data) + config = load_yaml_config(self.version) python_env = Virtualenv( version=self.version, @@ -1470,9 +1476,11 @@ def test_is_obsolete_with_json_different_build_image(self): env_json_data = '{"build": {"image": "readthedocs/build:2.0", "hash": "a1b2c3"}, "python": {"version": 2.7}}' # noqa with patch('os.path.exists') as exists, patch('readthedocs.doc_builder.python_environments.open', mock_open(read_data=env_json_data)) as _open: # noqa exists.return_value = True - self.assertTrue(python_env.is_obsolete) + obsolete = python_env.is_obsolete + self.assertTrue(obsolete) - def test_is_obsolete_with_project_different_build_image(self): + @mock.patch('readthedocs.doc_builder.config.load_config') + def test_is_obsolete_with_project_different_build_image(self, load_config): config_data = { 'build': { 'image': '2.0', @@ -1481,13 +1489,14 @@ def test_is_obsolete_with_project_different_build_image(self): 'version': 2.7, }, } - yaml_config = create_load(config_data)()[0] - config = ConfigWrapper(version=self.version, yaml_config=yaml_config) + load_config.side_effect = create_load(config_data) # Set container_image manually self.pip.container_image = 'readthedocs/build:latest' self.pip.save() + config = load_yaml_config(self.version) + python_env = Virtualenv( version=self.version, build_env=self.build_env, @@ -1498,7 +1507,8 @@ def test_is_obsolete_with_project_different_build_image(self): exists.return_value = True self.assertTrue(python_env.is_obsolete) - def test_is_obsolete_with_json_same_data_as_version(self): + @mock.patch('readthedocs.doc_builder.config.load_config') + def test_is_obsolete_with_json_same_data_as_version(self, load_config): config_data = { 'build': { 'image': '2.0', @@ -1507,8 +1517,8 @@ def test_is_obsolete_with_json_same_data_as_version(self): 'version': 3.5, }, } - yaml_config = create_load(config_data)()[0] - config = ConfigWrapper(version=self.version, yaml_config=yaml_config) + load_config.side_effect = create_load(config_data) + config = load_yaml_config(self.version) python_env = Virtualenv( version=self.version, @@ -1520,7 +1530,8 @@ def test_is_obsolete_with_json_same_data_as_version(self): exists.return_value = True self.assertFalse(python_env.is_obsolete) - def test_is_obsolete_with_json_different_build_hash(self): + @mock.patch('readthedocs.doc_builder.config.load_config') + def test_is_obsolete_with_json_different_build_hash(self, load_config): config_data = { 'build': { 'image': '2.0', @@ -1529,8 +1540,8 @@ def test_is_obsolete_with_json_different_build_hash(self): 'version': 2.7, }, } - yaml_config = create_load(config_data)()[0] - config = ConfigWrapper(version=self.version, yaml_config=yaml_config) + load_config.side_effect = create_load(config_data) + config = load_yaml_config(self.version) # Set container_image manually self.pip.container_image = 'readthedocs/build:2.0' @@ -1546,7 +1557,8 @@ def test_is_obsolete_with_json_different_build_hash(self): exists.return_value = True self.assertTrue(python_env.is_obsolete) - def test_is_obsolete_with_json_missing_build_hash(self): + @mock.patch('readthedocs.doc_builder.config.load_config') + def test_is_obsolete_with_json_missing_build_hash(self, load_config): config_data = { 'build': { 'image': '2.0', @@ -1556,8 +1568,8 @@ def test_is_obsolete_with_json_missing_build_hash(self): 'version': 2.7, }, } - yaml_config = create_load(config_data)()[0] - config = ConfigWrapper(version=self.version, yaml_config=yaml_config) + load_config.side_effect = create_load(config_data) + config = load_yaml_config(self.version) # Set container_image manually self.pip.container_image = 'readthedocs/build:2.0'