diff --git a/readthedocs/doc_builder/config.py b/readthedocs/doc_builder/config.py index fd4798debb8..51ce220cee1 100644 --- a/readthedocs/doc_builder/config.py +++ b/readthedocs/doc_builder/config.py @@ -2,6 +2,7 @@ load as load_config) +from .constants import BUILD_IMAGES, DOCKER_IMAGE from readthedocs.projects.exceptions import ProjectImportError @@ -49,25 +50,27 @@ def extra_requirements(self): @property def python_interpreter(self): - if 'version' in self._yaml_config.get('python', {}): - ver = self._yaml_config['python']['version'] - if str(ver).startswith('2'): - return 'python' - else: - return 'python3' - else: - return self._project.python_interpreter + ver = self.python_version + if ver in [2, 3]: + # Get the highest version of the major series version if user only + # gave us a version of '2', or '3' + ver = max(filter( + lambda x: x < ver + 1, + self._yaml_config.get_valid_python_versions(), + )) + return 'python{0}'.format(ver) @property def python_version(self): + # There should always be a version in the YAML config. If the config + # version is the default response of `2`, then assume we can use the + # Python.python_interpreter version to infer this value instead. + version = 2 if 'version' in self._yaml_config.get('python', {}): - ver = self._yaml_config['python']['version'] - return ver - else: - if self._project.python_interpreter == 'python': - return 2 - else: - return 3 + version = self._yaml_config['python']['version'] + if version == 2 and self._project.python_interpreter == 'python3': + version = 3 + return version @property def use_system_site_packages(self): @@ -125,20 +128,36 @@ def load_yaml_config(version): """ checkout_path = version.project.checkout_path(version.slug) + env_config = {} + + # Get build image to set up the python version validation. Pass in the + # build image python limitations to the loaded config so that the versions + # can be rejected at validation + build_image = BUILD_IMAGES.get( + version.project.container_image, + BUILD_IMAGES.get(DOCKER_IMAGE, None), + ) + if build_image: + env_config = { + 'python': build_image['python'], + } + try: + sphinx_env_config = env_config.copy() + sphinx_env_config.update({ + 'output_base': '', + 'type': 'sphinx', + 'name': version.slug, + }) config = load_config( path=checkout_path, - env_config={ - 'output_base': '', - 'type': 'sphinx', - 'name': version.slug, - }, + env_config=sphinx_env_config, )[0] except InvalidConfig: # This is a subclass of ConfigError, so has to come first raise except ConfigError: config = BuildConfig( - env_config={}, + env_config=env_config, raw_config={}, source_file='empty', source_position=0, diff --git a/readthedocs/doc_builder/constants.py b/readthedocs/doc_builder/constants.py index b182f35dc72..8a9a6afa086 100644 --- a/readthedocs/doc_builder/constants.py +++ b/readthedocs/doc_builder/constants.py @@ -15,9 +15,10 @@ PDF_RE = re.compile('Output written on (.*?)') +# Docker DOCKER_SOCKET = getattr(settings, 'DOCKER_SOCKET', 'unix:///var/run/docker.sock') DOCKER_VERSION = getattr(settings, 'DOCKER_VERSION', 'auto') -DOCKER_IMAGE = getattr(settings, 'DOCKER_IMAGE', 'rtfd-build') +DOCKER_IMAGE = getattr(settings, 'DOCKER_IMAGE', 'readthedocs/build:2.0') DOCKER_LIMITS = {'memory': '200m', 'time': 600} DOCKER_LIMITS.update(getattr(settings, 'DOCKER_LIMITS', {})) @@ -25,3 +26,16 @@ DOCKER_OOM_EXIT_CODE = 137 DOCKER_HOSTNAME_MAX_LEN = 64 + +# Build images +BUILD_IMAGES = { + 'readthedocs/build:1.0': { + 'python': {'supported_versions': [2, 2.7, 3, 3.3]}, + }, + 'readthedocs/build:2.0': { + 'python': {'supported_versions': [2, 2.7, 3, 3.5]}, + }, + 'readthedocs/build:latest': { + 'python': {'supported_versions': [2, 2.7, 3, 3.3, 3.4, 3.5, 3.6]}, + }, +} diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index e040fb1e460..a5a9ba2e201 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -18,6 +18,7 @@ from django.conf import settings from django.core.urlresolvers import reverse from django.utils.translation import ugettext_lazy as _ +from readthedocs_build.config import ConfigError from readthedocs.builds.constants import (LATEST, BUILD_STATE_CLONING, @@ -132,7 +133,12 @@ def run(self, pk, version_pk=None, build_pk=None, record=True, docker=False, status_code=423 ) - self.config = load_yaml_config(version=self.version) + try: + self.config = load_yaml_config(version=self.version) + except ConfigError as e: + raise BuildEnvironmentError( + 'Problem parsing YAML configuration. {0}'.format(str(e)) + ) if self.setup_env.failure or self.config is None: self._log('Failing build because of setup failure: %s' % self.setup_env.failure) diff --git a/readthedocs/rtd_tests/tests/test_builds.py b/readthedocs/rtd_tests/tests/test_builds.py index c0b2d6819dd..8dcbe230b12 100644 --- a/readthedocs/rtd_tests/tests/test_builds.py +++ b/readthedocs/rtd_tests/tests/test_builds.py @@ -10,7 +10,7 @@ from readthedocs.doc_builder.python_environments import Virtualenv from readthedocs.doc_builder.loader import get_builder_class from readthedocs.projects.tasks import UpdateDocsTask -from readthedocs.rtd_tests.tests.test_config_wrapper import get_build_config +from readthedocs.rtd_tests.tests.test_config_wrapper import create_load from ..mocks.environment import EnvironmentMockGroup @@ -40,8 +40,7 @@ def test_build(self): build_env = LocalEnvironment(project=project, version=version, build={}) python_env = Virtualenv(version=version, build_env=build_env) - yaml_config = get_build_config({}) - config = ConfigWrapper(version=version, yaml_config=yaml_config) + config = ConfigWrapper(version=version, yaml_config=create_load()()[0]) task = UpdateDocsTask(build_env=build_env, project=project, python_env=python_env, version=version, search=False, localmedia=False, config=config) task.build_docs() @@ -65,8 +64,7 @@ def test_build_respects_pdf_flag(self): build_env = LocalEnvironment(project=project, version=version, build={}) python_env = Virtualenv(version=version, build_env=build_env) - yaml_config = get_build_config({}) - config = ConfigWrapper(version=version, yaml_config=yaml_config) + config = ConfigWrapper(version=version, yaml_config=create_load()()[0]) task = UpdateDocsTask(build_env=build_env, project=project, python_env=python_env, version=version, search=False, localmedia=False, config=config) @@ -91,8 +89,7 @@ def test_build_respects_epub_flag(self): build_env = LocalEnvironment(project=project, version=version, build={}) python_env = Virtualenv(version=version, build_env=build_env) - yaml_config = get_build_config({}) - config = ConfigWrapper(version=version, yaml_config=yaml_config) + config = ConfigWrapper(version=version, yaml_config=create_load()()[0]) task = UpdateDocsTask(build_env=build_env, project=project, python_env=python_env, version=version, search=False, localmedia=False, config=config) task.build_docs() @@ -116,8 +113,9 @@ def test_build_respects_yaml(self): build_env = LocalEnvironment(project=project, version=version, build={}) python_env = Virtualenv(version=version, build_env=build_env) - yaml_config = get_build_config({'formats': ['epub']}) - config = ConfigWrapper(version=version, yaml_config=yaml_config) + config = ConfigWrapper(version=version, yaml_config=create_load({ + 'formats': ['epub'] + })()[0]) task = UpdateDocsTask(build_env=build_env, project=project, python_env=python_env, version=version, search=False, localmedia=False, config=config) task.build_docs() @@ -171,8 +169,7 @@ def test_build_pdf_latex_failures(self): build_env = LocalEnvironment(project=project, version=version, build={}) python_env = Virtualenv(version=version, build_env=build_env) - yaml_config = get_build_config({}) - config = ConfigWrapper(version=version, yaml_config=yaml_config) + config = ConfigWrapper(version=version, yaml_config=create_load()()[0]) task = UpdateDocsTask(build_env=build_env, project=project, python_env=python_env, version=version, search=False, localmedia=False, config=config) @@ -213,8 +210,7 @@ def test_build_pdf_latex_not_failure(self): build_env = LocalEnvironment(project=project, version=version, build={}) python_env = Virtualenv(version=version, build_env=build_env) - yaml_config = get_build_config({}) - config = ConfigWrapper(version=version, yaml_config=yaml_config) + config = ConfigWrapper(version=version, yaml_config=create_load()()[0]) task = UpdateDocsTask(build_env=build_env, project=project, python_env=python_env, version=version, search=False, localmedia=False, config=config) diff --git a/readthedocs/rtd_tests/tests/test_config_wrapper.py b/readthedocs/rtd_tests/tests/test_config_wrapper.py index 3a0ae4afd04..1240cdc77e6 100644 --- a/readthedocs/rtd_tests/tests/test_config_wrapper.py +++ b/readthedocs/rtd_tests/tests/test_config_wrapper.py @@ -1,99 +1,197 @@ +import mock from django.test import TestCase - from django_dynamic_fixture import get -from readthedocs_build.config import BuildConfig +from readthedocs_build.config import BuildConfig, ProjectConfig, InvalidConfig from readthedocs.builds.models import Version from readthedocs.projects.models import Project -from readthedocs.doc_builder.config import ConfigWrapper - - -def get_build_config(config, env_config=None, source_file='readthedocs.yml', - source_position=0): - config['name'] = 'test' - config['type'] = 'sphinx' - ret_config = BuildConfig( - {'output_base': ''}, - config, - source_file=source_file, - source_position=source_position) - ret_config.validate() - return ret_config - - -class ConfigWrapperTests(TestCase): +from readthedocs.doc_builder.config import ConfigWrapper, load_yaml_config + + +def create_load(config=None): + """Mock out the function of the build load function + + This will create a ProjectConfig list of BuildConfig objects and validate + them. The default load function iterates over files and builds up a list of + objects. Instead of mocking all of this, just mock the end result. + """ + if config is None: + config = {} + + def inner(path=None, env_config=None): + env_config_defaults = { + 'output_base': '', + 'name': '1', + 'type': 'sphinx', + } + if env_config is not None: + env_config_defaults.update(env_config) + yaml_config = ProjectConfig([ + BuildConfig(env_config_defaults, + config, + source_file='readthedocs.yml', + source_position=0) + ]) + yaml_config.validate() + return yaml_config + return inner + + +@mock.patch('readthedocs.doc_builder.config.load_config') +class LoadConfigTests(TestCase): def setUp(self): - self.project = get(Project, slug='test', python_interpreter='python', + self.project = get(Project, main_language_project=None, install_project=False, requirements_file='urls.py') - self.version = get(Version, project=self.project, slug='foobar') - - def test_python_version(self): - yaml_config = get_build_config({'python': {'version': 3}}) - config = ConfigWrapper(version=self.version, yaml_config=yaml_config) - self.assertEqual(config.python_version, 3) + self.version = get(Version, project=self.project) + + def test_python_supported_versions_default_image_1_0(self, load_config): + load_config.side_effect = create_load() + self.project.container_image = 'readthedocs/build:1.0' + self.project.save() + config = load_yaml_config(self.version) + self.assertEqual(load_config.call_count, 1) + load_config.assert_has_calls([ + mock.call(path=mock.ANY, env_config={ + 'python': {'supported_versions': [2, 2.7, 3, 3.3]}, + 'type': 'sphinx', + 'output_base': '', + 'name': mock.ANY + }), + ]) + self.assertEqual(config.python_version, 2) - yaml_config = get_build_config({}) - config = ConfigWrapper(version=self.version, yaml_config=yaml_config) + def test_python_supported_versions_image_2_0(self, load_config): + load_config.side_effect = create_load() + self.project.container_image = 'readthedocs/build:2.0' + self.project.save() + config = load_yaml_config(self.version) + self.assertEqual(load_config.call_count, 1) + load_config.assert_has_calls([ + mock.call(path=mock.ANY, env_config={ + 'python': {'supported_versions': [2, 2.7, 3, 3.5]}, + 'type': 'sphinx', + 'output_base': '', + 'name': mock.ANY + }), + ]) self.assertEqual(config.python_version, 2) - def test_python_interpreter(self): - yaml_config = get_build_config({'python': {'version': 3}}) - config = ConfigWrapper(version=self.version, yaml_config=yaml_config) - self.assertEqual(config.python_interpreter, 'python3') + def test_python_supported_versions_image_latest(self, load_config): + load_config.side_effect = create_load() + self.project.container_image = 'readthedocs/build:latest' + self.project.save() + config = load_yaml_config(self.version) + self.assertEqual(load_config.call_count, 1) + load_config.assert_has_calls([ + mock.call(path=mock.ANY, env_config={ + 'python': {'supported_versions': [2, 2.7, 3, 3.3, 3.4, 3.5, 3.6]}, + 'type': 'sphinx', + 'output_base': '', + 'name': mock.ANY + }), + ]) + self.assertEqual(config.python_version, 2) - yaml_config = get_build_config({}) - config = ConfigWrapper(version=self.version, yaml_config=yaml_config) - self.assertEqual(config.python_interpreter, 'python') + def test_python_default_version(self, load_config): + load_config.side_effect = create_load() + config = load_yaml_config(self.version) + self.assertEqual(config.python_version, 2) + self.assertEqual(config.python_interpreter, 'python2.7') + + def test_python_set_python_version_on_project(self, load_config): + load_config.side_effect = create_load() + self.project.container_image = 'readthedocs/build:2.0' + self.project.python_interpreter = 'python3' + self.project.save() + config = load_yaml_config(self.version) + self.assertEqual(config.python_version, 3) + self.assertEqual(config.python_interpreter, 'python3.5') + + def test_python_set_python_version_in_config(self, load_config): + load_config.side_effect = create_load({ + 'python': {'version': 3.5} + }) + self.project.container_image = 'readthedocs/build:2.0' + self.project.save() + config = load_yaml_config(self.version) + self.assertEqual(config.python_version, 3.5) + self.assertEqual(config.python_interpreter, 'python3.5') + + def test_python_invalid_version_in_config(self, load_config): + load_config.side_effect = create_load({ + 'python': {'version': 2.6} + }) + self.project.container_image = 'readthedocs/build:2.0' + self.project.save() + with self.assertRaises(InvalidConfig): + config = load_yaml_config(self.version) + + def test_install_project(self, load_config): + load_config.side_effect = create_load() + config = load_yaml_config(self.version) + self.assertEqual(config.install_project, False) - def test_install_project(self): - yaml_config = get_build_config({'python': {'setup_py_install': True}}) - config = ConfigWrapper(version=self.version, yaml_config=yaml_config) + load_config.side_effect = create_load({ + 'python': {'setup_py_install': True} + }) + config = load_yaml_config(self.version) self.assertEqual(config.install_project, True) - yaml_config = get_build_config({}) - config = ConfigWrapper(version=self.version, yaml_config=yaml_config) - self.assertEqual(config.install_project, False) - - def test_extra_requirements(self): - yaml_config = get_build_config({'python': { - 'pip_install': True, - 'extra_requirements': ['tests', 'docs']}}) - config = ConfigWrapper(version=self.version, yaml_config=yaml_config) + def test_extra_requirements(self, load_config): + load_config.side_effect = create_load({ + 'python': { + 'pip_install': True, + 'extra_requirements': ['tests', 'docs'] + } + }) + config = load_yaml_config(self.version) self.assertEqual(config.extra_requirements, ['tests', 'docs']) - yaml_config = get_build_config({'python': { - 'extra_requirements': ['tests', 'docs']}}) - config = ConfigWrapper(version=self.version, yaml_config=yaml_config) + load_config.side_effect = create_load({ + 'python': { + 'extra_requirements': ['tests', 'docs'] + } + }) + config = load_yaml_config(self.version) self.assertEqual(config.extra_requirements, []) - yaml_config = get_build_config({}) - config = ConfigWrapper(version=self.version, yaml_config=yaml_config) + load_config.side_effect = create_load() + config = load_yaml_config(self.version) self.assertEqual(config.extra_requirements, []) - yaml_config = get_build_config({'python': { - 'setup_py_install': True, - 'extra_requirements': ['tests', 'docs']}}) - config = ConfigWrapper(version=self.version, yaml_config=yaml_config) + load_config.side_effect = create_load({ + 'python': { + 'setup_py_install': True, + 'extra_requirements': ['tests', 'docs'] + } + }) + config = load_yaml_config(self.version) self.assertEqual(config.extra_requirements, []) - def test_conda(self): + def test_conda(self, load_config): to_find = 'urls.py' - yaml_config = get_build_config({'conda': {'file': to_find}}) - config = ConfigWrapper(version=self.version, yaml_config=yaml_config) + load_config.side_effect = create_load({ + 'conda': { + 'file': to_find + } + }) + config = load_yaml_config(self.version) self.assertEqual(config.use_conda, True) self.assertTrue(config.conda_file[-len(to_find):] == to_find) - yaml_config = get_build_config({}) - config = ConfigWrapper(version=self.version, yaml_config=yaml_config) + load_config.side_effect = create_load() + config = load_yaml_config(self.version) self.assertEqual(config.use_conda, False) self.assertEqual(config.conda_file, None) - def test_requirements_file(self): - yaml_config = get_build_config({'requirements_file': 'wsgi.py'}) - config = ConfigWrapper(version=self.version, yaml_config=yaml_config) + def test_requirements_file(self, load_config): + load_config.side_effect = create_load({ + 'requirements_file': 'wsgi.py' + }) + config = load_yaml_config(self.version) self.assertEqual(config.requirements_file, 'wsgi.py') - yaml_config = get_build_config({}) - config = ConfigWrapper(version=self.version, yaml_config=yaml_config) + load_config.side_effect = create_load() + config = load_yaml_config(self.version) self.assertEqual(config.requirements_file, 'urls.py') diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index 1833dbed533..05c8a08f302 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -240,7 +240,7 @@ def INSTALLED_APPS(self): # noqa # Docker DOCKER_ENABLE = False - DOCKER_IMAGE = 'readthedocs/build:14.04' + DOCKER_IMAGE = 'readthedocs/build:1.0' # All auth ACCOUNT_ADAPTER = 'readthedocs.core.adapters.AccountAdapter' diff --git a/requirements/pip.txt b/requirements/pip.txt index 200019866d0..b11bd910835 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -5,7 +5,7 @@ docutils==0.11 Sphinx==1.3.5 Pygments==2.0.2 mkdocs==0.14.0 -git+https://github.com/rtfd/readthedocs-build.git@db4ad19df4f432bfbbd56d05964c58b6634356f3#egg=readthedocs-build-2.0.6.dev +readthedocs-build==2.0.6 django==1.8.16 django-tastypie==0.12.2