Merge pull request #5155 from rtfd/humitos/latest-docker-images

Use latest docker images as default

Author: ericholscher

readthedocs/config/config.py

@@ -4,6 +4,7 @@
 
 import copy
 import os
+import re
 from contextlib import contextmanager
 
 from django.conf import settings
@@ -146,6 +147,8 @@ class BuildConfigBase:
         'mkdocs',
         'submodules',
     ]
+
+    default_build_image = DOCKER_DEFAULT_VERSION
     version = None
 
     def __init__(self, env_config, raw_config, source_file):
@@ -246,6 +249,40 @@ def python_full_version(self):
             )
         return ver
 
+    @property
+    def valid_build_images(self):
+        """
+        Return all the valid Docker image choices for ``build.image`` option.
+
+        The user can use any of this values in the YAML file. These values are
+        the keys of ``DOCKER_IMAGE_SETTINGS`` Django setting (without the
+        ``readthedocs/build`` part) plus ``stable`` and ``latest``.
+        """
+        images = {'stable', 'latest'}
+        for k in DOCKER_IMAGE_SETTINGS:
+            _, version = k.split(':')
+            if re.fullmatch(r'^[\d\.]+$', version):
+                images.add(version)
+        return images
+
+    def get_valid_python_versions_for_image(self, build_image):
+        """
+        Return all the valid Python versions for a Docker image.
+
+        The Docker image (``build_image``) has to be its complete name, already
+        validated: ``readthedocs/build:4.0``, not just ``4.0``.
+
+        Returns supported versions for the ``DOCKER_DEFAULT_VERSION`` if not
+        ``build_image`` found.
+        """
+
+        if build_image not in DOCKER_IMAGE_SETTINGS:
+            build_image = '{}:{}'.format(
+                DOCKER_DEFAULT_IMAGE,
+                self.default_build_image,
+            )
+        return DOCKER_IMAGE_SETTINGS[build_image]['python']['supported_versions']
+
     def as_dict(self):
         config = {}
         for name in self.PUBLIC_ATTRIBUTES:
@@ -268,18 +305,23 @@ class BuildConfigV1(BuildConfigBase):
         '"python.extra_requirements" section must be a list.'
     )
 
-    PYTHON_SUPPORTED_VERSIONS = [2, 2.7, 3, 3.5]
-    DOCKER_SUPPORTED_VERSIONS = ['1.0', '2.0', 'latest']
-
     version = '1'
 
     def get_valid_python_versions(self):
-        """Get all valid python versions."""
+        """
+        Return all valid Python versions.
+
+        .. note::
+
+            It does not take current build image used into account.
+        """
         try:
             return self.env_config['python']['supported_versions']
         except (KeyError, TypeError):
-            pass
-        return self.PYTHON_SUPPORTED_VERSIONS
+            versions = set()
+            for _, options in DOCKER_IMAGE_SETTINGS.items():
+                versions = versions.union(options['python']['supported_versions'])
+            return versions
 
     def get_valid_formats(self):  # noqa
         """Get all valid documentation formats."""
@@ -339,7 +381,7 @@ def validate_build(self):
                 with self.catch_validation_error('build'):
                     build['image'] = validate_choice(
                         str(_build['image']),
-                        self.DOCKER_SUPPORTED_VERSIONS,
+                        self.valid_build_images,
                     )
             if ':' not in build['image']:
                 # Prepend proper image name to user's image name
@@ -577,8 +619,6 @@ class BuildConfigV2(BuildConfigBase):
 
     version = '2'
     valid_formats = ['htmlzip', 'pdf', 'epub']
-    valid_build_images = ['1.0', '2.0', '3.0', 'stable', 'latest']
-    default_build_image = 'latest'
     valid_install_method = [PIP, SETUPTOOLS]
     valid_sphinx_builders = {
         'html': 'sphinx',
@@ -793,13 +833,7 @@ def get_valid_python_versions(self):
         This should be called after ``validate_build()``.
         """
         build_image = self.build.image
-        if build_image not in DOCKER_IMAGE_SETTINGS:
-            build_image = '{}:{}'.format(
-                DOCKER_DEFAULT_IMAGE,
-                self.default_build_image,
-            )
-        python = DOCKER_IMAGE_SETTINGS[build_image]['python']
-        return python['supported_versions']
+        return self.get_valid_python_versions_for_image(build_image)
 
     def validate_doc_types(self):
         """

readthedocs/config/tests/test_config.py

@@ -363,8 +363,8 @@ def test_it_validates_wrong_type_right_value(self):
         )
         build.validate()
         assert build.python.version == 3
-        assert build.python_interpreter == 'python3.5'
-        assert build.python_full_version == 3.5
+        assert build.python_interpreter == 'python3.7'
+        assert build.python_full_version == 3.7
 
     def test_it_validates_env_supported_versions(self):
         build = get_build_config(
@@ -483,7 +483,7 @@ def test_it_fails_if_build_is_invalid_option(self, tmpdir):
         apply_fs(tmpdir, yaml_config_dir)
         build = BuildConfigV1(
             {},
-            {'build': {'image': 3.0}},
+            {'build': {'image': 3.2}},
             source_file=str(tmpdir.join('readthedocs.yml')),
         )
         with raises(InvalidConfig) as excinfo:
@@ -513,7 +513,7 @@ def test_it_works_on_python_validation(self, tmpdir):
             {},
             {
                 'build': {'image': 'latest'},
-                'python': {'version': '3.3'},
+                'python': {'version': '3.6'},
             },
             source_file=str(tmpdir.join('readthedocs.yml')),
         )
@@ -538,7 +538,7 @@ def test_default(self, tmpdir):
             source_file=str(tmpdir.join('readthedocs.yml')),
         )
         build.validate()
-        assert build.build.image == 'readthedocs/build:2.0'
+        assert build.build.image == 'readthedocs/build:latest'
 
     @pytest.mark.parametrize(
         'image', ['latest', 'readthedocs/build:3.0', 'rtd/build:latest'],
@@ -712,7 +712,7 @@ def test_as_dict(tmpdir):
             'use_system_site_packages': False,
         },
         'build': {
-            'image': 'readthedocs/build:2.0',
+            'image': 'readthedocs/build:latest',
         },
         'conda': None,
         'sphinx': {

readthedocs/rtd_tests/tests/test_config_integration.py

@@ -145,7 +145,7 @@ def test_python_supported_versions_image_latest(self, load_config):
         config = load_yaml_config(self.version)
         self.assertEqual(
             config.get_valid_python_versions(),
-            [2, 2.7, 3, 3.3, 3.4, 3.5, 3.6],
+            [2, 2.7, 3, 3.5, 3.6, 3.7],
         )
 
     @mock.patch('readthedocs.doc_builder.config.load_config')
@@ -507,7 +507,7 @@ def test_python_version(self, checkout_path, tmpdir):
 
         config = self.get_update_docs_task().config
         assert config.python.version == 3
-        assert config.python_full_version == 3.6
+        assert config.python_full_version == 3.7
 
     @patch('readthedocs.doc_builder.environments.BuildEnvironment.run')
     def test_python_install_requirements(self, run, checkout_path, tmpdir):

readthedocs/settings/base.py

@@ -274,7 +274,7 @@ def USE_PROMOS(self):  # noqa
     # Docker
     DOCKER_ENABLE = False
     DOCKER_DEFAULT_IMAGE = 'readthedocs/build'
-    DOCKER_DEFAULT_VERSION = '2.0'
+    DOCKER_DEFAULT_VERSION = 'latest'
     DOCKER_IMAGE = '{}:{}'.format(DOCKER_DEFAULT_IMAGE, DOCKER_DEFAULT_VERSION)
     DOCKER_IMAGE_SETTINGS = {
         'readthedocs/build:1.0': {
@@ -286,14 +286,17 @@ def USE_PROMOS(self):  # noqa
         'readthedocs/build:3.0': {
             'python': {'supported_versions': [2, 2.7, 3, 3.3, 3.4, 3.5, 3.6]},
         },
-        'readthedocs/build:stable': {
-            'python': {'supported_versions': [2, 2.7, 3, 3.3, 3.4, 3.5, 3.6]},
-        },
-        'readthedocs/build:latest': {
-            'python': {'supported_versions': [2, 2.7, 3, 3.3, 3.4, 3.5, 3.6]},
+        'readthedocs/build:4.0': {
+            'python': {'supported_versions': [2, 2.7, 3, 3.5, 3.6, 3.7]},
         },
     }
 
+    # Alias tagged via ``docker tag`` on the build servers
+    DOCKER_IMAGE_SETTINGS.update({
+        'readthedocs/build:stable': DOCKER_IMAGE_SETTINGS.get('readthedocs/build:3.0'),
+        'readthedocs/build:latest': DOCKER_IMAGE_SETTINGS.get('readthedocs/build:4.0'),
+    })
+
     # All auth
     ACCOUNT_ADAPTER = 'readthedocs.core.adapters.AccountAdapter'
     ACCOUNT_EMAIL_REQUIRED = True