diff --git a/docs/guides/feature-flags.rst b/docs/guides/feature-flags.rst index fd73f90a178..ad0945a5f4f 100644 --- a/docs/guides/feature-flags.rst +++ b/docs/guides/feature-flags.rst @@ -20,6 +20,11 @@ The version of ``conda`` used in the build process could not be the latest one. This is because we use Miniconda, which its release process is a little more slow than ``conda`` itself. In case you prefer to use the latest ``conda`` version available, this is the flag you need. +``CONDA_APPEND_CORE_REQUIREMENTS``: :featureflags:`CONDA_APPEND_CORE_REQUIREMENTS` + +Makes Read the Docs to install all the requirements at once on ``conda create`` step. +This helps users to pin dependencies on conda and to improve build time. + ``DONT_OVERWRITE_SPHINX_CONTEXT``: :featureflags:`DONT_OVERWRITE_SPHINX_CONTEXT` ``DONT_SHALLOW_CLONE``: :featureflags:`DONT_SHALLOW_CLONE` diff --git a/readthedocs/doc_builder/python_environments.py b/readthedocs/doc_builder/python_environments.py index 10f400822da..1204a759efa 100644 --- a/readthedocs/doc_builder/python_environments.py +++ b/readthedocs/doc_builder/python_environments.py @@ -1,14 +1,16 @@ """An abstraction over virtualenv and Conda environments.""" import copy +import codecs import hashlib import itertools import json import logging import os import shutil +import yaml -from readthedocs.config import PIP, SETUPTOOLS +from readthedocs.config import PIP, SETUPTOOLS, ParseError, parse as parse_yaml from readthedocs.config.models import PythonInstall, PythonInstallRequirements from readthedocs.doc_builder.config import load_yaml_config from readthedocs.doc_builder.constants import DOCKER_IMAGE @@ -442,6 +444,10 @@ def setup_base(self): if self.project.has_feature(Feature.UPDATE_CONDA_STARTUP): self._update_conda_startup() + if self.project.has_feature(Feature.CONDA_APPEND_CORE_REQUIREMENTS): + self._append_core_requirements() + self._show_environment_yaml() + self.build_env.run( 'conda', 'env', @@ -455,10 +461,75 @@ def setup_base(self): cwd=self.checkout_path, ) - def install_core_requirements(self): - """Install basic Read the Docs requirements into the Conda env.""" + def _show_environment_yaml(self): + """Show ``environment.yml`` file in the Build output.""" + self.build_env.run( + 'cat', + self.config.conda.environment, + cwd=self.checkout_path, + ) + + def _append_core_requirements(self): + """ + Append Read the Docs dependencies to Conda environment file. + + This help users to pin their dependencies properly without us upgrading + them in the second ``conda install`` run. + + See https://github.com/readthedocs/readthedocs.org/pull/5631 + """ + try: + inputfile = codecs.open( + os.path.join( + self.checkout_path, + self.config.conda.environment, + ), + encoding='utf-8', + mode='r', + ) + environment = parse_yaml(inputfile) + except IOError: + log.warning( + 'There was an error while reading Conda environment file.', + ) + except ParseError: + log.warning( + 'There was an error while parsing Conda environment file.', + ) + else: + # Append conda dependencies directly to ``dependencies`` and pip + # dependencies to ``dependencies.pip`` + pip_requirements, conda_requirements = self._get_core_requirements() + dependencies = environment.get('dependencies', []) + pip_dependencies = {'pip': pip_requirements} + + for item in dependencies: + if isinstance(item, dict) and 'pip' in item: + pip_requirements.extend(item.get('pip', [])) + dependencies.remove(item) + break + + dependencies.append(pip_dependencies) + environment.update({'dependencies': dependencies}) + try: + outputfile = codecs.open( + os.path.join( + self.checkout_path, + self.config.conda.environment, + ), + encoding='utf-8', + mode='w', + ) + yaml.safe_dump(environment, outputfile) + except IOError: + log.warning( + 'There was an error while writing the new Conda ' + 'environment file.', + ) + + def _get_core_requirements(self): # Use conda for requirements it packages - requirements = [ + conda_requirements = [ 'mock', 'pillow', ] @@ -472,8 +543,22 @@ def install_core_requirements(self): pip_requirements.append('mkdocs') else: pip_requirements.append('readthedocs-sphinx-ext') - requirements.extend(['sphinx', 'sphinx_rtd_theme']) + conda_requirements.extend(['sphinx', 'sphinx_rtd_theme']) + + return pip_requirements, conda_requirements + + def install_core_requirements(self): + """Install basic Read the Docs requirements into the Conda env.""" + if self.project.has_feature(Feature.CONDA_APPEND_CORE_REQUIREMENTS): + # Skip install core requirements since they were already appended to + # the user's ``environment.yml`` and installed at ``conda env + # create`` step. + return + + pip_requirements, conda_requirements = self._get_core_requirements() + # Install requirements via ``conda install`` command if they were + # not appended to the ``environment.yml`` file. cmd = [ 'conda', 'install', @@ -482,12 +567,13 @@ def install_core_requirements(self): '--name', self.version.slug, ] - cmd.extend(requirements) + cmd.extend(conda_requirements) self.build_env.run( *cmd, cwd=self.checkout_path, ) + # Install requirements via ``pip install`` pip_cmd = [ self.venv_bin(filename='python'), '-m', diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index 866dbbcd4db..e4272ecd023 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -1374,6 +1374,7 @@ def add_features(sender, **kwargs): DEFAULT_TO_MKDOCS_0_17_3 = 'default_to_mkdocs_0_17_3' CLEAN_AFTER_BUILD = 'clean_after_build' UPDATE_CONDA_STARTUP = 'update_conda_startup' + CONDA_APPEND_CORE_REQUIREMENTS = 'conda_append_core_requirements' FEATURES = ( (USE_SPHINX_LATEST, _('Use latest version of Sphinx')), @@ -1418,6 +1419,10 @@ def add_features(sender, **kwargs): UPDATE_CONDA_STARTUP, _('Upgrade conda before creating the environment'), ), + ( + CONDA_APPEND_CORE_REQUIREMENTS, + _('Append Read the Docs core requirements to environment.yml file'), + ), ) projects = models.ManyToManyField(