Append core requirements to Conda environment file

We run 3 steps when a project depends on conda.

1. create the whole environment based on user's YAML file
2. run `conda install` with our own dependencies
3. run `pip install` with some of our dependencies that are not
published on conda repositories.

This commit changes this to make it in just one step (at environment
creation). To do this, it appends our own requirements to the
`environment.yml` file under `dependencies` and `dependencies.pip`
config (see https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#create-env-file-manually)

It also shows the resulting `environment.yml` in the build output.

This behavior is added behind a feature flag so we can test it first.

This allow users to be able to pin their dependencies as they want
without us upgrading them in the second step. Also, this should
improve the build time, since dependencies resolution is done just
once.

Related to

* #3829
* #5631

Author: humitos

readthedocs/doc_builder/python_environments.py

@@ -1,14 +1,16 @@
 """An abstraction over virtualenv and Conda environments."""
 
 import copy
+import codecs
 import hashlib
 import itertools
 import json
 import logging
 import os
 import shutil
+import yaml
 
-from readthedocs.config import PIP, SETUPTOOLS
+from readthedocs.config import PIP, SETUPTOOLS, ParseError, parse as parse_yaml
 from readthedocs.config.models import PythonInstall, PythonInstallRequirements
 from readthedocs.doc_builder.config import load_yaml_config
 from readthedocs.doc_builder.constants import DOCKER_IMAGE
@@ -442,6 +444,10 @@ def setup_base(self):
         if self.project.has_feature(Feature.UPDATE_CONDA_STARTUP):
             self._update_conda_startup()
 
+        if self.project.has_feature(Feature.CONDA_APPEND_CORE_REQUIREMENTS):
+            self._append_core_requirements()
+            self._show_environment_yaml()
+
         self.build_env.run(
             'conda',
             'env',
@@ -455,10 +461,75 @@ def setup_base(self):
             cwd=self.checkout_path,
         )
 
-    def install_core_requirements(self):
-        """Install basic Read the Docs requirements into the Conda env."""
+    def _show_environment_yaml(self):
+        """Show ``environment.yml`` file in the Build output."""
+        self.build_env.run(
+            'cat',
+            self.config.conda.environment,
+            cwd=self.checkout_path,
+        )
+
+    def _append_core_requirements(self):
+        """
+        Append Read the Docs dependencies to Conda environment file.
+
+        This help users to pin their dependencies properly without us upgrading
+        them in the second ``conda install`` run.
+
+        See https://github.com/readthedocs/readthedocs.org/pull/5631
+        """
+        try:
+            inputfile = codecs.open(
+                os.path.join(
+                    self.checkout_path,
+                    self.config.conda.environment,
+                ),
+                encoding='utf-8',
+                mode='r',
+            )
+            environment = parse_yaml(inputfile)
+        except IOError:
+            log.warning(
+                'There was an error while reading Conda environment file.',
+            )
+        except ParseError:
+            log.warning(
+                'There was an error while parsing Conda environment file.',
+            )
+        else:
+            # Append conda dependencies directly to ``dependencies`` and pip
+            # dependencies to ``dependencies.pip``
+            pip_requirements, conda_requirements = self._get_core_requirements()
+            dependencies = environment.get('dependencies', [])
+            pip_dependencies = {'pip': pip_requirements}
+
+            for item in dependencies:
+                if isinstance(item, dict) and 'pip' in item:
+                    pip_requirements.extend(item.get('pip', []))
+                    dependencies.remove(item)
+                    break
+
+            dependencies.append(pip_dependencies)
+            environment.update({'dependencies': dependencies})
+            try:
+                outputfile = codecs.open(
+                    os.path.join(
+                        self.checkout_path,
+                        self.config.conda.environment,
+                    ),
+                    encoding='utf-8',
+                    mode='w',
+                )
+                yaml.safe_dump(environment, outputfile)
+            except IOError:
+                log.warning(
+                    'There was an error while writing the new Conda ',
+                    'environment file.',
+                )
+
+    def _get_core_requirements(self):
         # Use conda for requirements it packages
-        requirements = [
+        conda_requirements = [
             'mock',
             'pillow',
         ]
@@ -472,8 +543,19 @@ def install_core_requirements(self):
             pip_requirements.append('mkdocs')
         else:
             pip_requirements.append('readthedocs-sphinx-ext')
-            requirements.extend(['sphinx', 'sphinx_rtd_theme'])
+            conda_requirements.extend(['sphinx', 'sphinx_rtd_theme'])
+
+        return pip_requirements, conda_requirements
+
+    def install_core_requirements(self):
+        """Install basic Read the Docs requirements into the Conda env."""
 
+        if self.project.has_feature(Feature.CONDA_APPEND_CORE_REQUIREMENTS):
+            return
+
+        pip_requirements, conda_requirements = self._get_core_requirements()
+        # Install requirements via ``conda install`` command if they were
+        # not appended to the ``environment.yml`` file.
         cmd = [
             'conda',
             'install',
@@ -482,12 +564,13 @@ def install_core_requirements(self):
             '--name',
             self.version.slug,
         ]
-        cmd.extend(requirements)
+        cmd.extend(conda_requirements)
         self.build_env.run(
             *cmd,
             cwd=self.checkout_path,
         )
 
+        # Install requirements via ``pip install``
         pip_cmd = [
             self.venv_bin(filename='python'),
             '-m',

readthedocs/projects/models.py

@@ -1374,6 +1374,7 @@ def add_features(sender, **kwargs):
     DEFAULT_TO_MKDOCS_0_17_3 = 'default_to_mkdocs_0_17_3'
     CLEAN_AFTER_BUILD = 'clean_after_build'
     UPDATE_CONDA_STARTUP = 'update_conda_startup'
+    CONDA_APPEND_CORE_REQUIREMENTS = 'conda_append_core_requirements'
 
     FEATURES = (
         (USE_SPHINX_LATEST, _('Use latest version of Sphinx')),
@@ -1418,6 +1419,10 @@ def add_features(sender, **kwargs):
             UPDATE_CONDA_STARTUP,
             _('Upgrade conda before creating the environment'),
         ),
+        (
+            CONDA_APPEND_CORE_REQUIREMENTS,
+            _('Append Read the Docs core requirements to environment.yml file'),
+        ),
     )
 
     projects = models.ManyToManyField(