python_environments.py

"""An abstraction over virtualenv and Conda environments."""

import copy
import os

import structlog
import yaml

from readthedocs.config import PIP, SETUPTOOLS, ParseError
from readthedocs.config import parse as parse_yaml
from readthedocs.config.models import PythonInstall, PythonInstallRequirements
from readthedocs.core.utils.filesystem import safe_open
from readthedocs.doc_builder.config import load_yaml_config
from readthedocs.projects.exceptions import UserFileNotFound
from readthedocs.projects.models import Feature

log = structlog.get_logger(__name__)


class PythonEnvironment:

    """An isolated environment into which Python packages can be installed."""

    def __init__(self, version, build_env, config=None):
        self.version = version
        self.project = version.project
        self.build_env = build_env
        if config:
            self.config = config
        else:
            self.config = load_yaml_config(version)
        # Compute here, since it's used a lot
        self.checkout_path = self.project.checkout_path(self.version.slug)
        log.bind(
            project_slug=self.project.slug,
            version_slug=self.version.slug,
        )

    def install_requirements(self):
        """Install all requirements from the config object."""
        for install in self.config.python.install:
            if isinstance(install, PythonInstallRequirements):
                self.install_requirements_file(install)
            if isinstance(install, PythonInstall):
                self.install_package(install)

    def install_package(self, install):
        """
        Install the package using pip or setuptools.

        :param install: A install object from the config module.
        :type install: readthedocs.config.models.PythonInstall
        """
        # NOTE: `venv_bin` requires `prefixes`.
        # However, it's overwritten in the subclasses and
        # it forces passing the `prefixes=` attribute.
        # I'm not sure how to solve this, so I'm skipping this check for now.
        # pylint: disable=no-value-for-parameter

        if install.method == PIP:
            # Prefix ./ so pip installs from a local path rather than pypi
            local_path = (
                os.path.join(".", install.path) if install.path != "." else install.path
            )
            extra_req_param = ""
            if install.extra_requirements:
                extra_req_param = "[{}]".format(",".join(install.extra_requirements))
            self.build_env.run(
                self.venv_bin(filename="python"),
                "-m",
                "pip",
                "install",
                "--upgrade",
                "--upgrade-strategy",
                "only-if-needed",
                "--no-cache-dir",
                "{path}{extra_requirements}".format(
                    path=local_path,
                    extra_requirements=extra_req_param,
                ),
                cwd=self.checkout_path,
                bin_path=self.venv_bin(),
            )
        elif install.method == SETUPTOOLS:
            self.build_env.run(
                self.venv_bin(filename="python"),
                os.path.join(install.path, "setup.py"),
                "install",
                "--force",
                cwd=self.checkout_path,
                bin_path=self.venv_bin(),
            )

    def venv_bin(self, prefixes, filename=None):
        """
        Return path to the virtualenv bin path, or a specific binary.

        :param filename: If specified, add this filename to the path return
        :param prefixes: List of path prefixes to include in the resulting path
        :returns: Path to virtualenv bin or filename in virtualenv bin
        """
        if filename is not None:
            prefixes.append(filename)
        return os.path.join(*prefixes)


class Virtualenv(PythonEnvironment):

    """
    A virtualenv_ environment.

    .. _virtualenv: https://virtualenv.pypa.io/
    """

    # pylint: disable=arguments-differ
    def venv_bin(self, filename=None):
        prefixes = ["$READTHEDOCS_VIRTUALENV_PATH", "bin"]
        return super().venv_bin(prefixes, filename=filename)

    def setup_base(self):
        """
        Create a virtualenv, invoking ``python -mvirtualenv``.

        .. note::

            ``--no-download`` was removed because of the pip breakage,
            it was sometimes installing pip 20.0 which broke everything
            https://github.com/readthedocs/readthedocs.org/issues/6585

            Important not to add empty string arguments, see:
            https://github.com/readthedocs/readthedocs.org/issues/7322
        """
        cli_args = [
            "-mvirtualenv",
            # Append the positional destination argument
            "$READTHEDOCS_VIRTUALENV_PATH",
        ]

        self.build_env.run(
            self.config.python_interpreter,
            *cli_args,
            # Don't use virtualenv bin that doesn't exist yet
            bin_path=None,
            # Don't use the project's root, some config files can interfere
            cwd=None,
        )

    def install_core_requirements(self):
        """Install basic Read the Docs requirements into the virtualenv."""
        pip_install_cmd = [
            self.venv_bin(filename="python"),
            "-m",
            "pip",
            "install",
            "--upgrade",
            "--no-cache-dir",
        ]

        self._install_latest_requirements(pip_install_cmd)

    def _install_latest_requirements(self, pip_install_cmd):
        """Install all the latest core requirements."""
        # First, upgrade pip and setuptools to their latest versions
        cmd = pip_install_cmd + ["pip", "setuptools"]
        self.build_env.run(
            *cmd,
            bin_path=self.venv_bin(),
            cwd=self.checkout_path,
        )

        # Second, install all the latest core requirements
        requirements = []

        if self.config.doctype == "mkdocs":
            requirements.append("mkdocs")
        else:
            requirements.append("sphinx")

            # Install ``readthedocs-sphinx-ext`` only on old projects
            if not self.project.has_feature(Feature.DISABLE_SPHINX_MANIPULATION):
                requirements.append("readthedocs-sphinx-ext")

        cmd = copy.copy(pip_install_cmd)
        cmd.extend(requirements)
        self.build_env.run(
            *cmd,
            bin_path=self.venv_bin(),
            cwd=self.checkout_path,
        )

    def install_requirements_file(self, install):
        """
        Install a requirements file using pip.

        :param install: A install object from the config module.
        :type install: readthedocs.config.models.PythonInstallRequirements
        """
        requirements_file_path = install.requirements
        if requirements_file_path:
            args = [
                self.venv_bin(filename="python"),
                "-m",
                "pip",
                "install",
            ]
            if self.project.has_feature(Feature.PIP_ALWAYS_UPGRADE):
                args += ["--upgrade"]
            args += [
                "--exists-action=w",
                "--no-cache-dir",
                "-r",
                requirements_file_path,
            ]
            self.build_env.run(
                *args,
                cwd=self.checkout_path,
                bin_path=self.venv_bin(),
            )


class Conda(PythonEnvironment):

    """
    A Conda_ environment.

    .. _Conda: https://conda.io/docs/
    """

    # pylint: disable=arguments-differ
    def venv_bin(self, filename=None):
        prefixes = ["$CONDA_ENVS_PATH", "$CONDA_DEFAULT_ENV", "bin"]
        return super().venv_bin(prefixes, filename=filename)

    def conda_bin_name(self):
        """
        Decide whether use ``mamba`` or ``conda`` to create the environment.

        ``mamba`` is really fast to solve dependencies and download channel
        metadata on startup.

        See https://github.com/QuantStack/mamba
        """
        return self.config.python_interpreter

    def setup_base(self):
        if self.project.has_feature(Feature.CONDA_APPEND_CORE_REQUIREMENTS):
            self._append_core_requirements()
            self._show_environment_yaml()

        self.build_env.run(
            self.conda_bin_name(),
            "env",
            "create",
            "--quiet",
            "--name",
            self.version.slug,
            "--file",
            self.config.conda.environment,
            bin_path=None,  # Don't use conda bin that doesn't exist yet
            cwd=self.checkout_path,
        )

    def _show_environment_yaml(self):
        """Show ``environment.yml`` file in the Build output."""
        self.build_env.run(
            "cat",
            self.config.conda.environment,
            cwd=self.checkout_path,
        )

    def _append_core_requirements(self):
        """
        Append Read the Docs dependencies to Conda environment file.

        This help users to pin their dependencies properly without us upgrading
        them in the second ``conda install`` run.

        See https://github.com/readthedocs/readthedocs.org/pull/5631
        """
        try:
            # Allow symlinks, but only the ones that resolve inside the base directory.
            inputfile = safe_open(
                os.path.join(
                    self.checkout_path,
                    self.config.conda.environment,
                ),
                "r",
                allow_symlinks=True,
                base_path=self.checkout_path,
            )
            if not inputfile:
                raise UserFileNotFound(
                    message_id=UserFileNotFound.FILE_NOT_FOUND,
                    format_values={
                        "filename": self.config.conda.environment,
                    },
                )
            environment = parse_yaml(inputfile)
        except IOError:
            log.warning(
                "There was an error while reading Conda environment file.",
            )
        except ParseError:
            log.warning(
                "There was an error while parsing Conda environment file.",
            )
        else:
            # Append conda dependencies directly to ``dependencies`` and pip
            # dependencies to ``dependencies.pip``
            pip_requirements, conda_requirements = self._get_core_requirements()
            dependencies = environment.get("dependencies", [])
            pip_dependencies = {"pip": pip_requirements}

            for item in dependencies:
                if isinstance(item, dict) and "pip" in item:
                    # NOTE: pip can be ``None``
                    pip_requirements.extend(item.get("pip") or [])
                    dependencies.remove(item)
                    break

            dependencies.append(pip_dependencies)
            dependencies.extend(conda_requirements)
            environment.update({"dependencies": dependencies})
            try:
                # Allow symlinks, but only the ones that resolve inside the base directory.
                outputfile = safe_open(
                    os.path.join(
                        self.checkout_path,
                        self.config.conda.environment,
                    ),
                    "w",
                    allow_symlinks=True,
                    base_path=self.checkout_path,
                )
                if not outputfile:
                    raise UserFileNotFound(
                        message_id=UserFileNotFound.FILE_NOT_FOUND,
                        format_values={
                            "filename": self.config.conda.environment,
                        },
                    )
                yaml.safe_dump(environment, outputfile)
            except IOError:
                log.warning(
                    "There was an error while writing the new Conda "
                    "environment file.",
                )

    def _get_core_requirements(self):
        # Use conda for requirements it packages
        conda_requirements = []

        # Install pip-only things.
        pip_requirements = []

        if self.config.doctype == "mkdocs":
            pip_requirements.append("mkdocs")
        else:
            pip_requirements.append("readthedocs-sphinx-ext")
            conda_requirements.extend(["sphinx"])

        return pip_requirements, conda_requirements

    def install_core_requirements(self):
        """Install basic Read the Docs requirements into the Conda env."""

        if self.project.has_feature(Feature.CONDA_APPEND_CORE_REQUIREMENTS):
            # Skip install core requirements since they were already appended to
            # the user's ``environment.yml`` and installed at ``conda env
            # create`` step.
            return

        pip_requirements, conda_requirements = self._get_core_requirements()
        # Install requirements via ``conda install`` command if they were
        # not appended to the ``environment.yml`` file.
        cmd = [
            self.conda_bin_name(),
            "install",
            "--yes",
            "--quiet",
            "--name",
            self.version.slug,
        ]
        cmd.extend(conda_requirements)
        self.build_env.run(
            *cmd,
            cwd=self.checkout_path,
            # TODO: on tests I found that we are not passing ``bin_path`` here
            # for some reason.
        )

        # Install requirements via ``pip install``
        pip_cmd = [
            self.venv_bin(filename="python"),
            "-m",
            "pip",
            "install",
            "-U",
            "--no-cache-dir",
        ]
        pip_cmd.extend(pip_requirements)
        self.build_env.run(
            *pip_cmd,
            bin_path=self.venv_bin(),
            cwd=self.checkout_path,  # noqa - no comma here in py27 :/
        )

    def install_requirements_file(self, install):
        # as the conda environment was created by using the ``environment.yml``
        # defined by the user, there is nothing to update at this point
        pass