BUILD: Simplifying contributor dependencies (#23522)

Author: datapythonista

ci/code_checks.sh

@@ -9,16 +9,19 @@
 # In the future we may want to add the validation of docstrings and other checks here.
 #
 # Usage:
-#   $ ./ci/code_checks.sh             # run all checks
-#   $ ./ci/code_checks.sh lint        # run linting only
-#   $ ./ci/code_checks.sh patterns    # check for patterns that should not exist
-#   $ ./ci/code_checks.sh doctests    # run doctests
+#   $ ./ci/code_checks.sh               # run all checks
+#   $ ./ci/code_checks.sh lint          # run linting only
+#   $ ./ci/code_checks.sh patterns      # check for patterns that should not exist
+#   $ ./ci/code_checks.sh doctests      # run doctests
+#   $ ./ci/code_checks.sh dependencies  # check that dependencies are consistent
 
 echo "inside $0"
 [[ $LINT ]] || { echo "NOT Linting. To lint use: LINT=true $0 $1"; exit 0; }
-[[ -z "$1" || "$1" == "lint" || "$1" == "patterns" || "$1" == "doctests" ]] || { echo "Unknown command $1. Usage: $0 [lint|patterns|doctests]"; exit 9999; }
+[[ -z "$1" || "$1" == "lint" || "$1" == "patterns" || "$1" == "doctests" || "$1" == "dependencies"  ]] \
+    || { echo "Unknown command $1. Usage: $0 [lint|patterns|doctests|dependencies]"; exit 9999; }
 
 source activate pandas
+BASE_DIR="$(dirname $0)/.."
 RET=0
 CHECK=$1
 
@@ -172,4 +175,11 @@ if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then
 
 fi
 
+### DEPENDENCIES ###
+if [[ -z "$CHECK" || "$CHECK" == "dependencies" ]]; then
+    MSG='Check that requirements-dev.txt has been generated from environment.yml' ; echo $MSG
+    $BASE_DIR/scripts/generate_pip_deps_from_conda.py --compare
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+fi
+
 exit $RET

ci/environment-dev.yaml

@@ -170,7 +170,7 @@ We'll now kick off a three-step process:
 .. code-block:: none
 
    # Create and activate the build environment
-   conda env create -f ci/environment-dev.yaml
+   conda env create -f environment.yml
    conda activate pandas-dev
 
    # or with older versions of Anaconda:
@@ -180,9 +180,6 @@ We'll now kick off a three-step process:
    python setup.py build_ext --inplace -j 4
    python -m pip install -e .
 
-   # Install the rest of the optional dependencies
-   conda install -c defaults -c conda-forge --file=ci/requirements-optional-conda.txt
-
 At this point you should be able to import pandas from your locally built version::
 
    $ python  # start an interpreter
@@ -221,14 +218,12 @@ You'll need to have at least python3.5 installed on your system.
    . ~/virtualenvs/pandas-dev/bin/activate
 
    # Install the build dependencies
-   python -m pip install -r ci/requirements_dev.txt
+   python -m pip install -r requirements-dev.txt
+
    # Build and install pandas
    python setup.py build_ext --inplace -j 4
    python -m pip install -e .
 
-   # Install additional dependencies
-   python -m pip install -r ci/requirements-optional-pip.txt
-
 Creating a branch
 -----------------
 

ci/requirements-optional-conda.txt

@@ -0,0 +1,53 @@
+name: pandas-dev
+channels:
+  - defaults
+  - conda-forge
+dependencies:
+  # required
+  - NumPy
+  - python=3
+  - python-dateutil>=2.5.0
+  - pytz
+
+  # development
+  - Cython>=0.28.2
+  - flake8
+  - flake8-comprehensions
+  - flake8-rst
+  - hypothesis>=3.58.0
+  - isort
+  - moto
+  - pytest>=3.6
+  - setuptools>=24.2.0
+  - sphinx
+  - sphinxcontrib-spelling
+
+  # optional
+  - beautifulsoup4>=4.2.1
+  - blosc
+  - bottleneck>=1.2.0
+  - fastparquet>=0.1.2
+  - gcsfs
+  - html5lib
+  - ipython>=5.6.0
+  - ipykernel
+  - jinja2
+  - lxml
+  - matplotlib>=2.0.0
+  - nbsphinx
+  - numexpr>=2.6.1
+  - openpyxl
+  - pyarrow>=0.7.0
+  - pymysql
+  - pytables>=3.4.2
+  - pytest-cov
+  - pytest-xdist
+  - s3fs
+  - scipy>=0.18.1
+  - seaborn
+  - sqlalchemy
+  - statsmodels
+  - xarray
+  - xlrd
+  - xlsxwriter
+  - xlwt

ci/requirements_dev.txt

@@ -1,5 +1,17 @@
-# This file was autogenerated by scripts/convert_deps.py
-# Do not modify directly
+NumPy
+python-dateutil>=2.5.0
+pytz
+Cython>=0.28.2
+flake8
+flake8-comprehensions
+flake8-rst
+hypothesis>=3.58.0
+isort
+moto
+pytest>=3.6
+setuptools>=24.2.0
+sphinx
+sphinxcontrib-spelling
 beautifulsoup4>=4.2.1
 blosc
 bottleneck>=1.2.0

doc/source/contributing.rst

@@ -0,0 +1,96 @@
+#!/usr/bin/env python
+"""
+Convert the conda environment.yml to the pip requirements-dev.txt,
+or check that they have the same packages (for the CI)
+
+Usage:
+
+    Generate `requirements-dev.txt`
+    $ ./conda_to_pip
+
+    Compare and fail (exit status != 0) if `requirements-dev.txt` has not been
+    generated with this script:
+    $ ./conda_to_pip --compare
+"""
+import argparse
+import os
+import re
+import sys
+import yaml
+
+
+EXCLUDE = {'python=3'}
+RENAME = {'pytables': 'tables'}
+
+
+def conda_package_to_pip(package):
+    """
+    Convert a conda package to its pip equivalent.
+
+    In most cases they are the same, those are the exceptions:
+    - Packages that should be excluded (in `EXCLUDE`)
+    - Packages that should be renamed (in `RENAME`)
+    - A package requiring a specific version, in conda is defined with a single
+      equal (e.g. ``pandas=1.0``) and in pip with two (e.g. ``pandas==1.0``)
+    """
+    if package in EXCLUDE:
+        return
+
+    if package in RENAME:
+        return RENAME[package]
+
+    return re.sub('(?<=[^<>])=', '==', package).strip()
+
+
+def main(conda_fname, pip_fname, compare=False):
+    """
+    Generate the pip dependencies file from the conda file, or compare that
+    they are synchronized (``compare=True``).
+
+    Parameters
+    ----------
+    conda_fname : str
+        Path to the conda file with dependencies (e.g. `environment.yml`).
+    pip_fname : str
+        Path to the pip file with dependencies (e.g. `requirements-dev.txt`).
+    compare : bool, default False
+        Whether to generate the pip file (``False``) or to compare if the
+        pip file has been generated with this script and the last version
+        of the conda file (``True``).
+
+    Returns
+    -------
+    bool
+        True if the comparison fails, False otherwise
+    """
+    with open(conda_fname) as conda_fd:
+        deps = yaml.safe_load(conda_fd)['dependencies']
+
+    pip_content = '\n'.join(filter(None, map(conda_package_to_pip, deps)))
+
+    if compare:
+        with open(pip_fname) as pip_fd:
+            return pip_content != pip_fd.read()
+    else:
+        with open(pip_fname, 'w') as pip_fd:
+            pip_fd.write(pip_content)
+        return False
+
+
+if __name__ == '__main__':
+    argparser = argparse.ArgumentParser(
+        description='convert (or compare) conda file to pip')
+    argparser.add_argument('--compare',
+                           action='store_true',
+                           help='compare whether the two files are equivalent')
+    args = argparser.parse_args()
+
+    repo_path = os.path.dirname(os.path.abspath(os.path.dirname(__file__)))
+    res = main(os.path.join(repo_path, 'environment.yml'),
+               os.path.join(repo_path, 'requirements-dev.txt'),
+               compare=args.compare)
+    if res:
+        sys.stderr.write('`requirements-dev.txt` has to be generated with '
+                         '`{}` after `environment.yml` is modified.\n'.format(
+                             sys.argv[0]))
+    sys.exit(res)