merge upstream master

Author: baidoosik

.coveragerc

@@ -17,6 +17,7 @@ exclude_lines =
     # Don't complain if tests don't hit defensive assertion code:
     raise AssertionError
     raise NotImplementedError
+    AbstractMethodError
 
     # Don't complain if non-runnable code isn't run:
     if 0:

.gitignore

@@ -62,6 +62,8 @@ dist
 coverage.xml
 coverage_html_report
 *.pytest_cache
+# hypothesis test database
+.hypothesis/
 
 # OS generated files #
 ######################

Makefile

@@ -13,7 +13,7 @@ build: clean_pyc
 	python setup.py build_ext --inplace
 
 lint-diff:
-	git diff master --name-only -- "*.py" | grep "pandas" | xargs flake8
+	git diff master --name-only -- "*.py" | grep -E "pandas|scripts" | xargs flake8
 
 develop: build
 	-python setup.py develop

ci/appveyor-27.yaml

@@ -13,7 +13,7 @@ dependencies:
   - matplotlib
   - numexpr
   - numpy=1.12*
-  - openpyxl
+  - openpyxl=2.5.5
   - pytables
   - python=2.7.*
   - pytz
@@ -28,3 +28,4 @@ dependencies:
   - pytest
   - pytest-xdist
   - moto
+  - hypothesis>=3.58.0

ci/appveyor-36.yaml

@@ -10,7 +10,7 @@ dependencies:
   - matplotlib
   - numexpr
   - numpy=1.14*
-  - openpyxl
+  - openpyxl=2.5.5
   - pyarrow
   - pytables
   - python-dateutil
@@ -25,3 +25,4 @@ dependencies:
   - cython>=0.28.2
   - pytest
   - pytest-xdist
+  - hypothesis>=3.58.0

ci/check_imports.py

@@ -9,6 +9,7 @@
     'html5lib',
     'ipython',
     'jinja2'
+    'hypothesis',
     'lxml',
     'numexpr',
     'openpyxl',

ci/circle-27-compat.yaml

@@ -7,8 +7,8 @@ dependencies:
   - cython=0.28.2
   - jinja2=2.8
   - numexpr=2.4.4 # we test that we correctly don't use an unsupported numexpr
-  - numpy=1.9.2
-  - openpyxl
+  - numpy=1.9.3
+  - openpyxl=2.5.5
   - psycopg2
   - pytables=3.2.2
   - python-dateutil=2.5.0
@@ -26,3 +26,4 @@ dependencies:
     - html5lib==1.0b2
     - beautifulsoup4==4.2.1
     - pymysql==0.6.0
+    - hypothesis>=3.58.0

ci/circle-35-ascii.yaml

@@ -11,3 +11,5 @@ dependencies:
   # universal
   - pytest
   - pytest-xdist
+  - pip:
+    - hypothesis>=3.58.0

ci/circle-36-locale.yaml

@@ -13,7 +13,7 @@ dependencies:
   - nomkl
   - numexpr
   - numpy
-  - openpyxl
+  - openpyxl=2.5.5
   - psycopg2
   - pymysql
   - pytables
@@ -31,3 +31,5 @@ dependencies:
   - pytest
   - pytest-xdist
   - moto
+  - pip:
+    - hypothesis>=3.58.0

ci/circle-36-locale_slow.yaml

@@ -14,7 +14,7 @@ dependencies:
   - nomkl
   - numexpr
   - numpy
-  - openpyxl
+  - openpyxl=2.5.5
   - psycopg2
   - pymysql
   - pytables
@@ -32,3 +32,5 @@ dependencies:
   - pytest
   - pytest-xdist
   - moto
+  - pip:
+    - hypothesis>=3.58.0

ci/doctests.sh

@@ -21,14 +21,14 @@ if [ "$DOCTEST" ]; then
 
     # DataFrame / Series docstrings
     pytest --doctest-modules -v pandas/core/frame.py \
-        -k"-assign -axes -combine -isin -itertuples -join -nlargest -nsmallest -nunique -pivot_table -quantile -query -reindex -reindex_axis -replace -round -set_index -stack -to_dict -to_records -to_stata -transform"
+        -k"-assign -axes -combine -isin -itertuples -join -nlargest -nsmallest -nunique -pivot_table -quantile -query -reindex -reindex_axis -replace -round -set_index -stack -to_dict -to_stata -transform"
 
     if [ $? -ne "0" ]; then
         RET=1
     fi
 
     pytest --doctest-modules -v pandas/core/series.py \
-        -k"-agg -map -nlargest -nonzero -nsmallest -reindex -searchsorted -to_dict"
+        -k"-nlargest -nonzero -nsmallest -reindex -searchsorted -to_dict"
 
     if [ $? -ne "0" ]; then
         RET=1

ci/environment-dev.yaml

@@ -7,6 +7,7 @@ dependencies:
   - NumPy
   - flake8
   - flake8-comprehensions
+  - hypothesis>=3.58.0
   - moto
   - pytest>=3.6
   - python-dateutil>=2.5.0

ci/lint.sh

@@ -20,14 +20,19 @@ if [ "$LINT" ]; then
 
     # pandas/_libs/src is C code, so no need to search there.
     echo "Linting *.py"
-    flake8 pandas --filename=*.py --exclude pandas/_libs/src --ignore=C406,C408,C409,C410,E402,E731,E741,W503
+    flake8 pandas --filename=*.py --exclude pandas/_libs/src --ignore=C406,C408,C409,E402,E731,E741,W503
+    if [ $? -ne "0" ]; then
+        RET=1
+    fi
+
+    flake8 scripts/tests --filename=*.py
     if [ $? -ne "0" ]; then
         RET=1
     fi
     echo "Linting *.py DONE"
 
     echo "Linting setup.py"
-    flake8 setup.py --ignore=C406,C408,C409,C410,E402,E731,E741,W503
+    flake8 setup.py --ignore=E402,E731,E741,W503
     if [ $? -ne "0" ]; then
         RET=1
     fi
@@ -41,21 +46,21 @@ if [ "$LINT" ]; then
     echo "Linting asv_bench/benchmarks/*.py DONE"
 
     echo "Linting scripts/*.py"
-    flake8 scripts --filename=*.py --ignore=C406,C408,C409,C410,E402,E731,E741,W503
+    flake8 scripts --filename=*.py --ignore=C408,E402,E731,E741,W503
     if [ $? -ne "0" ]; then
         RET=1
     fi
     echo "Linting scripts/*.py DONE"
 
     echo "Linting doc scripts"
-    flake8 doc/make.py doc/source/conf.py --ignore=C406,C408,C409,C410,E402,E731,E741,W503
+    flake8 doc/make.py doc/source/conf.py --ignore=E402,E731,E741,W503
     if [ $? -ne "0" ]; then
         RET=1
     fi
     echo "Linting doc scripts DONE"
 
     echo "Linting *.pyx"
-    flake8 pandas --filename=*.pyx --select=E501,E302,E203,E111,E114,E221,E303,E128,E231,E126,E265,E305,E301,E127,E261,E271,E129,W291,E222,E241,E123,F403,C400,C401,C402,C403,C404,C407,C411
+    flake8 pandas --filename=*.pyx --select=E501,E302,E203,E111,E114,E221,E303,E128,E231,E126,E265,E305,E301,E127,E261,E271,E129,W291,E222,E241,E123,F403,C400,C401,C402,C403,C404,C405,C406,C407,C408,C409,C410,C411
     if [ $? -ne "0" ]; then
         RET=1
     fi
@@ -175,7 +180,7 @@ if [ "$LINT" ]; then
         RET=1
     fi
     echo "Check for old-style classes DONE"
-    
+
     echo "Check for backticks incorrectly rendering because of missing spaces"
     grep -R --include="*.rst" -E "[a-zA-Z0-9]\`\`?[a-zA-Z0-9]" doc/source/
 

ci/requirements-optional-conda.txt

@@ -12,7 +12,7 @@ lxml
 matplotlib
 nbsphinx
 numexpr
-openpyxl
+openpyxl=2.5.5
 pyarrow
 pymysql
 pytables

ci/requirements-optional-pip.txt

@@ -14,7 +14,7 @@ lxml
 matplotlib
 nbsphinx
 numexpr
-openpyxl
+openpyxl=2.5.5
 pyarrow
 pymysql
 tables
@@ -28,4 +28,4 @@ statsmodels
 xarray
 xlrd
 xlsxwriter
-xlwt
+xlwt

ci/requirements_dev.txt

@@ -4,6 +4,7 @@ Cython>=0.28.2
 NumPy
 flake8
 flake8-comprehensions
+hypothesis>=3.58.0
 moto
 pytest>=3.6
 python-dateutil>=2.5.0

ci/script_single.sh

@@ -28,6 +28,8 @@ elif [ "$COVERAGE" ]; then
     echo pytest -s -m "single" -r xXs --strict --cov=pandas --cov-report xml:/tmp/cov-single.xml --junitxml=/tmp/single.xml $TEST_ARGS pandas
     pytest      -s -m "single" -r xXs --strict --cov=pandas --cov-report xml:/tmp/cov-single.xml --junitxml=/tmp/single.xml $TEST_ARGS pandas
 
+    echo pytest -s -r xXs --strict scripts
+    pytest      -s -r xXs --strict scripts
 else
     echo pytest -m "single" -r xXs --junitxml=/tmp/single.xml --strict $TEST_ARGS pandas
     pytest      -m "single" -r xXs --junitxml=/tmp/single.xml --strict $TEST_ARGS pandas # TODO: doctest

ci/travis-27-locale.yaml

@@ -7,7 +7,7 @@ dependencies:
   - cython=0.28.2
   - lxml
   - matplotlib=1.4.3
-  - numpy=1.9.2
+  - numpy=1.9.3
   - openpyxl=2.4.0
   - python-dateutil
   - python-blosc
@@ -22,6 +22,7 @@ dependencies:
   # universal
   - pytest
   - pytest-xdist
+  - hypothesis>=3.58.0
   - pip:
     - html5lib==1.0b2
     - beautifulsoup4==4.2.1

ci/travis-27.yaml

@@ -29,6 +29,7 @@ dependencies:
   - PyCrypto
   - pymysql=0.6.3
   - pytables
+  - blosc=1.14.3
   - python-blosc
   - python-dateutil=2.5.0
   - python=2.7*
@@ -44,6 +45,7 @@ dependencies:
   - pytest
   - pytest-xdist
   - moto
+  - hypothesis>=3.58.0
   - pip:
     - backports.lzma
     - cpplint

ci/travis-35-osx.yaml

@@ -12,7 +12,7 @@ dependencies:
   - nomkl
   - numexpr
   - numpy=1.10.4
-  - openpyxl
+  - openpyxl=2.5.5
   - pytables
   - python=3.5*
   - pytz
@@ -25,3 +25,4 @@ dependencies:
   - pytest-xdist
   - pip:
     - python-dateutil==2.5.3
+    - hypothesis>=3.58.0

ci/travis-36-doc.yaml

@@ -10,6 +10,7 @@ dependencies:
   - fastparquet
   - feather-format
   - html5lib
+  - hypothesis>=3.58.0
   - ipykernel
   - ipython
   - ipywidgets
@@ -21,7 +22,7 @@ dependencies:
   - notebook
   - numexpr
   - numpy=1.13*
-  - openpyxl
+  - openpyxl=2.5.5
   - pandoc
   - pyqt
   - pytables

ci/travis-36-numpydev.yaml

@@ -8,6 +8,7 @@ dependencies:
   # universal
   - pytest
   - pytest-xdist
+  - hypothesis>=3.58.0
   - pip:
     - "git+git://github.com/dateutil/dateutil.git"
     - "-f https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com"

ci/travis-36-slow.yaml

@@ -10,7 +10,7 @@ dependencies:
   - matplotlib
   - numexpr
   - numpy
-  - openpyxl
+  - openpyxl=2.5.5
   - patsy
   - psycopg2
   - pymysql
@@ -28,3 +28,4 @@ dependencies:
   - pytest
   - pytest-xdist
   - moto
+  - hypothesis>=3.58.0

ci/travis-36.yaml

@@ -18,7 +18,7 @@ dependencies:
   - nomkl
   - numexpr
   - numpy
-  - openpyxl
+  - openpyxl=2.5.5
   - psycopg2
   - pyarrow
   - pymysql
@@ -41,6 +41,7 @@ dependencies:
   - pytest-xdist
   - pytest-cov
   - moto
+  - hypothesis>=3.58.0
   - pip:
     - brotlipy
     - coverage

ci/travis-37.yaml

@@ -12,3 +12,4 @@ dependencies:
   - pytz
   - pytest
   - pytest-xdist
+  - hypothesis>=3.58.0

doc/source/api.rst

@@ -2352,6 +2352,7 @@ Computations / Descriptive Stats
    Resampler.std
    Resampler.sum
    Resampler.var
+   Resampler.quantile
 
 Style
 -----

doc/source/contributing.rst

@@ -820,6 +820,46 @@ Tests that we have ``parametrized`` are now accessible via the test name, for ex
    test_cool_feature.py::test_series[int8] PASSED
 
 
+.. _using-hypothesis:
+
+Using ``hypothesis``
+~~~~~~~~~~~~~~~~~~~~
+
+Hypothesis is a library for property-based testing.  Instead of explicitly
+parametrizing a test, you can describe *all* valid inputs and let Hypothesis
+try to find a failing input.  Even better, no matter how many random examples
+it tries, Hypothesis always reports a single minimal counterexample to your
+assertions - often an example that you would never have thought to test.
+
+See `Getting Started with Hypothesis <https://hypothesis.works/articles/getting-started-with-hypothesis/>`_
+for more of an introduction, then `refer to the Hypothesis documentation
+for details <https://hypothesis.readthedocs.io/en/latest/index.html>`_.
+
+.. code-block:: python
+
+    import json
+    from hypothesis import given, strategies as st
+
+    any_json_value = st.deferred(lambda: st.one_of(
+        st.none(), st.booleans(), st.floats(allow_nan=False), st.text(),
+        st.lists(any_json_value), st.dictionaries(st.text(), any_json_value)
+    ))
+
+    @given(value=any_json_value)
+    def test_json_roundtrip(value):
+        result = json.loads(json.dumps(value))
+        assert value == result
+
+This test shows off several useful features of Hypothesis, as well as
+demonstrating a good use-case: checking properties that should hold over
+a large or complicated domain of inputs.
+
+To keep the Pandas test suite running quickly, parametrized tests are
+preferred if the inputs or logic are simple, with Hypothesis tests reserved
+for cases with complex logic or where there are too many combinations of
+options or subtle interactions to test (or think of!) all of them.
+
+
 Running the test suite
 ----------------------