Merge tag 'v0.20.3' into releases

Version 0.20.3

* tag 'v0.20.3': (34 commits)
  RLS: v0.20.3
  TST/PKG: Move test HDF5 file to legacy (pandas-dev#16856)
  DOC: Final release notes
  CI: Pin to sphinx 1.5 for doc build
  DOC: Whatsnew updates (pandas-dev#16853)
  BUG: kind parameter on categorical argsort (pandas-dev#16834)
  BUG: allow empty multiindex (fixes .isin regression, GH16777) (pandas-dev#16782)
  BUG: fix missing sort keyword for PeriodIndex.join (pandas-dev#16586)
  DOC: Pin numpy at 1.11 for doc build
  MAINT: Remove fspath from 0.20.x branch
  BUG: TimedeltaIndex raising ValueError when slice indexing (pandas-dev#16637) (pandas-dev#16638)
  BUG: render dataframe as html do not produce duplicate element id's (pandas-dev#16780) (pandas-dev#16801)
  use network decorator on additional tests (pandas-dev#16824)
  BUG: rolling.cov with multi-index columns should presever the MI (pandas-dev#16825)
  BUG: Fix Series doesn't work in pd.astype(). Now treat Series as dict. (pandas-dev#16725)
  BUG: Fix read of py3 PeriodIndex DataFrame HDF made in py2 (pandas-dev#16781) (pandas-dev#16790)
  CI: use dist/trusty rather than os/linux (pandas-dev#16806)
  BUG: fix to_latex bold_rows option (pandas-dev#16708)
  Bug in pd.merge() when merge/join with multiple categorical columns (pandas-dev#16786)
  BUG: Load data from a CategoricalIndex for dtype comparison, closes #… (pandas-dev#16738)
  ...

Author: yarikoptic

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,4 +1,4 @@
  - [ ] closes #xxxx
  - [ ] tests added / passed
- - [ ] passes ``git diff upstream/master --name-only -- '*.py' | flake8 --diff``
+ - [ ] passes ``git diff upstream/master --name-only -- '*.py' | flake8 --diff`` (On Windows, ``git diff upstream/master -u -- "*.py" | flake8 --diff`` might work as an alternative.)
  - [ ] whatsnew entry

.travis.yml

@@ -34,57 +34,57 @@ matrix:
       language: generic
       env:
         - JOB="3.5_OSX" TEST_ARGS="--skip-slow --skip-network"
-    - os: linux
+    - dist: trusty
       env:
         - JOB="2.7_LOCALE" TEST_ARGS="--only-slow --skip-network" LOCALE_OVERRIDE="zh_CN.UTF-8"
       addons:
         apt:
           packages:
           - language-pack-zh-hans
-    - os: linux
+    - dist: trusty
       env:
         - JOB="2.7" TEST_ARGS="--skip-slow" LINT=true
       addons:
         apt:
           packages:
           - python-gtk2
-    - os: linux
+    - dist: trusty
       env:
         - JOB="3.5" TEST_ARGS="--skip-slow --skip-network" COVERAGE=true
       addons:
         apt:
           packages:
           - xsel
-    - os: linux
+    - dist: trusty
       env:
         - JOB="3.6" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate" CONDA_FORGE=true
     # In allow_failures
-    - os: linux
+    - dist: trusty
       env:
         - JOB="2.7_SLOW" TEST_ARGS="--only-slow --skip-network"
     # In allow_failures
-    - os: linux
+    - dist: trusty
       env:
         - JOB="2.7_BUILD_TEST" TEST_ARGS="--skip-slow" BUILD_TEST=true
     # In allow_failures
-    - os: linux
+    - dist: trusty
       env:
         - JOB="3.6_NUMPY_DEV" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate"
     # In allow_failures
-    - os: linux
+    - dist: trusty
       env:
         - JOB="3.5_DOC" DOC=true
     allow_failures:
-      - os: linux
+      - dist: trusty
         env:
           - JOB="2.7_SLOW" TEST_ARGS="--only-slow --skip-network"
-      - os: linux
+      - dist: trusty
         env:
           - JOB="2.7_BUILD_TEST" TEST_ARGS="--skip-slow" BUILD_TEST=true
-      - os: linux
+      - dist: trusty
         env:
           - JOB="3.6_NUMPY_DEV" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate"
-      - os: linux
+      - dist: trusty
         env:
           - JOB="3.5_DOC" DOC=true
 

ci/requirements-3.5_DOC.build

@@ -1,5 +1,5 @@
 python=3.5*
 python-dateutil
 pytz
-numpy
+numpy==1.11
 cython

ci/requirements-3.5_DOC.run

@@ -1,7 +1,7 @@
 ipython
 ipykernel
 ipywidgets
-sphinx
+sphinx=1.5*
 nbconvert
 nbformat
 notebook

doc/source/contributing.rst

@@ -525,6 +525,12 @@ run this slightly modified command::
 
    git diff master --name-only -- '*.py' | grep 'pandas/' | xargs flake8
 
+Note that on Windows, ``grep``, ``xargs``, and other tools are likely
+unavailable. However, this has been shown to work on smaller commits in the
+standard Windows command line::
+
+   git diff master -u -- "*.py" | flake8 --diff
+
 Backwards Compatibility
 ~~~~~~~~~~~~~~~~~~~~~~~
 

doc/source/io.rst

@@ -712,6 +712,16 @@ index column inference and discard the last column, pass ``index_col=False``:
     pd.read_csv(StringIO(data))
     pd.read_csv(StringIO(data), index_col=False)
 
+If a subset of data is being parsed using the ``usecols`` option, the
+``index_col`` specification is based on that subset, not the original data.
+
+.. ipython:: python
+
+    data = 'a,b,c\n4,apple,bat,\n8,orange,cow,'
+    print(data)
+    pd.read_csv(StringIO(data), usecols=['b', 'c'])
+    pd.read_csv(StringIO(data), usecols=['b', 'c'], index_col=0)
+
 .. _io.parse_dates:
 
 Date Handling
@@ -4415,34 +4425,6 @@ Performance
   `Here <http://stackoverflow.com/questions/14355151/how-to-make-pandas-hdfstore-put-operation-faster/14370190#14370190>`__
   for more information and some solutions.
 
-Experimental
-''''''''''''
-
-HDFStore supports ``Panel4D`` storage.
-
-.. ipython:: python
-   :okwarning:
-
-   wp = pd.Panel(randn(2, 5, 4), items=['Item1', 'Item2'],
-                 major_axis=pd.date_range('1/1/2000', periods=5),
-                 minor_axis=['A', 'B', 'C', 'D'])
-   p4d = pd.Panel4D({ 'l1' : wp })
-   p4d
-   store.append('p4d', p4d)
-   store
-
-These, by default, index the three axes ``items, major_axis,
-minor_axis``. On an ``AppendableTable`` it is possible to setup with the
-first append a different indexing scheme, depending on how you want to
-store your data. Pass the ``axes`` keyword with a list of dimensions
-(currently must by exactly 1 less than the total dimensions of the
-object). This cannot be changed after table creation.
-
-.. ipython:: python
-   :okwarning:
-
-   store.append('p4d2', p4d, axes=['labels', 'major_axis', 'minor_axis'])
-   store.select('p4d2', where='labels=l1 and items=Item1 and minor_axis=A')
 
 .. ipython:: python
    :suppress:

doc/source/release.rst

@@ -37,6 +37,44 @@ analysis / manipulation tool available in any language.
 * Binary installers on PyPI: http://pypi.python.org/pypi/pandas
 * Documentation: http://pandas.pydata.org
 
+pandas 0.20.3
+-------------
+
+**Release date:** July 7, 2017
+
+This is a minor bug-fix release in the 0.20.x series and includes some small regression fixes
+and bug fixes.  We recommend that all users upgrade to this version.
+
+See the :ref:`v0.20.3 Whatsnew <whatsnew_0203>` overview for an extensive list
+of all enhancements and bugs that have been fixed in 0.20.3
+
+Thanks
+~~~~~~
+
+A total of 20 people contributed to this release.  People with a "+" by their
+names contributed a patch for the first time.
+
+* Bran Yang
+* Chris
+* Chris Kerr +
+* DSM
+* David Gwynne
+* Douglas Rudd
+* Forbidden Donut +
+* Jeff Reback
+* Joris Van den Bossche
+* Karel De Brabandere +
+* Peter Quackenbush +
+* Pradyumna Reddy Chinthala +
+* Telt +
+* Tom Augspurger
+* chris-b1
+* gfyoung
+* ian +
+* jdeschenes +
+* kjford +
+* ri938 +
+
 pandas 0.20.2
 -------------
 

doc/source/whatsnew.rst

@@ -18,6 +18,8 @@ What's New
 
 These are new features and improvements of note in each release.
 
+.. include:: whatsnew/v0.20.3.txt
+
 .. include:: whatsnew/v0.20.2.txt
 
 .. include:: whatsnew/v0.20.0.txt

doc/source/whatsnew/v0.20.3.txt

@@ -0,0 +1,60 @@
+.. _whatsnew_0203:
+
+v0.20.3 (July 7, 2017)
+-----------------------
+
+This is a minor bug-fix release in the 0.20.x series and includes some small regression fixes
+and bug fixes. We recommend that all users upgrade to this version.
+
+.. contents:: What's new in v0.20.3
+    :local:
+    :backlinks: none
+
+.. _whatsnew_0203.bug_fixes:
+
+Bug Fixes
+~~~~~~~~~
+
+- Fixed a bug in failing to compute rolling computations of a column-MultiIndexed ``DataFrame`` (:issue:`16789`, :issue:`16825`)
+- Fixed a pytest marker failing downstream packages' tests suites (:issue:`16680`)
+
+Conversion
+^^^^^^^^^^
+
+- Bug in pickle compat prior to the v0.20.x series, when ``UTC`` is a timezone in a Series/DataFrame/Index (:issue:`16608`)
+- Bug in ``Series`` construction when passing a ``Series`` with ``dtype='category'`` (:issue:`16524`).
+- Bug in :meth:`DataFrame.astype` when passing a ``Series`` as the ``dtype`` kwarg. (:issue:`16717`).
+
+Indexing
+^^^^^^^^
+
+- Bug in ``Float64Index`` causing an empty array instead of ``None`` to be returned from ``.get(np.nan)`` on a Series whose index did not contain any ``NaN`` s (:issue:`8569`)
+- Bug in ``MultiIndex.isin`` causing an error when passing an empty iterable (:issue:`16777`)
+- Fixed a bug in a slicing DataFrame/Series that have a  ``TimedeltaIndex`` (:issue:`16637`)
+
+I/O
+^^^
+
+- Bug in :func:`read_csv` in which files weren't opened as binary files by the C engine on Windows, causing EOF characters mid-field, which would fail (:issue:`16039`, :issue:`16559`, :issue:`16675`)
+- Bug in :func:`read_hdf` in which reading a ``Series`` saved to an HDF file in 'fixed' format fails when an explicit ``mode='r'`` argument is supplied (:issue:`16583`)
+- Bug in :meth:`DataFrame.to_latex` where ``bold_rows`` was wrongly specified to be ``True`` by default, whereas in reality row labels remained non-bold whatever parameter provided. (:issue:`16707`)
+- Fixed an issue with :meth:`DataFrame.style` where generated element ids were not unique (:issue:`16780`)
+- Fixed loading a ``DataFrame`` with a ``PeriodIndex``, from a ``format='fixed'`` HDFStore, in Python 3, that was written in Python 2 (:issue:`16781`)
+
+Plotting
+^^^^^^^^
+
+- Fixed regression that prevented RGB and RGBA tuples from being used as color arguments (:issue:`16233`)
+- Fixed an issue with :meth:`DataFrame.plot.scatter` that incorrectly raised a ``KeyError`` when categorical data is used for plotting (:issue:`16199`)
+
+Reshaping
+^^^^^^^^^
+
+- ``PeriodIndex`` / ``TimedeltaIndex.join`` was missing the ``sort=`` kwarg (:issue:`16541`) 
+- Bug in joining on a ``MultiIndex`` with a ``category`` dtype for a level (:issue:`16627`).
+- Bug in :func:`merge` when merging/joining with multiple categorical columns (:issue:`16767`)
+
+Categorical
+^^^^^^^^^^^
+
+- Bug in ``DataFrame.sort_values`` not respecting the ``kind`` parameter with categorical data (:issue:`16793`)

pandas/_libs/src/parser/io.c

@@ -13,6 +13,10 @@ The full license is in the LICENSE file, distributed with this software.
 #include <sys/stat.h>
 #include <fcntl.h>
 
+#ifndef O_BINARY
+#define O_BINARY 0
+#endif /* O_BINARY */
+
 /*
   On-disk FILE, uncompressed
 */
@@ -23,26 +27,25 @@ void *new_file_source(char *fname, size_t buffer_size) {
         return NULL;
     }
 
-    fs->fd = open(fname, O_RDONLY);
+    fs->fd = open(fname, O_RDONLY | O_BINARY);
     if (fs->fd == -1) {
-        goto err_free;
+        free(fs);
+        return NULL;
     }
 
     // Only allocate this heap memory if we are not memory-mapping the file
     fs->buffer = (char *)malloc((buffer_size + 1) * sizeof(char));
 
     if (fs->buffer == NULL) {
-        goto err_free;
+        close(fs->fd);
+        free(fs);
+        return NULL;
     }
 
     memset(fs->buffer, '\0', buffer_size + 1);
     fs->size = buffer_size;
 
     return (void *)fs;
-
-err_free:
-    free(fs);
-    return NULL;
 }
 
 void *new_rd_source(PyObject *obj) {
@@ -184,37 +187,36 @@ void *new_mmap(char *fname) {
         fprintf(stderr, "new_file_buffer: malloc() failed.\n");
         return (NULL);
     }
-    mm->fd = open(fname, O_RDONLY);
+    mm->fd = open(fname, O_RDONLY | O_BINARY);
     if (mm->fd == -1) {
         fprintf(stderr, "new_file_buffer: open(%s) failed. errno =%d\n",
           fname, errno);
-        goto err_free;
+        free(mm);
+        return NULL;
     }
 
     if (fstat(mm->fd, &stat) == -1) {
         fprintf(stderr, "new_file_buffer: fstat() failed. errno =%d\n",
           errno);
-        goto err_close;
+        close(mm->fd);
+        free(mm);
+        return NULL;
     }
     filesize = stat.st_size; /* XXX This might be 32 bits. */
 
     mm->memmap = mmap(NULL, filesize, PROT_READ, MAP_SHARED, mm->fd, 0);
     if (mm->memmap == MAP_FAILED) {
         /* XXX Eventually remove this print statement. */
         fprintf(stderr, "new_file_buffer: mmap() failed.\n");
-        goto err_close;
+        close(mm->fd);
+        free(mm);
+        return NULL;
     }
 
     mm->size = (off_t)filesize;
     mm->position = 0;
 
     return mm;
-
-err_close:
-    close(mm->fd);
-err_free:
-    free(mm);
-    return NULL;
 }
 
 int del_mmap(void *ptr) {

pandas/compat/numpy/__init__.py

@@ -15,6 +15,7 @@
 _np_version_under1p11 = _nlv < '1.11'
 _np_version_under1p12 = _nlv < '1.12'
 _np_version_under1p13 = _nlv < '1.13'
+_np_version_under1p14 = _nlv < '1.14'
 
 if _nlv < '1.7.0':
     raise ImportError('this version of pandas is incompatible with '
@@ -74,4 +75,6 @@ def np_array_datetime64_compat(arr, *args, **kwargs):
            '_np_version_under1p10',
            '_np_version_under1p11',
            '_np_version_under1p12',
+           '_np_version_under1p13',
+           '_np_version_under1p14'
            ]