Merge remote-tracking branch 'upstream/master' into fix-nunique-groupby

Author: TomAugspurger

ci/deps/azure-macos-35.yaml

@@ -22,6 +22,7 @@ dependencies:
   - xlrd
   - xlsxwriter
   - xlwt
+  - pip
   - pip:
     - pyreadstat
     # universal

ci/run_tests.sh

@@ -50,9 +50,10 @@ do
     # if no tests are found (the case of "single and slow"), pytest exits with code 5, and would make the script fail, if not for the below code
     sh -c "$PYTEST_CMD; ret=\$?; [ \$ret = 5 ] && exit 0 || exit \$ret"
 
-    if [[ "$COVERAGE" && $? == 0 ]]; then
-        echo "uploading coverage for $TYPE tests"
-        echo "bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME"
-              bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME
-    fi
+    # 2019-08-21 disabling because this is hitting HTTP 400 errors GH#27602
+    # if [[ "$COVERAGE" && $? == 0 && "$TRAVIS_BRANCH" == "master" ]]; then
+    #    echo "uploading coverage for $TYPE tests"
+    #    echo "bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME"
+    #          bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME
+    # fi
 done

doc/source/user_guide/enhancingperf.rst

@@ -243,9 +243,9 @@ We've gotten another big improvement. Let's check again where the time is spent:
 
 .. ipython:: python
 
-   %prun -l 4 apply_integrate_f(df['a'].to_numpy(),
-                                df['b'].to_numpy(),
-                                df['N'].to_numpy())
+   %%prun -l 4 apply_integrate_f(df['a'].to_numpy(),
+                                 df['b'].to_numpy(),
+                                 df['N'].to_numpy())
 
 As one might expect, the majority of the time is now spent in ``apply_integrate_f``,
 so if we wanted to make anymore efficiencies we must continue to concentrate our

doc/source/user_guide/io.rst

@@ -28,6 +28,7 @@ The pandas I/O API is a set of top level ``reader`` functions accessed like
     :delim: ;
 
     text;`CSV <https://en.wikipedia.org/wiki/Comma-separated_values>`__;:ref:`read_csv<io.read_csv_table>`;:ref:`to_csv<io.store_in_csv>`
+    text;Fixed-Width Text File;:ref:`read_fwf<io.fwf_reader>`
     text;`JSON <https://www.json.org/>`__;:ref:`read_json<io.json_reader>`;:ref:`to_json<io.json_writer>`
     text;`HTML <https://en.wikipedia.org/wiki/HTML>`__;:ref:`read_html<io.read_html>`;:ref:`to_html<io.html>`
     text; Local clipboard;:ref:`read_clipboard<io.clipboard>`;:ref:`to_clipboard<io.clipboard>`
@@ -1372,6 +1373,7 @@ should pass the ``escapechar`` option:
    print(data)
    pd.read_csv(StringIO(data), escapechar='\\')
 
+.. _io.fwf_reader:
 .. _io.fwf:
 
 Files with fixed width columns

doc/source/whatsnew/v0.25.1.rst

@@ -1,127 +1,96 @@
 .. _whatsnew_0251:
 
-What's new in 0.25.1 (July XX, 2019)
-------------------------------------
-
-Enhancements
-~~~~~~~~~~~~
+What's new in 0.25.1 (August 21, 2019)
+--------------------------------------
 
+These are the changes in pandas 0.25.1. See :ref:`release` for a full changelog
+including other versions of pandas.
 
-.. _whatsnew_0251.enhancements.other:
-
-Other enhancements
-^^^^^^^^^^^^^^^^^^
+I/O and LZMA
+~~~~~~~~~~~~
 
--
--
--
+Some users may unknowingly have an incomplete Python installation lacking the `lzma` module from the standard library. In this case, `import pandas` failed due to an `ImportError` (:issue: `27575`).
+Pandas will now warn, rather than raising an `ImportError` if the `lzma` module is not present. Any subsequent attempt to use `lzma` methods will raise a `RuntimeError`.
+A possible fix for the lack of the `lzma` module is to ensure you have the necessary libraries and then re-install Python.
+For example, on MacOS installing Python with `pyenv` may lead to an incomplete Python installation due to unmet system dependencies at compilation time (like `xz`). Compilation will succeed, but Python might fail at run time. The issue can be solved by installing the necessary dependencies and then re-installing Python.
 
 .. _whatsnew_0251.bug_fixes:
 
 Bug fixes
 ~~~~~~~~~
 
-
 Categorical
 ^^^^^^^^^^^
 
-- Bug in :meth:`Categorical.fillna` would replace all values, not just those that are ``NaN`` (:issue:`26215`)
--
+- Bug in :meth:`Categorical.fillna` that would replace all values, not just those that are ``NaN`` (:issue:`26215`)
 
 Datetimelike
 ^^^^^^^^^^^^
-- Bug in :func:`to_datetime` where passing a timezone-naive :class:`DatetimeArray` or :class:`DatetimeIndex` and ``utc=True`` would incorrectly return a timezone-naive result (:issue:`27733`)
--
--
--
 
-Timedelta
-^^^^^^^^^
-
--
--
--
+- Bug in :func:`to_datetime` where passing a timezone-naive :class:`DatetimeArray` or :class:`DatetimeIndex` and ``utc=True`` would incorrectly return a timezone-naive result (:issue:`27733`)
+- Bug in :meth:`Period.to_timestamp` where a :class:`Period` outside the :class:`Timestamp` implementation bounds (roughly 1677-09-21 to 2262-04-11) would return an incorrect :class:`Timestamp` instead of raising ``OutOfBoundsDatetime`` (:issue:`19643`)
+- Bug in iterating over :class:`DatetimeIndex` when the underlying data is read-only (:issue:`28055`)
 
 Timezones
 ^^^^^^^^^
 
 - Bug in :class:`Index` where a numpy object array with a timezone aware :class:`Timestamp` and ``np.nan`` would not return a :class:`DatetimeIndex` (:issue:`27011`)
--
--
 
 Numeric
 ^^^^^^^
+
 - Bug in :meth:`Series.interpolate` when using a timezone aware :class:`DatetimeIndex` (:issue:`27548`)
 - Bug when printing negative floating point complex numbers would raise an ``IndexError`` (:issue:`27484`)
--
--
+- Bug where :class:`DataFrame` arithmetic operators such as :meth:`DataFrame.mul` with a :class:`Series` with axis=1 would raise an ``AttributeError`` on :class:`DataFrame` larger than the minimum threshold to invoke numexpr (:issue:`27636`)
+- Bug in :class:`DataFrame` arithmetic where missing values in results were incorrectly masked with ``NaN`` instead of ``Inf`` (:issue:`27464`)
 
 Conversion
 ^^^^^^^^^^
 
 - Improved the warnings for the deprecated methods :meth:`Series.real` and :meth:`Series.imag` (:issue:`27610`)
--
--
-
-Strings
-^^^^^^^
-
--
--
--
-
 
 Interval
 ^^^^^^^^
+
 - Bug in :class:`IntervalIndex` where `dir(obj)` would raise ``ValueError`` (:issue:`27571`)
--
--
--
 
 Indexing
 ^^^^^^^^
 
 - Bug in partial-string indexing returning a NumPy array rather than a ``Series`` when indexing with a scalar like ``.loc['2015']`` (:issue:`27516`)
 - Break reference cycle involving :class:`Index` and other index classes to allow garbage collection of index objects without running the GC. (:issue:`27585`, :issue:`27840`)
 - Fix regression in assigning values to a single column of a DataFrame with a ``MultiIndex`` columns (:issue:`27841`).
--
+- Fix regression in ``.ix`` fallback with an ``IntervalIndex`` (:issue:`27865`).
 
 Missing
 ^^^^^^^
 
--
--
--
-
-MultiIndex
-^^^^^^^^^^
-
--
--
--
+- Bug in :func:`pandas.isnull` or :func:`pandas.isna` when the input is a type e.g. ``type(pandas.Series())`` (:issue:`27482`)
 
 I/O
 ^^^
 
 - Avoid calling ``S3File.s3`` when reading parquet, as this was removed in s3fs version 0.3.0 (:issue:`27756`)
 - Better error message when a negative header is passed in :func:`pandas.read_csv` (:issue:`27779`)
--
+- Follow the ``min_rows`` display option (introduced in v0.25.0) correctly in the HTML repr in the notebook (:issue:`27991`).
 
 Plotting
 ^^^^^^^^
 
-- Added a pandas_plotting_backends entrypoint group for registering plot backends. See :ref:`extending.plotting-backends` for more (:issue:`26747`).
+- Added a ``pandas_plotting_backends`` entrypoint group for registering plot backends. See :ref:`extending.plotting-backends` for more (:issue:`26747`).
+- Fixed the re-instatement of Matplotlib datetime converters after calling
+  :meth:`pandas.plotting.deregister_matplotlib_converters` (:issue:`27481`).
 - Fix compatibility issue with matplotlib when passing a pandas ``Index`` to a plot call (:issue:`27775`).
--
 
 Groupby/resample/rolling
 ^^^^^^^^^^^^^^^^^^^^^^^^
 
+- Fixed regression in :meth:`pands.core.groupby.DataFrameGroupBy.quantile` raising when multiple quantiles are given (:issue:`27526`)
 - Bug in :meth:`pandas.core.groupby.DataFrameGroupBy.transform` where applying a timezone conversion lambda function would drop timezone information (:issue:`27496`)
+- Bug in :meth:`pandas.core.groupby.GroupBy.nth` where ``observed=False`` was being ignored for Categorical groupers (:issue:`26385`)
 - Bug in windowing over read-only arrays (:issue:`27766`)
 - Fixed segfault in `pandas.core.groupby.DataFrameGroupBy.quantile` when an invalid quantile was passed (:issue:`27470`)
 - Bug in :meth:`pandas.core.groupby.SeriesGroupBy.nunique` where ``NaT`` values were interfering with the count of unique values (:issue:`27951`)
--
 
 Reshaping
 ^^^^^^^^^
@@ -133,32 +102,13 @@ Reshaping
 
 Sparse
 ^^^^^^
-- Bug in reductions for :class:`Series` with Sparse dtypes (:issue:`27080`)
--
--
--
 
-
-Build Changes
-^^^^^^^^^^^^^
-
--
--
--
-
-ExtensionArray
-^^^^^^^^^^^^^^
-
--
--
--
+- Bug in reductions for :class:`Series` with Sparse dtypes (:issue:`27080`)
 
 Other
 ^^^^^
+
 - Bug in :meth:`Series.replace` and :meth:`DataFrame.replace` when replacing timezone-aware timestamps using a dict-like replacer (:issue:`27720`)
--
--
--
 
 .. _whatsnew_0.251.contributors:
 

doc/source/whatsnew/v0.7.3.rst

@@ -25,8 +25,6 @@ New features
    from pandas.tools.plotting import scatter_matrix
    scatter_matrix(df, alpha=0.2)        # noqa F821
 
-.. image:: ../savefig/scatter_matrix_kde.png
-   :width: 5in
 
 - Add ``stacked`` argument to Series and DataFrame's ``plot`` method for
   :ref:`stacked bar plots <visualization.barplot>`.
@@ -35,15 +33,11 @@ New features
 
    df.plot(kind='bar', stacked=True)    # noqa F821
 
-.. image:: ../savefig/bar_plot_stacked_ex.png
-   :width: 4in
 
 .. code-block:: python
 
    df.plot(kind='barh', stacked=True)   # noqa F821
 
-.. image:: ../savefig/barh_plot_stacked_ex.png
-   :width: 4in
 
 - Add log x and y :ref:`scaling options <visualization.basic>` to
   ``DataFrame.plot`` and ``Series.plot``

doc/source/whatsnew/v1.0.0.rst

@@ -158,7 +158,7 @@ MultiIndex
 I/O
 ^^^
 
--
+- :meth:`read_csv` now accepts binary mode file buffers when using the Python csv engine (:issue:`23779`)
 -
 
 Plotting
@@ -168,6 +168,7 @@ Plotting
 -
 - Bug in :meth:`DataFrame.plot` producing incorrect legend markers when plotting multiple series on the same axis (:issue:`18222`)
 - Bug in :meth:`DataFrame.plot` when ``kind='box'`` and data contains datetime or timedelta data. These types are now automatically dropped (:issue:`22799`)
+- Bug in :meth:`DataFrame.plot.line` and :meth:`DataFrame.plot.area` produce wrong xlim in x-axis (:issue:`27686`, :issue:`25160`, :issue:`24784`)
 
 Groupby/resample/rolling
 ^^^^^^^^^^^^^^^^^^^^^^^^

pandas/_libs/parsers.pyx

@@ -2,7 +2,6 @@
 # See LICENSE for the license
 import bz2
 import gzip
-import lzma
 import os
 import sys
 import time
@@ -59,9 +58,12 @@ from pandas.core.arrays import Categorical
 from pandas.core.dtypes.concat import union_categoricals
 import pandas.io.common as icom
 
+from pandas.compat import _import_lzma, _get_lzma_file
 from pandas.errors import (ParserError, DtypeWarning,
                            EmptyDataError, ParserWarning)
 
+lzma = _import_lzma()
+
 # Import CParserError as alias of ParserError for backwards compatibility.
 # Ultimately, we want to remove this import. See gh-12665 and gh-14479.
 CParserError = ParserError
@@ -645,9 +647,9 @@ cdef class TextReader:
                                      'zip file %s', str(zip_names))
             elif self.compression == 'xz':
                 if isinstance(source, str):
-                    source = lzma.LZMAFile(source, 'rb')
+                    source = _get_lzma_file(lzma)(source, 'rb')
                 else:
-                    source = lzma.LZMAFile(filename=source)
+                    source = _get_lzma_file(lzma)(filename=source)
             else:
                 raise ValueError('Unrecognized compression type: %s' %
                                  self.compression)

pandas/_libs/tslib.pyx

@@ -71,7 +71,7 @@ cdef inline object create_time_from_ts(
 
 @cython.wraparound(False)
 @cython.boundscheck(False)
-def ints_to_pydatetime(int64_t[:] arr, object tz=None, object freq=None,
+def ints_to_pydatetime(const int64_t[:] arr, object tz=None, object freq=None,
                        str box="datetime"):
     """
     Convert an i8 repr to an ndarray of datetimes, date, time or Timestamp

pandas/_libs/tslibs/period.pyx

@@ -21,7 +21,8 @@ PyDateTime_IMPORT
 
 from pandas._libs.tslibs.np_datetime cimport (
     npy_datetimestruct, dtstruct_to_dt64, dt64_to_dtstruct,
-    pandas_datetime_to_datetimestruct, NPY_DATETIMEUNIT, NPY_FR_D)
+    pandas_datetime_to_datetimestruct, check_dts_bounds,
+    NPY_DATETIMEUNIT, NPY_FR_D)
 
 cdef extern from "src/datetime/np_datetime.h":
     int64_t npy_datetimestruct_to_datetime(NPY_DATETIMEUNIT fr,
@@ -1011,7 +1012,7 @@ def dt64arr_to_periodarr(int64_t[:] dtarr, int freq, tz=None):
 
 @cython.wraparound(False)
 @cython.boundscheck(False)
-def periodarr_to_dt64arr(int64_t[:] periodarr, int freq):
+def periodarr_to_dt64arr(const int64_t[:] periodarr, int freq):
     """
     Convert array to datetime64 values from a set of ordinals corresponding to
     periods per period convention.
@@ -1024,9 +1025,8 @@ def periodarr_to_dt64arr(int64_t[:] periodarr, int freq):
 
     out = np.empty(l, dtype='i8')
 
-    with nogil:
-        for i in range(l):
-            out[i] = period_ordinal_to_dt64(periodarr[i], freq)
+    for i in range(l):
+        out[i] = period_ordinal_to_dt64(periodarr[i], freq)
 
     return out.base  # .base to access underlying np.ndarray
 
@@ -1179,14 +1179,15 @@ cpdef int64_t period_ordinal(int y, int m, int d, int h, int min,
     return get_period_ordinal(&dts, freq)
 
 
-cpdef int64_t period_ordinal_to_dt64(int64_t ordinal, int freq) nogil:
+cdef int64_t period_ordinal_to_dt64(int64_t ordinal, int freq) except? -1:
     cdef:
         npy_datetimestruct dts
 
     if ordinal == NPY_NAT:
         return NPY_NAT
 
     get_date_info(ordinal, freq, &dts)
+    check_dts_bounds(&dts)
     return dtstruct_to_dt64(&dts)