Merge branch '1.0.x' of https://github.com/pandas-dev/pandas into 1.0.x

Author: jbrockmendel

doc/redirects.csv

@@ -46,7 +46,10 @@ developer,development/developer
 extending,development/extending
 internals,development/internals
 
-# api
+# api moved function
+reference/api/pandas.io.json.json_normalize,pandas.json_normalize
+
+# api rename
 api,reference/index
 generated/pandas.api.extensions.ExtensionArray.argsort,../reference/api/pandas.api.extensions.ExtensionArray.argsort
 generated/pandas.api.extensions.ExtensionArray.astype,../reference/api/pandas.api.extensions.ExtensionArray.astype

doc/source/_static/css/pandas.css

@@ -12,10 +12,6 @@
   /*min-height: 150px; */
 }
 
-.intro-card .card-img-top {
-  margin: 10px;
-}
-
 .custom-button {
   background-color: #dcdcdc;
   border: none;

doc/source/conf.py

@@ -208,6 +208,7 @@
     "external_links": [],
     "github_url": "https://github.com/pandas-dev/pandas",
     "twitter_url": "https://twitter.com/pandas_dev",
+    "google_analytics_id": "UA-27880019-2",
 }
 
 # Add any paths that contain custom themes here, relative to this directory.

doc/source/reference/extensions.rst

@@ -66,7 +66,7 @@ behaves correctly.
 .. autosummary::
   :toctree: api/
 
-  api.indexers.check_bool_array_indexer
+  api.indexers.check_array_indexer
 
 
 The sentinel ``pandas.api.extensions.no_default`` is used as the default

doc/source/whatsnew/index.rst

@@ -17,6 +17,7 @@ Version 1.0
    :maxdepth: 2
 
    v1.0.0
+   v1.0.1
 
 Version 0.25
 ------------

doc/source/whatsnew/v1.0.0.rst

@@ -0,0 +1,79 @@
+.. _whatsnew_101:
+
+What's new in 1.0.1 (February 5, 2020)
+--------------------------------------
+
+These are the changes in pandas 1.0.1. See :ref:`release` for a full changelog
+including other versions of pandas.
+
+{{ header }}
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_101.regressions:
+
+Fixed regressions
+~~~~~~~~~~~~~~~~~
+
+- Fixed regression in :class:`DataFrame` setting values with a slice (e.g. ``df[-4:] = 1``) indexing by label instead of position (:issue:`31469`)
+- Fixed regression when indexing a ``Series`` or ``DataFrame`` indexed by ``DatetimeIndex`` with a slice containg a :class:`datetime.date` (:issue:`31501`)
+- Fixed regression in ``DataFrame.__setitem__`` raising an ``AttributeError`` with a :class:`MultiIndex` and a non-monotonic indexer (:issue:`31449`)
+- Fixed regression in :class:`Series` multiplication when multiplying a numeric :class:`Series` with >10000 elements with a timedelta-like scalar (:issue:`31457`)
+- Fixed regression in ``.groupby().agg()`` raising an ``AssertionError`` for some reductions like ``min`` on object-dtype columns (:issue:`31522`)
+- Fixed regression in ``.groupby()`` aggregations with categorical dtype using Cythonized reduction functions (e.g. ``first``) (:issue:`31450`)
+- Fixed regression in :meth:`GroupBy.apply` if called with a function which returned a non-pandas non-scalar object (e.g. a list or numpy array) (:issue:`31441`)
+- Fixed regression in :meth:`DataFrame.groupby` whereby taking the minimum or maximum of a column with period dtype would raise a ``TypeError``. (:issue:`31471`)
+- Fixed regression in :meth:`DataFrame.groupby` with an empty DataFrame grouping by a level of a MultiIndex (:issue:`31670`).
+- Fixed regression in :meth:`DataFrame.apply` with object dtype and non-reducing function (:issue:`31505`)
+- Fixed regression in :meth:`to_datetime` when parsing non-nanosecond resolution datetimes (:issue:`31491`)
+- Fixed regression in :meth:`~DataFrame.to_csv` where specifying an ``na_rep`` might truncate the values written (:issue:`31447`)
+- Fixed regression in :class:`Categorical` construction with ``numpy.str_`` categories (:issue:`31499`)
+- Fixed regression in :meth:`DataFrame.loc` and :meth:`DataFrame.iloc` when selecting a row containing a single ``datetime64`` or ``timedelta64`` column (:issue:`31649`)
+- Fixed regression where setting :attr:`pd.options.display.max_colwidth` was not accepting negative integer. In addition, this behavior has been deprecated in favor of using ``None`` (:issue:`31532`)
+- Fixed regression in objTOJSON.c fix return-type warning (:issue:`31463`)
+- Fixed regression in :meth:`qcut` when passed a nullable integer. (:issue:`31389`)
+- Fixed regression in assigning to a :class:`Series` using a nullable integer dtype (:issue:`31446`)
+- Fixed performance regression when indexing a ``DataFrame`` or ``Series`` with a :class:`MultiIndex` for the index using a list of labels (:issue:`31648`)
+- Fixed regression in :meth:`read_csv` used in file like object ``RawIOBase`` is not recognize ``encoding`` option (:issue:`31575`)
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_101.deprecations:
+
+Deprecations
+~~~~~~~~~~~~
+
+- Support for negative integer for :attr:`pd.options.display.max_colwidth` is deprecated in favor of using ``None`` (:issue:`31532`)
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_101.bug_fixes:
+
+Bug fixes
+~~~~~~~~~
+
+**Datetimelike**
+
+- Fixed bug in :meth:`to_datetime` raising when ``cache=True`` and out-of-bound values are present (:issue:`31491`)
+
+**Numeric**
+
+- Bug in dtypes being lost in ``DataFrame.__invert__`` (``~`` operator) with mixed dtypes (:issue:`31183`)
+  and for extension-array backed ``Series`` and ``DataFrame`` (:issue:`23087`)
+
+**Plotting**
+
+- Plotting tz-aware timeseries no longer gives UserWarning (:issue:`31205`)
+
+**Interval**
+
+- Bug in :meth:`Series.shift` with ``interval`` dtype raising a ``TypeError`` when shifting an interval array of integers or datetimes (:issue:`34195`)
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_101.contributors:
+
+Contributors
+~~~~~~~~~~~~
+
+.. contributors:: v1.0.0..v1.0.1|HEAD

doc/source/whatsnew/v1.0.1.rst

@@ -57,6 +57,16 @@ def get_authors(revision_range):
     pat = "^.*\\t(.*)$"
     lst_release, cur_release = [r.strip() for r in revision_range.split("..")]
 
+    if "|" in cur_release:
+        # e.g. v1.0.1|HEAD
+        maybe_tag, head = cur_release.split("|")
+        assert head == "HEAD"
+        if maybe_tag in this_repo.tags:
+            cur_release = maybe_tag
+        else:
+            cur_release = head
+        revision_range = f"{lst_release}..{cur_release}"
+
     # authors, in current release and previous to current release.
     cur = set(re.findall(pat, this_repo.git.shortlog("-s", revision_range), re.M))
     pre = set(re.findall(pat, this_repo.git.shortlog("-s", lst_release), re.M))

doc/sphinxext/announce.py

@@ -6,7 +6,13 @@
 
 This will be replaced with a message indicating the number of
 code contributors and commits, and then list each contributor
-individually.
+individually. For development versions (before a tag is available)
+use::
+
+    .. contributors:: v0.23.0..v0.23.1|HEAD
+
+While the v0.23.1 tag does not exist, that will use the HEAD of the
+branch as the end of the revision range.
 """
 from announce import build_components
 from docutils import nodes

doc/sphinxext/contributors.py

@@ -144,9 +144,7 @@ def _describe_option(pat="", _print_desc=True):
     if len(keys) == 0:
         raise OptionError("No such keys(s)")
 
-    s = ""
-    for k in keys:  # filter by pat
-        s += _build_option_description(k)
+    s = "\n".join([_build_option_description(k) for k in keys])
 
     if _print_desc:
         print(s)

pandas/_config/config.py

@@ -670,7 +670,9 @@ cdef class StringHashTable(HashTable):
             val = values[i]
 
             if isinstance(val, str):
-                v = get_c_string(val)
+                # GH#31499 if we have a np.str_ get_c_string wont recognize
+                #  it as a str, even though isinstance does.
+                v = get_c_string(<str>val)
             else:
                 v = get_c_string(self.na_string_sentinel)
             vecs[i] = v
@@ -703,7 +705,9 @@ cdef class StringHashTable(HashTable):
             val = values[i]
 
             if isinstance(val, str):
-                v = get_c_string(val)
+                # GH#31499 if we have a np.str_ get_c_string wont recognize
+                #  it as a str, even though isinstance does.
+                v = get_c_string(<str>val)
             else:
                 v = get_c_string(self.na_string_sentinel)
             vecs[i] = v

pandas/_libs/hashtable_class_helper.pxi.in

@@ -638,7 +638,7 @@ cdef class TextReader:
                 raise ValueError(f'Unrecognized compression type: '
                                  f'{self.compression}')
 
-            if self.encoding and isinstance(source, io.BufferedIOBase):
+            if self.encoding and isinstance(source, (io.BufferedIOBase, io.RawIOBase)):
                 source = io.TextIOWrapper(
                     source, self.encoding.decode('utf-8'), newline='')
 

pandas/_libs/parsers.pyx

@@ -114,7 +114,8 @@ cdef class Reducer:
                     if self.typ is not None:
                         # In this case, we also have self.index
                         name = labels[i]
-                        cached_typ = self.typ(chunk, index=self.index, name=name)
+                        cached_typ = self.typ(
+                            chunk, index=self.index, name=name, dtype=arr.dtype)
 
                 # use the cached_typ if possible
                 if cached_typ is not None:
@@ -501,9 +502,9 @@ def apply_frame_axis0(object frame, object f, object names,
 
             if not is_scalar(piece):
                 # Need to copy data to avoid appending references
-                if hasattr(piece, "copy"):
+                try:
                     piece = piece.copy(deep="all")
-                else:
+                except (TypeError, AttributeError):
                     piece = copy(piece)
 
             results.append(piece)

pandas/_libs/reduction.pyx

@@ -178,6 +178,8 @@ void *initObjToJSON(void) {
 
     /* Initialise numpy API */
     import_array();
+    // GH 31463
+    return NULL;
 }
 
 static TypeContext *createTypeContext(void) {

pandas/_libs/src/ujson/python/objToJSON.c

@@ -2,7 +2,7 @@
 Public API for Rolling Window Indexers.
 """
 
-from pandas.core.indexers import check_bool_array_indexer
+from pandas.core.indexers import check_array_indexer
 from pandas.core.window.indexers import BaseIndexer
 
-__all__ = ["check_bool_array_indexer", "BaseIndexer"]
+__all__ = ["check_array_indexer", "BaseIndexer"]

pandas/api/indexers/__init__.py

@@ -29,6 +29,7 @@
     is_categorical_dtype,
     is_complex_dtype,
     is_datetime64_any_dtype,
+    is_datetime64_dtype,
     is_datetime64_ns_dtype,
     is_extension_array_dtype,
     is_float_dtype,
@@ -191,6 +192,11 @@ def _reconstruct_data(values, dtype, original):
         if isinstance(original, ABCIndexClass):
             values = values.astype(object, copy=False)
     elif dtype is not None:
+        if is_datetime64_dtype(dtype):
+            dtype = "datetime64[ns]"
+        elif is_timedelta64_dtype(dtype):
+            dtype = "timedelta64[ns]"
+
         values = values.astype(dtype, copy=False)
 
     return values

pandas/core/algorithms.py

@@ -13,7 +13,7 @@
     is_list_like,
     is_sequence,
 )
-from pandas.core.dtypes.generic import ABCMultiIndex, ABCSeries
+from pandas.core.dtypes.generic import ABCSeries
 
 from pandas.core.construction import create_series_with_explicit_dtype
 
@@ -277,9 +277,8 @@ def apply_standard(self):
         if (
             self.result_type in ["reduce", None]
             and not self.dtypes.apply(is_extension_array_dtype).any()
-            # Disallow complex_internals since libreduction shortcut
-            #  cannot handle MultiIndex
-            and not isinstance(self.agg_axis, ABCMultiIndex)
+            # Disallow complex_internals since libreduction shortcut raises a TypeError
+            and not self.agg_axis._has_complex_internals
         ):
 
             values = self.values

pandas/core/apply.py

@@ -26,6 +26,7 @@
 from pandas.core.dtypes.missing import isna, notna
 
 from pandas.core import nanops, ops
+from pandas.core.indexers import check_array_indexer
 
 from .masked import BaseMaskedArray
 
@@ -369,6 +370,7 @@ def __setitem__(self, key, value):
             value = value[0]
             mask = mask[0]
 
+        key = check_array_indexer(self, key)
         self._data[key] = value
         self._mask[key] = mask
 

pandas/core/arrays/boolean.py

@@ -39,7 +39,7 @@
 )
 from pandas.core.dtypes.dtypes import CategoricalDtype
 from pandas.core.dtypes.generic import ABCIndexClass, ABCSeries
-from pandas.core.dtypes.inference import is_array_like, is_hashable
+from pandas.core.dtypes.inference import is_hashable
 from pandas.core.dtypes.missing import isna, notna
 
 from pandas.core import ops
@@ -54,7 +54,7 @@
 from pandas.core.base import NoNewAttributesMixin, PandasObject, _shared_docs
 import pandas.core.common as com
 from pandas.core.construction import array, extract_array, sanitize_array
-from pandas.core.indexers import check_bool_array_indexer
+from pandas.core.indexers import check_array_indexer, deprecate_ndim_indexing
 from pandas.core.missing import interpolate_2d
 from pandas.core.ops.common import unpack_zerodim_and_defer
 from pandas.core.sorting import nargsort
@@ -2001,14 +2001,11 @@ def __getitem__(self, key):
             else:
                 return self.categories[i]
 
-        if is_list_like(key) and not is_array_like(key):
-            key = np.asarray(key)
-
-        if com.is_bool_indexer(key):
-            key = check_bool_array_indexer(self, key)
+        key = check_array_indexer(self, key)
 
         result = self._codes[key]
         if result.ndim > 1:
+            deprecate_ndim_indexing(result)
             return result
         return self._constructor(result, dtype=self.dtype, fastpath=True)
 
@@ -2076,6 +2073,8 @@ def __setitem__(self, key, value):
 
         lindexer = self.categories.get_indexer(rvalue)
         lindexer = self._maybe_coerce_indexer(lindexer)
+
+        key = check_array_indexer(self, key)
         self._codes[key] = lindexer
 
     def _reverse_indexer(self) -> Dict[Hashable, np.ndarray]:

pandas/core/arrays/categorical.py

@@ -42,7 +42,7 @@
 from pandas.core.algorithms import checked_add_with_arr, take, unique1d, value_counts
 from pandas.core.arrays.base import ExtensionArray, ExtensionOpsMixin
 import pandas.core.common as com
-from pandas.core.indexers import check_bool_array_indexer
+from pandas.core.indexers import check_array_indexer
 from pandas.core.ops.common import unpack_zerodim_and_defer
 from pandas.core.ops.invalid import invalid_comparison, make_invalid_op
 
@@ -518,11 +518,20 @@ def __getitem__(self, key):
             return type(self)(val, dtype=self.dtype)
 
         if com.is_bool_indexer(key):
-            key = check_bool_array_indexer(self, key)
+            # first convert to boolean, because check_array_indexer doesn't
+            # allow object dtype
+            key = np.asarray(key, dtype=bool)
+            key = check_array_indexer(self, key)
             if key.all():
                 key = slice(0, None, None)
             else:
                 key = lib.maybe_booleans_to_slice(key.view(np.uint8))
+        elif isinstance(key, list) and len(key) == 1 and isinstance(key[0], slice):
+            # see https://github.com/pandas-dev/pandas/issues/31299, need to allow
+            # this for now (would otherwise raise in check_array_indexer)
+            pass
+        else:
+            key = check_array_indexer(self, key)
 
         is_period = is_period_dtype(self)
         if is_period:
@@ -592,6 +601,8 @@ def __setitem__(
                 f"or array of those. Got '{type(value).__name__}' instead."
             )
             raise TypeError(msg)
+
+        key = check_array_indexer(self, key)
         self._data[key] = value
         self._maybe_clear_freq()
 

pandas/core/arrays/datetimelike.py

@@ -25,6 +25,7 @@
 from pandas.core.dtypes.missing import isna
 
 from pandas.core import nanops, ops
+from pandas.core.indexers import check_array_indexer
 from pandas.core.ops import invalid_comparison
 from pandas.core.ops.common import unpack_zerodim_and_defer
 from pandas.core.tools.numeric import to_numeric
@@ -414,6 +415,7 @@ def __setitem__(self, key, value):
             value = value[0]
             mask = mask[0]
 
+        key = check_array_indexer(self, key)
         self._data[key] = value
         self._mask[key] = mask