Merge branch 'master' of https://github.com/pandas-dev/pandas into tz-localize-categories

Author: Marco Gorelli

asv_bench/benchmarks/frame_methods.py

@@ -609,4 +609,15 @@ def time_dataframe_describe(self):
         self.df.describe()
 
 
+class SelectDtypes:
+    params = [100, 1000]
+    param_names = ["n"]
+
+    def setup(self, n):
+        self.df = DataFrame(np.random.randn(10, n))
+
+    def time_select_dtypes(self, n):
+        self.df.select_dtypes(include="int")
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/rolling.py

@@ -25,6 +25,25 @@ def peakmem_rolling(self, constructor, window, dtype, method):
         getattr(self.roll, method)()
 
 
+class Apply:
+    params = (
+        ["DataFrame", "Series"],
+        [10, 1000],
+        ["int", "float"],
+        [sum, np.sum, lambda x: np.sum(x) + 5],
+        [True, False],
+    )
+    param_names = ["contructor", "window", "dtype", "function", "raw"]
+
+    def setup(self, constructor, window, dtype, function, raw):
+        N = 10 ** 5
+        arr = (100 * np.random.random(N)).astype(dtype)
+        self.roll = getattr(pd, constructor)(arr).rolling(window)
+
+    def time_rolling(self, constructor, window, dtype, function, raw):
+        self.roll.apply(function, raw=raw)
+
+
 class ExpandingMethods:
 
     params = (

doc/source/development/index.rst

@@ -16,4 +16,5 @@ Development
     internals
     extending
     developer
+    policies
     roadmap

doc/source/development/policies.rst

@@ -0,0 +1,57 @@
+.. _develop.policies:
+
+********
+Policies
+********
+
+.. _policies.version:
+
+Version Policy
+~~~~~~~~~~~~~~
+
+.. versionchanged:: 1.0.0
+
+Pandas uses a loose variant of semantic versioning (`SemVer`_) to govern
+deprecations, API compatibility, and version numbering.
+
+A pandas release number is made up of ``MAJOR.MINOR.PATCH``.
+
+API breaking changes should only occur in **major** releases. Theses changes
+will be documented, with clear guidance on what is changing, why it's changing,
+and how to migrate existing code to the new behavior.
+
+Whenever possible, a deprecation path will be provided rather than an outright
+breaking change.
+
+Pandas will introduce deprecations in **minor** releases. These deprecations
+will preserve the existing behavior while emitting a warning that provide
+guidance on:
+
+* How to achieve similar behavior if an alternative is available
+* The pandas version in which the deprecation will be enforced.
+
+We will not introduce new deprecations in patch releases.
+
+Deprecations will only be enforced in **major** releases. For example, if a
+behavior is deprecated in pandas 1.2.0, it will continue to work, with a
+warning, for all releases in the 1.x series. The behavior will change and the
+deprecation removed in the next next major release (2.0.0).
+
+.. note::
+
+   Pandas will sometimes make *behavior changing* bug fixes, as part of
+   minor or patch releases. Whether or not a change is a bug fix or an
+   API-breaking change is a judgement call. We'll do our best, and we
+   invite you to participate in development discussion on the issue
+   tracker or mailing list.
+
+These policies do not apply to features marked as **experimental** in the documentation.
+Pandas may change the behavior of experimental features at any time.
+
+Python Support
+~~~~~~~~~~~~~~
+
+Pandas will only drop support for specific Python versions (e.g. 3.5.x, 3.6.x) in
+pandas **major** releases.
+
+.. _SemVer: https://semver.org

doc/source/whatsnew/v0.25.2.rst

@@ -100,7 +100,8 @@ Other
 ^^^^^
 
 - Compatibility with Python 3.8 in :meth:`DataFrame.query` (:issue:`27261`)
--
+- Fix to ensure that tab-completion in an IPython console does not raise
+  warnings for deprecated attributes (:issue:`27900`).
 
 .. _whatsnew_0.252.contributors:
 

doc/source/whatsnew/v1.0.0.rst

@@ -3,10 +3,39 @@
 What's new in 1.0.0 (??)
 ------------------------
 
-.. warning::
+New Deprecation Policy
+~~~~~~~~~~~~~~~~~~~~~~
+
+Starting with Pandas 1.0.0, pandas will adopt a version of `SemVer`_.
+
+Historically, pandas has used a "rolling" deprecation policy, with occasional
+outright breaking API changes. Where possible, we would deprecate the behavior
+we'd like to change, giving an option to adopt the new behavior (via a keyword
+or an alternative method), and issuing a warning for users of the old behavior.
+Sometimes, a deprecation was not possible, and we would make an outright API
+breaking change.
+
+We'll continue to *introduce* deprecations in major and minor releases (e.g.
+1.0.0, 1.1.0, ...). Those deprecations will be *enforced* in the next major
+release.
+
+Note that *behavior changes* and *API breaking changes* are not identical. API
+breaking changes will only be released in major versions. If we consider a
+behavior to be a bug, and fixing that bug induces a behavior change, we'll
+release that change in a minor release. This is a sometimes difficult judgment
+call that we'll do our best on.
 
-   Starting with the 0.25.x series of releases, pandas only supports Python 3.5.3 and higher.
-   See `Dropping Python 2.7 <https://pandas.pydata.org/pandas-docs/version/0.24/install.html#install-dropping-27>`_ for more details.
+This doesn't mean that pandas' pace of development will slow down. In the `2019
+Pandas User Survey`_, about 95% of the respondents said they considered pandas
+"stable enough". This indicates there's an appetite for new features, even if it
+comes at the cost of break API. The difference is that now API breaking changes
+will be accompanied with a bump in the major version number (e.g. pandas 1.5.1
+-> 2.0.0).
+
+See :ref:`policies.version` for more.
+
+.. _2019 Pandas User Survey: http://dev.pandas.io/pandas-blog/2019-pandas-user-survey.html
+.. _SemVer: https://semver.org
 
 .. warning::
 
@@ -37,7 +66,7 @@ Other enhancements
   pandas (so it will become an integer or float dtype depending on the presence of missing data).
   (:issue:`28368`)
 - :meth:`DataFrame.to_json` now accepts an ``indent`` integer argument to enable pretty printing of JSON output (:issue:`12004`)
-
+- :meth:`read_stata` can read Stata 119 dta files. (:issue:`28250`)
 
 Build Changes
 ^^^^^^^^^^^^^
@@ -122,6 +151,7 @@ Performance improvements
 - Performance improvement in :func:`cut` when ``bins`` is an :class:`IntervalIndex` (:issue:`27668`)
 - Performance improvement in :meth:`DataFrame.corr` when ``method`` is ``"spearman"`` (:issue:`28139`)
 - Performance improvement in :meth:`DataFrame.replace` when provided a list of values to replace (:issue:`28099`)
+- Performance improvement in :meth:`DataFrame.select_dtypes` by using vectorization instead of iterating over a loop (:issue:`28317`)
 
 .. _whatsnew_1000.bug_fixes:
 
@@ -152,7 +182,8 @@ Datetimelike
 - Addition and subtraction of integer or integer-dtype arrays with :class:`Timestamp` will now raise ``NullFrequencyError`` instead of ``ValueError`` (:issue:`28268`)
 - Bug in :class:`Series` and :class:`DataFrame` with integer dtype failing to raise ``TypeError`` when adding or subtracting a ``np.datetime64`` object (:issue:`28080`)
 - Bug in :class:`Week` with ``weekday`` incorrectly raising ``AttributeError`` instead of ``TypeError`` when adding or subtracting an invalid type (:issue:`28530`)
-
+- Bug in :class:`DataFrame` arithmetic operations when operating with a :class:`Series` with dtype `'timedelta64[ns]'` (:issue:`28049`)
+-
 
 Timedelta
 ^^^^^^^^^
@@ -234,6 +265,7 @@ Plotting
 - Bug in :meth:`DataFrame.plot` when ``kind='box'`` and data contains datetime or timedelta data. These types are now automatically dropped (:issue:`22799`)
 - Bug in :meth:`DataFrame.plot.line` and :meth:`DataFrame.plot.area` produce wrong xlim in x-axis (:issue:`27686`, :issue:`25160`, :issue:`24784`)
 - Bug where :meth:`DataFrame.boxplot` would not accept a `color` parameter like `DataFrame.plot.box` (:issue:`26214`)
+- Bug in the ``xticks`` argument being ignored for :meth:`DataFrame.plot.bar` (:issue:`14119`)
 - :func:`set_option` now validates that the plot backend provided to ``'plotting.backend'`` implements the backend when the option is set, rather than when a plot is created (:issue:`28163`)
 
 Groupby/resample/rolling

pandas/_libs/lib.pyx

@@ -55,8 +55,7 @@ cimport pandas._libs.util as util
 from pandas._libs.util cimport is_nan, UINT64_MAX, INT64_MAX, INT64_MIN
 
 from pandas._libs.tslib import array_to_datetime
-from pandas._libs.tslibs.nattype cimport NPY_NAT
-from pandas._libs.tslibs.nattype import NaT
+from pandas._libs.tslibs.nattype cimport NPY_NAT, c_NaT as NaT
 from pandas._libs.tslibs.conversion cimport convert_to_tsobject
 from pandas._libs.tslibs.timedeltas cimport convert_to_timedelta64
 from pandas._libs.tslibs.timezones cimport get_timezone, tz_compare
@@ -525,9 +524,18 @@ def array_equivalent_object(left: object[:], right: object[:]) -> bool:
 
         # we are either not equal or both nan
         # I think None == None will be true here
-        if not (PyObject_RichCompareBool(x, y, Py_EQ) or
-                (x is None or is_nan(x)) and (y is None or is_nan(y))):
-            return False
+        try:
+            if not (PyObject_RichCompareBool(x, y, Py_EQ) or
+                    (x is None or is_nan(x)) and (y is None or is_nan(y))):
+                return False
+        except TypeError as err:
+            # Avoid raising TypeError on tzawareness mismatch
+            # TODO: This try/except can be removed if/when Timestamp
+            #  comparisons are change dto match datetime, see GH#28507
+            if "tz-naive and tz-aware" in str(err):
+                return False
+            raise
+
     return True
 
 

pandas/core/algorithms.py

@@ -39,7 +39,6 @@
     is_period_dtype,
     is_scalar,
     is_signed_integer_dtype,
-    is_sparse,
     is_timedelta64_dtype,
     is_unsigned_integer_dtype,
     needs_i8_conversion,
@@ -743,7 +742,7 @@ def value_counts(
 
     else:
 
-        if is_extension_array_dtype(values) or is_sparse(values):
+        if is_extension_array_dtype(values):
 
             # handle Categorical and sparse,
             result = Series(values)._values.value_counts(dropna=dropna)
@@ -1623,7 +1622,7 @@ def take_nd(
     out : ndarray or None, default None
         Optional output array, must be appropriate type to hold input and
         fill_value together, if indexer has any -1 value entries; call
-        _maybe_promote to determine this type for any fill_value
+        maybe_promote to determine this type for any fill_value
     fill_value : any, default np.nan
         Fill value to replace -1 values with
     mask_info : tuple of (ndarray, boolean)
@@ -1644,9 +1643,7 @@ def take_nd(
     if is_extension_array_dtype(arr):
         return arr.take(indexer, fill_value=fill_value, allow_fill=allow_fill)
 
-    if is_sparse(arr):
-        arr = arr.to_dense()
-    elif isinstance(arr, (ABCIndexClass, ABCSeries)):
+    if isinstance(arr, (ABCIndexClass, ABCSeries)):
         arr = arr._values
 
     arr = np.asarray(arr)

pandas/core/construction.py

@@ -9,7 +9,7 @@
 import numpy as np
 import numpy.ma as ma
 
-from pandas._libs import lib, tslibs
+from pandas._libs import lib
 from pandas._libs.tslibs import IncompatibleFrequency, OutOfBoundsDatetime
 
 from pandas.core.dtypes.cast import (
@@ -36,7 +36,7 @@
     is_timedelta64_ns_dtype,
     pandas_dtype,
 )
-from pandas.core.dtypes.dtypes import ExtensionDtype, registry
+from pandas.core.dtypes.dtypes import CategoricalDtype, ExtensionDtype, registry
 from pandas.core.dtypes.generic import (
     ABCExtensionArray,
     ABCIndexClass,
@@ -275,7 +275,7 @@ def array(
         if inferred_dtype == "period":
             try:
                 return period_array(data, copy=copy)
-            except tslibs.IncompatibleFrequency:
+            except IncompatibleFrequency:
                 # We may have a mixture of frequencies.
                 # We choose to return an ndarray, rather than raising.
                 pass
@@ -365,7 +365,9 @@ def extract_array(obj, extract_numpy=False):
     return obj
 
 
-def sanitize_array(data, index, dtype=None, copy=False, raise_cast_failure=False):
+def sanitize_array(
+    data, index, dtype=None, copy: bool = False, raise_cast_failure: bool = False
+):
     """
     Sanitize input data to an ndarray, copy if specified, coerce to the
     dtype if specified.
@@ -486,13 +488,19 @@ def sanitize_array(data, index, dtype=None, copy=False, raise_cast_failure=False
     return subarr
 
 
-def _try_cast(arr, dtype, copy, raise_cast_failure):
+def _try_cast(
+    arr,
+    dtype: Optional[Union[np.dtype, "ExtensionDtype"]],
+    copy: bool,
+    raise_cast_failure: bool,
+):
     """
     Convert input to numpy ndarray and optionally cast to a given dtype.
 
     Parameters
     ----------
-    arr : array-like
+    arr : ndarray, list, tuple, iterator (catchall)
+        Excludes: ExtensionArray, Series, Index.
     dtype : np.dtype, ExtensionDtype or None
     copy : bool
         If False, don't copy the data if not needed.
@@ -528,11 +536,13 @@ def _try_cast(arr, dtype, copy, raise_cast_failure):
         if is_categorical_dtype(dtype):
             # We *do* allow casting to categorical, since we know
             # that Categorical is the only array type for 'category'.
+            dtype = cast(CategoricalDtype, dtype)
             subarr = dtype.construct_array_type()(
                 arr, dtype.categories, ordered=dtype._ordered
             )
         elif is_extension_array_dtype(dtype):
             # create an extension array from its dtype
+            dtype = cast(ExtensionDtype, dtype)
             array_type = dtype.construct_array_type()._from_sequence
             subarr = array_type(arr, dtype=dtype, copy=copy)
         elif dtype is not None and raise_cast_failure:

pandas/core/dtypes/cast.py

@@ -796,7 +796,7 @@ def maybe_convert_objects(values: np.ndarray, convert_numeric: bool = True):
                 new_values = lib.maybe_convert_numeric(
                     values, set(), coerce_numeric=True
                 )
-            except Exception:
+            except (ValueError, TypeError):
                 pass
             else:
                 # if we are all nans then leave me alone
@@ -875,7 +875,7 @@ def soft_convert_objects(
     if numeric and is_object_dtype(values.dtype):
         try:
             converted = lib.maybe_convert_numeric(values, set(), coerce_numeric=True)
-        except Exception:
+        except (ValueError, TypeError):
             pass
         else:
             # If all NaNs, then do not-alter
@@ -953,9 +953,10 @@ def try_datetime(v):
             # we might have a sequence of the same-datetimes with tz's
             # if so coerce to a DatetimeIndex; if they are not the same,
             # then these stay as object dtype, xref GH19671
+            from pandas._libs.tslibs import conversion
+            from pandas import DatetimeIndex
+
             try:
-                from pandas._libs.tslibs import conversion
-                from pandas import DatetimeIndex
 
                 values, tz = conversion.datetime_to_datetime64(v)
                 return DatetimeIndex(values).tz_localize("UTC").tz_convert(tz=tz)
@@ -1310,9 +1311,8 @@ def construct_1d_ndarray_preserving_na(values, dtype=None, copy=False):
     >>> np.array([1.0, 2.0, None], dtype='str')
     array(['1.0', '2.0', 'None'], dtype='<U4')
 
-    >>> construct_1d_ndarray_preserving_na([1.0, 2.0, None], dtype='str')
-
-
+    >>> construct_1d_ndarray_preserving_na([1.0, 2.0, None], dtype=np.dtype('str'))
+    array(['1.0', '2.0', None], dtype=object)
     """
     subarr = np.array(values, dtype=dtype, copy=copy)
 

pandas/core/dtypes/common.py

@@ -888,7 +888,8 @@ def is_dtype_equal(source, target):
 
 
 def is_any_int_dtype(arr_or_dtype) -> bool:
-    """Check whether the provided array or dtype is of an integer dtype.
+    """
+    Check whether the provided array or dtype is of an integer dtype.
 
     In this function, timedelta64 instances are also considered "any-integer"
     type objects and will return True.