Merge remote-tracking branch 'upstream/main'

Author: kwhkim

asv_bench/benchmarks/tslibs/tz_convert.py

@@ -11,10 +11,14 @@
 
 try:
     old_sig = False
-    from pandas._libs.tslibs.tzconversion import tz_convert_from_utc
+    from pandas._libs.tslibs import tz_convert_from_utc
 except ImportError:
-    old_sig = True
-    from pandas._libs.tslibs.tzconversion import tz_convert as tz_convert_from_utc
+    try:
+        old_sig = False
+        from pandas._libs.tslibs.tzconversion import tz_convert_from_utc
+    except ImportError:
+        old_sig = True
+        from pandas._libs.tslibs.tzconversion import tz_convert as tz_convert_from_utc
 
 
 class TimeTZConvert:

azure-pipelines.yml

@@ -48,6 +48,7 @@ jobs:
         pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \
         python setup.py build_ext -q -j2 && \
         python -m pip install --no-build-isolation -e . && \
+        export PANDAS_CI=1 && \
         pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml"
       displayName: 'Run 32-bit manylinux2014 Docker Build / Tests'
 

doc/source/getting_started/intro_tutorials/04_plotting.rst

@@ -2,6 +2,12 @@
 
 {{ header }}
 
+How do I create plots in pandas?
+----------------------------------
+
+.. image:: ../../_static/schemas/04_plot_overview.svg
+   :align: center
+
 .. ipython:: python
 
     import pandas as pd
@@ -35,12 +41,6 @@
     </ul>
     </div>
 
-How to create plots in pandas?
-------------------------------
-
-.. image:: ../../_static/schemas/04_plot_overview.svg
-   :align: center
-
 .. raw:: html
 
     <ul class="task-bullet">
@@ -52,6 +52,7 @@ I want a quick visual check of the data.
 
     @savefig 04_airqual_quick.png
     air_quality.plot()
+    plt.show()
 
 With a ``DataFrame``, pandas creates by default one line plot for each of
 the columns with numeric data.
@@ -68,10 +69,19 @@ the columns with numeric data.
 
 I want to plot only the columns of the data table with the data from Paris.
 
+.. ipython:: python
+   :suppress:
+
+    # We need to clear the figure here as, within doc generation, the plot
+    # accumulates data on each plot(). This is not needed when running
+    # in a notebook, so is suppressed from output.
+    plt.clf()
+
 .. ipython:: python
 
     @savefig 04_airqual_paris.png
     air_quality["station_paris"].plot()
+    plt.show()
 
 To plot a specific column, use the selection method of the
 :ref:`subset data tutorial <10min_tut_03_subset>` in combination with the :meth:`~DataFrame.plot`
@@ -94,6 +104,7 @@ I want to visually compare the :math:`NO_2` values measured in London versus Par
 
     @savefig 04_airqual_scatter.png
     air_quality.plot.scatter(x="station_london", y="station_paris", alpha=0.5)
+    plt.show()
 
 .. raw:: html
 
@@ -125,6 +136,7 @@ method is applicable on the air quality example data:
 
     @savefig 04_airqual_boxplot.png
     air_quality.plot.box()
+    plt.show()
 
 .. raw:: html
 
@@ -148,6 +160,7 @@ I want each of the columns in a separate subplot.
 
     @savefig 04_airqual_area_subplot.png
     axs = air_quality.plot.area(figsize=(12, 4), subplots=True)
+    plt.show()
 
 Separate subplots for each of the data columns are supported by the ``subplots`` argument
 of the ``plot`` functions. The builtin options available in each of the pandas plot
@@ -180,9 +193,10 @@ I want to further customize, extend or save the resulting plot.
 
     fig, axs = plt.subplots(figsize=(12, 4))
     air_quality.plot.area(ax=axs)
-    @savefig 04_airqual_customized.png
     axs.set_ylabel("NO$_2$ concentration")
+    @savefig 04_airqual_customized.png
     fig.savefig("no2_concentrations.png")
+    plt.show()
 
 .. ipython:: python
    :suppress:
@@ -208,6 +222,7 @@ This strategy is applied in the previous example:
    air_quality.plot.area(ax=axs)                   # Use pandas to put the area plot on the prepared Figure/Axes
    axs.set_ylabel("NO$_2$ concentration")          # Do any Matplotlib customization you like
    fig.savefig("no2_concentrations.png")           # Save the Figure/Axes using the existing Matplotlib method.
+   plt.show()                                      # Display the plot
 
 .. raw:: html
 

doc/source/whatsnew/v1.5.0.rst

@@ -95,6 +95,8 @@ Other enhancements
 - :meth:`pd.concat` now raises when ``levels`` is given but ``keys`` is None (:issue:`46653`)
 - :meth:`pd.concat` now raises when ``levels`` contains duplicate values (:issue:`46653`)
 - Added ``numeric_only`` argument to :meth:`DataFrame.corr`, :meth:`DataFrame.corrwith`, and :meth:`DataFrame.cov` (:issue:`46560`)
+- A :class:`errors.PerformanceWarning` is now thrown when using ``string[pyarrow]`` dtype with methods that don't dispatch to ``pyarrow.compute`` methods (:issue:`42613`)
+- Added ``numeric_only`` argument to :meth:`Resampler.sum`, :meth:`Resampler.prod`, :meth:`Resampler.min`, :meth:`Resampler.max`, :meth:`Resampler.first`, and :meth:`Resampler.last` (:issue:`46442`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_150.notable_bug_fixes:
@@ -478,10 +480,12 @@ Datetimelike
 Timedelta
 ^^^^^^^^^
 - Bug in :func:`astype_nansafe` astype("timedelta64[ns]") fails when np.nan is included (:issue:`45798`)
+- Bug in constructing a :class:`Timedelta` with a ``np.timedelta64`` object and a ``unit`` sometimes silently overflowing and returning incorrect results instead of raising ``OutOfBoundsTimedelta`` (:issue:`46827`)
+-
 
 Time Zones
 ^^^^^^^^^^
--
+- Bug in :class:`Timestamp` constructor raising when passed a ``ZoneInfo`` tzinfo object (:issue:`46425`)
 -
 
 Numeric
@@ -500,6 +504,7 @@ Conversion
 - Bug in :func:`array` with ``FloatingDtype`` and values containing float-castable strings incorrectly raising (:issue:`45424`)
 - Bug when comparing string and datetime64ns objects causing ``OverflowError`` exception. (:issue:`45506`)
 - Bug in metaclass of generic abstract dtypes causing :meth:`DataFrame.apply` and :meth:`Series.apply` to raise for the built-in function ``type`` (:issue:`46684`)
+- Bug in :meth:`DataFrame.to_dict` for ``orient="list"`` or ``orient="index"`` was not returning native types (:issue:`46751`)
 
 Strings
 ^^^^^^^
@@ -569,6 +574,8 @@ I/O
 - Bug in Parquet roundtrip for Interval dtype with ``datetime64[ns]`` subtype (:issue:`45881`)
 - Bug in :func:`read_excel` when reading a ``.ods`` file with newlines between xml elements (:issue:`45598`)
 - Bug in :func:`read_parquet` when ``engine="fastparquet"`` where the file was not closed on error (:issue:`46555`)
+- :meth:`to_html` now excludes the ``border`` attribute from ``<table>`` elements when ``border`` keyword is set to ``False``.
+-
 
 Period
 ^^^^^^
@@ -599,7 +606,10 @@ Groupby/resample/rolling
 - Bug in :meth:`GroupBy.cummax` with ``int64`` dtype with leading value being the smallest possible int64 (:issue:`46382`)
 - Bug in :meth:`GroupBy.max` with empty groups and ``uint64`` dtype incorrectly raising ``RuntimeError`` (:issue:`46408`)
 - Bug in :meth:`.GroupBy.apply` would fail when ``func`` was a string and args or kwargs were supplied (:issue:`46479`)
--
+- Bug in :meth:`SeriesGroupBy.apply` would incorrectly name its result when there was a unique group (:issue:`46369`)
+- Bug in :meth:`Rolling.var` and :meth:`Rolling.std` would give non-zero result with window of same values (:issue:`42064`)
+- Bug in :meth:`.Rolling.var` would segfault calculating weighted variance when window size was larger than data size (:issue:`46760`)
+- Bug in :meth:`Grouper.__repr__` where ``dropna`` was not included. Now it is (:issue:`46754`)
 
 Reshaping
 ^^^^^^^^^

pandas/_libs/algos.pyx

@@ -1,6 +1,5 @@
-import cython
-from cython import Py_ssize_t
-
+cimport cython
+from cython cimport Py_ssize_t
 from libc.math cimport (
     fabs,
     sqrt,

pandas/_libs/algos_common_helper.pxi.in

@@ -65,7 +65,8 @@ def ensure_{{name}}(object arr, copy=True):
         if (<ndarray>arr).descr.type_num == NPY_{{c_type}}:
             return arr
         else:
-            return arr.astype(np.{{dtype}}, copy=copy)
+            # equiv: arr.astype(np.{{dtype}}, copy=copy)
+            return cnp.PyArray_Cast(<ndarray>arr, cnp.NPY_{{c_type}})
     else:
         return np.array(arr, dtype=np.{{dtype}})
 

pandas/_libs/groupby.pyx

@@ -1,7 +1,8 @@
-import cython
-from cython import Py_ssize_t
-
-from cython cimport floating
+cimport cython
+from cython cimport (
+    Py_ssize_t,
+    floating,
+)
 from libc.stdlib cimport (
     free,
     malloc,
@@ -268,7 +269,6 @@ def group_cumsum(
                         out[i, j] = na_val
                         continue
 
-
                 if isna_entry:
                     out[i, j] = na_val
                     if not skipna:
@@ -1220,7 +1220,7 @@ def group_nth(
                         if nobs[lab, j] == rank:
                             resx[lab, j] = val
 
-            # TODO: de-dup this whoel block with group_last?
+            # TODO: de-dup this whole block with group_last?
             for i in range(ncounts):
                 for j in range(K):
                     if nobs[i, j] < min_count:
@@ -1232,6 +1232,9 @@ def group_nth(
                         #  set a placeholder value in out[i, j].
                         if uses_mask:
                             result_mask[i, j] = True
+                            # set out[i, j] to 0 to be deterministic, as
+                            #  it was initialized with np.empty. Also ensures
+                            #  we can downcast out if appropriate.
                             out[i, j] = 0
                         elif numeric_object_t is float32_t or numeric_object_t is float64_t:
                             out[i, j] = NAN
@@ -1369,7 +1372,7 @@ cdef group_min_max(
     """
     cdef:
         Py_ssize_t i, j, N, K, lab, ngroups = len(counts)
-        numeric_t val, nan_val
+        numeric_t val
         ndarray[numeric_t, ndim=2] group_min_or_max
         int64_t[:, ::1] nobs
         bint uses_mask = mask is not None
@@ -1386,20 +1389,6 @@ cdef group_min_max(
     group_min_or_max = np.empty_like(out)
     group_min_or_max[:] = _get_min_or_max(<numeric_t>0, compute_max, is_datetimelike)
 
-    # NB: We do not define nan_val because there is no such thing
-    # for uint64_t.  We carefully avoid having to reference it in this
-    # case.
-    if numeric_t is int64_t:
-        nan_val = NPY_NAT
-    elif numeric_t is int32_t:
-        nan_val = util.INT32_MIN
-    elif numeric_t is int16_t:
-        nan_val = util.INT16_MIN
-    elif numeric_t is int8_t:
-        nan_val = util.INT8_MIN
-    elif numeric_t is float64_t or numeric_t is float32_t:
-        nan_val = NAN
-
     N, K = (<object>values).shape
 
     with nogil:
@@ -1442,11 +1431,11 @@ cdef group_min_max(
                         #  we can downcast out if appropriate.
                         out[i, j] = 0
                     elif numeric_t is float32_t or numeric_t is float64_t:
-                        out[i, j] = nan_val
+                        out[i, j] = NAN
                     elif numeric_t is int64_t:
                         # Per above, this is a placeholder in
                         #  non-is_datetimelike cases.
-                        out[i, j] = nan_val
+                        out[i, j] = NPY_NAT
                     else:
                         # placeholder, see above
                         out[i, j] = 0

pandas/_libs/hashing.pyx

@@ -1,8 +1,7 @@
 # Translated from the reference implementation
 # at https://github.com/veorq/SipHash
 
-import cython
-
+cimport cython
 from libc.stdlib cimport (
     free,
     malloc,

pandas/_libs/internals.pyx

@@ -1,9 +1,8 @@
 from collections import defaultdict
 
-import cython
-from cython import Py_ssize_t
-
+cimport cython
 from cpython.slice cimport PySlice_GetIndicesEx
+from cython cimport Py_ssize_t
 
 
 cdef extern from "Python.h":

pandas/_libs/interval.pyx

@@ -11,6 +11,7 @@ from cpython.datetime cimport (
 
 import_datetime()
 
+cimport cython
 from cpython.object cimport (
     Py_EQ,
     Py_GE,
@@ -20,9 +21,8 @@ from cpython.object cimport (
     Py_NE,
     PyObject_RichCompare,
 )
+from cython cimport Py_ssize_t
 
-import cython
-from cython import Py_ssize_t
 import numpy as np
 
 cimport numpy as cnp

pandas/_libs/join.pyx

@@ -1,5 +1,5 @@
-import cython
-from cython import Py_ssize_t
+cimport cython
+from cython cimport Py_ssize_t
 import numpy as np
 
 cimport numpy as cnp
@@ -233,6 +233,8 @@ cdef void _get_result_indexer(intp_t[::1] sorter, intp_t[::1] indexer) nogil:
         indexer[:] = -1
 
 
+@cython.wraparound(False)
+@cython.boundscheck(False)
 def ffill_indexer(const intp_t[:] indexer) -> np.ndarray:
     cdef:
         Py_ssize_t i, n = len(indexer)

pandas/_libs/lib.pyx

@@ -3,9 +3,7 @@ from decimal import Decimal
 from enum import Enum
 import warnings
 
-import cython
-from cython import Py_ssize_t
-
+cimport cython
 from cpython.datetime cimport (
     PyDate_Check,
     PyDateTime_Check,
@@ -25,7 +23,10 @@ from cpython.tuple cimport (
     PyTuple_New,
     PyTuple_SET_ITEM,
 )
-from cython cimport floating
+from cython cimport (
+    Py_ssize_t,
+    floating,
+)
 
 import_datetime()
 

pandas/_libs/missing.pyx

@@ -2,8 +2,8 @@ from decimal import Decimal
 import numbers
 from sys import maxsize
 
-import cython
-from cython import Py_ssize_t
+cimport cython
+from cython cimport Py_ssize_t
 import numpy as np
 
 cimport numpy as cnp

pandas/_libs/ops.pyx

@@ -1,5 +1,6 @@
 import operator
 
+cimport cython
 from cpython.object cimport (
     Py_EQ,
     Py_GE,
@@ -9,9 +10,8 @@ from cpython.object cimport (
     Py_NE,
     PyObject_RichCompareBool,
 )
+from cython cimport Py_ssize_t
 
-import cython
-from cython import Py_ssize_t
 import numpy as np
 
 from numpy cimport (