Merge branch 'main' of https://github.com/weikhor/pandas into enh-consistency-interval-range

Author: chean.wei.khor

.github/workflows/posix.yml

@@ -162,7 +162,7 @@ jobs:
       shell: bash
       run: |
         # TODO: re-enable cov, its slowing the tests down though
-        pip install Cython numpy python-dateutil pytz pytest>=6.0 pytest-xdist>=1.31.0 hypothesis>=5.5.3
+        pip install Cython numpy python-dateutil pytz pytest>=6.0 pytest-xdist>=1.31.0 pytest-asyncio hypothesis>=5.5.3
       if: ${{ env.IS_PYPY == 'true' }}
 
     - name: Build Pandas

asv_bench/benchmarks/categoricals.py

@@ -187,7 +187,7 @@ def time_remove_categories(self):
 class Rank:
     def setup(self):
         N = 10**5
-        ncats = 100
+        ncats = 15
 
         self.s_str = pd.Series(tm.makeCategoricalIndex(N, ncats)).astype(str)
         self.s_str_cat = pd.Series(self.s_str, dtype="category")

doc/source/whatsnew/v1.4.2.rst

@@ -19,6 +19,7 @@ Fixed regressions
 - Fixed memory performance regression in :meth:`Series.fillna` when called on a :class:`DataFrame` column with ``inplace=True`` (:issue:`46149`)
 - Provided an alternative solution for passing custom Excel formats in :meth:`.Styler.to_excel`, which was a regression based on stricter CSS validation. Examples available in the documentation for :meth:`.Styler.format` (:issue:`46152`)
 - Fixed regression in :meth:`DataFrame.replace` when a replacement value was also a target for replacement (:issue:`46306`)
+- Fixed regression in :meth:`DataFrame.replace` when the replacement value was explicitly ``None`` when passed in a dictionary to ``to_replace`` (:issue:`45601`, :issue:`45836`)
 - Fixed regression when setting values with :meth:`DataFrame.loc` losing :class:`MultiIndex` names if :class:`DataFrame`  was empty before (:issue:`46317`)
 -
 

doc/source/whatsnew/v1.5.0.rst

@@ -504,6 +504,7 @@ Groupby/resample/rolling
 - Bug in :meth:`.GroupBy.cumsum` with ``timedelta64[ns]`` dtype failing to recognize ``NaT`` as a null value (:issue:`46216`)
 - Bug in :meth:`GroupBy.cummin` and :meth:`GroupBy.cummax` with nullable dtypes incorrectly altering the original data in place (:issue:`46220`)
 - Bug in :meth:`GroupBy.cummax` with ``int64`` dtype with leading value being the smallest possible int64 (:issue:`46382`)
+- Bug in :meth:`GroupBy.max` with empty groups and ``uint64`` dtype incorrectly raising ``RuntimeError`` (:issue:`46408`)
 -
 
 Reshaping

pandas/_libs/hashing.pyx

@@ -53,7 +53,7 @@ def hash_object_array(
     """
     cdef:
         Py_ssize_t i, n
-        uint64_t[:] result
+        uint64_t[::1] result
         bytes data, k
         uint8_t *kb
         uint64_t *lens

pandas/_libs/hashtable_class_helper.pxi.in

@@ -503,7 +503,7 @@ cdef class {{name}}HashTable(HashTable):
             int ret = 0
             {{c_type}} val
             khiter_t k
-            intp_t[:] locs = np.empty(n, dtype=np.intp)
+            intp_t[::1] locs = np.empty(n, dtype=np.intp)
 
         with nogil:
             for i in range(n):
@@ -561,7 +561,7 @@ cdef class {{name}}HashTable(HashTable):
         """
         cdef:
             Py_ssize_t i, idx, count = count_prior, n = len(values)
-            intp_t[:] labels
+            intp_t[::1] labels
             int ret = 0
             {{c_type}} val, na_value2
             khiter_t k
@@ -710,7 +710,7 @@ cdef class {{name}}HashTable(HashTable):
         # tuple[np.ndarray[np.intp], np.ndarray[{{dtype}}]]
         cdef:
             Py_ssize_t i, n = len(values)
-            intp_t[:] labels
+            intp_t[::1] labels
             Py_ssize_t idx, count = 0
             int ret = 0
             {{c_type}} val
@@ -848,7 +848,7 @@ cdef class StringHashTable(HashTable):
             object val
             const char *v
             khiter_t k
-            intp_t[:] locs = np.empty(n, dtype=np.intp)
+            intp_t[::1] locs = np.empty(n, dtype=np.intp)
 
         # these by-definition *must* be strings
         vecs = <const char **>malloc(n * sizeof(char *))
@@ -946,8 +946,8 @@ cdef class StringHashTable(HashTable):
         """
         cdef:
             Py_ssize_t i, idx, count = count_prior, n = len(values)
-            intp_t[:] labels
-            int64_t[:] uindexer
+            intp_t[::1] labels
+            int64_t[::1] uindexer
             int ret = 0
             object val
             const char *v
@@ -1168,7 +1168,7 @@ cdef class PyObjectHashTable(HashTable):
             int ret = 0
             object val
             khiter_t k
-            intp_t[:] locs = np.empty(n, dtype=np.intp)
+            intp_t[::1] locs = np.empty(n, dtype=np.intp)
 
         for i in range(n):
             val = values[i]
@@ -1223,7 +1223,7 @@ cdef class PyObjectHashTable(HashTable):
         """
         cdef:
             Py_ssize_t i, idx, count = count_prior, n = len(values)
-            intp_t[:] labels
+            intp_t[::1] labels
             int ret = 0
             object val
             khiter_t k

pandas/_libs/hashtable_func_helper.pxi.in

@@ -85,7 +85,9 @@ cdef value_count_{{dtype}}(const {{dtype}}_t[:] values, bint dropna):
     {{endif}}
 
     # collect counts in the order corresponding to result_keys:
-    cdef int64_t[:] result_counts = np.empty(table.size, dtype=np.int64)
+    cdef:
+        int64_t[::1] result_counts = np.empty(table.size, dtype=np.int64)
+
     for i in range(table.size):
         {{if dtype == 'object'}}
         k = kh_get_{{ttype}}(table, result_keys.data[i])
@@ -366,7 +368,7 @@ def mode(ndarray[htfunc_t] values, bint dropna):
         ndarray[htfunc_t] keys
         ndarray[htfunc_t] modes
 
-        int64_t[:] counts
+        int64_t[::1] counts
         int64_t count, max_count = -1
         Py_ssize_t nkeys, k, j = 0
 

pandas/_libs/interval.pyx

@@ -5,11 +5,11 @@ from operator import (
 )
 
 from cpython.datetime cimport (
-    PyDateTime_IMPORT,
     PyDelta_Check,
+    import_datetime,
 )
 
-PyDateTime_IMPORT
+import_datetime()
 
 from cpython.object cimport (
     Py_EQ,

pandas/_libs/lib.pyx

@@ -9,9 +9,9 @@ from cython import Py_ssize_t
 from cpython.datetime cimport (
     PyDate_Check,
     PyDateTime_Check,
-    PyDateTime_IMPORT,
     PyDelta_Check,
     PyTime_Check,
+    import_datetime,
 )
 from cpython.iterator cimport PyIter_Check
 from cpython.number cimport PyNumber_Check
@@ -27,7 +27,7 @@ from cpython.tuple cimport (
 )
 from cython cimport floating
 
-PyDateTime_IMPORT
+import_datetime()
 
 import numpy as np
 
@@ -2470,8 +2470,8 @@ def maybe_convert_objects(ndarray[object] objects,
         ndarray[int64_t] ints
         ndarray[uint64_t] uints
         ndarray[uint8_t] bools
-        int64_t[:]  idatetimes
-        int64_t[:] itimedeltas
+        int64_t[::1]  idatetimes
+        int64_t[::1] itimedeltas
         Seen seen = Seen()
         object val
         float64_t fval, fnan = np.nan

pandas/_libs/ops.pyx

@@ -194,7 +194,7 @@ def scalar_binop(object[:] values, object val, object op) -> ndarray:
     """
     cdef:
         Py_ssize_t i, n = len(values)
-        object[:] result
+        object[::1] result
         object x
 
     result = np.empty(n, dtype=object)
@@ -231,7 +231,7 @@ def vec_binop(object[:] left, object[:] right, object op) -> ndarray:
     """
     cdef:
         Py_ssize_t i, n = len(left)
-        object[:] result
+        object[::1] result
 
     if n != <Py_ssize_t>len(right):
         raise ValueError(f'Arrays were different lengths: {n} vs {len(right)}')

pandas/_libs/parsers.pyx

@@ -1457,7 +1457,7 @@ cdef _categorical_convert(parser_t *parser, int64_t col,
         const char *word = NULL
 
         int64_t NA = -1
-        int64_t[:] codes
+        int64_t[::1] codes
         int64_t current_category = 0
 
         char *errors = "strict"

pandas/_libs/tslib.pyx

@@ -5,13 +5,13 @@ import cython
 from cpython.datetime cimport (
     PyDate_Check,
     PyDateTime_Check,
-    PyDateTime_IMPORT,
     datetime,
+    import_datetime,
     tzinfo,
 )
 
 # import datetime C API
-PyDateTime_IMPORT
+import_datetime()
 
 
 cimport numpy as cnp
@@ -63,7 +63,6 @@ from pandas._libs.tslibs.timestamps cimport _Timestamp
 from pandas._libs.tslibs.timestamps import Timestamp
 
 # Note: this is the only non-tslibs intra-pandas dependency here
-
 from pandas._libs.missing cimport checknull_with_nat_and_na
 from pandas._libs.tslibs.tzconversion cimport tz_localize_to_utc_single
 

pandas/_libs/tslibs/conversion.pyx

@@ -18,13 +18,13 @@ import pytz
 from cpython.datetime cimport (
     PyDate_Check,
     PyDateTime_Check,
-    PyDateTime_IMPORT,
     datetime,
+    import_datetime,
     time,
     tzinfo,
 )
 
-PyDateTime_IMPORT
+import_datetime()
 
 from pandas._libs.tslibs.base cimport ABCTimestamp
 from pandas._libs.tslibs.np_datetime cimport (
@@ -71,7 +71,7 @@ from pandas._libs.tslibs.nattype cimport (
 )
 from pandas._libs.tslibs.tzconversion cimport (
     bisect_right_i8,
-    tz_convert_utc_to_tzlocal,
+    localize_tzinfo_api,
     tz_localize_to_utc_single,
 )
 
@@ -556,7 +556,7 @@ cdef _TSObject _create_tsobject_tz_using_offset(npy_datetimestruct dts,
     if is_utc(tz):
         pass
     elif is_tzlocal(tz):
-        tz_convert_utc_to_tzlocal(obj.value, tz, &obj.fold)
+        localize_tzinfo_api(obj.value, tz, &obj.fold)
     else:
         trans, deltas, typ = get_dst_info(tz)
 
@@ -725,7 +725,7 @@ cdef inline void _localize_tso(_TSObject obj, tzinfo tz):
     elif obj.value == NPY_NAT:
         pass
     elif is_tzlocal(tz):
-        local_val = tz_convert_utc_to_tzlocal(obj.value, tz, &obj.fold)
+        local_val = obj.value + localize_tzinfo_api(obj.value, tz, &obj.fold)
         dt64_to_dtstruct(local_val, &obj.dts)
     else:
         # Adjust datetime64 timestamp, recompute datetimestruct

pandas/_libs/tslibs/dtypes.pyx

@@ -165,7 +165,7 @@ class FreqGroup(Enum):
     FR_MS = c_FreqGroup.FR_MS
     FR_US = c_FreqGroup.FR_US
     FR_NS = c_FreqGroup.FR_NS
-    FR_UND = -c_FreqGroup.FR_UND  # undefined
+    FR_UND = c_FreqGroup.FR_UND  # undefined
 
     @staticmethod
     def from_period_dtype_code(code: int) -> "FreqGroup":

pandas/_libs/tslibs/fields.pyi

@@ -22,7 +22,7 @@ def get_date_field(
     field: str,
 ) -> npt.NDArray[np.int32]: ...
 def get_timedelta_field(
-    tdindex: np.ndarray,  # const int64_t[:]
+    tdindex: npt.NDArray[np.int64],  # const int64_t[:]
     field: str,
 ) -> npt.NDArray[np.int32]: ...
 def isleapyear_arr(
@@ -31,7 +31,7 @@ def isleapyear_arr(
 def build_isocalendar_sarray(
     dtindex: npt.NDArray[np.int64],  # const int64_t[:]
 ) -> np.ndarray: ...
-def get_locale_names(name_type: str, locale: str | None = ...): ...
+def _get_locale_names(name_type: str, locale: str | None = ...): ...
 
 class RoundTo:
     @property

pandas/_libs/tslibs/fields.pyx

@@ -152,7 +152,7 @@ def get_date_name_field(const int64_t[:] dtindex, str field, object locale=None)
         if locale is None:
             names = np.array(DAYS_FULL, dtype=np.object_)
         else:
-            names = np.array(get_locale_names('f_weekday', locale),
+            names = np.array(_get_locale_names('f_weekday', locale),
                              dtype=np.object_)
         for i in range(count):
             if dtindex[i] == NPY_NAT:
@@ -167,7 +167,7 @@ def get_date_name_field(const int64_t[:] dtindex, str field, object locale=None)
         if locale is None:
             names = np.array(MONTHS_FULL, dtype=np.object_)
         else:
-            names = np.array(get_locale_names('f_month', locale),
+            names = np.array(_get_locale_names('f_month', locale),
                              dtype=np.object_)
         for i in range(count):
             if dtindex[i] == NPY_NAT:
@@ -574,7 +574,7 @@ def build_isocalendar_sarray(const int64_t[:] dtindex):
     return out
 
 
-def get_locale_names(name_type: str, locale: object = None):
+def _get_locale_names(name_type: str, locale: object = None):
     """
     Returns an array of localized day or month names.
 
@@ -650,7 +650,7 @@ class RoundTo:
         return 4
 
 
-cdef inline ndarray[int64_t] _floor_int64(int64_t[:] values, int64_t unit):
+cdef inline ndarray[int64_t] _floor_int64(const int64_t[:] values, int64_t unit):
     cdef:
         Py_ssize_t i, n = len(values)
         ndarray[int64_t] result = np.empty(n, dtype="i8")
@@ -668,7 +668,7 @@ cdef inline ndarray[int64_t] _floor_int64(int64_t[:] values, int64_t unit):
     return result
 
 
-cdef inline ndarray[int64_t] _ceil_int64(int64_t[:] values, int64_t unit):
+cdef inline ndarray[int64_t] _ceil_int64(const int64_t[:] values, int64_t unit):
     cdef:
         Py_ssize_t i, n = len(values)
         ndarray[int64_t] result = np.empty(n, dtype="i8")

pandas/_libs/tslibs/nattype.pxd

@@ -4,7 +4,6 @@ from numpy cimport int64_t
 
 cdef int64_t NPY_NAT
 
-cdef bint _nat_scalar_rules[6]
 cdef set c_nat_strings
 
 cdef class _NaT(datetime):