Merge branch 'master' of https://github.com/pandas-dev/pandas into clean-df-combine-first

Author: arw2019

.pre-commit-config.yaml

@@ -119,6 +119,12 @@ repos:
         entry: python scripts/validate_unwanted_patterns.py --validation-type="private_function_across_module"
         types: [python]
         exclude: ^(asv_bench|pandas/tests|doc)/
+    -   id: inconsistent-namespace-usage
+        name: 'Check for inconsistent use of pandas namespace in tests'
+        entry: python scripts/check_for_inconsistent_pandas_namespace.py
+        language: python
+        types: [python]
+        files: ^pandas/tests/
     -   id: FrameOrSeriesUnion
         name: Check for use of Union[Series, DataFrame] instead of FrameOrSeriesUnion alias
         entry: Union\[.*(Series.*DataFrame|DataFrame.*Series).*\]

ci/code_checks.sh

@@ -37,12 +37,6 @@ function invgrep {
     return $((! $EXIT_STATUS))
 }
 
-function check_namespace {
-    local -r CLASS=${1}
-    grep -R -l --include "*.py" " ${CLASS}(" pandas/tests | xargs grep -n "pd\.${CLASS}[(\.]"
-    test $? -gt 0
-}
-
 if [[ "$GITHUB_ACTIONS" == "true" ]]; then
     FLAKE8_FORMAT="##[error]%(path)s:%(row)s:%(col)s:%(code)s:%(text)s"
     INVGREP_PREPEND="##[error]"
@@ -120,13 +114,6 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     MSG='Check for use of {foo!r} instead of {repr(foo)}' ; echo $MSG
     invgrep -R --include=*.{py,pyx} '!r}' pandas
     RET=$(($RET + $?)) ; echo $MSG "DONE"
-
-    # -------------------------------------------------------------------------
-    MSG='Check for inconsistent use of pandas namespace in tests' ; echo $MSG
-    for class in "Series" "DataFrame" "Index" "MultiIndex" "Timestamp" "Timedelta" "TimedeltaIndex" "DatetimeIndex" "Categorical"; do
-        check_namespace ${class}
-        RET=$(($RET + $?))
-    done
     echo $MSG "DONE"
 fi
 

doc/source/whatsnew/v1.1.5.rst

@@ -24,6 +24,8 @@ Fixed regressions
 Bug fixes
 ~~~~~~~~~
 - Bug in metadata propagation for ``groupby`` iterator (:issue:`37343`)
+- Bug in indexing on a :class:`Series` with ``CategoricalDtype`` after unpickling (:issue:`37631`)
+- Bug in :class:`RollingGroupby` with the resulting :class:`MultiIndex` when grouping by a label that is in the index (:issue:`37641`)
 -
 
 .. ---------------------------------------------------------------------------

doc/source/whatsnew/v1.2.0.rst

@@ -231,6 +231,8 @@ Other enhancements
 - :class:`Window` now supports all Scipy window types in ``win_type`` with flexible keyword argument support (:issue:`34556`)
 - :meth:`testing.assert_index_equal` now has a ``check_order`` parameter that allows indexes to be checked in an order-insensitive manner (:issue:`37478`)
 - :func:`read_csv` supports memory-mapping for compressed files (:issue:`37621`)
+- Improve error reporting for :meth:`DataFrame.merge()` when invalid merge column definitions were given (:issue:`16228`)
+- Improve numerical stability for :meth:`Rolling.skew()`, :meth:`Rolling.kurt()`, :meth:`Expanding.skew()` and :meth:`Expanding.kurt()` through implementation of Kahan summation (:issue:`6929`)
 
 .. _whatsnew_120.api_breaking.python:
 
@@ -380,7 +382,7 @@ Categorical
 ^^^^^^^^^^^
 - :meth:`Categorical.fillna` will always return a copy, will validate a passed fill value regardless of whether there are any NAs to fill, and will disallow a ``NaT`` as a fill value for numeric categories (:issue:`36530`)
 - Bug in :meth:`Categorical.__setitem__` that incorrectly raised when trying to set a tuple value (:issue:`20439`)
--
+- Bug in :meth:`CategoricalIndex.equals` incorrectly casting non-category entries to ``np.nan`` (:issue:`37667`)
 
 Datetimelike
 ^^^^^^^^^^^^
@@ -412,7 +414,7 @@ Timezones
 ^^^^^^^^^
 
 - Bug in :func:`date_range` was raising AmbiguousTimeError for valid input with ``ambiguous=False`` (:issue:`35297`)
--
+- Bug in :meth:`Timestamp.replace` was losing fold information (:issue:`37610`)
 
 
 Numeric
@@ -465,6 +467,8 @@ Indexing
 - Bug in indexing on a :class:`Series` or :class:`DataFrame` with a :class:`MultiIndex` with a level named "0" (:issue:`37194`)
 - Bug in :meth:`Series.__getitem__` when using an unsigned integer array as an indexer giving incorrect results or segfaulting instead of raising ``KeyError`` (:issue:`37218`)
 - Bug in :meth:`Index.where` incorrectly casting numeric values to strings (:issue:`37591`)
+- Bug in :meth:`Series.loc` and :meth:`DataFrame.loc` raises when numeric label was given for object :class:`Index` although label was in :class:`Index` (:issue:`26491`)
+- Bug in :meth:`DataFrame.loc` returned requested key plus missing values when ``loc`` was applied to single level from :class:`MultiIndex` (:issue:`27104`)
 
 Missing
 ^^^^^^^
@@ -510,6 +514,8 @@ Plotting
 - Bug in :meth:`DataFrame.plot` was rotating xticklabels when ``subplots=True``, even if the x-axis wasn't an irregular time series (:issue:`29460`)
 - Bug in :meth:`DataFrame.plot` where a marker letter in the ``style`` keyword sometimes causes a ``ValueError`` (:issue:`21003`)
 - Twinned axes were losing their tick labels which should only happen to all but the last row or column of 'externally' shared axes (:issue:`33819`)
+- Bug in :meth:`DataFrameGroupBy.boxplot` when ``subplots=False``, a KeyError would raise (:issue:`16748`)
+
 
 Groupby/resample/rolling
 ^^^^^^^^^^^^^^^^^^^^^^^^

pandas/_libs/tslibs/nattype.pyx

@@ -774,7 +774,7 @@ default 'raise'
         microsecond : int, optional
         nanosecond : int, optional
         tzinfo : tz-convertible, optional
-        fold : int, optional, default is 0
+        fold : int, optional
 
         Returns
         -------

pandas/_libs/tslibs/period.pyx

@@ -2316,7 +2316,7 @@ class Period(_Period):
     freq : str, default None
         One of pandas period strings or corresponding objects.
     ordinal : int, default None
-        The period offset from the gregorian proleptic epoch.
+        The period offset from the proleptic Gregorian epoch.
     year : int, default None
         Year value of the period.
     month : int, default 1

pandas/_libs/tslibs/timestamps.pyx

@@ -1374,7 +1374,7 @@ default 'raise'
         microsecond=None,
         nanosecond=None,
         tzinfo=object,
-        fold=0,
+        fold=None,
     ):
         """
         implements datetime.replace, handles nanoseconds.
@@ -1390,7 +1390,7 @@ default 'raise'
         microsecond : int, optional
         nanosecond : int, optional
         tzinfo : tz-convertible, optional
-        fold : int, optional, default is 0
+        fold : int, optional
 
         Returns
         -------
@@ -1407,6 +1407,11 @@ default 'raise'
         # set to naive if needed
         tzobj = self.tzinfo
         value = self.value
+
+        # GH 37610. Preserve fold when replacing.
+        if fold is None:
+            fold = self.fold
+
         if tzobj is not None:
             value = tz_convert_from_utc_single(value, tzobj)
 

pandas/_libs/tslibs/tzconversion.pxd

@@ -2,7 +2,9 @@ from cpython.datetime cimport tzinfo
 from numpy cimport int64_t
 
 
-cdef int64_t tz_convert_utc_to_tzlocal(int64_t utc_val, tzinfo tz, bint* fold=*)
+cdef int64_t tz_convert_utc_to_tzlocal(
+    int64_t utc_val, tzinfo tz, bint* fold=*
+) except? -1
 cpdef int64_t tz_convert_from_utc_single(int64_t val, tzinfo tz)
 cdef int64_t tz_localize_to_utc_single(
     int64_t val, tzinfo tz, object ambiguous=*, object nonexistent=*

pandas/_libs/tslibs/tzconversion.pyx

@@ -355,7 +355,9 @@ cdef inline str _render_tstamp(int64_t val):
 # ----------------------------------------------------------------------
 # Timezone Conversion
 
-cdef int64_t tz_convert_utc_to_tzlocal(int64_t utc_val, tzinfo tz, bint* fold=NULL):
+cdef int64_t tz_convert_utc_to_tzlocal(
+    int64_t utc_val, tzinfo tz, bint* fold=NULL
+) except? -1:
     """
     Parameters
     ----------
@@ -500,9 +502,11 @@ cdef int64_t[:] _tz_convert_from_utc(const int64_t[:] vals, tzinfo tz):
     return converted
 
 
+# OSError may be thrown by tzlocal on windows at or close to 1970-01-01
+#  see https://github.com/pandas-dev/pandas/pull/37591#issuecomment-720628241
 cdef inline int64_t _tzlocal_get_offset_components(int64_t val, tzinfo tz,
                                                    bint to_utc,
-                                                   bint *fold=NULL):
+                                                   bint *fold=NULL) except? -1:
     """
     Calculate offset in nanoseconds needed to convert the i8 representation of
     a datetime from a tzlocal timezone to UTC, or vice-versa.
@@ -547,8 +551,10 @@ cdef inline int64_t _tzlocal_get_offset_components(int64_t val, tzinfo tz,
     return int(td.total_seconds() * 1_000_000_000)
 
 
+# OSError may be thrown by tzlocal on windows at or close to 1970-01-01
+#  see https://github.com/pandas-dev/pandas/pull/37591#issuecomment-720628241
 cdef int64_t _tz_convert_tzlocal_utc(int64_t val, tzinfo tz, bint to_utc=True,
-                                     bint* fold=NULL):
+                                     bint* fold=NULL) except? -1:
     """
     Convert the i8 representation of a datetime from a tzlocal timezone to
     UTC, or vice-versa.