Merge remote-tracking branch 'upstream/master' into typ_c_parser

Author: phofl

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -7,6 +7,7 @@ body:
   - type: checkboxes
     id: checks
     attributes:
+      label: Pandas version checks
       options:
         - label: >
             I have checked that this issue has not already been reported.

.github/ISSUE_TEMPLATE/documentation_improvement.yaml

@@ -6,6 +6,7 @@ labels: [Docs, Needs Triage]
 body:
   - type: checkboxes
     attributes:
+      label: Pandas version checks
       options:
         - label: >
             I have checked that the issue still exists on the latest versions of the docs

.github/ISSUE_TEMPLATE/installation_issue.yaml

@@ -7,6 +7,7 @@ body:
   - type: checkboxes
     id: checks
     attributes:
+      label: Installation check
       options:
         - label: >
             I have read the [installation guide](https://pandas.pydata.org/pandas-docs/stable/getting_started/install.html#installing-pandas).

.github/ISSUE_TEMPLATE/performance_issue.yaml

@@ -7,6 +7,7 @@ body:
   - type: checkboxes
     id: checks
     attributes:
+      label: Pandas version checks
       options:
         - label: >
             I have checked that this issue has not already been reported.

.github/ISSUE_TEMPLATE/submit_question.yml

@@ -11,6 +11,7 @@ body:
         usage questions, we ask that all usage questions are first asked on StackOverflow.
   - type: checkboxes
     attributes:
+      label: Research
       options:
         - label: >
             I have searched the [[pandas] tag](https://stackoverflow.com/questions/tagged/pandas)

asv_bench/benchmarks/arithmetic.py

@@ -144,7 +144,7 @@ def setup(self, op, shape):
         # should already be the case, but just to be sure
         df._consolidate_inplace()
 
-        # TODO: GH#33198 the setting here shoudlnt need two steps
+        # TODO: GH#33198 the setting here shouldn't need two steps
         arr1 = np.random.randn(n_rows, max(n_cols // 4, 3)).astype("f8")
         arr2 = np.random.randn(n_rows, n_cols // 2).astype("i8")
         arr3 = np.random.randn(n_rows, n_cols // 4).astype("f8")

asv_bench/benchmarks/io/csv.py

@@ -67,6 +67,18 @@ def time_frame_date_formatting(self):
         self.data.to_csv(self.fname, date_format="%Y%m%d")
 
 
+class ToCSVDatetimeIndex(BaseIO):
+
+    fname = "__test__.csv"
+
+    def setup(self):
+        rng = date_range("2000", periods=100_000, freq="S")
+        self.data = DataFrame({"a": 1}, index=rng)
+
+    def time_frame_date_formatting_index(self):
+        self.data.to_csv(self.fname, date_format="%Y-%m-%d %H:%M:%S")
+
+
 class ToCSVDatetimeBig(BaseIO):
 
     fname = "__test__.csv"

ci/deps/actions-38-db.yaml

@@ -12,7 +12,7 @@ dependencies:
   - pytest-cov>=2.10.1  # this is only needed in the coverage build, ref: GH 35737
 
   # pandas dependencies
-  - aiobotocore<2.0.0
+  - aiobotocore<2.0.0  # GH#44311 pinned to fix docbuild
   - beautifulsoup4
   - boto3
   - botocore>=1.11

ci/run_tests.sh

@@ -5,6 +5,9 @@
 # https://github.com/pytest-dev/pytest/issues/1075
 export PYTHONHASHSEED=$(python -c 'import random; print(random.randint(1, 4294967295))')
 
+# May help reproduce flaky CI builds if set in subsequent runs
+echo PYTHONHASHSEED=$PYTHONHASHSEED
+
 if [[ "not network" == *"$PATTERN"* ]]; then
     export http_proxy=http://1.2.3.4 https_proxy=http://1.2.3.4;
 fi

doc/source/whatsnew/v1.4.0.rst

@@ -537,7 +537,9 @@ Other Deprecations
 - Deprecated passing arguments as positional for :func:`read_fwf` other than ``filepath_or_buffer`` (:issue:`41485`):
 - Deprecated passing ``skipna=None`` for :meth:`DataFrame.mad` and :meth:`Series.mad`, pass ``skipna=True`` instead (:issue:`44580`)
 - Deprecated :meth:`DateOffset.apply`, use ``offset + other`` instead (:issue:`44522`)
+- Deprecated parameter ``names`` in :meth:`Index.copy` (:issue:`44916`)
 - A deprecation warning is now shown for :meth:`DataFrame.to_latex` indicating the arguments signature may change and emulate more the arguments to :meth:`.Styler.to_latex` in future versions (:issue:`44411`)
+- Deprecated :meth:`Categorical.replace`, use :meth:`Series.replace` instead (:issue:`44929`)
 -
 
 .. ---------------------------------------------------------------------------
@@ -589,6 +591,7 @@ Performance improvements
 - Performance improvement in :meth:`Series.to_frame` (:issue:`43558`)
 - Performance improvement in :meth:`Series.mad` (:issue:`43010`)
 - Performance improvement in :func:`merge` (:issue:`43332`)
+- Performance improvement in :func:`to_csv` when index column is a datetime and is formatted (:issue:`39413`)
 - Performance improvement in :func:`read_csv` when ``index_col`` was set with a numeric column (:issue:`44158`)
 - Performance improvement in :func:`concat` (:issue:`43354`)
 -
@@ -660,7 +663,7 @@ Conversion
 
 Strings
 ^^^^^^^
-- Fixed bug in checking for ``string[pyarrow]`` dtype incorrectly raising an ImportError when pyarrow is not installed (:issue:`44327`)
+- Fixed bug in checking for ``string[pyarrow]`` dtype incorrectly raising an ImportError when pyarrow is not installed (:issue:`44276`)
 -
 
 Interval
@@ -749,6 +752,7 @@ I/O
 - Bug in :func:`read_csv` raising ``ValueError`` when names was longer than header but equal to data rows for ``engine="python"`` (:issue:`38453`)
 - Bug in :class:`ExcelWriter`, where ``engine_kwargs`` were not passed through to all engines (:issue:`43442`)
 - Bug in :func:`read_csv` raising ``ValueError`` when ``parse_dates`` was used with ``MultiIndex`` columns (:issue:`8991`)
+- Bug in :func:`read_csv` not raising an ``ValueError`` when ``\n`` was specified as ``delimiter`` or ``sep`` which conflicts with ``lineterminator`` (:issue:`43528`)
 - Bug in :func:`read_csv` converting columns to numeric after date parsing failed (:issue:`11019`)
 - Bug in :func:`read_csv` not replacing ``NaN`` values with ``np.nan`` before attempting date conversion (:issue:`26203`)
 - Bug in :func:`read_csv` raising ``AttributeError`` when attempting to read a .csv file and infer index column dtype from an nullable integer type (:issue:`44079`)
@@ -806,6 +810,7 @@ Reshaping
 - Bug in :func:`crosstab` would fail when inputs are lists or tuples (:issue:`44076`)
 - Bug in :meth:`DataFrame.append` failing to retain ``index.name`` when appending a list of :class:`Series` objects (:issue:`44109`)
 - Fixed metadata propagation in :meth:`Dataframe.apply` method, consequently fixing the same issue for :meth:`Dataframe.transform`, :meth:`Dataframe.nunique` and :meth:`Dataframe.mode` (:issue:`28283`)
+- Bug in :func:`concat` casting levels of :class:`MultiIndex` to float if the only consist of missing values (:issue:`44900`)
 - Bug in :meth:`DataFrame.stack` with ``ExtensionDtype`` columns incorrectly raising (:issue:`43561`)
 - Bug in :meth:`Series.unstack` with object doing unwanted type inference on resulting columns (:issue:`44595`)
 - Bug in :class:`MultiIndex` failing join operations with overlapping ``IntervalIndex`` levels (:issue:`44096`)
@@ -856,6 +861,7 @@ Other
 - Bug in :meth:`DataFrame.shift` with ``axis=1`` and ``ExtensionDtype`` columns incorrectly raising when an incompatible ``fill_value`` is passed (:issue:`44564`)
 - Bug in :meth:`DataFrame.diff` when passing a NumPy integer object instead of an ``int`` object (:issue:`44572`)
 - Bug in :meth:`Series.replace` raising ``ValueError`` when using ``regex=True`` with a :class:`Series` containing ``np.nan`` values (:issue:`43344`)
+- Bug in :meth:`DataFrame.to_records` where an incorrect ``n`` was used when missing names were replaced by ``level_n`` (:issue:`44818`)
 
 .. ***DO NOT USE THIS SECTION***
 

environment.yml

@@ -105,7 +105,7 @@ dependencies:
 
   - pytables>=3.6.1  # pandas.read_hdf, DataFrame.to_hdf
   - s3fs>=0.4.0  # file IO when using 's3://...' path
-  - aiobotocore<2.0.0
+  - aiobotocore<2.0.0  # GH#44311 pinned to fix docbuild
   - fsspec>=0.7.4  # for generic remote file operations
   - gcsfs>=0.6.0  # file IO when using 'gcs://...' path
   - sqlalchemy  # pandas.read_sql, DataFrame.to_sql

pandas/_libs/parsers.pyx

@@ -558,18 +558,11 @@ cdef class TextReader:
         pass
 
     def __dealloc__(self):
-        self.close()
+        _close(self)
         parser_del(self.parser)
 
-    def close(self) -> None:
-        # also preemptively free all allocated memory
-        parser_free(self.parser)
-        if self.true_set:
-            kh_destroy_str_starts(self.true_set)
-            self.true_set = NULL
-        if self.false_set:
-            kh_destroy_str_starts(self.false_set)
-            self.false_set = NULL
+    def close(self):
+        _close(self)
 
     def _set_quoting(self, quote_char: str | bytes | None, quoting: int):
         if not isinstance(quoting, int):
@@ -1292,6 +1285,21 @@ cdef class TextReader:
                 return None
 
 
+# Factor out code common to TextReader.__dealloc__ and TextReader.close
+# It cannot be a class method, since calling self.close() in __dealloc__
+# which causes a class attribute lookup and violates best parctices
+# https://cython.readthedocs.io/en/latest/src/userguide/special_methods.html#finalization-method-dealloc
+cdef _close(TextReader reader):
+    # also preemptively free all allocated memory
+    parser_free(reader.parser)
+    if reader.true_set:
+        kh_destroy_str_starts(reader.true_set)
+        reader.true_set = NULL
+    if reader.false_set:
+        kh_destroy_str_starts(reader.false_set)
+        reader.false_set = NULL
+
+
 cdef:
     object _true_values = [b'True', b'TRUE', b'true']
     object _false_values = [b'False', b'FALSE', b'false']

pandas/_testing/__init__.py

@@ -28,17 +28,12 @@
 from pandas._typing import Dtype
 
 from pandas.core.dtypes.common import (
-    is_datetime64_dtype,
-    is_datetime64tz_dtype,
     is_float_dtype,
     is_integer_dtype,
-    is_period_dtype,
     is_sequence,
-    is_timedelta64_dtype,
     is_unsigned_integer_dtype,
     pandas_dtype,
 )
-from pandas.core.dtypes.dtypes import IntervalDtype
 
 import pandas as pd
 from pandas import (
@@ -112,14 +107,11 @@
 )
 from pandas.core.arrays import (
     BaseMaskedArray,
-    DatetimeArray,
     ExtensionArray,
     PandasArray,
-    PeriodArray,
-    TimedeltaArray,
-    period_array,
 )
 from pandas.core.arrays._mixins import NDArrayBackedExtensionArray
+from pandas.core.construction import extract_array
 
 if TYPE_CHECKING:
     from pandas import (
@@ -161,6 +153,17 @@
     + BYTES_DTYPES
 )
 
+NARROW_NP_DTYPES = [
+    np.float16,
+    np.float32,
+    np.int8,
+    np.int16,
+    np.int32,
+    np.uint8,
+    np.uint16,
+    np.uint32,
+]
+
 NULL_OBJECTS = [None, np.nan, pd.NaT, float("nan"), pd.NA, Decimal("NaN")]
 NP_NAT_OBJECTS = [
     cls("NaT", unit)
@@ -257,13 +260,6 @@ def box_expected(expected, box_cls, transpose=True):
             #  single-row special cases in datetime arithmetic
             expected = expected.T
             expected = pd.concat([expected] * 2, ignore_index=True)
-    elif box_cls is PeriodArray:
-        # the PeriodArray constructor is not as flexible as period_array
-        expected = period_array(expected)
-    elif box_cls is DatetimeArray:
-        expected = DatetimeArray(expected)
-    elif box_cls is TimedeltaArray:
-        expected = TimedeltaArray(expected)
     elif box_cls is np.ndarray or box_cls is np.array:
         expected = np.array(expected)
     elif box_cls is to_array:
@@ -274,21 +270,16 @@ def box_expected(expected, box_cls, transpose=True):
 
 
 def to_array(obj):
+    """
+    Similar to pd.array, but does not cast numpy dtypes to nullable dtypes.
+    """
     # temporary implementation until we get pd.array in place
     dtype = getattr(obj, "dtype", None)
 
-    if is_period_dtype(dtype):
-        return period_array(obj)
-    elif is_datetime64_dtype(dtype) or is_datetime64tz_dtype(dtype):
-        return DatetimeArray._from_sequence(obj)
-    elif is_timedelta64_dtype(dtype):
-        return TimedeltaArray._from_sequence(obj)
-    elif isinstance(obj, pd.core.arrays.BooleanArray):
-        return obj
-    elif isinstance(dtype, IntervalDtype):
-        return pd.core.arrays.IntervalArray(obj)
-    else:
-        return np.array(obj)
+    if dtype is None:
+        return np.asarray(obj)
+
+    return extract_array(obj, extract_numpy=True)
 
 
 # -----------------------------------------------------------------------------

pandas/compat/__init__.py

@@ -13,8 +13,6 @@
 from pandas._typing import F
 from pandas.compat.numpy import (
     is_numpy_dev,
-    np_array_datetime64_compat,
-    np_datetime64_compat,
     np_version_under1p19,
     np_version_under1p20,
 )
@@ -130,8 +128,6 @@ def get_lzma_file():
 
 __all__ = [
     "is_numpy_dev",
-    "np_array_datetime64_compat",
-    "np_datetime64_compat",
     "np_version_under1p19",
     "np_version_under1p20",
     "pa_version_under1p01",

pandas/compat/numpy/__init__.py

@@ -1,7 +1,4 @@
 """ support numpy compatibility across versions """
-
-import re
-
 import numpy as np
 
 from pandas.util.version import Version
@@ -29,44 +26,6 @@
     )
 
 
-_tz_regex = re.compile("[+-]0000$")
-
-
-def _tz_replacer(tstring):
-    if isinstance(tstring, str):
-        if tstring.endswith("Z"):
-            tstring = tstring[:-1]
-        elif _tz_regex.search(tstring):
-            tstring = tstring[:-5]
-    return tstring
-
-
-def np_datetime64_compat(tstring: str, unit: str = "ns"):
-    """
-    provide compat for construction of strings to numpy datetime64's with
-    tz-changes in 1.11 that make '2015-01-01 09:00:00Z' show a deprecation
-    warning, when need to pass '2015-01-01 09:00:00'
-    """
-    tstring = _tz_replacer(tstring)
-    return np.datetime64(tstring, unit)
-
-
-def np_array_datetime64_compat(arr, dtype="M8[ns]"):
-    """
-    provide compat for construction of an array of strings to a
-    np.array(..., dtype=np.datetime64(..))
-    tz-changes in 1.11 that make '2015-01-01 09:00:00Z' show a deprecation
-    warning, when need to pass '2015-01-01 09:00:00'
-    """
-    # is_list_like; can't import as it would be circular
-    if hasattr(arr, "__iter__") and not isinstance(arr, (str, bytes)):
-        arr = [_tz_replacer(s) for s in arr]
-    else:
-        arr = _tz_replacer(arr)
-
-    return np.array(arr, dtype=dtype)
-
-
 __all__ = [
     "np",
     "_np_version",

pandas/compat/pickle_compat.py

@@ -35,9 +35,6 @@ def load_reduce(self):
     args = stack.pop()
     func = stack[-1]
 
-    if len(args) and type(args[0]) is type:
-        n = args[0].__name__  # noqa
-
     try:
         stack[-1] = func(*args)
         return