Merge remote-tracking branch 'upstream/master' into expressions-dtypes-check

Author: jorisvandenbossche

.github/workflows/ci.yml

@@ -169,6 +169,7 @@ jobs:
         pytest pandas/tests/indexing/multiindex/test_setitem.py::TestMultiIndexSetItem::test_frame_setitem_multi_column
 
         pytest pandas/tests/api/
+        pytest pandas/tests/apply/
         pytest pandas/tests/arrays/
         pytest pandas/tests/base/
         pytest pandas/tests/computation/

Makefile

@@ -1,4 +1,4 @@
-.PHONY : develop build clean clean_pyc doc lint-diff black
+.PHONY : develop build clean clean_pyc doc lint-diff black test-scripts
 
 all: develop
 
@@ -25,3 +25,6 @@ doc:
 	cd doc; \
 	python make.py clean; \
 	python make.py html
+
+test-scripts:
+	pytest scripts

doc/source/whatsnew/v1.3.0.rst

@@ -225,6 +225,41 @@ In pandas 1.3.0, ``df`` continues to share data with ``values``
    np.shares_memory(df["A"], values)
 
 
+.. _whatsnew_130.notable_bug_fixes.setitem_never_inplace:
+
+Never Operate Inplace When Setting ``frame[keys] = values``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When setting multiple columns using ``frame[keys] = values`` new arrays will
+replace pre-existing arrays for these keys, which will *not* be over-written
+(:issue:`39510`).  As a result, the columns will retain the dtype(s) of ``values``,
+never casting to the dtypes of the existing arrays.
+
+.. ipython:: python
+
+   df = pd.DataFrame(range(3), columns=["A"], dtype="float64")
+   df[["A"]] = 5
+
+In the old behavior, ``5`` was cast to ``float64`` and inserted into the existing
+array backing ``df``:
+
+*pandas 1.2.x*
+
+.. code-block:: ipython
+
+   In [1]: df.dtypes
+   Out[1]:
+   A    float64
+
+In the new behavior, we get a new array, and retain an integer-dtyped ``5``:
+
+*pandas 1.3.0*
+
+.. ipython:: python
+
+   df.dtypes
+
+
 .. _whatsnew_130.notable_bug_fixes.setitem_with_bool_casting:
 
 Consistent Casting With Setting Into Boolean Series
@@ -300,7 +335,7 @@ Optional libraries below the lowest tested version may still work, but are not c
 +=================+=================+=========+
 | beautifulsoup4  | 4.6.0           |         |
 +-----------------+-----------------+---------+
-| fastparquet     | 0.3.2           |         |
+| fastparquet     | 0.4.0           |    X    |
 +-----------------+-----------------+---------+
 | fsspec          | 0.7.4           |         |
 +-----------------+-----------------+---------+
@@ -627,6 +662,7 @@ Other
 - Bug in :class:`Styler` where multiple elements in CSS-selectors were not correctly added to ``table_styles`` (:issue:`39942`)
 - Bug in :meth:`DataFrame.equals`, :meth:`Series.equals`, :meth:`Index.equals` with object-dtype containing ``np.datetime64("NaT")`` or ``np.timedelta64("NaT")`` (:issue:`39650`)
 - Bug in :func:`pandas.util.show_versions` where console JSON output was not proper JSON (:issue:`39701`)
+- Bug in :meth:`DataFrame.convert_dtypes` incorrectly raised ValueError when called on an empty DataFrame (:issue:`40393`)
 
 
 .. ---------------------------------------------------------------------------

pandas/_libs/index.pyx

@@ -1,5 +1,7 @@
 import warnings
 
+cimport cython
+
 import numpy as np
 
 cimport numpy as cnp
@@ -47,6 +49,7 @@ cdef inline bint is_definitely_invalid_key(object val):
 _SIZE_CUTOFF = 1_000_000
 
 
+@cython.freelist(32)
 cdef class IndexEngine:
 
     cdef readonly:

pandas/_libs/missing.pyx

@@ -104,6 +104,7 @@ cpdef bint checknull(object val):
      - np.datetime64 representation of NaT
      - np.timedelta64 representation of NaT
      - NA
+     - Decimal("NaN")
 
     Parameters
     ----------
@@ -143,6 +144,8 @@ cpdef bint checknull_old(object val):
      - NaT
      - np.datetime64 representation of NaT
      - np.timedelta64 representation of NaT
+     - NA
+     - Decimal("NaN")
 
     Parameters
     ----------
@@ -175,6 +178,8 @@ cpdef ndarray[uint8_t] isnaobj(ndarray arr):
      - NaT
      - np.datetime64 representation of NaT
      - np.timedelta64 representation of NaT
+     - NA
+     - Decimal("NaN")
 
     Parameters
     ----------
@@ -211,6 +216,7 @@ def isnaobj_old(arr: ndarray) -> ndarray:
      - NEGINF
      - NaT
      - NA
+     - Decimal("NaN")
 
     Parameters
     ----------
@@ -249,6 +255,8 @@ def isnaobj2d(arr: ndarray) -> ndarray:
      - NaT
      - np.datetime64 representation of NaT
      - np.timedelta64 representation of NaT
+     - NA
+     - Decimal("NaN")
 
     Parameters
     ----------
@@ -293,6 +301,8 @@ def isnaobj2d_old(arr: ndarray) -> ndarray:
      - NaT
      - np.datetime64 representation of NaT
      - np.timedelta64 representation of NaT
+     - NA
+     - Decimal("NaN")
 
     Parameters
     ----------

pandas/_libs/tslibs/ccalendar.pyi

@@ -0,0 +1,13 @@
+
+DAYS: list[str]
+MONTH_ALIASES: dict[int, str]
+MONTH_NUMBERS: dict[str, int]
+MONTHS: list[str]
+int_to_weekday: dict[int, str]
+
+def get_firstbday(year: int, month: int) -> int: ...
+def get_lastbday(year: int, month: int) -> int: ...
+def get_day_of_year(year: int, month: int, day: int) -> int: ...
+def get_iso_calendar(year: int, month: int, day: int) -> tuple[int, int, int]: ...
+def get_week_of_year(year: int, month: int, day: int) -> int: ...
+def get_days_in_month(year: int, month: int) -> int: ...

pandas/_libs/tslibs/strptime.pyi

@@ -0,0 +1,11 @@
+from typing import Optional
+
+import numpy as np
+
+def array_strptime(
+    values: np.ndarray,  # np.ndarray[object]
+    fmt: Optional[str],
+    exact: bool = True,
+    errors: str = "raise"
+) -> tuple[np.ndarray, np.ndarray]: ...
+# first  ndarray is M8[ns], second is object ndarray of Optional[tzinfo]

pandas/_libs/tslibs/timezones.pyi

@@ -0,0 +1,32 @@
+from datetime import (
+    datetime,
+    tzinfo,
+)
+from typing import (
+    Callable,
+    Optional,
+    Union,
+)
+
+import numpy as np
+
+# imported from dateutil.tz
+dateutil_gettz: Callable[[str], tzinfo]
+
+
+def tz_standardize(tz: tzinfo) -> tzinfo: ...
+
+def tz_compare(start: Optional[tzinfo], end: Optional[tzinfo]) -> bool: ...
+
+def infer_tzinfo(
+    start: Optional[datetime], end: Optional[datetime],
+) -> Optional[tzinfo]: ...
+
+# ndarrays returned are both int64_t
+def get_dst_info(tz: tzinfo) -> tuple[np.ndarray, np.ndarray, str]: ...
+
+def maybe_get_tz(tz: Optional[Union[str, int, np.int64, tzinfo]]) -> Optional[tzinfo]: ...
+
+def get_timezone(tz: tzinfo) -> Union[tzinfo, str]: ...
+
+def is_utc(tz: Optional[tzinfo]) -> bool: ...

pandas/_libs/tslibs/timezones.pyx

@@ -67,6 +67,7 @@ cdef inline bint treat_tz_as_dateutil(tzinfo tz):
     return hasattr(tz, '_trans_list') and hasattr(tz, '_trans_idx')
 
 
+# Returns str or tzinfo object
 cpdef inline object get_timezone(tzinfo tz):
     """
     We need to do several things here:
@@ -80,6 +81,8 @@ cpdef inline object get_timezone(tzinfo tz):
     the tz name. It needs to be a string so that we can serialize it with
     UJSON/pytables. maybe_get_tz (below) is the inverse of this process.
     """
+    if tz is None:
+        raise TypeError("tz argument cannot be None")
     if is_utc(tz):
         return tz
     else:
@@ -364,6 +367,8 @@ cpdef bint tz_compare(tzinfo start, tzinfo end):
     elif is_utc(end):
         # Ensure we don't treat tzlocal as equal to UTC when running in UTC
         return False
+    elif start is None or end is None:
+        return start is None and end is None
     return get_timezone(start) == get_timezone(end)
 
 

pandas/_libs/tslibs/tzconversion.pyi

@@ -0,0 +1,25 @@
+from datetime import (
+    timedelta,
+    tzinfo,
+)
+from typing import (
+    Iterable,
+    Optional,
+    Union,
+)
+
+import numpy as np
+
+def tz_convert_from_utc(
+    vals: np.ndarray,  # const int64_t[:]
+    tz: tzinfo,
+) -> np.ndarray: ...  # np.ndarray[np.int64]
+
+def tz_convert_from_utc_single(val: np.int64, tz: tzinfo) -> np.int64: ...
+
+def tz_localize_to_utc(
+    vals: np.ndarray,  # np.ndarray[np.int64]
+    tz: Optional[tzinfo],
+    ambiguous: Optional[Union[str, bool, Iterable[bool]]] = None,
+    nonexistent: Optional[Union[str, timedelta, np.timedelta64]] = None,
+) -> np.ndarray: ...  # np.ndarray[np.int64]

pandas/_libs/tslibs/vectorized.pyi

@@ -0,0 +1,47 @@
+"""
+For cython types that cannot be represented precisely, closest-available
+python equivalents are used, and the precise types kept as adjacent comments.
+"""
+from datetime import tzinfo
+from typing import (
+    Optional,
+    Union,
+)
+
+import numpy as np
+
+from pandas._libs.tslibs.dtypes import Resolution
+from pandas._libs.tslibs.offsets import BaseOffset
+
+def dt64arr_to_periodarr(
+    stamps: np.ndarray,  # const int64_t[:]
+    freq: int,
+    tz: Optional[tzinfo],
+) -> np.ndarray: ...  # np.ndarray[np.int64, ndim=1]
+
+
+def is_date_array_normalized(
+    stamps: np.ndarray,  # const int64_t[:]
+    tz: Optional[tzinfo] = None,
+) -> bool: ...
+
+
+def normalize_i8_timestamps(
+    stamps: np.ndarray,  # const int64_t[:]
+    tz: Optional[tzinfo],
+) -> np.ndarray: ...  # np.ndarray[np.int64]
+
+
+def get_resolution(
+    stamps: np.ndarray,  # const int64_t[:]
+    tz: Optional[tzinfo] = None,
+) -> Resolution: ...
+
+
+def ints_to_pydatetime(
+    arr: np.ndarray,  # const int64_t[:}]
+    tz: Optional[tzinfo] = None,
+    freq: Optional[Union[str, BaseOffset]] = None,
+    fold: bool = False,
+    box: str = "datetime",
+) -> np.ndarray: ...  # np.ndarray[object]

pandas/_testing/_io.py

@@ -93,9 +93,7 @@ def dec(f):
         is_decorating = not kwargs and len(args) == 1 and callable(args[0])
         if is_decorating:
             f = args[0]
-            # error: Incompatible types in assignment (expression has type
-            # "List[<nothing>]", variable has type "Tuple[Any, ...]")
-            args = []  # type: ignore[assignment]
+            args = ()
             return dec(f)
         else:
             return dec

pandas/compat/_optional.py

@@ -11,7 +11,7 @@
     "bs4": "4.6.0",
     "bottleneck": "1.2.1",
     "fsspec": "0.7.4",
-    "fastparquet": "0.3.2",
+    "fastparquet": "0.4.0",
     "gcsfs": "0.6.0",
     "lxml.etree": "4.3.0",
     "matplotlib": "2.2.3",

pandas/compat/numpy/__init__.py

@@ -27,41 +27,39 @@
 _tz_regex = re.compile("[+-]0000$")
 
 
-def tz_replacer(s):
-    if isinstance(s, str):
-        if s.endswith("Z"):
-            s = s[:-1]
-        elif _tz_regex.search(s):
-            s = s[:-5]
-    return s
+def _tz_replacer(tstring):
+    if isinstance(tstring, str):
+        if tstring.endswith("Z"):
+            tstring = tstring[:-1]
+        elif _tz_regex.search(tstring):
+            tstring = tstring[:-5]
+    return tstring
 
 
-def np_datetime64_compat(s, *args, **kwargs):
+def np_datetime64_compat(tstring: str, unit: str = "ns"):
     """
     provide compat for construction of strings to numpy datetime64's with
     tz-changes in 1.11 that make '2015-01-01 09:00:00Z' show a deprecation
     warning, when need to pass '2015-01-01 09:00:00'
     """
-    s = tz_replacer(s)
-    # error: No overload variant of "datetime64" matches argument types "Any",
-    # "Tuple[Any, ...]", "Dict[str, Any]"
-    return np.datetime64(s, *args, **kwargs)  # type: ignore[call-overload]
+    tstring = _tz_replacer(tstring)
+    return np.datetime64(tstring, unit)
 
 
-def np_array_datetime64_compat(arr, *args, **kwargs):
+def np_array_datetime64_compat(arr, dtype="M8[ns]"):
     """
     provide compat for construction of an array of strings to a
     np.array(..., dtype=np.datetime64(..))
     tz-changes in 1.11 that make '2015-01-01 09:00:00Z' show a deprecation
     warning, when need to pass '2015-01-01 09:00:00'
     """
-    # is_list_like
+    # is_list_like; can't import as it would be circular
     if hasattr(arr, "__iter__") and not isinstance(arr, (str, bytes)):
-        arr = [tz_replacer(s) for s in arr]
+        arr = [_tz_replacer(s) for s in arr]
     else:
-        arr = tz_replacer(arr)
+        arr = _tz_replacer(arr)
 
-    return np.array(arr, *args, **kwargs)
+    return np.array(arr, dtype=dtype)
 
 
 __all__ = [

pandas/compat/pickle_compat.py

@@ -49,7 +49,7 @@ def load_reduce(self):
                 return
             except TypeError:
                 pass
-        elif args and issubclass(args[0], BaseOffset):
+        elif args and isinstance(args[0], type) and issubclass(args[0], BaseOffset):
             # TypeError: object.__new__(Day) is not safe, use Day.__new__()
             cls = args[0]
             stack[-1] = cls.__new__(*args)