Merge remote-tracking branch 'upstream/main' into bisect

Author: simonjayhawkins

doc/source/reference/testing.rst

@@ -35,6 +35,7 @@ Exceptions and warnings
    errors.MergeError
    errors.NullFrequencyError
    errors.NumbaUtilError
+   errors.NumExprClobberingError
    errors.OptionError
    errors.OutOfBoundsDatetime
    errors.OutOfBoundsTimedelta

doc/source/whatsnew/v1.5.0.rst

@@ -151,7 +151,8 @@ Other enhancements
 - A :class:`errors.PerformanceWarning` is now thrown when using ``string[pyarrow]`` dtype with methods that don't dispatch to ``pyarrow.compute`` methods (:issue:`42613`)
 - Added ``numeric_only`` argument to :meth:`Resampler.sum`, :meth:`Resampler.prod`, :meth:`Resampler.min`, :meth:`Resampler.max`, :meth:`Resampler.first`, and :meth:`Resampler.last` (:issue:`46442`)
 - ``times`` argument in :class:`.ExponentialMovingWindow` now accepts ``np.timedelta64`` (:issue:`47003`)
-- :class:`DataError`, :class:`SpecificationError`, :class:`SettingWithCopyError`, and :class:`SettingWithCopyWarning` are now exposed in ``pandas.errors`` (:issue:`27656`)
+- :class:`DataError`, :class:`SpecificationError`, :class:`SettingWithCopyError`, :class:`SettingWithCopyWarning`, and :class:`NumExprClobberingError` are now exposed in ``pandas.errors`` (:issue:`27656`)
+- Added ``check_like`` argument to :func:`testing.assert_series_equal` (:issue:`47247`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_150.notable_bug_fixes:
@@ -434,6 +435,8 @@ Other API changes
   <https://developers.googleblog.com/2022/02/making-oauth-flows-safer.html?m=1#disallowed-oob>`_.
   The ``auth_local_webserver = False`` option is planned to stop working in
   October 2022. (:issue:`46312`)
+- :func:`read_json` now raises ``FileNotFoundError`` (previously ``ValueError``) when input is a string ending in ``.json``, ``.json.gz``, ``.json.bz2``, etc. but no such file exists. (:issue:`29102`)
+- Operations with :class:`Timestamp` or :class:`Timedelta` that would previously raise ``OverflowError`` instead raise ``OutOfBoundsDatetime`` or ``OutOfBoundsTimedelta`` where appropriate (:issue:`47268`)
 -
 
 .. ---------------------------------------------------------------------------
@@ -682,6 +685,8 @@ Other Deprecations
 - Deprecated the ``closed`` argument in :class:`IntervalArray` in favor of ``inclusive`` argument; In a future version passing ``closed`` will raise (:issue:`40245`)
 - Deprecated the ``closed`` argument in :class:`intervaltree` in favor of ``inclusive`` argument; In a future version passing ``closed`` will raise (:issue:`40245`)
 - Deprecated the ``closed`` argument in :class:`ArrowInterval` in favor of ``inclusive`` argument; In a future version passing ``closed`` will raise (:issue:`40245`)
+- Deprecated allowing ``unit="M"`` or ``unit="Y"`` in :class:`Timestamp` constructor with a non-round float value (:issue:`47267`)
+-
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_150.performance:
@@ -726,12 +731,14 @@ Datetimelike
 - Bug in :meth:`SeriesGroupBy.value_counts` index when passing categorical column (:issue:`44324`)
 - Bug in :meth:`DatetimeIndex.tz_localize` localizing to UTC failing to make a copy of the underlying data (:issue:`46460`)
 - Bug in :meth:`DatetimeIndex.resolution` incorrectly returning "day" instead of "nanosecond" for nanosecond-resolution indexes (:issue:`46903`)
+- Bug in :class:`Timestamp` with an integer or float value and ``unit="Y"`` or ``unit="M"`` giving slightly-wrong results (:issue:`47266`)
 -
 
 Timedelta
 ^^^^^^^^^
 - Bug in :func:`astype_nansafe` astype("timedelta64[ns]") fails when np.nan is included (:issue:`45798`)
 - Bug in constructing a :class:`Timedelta` with a ``np.timedelta64`` object and a ``unit`` sometimes silently overflowing and returning incorrect results instead of raising ``OutOfBoundsTimedelta`` (:issue:`46827`)
+- Bug in constructing a :class:`Timedelta` from a large integer or float with ``unit="W"`` silently overflowing and returning incorrect results instead of raising ``OutOfBoundsTimedelta`` (:issue:`47268`)
 -
 
 Time Zones
@@ -765,7 +772,7 @@ Strings
 Interval
 ^^^^^^^^
 - Bug in :meth:`IntervalArray.__setitem__` when setting ``np.nan`` into an integer-backed array raising ``ValueError`` instead of ``TypeError`` (:issue:`45484`)
--
+- Bug in :class:`IntervalDtype` when using datetime64[ns, tz] as a dtype string (:issue:`46999`)
 
 Indexing
 ^^^^^^^^

pandas/_libs/tslibs/conversion.pyx

@@ -1,5 +1,7 @@
 cimport cython
 
+import warnings
+
 import numpy as np
 
 cimport numpy as cnp
@@ -248,13 +250,35 @@ cdef _TSObject convert_to_tsobject(object ts, tzinfo tz, str unit,
         if ts == NPY_NAT:
             obj.value = NPY_NAT
         else:
+            if unit in ["Y", "M"]:
+                # GH#47266 cast_from_unit leads to weird results e.g. with "Y"
+                #  and 150 we'd get 2120-01-01 09:00:00
+                ts = np.datetime64(ts, unit)
+                return convert_to_tsobject(ts, tz, None, False, False)
+
             ts = ts * cast_from_unit(None, unit)
             obj.value = ts
             dt64_to_dtstruct(ts, &obj.dts)
     elif is_float_object(ts):
         if ts != ts or ts == NPY_NAT:
             obj.value = NPY_NAT
         else:
+            if unit in ["Y", "M"]:
+                if ts == int(ts):
+                    # GH#47266 Avoid cast_from_unit, which would give weird results
+                    #  e.g. with "Y" and 150.0 we'd get 2120-01-01 09:00:00
+                    return convert_to_tsobject(int(ts), tz, unit, False, False)
+                else:
+                    # GH#47267 it is clear that 2 "M" corresponds to 1970-02-01,
+                    #  but not clear what 2.5 "M" corresponds to, so we will
+                    #  disallow that case.
+                    warnings.warn(
+                        "Conversion of non-round float with unit={unit} is ambiguous "
+                        "and will raise in a future version.",
+                        FutureWarning,
+                        stacklevel=1,
+                    )
+
             ts = cast_from_unit(ts, unit)
             obj.value = ts
             dt64_to_dtstruct(ts, &obj.dts)

pandas/_libs/tslibs/fields.pyi

@@ -28,6 +28,7 @@ def get_date_field(
 def get_timedelta_field(
     tdindex: npt.NDArray[np.int64],  # const int64_t[:]
     field: str,
+    reso: int = ...,  # NPY_DATETIMEUNIT
 ) -> npt.NDArray[np.int32]: ...
 def isleapyear_arr(
     years: np.ndarray,

pandas/_libs/tslibs/fields.pyx

@@ -48,8 +48,8 @@ from pandas._libs.tslibs.np_datetime cimport (
     get_unit_from_dtype,
     npy_datetimestruct,
     pandas_datetime_to_datetimestruct,
+    pandas_timedelta_to_timedeltastruct,
     pandas_timedeltastruct,
-    td64_to_tdstruct,
 )
 
 
@@ -491,7 +491,11 @@ def get_date_field(const int64_t[:] dtindex, str field, NPY_DATETIMEUNIT reso=NP
 
 @cython.wraparound(False)
 @cython.boundscheck(False)
-def get_timedelta_field(const int64_t[:] tdindex, str field):
+def get_timedelta_field(
+    const int64_t[:] tdindex,
+    str field,
+    NPY_DATETIMEUNIT reso=NPY_FR_ns,
+):
     """
     Given a int64-based timedelta index, extract the days, hrs, sec.,
     field and return an array of these values.
@@ -510,7 +514,7 @@ def get_timedelta_field(const int64_t[:] tdindex, str field):
                     out[i] = -1
                     continue
 
-                td64_to_tdstruct(tdindex[i], &tds)
+                pandas_timedelta_to_timedeltastruct(tdindex[i], reso, &tds)
                 out[i] = tds.days
         return out
 
@@ -521,7 +525,7 @@ def get_timedelta_field(const int64_t[:] tdindex, str field):
                     out[i] = -1
                     continue
 
-                td64_to_tdstruct(tdindex[i], &tds)
+                pandas_timedelta_to_timedeltastruct(tdindex[i], reso, &tds)
                 out[i] = tds.seconds
         return out
 
@@ -532,7 +536,7 @@ def get_timedelta_field(const int64_t[:] tdindex, str field):
                     out[i] = -1
                     continue
 
-                td64_to_tdstruct(tdindex[i], &tds)
+                pandas_timedelta_to_timedeltastruct(tdindex[i], reso, &tds)
                 out[i] = tds.microseconds
         return out
 
@@ -543,7 +547,7 @@ def get_timedelta_field(const int64_t[:] tdindex, str field):
                     out[i] = -1
                     continue
 
-                td64_to_tdstruct(tdindex[i], &tds)
+                pandas_timedelta_to_timedeltastruct(tdindex[i], reso, &tds)
                 out[i] = tds.nanoseconds
         return out
 

pandas/_libs/tslibs/np_datetime.pxd

@@ -77,7 +77,6 @@ cdef check_dts_bounds(npy_datetimestruct *dts, NPY_DATETIMEUNIT unit=?)
 
 cdef int64_t dtstruct_to_dt64(npy_datetimestruct* dts) nogil
 cdef void dt64_to_dtstruct(int64_t dt64, npy_datetimestruct* out) nogil
-cdef void td64_to_tdstruct(int64_t td64, pandas_timedeltastruct* out) nogil
 
 cdef int64_t pydatetime_to_dt64(datetime val, npy_datetimestruct *dts)
 cdef int64_t pydate_to_dt64(date val, npy_datetimestruct *dts)

pandas/_libs/tslibs/np_datetime.pyx

@@ -221,14 +221,6 @@ cdef inline void dt64_to_dtstruct(int64_t dt64,
     return
 
 
-cdef inline void td64_to_tdstruct(int64_t td64,
-                                  pandas_timedeltastruct* out) nogil:
-    """Convenience function to call pandas_timedelta_to_timedeltastruct
-    with the by-far-most-common frequency NPY_FR_ns"""
-    pandas_timedelta_to_timedeltastruct(td64, NPY_FR_ns, out)
-    return
-
-
 # just exposed for testing at the moment
 def py_td64_to_tdstruct(int64_t td64, NPY_DATETIMEUNIT unit):
     cdef:

pandas/_libs/tslibs/timedeltas.pyx

@@ -161,42 +161,61 @@ def ints_to_pytimedelta(ndarray m8values, box=False):
         array of Timedelta or timedeltas objects
     """
     cdef:
+        NPY_DATETIMEUNIT reso = get_unit_from_dtype(m8values.dtype)
         Py_ssize_t i, n = m8values.size
         int64_t value
-        object[::1] result = np.empty(n, dtype=object)
-        NPY_DATETIMEUNIT reso = get_unit_from_dtype(m8values.dtype)
+        object res_val
+
+        # Note that `result` (and thus `result_flat`) is C-order and
+        #  `it` iterates C-order as well, so the iteration matches
+        #  See discussion at
+        #  github.com/pandas-dev/pandas/pull/46886#discussion_r860261305
+        ndarray result = cnp.PyArray_EMPTY(m8values.ndim, m8values.shape, cnp.NPY_OBJECT, 0)
+        object[::1] res_flat = result.ravel()     # should NOT be a copy
 
-    arr = m8values.view("i8")
+        ndarray arr = m8values.view("i8")
+        cnp.flatiter it = cnp.PyArray_IterNew(arr)
 
     for i in range(n):
+        # Analogous to: value = arr[i]
+        value = (<int64_t*>cnp.PyArray_ITER_DATA(it))[0]
 
-        value = arr[i]
         if value == NPY_NAT:
-            result[i] = <object>NaT
+            res_val = <object>NaT
         else:
             if box:
-                result[i] = _timedelta_from_value_and_reso(value, reso=reso)
+                res_val = _timedelta_from_value_and_reso(value, reso=reso)
             elif reso == NPY_DATETIMEUNIT.NPY_FR_ns:
-                result[i] = timedelta(microseconds=int(value) / 1000)
+                res_val = timedelta(microseconds=int(value) / 1000)
             elif reso == NPY_DATETIMEUNIT.NPY_FR_us:
-                result[i] = timedelta(microseconds=value)
+                res_val = timedelta(microseconds=value)
             elif reso == NPY_DATETIMEUNIT.NPY_FR_ms:
-                result[i] = timedelta(milliseconds=value)
+                res_val = timedelta(milliseconds=value)
             elif reso == NPY_DATETIMEUNIT.NPY_FR_s:
-                result[i] = timedelta(seconds=value)
+                res_val = timedelta(seconds=value)
             elif reso == NPY_DATETIMEUNIT.NPY_FR_m:
-                result[i] = timedelta(minutes=value)
+                res_val = timedelta(minutes=value)
             elif reso == NPY_DATETIMEUNIT.NPY_FR_h:
-                result[i] = timedelta(hours=value)
+                res_val = timedelta(hours=value)
             elif reso == NPY_DATETIMEUNIT.NPY_FR_D:
-                result[i] = timedelta(days=value)
+                res_val = timedelta(days=value)
             elif reso == NPY_DATETIMEUNIT.NPY_FR_W:
-                result[i] = timedelta(weeks=value)
+                res_val = timedelta(weeks=value)
             else:
                 # Month, Year, NPY_FR_GENERIC, pico, fempto, atto
                 raise NotImplementedError(reso)
 
-    return result.base  # .base to access underlying np.ndarray
+        # Note: we can index result directly instead of using PyArray_MultiIter_DATA
+        #  like we do for the other functions because result is known C-contiguous
+        #  and is the first argument to PyArray_MultiIterNew2.  The usual pattern
+        #  does not seem to work with object dtype.
+        #  See discussion at
+        #  github.com/pandas-dev/pandas/pull/46886#discussion_r860261305
+        res_flat[i] = res_val
+
+        cnp.PyArray_ITER_NEXT(it)
+
+    return result
 
 
 # ----------------------------------------------------------------------
@@ -316,6 +335,8 @@ cdef convert_to_timedelta64(object ts, str unit):
 
     Return an ns based int64
     """
+    # Caller is responsible for checking unit not in ["Y", "y", "M"]
+
     if checknull_with_nat(ts):
         return np.timedelta64(NPY_NAT, "ns")
     elif isinstance(ts, _Timedelta):
@@ -329,17 +350,9 @@ cdef convert_to_timedelta64(object ts, str unit):
         if ts == NPY_NAT:
             return np.timedelta64(NPY_NAT, "ns")
         else:
-            if unit in ["Y", "M", "W"]:
-                ts = np.timedelta64(ts, unit)
-            else:
-                ts = cast_from_unit(ts, unit)
-                ts = np.timedelta64(ts, "ns")
+            ts = _maybe_cast_from_unit(ts, unit)
     elif is_float_object(ts):
-        if unit in ["Y", "M", "W"]:
-            ts = np.timedelta64(int(ts), unit)
-        else:
-            ts = cast_from_unit(ts, unit)
-            ts = np.timedelta64(ts, "ns")
+        ts = _maybe_cast_from_unit(ts, unit)
     elif isinstance(ts, str):
         if (len(ts) > 0 and ts[0] == "P") or (len(ts) > 1 and ts[:2] == "-P"):
             ts = parse_iso_format_string(ts)
@@ -356,6 +369,20 @@ cdef convert_to_timedelta64(object ts, str unit):
     return ts.astype("timedelta64[ns]")
 
 
+cdef _maybe_cast_from_unit(ts, str unit):
+    # caller is responsible for checking
+    #  assert unit not in ["Y", "y", "M"]
+    try:
+        ts = cast_from_unit(ts, unit)
+    except OverflowError as err:
+        raise OutOfBoundsTimedelta(
+            f"Cannot cast {ts} from {unit} to 'ns' without overflow."
+        ) from err
+
+    ts = np.timedelta64(ts, "ns")
+    return ts
+
+
 @cython.boundscheck(False)
 @cython.wraparound(False)
 def array_to_timedelta64(
@@ -370,6 +397,8 @@ def array_to_timedelta64(
     -------
     np.ndarray[timedelta64ns]
     """
+    # Caller is responsible for checking
+    assert unit not in ["Y", "y", "M"]
 
     cdef:
         Py_ssize_t i, n = values.size
@@ -652,24 +681,20 @@ cdef inline timedelta_from_spec(object number, object frac, object unit):
     cdef:
         str n
 
-    try:
-        unit = ''.join(unit)
-
-        if unit in ["M", "Y", "y"]:
-            warnings.warn(
-                "Units 'M', 'Y' and 'y' do not represent unambiguous "
-                "timedelta values and will be removed in a future version.",
-                FutureWarning,
-                stacklevel=2,
-            )
+    unit = ''.join(unit)
+    if unit in ["M", "Y", "y"]:
+        warnings.warn(
+            "Units 'M', 'Y' and 'y' do not represent unambiguous "
+            "timedelta values and will be removed in a future version.",
+            FutureWarning,
+            stacklevel=3,
+        )
 
-        if unit == 'M':
-            # To parse ISO 8601 string, 'M' should be treated as minute,
-            # not month
-            unit = 'm'
-        unit = parse_timedelta_unit(unit)
-    except KeyError:
-        raise ValueError(f"invalid abbreviation: {unit}")
+    if unit == 'M':
+        # To parse ISO 8601 string, 'M' should be treated as minute,
+        # not month
+        unit = 'm'
+    unit = parse_timedelta_unit(unit)
 
     n = ''.join(number) + '.' + ''.join(frac)
     return cast_from_unit(float(n), unit)
@@ -696,7 +721,7 @@ cpdef inline str parse_timedelta_unit(str unit):
         return unit
     try:
         return timedelta_abbrevs[unit.lower()]
-    except (KeyError, AttributeError):
+    except KeyError:
         raise ValueError(f"invalid unit abbreviation: {unit}")
 
 # ----------------------------------------------------------------------

pandas/_libs/tslibs/timestamps.pyx

@@ -90,7 +90,10 @@ from pandas._libs.tslibs.np_datetime cimport (
     pydatetime_to_dt64,
 )
 
-from pandas._libs.tslibs.np_datetime import OutOfBoundsDatetime
+from pandas._libs.tslibs.np_datetime import (
+    OutOfBoundsDatetime,
+    OutOfBoundsTimedelta,
+)
 
 from pandas._libs.tslibs.offsets cimport (
     BaseOffset,
@@ -455,7 +458,7 @@ cdef class _Timestamp(ABCTimestamp):
             # Timedelta
             try:
                 return Timedelta(self.value - other.value)
-            except (OverflowError, OutOfBoundsDatetime) as err:
+            except (OverflowError, OutOfBoundsDatetime, OutOfBoundsTimedelta) as err:
                 if isinstance(other, _Timestamp):
                     if both_timestamps:
                         raise OutOfBoundsDatetime(