Remove undefined behavior from npy_datetimestruct_to_datetime

meson.build

@@ -7,7 +7,7 @@ project(
     meson_version: '>=1.0.1',
     default_options: [
         'buildtype=release',
-        'c_std=c99'
+        'c_std=c11'
     ]
 )
 

pandas/_libs/src/vendored/numpy/datetime/np_datetime.c

@@ -29,6 +29,29 @@ This file is derived from NumPy 1.7. See NUMPY_LICENSE.txt
 #include <numpy/ndarraytypes.h>
 #include "pandas/vendored/numpy/datetime/np_datetime.h"
 
+#if defined(_MSVC_VER)
+  #include <intsafe.h>
+  #define checked_int64_add(a, b, res) LongLongAdd(a, b, res)
+  #define checked_int64_sub(a, b, res) LongLongSub(a, b, res)
+  #define checked_int64_mul(a, b, res) LongLongMul(a, b, res)
+#else
+  #if !__has_builtin(__builtin_add_overflow)
+    _Static_assert(0, "Overflow checking not detected; please try a newer compiler");
+  #endif
+  #if _LP64 || __LP64__ || _ILP64 || __ILP64__
+    #define checked_int64_add(a, b, res) __builtin_saddl_overflow(a, b, res)
+    #define checked_int64_sub(a, b, res) __builtin_ssubl_overflow(a, b, res)
+    #define checked_int64_mul(a, b, res) __builtin_smull_overflow(a, b, res)
+  #else
+    #define checked_int64_add(a, b, res) __builtin_saddll_overflow(a, b, res)
+    #define checked_int64_sub(a, b, res) __builtin_ssubll_overflow(a, b, res)
+    #define checked_int64_mul(a, b, res) __builtin_smulll_overflow(a, b, res)
+  #endif
+#endif
+
+// CHECK_OVERFLOW can be used in functions which define a
+// OVERFLOW_OCCURRED goto label
+#define CHECK_OVERFLOW(FUNC) do { if ((FUNC) != 0) goto OVERFLOW_OCCURRED; } while (0)
 
 const int days_per_month_table[2][12] = {
     {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31},
@@ -301,104 +324,188 @@ PyObject *extract_utc_offset(PyObject *obj) {
     return tmp;
 }
 
+
+static inline int scaleYearToEpoch(int64_t year, int64_t* result) {
+  return checked_int64_sub(year, 1970, result);
+}
+
+static inline int scaleYearsToMonths(int64_t years, int64_t* result) {
+  return checked_int64_mul(years, 12, result);
+}
+
+static inline int scaleDaysToWeeks(int64_t days, int64_t* result) {
+  if (days >= 0) {
+    *result = days / 7;
+    return 0;
+  } else {
+    int res;
+    int64_t checked_days;
+    if ((res = checked_int64_sub(days, 6, &checked_days))) {
+      return res;
+    }
+
+    *result = checked_days / 7;
+    return 0;
+  }
+}
+
+static inline int scaleDaysToHours(int64_t days, int64_t* result) {
+  return checked_int64_mul(days, 24, result);
+}
+
+static inline int scaleHoursToMinutes(int64_t hours, int64_t* result) {
+  return checked_int64_mul(hours, 60, result);
+}
+
+static inline int scaleMinutesToSeconds(int64_t minutes, int64_t* result) {
+  return checked_int64_mul(minutes, 60, result);
+}
+
+static inline int scaleSecondsToMilliseconds(int64_t seconds, int64_t* result) {
+  return checked_int64_mul(seconds, 1000, result);
+}
+
+static inline int scaleSecondsToMicroseconds(int64_t seconds, int64_t* result) {
+  return checked_int64_mul(seconds, 1000000, result);
+}
+
+static inline int scaleMicrosecondsToNanoseconds(int64_t microseconds,
+                                                 int64_t* result) {
+  return checked_int64_mul(microseconds, 1000, result);
+}
+
+static inline int scaleMicrosecondsToPicoseconds(int64_t microseconds,
+                                                 int64_t* result) {
+  return checked_int64_mul(microseconds, 1000000, result);
+}
+
+static inline int64_t scalePicosecondsToFemtoseconds(int64_t picoseconds,
+                                                     int64_t* result) {
+  return checked_int64_mul(picoseconds, 1000, result);
+}
+
+static inline int64_t scalePicosecondsToAttoseconds(int64_t picoseconds,
+                                                    int64_t* result) {
+  return checked_int64_mul(picoseconds, 1000000, result);
+}
+
 /*
  * Converts a datetime from a datetimestruct to a datetime based
- * on a metadata unit. The date is assumed to be valid.
+ * on a metadata unit. Returns -1 on and sets PyErr on error.
  */
 npy_datetime npy_datetimestruct_to_datetime(NPY_DATETIMEUNIT base,
                                             const npy_datetimestruct *dts) {
-    npy_datetime ret;
-
-    if (base == NPY_FR_Y) {
-        /* Truncate to the year */
-        ret = dts->year - 1970;
-    } else if (base == NPY_FR_M) {
-        /* Truncate to the month */
-        ret = 12 * (dts->year - 1970) + (dts->month - 1);
-    } else {
-        /* Otherwise calculate the number of days to start */
-        npy_int64 days = get_datetimestruct_days(dts);
-
-        switch (base) {
-            case NPY_FR_W:
-                /* Truncate to weeks */
-                if (days >= 0) {
-                    ret = days / 7;
-                } else {
-                    ret = (days - 6) / 7;
-                }
-                break;
-            case NPY_FR_D:
-                ret = days;
-                break;
-            case NPY_FR_h:
-                ret = days * 24 + dts->hour;
-                break;
-            case NPY_FR_m:
-                ret = (days * 24 + dts->hour) * 60 + dts->min;
-                break;
-            case NPY_FR_s:
-                ret = ((days * 24 + dts->hour) * 60 + dts->min) * 60 + dts->sec;
-                break;
-            case NPY_FR_ms:
-                ret = (((days * 24 + dts->hour) * 60 + dts->min) * 60 +
-                       dts->sec) *
-                          1000 +
-                      dts->us / 1000;
-                break;
-            case NPY_FR_us:
-                ret = (((days * 24 + dts->hour) * 60 + dts->min) * 60 +
-                       dts->sec) *
-                          1000000 +
-                      dts->us;
-                break;
-            case NPY_FR_ns:
-                ret = ((((days * 24 + dts->hour) * 60 + dts->min) * 60 +
-                        dts->sec) *
-                           1000000 +
-                       dts->us) *
-                          1000 +
-                      dts->ps / 1000;
-                break;
-            case NPY_FR_ps:
-                ret = ((((days * 24 + dts->hour) * 60 + dts->min) * 60 +
-                        dts->sec) *
-                           1000000 +
-                       dts->us) *
-                          1000000 +
-                      dts->ps;
-                break;
-            case NPY_FR_fs:
-                /* only 2.6 hours */
-                ret = (((((days * 24 + dts->hour) * 60 + dts->min) * 60 +
-                         dts->sec) *
-                            1000000 +
-                        dts->us) *
-                           1000000 +
-                       dts->ps) *
-                          1000 +
-                      dts->as / 1000;
-                break;
-            case NPY_FR_as:
-                /* only 9.2 secs */
-                ret = (((((days * 24 + dts->hour) * 60 + dts->min) * 60 +
-                         dts->sec) *
-                            1000000 +
-                        dts->us) *
-                           1000000 +
-                       dts->ps) *
-                          1000000 +
-                      dts->as;
-                break;
-            default:
-                /* Something got corrupted */
-                PyErr_SetString(
-                    PyExc_ValueError,
-                    "NumPy datetime metadata with corrupt unit value");
-                return -1;
-        }
+    if ((base == NPY_FR_Y) || (base == NPY_FR_M)) {
+      int64_t years;
+      CHECK_OVERFLOW(scaleYearToEpoch(dts->year, &years));
+
+      if (base == NPY_FR_Y) {
+        return years;
+      }
+
+      int64_t months;
+      CHECK_OVERFLOW(scaleYearsToMonths(years, &months));
+
+      int64_t months_adder;
+      CHECK_OVERFLOW(checked_int64_sub(dts->month, 1, &months_adder));
+      CHECK_OVERFLOW(checked_int64_add(months, months_adder, &months));
+
+      if (base == NPY_FR_M) {
+        return months;
+      }
+    }
+
+    const int64_t days = get_datetimestruct_days(dts);
+    if (base == NPY_FR_D) {
+      return days;
+    }
+
+    if (base == NPY_FR_W) {
+      int64_t weeks;
+      CHECK_OVERFLOW(scaleDaysToWeeks(days, &weeks));
+      return weeks;
+    }
+
+    int64_t hours;
+    CHECK_OVERFLOW(scaleDaysToHours(days, &hours));
+    CHECK_OVERFLOW(checked_int64_add(hours, dts->hour, &hours));
+
+    if (base == NPY_FR_h) {
+      return hours;
     }
-    return ret;
+
+
+    int64_t minutes;
+    CHECK_OVERFLOW(scaleHoursToMinutes(hours, &minutes));
+    CHECK_OVERFLOW(checked_int64_add(minutes, dts->min, &minutes));
+
+    if (base == NPY_FR_m) {
+      return minutes;
+    }
+
+    int64_t seconds;
+    CHECK_OVERFLOW(scaleMinutesToSeconds(minutes, &seconds));
+    CHECK_OVERFLOW(checked_int64_add(seconds, dts->sec, &seconds));
+
+    if (base == NPY_FR_s) {
+      return seconds;
+    }
+
+    if (base == NPY_FR_ms) {
+      int64_t milliseconds;
+      CHECK_OVERFLOW(scaleSecondsToMilliseconds(seconds, &milliseconds));
+      CHECK_OVERFLOW(checked_int64_add(milliseconds, dts->us / 1000, &milliseconds));
+
+      return milliseconds;
+    }
+
+    int64_t microseconds;
+    CHECK_OVERFLOW(scaleSecondsToMicroseconds(seconds, &microseconds));
+    CHECK_OVERFLOW(checked_int64_add(microseconds, dts->us, &microseconds));
+
+    if (base == NPY_FR_us) {
+      return microseconds;
+    }
+
+    if (base == NPY_FR_ns) {
+      int64_t nanoseconds;
+      CHECK_OVERFLOW(scaleMicrosecondsToNanoseconds(microseconds, &nanoseconds));
+      CHECK_OVERFLOW(checked_int64_add(nanoseconds, dts->ps / 1000, &nanoseconds));
+
+      return nanoseconds;
+    }
+
+    int64_t picoseconds;
+    CHECK_OVERFLOW(scaleMicrosecondsToPicoseconds(microseconds, &picoseconds));
+    CHECK_OVERFLOW(checked_int64_add(picoseconds, dts->ps, &picoseconds));
+
+    if (base == NPY_FR_ps) {
+      return picoseconds;
+    }
+
+    if (base == NPY_FR_fs) {
+      int64_t femtoseconds;
+      CHECK_OVERFLOW(scalePicosecondsToFemtoseconds(picoseconds, &femtoseconds));
+      CHECK_OVERFLOW(checked_int64_add(femtoseconds, dts->as / 1000, &femtoseconds));
+      return femtoseconds;
+    }
+
+    if (base == NPY_FR_as) {
+      int64_t attoseconds;
+      CHECK_OVERFLOW(scalePicosecondsToAttoseconds(picoseconds, &attoseconds));
+      CHECK_OVERFLOW(checked_int64_add(attoseconds, dts->as, &attoseconds));
+      return attoseconds;
+    }
+
+    /* Something got corrupted */
+    PyErr_SetString(PyExc_ValueError,
+                    "NumPy datetime metadata with corrupt unit value");
+    return -1;
+
+OVERFLOW_OCCURRED:
+    PyErr_SetString(PyExc_OverflowError,
+                    "Overflow occurred in npy_datetimestruct_to_datetime");
+    return -1;
 }
 
 /*

pandas/_libs/tslib.pyx

@@ -488,7 +488,6 @@ cpdef array_to_datetime(
 
             elif PyDate_Check(val):
                 iresult[i] = pydate_to_dt64(val, &dts)
-                check_dts_bounds(&dts)
 
             elif is_datetime64_object(val):
                 iresult[i] = get_datetime64_nanos(val, NPY_FR_ns)

pandas/_libs/tslibs/conversion.pyx

@@ -764,5 +764,4 @@ cdef int64_t parse_pydatetime(
             result = val.as_unit("ns")._value
         else:
             result = pydatetime_to_dt64(val, dts)
-            check_dts_bounds(dts)
     return result

pandas/_libs/tslibs/np_datetime.pxd

@@ -65,7 +65,7 @@ cdef extern from "pandas/datetime/pd_datetime.h":
                                            npy_datetimestruct *result) nogil
 
     npy_datetime npy_datetimestruct_to_datetime(NPY_DATETIMEUNIT fr,
-                                                npy_datetimestruct *d) nogil
+                                                npy_datetimestruct *d) nogil except? -1
 
     void pandas_timedelta_to_timedeltastruct(npy_timedelta val,
                                              NPY_DATETIMEUNIT fr,
@@ -89,11 +89,11 @@ cdef check_dts_bounds(npy_datetimestruct *dts, NPY_DATETIMEUNIT unit=?)
 
 cdef int64_t pydatetime_to_dt64(
     datetime val, npy_datetimestruct *dts, NPY_DATETIMEUNIT reso=?
-)
+) except? -1
 cdef void pydatetime_to_dtstruct(datetime dt, npy_datetimestruct *dts) noexcept
 cdef int64_t pydate_to_dt64(
     date val, npy_datetimestruct *dts, NPY_DATETIMEUNIT reso=?
-)
+) except? -1
 cdef void pydate_to_dtstruct(date val, npy_datetimestruct *dts) noexcept
 
 cdef npy_datetime get_datetime64_value(object obj) noexcept nogil

pandas/_libs/tslibs/np_datetime.pyx

@@ -262,7 +262,7 @@ cdef void pydatetime_to_dtstruct(datetime dt, npy_datetimestruct *dts) noexcept:
 
 cdef int64_t pydatetime_to_dt64(datetime val,
                                 npy_datetimestruct *dts,
-                                NPY_DATETIMEUNIT reso=NPY_FR_ns):
+                                NPY_DATETIMEUNIT reso=NPY_FR_ns) except? -1:
     """
     Note we are assuming that the datetime object is timezone-naive.
     """
@@ -280,7 +280,7 @@ cdef void pydate_to_dtstruct(date val, npy_datetimestruct *dts) noexcept:
 
 cdef int64_t pydate_to_dt64(
     date val, npy_datetimestruct *dts, NPY_DATETIMEUNIT reso=NPY_FR_ns
-):
+) except? -1:
     pydate_to_dtstruct(val, dts)
     return npy_datetimestruct_to_datetime(reso, dts)
 

pandas/tests/tslibs/test_array_to_datetime.py

@@ -132,7 +132,7 @@ def test_coerce_outside_ns_bounds(invalid_date, errors):
     if errors == "raise":
         msg = "^Out of bounds nanosecond timestamp: .*, at position 0$"
 
-        with pytest.raises(ValueError, match=msg):
+        with pytest.raises(OverflowError, match=msg):
             tslib.array_to_datetime(**kwargs)
     else:  # coerce.
         result, _ = tslib.array_to_datetime(**kwargs)