Remove undefined behavior from npy_datetimestruct_to_datetime

asv_bench/benchmarks/tslibs/period.py

@@ -151,7 +151,11 @@ def setup(self, size, freq, tz):
             # tzlocal is cumbersomely slow, so skip to keep runtime in check
             raise NotImplementedError
 
-        arr = np.arange(10, dtype="i8").repeat(size // 10)
+        # we pick 2**55 because smaller values end up returning
+        # -1 from npy_datetimestruct_to_datetime with NPY_FR_Y frequency
+        # this artificially slows down functions since -1 is also the
+        # error sentinel
+        arr = np.arange(2**55, 2**55 + 10, dtype="i8").repeat(size // 10)
         self.i8values = arr
 
     def time_dt64arr_to_periodarr(self, size, freq, tz):

meson.build

@@ -7,7 +7,7 @@ project(
     meson_version: '>=1.2.1',
     default_options: [
         'buildtype=release',
-        'c_std=c99'
+        'c_std=c11'
     ]
 )
 

pandas/_libs/src/vendored/numpy/datetime/np_datetime.c

@@ -29,6 +29,58 @@ This file is derived from NumPy 1.7. See NUMPY_LICENSE.txt
 #include <numpy/arrayscalars.h>
 #include <numpy/ndarraytypes.h>
 
+#if defined(_WIN32)
+#ifndef ENABLE_INTSAFE_SIGNED_FUNCTIONS
+#define ENABLE_INTSAFE_SIGNED_FUNCTIONS
+#endif
+#include <intsafe.h>
+#define checked_int64_add(a, b, res) LongLongAdd(a, b, res)
+#define checked_int64_sub(a, b, res) LongLongSub(a, b, res)
+#define checked_int64_mul(a, b, res) LongLongMult(a, b, res)
+#else
+#if defined __has_builtin
+#if __has_builtin(__builtin_add_overflow)
+#if _LP64 || __LP64__ || _ILP64 || __ILP64__
+#define checked_int64_add(a, b, res) __builtin_saddl_overflow(a, b, res)
+#define checked_int64_sub(a, b, res) __builtin_ssubl_overflow(a, b, res)
+#define checked_int64_mul(a, b, res) __builtin_smull_overflow(a, b, res)
+#else
+#define checked_int64_add(a, b, res) __builtin_saddll_overflow(a, b, res)
+#define checked_int64_sub(a, b, res) __builtin_ssubll_overflow(a, b, res)
+#define checked_int64_mul(a, b, res) __builtin_smulll_overflow(a, b, res)
+#endif
+#else
+_Static_assert(0,
+               "Overflow checking not detected; please try a newer compiler");
+#endif
+// __has_builtin was added in gcc 10, but our muslinux_1_1 build environment
+// only has gcc-9.3, so fall back to __GNUC__ macro as long as we have that
+#elif __GNUC__ > 7
+#if _LP64 || __LP64__ || _ILP64 || __ILP64__
+#define checked_int64_add(a, b, res) __builtin_saddl_overflow(a, b, res)
+#define checked_int64_sub(a, b, res) __builtin_ssubl_overflow(a, b, res)
+#define checked_int64_mul(a, b, res) __builtin_smull_overflow(a, b, res)
+#else
+#define checked_int64_add(a, b, res) __builtin_saddll_overflow(a, b, res)
+#define checked_int64_sub(a, b, res) __builtin_ssubll_overflow(a, b, res)
+#define checked_int64_mul(a, b, res) __builtin_smulll_overflow(a, b, res)
+#endif
+#else
+_Static_assert(0, "__has_builtin not detected; please try a newer compiler");
+#endif
+#endif
+
+#define PD_CHECK_OVERFLOW(FUNC)                                                \
+  do {                                                                         \
+    if ((FUNC) != 0) {                                                         \
+      PyGILState_STATE gstate = PyGILState_Ensure();                           \
+      PyErr_SetString(PyExc_OverflowError,                                     \
+                      "Overflow occurred in npy_datetimestruct_to_datetime");  \
+      PyGILState_Release(gstate);                                              \
+      return -1;                                                               \
+    }                                                                          \
+  } while (0)
+
 const int days_per_month_table[2][12] = {
     {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31},
     {31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}};
@@ -299,96 +351,189 @@ PyObject *extract_utc_offset(PyObject *obj) {
   return tmp;
 }
 
+static inline int scaleYearToEpoch(int64_t year, int64_t *result) {
+  return checked_int64_sub(year, 1970, result);
+}
+
+static inline int scaleYearsToMonths(int64_t years, int64_t *result) {
+  return checked_int64_mul(years, 12, result);
+}
+
+static inline int scaleDaysToWeeks(int64_t days, int64_t *result) {
+  if (days >= 0) {
+    *result = days / 7;
+    return 0;
+  } else {
+    int res;
+    int64_t checked_days;
+    if ((res = checked_int64_sub(days, 6, &checked_days))) {
+      return res;
+    }
+
+    *result = checked_days / 7;
+    return 0;
+  }
+}
+
+static inline int scaleDaysToHours(int64_t days, int64_t *result) {
+  return checked_int64_mul(days, 24, result);
+}
+
+static inline int scaleHoursToMinutes(int64_t hours, int64_t *result) {
+  return checked_int64_mul(hours, 60, result);
+}
+
+static inline int scaleMinutesToSeconds(int64_t minutes, int64_t *result) {
+  return checked_int64_mul(minutes, 60, result);
+}
+
+static inline int scaleSecondsToMilliseconds(int64_t seconds, int64_t *result) {
+  return checked_int64_mul(seconds, 1000, result);
+}
+
+static inline int scaleSecondsToMicroseconds(int64_t seconds, int64_t *result) {
+  return checked_int64_mul(seconds, 1000000, result);
+}
+
+static inline int scaleMicrosecondsToNanoseconds(int64_t microseconds,
+                                                 int64_t *result) {
+  return checked_int64_mul(microseconds, 1000, result);
+}
+
+static inline int scaleMicrosecondsToPicoseconds(int64_t microseconds,
+                                                 int64_t *result) {
+  return checked_int64_mul(microseconds, 1000000, result);
+}
+
+static inline int64_t scalePicosecondsToFemtoseconds(int64_t picoseconds,
+                                                     int64_t *result) {
+  return checked_int64_mul(picoseconds, 1000, result);
+}
+
+static inline int64_t scalePicosecondsToAttoseconds(int64_t picoseconds,
+                                                    int64_t *result) {
+  return checked_int64_mul(picoseconds, 1000000, result);
+}
+
 /*
  * Converts a datetime from a datetimestruct to a datetime based
- * on a metadata unit. The date is assumed to be valid.
+ * on a metadata unit. Returns -1 on and sets PyErr on error.
  */
 npy_datetime npy_datetimestruct_to_datetime(NPY_DATETIMEUNIT base,
                                             const npy_datetimestruct *dts) {
-  npy_datetime ret;
-
-  if (base == NPY_FR_Y) {
-    /* Truncate to the year */
-    ret = dts->year - 1970;
-  } else if (base == NPY_FR_M) {
-    /* Truncate to the month */
-    ret = 12 * (dts->year - 1970) + (dts->month - 1);
-  } else {
-    /* Otherwise calculate the number of days to start */
-    npy_int64 days = get_datetimestruct_days(dts);
-
-    switch (base) {
-    case NPY_FR_W:
-      /* Truncate to weeks */
-      if (days >= 0) {
-        ret = days / 7;
-      } else {
-        ret = (days - 6) / 7;
-      }
-      break;
-    case NPY_FR_D:
-      ret = days;
-      break;
-    case NPY_FR_h:
-      ret = days * 24 + dts->hour;
-      break;
-    case NPY_FR_m:
-      ret = (days * 24 + dts->hour) * 60 + dts->min;
-      break;
-    case NPY_FR_s:
-      ret = ((days * 24 + dts->hour) * 60 + dts->min) * 60 + dts->sec;
-      break;
-    case NPY_FR_ms:
-      ret = (((days * 24 + dts->hour) * 60 + dts->min) * 60 + dts->sec) * 1000 +
-            dts->us / 1000;
-      break;
-    case NPY_FR_us:
-      ret = (((days * 24 + dts->hour) * 60 + dts->min) * 60 + dts->sec) *
-                1000000 +
-            dts->us;
-      break;
-    case NPY_FR_ns:
-      ret = ((((days * 24 + dts->hour) * 60 + dts->min) * 60 + dts->sec) *
-                 1000000 +
-             dts->us) *
-                1000 +
-            dts->ps / 1000;
-      break;
-    case NPY_FR_ps:
-      ret = ((((days * 24 + dts->hour) * 60 + dts->min) * 60 + dts->sec) *
-                 1000000 +
-             dts->us) *
-                1000000 +
-            dts->ps;
-      break;
-    case NPY_FR_fs:
-      /* only 2.6 hours */
-      ret = (((((days * 24 + dts->hour) * 60 + dts->min) * 60 + dts->sec) *
-                  1000000 +
-              dts->us) *
-                 1000000 +
-             dts->ps) *
-                1000 +
-            dts->as / 1000;
-      break;
-    case NPY_FR_as:
-      /* only 9.2 secs */
-      ret = (((((days * 24 + dts->hour) * 60 + dts->min) * 60 + dts->sec) *
-                  1000000 +
-              dts->us) *
-                 1000000 +
-             dts->ps) *
-                1000000 +
-            dts->as;
-      break;
-    default:
-      /* Something got corrupted */
-      PyErr_SetString(PyExc_ValueError,
-                      "NumPy datetime metadata with corrupt unit value");
-      return -1;
-    }
-  }
-  return ret;
+  if ((base == NPY_FR_Y) || (base == NPY_FR_M)) {
+    int64_t years;
+    PD_CHECK_OVERFLOW(scaleYearToEpoch(dts->year, &years));
+
+    if (base == NPY_FR_Y) {
+      return years;
+    }
+
+    int64_t months;
+    PD_CHECK_OVERFLOW(scaleYearsToMonths(years, &months));
+
+    int64_t months_adder;
+    PD_CHECK_OVERFLOW(checked_int64_sub(dts->month, 1, &months_adder));
+    PD_CHECK_OVERFLOW(checked_int64_add(months, months_adder, &months));
+
+    if (base == NPY_FR_M) {
+      return months;
+    }
+  }
+
+  const int64_t days = get_datetimestruct_days(dts);
+  if (base == NPY_FR_D) {
+    return days;
+  }
+
+  if (base == NPY_FR_W) {
+    int64_t weeks;
+    PD_CHECK_OVERFLOW(scaleDaysToWeeks(days, &weeks));
+    return weeks;
+  }
+
+  int64_t hours;
+  PD_CHECK_OVERFLOW(scaleDaysToHours(days, &hours));
+  PD_CHECK_OVERFLOW(checked_int64_add(hours, dts->hour, &hours));
+
+  if (base == NPY_FR_h) {
+    return hours;
+  }
+
+  int64_t minutes;
+  PD_CHECK_OVERFLOW(scaleHoursToMinutes(hours, &minutes));
+  PD_CHECK_OVERFLOW(checked_int64_add(minutes, dts->min, &minutes));
+
+  if (base == NPY_FR_m) {
+    return minutes;
+  }
+
+  int64_t seconds;
+  PD_CHECK_OVERFLOW(scaleMinutesToSeconds(minutes, &seconds));
+  PD_CHECK_OVERFLOW(checked_int64_add(seconds, dts->sec, &seconds));
+
+  if (base == NPY_FR_s) {
+    return seconds;
+  }
+
+  if (base == NPY_FR_ms) {
+    int64_t milliseconds;
+    PD_CHECK_OVERFLOW(scaleSecondsToMilliseconds(seconds, &milliseconds));
+    PD_CHECK_OVERFLOW(
+        checked_int64_add(milliseconds, dts->us / 1000, &milliseconds));
+
+    return milliseconds;
+  }
+
+  int64_t microseconds;
+  PD_CHECK_OVERFLOW(scaleSecondsToMicroseconds(seconds, &microseconds));
+  PD_CHECK_OVERFLOW(checked_int64_add(microseconds, dts->us, &microseconds));
+
+  if (base == NPY_FR_us) {
+    return microseconds;
+  }
+
+  if (base == NPY_FR_ns) {
+    int64_t nanoseconds;
+    PD_CHECK_OVERFLOW(
+        scaleMicrosecondsToNanoseconds(microseconds, &nanoseconds));
+    PD_CHECK_OVERFLOW(
+        checked_int64_add(nanoseconds, dts->ps / 1000, &nanoseconds));
+
+    return nanoseconds;
+  }
+
+  int64_t picoseconds;
+  PD_CHECK_OVERFLOW(scaleMicrosecondsToPicoseconds(microseconds, &picoseconds));
+  PD_CHECK_OVERFLOW(checked_int64_add(picoseconds, dts->ps, &picoseconds));
+
+  if (base == NPY_FR_ps) {
+    return picoseconds;
+  }
+
+  if (base == NPY_FR_fs) {
+    int64_t femtoseconds;
+    PD_CHECK_OVERFLOW(
+        scalePicosecondsToFemtoseconds(picoseconds, &femtoseconds));
+    PD_CHECK_OVERFLOW(
+        checked_int64_add(femtoseconds, dts->as / 1000, &femtoseconds));
+    return femtoseconds;
+  }
+
+  if (base == NPY_FR_as) {
+    int64_t attoseconds;
+    PD_CHECK_OVERFLOW(scalePicosecondsToAttoseconds(picoseconds, &attoseconds));
+    PD_CHECK_OVERFLOW(checked_int64_add(attoseconds, dts->as, &attoseconds));
+    return attoseconds;
+  }
+
+  /* Something got corrupted */
+  PyGILState_STATE gstate = PyGILState_Ensure();
+  PyErr_SetString(PyExc_ValueError,
+                  "NumPy datetime metadata with corrupt unit value");
+  PyGILState_Release(gstate);
+
+  return -1;
 }
 
 /*

pandas/_libs/tslib.pyx

@@ -34,7 +34,6 @@ cnp.import_array()
 from pandas._libs.tslibs.np_datetime cimport (
     NPY_DATETIMEUNIT,
     NPY_FR_ns,
-    check_dts_bounds,
     import_pandas_datetime,
     npy_datetimestruct,
     npy_datetimestruct_to_datetime,
@@ -98,8 +97,10 @@ def _test_parse_iso8601(ts: str):
     obj = _TSObject()
 
     string_to_dts(ts, &obj.dts, &out_bestunit, &out_local, &out_tzoffset, True)
-    obj.value = npy_datetimestruct_to_datetime(NPY_FR_ns, &obj.dts)
-    check_dts_bounds(&obj.dts)
+    try:
+        obj.value = npy_datetimestruct_to_datetime(NPY_FR_ns, &obj.dts)
+    except OverflowError as err:
+        raise OutOfBoundsDatetime(f"Out of bounds nanosecond timestamp: {ts}") from err
     if out_local == 1:
         obj.tzinfo = timezone(timedelta(minutes=out_tzoffset))
         obj.value = tz_localize_to_utc_single(obj.value, obj.tzinfo)
@@ -487,7 +488,6 @@ cpdef array_to_datetime(
 
             elif PyDate_Check(val):
                 iresult[i] = pydate_to_dt64(val, &dts, reso=creso)
-                check_dts_bounds(&dts, creso)
 
             elif is_datetime64_object(val):
                 iresult[i] = get_datetime64_nanos(val, creso)