BUG: pandas.to_datetime() does not respect exact format string with ISO8601

doc/source/whatsnew/v2.0.0.rst

@@ -644,6 +644,7 @@ Conversion
 - Bug in :meth:`Series.convert_dtypes` not converting dtype to nullable dtype when :class:`Series` contains ``NA`` and has dtype ``object`` (:issue:`48791`)
 - Bug where any :class:`ExtensionDtype` subclass with ``kind="M"`` would be interpreted as a timezone type (:issue:`34986`)
 - Bug in :class:`.arrays.ArrowExtensionArray` that would raise ``NotImplementedError`` when passed a sequence of strings or binary (:issue:`49172`)
+- Bug in :func:`to_datetime` was not respecting ``exact`` argument when ``format`` was an ISO8601 format (:issue:`12649`)
 
 Strings
 ^^^^^^^

pandas/_libs/tslib.pyi

@@ -24,6 +24,8 @@ def array_to_datetime(
     yearfirst: bool = ...,
     utc: bool = ...,
     require_iso8601: bool = ...,
+    format: str | None = ...,
+    exact: bool = ...,
 ) -> tuple[np.ndarray, tzinfo | None]: ...
 
 # returned ndarray may be object dtype or datetime64[ns]

pandas/_libs/tslib.pyx

@@ -446,6 +446,8 @@ cpdef array_to_datetime(
     bint yearfirst=False,
     bint utc=False,
     bint require_iso8601=False,
+    format: str | None=None,
+    bint exact=True,
 ):
     """
     Converts a 1D array of date-like values to a numpy array of either:
@@ -563,6 +565,16 @@ cpdef array_to_datetime(
                     iresult[i] = get_datetime64_nanos(val, NPY_FR_ns)
 
                 elif is_integer_object(val) or is_float_object(val):
+                    if require_iso8601:
+                        if is_coerce:
+                            iresult[i] = NPY_NAT
+                            continue
+                        elif is_raise:
+                            raise ValueError(
+                                f"time data \"{val}\" at position {i} doesn't "
+                                f"match format \"{format}\""
+                            )
+                        return values, tz_out
                     # these must be ns unit by-definition
                     seen_integer = True
 
@@ -593,7 +605,7 @@ cpdef array_to_datetime(
 
                     string_to_dts_failed = string_to_dts(
                         val, &dts, &out_bestunit, &out_local,
-                        &out_tzoffset, False
+                        &out_tzoffset, False, format, exact
                     )
                     if string_to_dts_failed:
                         # An error at this point is a _parsing_ error
@@ -609,7 +621,7 @@ cpdef array_to_datetime(
                             elif is_raise:
                                 raise ValueError(
                                     f"time data \"{val}\" at position {i} doesn't "
-                                    "match format specified"
+                                    f"match format \"{format}\""
                                 )
                             return values, tz_out
 

pandas/_libs/tslibs/np_datetime.pxd

@@ -95,6 +95,8 @@ cdef int string_to_dts(
     int* out_local,
     int* out_tzoffset,
     bint want_exc,
+    format: str | None = *,
+    bint exact = *
 ) except? -1
 
 cdef NPY_DATETIMEUNIT get_unit_from_dtype(cnp.dtype dtype)

pandas/_libs/tslibs/np_datetime.pyx

@@ -52,7 +52,8 @@ cdef extern from "src/datetime/np_datetime_strings.h":
     int parse_iso_8601_datetime(const char *str, int len, int want_exc,
                                 npy_datetimestruct *out,
                                 NPY_DATETIMEUNIT *out_bestunit,
-                                int *out_local, int *out_tzoffset)
+                                int *out_local, int *out_tzoffset,
+                                const char *format, int format_len, int exact)
 
 
 # ----------------------------------------------------------------------
@@ -277,14 +278,25 @@ cdef inline int string_to_dts(
     int* out_local,
     int* out_tzoffset,
     bint want_exc,
+    format: str | None=None,
+    bint exact=True,
 ) except? -1:
     cdef:
         Py_ssize_t length
         const char* buf
+        Py_ssize_t format_length
+        const char* format_buf
 
     buf = get_c_string_buf_and_size(val, &length)
+    if format is None:
+        format_buf = b''
+        format_length = 0
+        exact = False
+    else:
+        format_buf = get_c_string_buf_and_size(format, &format_length)
     return parse_iso_8601_datetime(buf, length, want_exc,
-                                   dts, out_bestunit, out_local, out_tzoffset)
+                                   dts, out_bestunit, out_local, out_tzoffset,
+                                   format_buf, format_length, exact)
 
 
 cpdef ndarray astype_overflowsafe(

pandas/_libs/tslibs/src/datetime/np_datetime_strings.c

@@ -66,10 +66,45 @@ This file implements string parsing and creation for NumPy datetime.
  *
  * Returns 0 on success, -1 on failure.
  */
+
+// This function will advance the pointer on format
+// and decrement characters_remaining by n on success
+// On failure will return -1 without incrementing
+static int compare_format(const char **format, int *characters_remaining,
+                 const char *compare_to, int n, const int exact) {
+  if (*characters_remaining < n) {
+    if (exact) {
+      // TODO(pandas-dev): in the future we should set a PyErr here
+      // to be very clear about what went wrong
+      return -1;
+    } else if (*characters_remaining) {
+      // TODO(pandas-dev): same return value in this function as
+      // above branch, but stub out a future where
+      // we have a better error message
+      return -1;
+    } else {
+        return 0;
+    }
+  } else {
+    if (strncmp(*format, compare_to, n)) {
+      // TODO(pandas-dev): PyErr to differentiate what went wrong
+      return -1;
+    } else {
+      *format += n;
+      *characters_remaining -= n;
+      return 0;
+    }
+  }
+  return 0;
+}
+
 int parse_iso_8601_datetime(const char *str, int len, int want_exc,
                             npy_datetimestruct *out,
                             NPY_DATETIMEUNIT *out_bestunit,
-                            int *out_local, int *out_tzoffset) {
+                            int *out_local, int *out_tzoffset,
+                            const char* format, int format_len, int exact) {
+    if (len < 0 || format_len < 0)
+        goto parse_error;
     int year_leap = 0;
     int i, numdigits;
     const char *substr;
@@ -104,6 +139,9 @@ int parse_iso_8601_datetime(const char *str, int len, int want_exc,
     while (sublen > 0 && isspace(*substr)) {
         ++substr;
         --sublen;
+        if (compare_format(&format, &format_len, " ", 1, exact)) {
+            goto parse_error;
+        }
     }
 
     /* Leading '-' sign for negative year */
@@ -117,6 +155,10 @@ int parse_iso_8601_datetime(const char *str, int len, int want_exc,
     }
 
     /* PARSE THE YEAR (4 digits) */
+    if (compare_format(&format, &format_len, "%Y", 2, exact)) {
+        goto parse_error;
+    }
+
     out->year = 0;
     if (sublen >= 4 && isdigit(substr[0]) && isdigit(substr[1]) &&
         isdigit(substr[2]) && isdigit(substr[3])) {
@@ -139,6 +181,9 @@ int parse_iso_8601_datetime(const char *str, int len, int want_exc,
         if (out_local != NULL) {
             *out_local = 0;
         }
+        if (format_len) {
+          goto parse_error;
+        }
         bestunit = NPY_FR_Y;
         goto finish;
     }
@@ -156,13 +201,20 @@ int parse_iso_8601_datetime(const char *str, int len, int want_exc,
         ymd_sep = valid_ymd_sep[i];
         ++substr;
         --sublen;
+
+        if (compare_format(&format, &format_len, &ymd_sep, 1, exact)) {
+            goto parse_error;
+        }
         /* Cannot have trailing separator */
         if (sublen == 0 || !isdigit(*substr)) {
             goto parse_error;
         }
     }
 
     /* PARSE THE MONTH */
+    if (compare_format(&format, &format_len, "%m", 2, exact)) {
+        goto parse_error;
+    }
     /* First digit required */
     out->month = (*substr - '0');
     ++substr;
@@ -190,6 +242,9 @@ int parse_iso_8601_datetime(const char *str, int len, int want_exc,
         if (!has_ymd_sep) {
             goto parse_error;
         }
+        if (format_len) {
+          goto parse_error;
+        }
         if (out_local != NULL) {
             *out_local = 0;
         }
@@ -203,9 +258,15 @@ int parse_iso_8601_datetime(const char *str, int len, int want_exc,
         }
         ++substr;
         --sublen;
+        if (compare_format(&format, &format_len, &ymd_sep, 1, exact)) {
+            goto parse_error;
+        }
     }
 
     /* PARSE THE DAY */
+    if (compare_format(&format, &format_len, "%d", 2, exact)) {
+        goto parse_error;
+    }
     /* First digit required */
     if (!isdigit(*substr)) {
         goto parse_error;
@@ -235,17 +296,26 @@ int parse_iso_8601_datetime(const char *str, int len, int want_exc,
         if (out_local != NULL) {
             *out_local = 0;
         }
+        if (format_len) {
+          goto parse_error;
+        }
         bestunit = NPY_FR_D;
         goto finish;
     }
 
     if ((*substr != 'T' && *substr != ' ') || sublen == 1) {
         goto parse_error;
     }
+        if (compare_format(&format, &format_len, substr, 1, exact)) {
+            goto parse_error;
+        }
     ++substr;
     --sublen;
 
     /* PARSE THE HOURS */
+    if (compare_format(&format, &format_len, "%H", 2, exact)) {
+        goto parse_error;
+    }
     /* First digit required */
     if (!isdigit(*substr)) {
         goto parse_error;
@@ -274,6 +344,9 @@ int parse_iso_8601_datetime(const char *str, int len, int want_exc,
         if (!hour_was_2_digits) {
             goto parse_error;
         }
+        if (format_len) {
+          goto parse_error;
+        }
         bestunit = NPY_FR_h;
         goto finish;
     }
@@ -286,6 +359,9 @@ int parse_iso_8601_datetime(const char *str, int len, int want_exc,
         if (sublen == 0 || !isdigit(*substr)) {
             goto parse_error;
         }
+        if (compare_format(&format, &format_len, ":", 1, exact)) {
+            goto parse_error;
+        }
     } else if (!isdigit(*substr)) {
         if (!hour_was_2_digits) {
             goto parse_error;
@@ -294,6 +370,9 @@ int parse_iso_8601_datetime(const char *str, int len, int want_exc,
     }
 
     /* PARSE THE MINUTES */
+    if (compare_format(&format, &format_len, "%M", 2, exact)) {
+        goto parse_error;
+    }
     /* First digit required */
     out->min = (*substr - '0');
     ++substr;
@@ -317,12 +396,18 @@ int parse_iso_8601_datetime(const char *str, int len, int want_exc,
 
     if (sublen == 0) {
         bestunit = NPY_FR_m;
+        if (format_len) {
+          goto parse_error;
+        }
         goto finish;
     }
 
     /* If we make it through this condition block, then the next
      * character is a digit. */
     if (has_hms_sep && *substr == ':') {
+        if (compare_format(&format, &format_len, ":", 1, exact)) {
+            goto parse_error;
+        }
         ++substr;
         --sublen;
         /* Cannot have a trailing ':' */
@@ -335,6 +420,9 @@ int parse_iso_8601_datetime(const char *str, int len, int want_exc,
     }
 
     /* PARSE THE SECONDS */
+    if (compare_format(&format, &format_len, "%S", 2, exact)) {
+        goto parse_error;
+    }
     /* First digit required */
     out->sec = (*substr - '0');
     ++substr;
@@ -360,12 +448,18 @@ int parse_iso_8601_datetime(const char *str, int len, int want_exc,
     if (sublen > 0 && *substr == '.') {
         ++substr;
         --sublen;
+        if (compare_format(&format, &format_len, ".", 1, exact)) {
+            goto parse_error;
+        }
     } else {
         bestunit = NPY_FR_s;
         goto parse_timezone;
     }
 
     /* PARSE THE MICROSECONDS (0 to 6 digits) */
+    if (compare_format(&format, &format_len, "%f", 2, exact)) {
+        goto parse_error;
+    }
     numdigits = 0;
     for (i = 0; i < 6; ++i) {
         out->us *= 10;
@@ -430,15 +524,24 @@ int parse_iso_8601_datetime(const char *str, int len, int want_exc,
     while (sublen > 0 && isspace(*substr)) {
         ++substr;
         --sublen;
+        if (compare_format(&format, &format_len, " ", 1, exact)) {
+            goto parse_error;
+        }
     }
 
     if (sublen == 0) {
         // Unlike NumPy, treating no time zone as naive
+        if (format_len > 0) {
+            goto parse_error;
+        }
         goto finish;
     }
 
     /* UTC specifier */
     if (*substr == 'Z') {
+        if (compare_format(&format, &format_len, "%Z", 2, exact)) {
+            goto parse_error;
+        }
         /* "Z" should be equivalent to tz offset "+00:00" */
         if (out_local != NULL) {
             *out_local = 1;
@@ -449,12 +552,18 @@ int parse_iso_8601_datetime(const char *str, int len, int want_exc,
         }
 
         if (sublen == 1) {
+            if (format_len > 0) {
+                goto parse_error;
+            }
             goto finish;
         } else {
             ++substr;
             --sublen;
         }
     } else if (*substr == '-' || *substr == '+') {
+        if (compare_format(&format, &format_len, "%z", 2, exact)) {
+            goto parse_error;
+        }
         /* Time zone offset */
         int offset_neg = 0, offset_hour = 0, offset_minute = 0;
 
@@ -538,9 +647,12 @@ int parse_iso_8601_datetime(const char *str, int len, int want_exc,
     while (sublen > 0 && isspace(*substr)) {
         ++substr;
         --sublen;
+        if (compare_format(&format, &format_len, " ", 1, exact)) {
+            goto parse_error;
+        }
     }
 
-    if (sublen != 0) {
+    if ((sublen != 0) || (format_len != 0)) {
         goto parse_error;
     }