Black-reformat; avoid .dt.tz_localize() bug

Found a bug, pandas-dev/pandas#27952, which
made our module output wrong results when there are lots of duplicates.
We work around the wrong Pandas behavior by writing a special code path
that avoids the buggy code.

[finishes #167858839]

Author: adamhooper

converttodate.py

@@ -6,25 +6,16 @@
 
 
 class InputFormat(Enum):
-    AUTO = 'auto'
-    US = 'us'
-    EU = 'eu'
+    AUTO = "auto"
+    US = "us"
+    EU = "eu"
 
     @property
     def kwargs(self):
         return {
-            InputFormat.AUTO: {
-                'infer_datetime_format': True,
-                'format': None
-            },
-            InputFormat.US: {
-                'infer_datetime_format': False,
-                'format': '%m/%d/%Y'
-            },
-            InputFormat.EU: {
-                'infer_datetime_format': False,
-                'format': '%d/%m/%Y'
-            }
+            InputFormat.AUTO: {"infer_datetime_format": True, "format": None},
+            InputFormat.US: {"infer_datetime_format": False, "format": "%m/%d/%Y"},
+            InputFormat.EU: {"infer_datetime_format": False, "format": "%d/%m/%Y"},
         }[self]
 
 
@@ -43,24 +34,26 @@ class ErrorCount:
     total: int = 0
     n_columns: int = 0
 
-    def __add__(self, rhs: 'ErrorCount') -> 'ErrorCount':
+    def __add__(self, rhs: "ErrorCount") -> "ErrorCount":
         """Add more errors to this ErrorCount."""
-        return ErrorCount(self.a_column or rhs.a_column,
-                          self.a_row or rhs.a_row,
-                          self.a_value or rhs.a_value,
-                          self.total + rhs.total,
-                          self.n_columns + rhs.n_columns)
+        return ErrorCount(
+            self.a_column or rhs.a_column,
+            self.a_row or rhs.a_row,
+            self.a_value or rhs.a_value,
+            self.total + rhs.total,
+            self.n_columns + rhs.n_columns,
+        )
 
     def __str__(self):
         if self.total == 1:
-            n_errors_str = 'is 1 error'
+            n_errors_str = "is 1 error"
         else:
-            n_errors_str = f'are {self.total} errors'
+            n_errors_str = f"are {self.total} errors"
 
         if self.n_columns == 1:
-            n_columns_str = '1 column'
+            n_columns_str = "1 column"
         else:
-            n_columns_str = f'{self.n_columns} columns'
+            n_columns_str = f"{self.n_columns} columns"
 
         return (
             f"'{self.a_value}' in row {self.a_row + 1} of "
@@ -76,7 +69,7 @@ def __len__(self):
         return self.total
 
     @staticmethod
-    def from_diff(in_series, out_series) -> 'ErrorCount':
+    def from_diff(in_series, out_series) -> "ErrorCount":
         in_na = in_series.isna()
         out_na = out_series.isna()
         out_errors = out_na.index[out_na & ~in_na]
@@ -92,27 +85,54 @@ def from_diff(in_series, out_series) -> 'ErrorCount':
 
 def render(table, params):
     # No processing if no columns selected
-    if not params['colnames']:
+    if not params["colnames"]:
         return table
 
-    input_format = InputFormat(params['input_format'])
+    input_format = InputFormat(params["input_format"])
 
     error_count = ErrorCount()
 
-    for column in params['colnames']:
+    for column in params["colnames"]:
         in_series = table[column]
 
         kwargs = {**input_format.kwargs}
 
         if is_numeric_dtype(in_series):
             # For now, assume value is year and cast to string
-            kwargs['format'] = '%Y'
-
-        out_series = pd.to_datetime(in_series, errors='coerce', exact=False,
-                                    cache=True, utc=True,
-                                    **kwargs).dt.tz_localize(None)
+            kwargs["format"] = "%Y"
+
+        # Build `out_series`, a pd.Series of datetime64[ns]
+        if hasattr(in_series, "cat"):
+            # Pandas `to_datetime()` sometimes converts to Categorical; and
+            # when it does, `series.dt.tz_localize()` doesn't unwrap the
+            # Categorical. We can't blame `to_datetime()` for returning a
+            # Categorical but we _can_ blame `.dt.tz_localize()` for not
+            # unwrapping it.
+            #
+            # The bug: https://github.com/pandas-dev/pandas/issues/27952
+            #
+            # Workaround is to basically do what `pd.to_datetime()` does
+            # with its cache, using the assumption that categories are unique.
+            # We `tz_localize()` before caching, for speedup.
+            #
+            # Nix this if-statement and code path when the Pandas bug is fixed.
+            text_values = in_series.cat.categories
+            date_values = pd.to_datetime(
+                text_values,
+                errors="coerce",
+                exact=False,
+                cache=False,
+                utc=True,
+                **kwargs,
+            ).tz_localize(None)
+            mapping = pd.Series(date_values, index=text_values)
+            out_series = in_series.map(mapping).astype("datetime64[ns]")
+        else:
+            out_series = pd.to_datetime(
+                in_series, errors="coerce", exact=False, cache=True, utc=True, **kwargs
+            ).dt.tz_localize(None)
 
-        if not params['error_means_null']:
+        if not params["error_means_null"]:
             error_count += ErrorCount.from_diff(in_series, out_series)
 
         table[column] = out_series
@@ -130,9 +150,9 @@ def _migrate_params_v0_to_v1(params):
     v1: 'error_means_null' (bool), 'input_format' (enum 'auto'|'us'|'eu')
     """
     return {
-        'colnames': params['colnames'],
-        'error_means_null': params['type_null'],
-        'input_format': ['auto', 'us', 'eu'][params['type_date']]
+        "colnames": params["colnames"],
+        "error_means_null": params["type_null"],
+        "input_format": ["auto", "us", "eu"][params["type_date"]],
     }
 
 
@@ -144,16 +164,13 @@ def _migrate_params_v1_to_v2(params):
 
     https://www.pivotaltracker.com/story/show/160463316
     """
-    return {
-        **params,
-        'colnames': [c for c in params['colnames'].split(',') if c],
-    }
+    return {**params, "colnames": [c for c in params["colnames"].split(",") if c]}
 
 
 def migrate_params(params):
-    if 'type_date' in params:
+    if "type_date" in params:
         params = _migrate_params_v0_to_v1(params)
-    if isinstance(params['colnames'], str):
+    if isinstance(params["colnames"], str):
         params = _migrate_params_v1_to_v2(params)
 
     return params