wip

MarcoGorelli · MarcoGorelli · commit f2d9eb93670e · 2023-01-27T10:43:07.000Z
diff --git a/doc/source/user_guide/io.rst b/doc/source/user_guide/io.rst
@@ -290,6 +290,16 @@ date_parser : function, default ``None``
   values from the columns defined by parse_dates into a single array and pass
   that; and 3) call date_parser once for each row using one or more strings
   (corresponding to the columns defined by parse_dates) as arguments.
+
+  .. deprecated:: 2.0.0
+   Use ``date_format`` instead, or read in as ``object`` and then apply
+   :func:`to_datetime` as-needed.
+date_format : str, default ``None``
+   If used in conjunction with ``parse_dates``, will parse dates according to this
+   format. For anything more complex (e.g. different formats for different columns),
+   please read in as ``object`` and then apply :func:`to_datetime` as-needed.
+
+    .. versionadded:: 2.0.0
 dayfirst : boolean, default ``False``
   DD/MM format dates, international and European format.
 cache_dates : boolean, default True
diff --git a/pandas/io/excel/_base.py b/pandas/io/excel/_base.py
@@ -250,6 +250,16 @@
     and pass that; and 3) call `date_parser` once for each row using one or
     more strings (corresponding to the columns defined by `parse_dates`) as
     arguments.
+
+  .. deprecated:: 2.0.0
+   Use ``date_format`` instead, or read in as ``object`` and then apply
+   :func:`to_datetime` as-needed.
+date_format : str, default ``None``
+   If used in conjunction with ``parse_dates``, will parse dates according to this
+   format. For anything more complex (e.g. different formats for different columns),
+   please read in as ``object`` and then apply :func:`to_datetime` as-needed.
+
+    .. versionadded:: 2.0.0
 thousands : str, default None
     Thousands separator for parsing string columns to numeric.  Note that
     this parameter is only necessary for columns stored as TEXT in Excel,
diff --git a/pandas/io/parsers/base_parser.py b/pandas/io/parsers/base_parser.py
@@ -116,6 +116,7 @@ def __init__(self, kwds) -> None:
         self.parse_dates = _validate_parse_dates_arg(kwds.pop("parse_dates", False))
         self._parse_date_cols: Iterable = []
         self.date_parser = kwds.pop("date_parser", None)
+        self.date_format = kwds.pop("date_format", None)
         self.dayfirst = kwds.pop("dayfirst", False)
         self.keep_date_col = kwds.pop("keep_date_col", False)
 
@@ -134,6 +135,7 @@ def __init__(self, kwds) -> None:
 
         self._date_conv = _make_date_converter(
             date_parser=self.date_parser,
+            date_format=self.date_format,
             dayfirst=self.dayfirst,
             cache_dates=self.cache_dates,
         )
@@ -1092,13 +1094,15 @@ def _make_date_converter(
     date_parser=None,
     dayfirst: bool = False,
     cache_dates: bool = True,
+    date_format=None,
 ):
     def converter(*date_cols):
         if date_parser is None:
             strs = parsing.concat_date_cols(date_cols)
 
             return tools.to_datetime(
                 ensure_object(strs),
+                format=date_format,
                 utc=False,
                 dayfirst=dayfirst,
                 errors="ignore",
@@ -1153,6 +1157,7 @@ def converter(*date_cols):
     "keep_date_col": False,
     "dayfirst": False,
     "date_parser": None,
+    "date_format": None,
     "usecols": None,
     # 'iterator': False,
     "chunksize": None,
diff --git a/pandas/io/parsers/readers.py b/pandas/io/parsers/readers.py
@@ -239,10 +239,7 @@
     say because of an unparsable value or a mixture of timezones, the column
     or index will be returned unaltered as an object data type. For
     non-standard datetime parsing, use ``pd.to_datetime`` after
-    ``pd.read_csv``. To parse an index or column with a mixture of timezones,
-    specify ``date_parser`` to be a partially-applied
-    :func:`pandas.to_datetime` with ``utc=True``. See
-    :ref:`io.csv.mixed_timezones` for more.
+    ``pd.read_csv``.
 
     Note: A fast-path exists for iso8601-formatted dates.
 infer_datetime_format : bool, default False
@@ -267,6 +264,16 @@
     and pass that; and 3) call `date_parser` once for each row using one or
     more strings (corresponding to the columns defined by `parse_dates`) as
     arguments.
+
+  .. deprecated:: 2.0.0
+   Use ``date_format`` instead, or read in as ``object`` and then apply
+   :func:`to_datetime` as-needed.
+date_format : str, default ``None``
+   If used in conjunction with ``parse_dates``, will parse dates according to this
+   format. For anything more complex (e.g. different formats for different columns),
+   please read in as ``object`` and then apply :func:`to_datetime` as-needed.
+
+    .. versionadded:: 2.0.0
 dayfirst : bool, default False
     DD/MM format dates, international and European format.
 cache_dates : bool, default True
@@ -546,7 +553,7 @@ def _read(
     # if we pass a date_parser and parse_dates=False, we should not parse the
     # dates GH#44366
     if kwds.get("parse_dates", None) is None:
-        if kwds.get("date_parser", None) is None:
+        if kwds.get("date_parser", None) is None and kwds.get("date_format") is None:
             kwds["parse_dates"] = False
         else:
             kwds["parse_dates"] = True
@@ -620,6 +627,7 @@ def read_csv(
     infer_datetime_format: bool | lib.NoDefault = ...,
     keep_date_col: bool = ...,
     date_parser=...,
+    date_format: str | None = ...,
     dayfirst: bool = ...,
     cache_dates: bool = ...,
     iterator: Literal[True],
@@ -676,6 +684,7 @@ def read_csv(
     infer_datetime_format: bool | lib.NoDefault = ...,
     keep_date_col: bool = ...,
     date_parser=...,
+    date_format: str | None = ...,
     dayfirst: bool = ...,
     cache_dates: bool = ...,
     iterator: bool = ...,
@@ -732,6 +741,7 @@ def read_csv(
     infer_datetime_format: bool | lib.NoDefault = ...,
     keep_date_col: bool = ...,
     date_parser=...,
+    date_format: str | None = ...,
     dayfirst: bool = ...,
     cache_dates: bool = ...,
     iterator: Literal[False] = ...,
@@ -788,6 +798,7 @@ def read_csv(
     infer_datetime_format: bool | lib.NoDefault = ...,
     keep_date_col: bool = ...,
     date_parser=...,
+    date_format: str | None = ...,
     dayfirst: bool = ...,
     cache_dates: bool = ...,
     iterator: bool = ...,
@@ -856,6 +867,7 @@ def read_csv(
     infer_datetime_format: bool | lib.NoDefault = lib.no_default,
     keep_date_col: bool = False,
     date_parser=None,
+    date_format: str | None = None,
     dayfirst: bool = False,
     cache_dates: bool = True,
     # Iteration
@@ -943,6 +955,7 @@ def read_table(
     infer_datetime_format: bool | lib.NoDefault = ...,
     keep_date_col: bool = ...,
     date_parser=...,
+    date_format: str | None = ...,
     dayfirst: bool = ...,
     cache_dates: bool = ...,
     iterator: Literal[True],
@@ -999,6 +1012,7 @@ def read_table(
     infer_datetime_format: bool | lib.NoDefault = ...,
     keep_date_col: bool = ...,
     date_parser=...,
+    date_format: str | None = ...,
     dayfirst: bool = ...,
     cache_dates: bool = ...,
     iterator: bool = ...,
@@ -1055,6 +1069,7 @@ def read_table(
     infer_datetime_format: bool | lib.NoDefault = ...,
     keep_date_col: bool = ...,
     date_parser=...,
+    date_format: str | None = ...,
     dayfirst: bool = ...,
     cache_dates: bool = ...,
     iterator: Literal[False] = ...,
@@ -1111,6 +1126,7 @@ def read_table(
     infer_datetime_format: bool | lib.NoDefault = ...,
     keep_date_col: bool = ...,
     date_parser=...,
+    date_format: str | None = ...,
     dayfirst: bool = ...,
     cache_dates: bool = ...,
     iterator: bool = ...,
@@ -1179,6 +1195,7 @@ def read_table(
     infer_datetime_format: bool | lib.NoDefault = lib.no_default,
     keep_date_col: bool = False,
     date_parser=None,
+    date_format: str | None = None,
     dayfirst: bool = False,
     cache_dates: bool = True,
     # Iteration
@@ -1207,6 +1224,17 @@ def read_table(
     storage_options: StorageOptions = None,
     use_nullable_dtypes: bool | lib.NoDefault = lib.no_default,
 ) -> DataFrame | TextFileReader:
+    if date_parser is not None:
+        warnings.warn(
+            "The argument 'date_parser' is deprecated and will "
+            "be removed in a future version. "
+            "Please use 'date_format' instead, or read your data in as 'object' dtype "
+            "and then call 'to_datetime'.",
+            FutureWarning,
+            stacklevel=find_stack_level(),
+        )
+    if date_parser is not None and date_format is not None:
+        raise TypeError("Cannot use both 'date_parser' and 'date_format'")
     # locals() should never be modified
     kwds = locals().copy()
     del kwds["filepath_or_buffer"]
@@ -1762,6 +1790,11 @@ def TextParser(*args, **kwds) -> TextFileReader:
     parse_dates : bool, default False
     keep_date_col : bool, default False
     date_parser : function, optional
+
+        .. deprecated:: 2.0.0
+    date_format : str, default ``None``
+
+        .. versionadded:: 2.0.0
     skiprows : list of integers
         Row numbers to skip
     skipfooter : int
diff --git a/pandas/tests/io/parser/test_parse_dates.py b/pandas/tests/io/parser/test_parse_dates.py

-Original file line number
+Diff line change
 .00 -98573.7302 871458.0640 389.0086
         """
+    )
 -    result = all_parsers.read_csv(
 +    result = all_parsers.read_csv_check_warnings(
 +        FutureWarning,
 +        "Please use 'date_format' instead",
         testdata,
         delim_whitespace=True,
         parse_dates=True,
 .00 -97.72
         """
+    )
 -    result = all_parsers.read_csv(
 +    result = all_parsers.read_csv_check_warnings(
 +        FutureWarning,
 +        "Please use 'date_format' instead",
         testdata,
         delim_whitespace=True,
         parse_dates=False,
         "keep_date_col": keep_date_col,
         "names": ["X0", "X1", "X2", "X3", "X4", "X5", "X6", "X7", "X8"],
+    }
 -    result = parser.read_csv(StringIO(data), **kwds)
 +    result = parser.read_csv_check_warnings(
 +        FutureWarning,
 +        "use 'date_format' instead",
 +        StringIO(data),
 +        **kwds,
 +    )
     expected = DataFrame(
+        [
         "parse_dates": parse_dates,
         "date_parser": pd.to_datetime,
+    }
 -    result = parser.read_csv(StringIO(data), **kwds)
 +    result = parser.read_csv_check_warnings(
 +        FutureWarning, "use 'date_format' instead", StringIO(data), **kwds
 +    )
     expected = DataFrame(
+        [
     data = """05/31/2012,15:30:00.029,1306.25,1,E,0,,1306.25
 /31/2012,15:30:00.029,1306.25,8,E,0,,1306.25"""
 -    result = parser.read_csv(
 -        StringIO(data), parse_dates=[[0, 1]], header=None, date_parser=Timestamp
 +    result = parser.read_csv_check_warnings(
 +        FutureWarning,
 +        "use 'date_format' instead",
 +        StringIO(data),
 +        parse_dates=[[0, 1]],
 +        header=None,
 +        date_parser=Timestamp,
+    )
     expected = DataFrame(
+        [
         "12345,1,-1,3,invoice_InvoiceResource,search\n"
+    )
 -    result = parser.read_csv(
 +    result = parser.read_csv_check_warnings(
 +        FutureWarning,
 +        "use 'date_format' instead",
         StringIO(data),
         index_col=0,
         parse_dates=[0],
 20090103,c,4,5
 """
     parser = all_parsers
 -    result = parser.read_csv(
 -        StringIO(data), date_parser=lambda x: datetime.strptime(x, "%Y%m%d")
 +    result = parser.read_csv_check_warnings(
 +        FutureWarning,
 +        "use 'date_format' instead",
 +        StringIO(data),
 +        date_parser=lambda x: datetime.strptime(x, "%Y%m%d"),
+    )
     expected = parser.read_csv(StringIO(data), parse_dates=True)
     tm.assert_frame_equal(result, expected)
 /02/2010,1,2
 """
     if "dayfirst" in kwargs:
 -        df = parser.read_csv(
 +        df = parser.read_csv_check_warnings(
 +            FutureWarning,
 +            "use 'date_format' instead",
             StringIO(data),
             names=["time", "Q", "NTU"],
             date_parser=lambda d: du_parse(d, **kwargs),
     else:
         msg = "got an unexpected keyword argument 'day_first'"
         with pytest.raises(TypeError, match=msg):
 -            parser.read_csv(
 +            parser.read_csv_check_warnings(
 +                FutureWarning,
 +                "use 'date_format' instead",
                 StringIO(data),
                 names=["time", "Q", "NTU"],
                 date_parser=lambda d: du_parse(d, **kwargs),
     parser = all_parsers
     data = "Date,test\n2012-01-01,1\n,2"
     parser.read_csv_check_warnings(
 -        UserWarning,
 +        FutureWarning,
         "The argument 'infer_datetime_format' is deprecated",
         StringIO(data),
         parse_dates=["Date"],
         infer_datetime_format=True,
+    )
 +@pytest.mark.parametrize(
 +    "reader", ["read_csv_check_warnings", "read_table_check_warnings"]
 +)
 +def test_parse_dates_date_parser_and_date_format(all_parsers, reader):
 +    # GH ???
 +    parser = all_parsers
 +    data = "Date,test\n2012-01-01,1\n,2"
 +    msg = "Cannot use both 'date_parser' and 'date_format'"
 +    with pytest.raises(TypeError, match=msg):
 +        getattr(parser, reader)(
 +            FutureWarning,
 +            "use 'date_format' instead",
 +            StringIO(data),
 +            parse_dates=["Date"],
 +            date_parser=pd.to_datetime,
 +            date_format="ISO8601",
 +            sep=",",
 +        )
++
++
 @xfail_pyarrow
 @pytest.mark.parametrize(
     "data,kwargs,expected",
 -01-06, 00:00:00, 1.0, 11.
 """
     parser = all_parsers
 -    result = parser.read_csv(
 +    result = parser.read_csv_check_warnings(
 +        FutureWarning,
 +        "use 'date_format' instead",
         StringIO(data),
         header=[0, 1],
         parse_dates={"date_time": [0, 1]},
+)
 def test_parse_date_time(all_parsers, data, kwargs, expected):
     parser = all_parsers
 -    result = parser.read_csv(StringIO(data), date_parser=pd.to_datetime, **kwargs)
 +    result = parser.read_csv_check_warnings(
 +        FutureWarning,
 +        "use 'date_format' instead",
 +        StringIO(data),
 +        date_parser=pd.to_datetime,
 +        **kwargs,
 +    )
     # Python can sometimes be flaky about how
     # the aggregated columns are entered, so
 def test_parse_date_fields(all_parsers):
     parser = all_parsers
     data = "year,month,day,a\n2001,01,10,10.\n2001,02,1,11."
 -    result = parser.read_csv(
 +    result = parser.read_csv_check_warnings(
 +        FutureWarning,
 +        "use 'date_format' instead",
         StringIO(data),
         header=0,
         parse_dates={"ymd": [0, 1, 2]},
 ,01,05,10,00,0,0.0,10.
 ,01,5,10,0,00,1.,11.
 """
 -    result = parser.read_csv(
 +    result = parser.read_csv_check_warnings(
 +        FutureWarning,
 +        "use 'date_format' instead",
         StringIO(data),
         header=0,
         date_parser=lambda x: pd.to_datetime(x, format="%Y %m %d %H %M %S"),
 ,01,05,10,00,0.123456,0.0,10.
 ,01,5,10,0,0.500000,1.,11.
 """
 -    result = parser.read_csv(
 +    result = parser.read_csv_check_warnings(
 +        FutureWarning,
 +        "use 'date_format' instead",
         StringIO(data),
         header=0,
         date_parser=lambda x: pd.to_datetime(x, format="%Y %m %d %H %M %S.%f"),
     def parse_function(yy, mm):
         return [date(year=int(y), month=int(m), day=1) for y, m in zip(yy, mm)]
 -    result = parser.read_csv(
 +    result = parser.read_csv_check_warnings(
 +        FutureWarning,
 +        "use 'date_format' instead",
         StringIO(data),
         header=0,
         parse_dates={"ym": [0, 1]},
             arr = [datetime.combine(d, t) for d, t in zip(dt, time)]
         return np.array(arr, dtype="datetime64[s]")
 -    result = parser.read_csv(
 +    result = parser.read_csv_check_warnings(
 +        FutureWarning,
 +        "use 'date_format' instead",
         StringIO(data),
         date_parser=date_parser,
         parse_dates={"datetime": ["date", "time"]},
+#
 -09-09
 """
 -    result = parser.read_csv(
 +    result = parser.read_csv_check_warnings(
 +        FutureWarning,
 +        "use 'date_format' instead",
         StringIO(data),
         na_values={"Test": ["#", "0"]},
         parse_dates=["Test"],