Issue 20927 fix resolves read_sas error for dates/datetimes greater than 2262-04-11

doc/source/whatsnew/v1.0.0.rst

@@ -384,6 +384,7 @@ I/O
 - Bug in :meth:`DataFrame.to_html` where the length of the ``formatters`` argument was not verified (:issue:`28469`)
 - Bug in :meth:`pandas.io.formats.style.Styler` formatting for floating values not displaying decimals correctly (:issue:`13257`)
 - Bug in :meth:`DataFrame.to_html` when using ``formatters=<list>`` and ``max_cols`` together. (:issue:`25955`)
+- :func:`read_sas()` now handles dates and datetimes larger than :attr:`Timestamp.max` returning them as :class:`datetime.datetime` objects (:issue:`20927`)
 
 Plotting
 ^^^^^^^^

pandas/io/sas/sas7bdat.py

@@ -13,19 +13,21 @@
 Reference for binary data compression:
   http://collaboration.cmc.ec.gc.ca/science/rpn/biblio/ddj/Website/articles/CUJ/1992/9210/ross/ross.htm
 """
-from datetime import datetime
+from datetime import datetime, timedelta
 import struct
 
 import numpy as np
 
-from pandas.errors import EmptyDataError
+from pandas.errors import EmptyDataError, OutOfBoundsDatetime
 
 import pandas as pd
 
 from pandas.io.common import BaseIterator, get_filepath_or_buffer
 from pandas.io.sas._sas import Parser
 import pandas.io.sas.sas_constants as const
 
+from warnings import warn
+
 
 class _subheader_pointer:
     pass
@@ -687,6 +689,39 @@ def _read_next_page(self):
 
         return False
 
+    @staticmethod
+    def _convert_dates(sas_dates):
+        try:
+            return pd.to_datetime(sas_dates, unit="d", origin="1960-01-01")
+        except OutOfBoundsDatetime:
+            # convert to datetime.datetime rather than np.datetime64
+            # NB generally better support in pandas for datetime than date
+            warn(
+                "SAS date > pandas.Timestamp.max, returning "
+                "datetime.datetime objects instead"
+            )
+            return sas_dates.apply(
+                lambda sas_date_float: datetime(1960, 1, 1)
+                + timedelta(days=sas_date_float)
+            )
+
+    @staticmethod
+    def _convert_datetimes(sas_datetimes):
+        try:
+            return pd.to_datetime(sas_datetimes, unit="s", origin="1960-01-01")
+        except OutOfBoundsDatetime:
+            # convert to datetime.datetime rather than np.datetime64
+            # SAS float64 lacks precision for more than ms resolution so the
+            # fit to datetime.datetime is ok
+            warn(
+                "SAS datetime > pandas.Timestamp.max, returning "
+                "datetime.datetime objects instead"
+            )
+            return sas_datetimes.apply(
+                lambda sas_dt_float: datetime(1960, 1, 1)
+                + timedelta(seconds=sas_dt_float)
+            )
+
     def _chunk_to_dataframe(self):
 
         n = self._current_row_in_chunk_index
@@ -703,15 +738,10 @@ def _chunk_to_dataframe(self):
                 rslt[name] = self._byte_chunk[jb, :].view(dtype=self.byte_order + "d")
                 rslt[name] = np.asarray(rslt[name], dtype=np.float64)
                 if self.convert_dates:
-                    unit = None
                     if self.column_formats[j] in const.sas_date_formats:
-                        unit = "d"
+                        rslt[name] = self._convert_dates(rslt[name])
                     elif self.column_formats[j] in const.sas_datetime_formats:
-                        unit = "s"
-                    if unit:
-                        rslt[name] = pd.to_datetime(
-                            rslt[name], unit=unit, origin="1960-01-01"
-                        )
+                        rslt[name] = self._convert_datetimes(rslt[name])
                 jb += 1
             elif self._column_types[j] == b"s":
                 rslt[name] = self._string_chunk[js, :]

pandas/tests/io/sas/test_sas7bdat.py

@@ -1,3 +1,4 @@
+from datetime import datetime
 import io
 import os
 
@@ -217,3 +218,85 @@ def test_zero_variables(datapath):
     fname = datapath("io", "sas", "data", "zero_variables.sas7bdat")
     with pytest.raises(EmptyDataError):
         pd.read_sas(fname)
+
+
+def round_datetime_to_ms(ts):
+    if isinstance(ts, datetime):
+        return ts.replace(microsecond=int(round(ts.microsecond, -3) / 1000) * 1000)
+    else:
+        return ts
+
+
+def test_max_sas_date(datapath):
+    # GH 20927
+    # NB. max datetime in SAS dataset is 31DEC9999:23:59:59.999
+    #    but this is read as 29DEC9999:23:59:59.998993 by a buggy
+    #    sas7bdat module
+    fname = datapath("io", "sas", "data", "max_sas_date.sas7bdat")
+    with pytest.warns(UserWarning):
+        df = pd.read_sas(fname, encoding="iso-8859-1")
+    # SAS likes to left pad strings with spaces - lstrip before comparing
+    df = df.applymap(lambda x: x.lstrip() if isinstance(x, str) else x)
+    # GH 19732: Timestamps imported from sas will incur floating point errors
+    df = df.applymap(round_datetime_to_ms)
+    # if there are any date/times > pandas.Timestamp.max then ALL in that chunk
+    # are returned as datetime.datetime
+    expected = pd.DataFrame(
+        {
+            "text": ["max", "normal"],
+            "dt_as_float": [253717747199.999, 1880323199.999],
+            "dt_as_dt": [
+                datetime(9999, 12, 29, 23, 59, 59, 999000),
+                datetime(2019, 8, 1, 23, 59, 59, 999000),
+            ],
+            "date_as_float": [float(2936547), float(21762)],
+            "date_as_date": [datetime(9999, 12, 29), datetime(2019, 8, 1)],
+        },
+        columns=["text", "dt_as_float", "dt_as_dt", "date_as_float", "date_as_date"],
+    )
+    tm.assert_frame_equal(df, expected)
+
+
+def test_max_sas_date_iterator(datapath):
+    # GH 20927
+    # when called as an iterator, only those chunks with a date > pd.Timestamp.max
+    # are returned as datetime.datetime, if this happens that whole chunk is returned
+    # as datetime.datetime
+    col_order = ["text", "dt_as_float", "dt_as_dt", "date_as_float", "date_as_date"]
+    fname = datapath("io", "sas", "data", "max_sas_date.sas7bdat")
+    results = []
+    with pytest.warns(UserWarning):
+        for df in pd.read_sas(fname, encoding="iso-8859-1", chunksize=1):
+            # SAS likes to left pad strings with spaces - lstrip before comparing
+            df = df.applymap(lambda x: x.lstrip() if isinstance(x, str) else x)
+            # GH 19732: Timestamps imported from sas will incur floating point errors
+            try:
+                df["dt_as_dt"] = df["dt_as_dt"].dt.round("us")
+            except pd._libs.tslibs.np_datetime.OutOfBoundsDatetime:
+                df = df.applymap(round_datetime_to_ms)
+            df.reset_index(inplace=True, drop=True)
+            results.append(df)
+    expected = [
+        pd.DataFrame(
+            {
+                "text": ["max"],
+                "dt_as_float": [253717747199.999],
+                "dt_as_dt": [datetime(9999, 12, 29, 23, 59, 59, 999000)],
+                "date_as_float": [float(2936547)],
+                "date_as_date": [datetime(9999, 12, 29)],
+            },
+            columns=col_order,
+        ),
+        pd.DataFrame(
+            {
+                "text": ["normal"],
+                "dt_as_float": [1880323199.999],
+                "dt_as_dt": [np.datetime64("2019-08-01 23:59:59.999")],
+                "date_as_float": [float(21762)],
+                "date_as_date": [np.datetime64("2019-08-01")],
+            },
+            columns=col_order,
+        ),
+    ]
+    for r, e in zip(results, expected):
+        tm.assert_frame_equal(r, e)