From 903237af19c746bbe134f9d161d62182d4a8cc4d Mon Sep 17 00:00:00 2001 From: Brock Date: Sat, 11 Jun 2022 11:37:29 -0700 Subject: [PATCH 1/9] ENH: Timestamp +- timedeltalike scalar support non-nano --- pandas/_libs/tslibs/timestamps.pyx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pandas/_libs/tslibs/timestamps.pyx b/pandas/_libs/tslibs/timestamps.pyx index 1d21f602fac05..d6c6644f7630d 100644 --- a/pandas/_libs/tslibs/timestamps.pyx +++ b/pandas/_libs/tslibs/timestamps.pyx @@ -400,10 +400,14 @@ cdef class _Timestamp(ABCTimestamp): new_value = int(self.value) + int(nanos) try: - result = type(self)._from_value_and_reso(new_value, reso=self._reso, tz=self.tzinfo) + result = type(self)._from_value_and_reso( + new_value, reso=self._reso, tz=self.tzinfo + ) except OverflowError as err: # TODO: don't hard-code nanosecond here - raise OutOfBoundsDatetime(f"Out of bounds nanosecond timestamp: {new_value}") from err + raise OutOfBoundsDatetime( + f"Out of bounds nanosecond timestamp: {new_value}" + ) from err if result is not NaT: result._set_freq(self._freq) # avoid warning in constructor From b0316a6fd042023830cd4dcb149e02192883f91f Mon Sep 17 00:00:00 2001 From: Brock Date: Mon, 13 Jun 2022 20:23:26 -0700 Subject: [PATCH 2/9] ENH: Timestamp.__sub__(datetime) with non-nano --- pandas/_libs/tslibs/timestamps.pyx | 19 +++--- .../tests/scalar/timestamp/test_timestamp.py | 60 +++++++++++++++++++ 2 files changed, 69 insertions(+), 10 deletions(-) diff --git a/pandas/_libs/tslibs/timestamps.pyx b/pandas/_libs/tslibs/timestamps.pyx index d6c6644f7630d..8edfa789ec807 100644 --- a/pandas/_libs/tslibs/timestamps.pyx +++ b/pandas/_libs/tslibs/timestamps.pyx @@ -413,9 +413,6 @@ cdef class _Timestamp(ABCTimestamp): result._set_freq(self._freq) # avoid warning in constructor return result - elif isinstance(self, _Timestamp) and self._reso != NPY_FR_ns: - raise NotImplementedError(self._reso) - elif is_integer_object(other): raise integer_op_not_supported(self) @@ -450,9 +447,6 @@ cdef class _Timestamp(ABCTimestamp): neg_other = -other return self + neg_other - elif isinstance(self, _Timestamp) and self._reso != NPY_FR_ns: - raise NotImplementedError(self._reso) - elif is_array(other): if other.dtype.kind in ['i', 'u']: raise integer_op_not_supported(self) @@ -483,10 +477,18 @@ cdef class _Timestamp(ABCTimestamp): "Cannot subtract tz-naive and tz-aware datetime-like objects." ) + # We allow silent casting to the lower resolution if and only + # if it is lossless. + if self._reso < other._reso: + other = (<_Timestamp>other)._as_reso(self._reso, round_ok=False) + elif self._reso > other._reso: + self = (<_Timestamp>self)._as_reso(other._reso, round_ok=False) + # scalar Timestamp/datetime - Timestamp/datetime -> yields a # Timedelta try: - return Timedelta(self.value - other.value) + res_value = self.value - other.value + return Timedelta._from_value_and_reso(res_value, self._reso) except (OverflowError, OutOfBoundsDatetime, OutOfBoundsTimedelta) as err: if isinstance(other, _Timestamp): if both_timestamps: @@ -507,9 +509,6 @@ cdef class _Timestamp(ABCTimestamp): return NotImplemented def __rsub__(self, other): - if self._reso != NPY_FR_ns: - raise NotImplementedError(self._reso) - if PyDateTime_Check(other): try: return type(self)(other) - self diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py index 79c8a300b34e3..db9c75eeb7485 100644 --- a/pandas/tests/scalar/timestamp/test_timestamp.py +++ b/pandas/tests/scalar/timestamp/test_timestamp.py @@ -22,6 +22,7 @@ from pandas._libs.tslibs.timezones import ( dateutil_gettz as gettz, get_timezone, + maybe_get_tz, tz_compare, ) from pandas.errors import OutOfBoundsDatetime @@ -712,6 +713,11 @@ def dt64(self, reso): def ts(self, dt64): return Timestamp._from_dt64(dt64) + @pytest.fixture + def ts_tz(self, ts, tz_aware_fixture): + tz = maybe_get_tz(tz_aware_fixture) + return Timestamp._from_value_and_reso(ts.value, ts._reso, tz) + def test_non_nano_construction(self, dt64, ts, reso): assert ts.value == dt64.view("i8") @@ -893,6 +899,60 @@ def test_addsub_timedeltalike_non_nano(self, dt64, ts, td): assert result._reso == ts._reso assert result == expected + @pytest.mark.xfail(reason="tz_localize not yet implemented for non-nano") + def test_addsub_offset(self, ts_tz): + # specifically non-Tick offset + off = offsets.YearBegin(1) + result = ts_tz + off + + assert isinstance(result, Timestamp) + assert result._reso == ts_tz._reso + # If ts_tz is ever on the last day of the year, the year would be + # incremented by one + assert result.year == ts_tz.year + assert result.day == 31 + assert result.month == 12 + assert tz_compare(result.tz, ts_tz.tz) + + def test_sub_datetimelike_mismatched_reso(self, ts_tz): + # case with non-lossy rounding + ts = ts_tz + + # choose a unit for `other` that doesn't match ts_tz's + unit = { + NpyDatetimeUnit.NPY_FR_us.value: "ms", + NpyDatetimeUnit.NPY_FR_ms.value: "s", + NpyDatetimeUnit.NPY_FR_s.value: "us", + }[ts._reso] + other = ts._as_unit(unit) + assert other._reso != ts._reso + + result = ts - other + assert isinstance(result, Timedelta) + assert result.value == 0 + assert result._reso == min(ts._reso, other._reso) + + result = other - ts + assert isinstance(result, Timedelta) + assert result.value == 0 + assert result._reso == min(ts._reso, other._reso) + + # TODO: clarify in message that add/sub is allowed only when lossless? + msg = "Cannot losslessly convert units" + if ts._reso < other._reso: + # Case where rounding is lossy + other2 = other + Timedelta._from_value_and_reso(1, other._reso) + with pytest.raises(ValueError, match=msg): + ts - other2 + with pytest.raises(ValueError, match=msg): + other2 - ts + else: + ts2 = ts + Timedelta._from_value_and_reso(1, ts._reso) + with pytest.raises(ValueError, match=msg): + ts2 - other + with pytest.raises(ValueError, match=msg): + other - ts2 + class TestAsUnit: def test_as_unit(self): From ecc7d80826dcfbec112c3594d65156f01092b47e Mon Sep 17 00:00:00 2001 From: Brock Date: Tue, 14 Jun 2022 13:09:15 -0700 Subject: [PATCH 3/9] better exception message --- pandas/_libs/tslibs/timestamps.pyx | 15 +++++++++++---- pandas/tests/scalar/timestamp/test_timestamp.py | 16 +++++++++++++--- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/pandas/_libs/tslibs/timestamps.pyx b/pandas/_libs/tslibs/timestamps.pyx index 8edfa789ec807..da2377a9b085c 100644 --- a/pandas/_libs/tslibs/timestamps.pyx +++ b/pandas/_libs/tslibs/timestamps.pyx @@ -479,10 +479,17 @@ cdef class _Timestamp(ABCTimestamp): # We allow silent casting to the lower resolution if and only # if it is lossless. - if self._reso < other._reso: - other = (<_Timestamp>other)._as_reso(self._reso, round_ok=False) - elif self._reso > other._reso: - self = (<_Timestamp>self)._as_reso(other._reso, round_ok=False) + try: + if self._reso < other._reso: + other = (<_Timestamp>other)._as_reso(self._reso, round_ok=False) + elif self._reso > other._reso: + self = (<_Timestamp>self)._as_reso(other._reso, round_ok=False) + except ValueError as err: + raise ValueError( + "Timestamp subtraction with mismatched resolutions is not " + "allowed when casting to the lower resolution would require " + "lossy rounding." + ) from err # scalar Timestamp/datetime - Timestamp/datetime -> yields a # Timedelta diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py index db9c75eeb7485..f7f19e49d0bac 100644 --- a/pandas/tests/scalar/timestamp/test_timestamp.py +++ b/pandas/tests/scalar/timestamp/test_timestamp.py @@ -914,11 +914,22 @@ def test_addsub_offset(self, ts_tz): assert result.month == 12 assert tz_compare(result.tz, ts_tz.tz) + result = ts_tz - off + + assert isinstance(result, Timestamp) + assert result._reso == ts_tz._reso + assert result.year == ts_tz.year - 1 + assert result.day == 31 + assert result.month == 12 + assert tz_compare(result.tz, ts_tz.tz) + def test_sub_datetimelike_mismatched_reso(self, ts_tz): # case with non-lossy rounding ts = ts_tz - # choose a unit for `other` that doesn't match ts_tz's + # choose a unit for `other` that doesn't match ts_tz's; + # this construction ensures we get cases with other._reso < ts._reso + # and cases with other._reso > ts._reso unit = { NpyDatetimeUnit.NPY_FR_us.value: "ms", NpyDatetimeUnit.NPY_FR_ms.value: "s", @@ -937,8 +948,7 @@ def test_sub_datetimelike_mismatched_reso(self, ts_tz): assert result.value == 0 assert result._reso == min(ts._reso, other._reso) - # TODO: clarify in message that add/sub is allowed only when lossless? - msg = "Cannot losslessly convert units" + msg = "Timestamp subtraction with mismatched resolutions" if ts._reso < other._reso: # Case where rounding is lossy other2 = other + Timedelta._from_value_and_reso(1, other._reso) From 5549bea415f7b8d8b2aeeb4c763ff6952b7e2ecb Mon Sep 17 00:00:00 2001 From: Patrick Hoefler <61934744+phofl@users.noreply.github.com> Date: Tue, 14 Jun 2022 18:32:19 +0200 Subject: [PATCH 4/9] BUG: concat not sorting mixed column names when None is included (#47331) * REGR: concat not sorting columns for mixed column names * Fix none in columns * BUG: concat not sorting column names when None is included * Update doc/source/whatsnew/v1.5.0.rst Co-authored-by: Matthew Roeschke * Add gh reference Co-authored-by: Matthew Roeschke --- doc/source/whatsnew/v1.5.0.rst | 1 + pandas/core/algorithms.py | 7 +++++-- pandas/tests/reshape/concat/test_concat.py | 6 +++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/doc/source/whatsnew/v1.5.0.rst b/doc/source/whatsnew/v1.5.0.rst index 5891eeea98cbb..681139fb51272 100644 --- a/doc/source/whatsnew/v1.5.0.rst +++ b/doc/source/whatsnew/v1.5.0.rst @@ -924,6 +924,7 @@ Reshaping - Bug in :func:`get_dummies` that selected object and categorical dtypes but not string (:issue:`44965`) - Bug in :meth:`DataFrame.align` when aligning a :class:`MultiIndex` to a :class:`Series` with another :class:`MultiIndex` (:issue:`46001`) - Bug in concanenation with ``IntegerDtype``, or ``FloatingDtype`` arrays where the resulting dtype did not mirror the behavior of the non-nullable dtypes (:issue:`46379`) +- Bug in :func:`concat` not sorting the column names when ``None`` is included (:issue:`47331`) - Bug in :func:`concat` with identical key leads to error when indexing :class:`MultiIndex` (:issue:`46519`) - Bug in :meth:`DataFrame.join` with a list when using suffixes to join DataFrames with duplicate column names (:issue:`46396`) - Bug in :meth:`DataFrame.pivot_table` with ``sort=False`` results in sorted index (:issue:`17041`) diff --git a/pandas/core/algorithms.py b/pandas/core/algorithms.py index 888e943488953..cf73fd7c8929e 100644 --- a/pandas/core/algorithms.py +++ b/pandas/core/algorithms.py @@ -1771,9 +1771,12 @@ def safe_sort( def _sort_mixed(values) -> np.ndarray: """order ints before strings in 1d arrays, safe in py3""" str_pos = np.array([isinstance(x, str) for x in values], dtype=bool) - nums = np.sort(values[~str_pos]) + none_pos = np.array([x is None for x in values], dtype=bool) + nums = np.sort(values[~str_pos & ~none_pos]) strs = np.sort(values[str_pos]) - return np.concatenate([nums, np.asarray(strs, dtype=object)]) + return np.concatenate( + [nums, np.asarray(strs, dtype=object), np.array(values[none_pos])] + ) def _sort_tuples(values: np.ndarray) -> np.ndarray: diff --git a/pandas/tests/reshape/concat/test_concat.py b/pandas/tests/reshape/concat/test_concat.py index eb44b4889afb8..17c797fc36159 100644 --- a/pandas/tests/reshape/concat/test_concat.py +++ b/pandas/tests/reshape/concat/test_concat.py @@ -469,12 +469,12 @@ def __iter__(self): tm.assert_frame_equal(concat(CustomIterator2(), ignore_index=True), expected) def test_concat_order(self): - # GH 17344 + # GH 17344, GH#47331 dfs = [DataFrame(index=range(3), columns=["a", 1, None])] - dfs += [DataFrame(index=range(3), columns=[None, 1, "a"]) for i in range(100)] + dfs += [DataFrame(index=range(3), columns=[None, 1, "a"]) for _ in range(100)] result = concat(dfs, sort=True).columns - expected = dfs[0].columns + expected = Index([1, "a", None]) tm.assert_index_equal(result, expected) def test_concat_different_extension_dtypes_upcasts(self): From e6ff89c1da87cdd5c75d91884bae014a80470dce Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Tue, 14 Jun 2022 18:49:16 +0200 Subject: [PATCH 5/9] Add run-tests action (#47292) * Add run-tests action * Fix * Fix * Fix * Update macos-windows.yml * Update posix.yml * Update python-dev.yml * Update action.yml * Update macos-windows.yml * Update posix.yml * Update python-dev.yml * Update python-dev.yml * Update python-dev.yml * Update python-dev.yml * Update python-dev.yml * Update python-dev.yml * Update python-dev.yml * Update python-dev.yml * Update python-dev.yml --- .github/actions/run-tests/action.yml | 27 +++++++++++++++++++ .github/workflows/macos-windows.yml | 16 +---------- .github/workflows/posix.yml | 19 +------------ .github/workflows/python-dev.yml | 40 +++++++--------------------- 4 files changed, 39 insertions(+), 63 deletions(-) create mode 100644 .github/actions/run-tests/action.yml diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml new file mode 100644 index 0000000000000..2a7601f196ec4 --- /dev/null +++ b/.github/actions/run-tests/action.yml @@ -0,0 +1,27 @@ +name: Run tests and report results +runs: + using: composite + steps: + - name: Test + run: ci/run_tests.sh + shell: bash -el {0} + + - name: Publish test results + uses: actions/upload-artifact@v2 + with: + name: Test results + path: test-data.xml + if: failure() + + - name: Report Coverage + run: coverage report -m + shell: bash -el {0} + if: failure() + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v2 + with: + flags: unittests + name: codecov-pandas + fail_ci_if_error: false + if: failure() diff --git a/.github/workflows/macos-windows.yml b/.github/workflows/macos-windows.yml index 26e6c8699ca64..4c48d83b68947 100644 --- a/.github/workflows/macos-windows.yml +++ b/.github/workflows/macos-windows.yml @@ -53,18 +53,4 @@ jobs: uses: ./.github/actions/build_pandas - name: Test - run: ci/run_tests.sh - - - name: Publish test results - uses: actions/upload-artifact@v3 - with: - name: Test results - path: test-data.xml - if: failure() - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2 - with: - flags: unittests - name: codecov-pandas - fail_ci_if_error: false + uses: ./.github/actions/run-tests diff --git a/.github/workflows/posix.yml b/.github/workflows/posix.yml index 061b2b361ca62..831bbd8bb3233 100644 --- a/.github/workflows/posix.yml +++ b/.github/workflows/posix.yml @@ -157,23 +157,6 @@ jobs: uses: ./.github/actions/build_pandas - name: Test - run: ci/run_tests.sh + uses: ./.github/actions/run-tests # TODO: Don't continue on error for PyPy continue-on-error: ${{ env.IS_PYPY == 'true' }} - - - name: Build Version - run: conda list - - - name: Publish test results - uses: actions/upload-artifact@v3 - with: - name: Test results - path: test-data.xml - if: failure() - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2 - with: - flags: unittests - name: codecov-pandas - fail_ci_if_error: false diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index 753e288f5e391..09639acafbba1 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -57,40 +57,20 @@ jobs: - name: Install dependencies shell: bash -el {0} run: | - python -m pip install --upgrade pip setuptools wheel - pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy - pip install git+https://github.com/nedbat/coveragepy.git - pip install cython python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov - pip list + python3 -m pip install --upgrade pip setuptools wheel + python3 -m pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy + python3 -m pip install git+https://github.com/nedbat/coveragepy.git + python3 -m pip install cython python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17 + python3 -m pip list - name: Build Pandas run: | - python setup.py build_ext -q -j2 - python -m pip install -e . --no-build-isolation --no-use-pep517 + python3 setup.py build_ext -q -j2 + python3 -m pip install -e . --no-build-isolation --no-use-pep517 - name: Build Version run: | - python -c "import pandas; pandas.show_versions();" + python3 -c "import pandas; pandas.show_versions();" - - name: Test with pytest - shell: bash -el {0} - run: | - ci/run_tests.sh - - - name: Publish test results - uses: actions/upload-artifact@v3 - with: - name: Test results - path: test-data.xml - if: failure() - - - name: Report Coverage - run: | - coverage report -m - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2 - with: - flags: unittests - name: codecov-pandas - fail_ci_if_error: true + - name: Test + uses: ./.github/actions/run-tests From ae2e6de5c5c7847e240632f8095c1f93c593094d Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 14 Jun 2022 09:54:15 -0700 Subject: [PATCH 6/9] ENH: Timestamp pickle support non-nano tzaware (#47340) --- pandas/_libs/tslibs/timestamps.pyx | 9 +-------- pandas/tests/scalar/timestamp/test_timestamp.py | 5 ++++- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/pandas/_libs/tslibs/timestamps.pyx b/pandas/_libs/tslibs/timestamps.pyx index da2377a9b085c..2694991b54d4a 100644 --- a/pandas/_libs/tslibs/timestamps.pyx +++ b/pandas/_libs/tslibs/timestamps.pyx @@ -157,14 +157,7 @@ cdef inline _Timestamp create_timestamp_from_ts( def _unpickle_timestamp(value, freq, tz, reso=NPY_FR_ns): # GH#41949 dont warn on unpickle if we have a freq - if reso == NPY_FR_ns: - ts = Timestamp(value, tz=tz) - else: - if tz is not None: - raise NotImplementedError - abbrev = npy_unit_to_abbrev(reso) - dt64 = np.datetime64(value, abbrev) - ts = Timestamp._from_dt64(dt64) + ts = Timestamp._from_value_and_reso(value, reso, tz) ts._set_freq(freq) return ts diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py index f7f19e49d0bac..a02268956651c 100644 --- a/pandas/tests/scalar/timestamp/test_timestamp.py +++ b/pandas/tests/scalar/timestamp/test_timestamp.py @@ -842,7 +842,10 @@ def test_cmp_cross_reso_reversed_dt64(self): assert other.asm8 < ts - def test_pickle(self, ts): + def test_pickle(self, ts, tz_aware_fixture): + tz = tz_aware_fixture + tz = maybe_get_tz(tz) + ts = Timestamp._from_value_and_reso(ts.value, ts._reso, tz) rt = tm.round_trip_pickle(ts) assert rt._reso == ts._reso assert rt == ts From 7d39cb37fed11420d2c09ecf3d78c7f61c9c569f Mon Sep 17 00:00:00 2001 From: Brock Date: Fri, 10 Jun 2022 16:32:11 -0700 Subject: [PATCH 7/9] ENH: DTA to_pydatetime, time, timetz, date, iter support non-nano --- pandas/_libs/tslibs/vectorized.pyi | 1 + pandas/_libs/tslibs/vectorized.pyx | 13 ++++++++---- pandas/core/arrays/datetimelike.py | 5 +++-- pandas/core/arrays/datetimes.py | 14 ++++++++----- pandas/tests/arrays/test_datetimes.py | 29 +++++++++++++++++++++++++++ 5 files changed, 51 insertions(+), 11 deletions(-) diff --git a/pandas/_libs/tslibs/vectorized.pyi b/pandas/_libs/tslibs/vectorized.pyi index b0c7644e892d2..7eb4695b9ca2c 100644 --- a/pandas/_libs/tslibs/vectorized.pyi +++ b/pandas/_libs/tslibs/vectorized.pyi @@ -37,6 +37,7 @@ def ints_to_pydatetime( freq: BaseOffset | None = ..., fold: bool = ..., box: str = ..., + reso: int = ..., # NPY_DATETIMEUNIT ) -> npt.NDArray[np.object_]: ... def tz_convert_from_utc( stamps: npt.NDArray[np.int64], diff --git a/pandas/_libs/tslibs/vectorized.pyx b/pandas/_libs/tslibs/vectorized.pyx index 75efe6d4113cf..e5b0ba3f4dd80 100644 --- a/pandas/_libs/tslibs/vectorized.pyx +++ b/pandas/_libs/tslibs/vectorized.pyx @@ -100,7 +100,8 @@ def ints_to_pydatetime( tzinfo tz=None, BaseOffset freq=None, bint fold=False, - str box="datetime" + str box="datetime", + NPY_DATETIMEUNIT reso=NPY_FR_ns, ) -> np.ndarray: # stamps is int64, arbitrary ndim """ @@ -126,12 +127,14 @@ def ints_to_pydatetime( * If time, convert to datetime.time * If Timestamp, convert to pandas.Timestamp + reso : NPY_DATETIMEUNIT, default NPY_FR_ns + Returns ------- ndarray[object] of type specified by box """ cdef: - Localizer info = Localizer(tz, reso=NPY_FR_ns) + Localizer info = Localizer(tz, reso=reso) int64_t utc_val, local_val Py_ssize_t i, n = stamps.size Py_ssize_t pos = -1 # unused, avoid not-initialized warning @@ -179,10 +182,12 @@ def ints_to_pydatetime( # find right representation of dst etc in pytz timezone new_tz = tz._tzinfos[tz._transition_info[pos]] - dt64_to_dtstruct(local_val, &dts) + pandas_datetime_to_datetimestruct(local_val, reso, &dts) if use_ts: - res_val = create_timestamp_from_ts(utc_val, dts, new_tz, freq, fold) + res_val = create_timestamp_from_ts( + utc_val, dts, new_tz, freq, fold, reso=reso + ) elif use_pydt: res_val = datetime( dts.year, dts.month, dts.day, dts.hour, dts.min, dts.sec, dts.us, diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 1dfb070e29c30..1124a5dbca6ce 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -426,15 +426,16 @@ def astype(self, dtype, copy: bool = True): if self.dtype.kind == "M": # *much* faster than self._box_values # for e.g. test_get_loc_tuple_monotonic_above_size_cutoff - i8data = self.asi8.ravel() + i8data = self.asi8 converted = ints_to_pydatetime( i8data, # error: "DatetimeLikeArrayMixin" has no attribute "tz" tz=self.tz, # type: ignore[attr-defined] freq=self.freq, box="timestamp", + reso=self._reso, ) - return converted.reshape(self.shape) + return converted elif self.dtype.kind == "m": return ints_to_pytimedelta(self._ndarray, box=True) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index d297d3e9f8e4e..18133d7cf25ea 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -653,7 +653,11 @@ def __iter__(self): start_i = i * chunksize end_i = min((i + 1) * chunksize, length) converted = ints_to_pydatetime( - data[start_i:end_i], tz=self.tz, freq=self.freq, box="timestamp" + data[start_i:end_i], + tz=self.tz, + freq=self.freq, + box="timestamp", + reso=self._reso, ) yield from converted @@ -1044,7 +1048,7 @@ def to_pydatetime(self) -> npt.NDArray[np.object_]: ------- datetimes : ndarray[object] """ - return ints_to_pydatetime(self.asi8, tz=self.tz) + return ints_to_pydatetime(self.asi8, tz=self.tz, reso=self._reso) def normalize(self) -> DatetimeArray: """ @@ -1301,7 +1305,7 @@ def time(self) -> npt.NDArray[np.object_]: # keeping their timezone and not using UTC timestamps = self._local_timestamps() - return ints_to_pydatetime(timestamps, box="time") + return ints_to_pydatetime(timestamps, box="time", reso=self._reso) @property def timetz(self) -> npt.NDArray[np.object_]: @@ -1311,7 +1315,7 @@ def timetz(self) -> npt.NDArray[np.object_]: The time part of the Timestamps. """ - return ints_to_pydatetime(self.asi8, self.tz, box="time") + return ints_to_pydatetime(self.asi8, self.tz, box="time", reso=self._reso) @property def date(self) -> npt.NDArray[np.object_]: @@ -1326,7 +1330,7 @@ def date(self) -> npt.NDArray[np.object_]: # keeping their timezone and not using UTC timestamps = self._local_timestamps() - return ints_to_pydatetime(timestamps, box="date") + return ints_to_pydatetime(timestamps, box="date", reso=self._reso) def isocalendar(self) -> DataFrame: """ diff --git a/pandas/tests/arrays/test_datetimes.py b/pandas/tests/arrays/test_datetimes.py index d4f8e5b76a7c5..d82e865b069aa 100644 --- a/pandas/tests/arrays/test_datetimes.py +++ b/pandas/tests/arrays/test_datetimes.py @@ -126,6 +126,35 @@ def test_to_period(self, dta_dti): tm.assert_extension_array_equal(result, expected) + def test_iter(self, dta): + res = next(iter(dta)) + expected = dta[0] + + assert type(res) is pd.Timestamp + assert res.value == expected.value + assert res._reso == expected._reso + assert res == expected + + def test_astype_object(self, dta): + result = dta.astype(object) + assert all(x._reso == dta._reso for x in result) + assert all(x == y for x, y in zip(result, dta)) + + def test_to_pydatetime(self, dta_dti): + dta, dti = dta_dti + + result = dta.to_pydatetime() + expected = dti.to_pydatetime() + tm.assert_numpy_array_equal(result, expected) + + @pytest.mark.parametrize("meth", ["time", "timetz", "date"]) + def test_time_date(self, dta_dti, meth): + dta, dti = dta_dti + + result = getattr(dta, meth) + expected = getattr(dti, meth) + tm.assert_numpy_array_equal(result, expected) + class TestDatetimeArrayComparisons: # TODO: merge this into tests/arithmetic/test_datetime64 once it is From 429c0df5f4cd69bdf5d246a4f84423afdbaf767a Mon Sep 17 00:00:00 2001 From: Brock Date: Tue, 14 Jun 2022 13:10:14 -0700 Subject: [PATCH 8/9] cast in liboffsets --- pandas/_libs/tslibs/offsets.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/_libs/tslibs/offsets.pyx b/pandas/_libs/tslibs/offsets.pyx index f1ebe9dd6348f..f29e6a2d0d9ea 100644 --- a/pandas/_libs/tslibs/offsets.pyx +++ b/pandas/_libs/tslibs/offsets.pyx @@ -3115,7 +3115,7 @@ cdef class FY5253Quarter(FY5253Mixin): for qlen in qtr_lens: if qlen * 7 <= tdelta.days: num_qtrs += 1 - tdelta -= Timedelta(days=qlen * 7) + tdelta -= (<_Timedelta>Timedelta(days=qlen * 7))._as_reso(norm._reso) else: break else: From bdec4f433da2344d231315532f3381ab26b4b4ea Mon Sep 17 00:00:00 2001 From: Brock Date: Tue, 14 Jun 2022 14:50:34 -0700 Subject: [PATCH 9/9] mypy fixup --- pandas/core/arrays/datetimelike.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 1124a5dbca6ce..b9388c8047df9 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -424,13 +424,13 @@ def astype(self, dtype, copy: bool = True): if is_object_dtype(dtype): if self.dtype.kind == "M": + self = cast("DatetimeArray", self) # *much* faster than self._box_values # for e.g. test_get_loc_tuple_monotonic_above_size_cutoff i8data = self.asi8 converted = ints_to_pydatetime( i8data, - # error: "DatetimeLikeArrayMixin" has no attribute "tz" - tz=self.tz, # type: ignore[attr-defined] + tz=self.tz, freq=self.freq, box="timestamp", reso=self._reso,