Skip to content

BUG: GH29461 strftime() with nanoseconds for Timestamp #34317

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 30 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
12d0b4d
TST: GH28813 test .diff() on Sparse dtype
matteosantama May 19, 2020
02c4a85
TST: GH28813 test .diff() on Sparse dtype
matteosantama May 20, 2020
7e3256b
TST: GH28813 pull sparse diff() test into its own function
matteosantama May 20, 2020
f00e48a
Merge branch 'sparse_diff'
matteosantama May 22, 2020
a90f928
Merge branch 'master' of https://github.com/pandas-dev/pandas
matteosantama May 22, 2020
26a920c
BUG: GH29461 display nanoseconds with strftime()
matteosantama May 22, 2020
3a529ac
BUG: GH29461 don't display nanoseconds in strftime if none exists for…
matteosantama May 22, 2020
d0124aa
update whatsnew doc
matteosantama May 22, 2020
0c0aaf2
black formatting
matteosantama May 22, 2020
abdbe4e
remove trailing whitespace to conform to CI
matteosantama May 22, 2020
97ae6b3
still trying to pass linting checks
matteosantama May 22, 2020
1132aba
Merge branch 'master' of https://github.com/pandas-dev/pandas
matteosantama May 23, 2020
0e42026
Merge branch 'master' of github.com:matteosantama/pandas into strftime
matteosantama May 23, 2020
ab0c9d4
Add strftime benchmarks
matteosantama May 23, 2020
532b19c
Early exit strftime if no nanoseconds
matteosantama May 23, 2020
88fba5e
Make loop more pythonic
matteosantama May 24, 2020
2ac690c
Fix benchmark test
matteosantama May 24, 2020
3b533c9
Remove whitespace
matteosantama May 24, 2020
f6ba1c9
Remove extra function call
matteosantama May 26, 2020
07b27e2
Benchmark series.strftime()
matteosantama May 26, 2020
4b57c4f
Merge branch 'master' of https://github.com/pandas-dev/pandas into st…
matteosantama May 27, 2020
d72222a
Use explicitly named parameters in testing
Jun 8, 2020
e920d20
Use regex for replacing %f
Jun 8, 2020
34db469
Clean up Timestamp._time_repr to use new strftime functionality
Jun 8, 2020
fbe286e
Commiting so I can merge master
Jun 8, 2020
54d30eb
Merge branch 'master' of https://github.com/pandas-dev/pandas into st…
Jun 8, 2020
90629c5
Use regex for replacing %f
Jun 9, 2020
821dfbb
Clean up _time_repr to use new strftime functionality
Jun 9, 2020
16b0f9f
Test for all datetime strftime directives
matteosantama Jun 9, 2020
b8565f2
Test for all datetime strftime directives
matteosantama Jun 9, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 21 additions & 12 deletions asv_bench/benchmarks/timeseries.py
Original file line number Diff line number Diff line change
Expand Up @@ -394,33 +394,42 @@ def time_dup_string_tzoffset_dates(self, cache):

class DatetimeAccessor:

params = [None, "US/Eastern", "UTC", dateutil.tz.tzutc()]
param_names = "tz"

def setup(self, tz):
params = (
[None, "US/Eastern", "UTC", dateutil.tz.tzutc()],
["%Y-%m-%d %H:%M:%S.%f%z", "%Y-%m-%d %H:%M:%S%z"],
["T", "S", "NS"],
)
param_names = ["tz", "fmt", "frequency"]

def setup(self, tz, fmt, frequency):
N = 100000
self.series = Series(date_range(start="1/1/2000", periods=N, freq="T", tz=tz))
self.series = Series(
date_range(start="1/1/2000", periods=N, freq=frequency, tz=tz)
)

def time_dt_accessor(self, tz):
def time_dt_accessor(self, tz, fmt, frequency):
self.series.dt

def time_dt_accessor_normalize(self, tz):
def time_dt_accessor_normalize(self, tz, fmt, frequency):
self.series.dt.normalize()

def time_dt_accessor_month_name(self, tz):
def time_dt_accessor_month_name(self, tz, fmt, frequency):
self.series.dt.month_name()

def time_dt_accessor_day_name(self, tz):
def time_dt_accessor_day_name(self, tz, fmt, frequency):
self.series.dt.day_name()

def time_dt_accessor_time(self, tz):
def time_dt_accessor_time(self, tz, fmt, frequency):
self.series.dt.time

def time_dt_accessor_date(self, tz):
def time_dt_accessor_date(self, tz, fmt, frequency):
self.series.dt.date

def time_dt_accessor_year(self, tz):
def time_dt_accessor_year(self, tz, fmt, frequency):
self.series.dt.year

def time_dt_accessor_strftime(self, tz, fmt, frequency):
self.series.dt.strftime(fmt)


from .pandas_vb_common import setup # noqa: F401 isort:skip
11 changes: 11 additions & 0 deletions asv_bench/benchmarks/tslibs/timestamp.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,17 @@ def time_month_name(self, tz, freq):
self.ts.month_name()


class TimestampMethods:
params = ["%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M:%S.%f"]
param_names = ["fmt"]

def setup(self, fmt):
self.ts = Timestamp("2020-05-23 18:06:13.123456789")

def time_strftime(self, fmt):
self.ts.strftime(fmt)


class TimestampOps:
params = [None, "US/Eastern", pytz.UTC, dateutil.tz.tzutc()]
param_names = ["tz"]
Expand Down
1 change: 1 addition & 0 deletions doc/source/whatsnew/v1.1.0.rst
Original file line number Diff line number Diff line change
Expand Up @@ -815,6 +815,7 @@ Datetimelike
- Bug in :meth:`DatetimeIndex.intersection` and :meth:`TimedeltaIndex.intersection` with results not having the correct ``name`` attribute (:issue:`33904`)
- Bug in :meth:`DatetimeArray.__setitem__`, :meth:`TimedeltaArray.__setitem__`, :meth:`PeriodArray.__setitem__` incorrectly allowing values with ``int64`` dtype to be silently cast (:issue:`33717`)
- Bug in subtracting :class:`TimedeltaIndex` from :class:`Period` incorrectly raising ``TypeError`` in some cases where it should succeed and ``IncompatibleFrequency`` in some cases where it should raise ``TypeError`` (:issue:`33883`)
- Bug in :meth:`Timestamp.strftime` did not display full nanosecond precision (:issue:`29461`)

Timedelta
^^^^^^^^^
Expand Down
22 changes: 14 additions & 8 deletions pandas/_libs/tslibs/timestamps.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ construction requirements, we need to do object instantiation in python
shadows the python class, where we do any heavy lifting.
"""
import warnings
import time as _time
import re

import numpy as np
cimport numpy as cnp
Expand Down Expand Up @@ -493,14 +495,7 @@ cdef class _Timestamp(ABCTimestamp):

@property
def _time_repr(self) -> str:
result = f'{self.hour:02d}:{self.minute:02d}:{self.second:02d}'

if self.nanosecond != 0:
result += f'.{self.nanosecond + 1000 * self.microsecond:09d}'
elif self.microsecond != 0:
result += f'.{self.microsecond:06d}'

return result
return self.strftime('%H:%M:%S.%f')

@property
def _short_repr(self) -> str:
Expand Down Expand Up @@ -1455,6 +1450,17 @@ default 'raise'
np.array([self.value], dtype="i8"), tz=own_tz)
return Timestamp(normalized[0]).tz_localize(own_tz)

def strftime(self, format: str) -> str:
# time.strftime() doesn't support %f so we manually replace it
if '%f' in format:
# always show six digits of microseconds, even if its 0s
replacement = f'{self.microsecond:06d}'
# only show nanoseconds if we have them (for comparison to datetime)
if self.nanosecond:
replacement = f'{self.microsecond * 1000 + self.nanosecond:09d}'
format = re.sub('%f', replacement, format)
return _time.strftime(format, self.timetuple())


# Add the min and max fields at the class level
cdef int64_t _NS_UPPER_BOUND = np.iinfo(np.int64).max
Expand Down
66 changes: 65 additions & 1 deletion pandas/tests/scalar/timestamp/test_timestamp.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
from pandas.compat.numpy import np_datetime64_compat
import pandas.util._test_decorators as td

from pandas import NaT, Timedelta, Timestamp
from pandas import NaT, Timedelta, Timestamp, to_datetime
import pandas._testing as tm

from pandas.tseries import offsets
Expand Down Expand Up @@ -381,6 +381,56 @@ def test_tz_conversion_freq(self, tz_naive_fixture):
t2 = Timestamp("2019-01-02 12:00", tz="UTC", freq="T")
assert t2.tz_convert(tz="UTC").freq == t2.freq

@pytest.mark.parametrize(
"_input,fmt,_output",
[
("2020-05-22 11:07:30", "%Y-%m-%d", "2020-05-22"),
("2020-05-22 11:07:30.123456", "%Y-%m-%d %f", "2020-05-22 123456"),
("2020-05-22 11:07:30.123456789", "%f", "123456789"),
],
)
def test_strftime(self, _input, fmt, _output):
ts = Timestamp(_input)
result = ts.strftime(fmt)
assert result == _output

@pytest.mark.parametrize(
"fmt",
[
"%a",
"%A",
"%w",
"%d",
"%b",
"%B",
"%m",
"%y",
"%Y",
"%H",
"%I",
"%p",
"%M",
"%S",
"%f",
"%z",
"%Z",
"%j",
"%U",
"%W",
"%c",
"%x",
"%X",
"%G",
"%u",
"%V",
],
)
def test_strftime_components(self, fmt):
ts = Timestamp("2020-06-09 09:04:11.123456", tz="UTC")
dt = to_datetime(ts)
assert isinstance(ts, Timestamp) and isinstance(dt, datetime)
assert ts.strftime(fmt) == dt.strftime(fmt)


class TestTimestampNsOperations:
def test_nanosecond_string_parsing(self):
Expand Down Expand Up @@ -442,6 +492,20 @@ def test_nanosecond_timestamp(self):
assert t.value == expected
assert t.nanosecond == 10

@pytest.mark.parametrize(
"date",
[
"2020-05-22 08:53:19.123456789",
"2020-05-22 08:53:19.123456",
"2020-05-22 08:53:19",
],
)
@pytest.mark.parametrize("fmt", ["%m/%d/%Y %H:%M:%S.%f", "%m%d%Y%H%M%S%f"])
def test_nanosecond_roundtrip(self, date, fmt):
ts = Timestamp(date)
string = ts.strftime(fmt)
assert ts == to_datetime(string, format=fmt)


class TestTimestampToJulianDate:
def test_compare_1700(self):
Expand Down