Better datetime to ms (pandas-dev#549)

* Avoid floating point in datetime_to_ms (Issue 483)

* Added integration test for datetime_to_ms bug

* Truncate microseconds not round

Author: srf94

CHANGES.md

@@ -2,6 +2,7 @@
 
 ### 1.66
   * Bugfix: #168 Do not allow empty string as a column name
+  * Bugfix: #483 Remove potential floating point error from datetime_to_ms
 
 ### 1.65 (2018-04-16)
   * Bugfix: #534 VersionStore: overwriting a symbol with different dtype (but same data format) does not

arctic/date/_util.py

@@ -163,7 +163,8 @@ def _add_tzone(dtm):
 def datetime_to_ms(d):
     """Convert a Python datetime object to a millisecond epoch (UTC) time value."""
     try:
-        return long((calendar.timegm(_add_tzone(d).utctimetuple()) + d.microsecond / 1000000.0) * 1e3)
+        millisecond = d.microsecond // 1000
+        return calendar.timegm(_add_tzone(d).utctimetuple()) * 1000 + millisecond
     except AttributeError:
         raise TypeError('expect Python datetime object, not %s' % type(d))
 

tests/integration/tickstore/test_ts_write.py

@@ -3,9 +3,8 @@
 import numpy as np
 from pandas.util.testing import assert_frame_equal
 import pytest
-
-from arctic import arctic as m
-from arctic.date import mktz
+import pytz
+from arctic.date import mktz, DateRange
 from arctic.exceptions import OverlappingDataException, \
     NoDataFoundException
 from arctic.tickstore.tickstore import SYMBOL, START, END, COUNT, COLUMNS
@@ -79,7 +78,7 @@ def test_ts_write_pandas(tickstore_lib):
 
     read = tickstore_lib.read('SYM', columns=None)
     assert_frame_equal(read, data, check_names=False)
-    
+
 
 def test_ts_write_named_col(tickstore_lib):
     data = DUMMY_DATA
@@ -96,3 +95,16 @@ def test_ts_write_named_col(tickstore_lib):
 
     read = tickstore_lib.read('SYM')
     assert(read.index.name is None)
+
+
+def test_millisecond_roundtrip(tickstore_lib):
+    test_time = dt(2004, 1, 14, 8, 30, 4, 807000, tzinfo=pytz.utc)
+
+    data = [{'index': test_time, 'price': 9142.12, 'qualifiers': ''}]
+    tickstore_lib.write('blah', data)
+
+    data_range = DateRange(dt(2004, 1, 14, tzinfo=pytz.utc),
+                           dt(2004, 1, 15, tzinfo=pytz.utc))
+    reread = tickstore_lib.read('blah', data_range)
+
+    assert reread.index[0].to_datetime() == test_time

tests/unit/date/test_datetime_to_ms_roundtrip.py

@@ -71,3 +71,14 @@ def test_datetime_roundtrip_est_tz():
     pdt = datetime.datetime(2012, 1, 12, 12, 12, 12, 123000, tzinfo=mktz('EST'))
     pdt2 = ms_to_datetime(datetime_to_ms(pdt))
     assert pdt2.replace(tzinfo=mktz()) == pdt
+
+
+@pytest.mark.parametrize("microseconds,expected", [
+    (807000, 1074069004807),
+    (807243, 1074069004807),
+    (807675, 1074069004807)
+])
+def test_millisecond_conversion(microseconds, expected):
+    pdt = datetime.datetime(2004, 1, 14, 8, 30, 4, microseconds, tzinfo=pytz.utc)
+    pdt2 = datetime_to_ms(pdt)
+    assert pdt2 == expected