Skip to content

Commit 7b9a31d

Browse files
dimospeddimosped
authored
Merge pull request pandas-dev#574 from scriada/add-host-to-VersionedItem
add host to VersionedItem (adds feature pandas-dev#595)
2 parents bdb2868 + b79c082 commit 7b9a31d

File tree

7 files changed

+77
-41
lines changed

7 files changed

+77
-41
lines changed

‎CHANGES.md

+1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
* Feature: #571 Removed the Cython LZ4 code, use the latest python-lz4
66
* Feature: #557 Threadpool based compression. Speed imrpovement and tuning benchmarks.
77
* Bugfix: fix tickstore unicode handling, support both unicode and utf-8 arrays
8+
* Feature: #595 add host attribute to VersionedItem.
89

910
### 1.67.1 (2018-07-11)
1011
* Bugfix: #579 Fix symbol corruption due to restore_version and append

‎arctic/store/audit.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ def __init__(self, version_store, symbol, user, log, modify_timeseries=None, aud
9292
versions = [x['version'] for x in self._version_store.list_versions(self._symbol, latest_only=True)]
9393
versions.append(0)
9494
self.base_ts = VersionedItem(symbol=self._symbol, library=None,
95-
version=versions[0], metadata=None, data=None)
95+
version=versions[0], metadata=None, data=None, host=None)
9696
except OperationFailure:
9797
#TODO: Current errors in mongo "Incorrect Number of Segments Returned"
9898
# This workaround should be removed once underlying problem is resolved.

‎arctic/store/version_store.py

+13-6
Original file line numberDiff line numberDiff line change
@@ -390,7 +390,8 @@ def _do_read(self, symbol, version, from_version=None, **kwargs):
390390
handler = self._read_handler(version, symbol)
391391
data = handler.read(self._arctic_lib, version, symbol, from_version=from_version, **kwargs)
392392
return VersionedItem(symbol=symbol, library=self._arctic_lib.get_name(), version=version['version'],
393-
metadata=version.pop('metadata', None), data=data)
393+
metadata=version.pop('metadata', None), data=data,
394+
host=self._arctic_lib.arctic.mongo_host)
394395
_do_read_retry = mongo_retry(_do_read)
395396

396397
@mongo_retry
@@ -416,7 +417,8 @@ def read_metadata(self, symbol, as_of=None, allow_secondary=None):
416417
"""
417418
_version = self._read_metadata(symbol, as_of=as_of, read_preference=self._read_preference(allow_secondary))
418419
return VersionedItem(symbol=symbol, library=self._arctic_lib.get_name(), version=_version['version'],
419-
metadata=_version.pop('metadata', None), data=None)
420+
metadata=_version.pop('metadata', None), data=None,
421+
host=self._arctic_lib.arctic.mongo_host)
420422

421423
def _read_metadata(self, symbol, as_of=None, read_preference=None):
422424
if read_preference is None:
@@ -496,7 +498,8 @@ def append(self, symbol, data, metadata=None, prune_previous_version=True, upser
496498

497499
if len(data) == 0 and previous_version is not None:
498500
return VersionedItem(symbol=symbol, library=self._arctic_lib.get_name(), version=previous_version['version'],
499-
metadata=version.pop('metadata', None), data=None)
501+
metadata=version.pop('metadata', None), data=None,
502+
host=self._arctic_lib.arctic.mongo_host)
500503

501504
if upsert and previous_version is None:
502505
return self.write(symbol=symbol, data=data, prune_previous_version=prune_previous_version, metadata=metadata)
@@ -549,7 +552,8 @@ def append(self, symbol, data, metadata=None, prune_previous_version=True, upser
549552
self._insert_version(version)
550553

551554
return VersionedItem(symbol=symbol, library=self._arctic_lib.get_name(), version=version['version'],
552-
metadata=version.pop('metadata', None), data=None)
555+
metadata=version.pop('metadata', None), data=None,
556+
host=self._arctic_lib.arctic.mongo_host)
553557

554558
def _publish_change(self, symbol, version):
555559
if self._publish_changes:
@@ -605,7 +609,8 @@ def write(self, symbol, data, metadata=None, prune_previous_version=True, **kwar
605609
logger.debug('Finished writing versions for %s', symbol)
606610

607611
return VersionedItem(symbol=symbol, library=self._arctic_lib.get_name(), version=version['version'],
608-
metadata=version.pop('metadata', None), data=None)
612+
metadata=version.pop('metadata', None), data=None,
613+
host=self._arctic_lib.arctic.mongo_host)
609614

610615
def _add_new_version_using_reference(self, symbol, new_version, reference_version, prune_previous_version):
611616
# Attention: better not use this method following an append.
@@ -650,7 +655,8 @@ def _add_new_version_using_reference(self, symbol, new_version, reference_versio
650655
self._publish_change(symbol, new_version)
651656

652657
return VersionedItem(symbol=symbol, library=self._arctic_lib.get_name(), version=new_version['version'],
653-
metadata=new_version.get('metadata'), data=None)
658+
metadata=new_version.get('metadata'), data=None,
659+
host=self._arctic_lib.arctic.mongo_host)
654660

655661
@mongo_retry
656662
def write_metadata(self, symbol, metadata, prune_previous_version=True, **kwargs):
@@ -738,6 +744,7 @@ def restore_version(self, symbol, as_of, prune_previous_version=True):
738744
if self._last_version_seqnum(symbol) == version_to_restore['version']:
739745
return VersionedItem(symbol=symbol, library=self._arctic_lib.get_name(),
740746
version=version_to_restore['version'],
747+
host=self._arctic_lib.arctic.mongo_host,
741748
metadata=version_to_restore.pop('metadata', None), data=None)
742749

743750
# Read the existing data from as_of

‎arctic/store/versioned_item.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
from collections import namedtuple
22

33

4-
class VersionedItem(namedtuple('VersionedItem', ['symbol', 'library', 'data', 'version', 'metadata'])):
4+
class VersionedItem(namedtuple('VersionedItem', ['symbol', 'library', 'data', 'version', 'metadata', 'host'])):
55
"""
66
Class representing a Versioned object in VersionStore.
77
"""
@@ -12,8 +12,8 @@ def __repr__(self):
1212
return str(self)
1313

1414
def __str__(self):
15-
return "VersionedItem(symbol=%s,library=%s,data=%s,version=%s,metadata=%s" % \
16-
(self.symbol, self.library, type(self.data), self.version, self.metadata)
15+
return "VersionedItem(symbol=%s,library=%s,data=%s,version=%s,metadata=%s,host=%s)" % \
16+
(self.symbol, self.library, type(self.data), self.version, self.metadata, self.host)
1717

1818

1919
ChangedItem = namedtuple('ChangedItem', ['symbol', 'orig_version', 'new_version', 'changes'])

‎tests/unit/store/test_version_item.py

+6-3
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,11 @@ def test_versioned_item_str():
88
library="ONEMINUTE",
99
data=pd.DataFrame(),
1010
version=1.0,
11+
host='myhost',
1112
metadata={'metadata': 'foo'})
1213

1314
expected = "VersionedItem(symbol=sym,library=ONEMINUTE," + \
14-
"data=<class 'pandas.core.frame.DataFrame'>,version=1.0,metadata={'metadata': 'foo'}"
15+
"data=<class 'pandas.core.frame.DataFrame'>,version=1.0,metadata={'metadata': 'foo'},host=myhost)"
1516
assert str(item) == expected
1617
assert repr(item) == expected
1718

@@ -21,7 +22,8 @@ def test_versioned_item_str_handles_none():
2122
library=None,
2223
data=None,
2324
version=None,
24-
metadata=None)
25+
metadata=None,
26+
host=None)
2527

2628
assert str(item)
2729

@@ -31,5 +33,6 @@ def test_versioned_item_metadata_dict():
3133
library="test_lib",
3234
data=None,
3335
version=1.2,
34-
metadata=None)
36+
metadata=None,
37+
host=None)
3538
assert(item.metadata_dict() == {'symbol': 'test', 'library': 'test_lib', 'version': 1.2})

‎tests/unit/store/test_version_store.py

+20-8
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,8 @@ def test_write_check_quota():
157157
_collection=Mock(),
158158
_version_nums=Mock(find_one_and_update=Mock(return_value={'version':1})),
159159
_versions=Mock(insert_one=lambda x:None),
160-
_arctic_lib=create_autospec(ArcticLibraryBinding),
160+
_arctic_lib=create_autospec(ArcticLibraryBinding,
161+
arctic=create_autospec(Arctic, mongo_host='some_host')),
161162
_publish_changes=False)
162163
vs._collection.database.connection.nodes = []
163164
vs._write_handler.return_value = write_handler
@@ -327,7 +328,8 @@ def _create_mock_versionstore():
327328
vs._add_new_version_using_reference.side_effect = lambda *args: VersionStore._add_new_version_using_reference(vs, *args)
328329
vs._last_version_seqnum = lambda version: VersionStore._last_version_seqnum(vs, version)
329330
vs.write.return_value = VersionedItem(symbol=TEST_SYMBOL, library=vs._arctic_lib.get_name(),
330-
version=TPL_VERSION['version'] + 1, metadata=META_TO_WRITE, data=None)
331+
version=TPL_VERSION['version'] + 1, metadata=META_TO_WRITE, data=None,
332+
host=vs._arctic_lib.arctic.mongo_host)
331333
return vs
332334

333335

@@ -351,6 +353,7 @@ def test_write_metadata_with_previous_data():
351353

352354
expected_ret_val = VersionedItem(symbol=TEST_SYMBOL,
353355
library=vs._arctic_lib.get_name(),
356+
host=vs._arctic_lib.arctic.mongo_host,
354357
version=TPL_VERSION['version'] + 1,
355358
metadata=META_TO_WRITE,
356359
data=None)
@@ -376,6 +379,7 @@ def test_write_empty_metadata():
376379

377380
expected_ret_val = VersionedItem(symbol=TEST_SYMBOL,
378381
library=vs._arctic_lib.get_name(),
382+
host=vs._arctic_lib.arctic.mongo_host,
379383
version=TPL_VERSION['version'] + 1,
380384
metadata=None,
381385
data=None)
@@ -416,9 +420,11 @@ def test_restore_version():
416420

417421
LASTEST_VERSION = dict(TPL_VERSION, version=TPL_VERSION['version']+1, metadata={'something': 'different'})
418422
last_item = VersionedItem(symbol=TEST_SYMBOL, library=vs._arctic_lib.get_name(),
423+
host=vs._arctic_lib.arctic.mongo_host,
419424
version=LASTEST_VERSION, metadata=LASTEST_VERSION['metadata'], data="hello world")
420425
new_version = dict(LASTEST_VERSION, version=LASTEST_VERSION['version'] + 1)
421426
new_item = VersionedItem(symbol=TEST_SYMBOL, library=vs._arctic_lib.get_name(),
427+
host=vs._arctic_lib.arctic.mongo_host,
422428
version=new_version, metadata=new_version['metadata'], data=last_item.data)
423429

424430
vs.write.return_value = new_item
@@ -479,7 +485,8 @@ def test_write_error_clean_retry():
479485
_collection=Mock(),
480486
_version_nums=Mock(find_one_and_update=Mock(return_value={'version': 1})),
481487
_versions=Mock(insert_one=Mock(__name__="insert_one"), find_one=Mock(__name__="find_one")),
482-
_arctic_lib=create_autospec(ArcticLibraryBinding),
488+
_arctic_lib=create_autospec(ArcticLibraryBinding,
489+
arctic=create_autospec(Arctic, mongo_host='some_host')),
483490
_publish_changes=False)
484491
vs._insert_version = lambda version: VersionStore._insert_version(vs, version)
485492
vs._collection.database.connection.nodes = []
@@ -498,7 +505,8 @@ def test_write_insert_version_duplicatekey():
498505
_collection=Mock(),
499506
_version_nums=Mock(find_one_and_update=Mock(return_value={'version': 1})),
500507
_versions=Mock(insert_one=Mock(__name__="insert_one"), find_one=Mock(__name__="find_one")),
501-
_arctic_lib=create_autospec(ArcticLibraryBinding),
508+
_arctic_lib=create_autospec(ArcticLibraryBinding,
509+
arctic=create_autospec(Arctic, mongo_host='some_host')),
502510
_publish_changes=False)
503511
vs._insert_version = lambda version: VersionStore._insert_version(vs, version)
504512
vs._versions.insert_one.side_effect = [DuplicateKeyError("dup key error"), None]
@@ -518,7 +526,8 @@ def test_write_insert_version_operror():
518526
_collection=Mock(),
519527
_version_nums=Mock(find_one_and_update=Mock(return_value={'version': 1})),
520528
_versions=Mock(insert_one=Mock(__name__="insert_one"), find_one=Mock(__name__="find_one")),
521-
_arctic_lib=create_autospec(ArcticLibraryBinding),
529+
_arctic_lib=create_autospec(ArcticLibraryBinding,
530+
arctic=create_autospec(Arctic, mongo_host='some_host')),
522531
_publish_changes=False)
523532
vs._insert_version = lambda version: VersionStore._insert_version(vs, version)
524533
vs._versions.insert_one.side_effect = [OperationFailure("mongo op error"), None]
@@ -541,7 +550,8 @@ def test_append_error_clean_retry():
541550
_collection=Mock(),
542551
_version_nums=Mock(find_one_and_update=Mock(return_value={'version': previous_version['version']+1})),
543552
_versions=Mock(insert_one=Mock(__name__="insert_one"), find_one=Mock(__name__="find_one", return_value=previous_version)),
544-
_arctic_lib=create_autospec(ArcticLibraryBinding),
553+
_arctic_lib=create_autospec(ArcticLibraryBinding,
554+
arctic=create_autospec(Arctic, mongo_host='some_host')),
545555
_publish_changes=False)
546556
vs._insert_version = lambda version: VersionStore._insert_version(vs, version)
547557
vs._collection.database.connection.nodes = []
@@ -562,7 +572,8 @@ def test_append_insert_version_duplicatekey():
562572
_collection=Mock(),
563573
_version_nums=Mock(find_one_and_update=Mock(return_value={'version': previous_version['version']+1})),
564574
_versions=Mock(insert_one=Mock(__name__="insert_one"), find_one=Mock(__name__="find_one", return_value=previous_version)),
565-
_arctic_lib=create_autospec(ArcticLibraryBinding),
575+
_arctic_lib=create_autospec(ArcticLibraryBinding,
576+
arctic=create_autospec(Arctic, mongo_host='some_host')),
566577
_publish_changes=False)
567578
vs._insert_version = lambda version: VersionStore._insert_version(vs, version)
568579
vs._versions.insert_one.side_effect = [DuplicateKeyError("dup key error"), None]
@@ -583,7 +594,8 @@ def test_append_insert_version_operror():
583594
_collection=Mock(),
584595
_version_nums=Mock(find_one_and_update=Mock(return_value={'version': previous_version['version']+1})),
585596
_versions=Mock(insert_one=Mock(__name__="insert_one"), find_one=Mock(__name__="find_one", return_value=previous_version)),
586-
_arctic_lib=create_autospec(ArcticLibraryBinding),
597+
_arctic_lib=create_autospec(ArcticLibraryBinding,
598+
arctic=create_autospec(Arctic, mongo_host='some_host')),
587599
_publish_changes=False)
588600
vs._insert_version = lambda version: VersionStore._insert_version(vs, version)
589601
vs._versions.insert_one.side_effect = [OperationFailure("mongo op error"), None]

0 commit comments

Comments
 (0)