Merge pull request pandas-dev#89 from manahl/python3

Python3 support

Author: bmoscon

.travis.yml

@@ -0,0 +1,26 @@
+sudo: false
+language: python
+python:
+  - "2.7"
+  - "3.3"
+install:
+  - pip install --upgrade pip
+  - pip install python-dateutil --upgrade 
+  - pip install pytz --upgrade 
+  - pip install tzlocal --upgrade 
+  - pip install cython --upgrade
+  - pip install pymongo --upgrade
+  - pip install numpy --upgrade
+  - pip install pandas --upgrade 
+  - pip install decorator --upgrade
+  - pip install enum34 --upgrade
+  - pip install lz4 --upgrade
+  - pip install mock --upgrade
+  - pip install mockextras
+  - pip install pytest --upgrade
+  - pip install pytest-cov --upgrade
+  - pip install pytest-dbfixtures --upgrade
+  - pip install pytest-timeout --upgrade
+  - pip install pytest-xdist --upgrade
+  - pip install setuptools-git --upgrade
+script: python setup.py test

CHANGES.md

@@ -1,5 +1,9 @@
 ## Changelog
 
+### 1.19
+
+  * Feature: Add python 3.3 support
+
 ### 1.18 (2016-01-05)
 
   * Bugfix: #81 Fix broken read of multi-index DataFrame written by old version of Arctic

README.md

@@ -71,7 +71,7 @@ shows how.](howtos/how_to_custom_arctic_library.py)
 
 Arctic provides namespaced *libraries* of data.  These libraries allow
 bucketing data by *source*, *user* or some other metric (for example frequency:
-End-Of-Day; Minute Bars; etc.).  
+End-Of-Day; Minute Bars; etc.).
 
 Arctic supports multiple data libraries per user.  A user (or namespace)
 maps to a MongoDB database (the granularity of mongo authentication).  The library
@@ -105,7 +105,7 @@ Arctic storage implementations are **pluggable**.  VersionStore is the default.
 
 Arctic currently works with:
 
- * Python 2.7
+ * Python 2.7 or 3.3
  * pymongo >= 3.0
  * Pandas
  * MongoDB >= 2.4.x
@@ -133,4 +133,3 @@ Contributions welcome!
 ## License
 
 Arctic is licensed under the GNU LGPL v2.1.  A copy of which is included in [LICENSE](LICENSE)
-

arctic/_compression.py

@@ -7,7 +7,7 @@
 try:
     from . import _compress as clz4
 except ImportError:
-    logger.warn("Couldn't import cython lz4")
+    logger.warning("Couldn't import cython lz4")
     import lz4 as clz4
 
 

arctic/_util.py

@@ -1,16 +1,15 @@
 from pandas import DataFrame
 from pandas.util.testing import assert_frame_equal
 from pymongo.errors import OperationFailure
-import string
 import logging
 
 logger = logging.getLogger(__name__)
 
 
 def indent(s, num_spaces):
-    s = string.split(s, '\n')
+    s = s.split('\n')
     s = [(num_spaces * ' ') + line for line in s]
-    s = string.join(s, '\n')
+    s = '\n'.join(s)
     return s
 
 
@@ -31,7 +30,7 @@ def enable_sharding(arctic, library_name, hashed=False):
     library_name = lib.get_top_level_collection().name
     try:
         c.admin.command('enablesharding', dbname)
-    except OperationFailure, e:
+    except OperationFailure as e:
         if not 'failed: already enabled' in str(e):
             raise
     if not hashed:

arctic/arctic.py

@@ -11,6 +11,7 @@
 from .hooks import get_mongodb_uri
 from .store import version_store
 from .tickstore import tickstore, toplevel
+from six import string_types
 
 
 __all__ = ['Arctic', 'VERSION_STORE', 'TICK_STORE', 'register_library_type']
@@ -92,7 +93,7 @@ def __init__(self, mongo_host, app_name=APPLICATION_NAME, allow_secondary=False,
         self._server_selection_timeout = serverSelectionTimeoutMS
         self._lock = threading.Lock()
 
-        if isinstance(mongo_host, basestring):
+        if isinstance(mongo_host, string_types):
             self.mongo_host = mongo_host
         else:
             self.__conn = mongo_host
@@ -230,7 +231,7 @@ def get_library(self, library):
             error = None
             l = ArcticLibraryBinding(self, library)
             lib_type = l.get_library_type()
-        except (OperationFailure, AutoReconnect), e:
+        except (OperationFailure, AutoReconnect) as e:
             error = e
 
         if error:
@@ -249,7 +250,7 @@ def get_library(self, library):
         return self._library_cache[library]
 
     def __getitem__(self, key):
-        if isinstance(key, basestring):
+        if isinstance(key, string_types):
             return self.get_library(key)
         else:
             raise ArcticException("Unrecognised library specification - use [libraryName]")
@@ -424,18 +425,18 @@ def to_gigabytes(bytes_):
                                           to_gigabytes(self.quota)))
 
         # Quota not exceeded, print an informational message and return
-        avg_size = size / count if count > 1 else 100 * 1024
+        avg_size = size // count if count > 1 else 100 * 1024
         remaining = self.quota - size
         remaining_count = remaining / avg_size
         if remaining_count < 100:
-            logger.warn("Mongo Quota: %.3f / %.0f GB used" % (to_gigabytes(size),
+            logger.warning("Mongo Quota: %.3f / %.0f GB used" % (to_gigabytes(size),
                                                               to_gigabytes(self.quota)))
         else:
             logger.info("Mongo Quota: %.3f / %.0f GB used" % (to_gigabytes(size),
                                                               to_gigabytes(self.quota)))
 
         # Set-up a timer to prevent us for checking for a few writes.
-        self.quota_countdown = max(remaining_count / 2, 1)
+        self.quota_countdown = int(max(remaining_count // 2, 1))
 
     def get_library_type(self):
         return self.get_library_metadata(ArcticLibraryBinding.TYPE_FIELD)

arctic/auth.py

@@ -26,5 +26,5 @@ def get_auth(host, app_name, database_name):
     """
     Authentication hook to allow plugging in custom authentication credential providers
     """
-    from hooks import _get_auth_hook
+    from .hooks import _get_auth_hook
     return _get_auth_hook(host, app_name, database_name)

arctic/date/_daterange.py

@@ -170,6 +170,13 @@ def __eq__(self, rhs):
             return False
         return self.end == rhs.end and self.start == rhs.start
 
+    def __lt__(self, other):
+        if self.start is None:
+            return True
+        if other.start is None:
+            return False
+        return self.start < other.start
+
     def __hash__(self):
         return hash((self.start, self.end, self.step, self.interval))
 

arctic/date/_util.py

@@ -6,8 +6,11 @@
 from ._generalslice import OPEN_OPEN, CLOSED_CLOSED, OPEN_CLOSED, CLOSED_OPEN
 from ._parse import parse
 from ._mktz import mktz
+import sys
+if sys.version_info > (3,):
+    long = int
 
-
+    
 # Support standard brackets syntax for open/closed ranges.
 Ranges = {'()': OPEN_OPEN,
           '(]': OPEN_CLOSED,

arctic/decorators.py

@@ -81,7 +81,7 @@ def _handle_error(f, e, retry_count, **kwargs):
         logger.error('Too many retries %s [%s], raising' % (f.__name__, e))
         e.traceback = sys.exc_info()[2]
         raise
-    log_fn = logger.warn if retry_count > 2 else logger.debug
+    log_fn = logger.warning if retry_count > 2 else logger.debug
     log_fn('%s %s [%s], retrying %i' % (type(e), f.__name__, e, retry_count))
     # Log operation failure errors
     _log_exception(f.__name__, e, retry_count, **kwargs)

arctic/fixtures/mongo.py

@@ -19,7 +19,7 @@
 import os
 import pytest
 
-from path import path
+from path import Path as path
 from tempfile import mkdtemp
 
 from pytest_dbfixtures.executors import TCPExecutor

arctic/hosts.py

@@ -4,6 +4,7 @@
 import re
 import logging
 from weakref import WeakValueDictionary
+import six
 
 __all__ = ['get_arctic_lib', 'get_arctic_for_library']
 
@@ -41,7 +42,7 @@ def get_arctic_lib(connection_string, **kwargs):
 
 def _get_arctic(instance, **kwargs):
     # Consider any kwargs passed to the Arctic as discriminators for the cache
-    key = instance, frozenset(kwargs.iteritems())
+    key = instance, frozenset(six.iteritems(kwargs))
 
     # Don't create lots of Arctic instances
     arctic = arctic_cache.get(key, None)

arctic/multi_index.py

@@ -9,6 +9,7 @@
 
 import numpy as np
 import pandas as pd
+import six
 
 
 logger = logging.getLogger(__name__)
@@ -42,7 +43,7 @@ def fancy_group_by(df, grouping_level=0, aggregate_level=1, method='last', max_=
     if method not in ('first', 'last'):
         raise ValueError('Invalid method')
 
-    if isinstance(aggregate_level, basestring):
+    if isinstance(aggregate_level, six.string_types):
         aggregate_level = df.index.names.index(aggregate_level)
 
     # Trim any rows outside the aggregate value bounds

arctic/scripts/arctic_create_user.py

@@ -37,7 +37,7 @@ def main():
         write_access = args.write
         p = args.password
         if p is None:
-            p = base64.b64encode(uuid.uuid4().bytes).replace('/', '')[:12]
+            p = base64.b64encode(uuid.uuid4().bytes).replace(b'/', b'')[:12]
         db = args.db
         if not db:
             # Users always have write access to their database

arctic/scripts/arctic_delete_library.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 import optparse
 import pymongo
 import logging
@@ -28,7 +29,7 @@ def main():
     if not opts.library:
         parser.error('Must specify the full path of the library e.g. arctic_jblackburn.lib!')
 
-    print "Deleting: %s on mongo %s" % (opts.library, opts.host)
+    print("Deleting: %s on mongo %s" % (opts.library, opts.host))
     c = pymongo.MongoClient(get_mongodb_uri(opts.host))
 
     db_name = opts.library[:opts.library.index('.')] if '.' in opts.library else None

arctic/scripts/arctic_enable_sharding.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 import optparse
 import pymongo
 
@@ -25,7 +26,7 @@ def main():
     if not opts.library or '.' not in opts.library:
         parser.error('must specify the full path of the library e.g. arctic_jblackburn.lib!')
 
-    print "Enabling-sharding: %s on mongo %s" % (opts.library, opts.host)
+    print("Enabling-sharding: %s on mongo %s" % (opts.library, opts.host))
 
     c = pymongo.MongoClient(get_mongodb_uri(opts.host))
     credentials = get_auth(opts.host, 'admin', 'admin')

arctic/scripts/arctic_init_library.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 import argparse
 import pymongo
 import logging
@@ -36,7 +37,7 @@ def main():
         parser.error('Must specify the full path of the library e.g. user.library!')
     db_name, _ = ArcticLibraryBinding._parse_db_lib(opts.library)
 
-    print "Initializing: %s on mongo %s" % (opts.library, opts.host)
+    print("Initializing: %s on mongo %s" % (opts.library, opts.host))
     c = pymongo.MongoClient(get_mongodb_uri(opts.host))
 
     if not do_db_auth(opts.host, c, db_name):

arctic/scripts/arctic_prune_versions.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 import optparse
 import pymongo
 import logging
@@ -36,8 +37,8 @@ def main():
         parser.error('Must specify the Arctic library e.g. arctic_jblackburn.library!')
     db_name, _ = ArcticLibraryBinding._parse_db_lib(opts.library)
 
-    print "Pruning (old) versions in : %s on mongo %s" % (opts.library, opts.host)
-    print "Keeping all versions <= %s mins old" % (opts.keep_mins)
+    print("Pruning (old) versions in : %s on mongo %s" % (opts.library, opts.host))
+    print("Keeping all versions <= %s mins old" % (opts.keep_mins))
     c = pymongo.MongoClient(get_mongodb_uri(opts.host))
 
     if not do_db_auth(opts.host, c, db_name):

arctic/store/_ndarray_store.py

@@ -1,6 +1,5 @@
 import logging
 import hashlib
-import pprint
 
 from bson.binary import Binary
 import numpy as np
@@ -13,6 +12,8 @@
 
 from .._compression import compress_array, decompress
 from ..exceptions import ConcurrentModificationException
+from six.moves import xrange
+
 
 logger = logging.getLogger(__name__)
 
@@ -116,7 +117,7 @@ def _ensure_index(collection):
             collection.create_index([('symbol', pymongo.ASCENDING),
                                      ('parent', pymongo.ASCENDING),
                                      ('segment', pymongo.ASCENDING)], unique=True, background=True)
-        except OperationFailure, e:
+        except OperationFailure as e:
             if "can't use unique indexes" in str(e):
                 return
             raise
@@ -199,7 +200,7 @@ def _do_read(self, collection, version, symbol, index_range=None):
                                       collection.find_one({'_id': x['_id']}),
                                       collection.find_one({'_id': x['_id']}))
                 raise
-        data = ''.join(segments)
+        data = b''.join(segments)
 
         # Check that the correct number of segments has been returned
         if segment_count is not None and i + 1 != segment_count:
@@ -401,7 +402,7 @@ def _do_write(self, collection, version, symbol, item, previous_version, segment
         sze = int(item.dtype.itemsize * np.prod(item.shape[1:]))
 
         # chunk and store the data by (uncompressed) size
-        chunk_size = _CHUNK_SIZE / sze
+        chunk_size = int(_CHUNK_SIZE / sze)
 
         previous_shas = []
         if previous_version:

arctic/store/_pandas_ndarray_store.py

@@ -39,7 +39,7 @@ def _index_to_records(self, df):
         if isinstance(index, MultiIndex):
             # array of tuples to numpy cols. copy copy copy
             if len(df) > 0:
-                ix_vals = map(np.array, [index.get_level_values(i) for i in range(index.nlevels)])
+                ix_vals = list(map(np.array, [index.get_level_values(i) for i in range(index.nlevels)]))
             else:
                 # empty multi index has no size, create empty arrays for recarry..
                 ix_vals = [np.array([]) for n in index.names]
@@ -93,7 +93,7 @@ def to_records(self, df):
         metadata['columns'] = columns
         names = index_names + columns
         arrays = ix_vals + column_vals
-        arrays = map(_to_primitive, arrays)
+        arrays = list(map(_to_primitive, arrays))
         dtype = np.dtype([(str(x), v.dtype) if len(v.shape) == 1 else (str(x), v.dtype, v.shape[1]) for x, v in zip(names, arrays)],
                          metadata=metadata)
 
@@ -186,7 +186,7 @@ def _index_range(self, version, symbol, date_range=None, **kwargs):
                     return -1, -1
                 idxstart = min(np.searchsorted(dts, start), len(dts) - 1)
                 idxend = min(np.searchsorted(dts, end), len(dts) - 1)
-                return index['index'][idxstart], index['index'][idxend] + 1
+                return int(index['index'][idxstart]), int(index['index'][idxend] + 1)
         return super(PandasStore, self)._index_range(version, symbol, **kwargs)
 
     def _daterange(self, recarr, date_range):