Merge pull request pandas-dev#676 from shashank88/small_fixes

Multiple small formatting changes and enabling pycodestyle in ci

Author: shashank88

.travis.yml

@@ -8,6 +8,8 @@ python:
 services:
    - mongodb
 
+before_install:
+  - pip install pycodestyle
 install:
   - mongo --version
   - pip install --upgrade pip
@@ -31,3 +33,5 @@ install:
 script:
   - pip freeze
   - python setup.py test --pytest-args=-v
+  # Skipped codes in setup.cfg. Tests not checked for now.
+  - pycodestyle arctic

arctic/_compression.py

@@ -7,8 +7,9 @@
 except ImportError as e:
     from lz4 import compress as lz4_compress, compressHC as lz4_compressHC, decompress as lz4_decompress
 
+# ENABLE_PARALLEL mutated in global_scope. Do not remove.
 from ._config import ENABLE_PARALLEL, LZ4_HIGH_COMPRESSION, LZ4_WORKERS, LZ4_N_PARALLEL, LZ4_MINSZ_PARALLEL, \
-    BENCHMARK_MODE
+    BENCHMARK_MODE  # noqa # pylint: disable=unused-import
 
 logger = logging.getLogger(__name__)
 
@@ -78,8 +79,10 @@ def compress_array(str_list, withHC=LZ4_HIGH_COMPRESSION):
 
     do_compress = lz4_compressHC if withHC else lz4_compress
 
-    use_parallel = ENABLE_PARALLEL and withHC or \
-                   len(str_list) > LZ4_N_PARALLEL and len(str_list[0]) > LZ4_MINSZ_PARALLEL
+    def can_parallelize_strlist(strlist):
+        return len(strlist) > LZ4_N_PARALLEL and len(strlist[0]) > LZ4_MINSZ_PARALLEL
+
+    use_parallel = (ENABLE_PARALLEL and withHC) or can_parallelize_strlist(str_list)
 
     if BENCHMARK_MODE or use_parallel:
         if _compress_thread_pool is None:

arctic/_config.py

@@ -81,7 +81,7 @@ class FwPointersCfg(Enum):
 LZ4_N_PARALLEL = os.environ.get('LZ4_N_PARALLEL', 16)
 
 # Minimum data size to use parallel compression
-LZ4_MINSZ_PARALLEL = os.environ.get('LZ4_MINSZ_PARALLEL', 0.5*1024**2)  # 0.5 MB
+LZ4_MINSZ_PARALLEL = os.environ.get('LZ4_MINSZ_PARALLEL', 0.5 * 1024 ** 2)  # 0.5 MB
 
 # Enable this when you run the benchmark_lz4.py
 BENCHMARK_MODE = False

arctic/arctic.py

@@ -247,18 +247,18 @@ def initialize_library(self, library, lib_type=VERSION_STORE, **kwargs):
         kwargs :
             Arguments passed to the Library type for initialization.
         """
-        l = ArcticLibraryBinding(self, library)
+        lib = ArcticLibraryBinding(self, library)
         # check that we don't create too many namespaces
         # can be disabled check_library_count=False
         check_library_count = kwargs.pop('check_library_count', True)
-        if len(self._conn[l.database_name].list_collection_names()) > 5000 and check_library_count:
+        if len(self._conn[lib.database_name].list_collection_names()) > 5000 and check_library_count:
             raise ArcticException("Too many namespaces %s, not creating: %s" %
-                                  (len(self._conn[l.database_name].list_collection_names()), library))
-        l.set_library_type(lib_type)
-        LIBRARY_TYPES[lib_type].initialize_library(l, **kwargs)
+                                  (len(self._conn[lib.database_name].list_collection_names()), library))
+        lib.set_library_type(lib_type)
+        LIBRARY_TYPES[lib_type].initialize_library(lib, **kwargs)
         # Add a 10G quota just in case the user is calling this with API.
-        if not l.get_quota():
-            l.set_quota(10 * 1024 * 1024 * 1024)
+        if not lib.get_quota():
+            lib.set_quota(10 * 1024 * 1024 * 1024)
 
     @mongo_retry
     def delete_library(self, library):
@@ -270,19 +270,19 @@ def delete_library(self, library):
         library : `str`
             The name of the library. e.g. 'library' or 'user.library'
         """
-        l = ArcticLibraryBinding(self, library)
-        colname = l.get_top_level_collection().name
-        if not [c for c in l._db.list_collection_names(False) if re.match(r"^{}([\.].*)?$".format(colname), c)]:
+        lib = ArcticLibraryBinding(self, library)
+        colname = lib.get_top_level_collection().name
+        if not [c for c in lib._db.list_collection_names(False) if re.match(r"^{}([\.].*)?$".format(colname), c)]:
             logger.info('Nothing to delete. Arctic library %s does not exist.' % colname)
         logger.info('Dropping collection: %s' % colname)
-        l._db.drop_collection(colname)
-        for coll in l._db.list_collection_names():
+        lib._db.drop_collection(colname)
+        for coll in lib._db.list_collection_names():
             if coll.startswith(colname + '.'):
                 logger.info('Dropping collection: %s' % coll)
-                l._db.drop_collection(coll)
+                lib._db.drop_collection(coll)
         if library in self._library_cache:
             del self._library_cache[library]
-            del self._library_cache[l.get_name()]
+            del self._library_cache[lib.get_name()]
 
     def get_library(self, library):
         """
@@ -299,8 +299,8 @@ def get_library(self, library):
 
         try:
             error = None
-            l = ArcticLibraryBinding(self, library)
-            lib_type = l.get_library_type()
+            lib = ArcticLibraryBinding(self, library)
+            lib_type = lib.get_library_type()
         except (OperationFailure, AutoReconnect) as e:
             error = e
 
@@ -313,10 +313,10 @@ def get_library(self, library):
         elif lib_type not in LIBRARY_TYPES:
             raise LibraryNotFoundException("Couldn't load LibraryType '%s' for '%s' (has the class been registered?)" %
                                            (lib_type, library))
-        instance = LIBRARY_TYPES[lib_type](l)
+        instance = LIBRARY_TYPES[lib_type](lib)
         self._library_cache[library] = instance
         # The library official name may be different from 'library': e.g. 'library' vs 'user.library'
-        self._library_cache[l.get_name()] = instance
+        self._library_cache[lib.get_name()] = instance
         return self._library_cache[library]
 
     def __getitem__(self, key):
@@ -338,8 +338,7 @@ def set_quota(self, library, quota):
         quota : `int`
             Advisory quota for the library - in bytes
         """
-        l = ArcticLibraryBinding(self, library)
-        l.set_quota(quota)
+        ArcticLibraryBinding(self, library).set_quota(quota)
 
     def get_quota(self, library):
         """
@@ -350,8 +349,7 @@ def get_quota(self, library):
         library : `str`
             The name of the library. e.g. 'library' or 'user.library'
         """
-        l = ArcticLibraryBinding(self, library)
-        return l.get_quota()
+        return ArcticLibraryBinding(self, library).get_quota()
 
     def check_quota(self, library):
         """
@@ -366,8 +364,7 @@ def check_quota(self, library):
         ------
         arctic.exceptions.QuotaExceededException if the quota has been exceeded
         """
-        l = ArcticLibraryBinding(self, library)
-        l.check_quota()
+        ArcticLibraryBinding(self, library).check_quota()
 
     def rename_library(self, from_lib, to_lib):
         """
@@ -386,18 +383,18 @@ def rename_library(self, from_lib, to_lib):
                 raise ValueError("Collection can only be renamed in the same database")
             to_colname = to_lib.split('.')[1]
 
-        l = ArcticLibraryBinding(self, from_lib)
-        colname = l.get_top_level_collection().name
+        lib = ArcticLibraryBinding(self, from_lib)
+        colname = lib.get_top_level_collection().name
 
         logger.info('Renaming collection: %s' % colname)
-        l._db[colname].rename(to_colname)
-        for coll in l._db.list_collection_names():
+        lib._db[colname].rename(to_colname)
+        for coll in lib._db.list_collection_names():
             if coll.startswith(colname + '.'):
-                l._db[coll].rename(coll.replace(colname, to_colname))
+                lib._db[coll].rename(coll.replace(colname, to_colname))
 
         if from_lib in self._library_cache:
             del self._library_cache[from_lib]
-            del self._library_cache[l.get_name()]
+            del self._library_cache[lib.get_name()]
 
     def get_library_type(self, lib):
         """
@@ -408,8 +405,7 @@ def get_library_type(self, lib):
         lib: str
             the library
         """
-        l = ArcticLibraryBinding(self, lib)
-        return l.get_library_type()
+        return ArcticLibraryBinding(self, lib).get_library_type()
 
 
 class ArcticLibraryBinding(object):
@@ -565,14 +561,14 @@ def to_gigabytes(bytes_):
             remaining_count = remaining / avg_size
             if remaining_count < 100 or float(remaining) / self.quota < 0.1:
                 logger.warning("Mongo Quota: %s %.3f / %.0f GB used" % (
-                                '.'.join([self.database_name, self.library]),
-                                to_gigabytes(size),
-                                to_gigabytes(self.quota)))
+                    '.'.join([self.database_name, self.library]),
+                    to_gigabytes(size),
+                    to_gigabytes(self.quota)))
             else:
                 logger.info("Mongo Quota: %s %.3f / %.0f GB used" % (
-                                '.'.join([self.database_name, self.library]),
-                                to_gigabytes(size),
-                                to_gigabytes(self.quota)))
+                    '.'.join([self.database_name, self.library]),
+                    to_gigabytes(size),
+                    to_gigabytes(self.quota)))
 
             # Set-up a timer to prevent us for checking for a few writes.
             # This will check every average half-life

arctic/chunkstore/chunkstore.py

@@ -742,13 +742,14 @@ def stats(self):
         res['chunks'] = db.command('collstats', self._collection.name)
         res['symbols'] = db.command('collstats', self._symbols.name)
         res['metadata'] = db.command('collstats', self._mdata.name)
-        res['totals'] = {'count': res['chunks']['count'],
-                         'size': res['chunks']['size'] + res['symbols']['size'] + res['metadata']['size'],
-                        }
+        res['totals'] = {
+            'count': res['chunks']['count'],
+            'size': res['chunks']['size'] + res['symbols']['size'] + res['metadata']['size'],
+        }
         return res
 
     def has_symbol(self, symbol):
-        '''
+        """
         Check if symbol exists in collection
 
         Parameters
@@ -759,5 +760,5 @@ def has_symbol(self, symbol):
         Returns
         -------
         bool
-        '''
+        """
         return self._get_symbol_info(symbol) is not None

arctic/date/_daterange.py

@@ -82,22 +82,22 @@ def intersection(self, other):
         Create a new DateRange representing the maximal range enclosed by this range and other
         """
         startopen = other.startopen if self.start is None \
-                    else self.startopen if other.start is None \
-                    else other.startopen if self.start < other.start \
-                    else self.startopen if self.start > other.start \
-                    else (self.startopen or other.startopen)
+            else self.startopen if other.start is None \
+            else other.startopen if self.start < other.start \
+            else self.startopen if self.start > other.start \
+            else (self.startopen or other.startopen)
         endopen = other.endopen if self.end is None \
-                   else self.endopen if other.end is None \
-                   else other.endopen if self.end > other.end \
-                   else self.endopen if self.end < other.end \
-                   else (self.endopen or other.endopen)
+            else self.endopen if other.end is None \
+            else other.endopen if self.end > other.end \
+            else self.endopen if self.end < other.end \
+            else (self.endopen or other.endopen)
 
         new_start = self.start if other.start is None \
-                    else other.start if self.start is None \
-                    else max(self.start, other.start)
+            else other.start if self.start is None \
+            else max(self.start, other.start)
         new_end = self.end if other.end is None \
-                   else other.end if self.end is None \
-                   else min(self.end, other.end)
+            else other.end if self.end is None \
+            else min(self.end, other.end)
 
         interval = INTERVAL_LOOKUP[(startopen, endopen)]
 

arctic/date/_util.py

@@ -85,7 +85,7 @@ def string_to_daterange(str_range, delimiter='-', as_dates=False, interval=CLOSE
 def to_dt(date, default_tz=None):
     """
     Returns a non-naive datetime.datetime.
-    
+
     Interprets numbers as ms-since-epoch.
 
     Parameters
@@ -116,7 +116,7 @@ def to_pandas_closed_closed(date_range, add_tz=True):
 
     Parameters
     ----------
-    date_range : `DateRange` object 
+    date_range : `DateRange` object
         converted to CLOSED_CLOSED form for Pandas slicing
 
     add_tz : `bool`

arctic/fixtures/arctic.py

@@ -35,6 +35,7 @@ def arctic_secondary(mongo_server, arctic):
     arctic = m.Arctic(mongo_host=mongo_server.api, allow_secondary=True)
     return arctic
 
+
 @pytest.fixture(scope="function")
 def multicolumn_store_with_uncompressed_write(mongo_server):
     """

arctic/scripts/arctic_fsck.py

@@ -55,14 +55,14 @@ def main():
                      final_stats['versions'].get('avgObjSize', 0) / 1024. / 1024.))
         logger.info("Versions: %10.2fMB Change(+/-) %.2fMB" %
                     (final_stats['versions']['size'] / 1024. / 1024.,
-                    (final_stats['versions']['size'] - orig_stats['versions']['size']) / 1024. / 1024.))
+                     (final_stats['versions']['size'] - orig_stats['versions']['size']) / 1024. / 1024.))
         logger.info('Chunk Count: %7d   Change(+/-) %6d  (av: %.2fMB)' %
                     (final_stats['chunks']['count'],
                      final_stats['chunks']['count'] - orig_stats['chunks']['count'],
                      final_stats['chunks'].get('avgObjSize', 0) / 1024. / 1024.))
         logger.info("Chunks: %12.2fMB Change(+/-) %6.2fMB" %
                     (final_stats['chunks']['size'] / 1024. / 1024.,
-                    (final_stats['chunks']['size'] - orig_stats['chunks']['size']) / 1024. / 1024.))
+                     (final_stats['chunks']['size'] - orig_stats['chunks']['size']) / 1024. / 1024.))
         logger.info('----------------------------')
 
     if not opts.f:

arctic/scripts/arctic_init_library.py

@@ -26,12 +26,16 @@ def main():
     parser.add_argument("--host", default='localhost', help="Hostname, or clustername. Default: localhost")
     parser.add_argument("--library", help="The name of the library. e.g. 'arctic_jblackburn.lib'")
     parser.add_argument("--type", default=VERSION_STORE, choices=sorted(LIBRARY_TYPES.keys()),
-                                    help="The type of the library, as defined in "
-                                         "arctic.py. Default: %s" % VERSION_STORE)
+                        help="The type of the library, as defined in "
+                             "arctic.py. Default: %s" % VERSION_STORE)
     parser.add_argument("--quota", default=10, help="Quota for the library in GB. A quota of 0 is unlimited."
                                                     "Default: 10")
-    parser.add_argument("--hashed", action="store_true", default=False, help="Use hashed based sharding. Useful where SYMBOLs share a common prefix (e.g. Bloomberg BBGXXXX symbols)"
-                                                        "Default: False")
+    parser.add_argument(
+        "--hashed",
+        action="store_true",
+        default=False,
+        help="Use hashed based sharding. Useful where SYMBOLs share a common prefix (e.g. Bloomberg BBGXXXX symbols) "
+             "Default: False")
 
     opts = parser.parse_args()
 

arctic/serialization/incremental.py

@@ -196,23 +196,23 @@ def _generator(self, from_idx, to_idx, get_bytes=False):
         # Note that the range is: [from_idx, to_idx)
         self._lazy_init()
 
-        my_lenth = len(self)
+        my_length = len(self)
 
         # Take into account default arguments and negative indexing (from end offset)
         from_idx = 0 if from_idx is None else from_idx
         if from_idx < 0:
-            from_idx = my_lenth + from_idx
-        to_idx = my_lenth if to_idx is None else min(to_idx, my_lenth)
+            from_idx = my_length + from_idx
+        to_idx = my_length if to_idx is None else min(to_idx, my_length)
         if to_idx < 0:
-            to_idx = my_lenth + to_idx
+            to_idx = my_length + to_idx
 
         # No data, finish iteration
-        if my_lenth == 0 or from_idx >= my_lenth or from_idx >= to_idx:
+        if my_length == 0 or from_idx >= my_length or from_idx >= to_idx:
             return
 
         # Perform serialization for each chunk
         while from_idx < to_idx:
-            curr_stop = min(from_idx+self._rows_per_chunk, to_idx)
+            curr_stop = min(from_idx + self._rows_per_chunk, to_idx)
 
             chunk, _ = self._serializer.serialize(
                 self.input_data[from_idx: curr_stop],

arctic/serialization/numpy_arrays.py

@@ -186,7 +186,7 @@ def serialize(self, df):
         return ret
 
     def deserialize(self, data, columns=None):
-        '''
+        """
         Deserializes SON to a DataFrame
 
         Parameters
@@ -199,8 +199,8 @@ def deserialize(self, data, columns=None):
         Returns
         -------
         pandas dataframe or series
-        '''
-        if data == []:
+        """
+        if not data:
             return pd.DataFrame()
 
         meta = data[0][METADATA] if isinstance(data, list) else data[METADATA]

arctic/serialization/numpy_records.py

@@ -3,6 +3,7 @@
 import numpy as np
 from pandas import DataFrame, MultiIndex, Series, DatetimeIndex, Index
 
+# Used in global scope, do not remove.
 from .._config import FAST_CHECK_DF_SERIALIZABLE
 from .._util import NP_OBJECT_DTYPE
 from ..exceptions import ArcticException