BLD: rename / move some extensions

xref pandas-dev#12588

pandas.parser -> io/libparsers.pyx
pandas.json -> pandas.io.json.libjson
pandas.io.sas.saslib -> libsas
pandas.msgpack -> pandas.io.msgpack
pandas._testing -> pandas.util.libtesting
pandas._sparse -> pandas.sparse.libsparse
pandas._hash -> pandas.tools.libhash
pandas.tslib -> pandas.libs.tslib
pandas.index -> pandas.libs.index
pandas.algos -> pandas.libs.algos
pandas.lib -> pandas.libs.lib
pandas.hashtable -> pandas.libs.hashtable
pandas._window -> pandas.core.libwindow
pandas._join -> pandas.libs.join
move algos*.in, index*.in, hashtable*.in to libs
pandas._period -> pandas.libs.period

Author: jreback

Makefile

@@ -1,4 +1,4 @@
-tseries: pandas/lib.pyx pandas/tslib.pyx pandas/hashtable.pyx
+tseries: pandas/libs/lib.pyx pandas/libs/tslib.pyx pandas/libs/hashtable.pyx
 	python setup.py build_ext --inplace
 
 .PHONY : develop build clean clean_pyc tseries doc

asv_bench/benchmarks/binary_ops.py

@@ -107,4 +107,4 @@ def setup(self):
         self.s = Series(date_range('20010101', periods=self.N, freq='T', tz='US/Eastern'))
         self.ts = self.s[self.halfway]
 
-        self.s2 = Series(date_range('20010101', periods=self.N, freq='s',  tz='US/Eastern'))
+        self.s2 = Series(date_range('20010101', periods=self.N, freq='s',  tz='US/Eastern'))

asv_bench/benchmarks/pandas_vb_common.py

@@ -8,16 +8,22 @@
 import random
 import numpy as np
 import threading
+from importlib import import_module
+
 try:
     from pandas.compat import range
 except ImportError:
     pass
 
 np.random.seed(1234)
-try:
-    import pandas._tseries as lib
-except:
-    import pandas.lib as lib
+
+# try em until it works!
+for imp in ['pandas_tseries', 'pandas.lib', 'pandas.libs.lib']:
+    try:
+        lib = import_module(imp)
+        break
+    except:
+        pass
 
 try:
     Panel = Panel

asv_bench/benchmarks/panel_methods.py

@@ -21,4 +21,4 @@ def time_shift(self):
         self.panel.shift(1)
 
     def time_shift_minor(self):
-        self.panel.shift(1, axis='minor')
+        self.panel.shift(1, axis='minor')

doc/source/whatsnew/v0.20.0.txt

@@ -447,6 +447,35 @@ New Behavior:
    In [11]: index.memory_usage(deep=True)
    Out[11]: 260
 
+.. _whatsnew_0200.api_breaking.extensions:
+
+Extension Modules Moved
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Some formerly public c/c++/cython extension modules have been moved and/or renamed. These are all removed from the public API.
+If indicated, a deprecation warning will be issued if you reference that module. (:issue:`12588`)
+
+.. csv-table::
+    :header: "Previous Location", "New Location", "Deprecated"
+    :widths: 30, 30, 4
+
+    "pandas.json", "pandas.io.json.libjson", "X"
+    "pandas.parser", "pandas.io.libparsers", "X"
+    "pandas.lib", "pandas.libs.lib", "X"
+    "pandas.tslib", "pandas.libs.tslib", "X"
+    "pandas.io.sas.saslib", "pandas.io.sas.libsas", ""
+    "pandas.msgpack", "pandas.io.msgpack", ""
+    "pandas.index", "pandas.libs.index", ""
+    "pandas.algos", "pandas.libs.algos", ""
+    "pandas.hashtable", "pandas.libs.hashtable", ""
+    "pandas._testing", "pandas.util.libtesting", ""
+    "pandas._sparse", "pandas.sparse.libsparse", ""
+    "pandas._hash", "pandas.tools.libhash", ""
+    "pandas._window", "pandas.core.libwindow", ""
+    "pandas._join", "pandas.libs.join", ""
+    "pandas._period", "pandas.libs.period", ""
+
+
 .. _whatsnew_0200.api_breaking.groupby_describe:
 
 Groupby Describe Formatting

pandas/__init__.py

@@ -23,7 +23,9 @@
 from pandas.compat.numpy import *
 
 try:
-    from pandas import hashtable, tslib, lib
+    from pandas.libs import (hashtable as _hashtable,
+                             lib as _lib,
+                             tslib as _tslib)
 except ImportError as e:  # pragma: no cover
     # hack but overkill to use re
     module = str(e).lstrip('cannot import name ')
@@ -53,11 +55,17 @@
 from pandas.tools.util import to_numeric
 from pandas.core.reshape import melt
 from pandas.util.print_versions import show_versions
-
 from pandas.io.api import *
-
 from pandas.util._tester import test
 
+# extension module deprecations
+from pandas.util.depr_module import _DeprecatedModule
+
+json = _DeprecatedModule(deprmod='pandas.json', deprmodto='pandas.io.json.libjson')
+parser = _DeprecatedModule(deprmod='pandas.parser', deprmodto='pandas.io.libparsers')
+lib = _DeprecatedModule(deprmod='pandas.lib', deprmodto='pandas.libs.lib')
+tslib = _DeprecatedModule(deprmod='pandas.tslib', deprmodto='pandas.libs.tslib')
+
 # use the closest tagged version if possible
 from ._version import get_versions
 v = get_versions()

pandas/compat/pickle_compat.py

@@ -61,7 +61,13 @@ def load_reduce(self):
     ('pandas.core.base', 'FrozenList'): ('pandas.indexes.frozen', 'FrozenList'),
 
     # 10890
-    ('pandas.core.series', 'TimeSeries'): ('pandas.core.series', 'Series')
+    ('pandas.core.series', 'TimeSeries'): ('pandas.core.series', 'Series'),
+
+    # 12588, extensions moving
+    ('pandas._sparse', 'BlockIndex'): ('pandas.sparse.libsparse', 'BlockIndex'),
+    ('pandas.tslib', 'Timestamp'): ('pandas.libs.tslib', 'Timestamp'),
+    ('pandas.tslib', '__nat_unpickle'): ('pandas.libs.tslib', '__nat_unpickle'),
+    ('pandas._period', 'Period'): ('pandas.libs.period', 'Period')
     }
 
 

pandas/computation/scope.py

@@ -46,7 +46,7 @@ def _raw_hex_id(obj):
 
 
 _DEFAULT_GLOBALS = {
-    'Timestamp': pd.lib.Timestamp,
+    'Timestamp': pd.libs.lib.Timestamp,
     'datetime': datetime.datetime,
     'True': True,
     'False': False,

pandas/core/algorithms.py

@@ -6,7 +6,7 @@
 from warnings import warn
 import numpy as np
 
-from pandas import compat, lib, tslib, _np_version_under1p8
+from pandas import compat, _np_version_under1p8
 from pandas.types.cast import _maybe_promote
 from pandas.types.generic import ABCSeries, ABCIndex
 from pandas.types.common import (is_unsigned_integer_dtype,
@@ -34,10 +34,9 @@
 from pandas.types.missing import isnull
 
 import pandas.core.common as com
-import pandas.algos as algos
-import pandas.hashtable as htable
 from pandas.compat import string_types
-from pandas.tslib import iNaT
+from pandas.libs import algos, lib, hashtable as htable
+from pandas.libs.tslib import iNaT
 
 
 # --------------- #
@@ -1412,7 +1411,7 @@ def diff(arr, n, axis=0):
     if needs_i8_conversion(arr):
         dtype = np.float64
         arr = arr.view('i8')
-        na = tslib.iNaT
+        na = iNaT
         is_timedelta = True
     elif issubclass(dtype.type, np.integer):
         dtype = np.float64

pandas/core/base.py

@@ -12,7 +12,7 @@
 
 from pandas.core import common as com
 import pandas.core.nanops as nanops
-import pandas.lib as lib
+import pandas.libs.lib as lib
 from pandas.compat.numpy import function as nv
 from pandas.util.decorators import (Appender, cache_readonly,
                                     deprecate_kwarg, Substitution)

pandas/core/categorical.py

@@ -4,9 +4,9 @@
 from warnings import warn
 import types
 
-from pandas import compat, lib
+from pandas import compat
 from pandas.compat import u, lzip
-import pandas.algos as _algos
+from pandas.libs import lib, algos as libalgos
 
 from pandas.types.generic import ABCSeries, ABCIndexClass, ABCCategoricalIndex
 from pandas.types.missing import isnull, notnull
@@ -1817,8 +1817,8 @@ def _reverse_indexer(self):
 
         """
         categories = self.categories
-        r, counts = _algos.groupsort_indexer(self.codes.astype('int64'),
-                                             categories.size)
+        r, counts = libalgos.groupsort_indexer(self.codes.astype('int64'),
+                                               categories.size)
         counts = counts.cumsum()
         result = [r[counts[indexer]:counts[indexer + 1]]
                   for indexer in range(len(counts) - 1)]
@@ -1897,7 +1897,7 @@ def mode(self):
         modes : `Categorical` (sorted)
         """
 
-        import pandas.hashtable as htable
+        import pandas.libs.hashtable as htable
         good = self._codes != -1
         values = sorted(htable.mode_int64(_ensure_int64(self._codes[good])))
         result = self._constructor(values=values, categories=self.categories,

pandas/core/common.py

@@ -8,8 +8,8 @@
 from functools import partial
 
 import numpy as np
-import pandas.lib as lib
-import pandas.tslib as tslib
+from pandas.libs import lib, tslib
+
 from pandas import compat
 from pandas.compat import long, zip, iteritems
 from pandas.core.config import get_option
@@ -476,7 +476,6 @@ def _where_compat(mask, arr1, arr2):
         new_vals = np.where(mask, arr1.view('i8'), arr2.view('i8'))
         return new_vals.view(_NS_DTYPE)
 
-    import pandas.tslib as tslib
     if arr1.dtype == _NS_DTYPE:
         arr1 = tslib.ints_to_pydatetime(arr1.view('i8'))
     if arr2.dtype == _NS_DTYPE:

pandas/core/frame.py

@@ -71,7 +71,7 @@
 from pandas.core.series import Series
 from pandas.core.categorical import Categorical
 import pandas.computation.expressions as expressions
-import pandas.core.algorithms as algos
+import pandas.core.algorithms as algorithms
 from pandas.computation.eval import eval as _eval
 from pandas.compat import (range, map, zip, lrange, lmap, lzip, StringIO, u,
                            OrderedDict, raise_with_traceback)
@@ -93,8 +93,7 @@
 from pandas.formats.printing import pprint_thing
 import pandas.tools.plotting as gfx
 
-import pandas.lib as lib
-import pandas.algos as _algos
+from pandas.libs import lib, algos as libalgos
 
 from pandas.core.config import get_option
 
@@ -2776,8 +2775,8 @@ def _reindex_multi(self, axes, copy, fill_value):
 
         if row_indexer is not None and col_indexer is not None:
             indexer = row_indexer, col_indexer
-            new_values = algos.take_2d_multi(self.values, indexer,
-                                             fill_value=fill_value)
+            new_values = algorithms.take_2d_multi(self.values, indexer,
+                                                  fill_value=fill_value)
             return self._constructor(new_values, index=new_index,
                                      columns=new_columns)
         else:
@@ -3162,12 +3161,11 @@ def duplicated(self, subset=None, keep='first'):
         duplicated : Series
         """
         from pandas.core.sorting import get_group_index
-        from pandas.hashtable import duplicated_int64, _SIZE_HINT_LIMIT
+        from pandas.libs.hashtable import duplicated_int64, _SIZE_HINT_LIMIT
 
         def f(vals):
-            labels, shape = algos.factorize(vals,
-                                            size_hint=min(len(self),
-                                                          _SIZE_HINT_LIMIT))
+            labels, shape = algorithms.factorize(
+                vals, size_hint=min(len(self), _SIZE_HINT_LIMIT))
             return labels.astype('i8', copy=False), len(shape)
 
         if subset is None:
@@ -3419,7 +3417,7 @@ def nlargest(self, n, columns, keep='first'):
         1  10  b   2
         2   8  d NaN
         """
-        return algos.select_n_frame(self, columns, n, 'nlargest', keep)
+        return algorithms.select_n_frame(self, columns, n, 'nlargest', keep)
 
     def nsmallest(self, n, columns, keep='first'):
         """Get the rows of a DataFrame sorted by the `n` smallest
@@ -3453,7 +3451,7 @@ def nsmallest(self, n, columns, keep='first'):
         0  1  a   1
         2  8  d NaN
         """
-        return algos.select_n_frame(self, columns, n, 'nsmallest', keep)
+        return algorithms.select_n_frame(self, columns, n, 'nsmallest', keep)
 
     def swaplevel(self, i=-2, j=-1, axis=0):
         """
@@ -4721,10 +4719,10 @@ def corr(self, method='pearson', min_periods=1):
         mat = numeric_df.values
 
         if method == 'pearson':
-            correl = _algos.nancorr(_ensure_float64(mat), minp=min_periods)
+            correl = libalgos.nancorr(_ensure_float64(mat), minp=min_periods)
         elif method == 'spearman':
-            correl = _algos.nancorr_spearman(_ensure_float64(mat),
-                                             minp=min_periods)
+            correl = libalgos.nancorr_spearman(_ensure_float64(mat),
+                                               minp=min_periods)
         else:
             if min_periods is None:
                 min_periods = 1
@@ -4784,8 +4782,8 @@ def cov(self, min_periods=None):
                 baseCov = np.cov(mat.T)
             baseCov = baseCov.reshape((len(cols), len(cols)))
         else:
-            baseCov = _algos.nancorr(_ensure_float64(mat), cov=True,
-                                     minp=min_periods)
+            baseCov = libalgos.nancorr(_ensure_float64(mat), cov=True,
+                                       minp=min_periods)
 
         return self._constructor(baseCov, index=idx, columns=cols)
 
@@ -5651,7 +5649,7 @@ def _list_of_series_to_arrays(data, columns, coerce_float=False, dtype=None):
             indexer = indexer_cache[id(index)] = index.get_indexer(columns)
 
         values = _values_from_object(s)
-        aligned_values.append(algos.take_1d(values, indexer))
+        aligned_values.append(algorithms.take_1d(values, indexer))
 
     values = np.vstack(aligned_values)
 

pandas/core/generic.py

@@ -6,11 +6,9 @@
 import gc
 
 import numpy as np
-import pandas.lib as lib
-
 import pandas as pd
 
-
+from pandas.libs import tslib, lib
 from pandas.types.common import (_coerce_to_dtype,
                                  _ensure_int64,
                                  needs_i8_conversion,
@@ -6037,7 +6035,7 @@ def cum_func(self, axis=None, skipna=True, *args, **kwargs):
                 issubclass(y.dtype.type, (np.datetime64, np.timedelta64))):
             result = accum_func(y, axis)
             mask = isnull(self)
-            np.putmask(result, mask, pd.tslib.iNaT)
+            np.putmask(result, mask, tslib.iNaT)
         elif skipna and not issubclass(y.dtype.type, (np.integer, np.bool_)):
             mask = isnull(self)
             np.putmask(y, mask, mask_a)