BLD: consolidate remaining extensions

moves extensions to pandas/_libs, which holds the extension code
and also the generated builds (as its importable).

pandas/_libs/src is now almost an includes dir, holding low-frequency changing code.
This consolidates the import process making it more uniform and
consistent throughout the codebase.

Finally this cleans up the remaining top-level namespace (with some deprecations in place for
example pandas.lib, pandas.tslib, pandas.json, pandas.parser.

I listed all of the changes in the whatsnew, but I
don't think worthwhile deprecating anything else.

Author: Jeff Reback <[email protected]>

Closes pandas-dev#15537 from jreback/extensions3 and squashes the following commits:

a6d6cfa [Jeff Reback] BLD: rename / move some extensions

Author: jreback

Makefile

@@ -1,4 +1,4 @@
-tseries: pandas/lib.pyx pandas/tslib.pyx pandas/hashtable.pyx
+tseries: pandas/_libs/lib.pyx pandas/_libs/tslib.pyx pandas/_libs/hashtable.pyx
 	python setup.py build_ext --inplace
 
 .PHONY : develop build clean clean_pyc tseries doc

asv_bench/benchmarks/binary_ops.py

@@ -107,4 +107,4 @@ def setup(self):
         self.s = Series(date_range('20010101', periods=self.N, freq='T', tz='US/Eastern'))
         self.ts = self.s[self.halfway]
 
-        self.s2 = Series(date_range('20010101', periods=self.N, freq='s',  tz='US/Eastern'))
+        self.s2 = Series(date_range('20010101', periods=self.N, freq='s',  tz='US/Eastern'))

asv_bench/benchmarks/pandas_vb_common.py

@@ -8,16 +8,22 @@
 import random
 import numpy as np
 import threading
+from importlib import import_module
+
 try:
     from pandas.compat import range
 except ImportError:
     pass
 
 np.random.seed(1234)
-try:
-    import pandas._tseries as lib
-except:
-    import pandas.lib as lib
+
+# try em until it works!
+for imp in ['pandas_tseries', 'pandas.lib', 'pandas._libs.lib']:
+    try:
+        lib = import_module(imp)
+        break
+    except:
+        pass
 
 try:
     Panel = Panel

asv_bench/benchmarks/panel_methods.py

@@ -21,4 +21,4 @@ def time_shift(self):
         self.panel.shift(1)
 
     def time_shift_minor(self):
-        self.panel.shift(1, axis='minor')
+        self.panel.shift(1, axis='minor')

doc/source/whatsnew/v0.20.0.txt

@@ -484,6 +484,35 @@ New Behavior:
    In [11]: index.memory_usage(deep=True)
    Out[11]: 260
 
+.. _whatsnew_0200.api_breaking.extensions:
+
+Extension Modules Moved
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Some formerly public c/c++/cython extension modules have been moved and/or renamed. These are all removed from the public API.
+If indicated, a deprecation warning will be issued if you reference that module. (:issue:`12588`)
+
+.. csv-table::
+    :header: "Previous Location", "New Location", "Deprecated"
+    :widths: 30, 30, 4
+
+    "pandas.lib", "pandas._libs.lib", "X"
+    "pandas.tslib", "pandas._libs.tslib", "X"
+    "pandas._join", "pandas._libs.join", ""
+    "pandas._period", "pandas._libs.period", ""
+    "pandas.msgpack", "pandas.io.msgpack", ""
+    "pandas.index", "pandas._libs.index", ""
+    "pandas.algos", "pandas._libs.algos", ""
+    "pandas.hashtable", "pandas._libs.hashtable", ""
+    "pandas.json", "pandas.io.json.libjson", "X"
+    "pandas.parser", "pandas.io.libparsers", "X"
+    "pandas.io.sas.saslib", "pandas.io.sas.libsas", ""
+    "pandas._testing", "pandas.util.libtesting", ""
+    "pandas._sparse", "pandas.sparse.libsparse", ""
+    "pandas._hash", "pandas.tools.libhash", ""
+    "pandas._window", "pandas.core.libwindow", ""
+
+
 .. _whatsnew_0200.api_breaking.groupby_describe:
 
 Groupby Describe Formatting

pandas/__init__.py

@@ -23,7 +23,9 @@
 from pandas.compat.numpy import *
 
 try:
-    from pandas import hashtable, tslib, lib
+    from pandas._libs import (hashtable as _hashtable,
+                             lib as _lib,
+                             tslib as _tslib)
 except ImportError as e:  # pragma: no cover
     # hack but overkill to use re
     module = str(e).lstrip('cannot import name ')
@@ -52,11 +54,17 @@
 from pandas.tools.util import to_numeric
 from pandas.core.reshape import melt
 from pandas.util.print_versions import show_versions
-
 from pandas.io.api import *
-
 from pandas.util._tester import test
 
+# extension module deprecations
+from pandas.util.depr_module import _DeprecatedModule
+
+json = _DeprecatedModule(deprmod='pandas.json', deprmodto='pandas.io.json.libjson')
+parser = _DeprecatedModule(deprmod='pandas.parser', deprmodto='pandas.io.libparsers')
+lib = _DeprecatedModule(deprmod='pandas.lib', deprmodto='pandas._libs.lib')
+tslib = _DeprecatedModule(deprmod='pandas.tslib', deprmodto='pandas._libs.tslib')
+
 # use the closest tagged version if possible
 from ._version import get_versions
 v = get_versions()

pandas/_libs/__init__.py

@@ -0,0 +1,8 @@
+# flake8: noqa
+
+from .tslib import iNaT, NaT, Timestamp, Timedelta, OutOfBoundsDatetime
+
+# TODO
+# period is directly dependent on tslib and imports python
+# modules, so exposing Period as an alias is currently not possible
+# from period import Period

pandas/algos.pyx renamed to pandas/_libs/algos.pyx

@@ -37,7 +37,7 @@ float64 = np.dtype(np.float64)
 cdef double NaN = <double> np.NaN
 cdef double nan = NaN
 
-cdef extern from "src/headers/math.h":
+cdef extern from "../src/headers/math.h":
     double sqrt(double x) nogil
     double fabs(double) nogil
 
@@ -46,7 +46,7 @@ from util cimport numeric, get_nat
 
 cimport lib
 from lib cimport is_null_datetimelike
-from pandas import lib
+from pandas._libs import lib
 
 cdef int64_t iNaT = get_nat()
 

pandas/src/algos_common_helper.pxi.in renamed to pandas/_libs/algos_common_helper.pxi.in

@@ -433,7 +433,7 @@ def arrmap_{{name}}(ndarray[{{c_type}}] index, object func):
 
     cdef ndarray[object] result = np.empty(length, dtype=np.object_)
 
-    from pandas.lib import maybe_convert_objects
+    from pandas._libs.lib import maybe_convert_objects
 
     for i in range(length):
         result[i] = func(index[i])

pandas/src/algos_groupby_helper.pxi.in renamed to pandas/_libs/algos_groupby_helper.pxi.in

@@ -22,7 +22,7 @@ cdef extern from "numpy/npy_math.h":
 cimport cython
 cimport numpy as cnp
 
-from pandas.lib import checknull
+from pandas._libs.lib import checknull
 
 cnp.import_array()
 cnp.import_ufunc()

pandas/src/algos_rank_helper.pxi.in renamed to pandas/_libs/algos_rank_helper.pxi.in

@@ -17,8 +17,8 @@ import numpy as np
 
 cimport tslib
 from hashtable cimport *
-from pandas import algos, tslib, hashtable as _hash
-from pandas.tslib import Timestamp, Timedelta
+from pandas._libs import tslib, algos, hashtable as _hash
+from pandas._libs.tslib import Timestamp, Timedelta
 
 from datetime cimport (get_datetime64_value, _pydatetime_to_dts,
                        pandas_datetimestruct)

pandas/src/algos_take_helper.pxi.in renamed to pandas/_libs/algos_take_helper.pxi.in

@@ -32,10 +32,10 @@ float64 = np.dtype(np.float64)
 cdef double NaN = <double> np.NaN
 cdef double nan = NaN
 
-from pandas.algos import groupsort_indexer, ensure_platform_int
+from pandas._libs.algos import groupsort_indexer, ensure_platform_int
 from pandas.core.algorithms import take_nd
 
-include "joins_func_helper.pxi"
+include "join_func_helper.pxi"
 
 
 def inner_join(ndarray[int64_t] left, ndarray[int64_t] right,

pandas/hashtable.pxd renamed to pandas/_libs/hashtable.pxd

@@ -16,19 +16,15 @@ cdef extern from "datetime_helper.h":
 from libc.stdlib cimport free
 
 from pandas import compat
-
-from pandas.tseries import offsets
-from pandas.tseries.tools import parse_time_string
+from pandas.compat import PY2
 
 cimport cython
 from datetime cimport *
-cimport util
-cimport lib
+cimport util, lib
 from lib cimport is_null_datetimelike, is_period
-import lib
-from pandas import tslib
-from tslib import Timedelta, Timestamp, iNaT, NaT
-from tslib import have_pytz, _get_utcoffset
+from pandas._libs import tslib, lib
+from pandas._libs.tslib import (Timedelta, Timestamp, iNaT,
+                                NaT, have_pytz, _get_utcoffset)
 from tslib cimport (
     maybe_get_tz,
     _is_utc,
@@ -37,12 +33,10 @@ from tslib cimport (
     _nat_scalar_rules,
 )
 
+from pandas.tseries import offsets
+from pandas.tseries.tools import parse_time_string
 from pandas.tseries import frequencies
 
-from sys import version_info
-
-cdef bint PY2 = version_info[0] == 2
-
 cdef int64_t NPY_NAT = util.get_nat()
 
 cdef int RESO_US = frequencies.RESO_US
@@ -474,7 +468,7 @@ def extract_ordinals(ndarray[object] values, freq):
         p = values[i]
 
         if is_null_datetimelike(p):
-            ordinals[i] = tslib.iNaT
+            ordinals[i] = iNaT
         else:
             try:
                 ordinals[i] = p.ordinal
@@ -485,9 +479,9 @@ def extract_ordinals(ndarray[object] values, freq):
 
             except AttributeError:
                 p = Period(p, freq=freq)
-                if p is tslib.NaT:
+                if p is NaT:
                     # input may contain NaT-like string
-                    ordinals[i] = tslib.iNaT
+                    ordinals[i] = iNaT
                 else:
                     ordinals[i] = p.ordinal
 
@@ -716,8 +710,8 @@ cdef class _Period(object):
         """
         Fast creation from an ordinal and freq that are already validated!
         """
-        if ordinal == tslib.iNaT:
-            return tslib.NaT
+        if ordinal == iNaT:
+            return NaT
         else:
             self = _Period.__new__(cls)
             self.ordinal = ordinal
@@ -730,7 +724,7 @@ cdef class _Period(object):
                 msg = _DIFFERENT_FREQ.format(self.freqstr, other.freqstr)
                 raise IncompatibleFrequency(msg)
             return PyObject_RichCompareBool(self.ordinal, other.ordinal, op)
-        elif other is tslib.NaT:
+        elif other is NaT:
             return _nat_scalar_rules[op]
         # index/series like
         elif hasattr(other, '_typ'):
@@ -776,8 +770,8 @@ cdef class _Period(object):
                                   offsets.Tick, offsets.DateOffset,
                                   Timedelta)):
                 return self._add_delta(other)
-            elif other is tslib.NaT:
-                return tslib.NaT
+            elif other is NaT:
+                return NaT
             elif lib.is_integer(other):
                 ordinal = self.ordinal + other * self.freq.n
                 return Period(ordinal=ordinal, freq=self.freq)
@@ -808,8 +802,8 @@ cdef class _Period(object):
             else:  # pragma: no cover
                 return NotImplemented
         elif isinstance(other, Period):
-            if self is tslib.NaT:
-                return tslib.NaT
+            if self is NaT:
+                return NaT
             return NotImplemented
         else:
             return NotImplemented
@@ -1164,7 +1158,7 @@ class Period(_Period):
             if (year is None and month is None and
                         quarter is None and day is None and
                         hour is None and minute is None and second is None):
-                ordinal = tslib.iNaT
+                ordinal = iNaT
             else:
                 if freq is None:
                     raise ValueError("If value is None, freq cannot be None")
@@ -1190,7 +1184,7 @@ class Period(_Period):
                 ordinal = converted.ordinal
 
         elif is_null_datetimelike(value) or value in tslib._nat_strings:
-            ordinal = tslib.iNaT
+            ordinal = iNaT
 
         elif isinstance(value, compat.string_types) or lib.is_integer(value):
             if lib.is_integer(value):

pandas/hashtable.pyx renamed to pandas/_libs/hashtable.pyx

@@ -180,7 +180,7 @@ void initObjToJSON(void)
         Py_DECREF(mod_pandas);
     }
 
-    mod_tslib = PyImport_ImportModule("pandas.tslib");
+    mod_tslib = PyImport_ImportModule("pandas._libs.tslib");
     if (mod_tslib) {
         cls_nat = (PyTypeObject *)PyObject_GetAttrString(mod_tslib, "NaTType");
         Py_DECREF(mod_tslib);

pandas/src/hashtable_class_helper.pxi.in renamed to pandas/_libs/hashtable_class_helper.pxi.in

@@ -80,7 +80,7 @@ static PyMethodDef ujsonMethods[] = {
 
 static struct PyModuleDef moduledef = {
     PyModuleDef_HEAD_INIT,
-    "_pandasujson",
+    "_libjson",
     0,            /* m_doc */
     -1,           /* m_size */
     ujsonMethods, /* m_methods */
@@ -90,14 +90,14 @@ static struct PyModuleDef moduledef = {
     NULL          /* m_free */
 };
 
-#define PYMODINITFUNC PyMODINIT_FUNC PyInit_json(void)
+#define PYMODINITFUNC PyMODINIT_FUNC PyInit_libjson(void)
 #define PYMODULE_CREATE() PyModule_Create(&moduledef)
 #define MODINITERROR return NULL
 
 #else
 
-#define PYMODINITFUNC PyMODINIT_FUNC initjson(void)
-#define PYMODULE_CREATE() Py_InitModule("json", ujsonMethods)
+#define PYMODINITFUNC PyMODINIT_FUNC initlibjson(void)
+#define PYMODULE_CREATE() Py_InitModule("libjson", ujsonMethods)
 #define MODINITERROR return
 
 #endif

pandas/src/hashtable_func_helper.pxi.in renamed to pandas/_libs/hashtable_func_helper.pxi.in

@@ -62,7 +62,13 @@ def load_reduce(self):
 
     # 10890
     ('pandas.core.series', 'TimeSeries'): ('pandas.core.series', 'Series'),
-    ('pandas.sparse.series', 'SparseTimeSeries'): ('pandas.sparse.series', 'SparseSeries')
+    ('pandas.sparse.series', 'SparseTimeSeries'): ('pandas.sparse.series', 'SparseSeries'),
+
+    # 12588, extensions moving
+    ('pandas._sparse', 'BlockIndex'): ('pandas.sparse.libsparse', 'BlockIndex'),
+    ('pandas.tslib', 'Timestamp'): ('pandas._libs.tslib', 'Timestamp'),
+    ('pandas.tslib', '__nat_unpickle'): ('pandas._libs.tslib', '__nat_unpickle'),
+    ('pandas._period', 'Period'): ('pandas._libs.period', 'Period')
     }
 
 

pandas/index.pyx renamed to pandas/_libs/index.pyx

@@ -1,4 +1,5 @@
-"""Module for scope operations
+"""
+Module for scope operations
 """
 
 import sys
@@ -10,7 +11,8 @@
 
 import numpy as np
 
-import pandas as pd
+import pandas
+import pandas as pd  # noqa
 from pandas.compat import DeepChainMap, map, StringIO
 from pandas.core.base import StringMixin
 import pandas.computation as compu
@@ -46,7 +48,7 @@ def _raw_hex_id(obj):
 
 
 _DEFAULT_GLOBALS = {
-    'Timestamp': pd.lib.Timestamp,
+    'Timestamp': pandas._libs.lib.Timestamp,
     'datetime': datetime.datetime,
     'True': True,
     'False': False,