BLD: make all extensions private

closes pandas-dev#12588

Author: jreback

doc/source/whatsnew/v0.20.0.txt

@@ -246,6 +246,22 @@ Using ``.iloc``. Here we will get the location of the 'A' column, then use *posi
   df.iloc[[0, 2], df.columns.get_loc('A')]
 
 
+.. _whatsnew_0200.api_breaking.extensions:
+
+Extensions have been made private
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+All pandas compiled extensions are now imported privately to the pandas namespace. These are now imported as ``_*`` (:issue:`12588`)
+
+This includes the following c-extensions:
+
+- tslib
+- hashtable
+- parser
+- algos
+- lib
+- json
+
 .. _whatsnew_0200.api_breaking.index_map:
 
 Map on Index types now return other Index types

pandas/__init__.py

@@ -23,7 +23,7 @@
 from pandas.compat.numpy import *
 
 try:
-    from pandas import hashtable, tslib, lib
+    from pandas import hashtable, _tslib, _lib
 except ImportError as e:  # pragma: no cover
     # hack but overkill to use re
     module = str(e).lstrip('cannot import name ')

pandas/algos.pyx

@@ -46,7 +46,7 @@ from util cimport numeric, get_nat
 
 cimport lib
 from lib cimport is_null_datetimelike
-from pandas import lib
+from pandas import _lib as lib
 
 cdef int64_t iNaT = get_nat()
 

pandas/hashtable.pyx

@@ -4,6 +4,7 @@ from cpython cimport PyObject, Py_INCREF, PyList_Check, PyTuple_Check
 
 from khash cimport *
 from numpy cimport *
+from hashtable cimport *
 
 from libc.stdlib cimport malloc, free
 from cpython cimport (PyMem_Malloc, PyMem_Realloc, PyMem_Free,
@@ -22,8 +23,6 @@ cdef extern from "numpy/npy_math.h":
 cimport cython
 cimport numpy as cnp
 
-from pandas.lib import checknull
-
 cnp.import_array()
 cnp.import_ufunc()
 
@@ -41,7 +40,6 @@ cdef extern from "Python.h":
 
 cdef size_t _INIT_VEC_CAP = 128
 
-
 include "hashtable_class_helper.pxi"
 include "hashtable_func_helper.pxi"
 

pandas/lib.pyx

@@ -57,8 +57,7 @@ from datetime cimport *
 
 from tslib cimport (convert_to_tsobject, convert_to_timedelta64,
                     _check_all_nulls)
-import tslib
-from tslib import NaT, Timestamp, Timedelta
+from _tslib import NaT, Timestamp, Timedelta
 
 cdef int64_t NPY_NAT = util.get_nat()
 

pandas/src/period.pyx

@@ -25,8 +25,8 @@ from datetime cimport *
 cimport util
 cimport lib
 from lib cimport is_null_datetimelike, is_period
-import lib
-from pandas import tslib
+import _lib as lib
+from pandas import _tslib as tslib
 from tslib import Timedelta, Timestamp, iNaT, NaT
 from tslib import have_pytz, _get_utcoffset
 from tslib cimport (

pandas/tests/api/test_api.py

@@ -33,13 +33,8 @@ class TestPDApi(Base, tm.TestCase):
     # top-level sub-packages
     lib = ['api', 'compat', 'computation', 'core',
            'indexes', 'formats', 'pandas',
-           'test', 'tools', 'tseries',
-           'types', 'util', 'options', 'io']
-
-    # top-level packages that are c-imports, should rename to _*
-    # to avoid naming conflicts
-    lib_to_rename = ['algos', 'hashtable', 'tslib', 'msgpack', 'sparse',
-                     'json', 'lib', 'index', 'parser']
+           'test', 'tools', 'tseries', 'msgpack',
+           'types', 'util', 'options', 'sparse', 'io']
 
     # these are already deprecated; awaiting removal
     deprecated_modules = ['stats', 'datetools']

setup.py

@@ -113,11 +113,11 @@ def is_platform_mac():
 _pxi_dep_template = {
     'algos': ['algos_common_helper.pxi.in', 'algos_groupby_helper.pxi.in',
               'algos_take_helper.pxi.in', 'algos_rank_helper.pxi.in'],
-    '_join': ['join_helper.pxi.in', 'joins_func_helper.pxi.in'],
+    'join': ['join_helper.pxi.in', 'joins_func_helper.pxi.in'],
     'hashtable': ['hashtable_class_helper.pxi.in',
                   'hashtable_func_helper.pxi.in'],
     'index': ['index_class_helper.pxi.in'],
-    '_sparse': ['sparse_op_helper.pxi.in']
+    'sparse': ['sparse_op_helper.pxi.in']
 }
 _pxifiles = []
 _pxi_dep = {}
@@ -471,54 +471,54 @@ def pxd(name):
 libraries = ['m'] if not is_platform_windows() else []
 
 ext_data = dict(
-    lib={'pyxfile': 'lib',
-         'pxdfiles': [],
-         'depends': lib_depends},
+    _lib={'pyxfile': 'lib',
+          'pxdfiles': [],
+          'depends': lib_depends},
     hashtable={'pyxfile': 'hashtable',
-               'pxdfiles': ['hashtable'],
+               'pxdfiles': ['hashtable', 'src/util', 'src/khash'],
                'depends': (['pandas/src/klib/khash_python.h']
                            + _pxi_dep['hashtable'])},
-    tslib={'pyxfile': 'tslib',
-           'depends': tseries_depends,
-           'sources': ['pandas/src/datetime/np_datetime.c',
-                       'pandas/src/datetime/np_datetime_strings.c',
-                       'pandas/src/period_helper.c']},
+    _tslib={'pyxfile': 'tslib',
+            'depends': tseries_depends,
+            'sources': ['pandas/src/datetime/np_datetime.c',
+                        'pandas/src/datetime/np_datetime_strings.c',
+                        'pandas/src/period_helper.c']},
     _period={'pyxfile': 'src/period',
              'depends': tseries_depends,
              'sources': ['pandas/src/datetime/np_datetime.c',
                          'pandas/src/datetime/np_datetime_strings.c',
                          'pandas/src/period_helper.c']},
-    index={'pyxfile': 'index',
-           'sources': ['pandas/src/datetime/np_datetime.c',
-                       'pandas/src/datetime/np_datetime_strings.c'],
-           'pxdfiles': ['src/util', 'hashtable'],
-           'depends': _pxi_dep['index']},
-    algos={'pyxfile': 'algos',
-           'pxdfiles': ['src/util', 'hashtable'],
-           'depends': _pxi_dep['algos']},
+    _index={'pyxfile': 'index',
+            'sources': ['pandas/src/datetime/np_datetime.c',
+                        'pandas/src/datetime/np_datetime_strings.c'],
+            'pxdfiles': ['src/util', 'hashtable'],
+            'depends': _pxi_dep['index']},
+    _algos={'pyxfile': 'algos',
+            'pxdfiles': ['src/util', 'hashtable'],
+            'depends': _pxi_dep['algos']},
     _join={'pyxfile': 'src/join',
            'pxdfiles': ['src/util', 'hashtable'],
-           'depends': _pxi_dep['_join']},
+           'depends': _pxi_dep['join']},
     _window={'pyxfile': 'window',
              'pxdfiles': ['src/skiplist', 'src/util'],
              'depends': ['pandas/src/skiplist.pyx',
                          'pandas/src/skiplist.h']},
-    parser={'pyxfile': 'parser',
-            'depends': ['pandas/src/parser/tokenizer.h',
-                        'pandas/src/parser/io.h',
-                        'pandas/src/numpy_helper.h'],
-            'sources': ['pandas/src/parser/tokenizer.c',
-                        'pandas/src/parser/io.c']},
+    _parser={'pyxfile': 'parser',
+             'depends': ['pandas/src/parser/tokenizer.h',
+                         'pandas/src/parser/io.h',
+                         'pandas/src/numpy_helper.h'],
+             'sources': ['pandas/src/parser/tokenizer.c',
+                         'pandas/src/parser/io.c']},
     _sparse={'pyxfile': 'src/sparse',
              'depends': ([srcpath('sparse', suffix='.pyx')] +
-                         _pxi_dep['_sparse'])},
+                         _pxi_dep['sparse'])},
     _testing={'pyxfile': 'src/testing',
               'depends': [srcpath('testing', suffix='.pyx')]},
     _hash={'pyxfile': 'src/hash',
            'depends': [srcpath('hash', suffix='.pyx')]},
 )
 
-ext_data["io.sas.saslib"] = {'pyxfile': 'io/sas/saslib'}
+ext_data["io.sas._saslib"] = {'pyxfile': 'io/sas/saslib'}
 
 extensions = []
 
@@ -583,7 +583,7 @@ def pxd(name):
             root, _ = os.path.splitext(ext.sources[0])
             ext.sources[0] = root + suffix
 
-ujson_ext = Extension('pandas.json',
+ujson_ext = Extension('pandas._json',
                       depends=['pandas/src/ujson/lib/ultrajson.h',
                                'pandas/src/datetime_helper.h',
                                'pandas/src/numpy_helper.h'],