BLD: added caching to setup.py - speeds up testing dramatically.

Cythoning results and compiled extension are cached.

This is disabled by default, to enable it uncomment the line
specifying the cache directory at the top of setup.py.

you need to occasionally empty the cache manually, to keep
it from from growing too much.
BLD: added caching to setup.sys - speeds up testing dramatically.

Cythoning results and compiled extension are cached.

This is disabled by default, to enable it uncomment the line
specifying the cache directory at the top of setup.py.

you need to occasionally empty the cache manually, to keep
it from from growing too much.

Author: y-p

setup.py

@@ -11,6 +11,21 @@
 import shutil
 import warnings
 
+try:
+    BUILD_CACHE_DIR=None
+
+    # uncomment to activate the build cache
+    #BUILD_CACHE_DIR="/tmp/.pandas_build_cache/"
+
+    if os.isdir(BUILD_CACHE_DIR):
+        print("--------------------------------------------------------")
+        print("BUILD CACHE ACTIVATED. be careful, this is experimental.")
+        print("--------------------------------------------------------")
+    else:
+        BUILD_CACHE_DIR=None
+except :
+    pass
+
 # may need to work around setuptools bug by providing a fake Pyrex
 try:
     import Cython
@@ -87,8 +102,15 @@
 
 from distutils.extension import Extension
 from distutils.command.build import build
-from distutils.command.build_ext import build_ext
 from distutils.command.sdist import sdist
+from distutils.command.build_ext import build_ext
+
+try:
+    from Cython.Distutils import build_ext
+    #from Cython.Distutils import Extension # to get pyrex debugging symbols
+    cython=True
+except ImportError:
+    cython=False
 
 from os.path import splitext, basename, join as pjoin
 
@@ -314,38 +336,213 @@ def build_extensions(self):
         for ext in self.extensions:
             self.build_extension(ext)
 
+class CompilationCacheMixin(object):
+    def __init__(self,*args,**kwds):
+        cache_dir=kwds.pop("cache_dir",BUILD_CACHE_DIR)
+        self.cache_dir=cache_dir
+        if not os.path.isdir(cache_dir):
+            raise Exception("Error: path to Cache directory [%s] is not a dir");
+
+    def _copy_from_cache(self,hash,target):
+        src=os.path.join(self.cache_dir,hash)
+        if os.path.exists(src):
+            #            print("Cache HIT: asked to copy file %s in %s"  % (src,os.path.abspath(target)))
+            s="."
+            for d in target.split(os.path.sep)[:-1]:
+                s=os.path.join(s,d)
+                if not os.path.exists(s):
+                    os.mkdir(s)
+            shutil.copyfile(src,target)
+
+            return True
+
+        return False
+
+    def _put_to_cache(self,hash,src):
+        target=os.path.join(self.cache_dir,hash)
+        # print( "Cache miss: asked to copy file from %s to %s" % (src,target))
+        s="."
+        for d in target.split(os.path.sep)[:-1]:
+            s=os.path.join(s,d)
+            if not os.path.exists(s):
+                os.mkdir(s)
+        shutil.copyfile(src,target)
+
+    def _hash_obj(self,obj):
+        """
+        you should override this method to provide a sensible
+        implementation of hashing functions for your intended objects
+        """
+        try:
+            return hash(obj)
+        except:
+            raise NotImplementedError("You must override this method")
+
+    # this is missing in 2.5, mro will do the right thing
+    def get_ext_fullpath(self, ext_name):
+        """Returns the path of the filename for a given extension.
+
+        The file is located in `build_lib` or directly in the package
+        (inplace option).
+        """
+        import string
+        # makes sure the extension name is only using dots
+        all_dots = string.maketrans('/'+os.sep, '..')
+        ext_name = ext_name.translate(all_dots)
+
+        fullname = self.get_ext_fullname(ext_name)
+        modpath = fullname.split('.')
+        filename = self.get_ext_filename(ext_name)
+        filename = os.path.split(filename)[-1]
+
+        if not self.inplace:
+            # no further work needed
+            # returning :
+            #   build_dir/package/path/filename
+            filename = os.path.join(*modpath[:-1]+[filename])
+            return os.path.join(self.build_lib, filename)
+
+        # the inplace option requires to find the package directory
+        # using the build_py command for that
+        package = '.'.join(modpath[0:-1])
+        build_py = self.get_finalized_command('build_py')
+        package_dir = os.path.abspath(build_py.get_package_dir(package))
+
+        # returning
+        #   package_dir/filename
+        return os.path.join(package_dir, filename)
+
+class CompilationCacheExtMixin(CompilationCacheMixin):
+    def __init__(self,*args,**kwds):
+        CompilationCacheMixin.__init__(self,*args,**kwds)
+
+    def _hash_file(self,fname):
+        from hashlib import sha1
+        try:
+            hash=sha1()
+            hash.update(self.build_lib.encode('utf-8'))
+            try:
+                if sys.version_info[0] >= 3:
+                    import io
+                    f=io.open(fname,"rb")
+                else:
+                    f=open(fname)
+
+                first_line=f.readline()
+                # ignore cython generation timestamp header
+                if "Generated by Cython" not in first_line.decode('utf-8'):
+                    hash.update(first_line)
+                hash.update(f.read())
+                return hash.hexdigest()
+
+            except:
+                raise
+                return None
+            finally:
+                f.close()
+
+        except IOError:
+            return None
+
+    def _hash_obj(self,ext):
+        from hashlib import sha1
+
+        sources = ext.sources
+        if sources is None or \
+          (not hasattr(sources,'__iter__') ) or \
+          isinstance(sources,str) or \
+          sys.version[0]==2 and isinstance(sources,unicode): #argh
+            return False
+
+        sources = list(sources) + ext.depends
+        hash=sha1()
+        try:
+            for fname in sources:
+                fhash=self._hash_file(fname)
+                if fhash:
+                    hash.update(fhash.encode('utf-8'))
+        except:
+            return None
+
+        return hash.hexdigest()
+
+class CachingBuildExt(build_ext,CompilationCacheExtMixin):
+    def __init__(self,*args,**kwds):
+        CompilationCacheExtMixin.__init__(self,*args,**kwds)
+        kwds.pop("cache_dir",None)
+        build_ext.__init__(self,*args,**kwds)
+
+    def build_extension(self, ext,*args,**kwds):
+        ext_path = self.get_ext_fullpath(ext.name)
+        build_path = os.path.join(self.build_lib,os.path.basename(ext_path))
+
+        hash=self._hash_obj(ext)
+        if  hash and self._copy_from_cache(hash,ext_path):
+            return
+
+        build_ext.build_extension(self,ext,*args,**kwds)
+
+        hash=self._hash_obj(ext)
+        if os.path.exists(build_path):
+            self._put_to_cache(hash,build_path) # build_ext
+        if os.path.exists(ext_path):
+            self._put_to_cache(hash,ext_path) # develop
+
+
+    def cython_sources(self, sources, extension):
+        import re
+        cplus = self.cython_cplus or getattr(extension, 'cython_cplus', 0) or \
+                (extension.language and extension.language.lower() == 'c++')
+        target_ext = '.c'
+        if cplus:
+            target_ext = '.cpp'
+
+        for i,s in enumerate(sources):
+            if not re.search("\.(pyx|pxi|pxd)$",s):
+                continue
+            ext_dir=os.path.dirname(s)
+            ext_basename=re.sub("\.[^\.]+$","",os.path.basename(s))
+            ext_basename +=  target_ext
+            target= os.path.join(ext_dir,ext_basename)
+            hash=self._hash_file(s)
+            sources[i]=target
+            if hash and self._copy_from_cache(hash,target):
+                continue
+            build_ext.cython_sources(self,[s],extension)
+            self._put_to_cache(hash,target)
+
+        return sources
+
+class CythonCommand(build_ext):
+    """Custom distutils command subclassed from Cython.Distutils.build_ext
+    to compile pyx->c, and stop there. All this does is override the
+    C-compile method build_extension() with a no-op."""
+    def build_extension(self, ext):
+        pass
+
+class DummyBuildSrc(Command):
+    """ numpy's build_src command interferes with Cython's build_ext.
+    """
+    user_options = []
+    def initialize_options(self):
+        self.py_modules_dict = {}
+    def finalize_options(self):
+        pass
+    def run(self):
+        pass
+
 cmdclass = {'clean': CleanCommand,
             'build': build}
-
-try:
-    from Cython.Distutils import build_ext
-    #from Cython.Distutils import Extension # to get pyrex debugging symbols
-    cython=True
-except ImportError:
-    cython=False
-    suffix = '.c'
-    cmdclass['build_ext'] = CheckingBuildExt
-else:
+if cython:
     suffix = '.pyx'
-    class CythonCommand(build_ext):
-        """Custom distutils command subclassed from Cython.Distutils.build_ext
-        to compile pyx->c, and stop there. All this does is override the
-        C-compile method build_extension() with a no-op."""
-        def build_extension(self, ext):
-            pass
-
-    class DummyBuildSrc(Command):
-        """ numpy's build_src command interferes with Cython's build_ext.
-        """
-        user_options = []
-        def initialize_options(self):
-            self.py_modules_dict = {}
-        def finalize_options(self):
-            pass
-        def run(self):
-            pass
+    cmdclass['build_ext'] = build_ext
+    if BUILD_CACHE_DIR: # use the cache
+        cmdclass['build_ext'] = CachingBuildExt
+else:
 
+    suffix = '.c'
     cmdclass['build_src'] = DummyBuildSrc
+
     cmdclass['cython'] = CythonCommand
     cmdclass['build_ext'] =  build_ext
     cmdclass['sdist'] =  CheckSDist