wip on pandas-dev#8882, releasing the GIL

Author: jreback

pandas/util/testing.py

@@ -1782,3 +1782,36 @@ def use_numexpr(use, min_elements=expr._MIN_ELEMENTS):
 for name, obj in inspect.getmembers(sys.modules[__name__]):
     if inspect.isfunction(obj) and name.startswith('assert'):
         setattr(TestCase, name, staticmethod(obj))
+
+def test_parallel(num_threads=2):
+    """Decorator to run the same function multiple times in parallel.
+
+    Parameters
+    ----------
+    num_threads : int, optional
+        The number of times the function is run in parallel.
+
+    Notes
+    -----
+    This decorator does not pass the return value of the decorated function.
+
+    Original from scikit-image: https://github.com/scikit-image/scikit-image/pull/1519
+
+    """
+
+    assert num_threads > 0
+    import threading
+
+    def wrapper(func):
+        @wraps(func)
+        def inner(*args, **kwargs):
+            threads = []
+            for i in range(num_threads):
+                thread = threading.Thread(target=func, args=args, kwargs=kwargs)
+                threads.append(thread)
+            for thread in threads:
+                thread.start()
+            for thread in threads:
+                thread.join()
+        return inner
+    return wrapper

vb_suite/gil.py

@@ -0,0 +1,42 @@
+from vbench.api import Benchmark
+from datetime import datetime
+
+common_setup = """from pandas_vb_common import *
+from pandas.util.testing import test_parallel
+"""
+
+setup = common_setup + """
+N = 1000000
+ngroups = 100
+
+def get_test_data(ngroups=100, n=N):
+    unique_groups = range(ngroups)
+    arr = np.asarray(np.tile(unique_groups, n / ngroups), dtype=object)
+
+    if len(arr) < n:
+        arr = np.asarray(list(arr) + unique_groups[:n - len(arr)],
+                         dtype=object)
+
+    random.shuffle(arr)
+    return arr
+
+df = DataFrame({'key1' : get_test_data(ngroups=ngroups),
+                'key2' : get_test_data(ngroups=ngroups),
+                'data1' : np.random.randn(N),
+                'data2' : np.random.randn(N)})
+"""
+
+setup = common_setup + """
+
+def f():
+    df.groupby('key1')['data1'].sum()
+
+@test_parallel()
+def pf():
+    f()
+
+"""
+
+groupby_frame_cython_many_columns_nogil = Benchmark(
+    'f()', setup,
+    start_date=datetime(2011, 8, 1))