Merge branch 'master' into bugfix/GR-12657

Author: ppisl

CHANGELOG.md

@@ -3,6 +3,23 @@
 This changelog summarizes major changes between GraalVM versions of the Python
 language runtime. The main focus is on user-observable behavior of the engine.
 
+## Version 1.0.0 RC10
+
+* Improve performance of C API upcalls
+* Improve performance of classmethods, staticmethods, `globals()`, and `locals()`
+* Improve performance of various string and bytes operations
+* Initial support for the `_thread` builtin module (actual multi-threading is still disabled, the API defaults to a dummy implementation)
+* Implement the `zipimporter` module
+* Support assignment to `object.__class__`
+* Use the new Truffle filesystem API to get/set the current working directory
+* Attempt our best to report side-effects in KEY_INFO
+* The KEYS message now responds with attributes and methods, never dict keys
+* Support the `input` builtin
+* Add DEBUG launcher options for performance debugging
+* Ensure context isolation for file descriptors and child PIDs
+* Fix passing custom locals and globals through `exec` and `eval`
+* Fixes to builtin `help`
+
 ## Version 1.0.0 RC9
 
 * Support `help` in the builtin Python shell

graalpython/benchmarks/src/harness.py

@@ -55,6 +55,109 @@
 ATTR_TEARDOWN = '__teardown__'
 
 
+# ----------------------------------------------------------------------------------------------------------------------
+#
+# the CUSUM method adapted for warmup detection within a given threshold (initial iterations)
+#
+# ----------------------------------------------------------------------------------------------------------------------
+def zeros(n):
+    return [0 for _ in range(n)]
+
+
+def append(arr, val):
+    if isinstance(arr, list):
+        return arr + [val]
+    else:
+        return [val] + arr
+
+
+def cusum(values, threshold=1.0, drift=0.0):
+    csum_pos, csum_neg = zeros(len(values)), zeros(len(values))
+    change_points = []
+    for i in range(1, len(values)):
+        diff = values[i] - values[i - 1]
+        csum_pos[i] = csum_pos[i-1] + diff - drift
+        csum_neg[i] = csum_neg[i-1] - diff - drift
+
+        if csum_pos[i] < 0:
+            csum_pos[i] = 0
+        if csum_neg[i] < 0:
+            csum_neg[i] = 0
+
+        if csum_pos[i] > threshold or csum_neg[i] > threshold:
+            change_points = append(change_points, i)
+            csum_pos[i], csum_neg[i] = 0, 0
+
+    return change_points
+
+
+def avg(values):
+    return float(sum(values)) / len(values)
+
+
+def norm(values):
+    _max, _min  = max(values), min(values)
+    return [float(v - _min) / (_max - _min) * 100.0 for v in values]
+
+
+def pairwise_slopes(values, cp):
+    return [abs(float(values[i+1] - values[i]) / float(cp[i+1] - cp[i])) for i in range(len(values)-1)]
+
+
+def last_n_percent_runs(values, n=0.1):
+    assert 0.0 < n <= 1.0
+    end_runs_idx = len(values) - int(len(values) * n)
+    end_runs_idx = len(values) - 1 if end_runs_idx >= len(values) else end_runs_idx
+    return values[end_runs_idx:], list(range(end_runs_idx, len(values)))
+
+
+def first_n_percent_runs(values, n=0.1):
+    assert 0.0 < n <= 1.0
+    first_run_idx = int(len(values) * n)
+    return first_run_idx -1 if first_run_idx == len(values) else first_run_idx
+
+
+def detect_warmup(values, cp_threshold=0.03, stability_slope_grade=0.01):
+    """
+    detect the point of warmup point (iteration / run)
+
+    :param values: the durations for each run
+    :param cp_threshold:  the percent in value difference for a point to be considered a change point (percentage)
+    :param stability_slope_grade: the slope grade (percentage). A grade of 1% corresponds to a slope of 0.5 degrees
+    :return: the change point or -1 if not detected
+    """
+    # normalize all
+    stability_slope_grade *= 100.0
+    cp_threshold *= 100
+    values = norm(values)
+
+    try:
+        cp = cusum(values, threshold=cp_threshold)
+        rolling_avg = [avg(values[i:]) for i in cp]
+
+        def warmup(cp_index):
+            val_idx = cp[cp_index] + 1
+            return val_idx if val_idx < len(values) else -1
+
+        # find the point where the duration avg is below the cp threshold
+        for i, d in enumerate(rolling_avg):
+            if d <= cp_threshold:
+                return warmup(i)
+
+        # could not find something below the CP threshold (noise in the data), use the stabilisation of slopes
+        last_n_vals, last_n_idx = last_n_percent_runs(values, 0.1)
+        slopes = pairwise_slopes(rolling_avg + last_n_vals, cp + last_n_idx)
+
+        for i, d in enumerate(slopes):
+            if d <= stability_slope_grade:
+                return warmup(i)
+
+        return -1
+    except Exception as e:
+        print("exception occurred while detecting warmup: %s" % e)
+        return -1
+
+
 def ccompile(name, code):
     from importlib import invalidate_caches
     from distutils.core import setup, Extension
@@ -89,7 +192,14 @@ def _as_int(value):
 
 
 class BenchRunner(object):
-    def __init__(self, bench_file, bench_args=None, iterations=1, warmup=0):
+    def __init__(self, bench_file, bench_args=None, iterations=1, warmup=-1, warmup_runs=0):
+        assert isinstance(iterations, int), \
+            "BenchRunner iterations argument must be an int, got %s instead" % iterations
+        assert isinstance(warmup, int), \
+            "BenchRunner warmup argument must be an int, got %s instead" % warmup
+        assert isinstance(warmup_runs, int), \
+            "BenchRunner warmup_runs argument must be an int, got %s instead" % warmup_runs
+
         if bench_args is None:
             bench_args = []
         self.bench_module = BenchRunner.get_bench_module(bench_file)
@@ -98,10 +208,8 @@ def __init__(self, bench_file, bench_args=None, iterations=1, warmup=0):
         _iterations = _as_int(iterations)
         self._run_once = _iterations <= 1
         self.iterations = 1 if self._run_once else _iterations
-
-        assert isinstance(self.iterations, int)
-        self.warmup = _as_int(warmup)
-        assert isinstance(self.warmup, int)
+        self.warmup_runs = warmup_runs if warmup_runs > 0 else 0
+        self.warmup = warmup if warmup > 0 else -1
 
     @staticmethod
     def get_bench_module(bench_file):
@@ -139,9 +247,10 @@ def _call_attr(self, attr_name, *args):
 
     def run(self):
         if self._run_once:
-            print("### %s, exactly one iteration (no warmup curves)" % (self.bench_module.__name__))
+            print("### %s, exactly one iteration (no warmup curves)" % self.bench_module.__name__)
         else:
-            print("### %s, %s warmup iterations, %s bench iterations " % (self.bench_module.__name__, self.warmup, self.iterations))
+            print("### %s, %s warmup iterations, %s bench iterations " % (self.bench_module.__name__,
+                                                                          self.warmup_runs, self.iterations))
 
         # process the args if the processor function is defined
         args = self._call_attr(ATTR_PROCESS_ARGS, *self.bench_args)
@@ -159,9 +268,9 @@ def run(self):
         bench_func = self._get_attr(ATTR_BENCHMARK)
         durations = []
         if bench_func and hasattr(bench_func, '__call__'):
-            if self.warmup:
-                print("### warming up for %s iterations ... " % self.warmup)
-                for _ in range(self.warmup):
+            if self.warmup_runs:
+                print("### (pre)warming up for %s iterations ... " % self.warmup_runs)
+                for _ in range(self.warmup_runs):
                     bench_func(*args)
 
             for iteration in range(self.iterations):
@@ -173,21 +282,46 @@ def run(self):
                 if self._run_once:
                     print("@@@ name=%s, duration=%s" % (self.bench_module.__name__, duration_str))
                 else:
-                    print("### iteration=%s, name=%s, duration=%s" % (iteration, self.bench_module.__name__, duration_str))
+                    print("### iteration=%s, name=%s, duration=%s" % (iteration, self.bench_module.__name__,
+                                                                      duration_str))
 
         print(_HRULE)
         print("### teardown ... ")
         self._call_attr(ATTR_TEARDOWN)
         print("### benchmark complete")
         print(_HRULE)
-        print("### BEST     duration: %.3f s" % min(durations))
-        print("### WORST    duration: %.3f s" % max(durations))
-        print("### AVG      duration: %.3f" % (sum(durations) / len(durations)))
+
+        # summary
+        if self._run_once:
+            print("### SINGLE RUN        duration: %.3f s" % durations[0])
+        else:
+            print("### BEST                duration: %.3f s" % min(durations))
+            print("### WORST               duration: %.3f s" % max(durations))
+            print("### AVG (all runs)      duration: %.3f s" % (sum(durations) / len(durations)))
+            warmup_iter = self.warmup if self.warmup > 0 else detect_warmup(durations)
+            # if we cannot detect a warmup starting point but we performed some pre runs, we take a starting point
+            # after the 10% of the first runs ...
+            if warmup_iter < 0 and self.warmup_runs > 0:
+                print("### warmup could not be detected, but %s pre-runs were executed.\n"
+                      "### we assume the benchmark is warmed up and pick an iteration "
+                      "in the first 10%% of the runs" % self.warmup_runs)
+                warmup_iter = first_n_percent_runs(durations, 0.1)
+
+            if warmup_iter > 0:
+                print("### WARMUP %s at iteration: %d" % ("specified" if self.warmup > 0 else "detected", warmup_iter))
+                no_warmup_durations = durations[warmup_iter:]
+                print("### AVG (no warmup)     duration: %.3f s" % (sum(no_warmup_durations) / len(no_warmup_durations)))
+            else:
+                print("### WARMUP iteration not specified or could not be detected")
+
+        print(_HRULE)
+        print("### RAW DURATIONS: %s" % str(durations))
         print(_HRULE)
 
 
 def run_benchmark(args):
-    warmup = 0
+    warmup = -1
+    warmup_runs = 0
     iterations = 1
     bench_file = None
     bench_args = []
@@ -208,6 +342,12 @@ def run_benchmark(args):
         elif arg.startswith("--warmup"):
             warmup = _as_int(arg.split("=")[1])
 
+        elif arg == '-r':
+            i += 1
+            warmup_runs = _as_int(args[i])
+        elif arg.startswith("--warmup-runs"):
+            warmup_runs = _as_int(arg.split("=")[1])
+
         elif arg == '-p':
             i += 1
             paths = args[i].split(",")
@@ -229,7 +369,7 @@ def run_benchmark(args):
     else:
         print("### no extra module search paths specified")
 
-    BenchRunner(bench_file, bench_args=bench_args, iterations=iterations, warmup=warmup).run()
+    BenchRunner(bench_file, bench_args=bench_args, iterations=iterations, warmup=warmup, warmup_runs=warmup_runs).run()
 
 
 if __name__ == '__main__':

graalpython/com.oracle.graal.python.cext/src/abstract.c

@@ -136,6 +136,10 @@ PyObject * PyNumber_Invert(PyObject *o) {
 	return do_unaryop(o, INVERT);
 }
 
+PyObject * PyNumber_Power(PyObject *v, PyObject *w, PyObject *z) {
+    return UPCALL_O(PY_BUILTIN, polyglot_from_string("pow", SRC_CS), native_to_java(v), native_to_java(w), native_to_java(z));
+}
+
 UPCALL_ID(PyNumber_Index);
 PyObject * PyNumber_Index(PyObject *o) {
     if (o == NULL) {

graalpython/com.oracle.graal.python.cext/src/capi.c

@@ -129,9 +129,13 @@ declare_type(PyFloat_Type, float, PyFloatObject);
 declare_type(PySlice_Type, slice, PySliceObject);
 declare_type(PyByteArray_Type, bytearray, PyByteArrayObject);
 declare_type(PyCFunction_Type, builtin_function_or_method, PyCFunctionObject);
+declare_type(PyWrapperDescr_Type, method_descriptor, PyWrapperDescrObject); // LS: previously wrapper_descriptor
+// tfel: Both method_descriptor maps to both PyWrapperDescr_Type and
+// PyMethodDescr_Type. This reflects our interpreter, but we need to make sure
+// that the dynamic type for method_descriptor is always going to be
+// PyMethodDescr_Type, so these two declarations cannot be in the wrong order
 declare_type(PyMethodDescr_Type, method_descriptor, PyMethodDescrObject);
 declare_type(PyGetSetDescr_Type, getset_descriptor, PyGetSetDescrObject);
-declare_type(PyWrapperDescr_Type, method_descriptor, PyWrapperDescrObject); // LS: previously wrapper_descriptor
 declare_type(PyMemberDescr_Type, property, PyMemberDescrObject); // LS: previously member_descriptor
 declare_type(_PyExc_BaseException, BaseException, PyBaseExceptionObject);
 declare_type(PyBuffer_Type, buffer, PyBufferDecorator);
@@ -298,6 +302,10 @@ const char* PyTruffle_StringToCstr(void* o, int32_t strLen) {
     return str;
 }
 
+const char* PyTruffle_CstrToString(const char* o) {
+    return polyglot_from_string(o, SRC_CS);
+}
+
 #define PRIMITIVE_ARRAY_TO_NATIVE(__jtype__, __ctype__, __polyglot_type__, __element_cast__) \
     void* PyTruffle_##__jtype__##ArrayToNative(const void* jarray, int64_t len) { \
         int64_t i; \

graalpython/com.oracle.graal.python.cext/src/dictobject.c

@@ -122,3 +122,47 @@ int PyDict_Update(PyObject *a, PyObject *b) {
         return 0;
     }
 }
+
+PyObject* _PyObject_GenericGetDict(PyObject* obj) {
+    PyObject** dictptr = _PyObject_GetDictPtr(obj);
+    if (dictptr == NULL) {
+        return NULL;
+    }
+    PyObject* dict = *dictptr;
+    if (dict == NULL) {
+        *dictptr = dict = PyDict_New();
+    }
+    return dict;
+}
+
+PyObject* PyObject_GenericGetDict(PyObject* obj, void* context) {
+    PyObject* d = _PyObject_GenericGetDict(obj);
+    if (d == NULL) {
+        PyErr_SetString(PyExc_AttributeError, "This object has no __dict__");
+    }
+    return d;
+}
+
+PyObject** _PyObject_GetDictPtr(PyObject* obj) {
+    Py_ssize_t dictoffset;
+    PyTypeObject *tp = Py_TYPE(obj);
+
+    dictoffset = tp->tp_dictoffset;
+    if (dictoffset == 0) {
+        return NULL;
+    }
+    if (dictoffset < 0) {
+        Py_ssize_t nitems = ((PyVarObject *)obj)->ob_size;
+        if (nitems < 0) {
+            nitems = -nitems;
+        }
+
+        size_t size = tp->tp_basicsize + nitems * tp->tp_itemsize;
+        if (size % SIZEOF_VOID_P != 0) {
+            // round to full pointer boundary
+            size += SIZEOF_VOID_P - (size % SIZEOF_VOID_P);
+        }
+        dictoffset += (long)size;
+    }
+    return (PyObject **) ((char *)obj + dictoffset);
+}

graalpython/com.oracle.graal.python.cext/src/typeobject.c

@@ -232,6 +232,8 @@ int PyType_Ready(PyTypeObject* cls) {
     PyDict_SetItemString(native_members, "tp_name", polyglot_from_string(cls->tp_name, SRC_CS));
     PyDict_SetItemString(native_members, "tp_doc", polyglot_from_string(cls->tp_doc ? cls->tp_doc : "", SRC_CS));
     PyDict_SetItemString(native_members, "tp_basicsize", PyLong_FromSsize_t(cls->tp_basicsize));
+    PyDict_SetItemString(native_members, "tp_itemsize", PyLong_FromSsize_t(cls->tp_itemsize));
+    PyDict_SetItemString(native_members, "tp_dictoffset", PyLong_FromSsize_t(cls->tp_dictoffset));
     const char* class_name = cls->tp_name;
     PyTypeObject* javacls = polyglot_invoke(PY_TRUFFLE_CEXT,
                                             "PyType_Ready",