[GR-58473][GR-58526][GR-58707] Async GC with RSS guidance

PullRequest: graalpython/3499

Author: qunaibit

graalpython/com.oracle.graal.python.cext/include/Python.h

@@ -1,4 +1,4 @@
-/* Copyright (c) 2022, 2023, Oracle and/or its affiliates.
+/* Copyright (c) 2022, 2024, Oracle and/or its affiliates.
  * Copyright (C) 1996-2022 Python Software Foundation
  *
  * Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
@@ -64,6 +64,7 @@ struct timeval;
 #include "boolobject.h"
 #include "floatobject.h"
 #include "complexobject.h"
+#include "rangeobject.h"
 #include "memoryobject.h"
 #include "tupleobject.h"
 #include "listobject.h"

graalpython/com.oracle.graal.python.cext/include/rangeobject.h

@@ -0,0 +1,32 @@
+/* Copyright (c) 2018, 2024, Oracle and/or its affiliates.
+ * Copyright (C) 1996-2020 Python Software Foundation
+ *
+ * Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+ */
+
+/* Range object interface */
+
+#ifndef Py_RANGEOBJECT_H
+#define Py_RANGEOBJECT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+A range object represents an integer range.  This is an immutable object;
+a range cannot change its value after creation.
+
+Range objects behave like the corresponding tuple objects except that
+they are represented by a start, stop, and step datamembers.
+*/
+
+PyAPI_DATA(PyTypeObject) PyRange_Type;
+PyAPI_DATA(PyTypeObject) PyRangeIter_Type;
+PyAPI_DATA(PyTypeObject) PyLongRangeIter_Type;
+
+#define PyRange_Check(op) Py_IS_TYPE(op, &PyRange_Type)
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_RANGEOBJECT_H */

graalpython/com.oracle.graal.python.cext/src/capi.c

@@ -566,6 +566,44 @@ PyAPI_FUNC(void) PyTruffle_ObjectArrayRelease(PyObject** array, int32_t size) {
     }
 }
 
+#if defined(__APPLE__) && defined(__MACH__)
+#include <mach/mach.h>
+#elif defined(_WIN32)
+#include <windows.h>
+#include "psapi.h"
+#endif
+
+PyAPI_FUNC(size_t) PyTruffle_GetCurrentRSS() {
+    size_t rss = 0;
+#if defined(__APPLE__) && defined(__MACH__)
+    // MacOS
+    struct mach_task_basic_info info;
+    mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT;
+    if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t) &info, &infoCount) == KERN_SUCCESS) {
+        rss = (size_t)info.resident_size;
+    }
+
+#elif defined(__linux__) || defined(__gnu_linux__)
+    // Linux
+    FILE* fp = NULL;
+    if ((fp = fopen( "/proc/self/statm", "r" )) != NULL) {
+        if (fscanf(fp, "%*s%ld", (long) &rss)) {
+            rss *= (uint64_t) sysconf( _SC_PAGESIZE);
+        }
+        fclose(fp);
+    }
+
+#elif defined(_WIN32)
+    // Windows
+    PROCESS_MEMORY_COUNTERS pmc;
+    if (GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc))) {
+        rss = pmc.WorkingSetSize;
+    }
+#endif
+    return rss / (1024 * 1024 /* bytes -> megabytes*/);
+}
+
+
 #define ReadMember(object, offset, T) ((T*)(((char*)object) + offset))[0]
 
 PyAPI_FUNC(int) ReadShortMember(void* object, Py_ssize_t offset) {

graalpython/com.oracle.graal.python.cext/src/tupleobject.c

@@ -1418,5 +1418,9 @@ PyTruffleTuple_GetItems(PyObject *op)
  */
 PyObject*
 _PyTuple_GET_ITEM(PyObject* a, Py_ssize_t b) {
-    return PyTruffleTuple_GetItems(a)[b];
+    PyObject **ob_item = PyTruffleTuple_GetItems(a);
+    if (ob_item) {
+        return ob_item[b];
+    }
+    return NULL; // an exception has happend during transtion
 }

graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_gc.py

@@ -896,3 +896,7 @@ def test_module_globals(self):
         ####################################### GC #######################################
         self._trigger_gc()
         ##################################################################################
+
+@skipIf(not (GRAALPY) or RUNS_ON_LLVM, "Internal GraalPy RSS function")
+def test_current_rss_monitor():
+    assert __graalpython__.get_current_rss() > 0

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GcModuleBuiltins.java

@@ -57,7 +57,7 @@
 import com.oracle.graal.python.nodes.call.special.CallBinaryMethodNode;
 import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode;
 import com.oracle.graal.python.nodes.function.PythonBuiltinNode;
-import com.oracle.graal.python.nodes.function.builtins.PythonClinicBuiltinNode;
+import com.oracle.graal.python.nodes.function.builtins.PythonBinaryClinicBuiltinNode;
 import com.oracle.graal.python.nodes.function.builtins.PythonUnaryClinicBuiltinNode;
 import com.oracle.graal.python.nodes.function.builtins.clinic.ArgumentClinicProvider;
 import com.oracle.graal.python.runtime.GilNode;
@@ -112,7 +112,7 @@ public void initialize(Python3Core core) {
     @Builtin(name = "collect", parameterNames = {"$self", "generation"}, declaresExplicitSelf = true)
     @ArgumentClinic(name = "generation", conversion = ClinicConversion.Int, defaultValue = "2")
     @GenerateNodeFactory
-    abstract static class GcCollectNode extends PythonClinicBuiltinNode {
+    abstract static class GcCollectNode extends PythonBinaryClinicBuiltinNode {
         private static final NativeCAPISymbol SYMBOL = NativeCAPISymbol.FUN_GRAALPY_GC_COLLECT;
         private static final CApiTiming C_API_TIMING = CApiTiming.create(true, SYMBOL.getName());
 

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java

@@ -78,8 +78,6 @@
 import java.util.List;
 import java.util.logging.Level;
 
-import com.oracle.graal.python.nodes.util.ToNativePrimitiveStorageNode;
-import com.oracle.graal.python.runtime.sequence.storage.NativePrimitiveSequenceStorage;
 import org.graalvm.home.Version;
 import org.graalvm.nativeimage.ImageInfo;
 
@@ -148,6 +146,7 @@
 import com.oracle.graal.python.nodes.statement.AbstractImportNode;
 import com.oracle.graal.python.nodes.truffle.PythonArithmeticTypes;
 import com.oracle.graal.python.nodes.util.CastToTruffleStringNode;
+import com.oracle.graal.python.nodes.util.ToNativePrimitiveStorageNode;
 import com.oracle.graal.python.runtime.PosixSupportLibrary;
 import com.oracle.graal.python.runtime.PythonContext;
 import com.oracle.graal.python.runtime.PythonImageBuildOptions;
@@ -156,6 +155,7 @@
 import com.oracle.graal.python.runtime.exception.PythonExitException;
 import com.oracle.graal.python.runtime.object.PythonObjectFactory;
 import com.oracle.graal.python.runtime.sequence.PSequence;
+import com.oracle.graal.python.runtime.sequence.storage.NativePrimitiveSequenceStorage;
 import com.oracle.graal.python.runtime.sequence.storage.NativeSequenceStorage;
 import com.oracle.graal.python.runtime.sequence.storage.SequenceStorage;
 import com.oracle.graal.python.util.PythonUtils;
@@ -1131,4 +1131,15 @@ Object doClear() {
             return PNone.NONE;
         }
     }
+
+    @Builtin(name = "get_current_rss", maxNumOfPositionalArgs = 0)
+    @GenerateNodeFactory
+    public abstract static class GetCurrentRSS extends PythonBuiltinNode {
+
+        @Specialization
+        @TruffleBoundary
+        Object currentRSS() {
+            return getContext().getCApiContext().getCurrentRSS();
+        }
+    }
 }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextWeakrefBuiltins.java

@@ -79,8 +79,14 @@ static Object refType(Object object, Object callback,
     @CApiBuiltin(ret = PyObjectBorrowed, args = {PyObject}, call = Direct)
     abstract static class PyWeakref_GetObject extends CApiUnaryBuiltinNode {
         @Specialization
-        static Object call(PReferenceType self) {
-            return self.getPyObject();
+        static Object call(Object reference) {
+            if (reference instanceof PReferenceType ref) {
+                return ref.getPyObject();
+            }
+            /*
+             * This weak reference has died in the managed side due to its referent being collected.
+             */
+            return PNone.NONE;
         }
     }
 }