From b7673d5c159cfd6821b33e44e70e0a2cad61153e Mon Sep 17 00:00:00 2001 From: wangbill <12449837+YunchuWang@users.noreply.github.com> Date: Tue, 24 Sep 2024 10:32:31 -0700 Subject: [PATCH 1/2] feat: support for event loop debugging and slow tasks detection --- azure_functions_worker/constants.py | 5 +++++ azure_functions_worker/dispatcher.py | 20 +++++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/azure_functions_worker/constants.py b/azure_functions_worker/constants.py index b916252c..41634119 100644 --- a/azure_functions_worker/constants.py +++ b/azure_functions_worker/constants.py @@ -93,3 +93,8 @@ # Appsetting to specify AppInsights connection string APPLICATIONINSIGHTS_CONNECTION_STRING = "APPLICATIONINSIGHTS_CONNECTION_STRING" + +# New constant for event loop profiling +PYTHON_ENABLE_EVENT_LOOP_DEBUGGING = 'PYTHON_ENABLE_EVENT_LOOP_DEBUGGING' +PYTHON_DEBUGGING_EVENT_LOOP_SLOW_CALLBACK_THRESHOLD = 'PYTHON_DEBUGGING_EVENT_LOOP_SLOW_CALLBAK_THRESHOLD' +PYTHON_DEBUGGING_EVENT_LOOP_SLOW_CALLBACK_THRESHOLD_DEFAULT = 600 # 10 minutes diff --git a/azure_functions_worker/dispatcher.py b/azure_functions_worker/dispatcher.py index 897a3499..7966fe98 100644 --- a/azure_functions_worker/dispatcher.py +++ b/azure_functions_worker/dispatcher.py @@ -28,7 +28,10 @@ METADATA_PROPERTIES_WORKER_INDEXED, PYTHON_AZURE_MONITOR_LOGGER_NAME, PYTHON_AZURE_MONITOR_LOGGER_NAME_DEFAULT, + PYTHON_DEBUGGING_EVENT_LOOP_SLOW_CALLBACK_THRESHOLD, + PYTHON_DEBUGGING_EVENT_LOOP_SLOW_CALLBACK_THRESHOLD_DEFAULT, PYTHON_ENABLE_DEBUG_LOGGING, + PYTHON_ENABLE_EVENT_LOOP_DEBUGGING, PYTHON_ENABLE_INIT_INDEXING, PYTHON_ENABLE_OPENTELEMETRY, PYTHON_ENABLE_OPENTELEMETRY_DEFAULT, @@ -353,6 +356,15 @@ def update_opentelemetry_status(self): "Cannot import OpenTelemetry libraries." ) + + def _setup_event_loop_profiling(self): + if is_envvar_true(PYTHON_ENABLE_EVENT_LOOP_DEBUGGING): + self._loop.set_debug(True) + self._loop.slow_callback_duration = get_app_setting(PYTHON_DEBUGGING_EVENT_LOOP_SLOW_CALLBACK_THRESHOLD, + PYTHON_DEBUGGING_EVENT_LOOP_SLOW_CALLBACK_THRESHOLD_DEFAULT) # 10 minutes in seconds + logger.info("Event loop debug mode enabled with 10-minute slow callback detection") + + async def _handle__worker_init_request(self, request): logger.info('Received WorkerInitRequest, ' 'python version %s, ' @@ -381,6 +393,10 @@ async def _handle__worker_init_request(self, request): constants.RPC_HTTP_TRIGGER_METADATA_REMOVED: _TRUE, constants.SHARED_MEMORY_DATA_TRANSFER: _TRUE, } + + # Set up event loop profiling + self._setup_event_loop_profiling() + if get_app_setting(setting=PYTHON_ENABLE_OPENTELEMETRY, default_value=PYTHON_ENABLE_OPENTELEMETRY_DEFAULT): self.initialize_azure_monitor() @@ -760,7 +776,9 @@ async def _handle__function_environment_reload_request(self, request): env_vars = func_env_reload_request.environment_variables for var in env_vars: os.environ[var] = env_vars[var] - + # Enable event loop debug mode + self._setup_event_loop_profiling() + # Apply PYTHON_THREADPOOL_THREAD_COUNT self._stop_sync_call_tp() self._sync_call_tp = ( From 9748a90953101114ae28a17b1eb63c0f8003da20 Mon Sep 17 00:00:00 2001 From: wangbill <12449837+YunchuWang@users.noreply.github.com> Date: Tue, 24 Sep 2024 10:58:18 -0700 Subject: [PATCH 2/2] fix: linting --- azure_functions_worker/constants.py | 5 +++-- azure_functions_worker/dispatcher.py | 13 +++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/azure_functions_worker/constants.py b/azure_functions_worker/constants.py index 41634119..62655808 100644 --- a/azure_functions_worker/constants.py +++ b/azure_functions_worker/constants.py @@ -96,5 +96,6 @@ # New constant for event loop profiling PYTHON_ENABLE_EVENT_LOOP_DEBUGGING = 'PYTHON_ENABLE_EVENT_LOOP_DEBUGGING' -PYTHON_DEBUGGING_EVENT_LOOP_SLOW_CALLBACK_THRESHOLD = 'PYTHON_DEBUGGING_EVENT_LOOP_SLOW_CALLBAK_THRESHOLD' -PYTHON_DEBUGGING_EVENT_LOOP_SLOW_CALLBACK_THRESHOLD_DEFAULT = 600 # 10 minutes +PYTHON_DEBUGGING_EVENT_LOOP_SLOW_CALLBACK_THRESHOLD = \ + 'PYTHON_DEBUGGING_EVENT_LOOP_SLOW_CALLBAK_THRESHOLD' +PYTHON_DEBUGGING_EVENT_LOOP_SLOW_CALLBACK_THRESHOLD_DEFAULT = 600 # 10 minutes diff --git a/azure_functions_worker/dispatcher.py b/azure_functions_worker/dispatcher.py index 7966fe98..21f4e03d 100644 --- a/azure_functions_worker/dispatcher.py +++ b/azure_functions_worker/dispatcher.py @@ -356,14 +356,15 @@ def update_opentelemetry_status(self): "Cannot import OpenTelemetry libraries." ) - def _setup_event_loop_profiling(self): if is_envvar_true(PYTHON_ENABLE_EVENT_LOOP_DEBUGGING): self._loop.set_debug(True) - self._loop.slow_callback_duration = get_app_setting(PYTHON_DEBUGGING_EVENT_LOOP_SLOW_CALLBACK_THRESHOLD, - PYTHON_DEBUGGING_EVENT_LOOP_SLOW_CALLBACK_THRESHOLD_DEFAULT) # 10 minutes in seconds - logger.info("Event loop debug mode enabled with 10-minute slow callback detection") - + self._loop.slow_callback_duration = \ + get_app_setting( + PYTHON_DEBUGGING_EVENT_LOOP_SLOW_CALLBACK_THRESHOLD, + PYTHON_DEBUGGING_EVENT_LOOP_SLOW_CALLBACK_THRESHOLD_DEFAULT) + logger.info('Event loop debug mode enabled with 10-minute slow ' + 'callback detection') async def _handle__worker_init_request(self, request): logger.info('Received WorkerInitRequest, ' @@ -778,7 +779,7 @@ async def _handle__function_environment_reload_request(self, request): os.environ[var] = env_vars[var] # Enable event loop debug mode self._setup_event_loop_profiling() - + # Apply PYTHON_THREADPOOL_THREAD_COUNT self._stop_sync_call_tp() self._sync_call_tp = (