feat: Add env var to control sending telemetry to appinsights in functions worker (#1621)

* str

* change

* otel

* lint

* test

* changes

* Update pyproject.toml

* fix test

Author: lzchen

azure_functions_worker/constants.py

@@ -82,14 +82,18 @@
 BASE_EXT_SUPPORTED_PY_MINOR_VERSION = 8
 
 # Appsetting to turn on OpenTelemetry support/features
-# Includes turning on Azure monitor distro to send telemetry to AppInsights
+# A value of "true" enables the setting
 PYTHON_ENABLE_OPENTELEMETRY = "PYTHON_ENABLE_OPENTELEMETRY"
-PYTHON_ENABLE_OPENTELEMETRY_DEFAULT = False
+
+# Appsetting to turn on ApplicationInsights support/features
+# A value of "true" enables the setting
+PYTHON_APPLICATIONINSIGHTS_ENABLE_TELEMETRY = \
+    "PYTHON_APPLICATIONINSIGHTS_ENABLE_TELEMETRY"
 
 # Appsetting to specify root logger name of logger to collect telemetry for
-# Used by Azure monitor distro
-PYTHON_AZURE_MONITOR_LOGGER_NAME = "PYTHON_AZURE_MONITOR_LOGGER_NAME"
-PYTHON_AZURE_MONITOR_LOGGER_NAME_DEFAULT = ""
+# Used by Azure monitor distro (Application Insights)
+PYTHON_APPLICATIONINSIGHTS_LOGGER_NAME = "PYTHON_APPLICATIONINSIGHTS_LOGGER_NAME"
+PYTHON_APPLICATIONINSIGHTS_LOGGER_NAME_DEFAULT = ""
 
 # Appsetting to specify AppInsights connection string
 APPLICATIONINSIGHTS_CONNECTION_STRING = "APPLICATIONINSIGHTS_CONNECTION_STRING"

azure_functions_worker/dispatcher.py

@@ -26,12 +26,12 @@
     APPLICATIONINSIGHTS_CONNECTION_STRING,
     HTTP_URI,
     METADATA_PROPERTIES_WORKER_INDEXED,
-    PYTHON_AZURE_MONITOR_LOGGER_NAME,
-    PYTHON_AZURE_MONITOR_LOGGER_NAME_DEFAULT,
+    PYTHON_APPLICATIONINSIGHTS_LOGGER_NAME,
+    PYTHON_APPLICATIONINSIGHTS_LOGGER_NAME_DEFAULT,
     PYTHON_ENABLE_DEBUG_LOGGING,
     PYTHON_ENABLE_INIT_INDEXING,
+    PYTHON_APPLICATIONINSIGHTS_ENABLE_TELEMETRY,
     PYTHON_ENABLE_OPENTELEMETRY,
-    PYTHON_ENABLE_OPENTELEMETRY_DEFAULT,
     PYTHON_LANGUAGE_RUNTIME,
     PYTHON_ROLLBACK_CWD_PATH,
     PYTHON_SCRIPT_FILE_NAME,
@@ -103,8 +103,10 @@ def __init__(self, loop: BaseEventLoop, host: str, port: int,
         self._function_metadata_result = None
         self._function_metadata_exception = None
 
-        # Used for checking if open telemetry is enabled
+        # Used for checking if appinsights is enabled
         self._azure_monitor_available = False
+        # Used for checking if open telemetry is enabled
+        self._otel_libs_available = False
         self._context_api = None
         self._trace_context_propagator = None
 
@@ -318,8 +320,8 @@ def initialize_azure_monitor(self):
                     setting=APPLICATIONINSIGHTS_CONNECTION_STRING
                 ),
                 logger_name=get_app_setting(
-                    setting=PYTHON_AZURE_MONITOR_LOGGER_NAME,
-                    default_value=PYTHON_AZURE_MONITOR_LOGGER_NAME_DEFAULT
+                    setting=PYTHON_APPLICATIONINSIGHTS_LOGGER_NAME,
+                    default_value=PYTHON_APPLICATIONINSIGHTS_LOGGER_NAME_DEFAULT
                 ),
             )
             self._azure_monitor_available = True
@@ -381,12 +383,15 @@ async def _handle__worker_init_request(self, request):
             constants.RPC_HTTP_TRIGGER_METADATA_REMOVED: _TRUE,
             constants.SHARED_MEMORY_DATA_TRANSFER: _TRUE,
         }
-        if get_app_setting(setting=PYTHON_ENABLE_OPENTELEMETRY,
-                           default_value=PYTHON_ENABLE_OPENTELEMETRY_DEFAULT):
+
+        if is_envvar_true(PYTHON_APPLICATIONINSIGHTS_ENABLE_TELEMETRY):
             self.initialize_azure_monitor()
 
-            if self._azure_monitor_available:
-                capabilities[constants.WORKER_OPEN_TELEMETRY_ENABLED] = _TRUE
+        if is_envvar_true(PYTHON_ENABLE_OPENTELEMETRY):
+            self._otel_libs_available = True
+
+        if self._azure_monitor_available or self._otel_libs_available:
+            capabilities[constants.WORKER_OPEN_TELEMETRY_ENABLED] = _TRUE
 
         if DependencyManager.should_load_cx_dependencies():
             DependencyManager.prioritize_customer_dependencies()
@@ -662,7 +667,7 @@ async def _handle__invocation_request(self, request):
                     args[name] = bindings.Out()
 
             if fi.is_async:
-                if self._azure_monitor_available:
+                if self._azure_monitor_available or self._otel_libs_available:
                     self.configure_opentelemetry(fi_context)
 
                 call_result = \
@@ -779,14 +784,14 @@ async def _handle__function_environment_reload_request(self, request):
             bindings.load_binding_registry()
 
             capabilities = {}
-            if get_app_setting(
-                    setting=PYTHON_ENABLE_OPENTELEMETRY,
-                    default_value=PYTHON_ENABLE_OPENTELEMETRY_DEFAULT):
+            if is_envvar_true(PYTHON_ENABLE_OPENTELEMETRY):
+                self._otel_libs_available = True
+            if is_envvar_true(PYTHON_APPLICATIONINSIGHTS_ENABLE_TELEMETRY):
                 self.initialize_azure_monitor()
 
-                if self._azure_monitor_available:
-                    capabilities[constants.WORKER_OPEN_TELEMETRY_ENABLED] = (
-                        _TRUE)
+            if self._azure_monitor_available or self._otel_libs_available:
+                capabilities[constants.WORKER_OPEN_TELEMETRY_ENABLED] = (
+                    _TRUE)
 
             if is_envvar_true(PYTHON_ENABLE_INIT_INDEXING):
                 try:
@@ -996,7 +1001,7 @@ def _run_sync_func(self, invocation_id, context, func, params):
         # invocation_id from ThreadPoolExecutor's threads.
         context.thread_local_storage.invocation_id = invocation_id
         try:
-            if self._azure_monitor_available:
+            if self._azure_monitor_available or self._otel_libs_available:
                 self.configure_opentelemetry(context)
             return ExtensionManager.get_sync_invocation_wrapper(context,
                                                                 func)(params)

azure_functions_worker/utils/app_setting_manager.py

@@ -7,6 +7,7 @@
     FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED,
     PYTHON_ENABLE_DEBUG_LOGGING,
     PYTHON_ENABLE_INIT_INDEXING,
+    PYTHON_APPLICATIONINSIGHTS_ENABLE_TELEMETRY,
     PYTHON_ENABLE_OPENTELEMETRY,
     PYTHON_ENABLE_WORKER_EXTENSIONS,
     PYTHON_ENABLE_WORKER_EXTENSIONS_DEFAULT,
@@ -29,7 +30,8 @@ def get_python_appsetting_state():
          FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED,
          PYTHON_SCRIPT_FILE_NAME,
          PYTHON_ENABLE_INIT_INDEXING,
-         PYTHON_ENABLE_OPENTELEMETRY]
+         PYTHON_ENABLE_OPENTELEMETRY,
+         PYTHON_APPLICATIONINSIGHTS_ENABLE_TELEMETRY]
 
     app_setting_states = "".join(
         f"{app_setting}: {current_vars[app_setting]} | "

pyproject.toml

@@ -45,6 +45,7 @@ Repository = "https://github.com/Azure/azure-functions-python-worker"
 dev = [
     "azure-eventhub", # Used for EventHub E2E tests
     "azure-functions-durable", # Used for Durable E2E tests
+    "azure-monitor-opentelemetry", # Used for Azure Monitor unit tests
     "flask",
     "fastapi~=0.103.2",
     "pydantic",
@@ -55,6 +56,7 @@ dev = [
     "requests==2.*",
     "coverage",
     "pytest-sugar",
+    "opentelemetry-api", # Used for OpenTelemetry unit tests
     "pytest-cov",
     "pytest-xdist",
     "pytest-randomly",

tests/unittests/test_opentelemetry.py

@@ -23,8 +23,9 @@ def test_update_opentelemetry_status_import_error(self):
         # Patch the built-in import mechanism
         with patch('builtins.__import__', side_effect=ImportError):
             self.dispatcher.update_opentelemetry_status()
-            # Verify that otel_libs_available is set to False due to ImportError
-            self.assertFalse(self.dispatcher._azure_monitor_available)
+            # Verify that context variables are None due to ImportError
+            self.assertIsNone(self.dispatcher._context_api)
+            self.assertIsNone(self.dispatcher._trace_context_propagator)
 
     @patch('builtins.__import__')
     def test_update_opentelemetry_status_success(
@@ -54,12 +55,12 @@ def test_initialize_azure_monitor_import_error(
         with patch('builtins.__import__', side_effect=ImportError):
             self.dispatcher.initialize_azure_monitor()
             mock_update_ot.assert_called_once()
-            # Verify that otel_libs_available is set to False due to ImportError
+            # Verify that azure_monitor_available is set to False due to ImportError
             self.assertFalse(self.dispatcher._azure_monitor_available)
 
-    @patch.dict(os.environ, {'PYTHON_ENABLE_OPENTELEMETRY': 'true'})
+    @patch.dict(os.environ, {'PYTHON_APPLICATIONINSIGHTS_ENABLE_TELEMETRY': 'true'})
     @patch('builtins.__import__')
-    def test_init_request_otel_capability_enabled_app_setting(
+    def test_init_request_initialize_azure_monitor_enabled_app_setting(
             self,
             mock_imports,
     ):
@@ -78,13 +79,15 @@ def test_init_request_otel_capability_enabled_app_setting(
         self.assertEqual(init_response.worker_init_response.result.status,
                          protos.StatusResult.Success)
 
+        # Verify azure_monitor_available is set to True
+        self.assertTrue(self.dispatcher._azure_monitor_available)
         # Verify that WorkerOpenTelemetryEnabled capability is set to _TRUE
         capabilities = init_response.worker_init_response.capabilities
         self.assertIn("WorkerOpenTelemetryEnabled", capabilities)
         self.assertEqual(capabilities["WorkerOpenTelemetryEnabled"], "true")
 
     @patch("azure_functions_worker.dispatcher.Dispatcher.initialize_azure_monitor")
-    def test_init_request_otel_capability_disabled_app_setting(
+    def test_init_request_initialize_azure_monitor_default_app_setting(
         self,
         mock_initialize_azmon,
     ):
@@ -103,8 +106,113 @@ def test_init_request_otel_capability_disabled_app_setting(
                          protos.StatusResult.Success)
 
         # Azure monitor initialized not called
+        # Since default behavior is not enabled
         mock_initialize_azmon.assert_not_called()
 
+        # Verify azure_monitor_available is set to False
+        self.assertFalse(self.dispatcher._azure_monitor_available)
+        # Verify that WorkerOpenTelemetryEnabled capability is not set
+        capabilities = init_response.worker_init_response.capabilities
+        self.assertNotIn("WorkerOpenTelemetryEnabled", capabilities)
+
+    @patch.dict(os.environ, {'PYTHON_APPLICATIONINSIGHTS_ENABLE_TELEMETRY': 'false'})
+    @patch("azure_functions_worker.dispatcher.Dispatcher.initialize_azure_monitor")
+    def test_init_request_initialize_azure_monitor_disabled_app_setting(
+        self,
+        mock_initialize_azmon,
+    ):
+
+        init_request = protos.StreamingMessage(
+            worker_init_request=protos.WorkerInitRequest(
+                host_version="2.3.4",
+                function_app_directory=str(FUNCTION_APP_DIRECTORY)
+            )
+        )
+
+        init_response = self.loop.run_until_complete(
+            self.dispatcher._handle__worker_init_request(init_request))
+
+        self.assertEqual(init_response.worker_init_response.result.status,
+                         protos.StatusResult.Success)
+
+        # Azure monitor initialized not called
+        mock_initialize_azmon.assert_not_called()
+
+        # Verify azure_monitor_available is set to False
+        self.assertFalse(self.dispatcher._azure_monitor_available)
+        # Verify that WorkerOpenTelemetryEnabled capability is not set
+        capabilities = init_response.worker_init_response.capabilities
+        self.assertNotIn("WorkerOpenTelemetryEnabled", capabilities)
+
+    @patch.dict(os.environ, {'PYTHON_ENABLE_OPENTELEMETRY': 'true'})
+    def test_init_request_enable_opentelemetry_enabled_app_setting(
+        self,
+    ):
+
+        init_request = protos.StreamingMessage(
+            worker_init_request=protos.WorkerInitRequest(
+                host_version="2.3.4",
+                function_app_directory=str(FUNCTION_APP_DIRECTORY)
+            )
+        )
+
+        init_response = self.loop.run_until_complete(
+            self.dispatcher._handle__worker_init_request(init_request))
+
+        self.assertEqual(init_response.worker_init_response.result.status,
+                         protos.StatusResult.Success)
+
+        # Verify otel_libs_available is set to True
+        self.assertTrue(self.dispatcher._otel_libs_available)
+        # Verify that WorkerOpenTelemetryEnabled capability is set to _TRUE
+        capabilities = init_response.worker_init_response.capabilities
+        self.assertIn("WorkerOpenTelemetryEnabled", capabilities)
+        self.assertEqual(capabilities["WorkerOpenTelemetryEnabled"], "true")
+
+    @patch.dict(os.environ, {'PYTHON_ENABLE_OPENTELEMETRY': 'false'})
+    def test_init_request_enable_opentelemetry_default_app_setting(
+        self,
+    ):
+
+        init_request = protos.StreamingMessage(
+            worker_init_request=protos.WorkerInitRequest(
+                host_version="2.3.4",
+                function_app_directory=str(FUNCTION_APP_DIRECTORY)
+            )
+        )
+
+        init_response = self.loop.run_until_complete(
+            self.dispatcher._handle__worker_init_request(init_request))
+
+        self.assertEqual(init_response.worker_init_response.result.status,
+                         protos.StatusResult.Success)
+
+        # Verify otel_libs_available is set to False by default
+        self.assertFalse(self.dispatcher._otel_libs_available)
+        # Verify that WorkerOpenTelemetryEnabled capability is not set
+        capabilities = init_response.worker_init_response.capabilities
+        self.assertNotIn("WorkerOpenTelemetryEnabled", capabilities)
+
+    @patch.dict(os.environ, {'PYTHON_APPLICATIONINSIGHTS_ENABLE_TELEMETRY': 'false'})
+    def test_init_request_enable_azure_monitor_disabled_app_setting(
+        self,
+    ):
+
+        init_request = protos.StreamingMessage(
+            worker_init_request=protos.WorkerInitRequest(
+                host_version="2.3.4",
+                function_app_directory=str(FUNCTION_APP_DIRECTORY)
+            )
+        )
+
+        init_response = self.loop.run_until_complete(
+            self.dispatcher._handle__worker_init_request(init_request))
+
+        self.assertEqual(init_response.worker_init_response.result.status,
+                         protos.StatusResult.Success)
+
+        # Verify otel_libs_available is set to False by default
+        self.assertFalse(self.dispatcher._otel_libs_available)
         # Verify that WorkerOpenTelemetryEnabled capability is not set
         capabilities = init_response.worker_init_response.capabilities
         self.assertNotIn("WorkerOpenTelemetryEnabled", capabilities)