Fix version of Flask dependency werkzeug (open-telemetry#1980)

Feature/add new process metrics (open-telemetry#1948)

Using new cloud resource id attribute (open-telemetry#1976)

Fixes open-telemetry#1970,remove pkg resources and use importlib_metadata

Author: ocelotl

CHANGELOG.md

@@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+
+### Added
+- `opentelemetry-instrumentation-system-metrics` Add support for collecting process metrics
+  ([#1948](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1948))
+
+### Fixed
+
+- Fix version of Flask dependency `werkzeug`
+  ([#1980](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1980))
+- `opentelemetry-instrumentation`, `opentelemetry-instrumentation-aiohttp-client` Use importlib-metadata for entry points instead of pkg_resources
+  ([#2124](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2124))
+
 ## Version 1.20.0/0.41b0 (2023-09-01)
 
 ### Fixed
@@ -352,7 +364,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `opentelemetry-instrumentation-sqlalchemy` Added span for the connection phase ([#1133](https://github.com/open-telemetry/opentelemetry-python-contrib/issues/1133))
 - Add metric instrumentation in asgi
   ([#1197](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1197))
-- Add metric instumentation for flask
+- Add metric instrumentation for flask
   ([#1186](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1186))
 - Add a test for asgi using NoOpTracerProvider
   ([#1367](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1367))

instrumentation/opentelemetry-instrumentation-aiohttp-client/tests/test_aiohttp_client_integration.py

@@ -22,9 +22,9 @@
 
 import aiohttp
 import aiohttp.test_utils
+import importlib_metadata
 import yarl
 from http_server_mock import HttpServerMock
-from pkg_resources import iter_entry_points
 
 from opentelemetry import context
 from opentelemetry import trace as trace_api
@@ -574,8 +574,8 @@ def response_hook(
 
 class TestLoadingAioHttpInstrumentor(unittest.TestCase):
     def test_loading_instrumentor(self):
-        entry_points = iter_entry_points(
-            "opentelemetry_instrumentor", "aiohttp-client"
+        entry_points = importlib_metadata.entry_points(
+            group="opentelemetry_instrumentor", name="aiohttp-client"
         )
 
         instrumentor = next(entry_points).load()()

instrumentation/opentelemetry-instrumentation-flask/pyproject.toml

@@ -36,6 +36,7 @@ dependencies = [
 [project.optional-dependencies]
 instruments = [
   "flask >= 1.0, < 3.0",
+  "werkzeug < 3.0.0"
 ]
 test = [
   "opentelemetry-instrumentation-flask[instruments]",

instrumentation/opentelemetry-instrumentation-sklearn/src/opentelemetry/instrumentation/sklearn/__init__.py

@@ -35,7 +35,7 @@
     ).instrument()
 
 
-Model intrumentation example:
+Model instrumentation example:
 
 .. code-block:: python
 
@@ -291,7 +291,7 @@ class descendent) is being instrumented with opentelemetry. Within a
         SklearnInstrumentor(packages=packages).instrument()
 
 
-    Model intrumentation example:
+    Model instrumentation example:
 
     .. code-block:: python
 

instrumentation/opentelemetry-instrumentation-system-metrics/src/opentelemetry/instrumentation/system_metrics/__init__.py

@@ -36,6 +36,10 @@
         "system.thread_count": None
         "process.runtime.memory": ["rss", "vms"],
         "process.runtime.cpu.time": ["user", "system"],
+        "process.runtime.gc_count": None,
+        "process.runtime.thread_count": None,
+        "process.runtime.cpu.utilization": None,
+        "process.runtime.context_switches": ["involuntary", "voluntary"],
     }
 
 Usage
@@ -63,6 +67,7 @@
         "system.network.io": ["transmit", "receive"],
         "process.runtime.memory": ["rss", "vms"],
         "process.runtime.cpu.time": ["user", "system"],
+        "process.runtime.context_switches": ["involuntary", "voluntary"],
     }
     SystemMetricsInstrumentor(config=configuration).instrument()
 
@@ -105,6 +110,9 @@
     "process.runtime.memory": ["rss", "vms"],
     "process.runtime.cpu.time": ["user", "system"],
     "process.runtime.gc_count": None,
+    "process.runtime.thread_count": None,
+    "process.runtime.cpu.utilization": None,
+    "process.runtime.context_switches": ["involuntary", "voluntary"],
 }
 
 
@@ -150,6 +158,9 @@ def __init__(
         self._runtime_memory_labels = self._labels.copy()
         self._runtime_cpu_time_labels = self._labels.copy()
         self._runtime_gc_count_labels = self._labels.copy()
+        self._runtime_thread_count_labels = self._labels.copy()
+        self._runtime_cpu_utilization_labels = self._labels.copy()
+        self._runtime_context_switches_labels = self._labels.copy()
 
     def instrumentation_dependencies(self) -> Collection[str]:
         return _instruments
@@ -347,6 +358,29 @@ def _instrument(self, **kwargs):
                 unit="bytes",
             )
 
+        if "process.runtime.thread_count" in self._config:
+            self._meter.create_observable_up_down_counter(
+                name=f"process.runtime.{self._python_implementation}.thread_count",
+                callbacks=[self._get_runtime_thread_count],
+                description="Runtime active threads count",
+            )
+
+        if "process.runtime.cpu.utilization" in self._config:
+            self._meter.create_observable_gauge(
+                name=f"process.runtime.{self._python_implementation}.cpu.utilization",
+                callbacks=[self._get_runtime_cpu_utilization],
+                description="Runtime CPU utilization",
+                unit="1",
+            )
+
+        if "process.runtime.context_switches" in self._config:
+            self._meter.create_observable_counter(
+                name=f"process.runtime.{self._python_implementation}.context_switches",
+                callbacks=[self._get_runtime_context_switches],
+                description="Runtime context switches",
+                unit="switches",
+            )
+
     def _uninstrument(self, **__):
         pass
 
@@ -646,3 +680,34 @@ def _get_runtime_gc_count(
         for index, count in enumerate(gc.get_count()):
             self._runtime_gc_count_labels["count"] = str(index)
             yield Observation(count, self._runtime_gc_count_labels.copy())
+
+    def _get_runtime_thread_count(
+        self, options: CallbackOptions
+    ) -> Iterable[Observation]:
+        """Observer callback for runtime active thread count"""
+        yield Observation(
+            self._proc.num_threads(), self._runtime_thread_count_labels.copy()
+        )
+
+    def _get_runtime_cpu_utilization(
+        self, options: CallbackOptions
+    ) -> Iterable[Observation]:
+        """Observer callback for runtime CPU utilization"""
+        proc_cpu_percent = self._proc.cpu_percent()
+        yield Observation(
+            proc_cpu_percent,
+            self._runtime_cpu_utilization_labels.copy(),
+        )
+
+    def _get_runtime_context_switches(
+        self, options: CallbackOptions
+    ) -> Iterable[Observation]:
+        """Observer callback for runtime context switches"""
+        ctx_switches = self._proc.num_ctx_switches()
+        for metric in self._config["process.runtime.context_switches"]:
+            if hasattr(ctx_switches, metric):
+                self._runtime_context_switches_labels["type"] = metric
+                yield Observation(
+                    getattr(ctx_switches, metric),
+                    self._runtime_context_switches_labels.copy(),
+                )

instrumentation/opentelemetry-instrumentation-system-metrics/tests/test_system_metrics.py

@@ -18,13 +18,14 @@
 from platform import python_implementation
 from unittest import mock
 
-from opentelemetry.instrumentation.system_metrics import (
-    SystemMetricsInstrumentor,
-)
 from opentelemetry.sdk.metrics import MeterProvider
 from opentelemetry.sdk.metrics.export import InMemoryMetricReader
 from opentelemetry.test.test_base import TestBase
 
+from opentelemetry.instrumentation.system_metrics import (
+    SystemMetricsInstrumentor,
+)
+
 
 def _mock_netconnection():
     NetConnection = namedtuple(
@@ -96,7 +97,7 @@ def test_system_metrics_instrument(self):
             for scope_metrics in resource_metrics.scope_metrics:
                 for metric in scope_metrics.metrics:
                     metric_names.append(metric.name)
-        self.assertEqual(len(metric_names), 18)
+        self.assertEqual(len(metric_names), 21)
 
         observer_names = [
             "system.cpu.time",
@@ -117,6 +118,9 @@ def test_system_metrics_instrument(self):
             f"process.runtime.{self.implementation}.memory",
             f"process.runtime.{self.implementation}.cpu_time",
             f"process.runtime.{self.implementation}.gc_count",
+            f"process.runtime.{self.implementation}.thread_count",
+            f"process.runtime.{self.implementation}.context_switches",
+            f"process.runtime.{self.implementation}.cpu.utilization",
         ]
 
         for observer in metric_names:
@@ -128,6 +132,9 @@ def test_runtime_metrics_instrument(self):
             "process.runtime.memory": ["rss", "vms"],
             "process.runtime.cpu.time": ["user", "system"],
             "process.runtime.gc_count": None,
+            "process.runtime.thread_count": None,
+            "process.runtime.cpu.utilization": None,
+            "process.runtime.context_switches": ["involuntary", "voluntary"],
         }
 
         reader = InMemoryMetricReader()
@@ -140,12 +147,15 @@ def test_runtime_metrics_instrument(self):
             for scope_metrics in resource_metrics.scope_metrics:
                 for metric in scope_metrics.metrics:
                     metric_names.append(metric.name)
-        self.assertEqual(len(metric_names), 3)
+        self.assertEqual(len(metric_names), 6)
 
         observer_names = [
             f"process.runtime.{self.implementation}.memory",
             f"process.runtime.{self.implementation}.cpu_time",
             f"process.runtime.{self.implementation}.gc_count",
+            f"process.runtime.{self.implementation}.thread_count",
+            f"process.runtime.{self.implementation}.context_switches",
+            f"process.runtime.{self.implementation}.cpu.utilization",
         ]
 
         for observer in metric_names:
@@ -161,9 +171,9 @@ def _assert_metrics(self, observer_name, reader, expected):
                     for data_point in metric.data.data_points:
                         for expect in expected:
                             if (
-                                dict(data_point.attributes)
-                                == expect.attributes
-                                and metric.name == observer_name
+                                    dict(data_point.attributes)
+                                    == expect.attributes
+                                    and metric.name == observer_name
                             ):
                                 self.assertEqual(
                                     data_point.value,
@@ -782,3 +792,37 @@ def test_runtime_get_count(self, mock_gc_get_count):
         self._test_metrics(
             f"process.runtime.{self.implementation}.gc_count", expected
         )
+
+    @mock.patch("psutil.Process.num_ctx_switches")
+    def test_runtime_context_switches(self, mock_process_num_ctx_switches):
+        PCtxSwitches = namedtuple("PCtxSwitches", ["voluntary", "involuntary"])
+
+        mock_process_num_ctx_switches.configure_mock(
+            **{"return_value": PCtxSwitches(voluntary=1, involuntary=2)}
+        )
+
+        expected = [
+            _SystemMetricsResult({"type": "voluntary"}, 1),
+            _SystemMetricsResult({"type": "involuntary"}, 2),
+        ]
+        self._test_metrics(
+            f"process.runtime.{self.implementation}.context_switches", expected
+        )
+
+    @mock.patch("psutil.Process.num_threads")
+    def test_runtime_thread_num(self, mock_process_thread_num):
+        mock_process_thread_num.configure_mock(**{"return_value": 42})
+
+        expected = [_SystemMetricsResult({}, 42)]
+        self._test_metrics(
+            f"process.runtime.{self.implementation}.thread_count", expected
+        )
+
+    @mock.patch("psutil.Process.cpu_percent")
+    def test_runtime_cpu_percent(self, mock_process_cpu_percent):
+        mock_process_cpu_percent.configure_mock(**{"return_value": 42})
+
+        expected = [_SystemMetricsResult({}, 42)]
+        self._test_metrics(
+            f"process.runtime.{self.implementation}.cpu.utilization", expected
+        )

instrumentation/opentelemetry-instrumentation-wsgi/tests/__init__.py

@@ -1,4 +1,4 @@
-# Copyright The OpenTelemetry Authors
+# Copyright The Open Telemetry Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import pkg_resources
 
 # IMPORTANT: Only the wsgi module needs this because it is always the first
 # package that uses the `{rootdir}/*/tests/` path and gets installed by
@@ -20,4 +19,5 @@
 # Naming the tests module as a namespace package ensures that
 # relative imports will resolve properly for subsequent test packages,
 # as it enables searching for a composite of multiple test modules.
-pkg_resources.declare_namespace(__name__)
+
+# No need for additional code here for Python 3.3+

opentelemetry-distro/tests/test_distro.py

@@ -15,8 +15,7 @@
 
 import os
 from unittest import TestCase
-
-from pkg_resources import DistributionNotFound, require
+import importlib_metadata
 
 from opentelemetry.distro import OpenTelemetryDistro
 from opentelemetry.environment_variables import (
@@ -25,12 +24,11 @@
 )
 from opentelemetry.sdk.environment_variables import OTEL_EXPORTER_OTLP_PROTOCOL
 
-
 class TestDistribution(TestCase):
     def test_package_available(self):
         try:
-            require(["opentelemetry-distro"])
-        except DistributionNotFound:
+            importlib_metadata.distribution("opentelemetry-distro")
+        except importlib_metadata.PackageNotFoundError:
             self.fail("opentelemetry-distro not installed")
 
     def test_default_configuration(self):

opentelemetry-instrumentation/src/opentelemetry/instrumentation/auto_instrumentation/__init__.py

@@ -21,7 +21,7 @@
 from re import sub
 from shutil import which
 
-from pkg_resources import iter_entry_points
+from importlib_metadata import entry_points
 
 from opentelemetry.instrumentation.version import __version__
 
@@ -50,8 +50,8 @@ def run() -> None:
 
     argument_otel_environment_variable = {}
 
-    for entry_point in iter_entry_points(
-        "opentelemetry_environment_variables"
+    for entry_point in entry_points(
+        group="opentelemetry_environment_variables"
     ):
         environment_variable_module = entry_point.load()