Moved common function into utils

Readded removed dependency

Added dependency for docker test

Fixed linting errors

Placed yarl dependency and moved common function into utils

Author: Ryo Kather

dev-requirements.txt

@@ -13,3 +13,4 @@ readme-renderer~=24.0
 grpcio-tools==1.29.0
 mypy-protobuf>=1.23
 protobuf>=3.13.0
+yarl~=1.6

instrumentation/opentelemetry-instrumentation-aiohttp-client/src/opentelemetry/instrumentation/aiohttp_client/__init__.py

@@ -65,7 +65,6 @@ def strip_query_params(url: yarl.URL) -> str:
 import types
 import typing
 from typing import Collection
-import yarl
 
 import aiohttp
 import wrapt
@@ -77,6 +76,7 @@ def strip_query_params(url: yarl.URL) -> str:
 from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
 from opentelemetry.instrumentation.utils import (
     http_status_to_status_code,
+    remove_url_credentials,
     unwrap,
 )
 from opentelemetry.propagate import inject
@@ -172,10 +172,7 @@ async def on_request_start(
         )
 
         if trace_config_ctx.span.is_recording():
-            try:
-                url = yarl.URL(params.url).with_user(None)
-            except ValueError: # invalid url was passed
-                pass
+            url = remove_url_credentials(params.url)
 
             attributes = {
                 SpanAttributes.HTTP_METHOD: http_method,

instrumentation/opentelemetry-instrumentation-asgi/src/opentelemetry/instrumentation/asgi/__init__.py

@@ -20,15 +20,17 @@
 
 import typing
 import urllib
-import yarl
 from functools import wraps
 from typing import Tuple
 
 from asgiref.compatibility import guarantee_single_callable
 
 from opentelemetry import context, trace
 from opentelemetry.instrumentation.asgi.version import __version__  # noqa
-from opentelemetry.instrumentation.utils import http_status_to_status_code
+from opentelemetry.instrumentation.utils import (
+    http_status_to_status_code,
+    remove_url_credentials,
+)
 from opentelemetry.propagate import extract
 from opentelemetry.propagators.textmap import Getter
 from opentelemetry.semconv.trace import SpanAttributes
@@ -81,18 +83,13 @@ def collect_request_attributes(scope):
             query_string = query_string.decode("utf8")
         http_url = http_url + ("?" + urllib.parse.unquote(query_string))
 
-    try:
-        http_url = str(yarl.URL(http_url).with_user(None))
-    except ValueError: # invalid url was passed
-        pass
-
     result = {
         SpanAttributes.HTTP_SCHEME: scope.get("scheme"),
         SpanAttributes.HTTP_HOST: server_host,
         SpanAttributes.NET_HOST_PORT: port,
         SpanAttributes.HTTP_FLAVOR: scope.get("http_version"),
         SpanAttributes.HTTP_TARGET: scope.get("path"),
-        SpanAttributes.HTTP_URL: http_url,
+        SpanAttributes.HTTP_URL: str(remove_url_credentials(http_url)),
     }
     http_method = scope.get("method")
     if http_method:

instrumentation/opentelemetry-instrumentation-asgi/tests/test_asgi_middleware.py

@@ -429,7 +429,7 @@ def test_response_attributes(self):
     def test_response_attributes_invalid_status_code(self):
         otel_asgi.set_status_code(self.span, "Invalid Status Code")
         self.assertEqual(self.span.set_status.call_count, 1)
-    
+
     def test_credential_removal(self):
         self.scope["server"] = ("username:[email protected]", 80)
         self.scope["path"] = "/status/200"

instrumentation/opentelemetry-instrumentation-requests/src/opentelemetry/instrumentation/requests/__init__.py

@@ -36,7 +36,6 @@
 import functools
 import types
 from typing import Collection
-import yarl
 
 from requests.models import Response
 from requests.sessions import Session
@@ -46,7 +45,10 @@
 from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
 from opentelemetry.instrumentation.requests.package import _instruments
 from opentelemetry.instrumentation.requests.version import __version__
-from opentelemetry.instrumentation.utils import http_status_to_status_code
+from opentelemetry.instrumentation.utils import (
+    http_status_to_status_code,
+    remove_url_credentials,
+)
 from opentelemetry.propagate import inject
 from opentelemetry.semconv.trace import SpanAttributes
 from opentelemetry.trace import SpanKind, get_tracer
@@ -125,10 +127,7 @@ def _instrumented_requests_call(
         if not span_name or not isinstance(span_name, str):
             span_name = get_default_span_name(method)
 
-        try:
-            url = str(yarl.URL(url).with_user(None))
-        except ValueError: # invalid url was passed
-            pass
+        url = str(remove_url_credentials(url))
 
         labels = {}
         labels[SpanAttributes.HTTP_METHOD] = method

instrumentation/opentelemetry-instrumentation-tornado/src/opentelemetry/instrumentation/tornado/client.py

@@ -14,12 +14,13 @@
 
 import functools
 
-import yarl
-
 from tornado.httpclient import HTTPError, HTTPRequest
 
 from opentelemetry import trace
-from opentelemetry.instrumentation.utils import http_status_to_status_code
+from opentelemetry.instrumentation.utils import (
+    http_status_to_status_code,
+    remove_url_credentials,
+)
 from opentelemetry.propagate import inject
 from opentelemetry.semconv.trace import SpanAttributes
 from opentelemetry.trace.status import Status
@@ -62,13 +63,8 @@ def fetch_async(tracer, request_hook, response_hook, func, _, args, kwargs):
         request_hook(span, request)
 
     if span.is_recording():
-        try:
-            url = str(yarl.URL(request.url).with_user(None))
-        except ValueError: # invalid url was passed
-            pass
-
         attributes = {
-            SpanAttributes.HTTP_URL: url,
+            SpanAttributes.HTTP_URL: str(remove_url_credentials(request.url)),
             SpanAttributes.HTTP_METHOD: request.method,
         }
         for key, value in attributes.items():

instrumentation/opentelemetry-instrumentation-tornado/tests/test_instrumentation.py

@@ -456,7 +456,9 @@ def test_response_headers(self):
         set_global_response_propagator(orig)
 
     def test_credential_removal(self):
-        response = self.fetch("http://username:[email protected]/status/200")
+        response = self.fetch(
+            "http://username:[email protected]/status/200"
+        )
         self.assertEqual(response.code, 200)
 
         spans = self.sorted_spans(self.memory_exporter.get_finished_spans())

instrumentation/opentelemetry-instrumentation-urllib/src/opentelemetry/instrumentation/urllib/__init__.py

@@ -39,7 +39,6 @@
 import functools
 import types
 from typing import Collection
-import yarl
 from urllib.request import (  # pylint: disable=no-name-in-module,import-error
     OpenerDirector,
     Request,
@@ -49,7 +48,10 @@
 from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
 from opentelemetry.instrumentation.urllib.package import _instruments
 from opentelemetry.instrumentation.urllib.version import __version__
-from opentelemetry.instrumentation.utils import http_status_to_status_code
+from opentelemetry.instrumentation.utils import (
+    http_status_to_status_code,
+    remove_url_credentials,
+)
 from opentelemetry.propagate import inject
 from opentelemetry.semconv.trace import SpanAttributes
 from opentelemetry.trace import SpanKind, get_tracer
@@ -143,14 +145,11 @@ def _instrumented_open_call(
         if not span_name or not isinstance(span_name, str):
             span_name = get_default_span_name(method)
 
-        try:
-            url = str(yarl.URL(url).with_user(None))
-        except ValueError: # invalid url was passed
-            pass
+        url = str(remove_url_credentials(url))
 
         labels = {
             SpanAttributes.HTTP_METHOD: method,
-            SpanAttributes.HTTP_URL: url
+            SpanAttributes.HTTP_URL: url,
         }
 
         with tracer.start_as_current_span(

instrumentation/opentelemetry-instrumentation-urllib/tests/test_urllib_integration.py

@@ -327,6 +327,7 @@ def test_credential_removal(self):
         span = self.assert_span()
         self.assertEqual(span.attributes[SpanAttributes.HTTP_URL], self.URL)
 
+
 class TestRequestsIntegration(RequestsIntegrationTestBase, TestBase):
     @staticmethod
     def perform_request(url: str, opener: OpenerDirector = None):

instrumentation/opentelemetry-instrumentation-wsgi/src/opentelemetry/instrumentation/wsgi/__init__.py

@@ -57,10 +57,12 @@ def hello():
 import functools
 import typing
 import wsgiref.util as wsgiref_util
-import yarl
 
 from opentelemetry import context, trace
-from opentelemetry.instrumentation.utils import http_status_to_status_code
+from opentelemetry.instrumentation.utils import (
+    http_status_to_status_code,
+    remove_url_credentials,
+)
 from opentelemetry.instrumentation.wsgi.version import __version__
 from opentelemetry.propagate import extract
 from opentelemetry.propagators.textmap import Getter
@@ -129,12 +131,9 @@ def collect_request_attributes(environ):
     if target is not None:
         result[SpanAttributes.HTTP_TARGET] = target
     else:
-        try:
-            url = str(yarl.URL(wsgiref_util.request_uri(environ)).with_user(None))
-        except ValueError: # invalid url was passed
-            pass
-
-        result[SpanAttributes.HTTP_URL] = url
+        result[SpanAttributes.HTTP_URL] = str(
+            remove_url_credentials(wsgiref_util.request_uri(environ))
+        )
 
     remote_addr = environ.get("REMOTE_ADDR")
     if remote_addr:

opentelemetry-instrumentation/src/opentelemetry/instrumentation/utils.py

@@ -16,6 +16,8 @@
 
 from wrapt import ObjectProxy
 
+from yarl import URL
+
 from opentelemetry.trace import StatusCode
 
 
@@ -60,3 +62,12 @@ def unwrap(obj, attr: str):
     func = getattr(obj, attr, None)
     if func and isinstance(func, ObjectProxy) and hasattr(func, "__wrapped__"):
         setattr(obj, attr, func.__wrapped__)
+
+
+def remove_url_credentials(url: str) -> URL:
+    """Given a string url, attempt to remove the username and password"""
+    try:
+        url = URL(url).with_user(None)
+    except ValueError:  # invalid url was passed
+        pass
+    return url

tox.ini

@@ -233,8 +233,8 @@ commands_pre =
   grpc: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-grpc[test]
 
   falcon,flask,django,pyramid,tornado,starlette,fastapi: pip install {toxinidir}/util/opentelemetry-util-http[test]
-  wsgi,falcon,flask,django,pyramid: pip install yarl {toxinidir}/instrumentation/opentelemetry-instrumentation-wsgi[test]
-  asgi,starlette,fastapi: pip install yarl {toxinidir}/instrumentation/opentelemetry-instrumentation-asgi[test]
+  wsgi,falcon,flask,django,pyramid: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-wsgi[test]
+  asgi,starlette,fastapi: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-asgi[test]
 
   asyncpg: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-asyncpg[test]
 
@@ -245,7 +245,7 @@ commands_pre =
 
   flask: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-flask[test]
 
-  urllib: pip install yarl {toxinidir}/instrumentation/opentelemetry-instrumentation-urllib[test]
+  urllib: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-urllib[test]
 
   urllib3: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-urllib3[test]
 
@@ -273,11 +273,11 @@ commands_pre =
 
   redis: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-redis[test]
 
-  requests: pip install yarl {toxinidir}/instrumentation/opentelemetry-instrumentation-requests[test]
+  requests: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-requests[test]
 
   starlette: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-starlette[test]
 
-  tornado: pip install yarl {toxinidir}/instrumentation/opentelemetry-instrumentation-tornado[test]
+  tornado: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-tornado[test]
 
   jinja2: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-jinja2[test]
 
@@ -287,10 +287,6 @@ commands_pre =
 
   aiopg: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-dbapi pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-aiopg[test]
 
-  asgi: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-asgi[test]
-
-  wsgi: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-wsgi[test]
-
   datadog: pip install flaky {toxinidir}/exporter/opentelemetry-exporter-datadog[test]
 
   sklearn: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-sklearn[test]