Preserve ServiceName (segment name) when tracing external http services with dynamic urls

Author: shcherbin

aws_xray_sdk/ext/aiohttp/client.py

@@ -8,7 +8,7 @@
 from aws_xray_sdk.core import xray_recorder
 from aws_xray_sdk.core.models import http
 from aws_xray_sdk.core.utils import stacktrace
-from aws_xray_sdk.ext.util import inject_trace_header, strip_url
+from aws_xray_sdk.ext.util import inject_trace_header, parse_hostname
 
 # All aiohttp calls will entail outgoing HTTP requests, only in some ad-hoc
 # exceptions the namespace will be flip back to local.
@@ -22,7 +22,7 @@
 
 
 async def begin_subsegment(session, trace_config_ctx, params):
-    name = trace_config_ctx.name if trace_config_ctx.name else strip_url(str(params.url))
+    name = trace_config_ctx.name if trace_config_ctx.name else parse_hostname(str(params.url))
     subsegment = xray_recorder.begin_subsegment(name, REMOTE_NAMESPACE)
 
     # No-op if subsegment is `None` due to `LOG_ERROR`.

aws_xray_sdk/ext/httplib/patch.py

@@ -8,7 +8,7 @@
 from aws_xray_sdk.core.models import http
 from aws_xray_sdk.core.exceptions.exceptions import SegmentNotFoundException
 from aws_xray_sdk.core.patcher import _PATCHED_MODULES
-from aws_xray_sdk.ext.util import inject_trace_header, strip_url, unwrap
+from aws_xray_sdk.ext.util import inject_trace_header, parse_hostname, unwrap
 
 if sys.version_info >= (3, 0, 0):
     PY2 = False
@@ -57,7 +57,7 @@ def _xray_traced_http_getresponse(wrapped, instance, args, kwargs):
 
     return xray_recorder.record_subsegment(
         wrapped, instance, args, kwargs,
-        name=strip_url(xray_data.url),
+        name=parse_hostname(xray_data.url),
         namespace='remote',
         meta_processor=http_response_processor,
     )
@@ -111,7 +111,7 @@ def decompose_args(method, url, body, headers, encode_chunked=False):
         # we add a segment here in case connect fails
         return xray_recorder.record_subsegment(
             wrapped, instance, args, kwargs,
-            name=strip_url(xray_data.url),
+            name=parse_hostname(xray_data.url),
             namespace='remote',
             meta_processor=http_send_request_processor
         )
@@ -141,7 +141,7 @@ def _xray_traced_http_client_read(wrapped, instance, args, kwargs):
 
     return xray_recorder.record_subsegment(
         wrapped, instance, args, kwargs,
-        name=strip_url(xray_data.url),
+        name=parse_hostname(xray_data.url),
         namespace='remote',
         meta_processor=http_read_processor
     )

aws_xray_sdk/ext/requests/patch.py

@@ -2,7 +2,7 @@
 
 from aws_xray_sdk.core import xray_recorder
 from aws_xray_sdk.core.models import http
-from aws_xray_sdk.ext.util import inject_trace_header, strip_url
+from aws_xray_sdk.ext.util import inject_trace_header, parse_hostname
 
 
 def patch():
@@ -26,7 +26,7 @@ def _xray_traced_requests(wrapped, instance, args, kwargs):
 
     return xray_recorder.record_subsegment(
         wrapped, instance, args, kwargs,
-        name=strip_url(url),
+        name=parse_hostname(url),
         namespace='remote',
         meta_processor=requests_processor,
     )

aws_xray_sdk/ext/util.py

@@ -1,10 +1,10 @@
 import re
 
-from aws_xray_sdk.core.models.trace_header import TraceHeader
-from aws_xray_sdk.core.models import http
-
 import wrapt
+from six.moves.urllib.parse import urlparse
 
+from aws_xray_sdk.core.models import http
+from aws_xray_sdk.core.models.trace_header import TraceHeader
 
 first_cap_re = re.compile('(.)([A-Z][a-z]+)')
 all_cap_re = re.compile('([a-z0-9])([A-Z])')
@@ -118,6 +118,16 @@ def strip_url(url):
     return url.partition('?')[0] if url else url
 
 
+def parse_hostname(url):
+    """
+    Parses the hostname from the given url.
+    The hostname is used as a segment name for external http requests.
+    :param url:
+    :return: hostname string
+    """
+    return urlparse(url).hostname
+
+
 def unwrap(obj, attr):
     """
     Will unwrap a `wrapt` attribute

setup.py

@@ -46,6 +46,7 @@
         'enum34;python_version<"3.4"',
         'wrapt',
         'future',
+        'six',
         'botocore>=1.11.3',
     ],
 

tests/ext/aiohttp/test_client.py

@@ -66,7 +66,7 @@ async def test_error(loop, recorder):
             pass
 
     subsegment = xray_recorder.current_segment().subsegments[0]
-    assert subsegment.name == url
+    assert subsegment.name == BASE_URL
     assert subsegment.error
 
     http_meta = subsegment.http
@@ -85,7 +85,7 @@ async def test_throttle(loop, recorder):
             pass
 
     subsegment = xray_recorder.current_segment().subsegments[0]
-    assert subsegment.name == url
+    assert subsegment.name == BASE_URL
     assert subsegment.error
     assert subsegment.throttle
 
@@ -105,7 +105,7 @@ async def test_fault(loop, recorder):
             pass
 
     subsegment = xray_recorder.current_segment().subsegments[0]
-    assert subsegment.name == url
+    assert subsegment.name == BASE_URL
     assert subsegment.fault
 
     http_meta = subsegment.http

tests/ext/httplib/test_httplib.py

@@ -4,7 +4,6 @@
 from aws_xray_sdk.core import patch
 from aws_xray_sdk.core import xray_recorder
 from aws_xray_sdk.core.context import Context
-from aws_xray_sdk.ext.util import strip_url
 
 if sys.version_info >= (3, 0, 0):
     import http.client as httplib
@@ -57,7 +56,7 @@ def test_ok():
     url = 'https://{}/status/{}?foo=bar&baz=foo'.format(BASE_URL, status_code)
     _do_req(url)
     subsegment = xray_recorder.current_segment().subsegments[1]
-    assert subsegment.name == strip_url(url)
+    assert subsegment.name == BASE_URL
 
     http_meta = subsegment.http
     assert http_meta['request']['url'] == url
@@ -70,7 +69,7 @@ def test_error():
     url = 'https://{}/status/{}'.format(BASE_URL, status_code)
     _do_req(url, 'POST')
     subsegment = xray_recorder.current_segment().subsegments[1]
-    assert subsegment.name == url
+    assert subsegment.name == BASE_URL
     assert subsegment.error
 
     http_meta = subsegment.http
@@ -84,7 +83,7 @@ def test_throttle():
     url = 'https://{}/status/{}'.format(BASE_URL, status_code)
     _do_req(url, 'HEAD')
     subsegment = xray_recorder.current_segment().subsegments[1]
-    assert subsegment.name == url
+    assert subsegment.name == BASE_URL
     assert subsegment.error
     assert subsegment.throttle
 
@@ -99,7 +98,7 @@ def test_fault():
     url = 'https://{}/status/{}'.format(BASE_URL, status_code)
     _do_req(url, 'PUT')
     subsegment = xray_recorder.current_segment().subsegments[1]
-    assert subsegment.name == url
+    assert subsegment.name == BASE_URL
     assert subsegment.fault
 
     http_meta = subsegment.http
@@ -126,7 +125,7 @@ def test_correct_identify_http():
     url = 'http://{}/status/{}?foo=bar&baz=foo'.format(BASE_URL, status_code)
     _do_req(url, use_https=False)
     subsegment = xray_recorder.current_segment().subsegments[0]
-    assert subsegment.name == strip_url(url)
+    assert subsegment.name == BASE_URL
 
     http_meta = subsegment.http
     assert http_meta['request']['url'].split(":")[0] == 'http'
@@ -137,7 +136,7 @@ def test_correct_identify_https():
     url = 'https://{}/status/{}?foo=bar&baz=foo'.format(BASE_URL, status_code)
     _do_req(url, use_https=True)
     subsegment = xray_recorder.current_segment().subsegments[0]
-    assert subsegment.name == strip_url(url)
+    assert subsegment.name == BASE_URL
 
     https_meta = subsegment.http
     assert https_meta['request']['url'].split(":")[0] == 'https'

tests/ext/requests/test_requests.py

@@ -4,7 +4,6 @@
 from aws_xray_sdk.core import patch
 from aws_xray_sdk.core import xray_recorder
 from aws_xray_sdk.core.context import Context
-from aws_xray_sdk.ext.util import strip_url
 
 
 patch(('requests',))
@@ -32,7 +31,7 @@ def test_ok():
     url = 'http://{}/status/{}?foo=bar'.format(BASE_URL, status_code)
     requests.get(url)
     subsegment = xray_recorder.current_segment().subsegments[0]
-    assert subsegment.name == strip_url(url)
+    assert subsegment.name == BASE_URL
 
     http_meta = subsegment.http
     assert http_meta['request']['url'] == url
@@ -45,7 +44,7 @@ def test_error():
     url = 'http://{}/status/{}'.format(BASE_URL, status_code)
     requests.post(url)
     subsegment = xray_recorder.current_segment().subsegments[0]
-    assert subsegment.name == url
+    assert subsegment.name == BASE_URL
     assert subsegment.error
 
     http_meta = subsegment.http
@@ -59,7 +58,7 @@ def test_throttle():
     url = 'http://{}/status/{}'.format(BASE_URL, status_code)
     requests.head(url)
     subsegment = xray_recorder.current_segment().subsegments[0]
-    assert subsegment.name == url
+    assert subsegment.name == BASE_URL
     assert subsegment.error
     assert subsegment.throttle
 
@@ -74,7 +73,7 @@ def test_fault():
     url = 'http://{}/status/{}'.format(BASE_URL, status_code)
     requests.put(url)
     subsegment = xray_recorder.current_segment().subsegments[0]
-    assert subsegment.name == url
+    assert subsegment.name == BASE_URL
     assert subsegment.fault
 
     http_meta = subsegment.http