feat: enable top-level span events (#12645)

quinna-h · web-flow · commit a754504fcd37 · 2025-04-02T16:05:23.000Z
Enable encoding top-level span events according to [RFC](https://docs.google.com/document/d/1cVod_VI7Yruq8U9dfMRFJd7npDu-uBpste2IB04GyaQ/edit?tab=t.0#heading=h.z8rsw69ccced) for native span events implementation. If `DD_TRACE_NATIVE_SPAN_EVENTS` is True, the encoder version should be `v0.4` and we should encode span events as a top level field. - Change type of SpanEvent attributes from `_JSONType` to `AttributeValue` (matching OTel). microbenchmark results (normal span events vs native span events): ``` encoder-one-trace-span-event 2.38 ms: 1.03x slower encoder-one-trace-top-level-span-event 629 us: 3.67x faster ``` ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)
diff --git a/ddtrace/_trace/span.py b/ddtrace/_trace/span.py
@@ -22,6 +22,7 @@
 from ddtrace._trace._span_pointer import _SpanPointer
 from ddtrace._trace._span_pointer import _SpanPointerDirection
 from ddtrace._trace.context import Context
+from ddtrace._trace.types import _AttributeValueType
 from ddtrace._trace.types import _MetaDictType
 from ddtrace._trace.types import _MetricDictType
 from ddtrace._trace.types import _TagNameType
@@ -53,7 +54,6 @@
 from ddtrace.internal.logger import get_logger
 from ddtrace.internal.sampling import SamplingMechanism
 from ddtrace.internal.sampling import set_sampling_decision_maker
-from ddtrace.settings._config import _JSONType
 
 
 _NUMERIC_TAGS = (_ANALYTICS_SAMPLE_RATE_KEY,)
@@ -63,16 +63,16 @@ class SpanEvent:
     __slots__ = ["name", "attributes", "time_unix_nano"]
 
     def __init__(
-        self, name: str, attributes: Optional[Dict[str, _JSONType]] = None, time_unix_nano: Optional[int] = None
+        self,
+        name: str,
+        attributes: Optional[Dict[str, _AttributeValueType]] = None,
+        time_unix_nano: Optional[int] = None,
     ):
         self.name: str = name
-        if attributes is None:
-            self.attributes = {}
-        else:
-            self.attributes = attributes
         if time_unix_nano is None:
             time_unix_nano = time_ns()
         self.time_unix_nano: int = time_unix_nano
+        self.attributes: dict = attributes if attributes else {}
 
     def __dict__(self):
         d = {"name": self.name, "time_unix_nano": self.time_unix_nano}
@@ -89,6 +89,12 @@ def __str__(self):
         attrs_str = ",".join(f"{k}:{v}" for k, v in self.attributes.items())
         return f"name={self.name} time={self.time_unix_nano} attributes={attrs_str}"
 
+    def __iter__(self):
+        yield "name", self.name
+        yield "time_unix_nano", self.time_unix_nano
+        if self.attributes:
+            yield "attributes", self.attributes
+
 
 log = get_logger(__name__)
 
@@ -180,7 +186,6 @@ def __init__(
             if config._raise:
                 raise TypeError("parent_id must be an integer")
             return
-
         self.name = name
         self.service = service
         self._resource = [resource or name]
@@ -496,7 +501,7 @@ def get_metric(self, key: _TagNameType) -> Optional[NumericType]:
         return self._metrics.get(key)
 
     def _add_event(
-        self, name: str, attributes: Optional[Dict[str, _JSONType]] = None, timestamp: Optional[int] = None
+        self, name: str, attributes: Optional[Dict[str, _AttributeValueType]] = None, timestamp: Optional[int] = None
     ) -> None:
         self._events.append(SpanEvent(name, attributes, timestamp))
 
@@ -587,7 +592,6 @@ def set_exc_info(
             return
 
         self.error = 1
-
         tb = self._get_traceback(exc_type, exc_val, exc_tb, limit=limit)
 
         # readable version of type (e.g. exceptions.ZeroDivisionError)
@@ -614,7 +618,7 @@ def set_exc_info(
     def record_exception(
         self,
         exception: BaseException,
-        attributes: Optional[Dict[str, _JSONType]] = None,
+        attributes: Optional[Dict[str, _AttributeValueType]] = None,
         timestamp: Optional[int] = None,
         escaped=False,
     ) -> None:
diff --git a/ddtrace/_trace/types.py b/ddtrace/_trace/types.py
@@ -1,4 +1,5 @@
 from typing import Dict
+from typing import Sequence
 from typing import Text
 from typing import Union
 
@@ -8,3 +9,13 @@
 _TagNameType = Union[Text, bytes]
 _MetaDictType = Dict[_TagNameType, Text]
 _MetricDictType = Dict[_TagNameType, NumericType]
+_AttributeValueType = Union[
+    str,
+    bool,
+    int,
+    float,
+    Sequence[str],
+    Sequence[bool],
+    Sequence[int],
+    Sequence[float],
+]
diff --git a/ddtrace/internal/_encoding.pyx b/ddtrace/internal/_encoding.pyx
@@ -23,6 +23,7 @@ from ..constants import _ORIGIN_KEY as ORIGIN_KEY
 from .constants import SPAN_LINKS_KEY
 from .constants import SPAN_EVENTS_KEY
 from .constants import MAX_UINT_64BITS
+from ..settings._agent import config as agent_config
 
 
 DEF MSGPACK_ARRAY_LENGTH_PREFIX_SIZE = 5
@@ -49,6 +50,7 @@ cdef extern from "pack.h":
     int msgpack_pack_bin(msgpack_packer* pk, size_t l)
     int msgpack_pack_raw_body(msgpack_packer* pk, char* body, size_t l)
     int msgpack_pack_unicode(msgpack_packer* pk, object o, long long limit)
+    int msgpack_pack_uint8(msgpack_packer* pk, stdint.uint8_t d)
     int msgpack_pack_uint32(msgpack_packer* pk, stdint.uint32_t d)
     int msgpack_pack_uint64(msgpack_packer* pk, stdint.uint64_t d)
     int msgpack_pack_int32(msgpack_packer* pk, stdint.int32_t d)
@@ -101,6 +103,10 @@ cdef inline int pack_bytes(msgpack_packer *pk, char *bs, Py_ssize_t l):
         ret = msgpack_pack_raw_body(pk, bs, l)
     return ret
 
+cdef inline int pack_bool(msgpack_packer *pk, bint n) except? -1:
+    if n:
+        return msgpack_pack_true(pk)
+    return msgpack_pack_false(pk)
 
 cdef inline int pack_number(msgpack_packer *pk, object n) except? -1:
     if n is None:
@@ -548,6 +554,11 @@ cdef class MsgpackEncoderBase(BufferedEncoder):
 
 
 cdef class MsgpackEncoderV04(MsgpackEncoderBase):
+    cdef bint top_level_span_event_encoding
+
+    def __cinit__(self, size_t max_size, size_t max_item_size):
+        self.top_level_span_event_encoding = agent_config.trace_native_span_events
+
     cpdef flush(self):
         with self._lock:
             try:
@@ -611,6 +622,57 @@ cdef class MsgpackEncoderV04(MsgpackEncoderBase):
                     return ret
         return 0
 
+    cdef inline int _pack_span_events(self, list span_events) except? -1:
+        cdef int ret
+        cdef int L
+        cdef str attr_k
+        cdef object attr_v
+        cdef object event
+        ret = msgpack_pack_array(&self.pk, len(span_events))
+        if ret != 0:
+            return ret
+
+        for event in span_events:
+            L = 2 + bool(event.attributes)
+            ret = msgpack_pack_map(&self.pk, L)
+            if ret != 0:
+                return ret
+
+            ret = pack_bytes(&self.pk, <char*> b"name", 4)
+            if ret != 0:
+                return ret
+
+            ret = pack_text(&self.pk, event.name)
+            if ret != 0:
+                return ret
+
+            ret = pack_bytes(&self.pk, <char*> b"time_unix_nano", 14)
+            if ret != 0:
+                return ret
+
+            ret = pack_number(&self.pk, event.time_unix_nano)
+            if ret != 0:
+                return ret
+
+            if event.attributes:
+                ret = pack_bytes(&self.pk, <char*> b"attributes", 10)
+                if ret != 0:
+                    return ret
+
+                ret = msgpack_pack_map(&self.pk, len(event.attributes))
+                if ret != 0:
+                    return ret
+
+                for attr_k, attr_v in event.attributes.items():
+                    ret = pack_text(&self.pk, attr_k)
+                    if ret != 0:
+                        return ret
+
+                    ret = self.pack_span_event_attributes(attr_v)
+                    if ret != 0:
+                        return ret
+        return ret
+
     cdef inline int _pack_meta(self, object meta, char *dd_origin, str span_events) except? -1:
         cdef Py_ssize_t L
         cdef int ret
@@ -679,13 +741,20 @@ cdef class MsgpackEncoderV04(MsgpackEncoderBase):
         has_error = <bint> (span.error != 0)
         has_span_type = <bint> (span.span_type is not None)
         has_span_events = <bint> (len(span._events) > 0)
-        has_meta = <bint> (len(span._meta) > 0 or dd_origin is not NULL or has_span_events)
         has_metrics = <bint> (len(span._metrics) > 0)
         has_parent_id = <bint> (span.parent_id is not None)
         has_links = <bint> (len(span._links) > 0)
         has_meta_struct = <bint> (len(span._meta_struct) > 0)
+        has_meta = <bint> (
+            len(span._meta) > 0
+            or dd_origin is not NULL
+            or (not self.top_level_span_event_encoding and has_span_events)
+        )
 
+        # do not include in meta
         L = 7 + has_span_type + has_meta + has_metrics + has_error + has_parent_id + has_links + has_meta_struct
+        if self.top_level_span_event_encoding:
+            L += has_span_events
 
         ret = msgpack_pack_map(&self.pk, L)
 
@@ -771,13 +840,21 @@ cdef class MsgpackEncoderV04(MsgpackEncoderBase):
                 if ret != 0:
                     return ret
 
+            if has_span_events and self.top_level_span_event_encoding:
+                ret = pack_bytes(&self.pk, <char *> b"span_events", 11)
+                if ret != 0:
+                    return ret
+                ret = self._pack_span_events(span._events)
+                if ret != 0:
+                    return ret
+
             if has_meta:
                 ret = pack_bytes(&self.pk, <char *> b"meta", 4)
                 if ret != 0:
                     return ret
 
                 span_events = ""
-                if has_span_events:
+                if has_span_events and not self.top_level_span_event_encoding:
                     span_events = json_dumps([vars(event)()  for event in span._events])
                 ret = self._pack_meta(span._meta, <char *> dd_origin, span_events)
                 if ret != 0:
@@ -812,6 +889,84 @@ cdef class MsgpackEncoderV04(MsgpackEncoderBase):
 
         return ret
 
+    cdef int pack_span_event_attributes(self, object attr, int depth=0) except ? -1:
+        cdef int ret
+        cdef object elt
+
+        ret = msgpack_pack_map(&self.pk, 2)
+        if ret != 0:
+            return ret
+        ret = pack_bytes(&self.pk, <char*> b"type", 4)
+        if ret != 0:
+            return ret
+
+        if isinstance(attr, str):
+            ret = msgpack_pack_uint8(&self.pk, 0)
+            if ret != 0:
+                return ret
+            ret = pack_bytes(&self.pk, <char*> b"string_value", 12)
+            if ret != 0:
+                return ret
+            ret = pack_text(&self.pk, attr)
+            if ret != 0:
+                return ret
+        elif isinstance(attr, bool):
+            ret = msgpack_pack_uint8(&self.pk, 1)
+            if ret != 0:
+                return ret
+            ret = pack_bytes(&self.pk, <char*> b"bool_value", 10)
+            if ret != 0:
+                return ret
+            ret = pack_bool(&self.pk, attr)
+            if ret != 0:
+                return ret
+        elif isinstance(attr, int):
+            ret = msgpack_pack_uint8(&self.pk, 2)
+            if ret != 0:
+                return ret
+            ret = pack_bytes(&self.pk, <char*> b"int_value", 9)
+            if ret != 0:
+                return ret
+            ret = pack_number(&self.pk, attr)
+            if ret != 0:
+                return ret
+        elif isinstance(attr, float):
+            ret = msgpack_pack_uint8(&self.pk, 3)
+            if ret != 0:
+                return ret
+            ret = pack_bytes(&self.pk, <char*> b"double_value", 12)
+            if ret != 0:
+                return ret
+            ret = pack_number(&self.pk, attr)
+            if ret != 0:
+                return ret
+        elif isinstance(attr, list):
+            if depth != 0:
+                raise ValueError("Nested list found; cannot encode")
+            ret = msgpack_pack_uint8(&self.pk, 4)
+            if ret != 0:
+                return ret
+            ret = pack_bytes(&self.pk, <char*> b"array_value", 11)
+            if ret != 0:
+                return ret
+            ret = msgpack_pack_map(&self.pk, 1)
+            if ret != 0:
+                return ret
+            ret = pack_bytes(&self.pk, <char*> b"values", 6)
+            if ret != 0:
+                return ret
+            ret = msgpack_pack_array(&self.pk, len(attr))
+            if ret != 0:
+                return ret
+
+            for elt in attr:
+                ret = self.pack_span_event_attributes(elt, depth+1)
+                if ret != 0:
+                    return ret
+        else:
+            raise ValueError(f"Unsupported type for SpanEvent attribute: {type(attr)}")
+
+        return ret
 
 cdef class MsgpackEncoderV05(MsgpackEncoderBase):
     cdef MsgpackStringTable _st
diff --git a/ddtrace/internal/encoding.py b/ddtrace/internal/encoding.py
@@ -6,6 +6,7 @@
 from typing import Optional  # noqa:F401
 from typing import Tuple  # noqa:F401
 
+from ..settings._agent import config as agent_config  # noqa:F401
 from ._encoding import ListStringTable
 from ._encoding import MsgpackEncoderV04
 from ._encoding import MsgpackEncoderV05
@@ -86,6 +87,9 @@ def _span_to_dict(span):
         if span._links:
             d["span_links"] = [link.to_dict() for link in span._links]
 
+        if span._events and agent_config.trace_native_span_events:
+            d["span_events"] = [dict(event) for event in span._events]
+
         return d
 
 
diff --git a/ddtrace/internal/writer/writer.py b/ddtrace/internal/writer/writer.py
@@ -468,6 +468,11 @@ def __init__(
             default_api_version = "v0.4"
 
         self._api_version = api_version or config._trace_api or default_api_version
+
+        if agent_config.trace_native_span_events:
+            log.warning("Setting api version to v0.4; DD_TRACE_NATIVE_SPAN_EVENTS is not compatible with v0.5")
+            self._api_version = "v0.4"
+
         if is_windows and self._api_version == "v0.5":
             raise RuntimeError(
                 "There is a known compatibility issue with v0.5 API and Windows, "
diff --git a/ddtrace/settings/_agent.py b/ddtrace/settings/_agent.py
@@ -141,6 +141,14 @@ class AgentConfig(DDConfig):
         help_type="Int",
         help="Stores the port of the agent",
     )
+
+    trace_native_span_events = DDConfig.v(
+        bool,
+        "trace_native_span_events",
+        default=False,
+        help_type="Boolean",
+        help="Stores whether native span events are enabled",
+    )
     # Effective trace agent URL (this is the one that will be used)
     trace_agent_url = DDConfig.d(str, _derive_trace_url)
     # Effective DogStatsD URL (this is the one that will be used)
diff --git a/releasenotes/notes/add-native-span-events-b04983442822d1b0.yaml b/releasenotes/notes/add-native-span-events-b04983442822d1b0.yaml
@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Adds configuration for encoding span events as a top-level field in v0.4 payloads and introduces environment variable ``DD_TRACE_NATIVE_SPAN_EVENTS`` (disabled by default).
+    This requires agent version 7.63.0 or later.
diff --git a/tests/telemetry/test_writer.py b/tests/telemetry/test_writer.py
@@ -483,6 +483,7 @@ def test_app_started_event_configuration_override(test_agent_session, run_python
         {"name": "DD_TRACE_LOG_FILE_SIZE_BYTES", "origin": "default", "value": 15728640},
         {"name": "DD_TRACE_LOG_STREAM_HANDLER", "origin": "default", "value": True},
         {"name": "DD_TRACE_METHODS", "origin": "default", "value": None},
+        {"name": "DD_TRACE_NATIVE_SPAN_EVENTS", "origin": "default", "value": False},
         {"name": "DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP", "origin": "env_var", "value": ".*"},
         {"name": "DD_TRACE_OTEL_ENABLED", "origin": "env_var", "value": True},
         {"name": "DD_TRACE_PARTIAL_FLUSH_ENABLED", "origin": "env_var", "value": False},
diff --git a/tests/tracer/test_encoders.py b/tests/tracer/test_encoders.py
