Skip to content

feat: [SVLS-5677] DynamoDB Stream event span pointers #522

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Oct 30, 2024
Merged

feat: [SVLS-5677] DynamoDB Stream event span pointers #522

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
7a646b0
chore: [SVLS-5677] update dependencies a bit, will revert and apply c…
apiarian-datadog Oct 16, 2024
91c7aba
chore: [SVLS-5677] handy to have rust in our test container when buil…
apiarian-datadog Oct 16, 2024
96dfea9
chore: [SVLS-5677] fix moved private method import line
apiarian-datadog Oct 16, 2024
b62a8cc
feat: [SVLS-5677] DynamoDB Stream event span pointers
apiarian-datadog Oct 16, 2024
2061b9a
chore: [SVLS-5677] upgrade ddtrace to 2.15.0
apiarian-datadog Oct 30, 2024
540f173
chore: [SVLS-5677] update integration test snapshots to include span …
apiarian-datadog Oct 30, 2024
79047da
chore: remove unnecessary string wrappers around exception logging
apiarian-datadog Oct 30, 2024
e5451e5
chore: flip conditionals
apiarian-datadog Oct 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 68 additions & 1 deletion datadog_lambda/span_pointers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,10 @@
import logging
from typing import List

from ddtrace._trace.utils_botocore.span_pointers import (
from ddtrace._trace.utils_botocore.span_pointers.dynamodb import (
_aws_dynamodb_item_span_pointer_description,
)
from ddtrace._trace.utils_botocore.span_pointers.s3 import (
_aws_s3_object_span_pointer_description,
)
from ddtrace._trace._span_pointer import _SpanPointerDirection
Expand All @@ -21,6 +24,9 @@ def calculate_span_pointers(
if event_source.equals(EventTypes.S3):
return _calculate_s3_span_pointers_for_event(event)

elif event_source.equals(EventTypes.DYNAMODB):
return _calculate_dynamodb_span_pointers_for_event(event)

except Exception as e:
logger.warning(
"failed to calculate span pointers for event: %s",
Expand Down Expand Up @@ -89,3 +95,64 @@ def _calculate_s3_span_pointers_for_object_created_s3_information(
str(e),
)
return []


def _calculate_dynamodb_span_pointers_for_event(event) -> List[_SpanPointerDescription]:
# Example event:
# https://docs.aws.amazon.com/lambda/latest/dg/with-ddb.html

return list(
chain.from_iterable(
_calculate_dynamodb_span_pointers_for_event_record(record)
for record in event.get("Records", [])
)
)


def _calculate_dynamodb_span_pointers_for_event_record(
record,
) -> List[_SpanPointerDescription]:
try:
table_name = _extract_table_name_from_dynamodb_stream_record(record)
primary_key = record["dynamodb"]["Keys"]

except Exception as e:
logger.warning(
"missing DynamoDB information required to make a span pointer: %s",
str(e),
)
return []

try:
return [
_aws_dynamodb_item_span_pointer_description(
pointer_direction=_SpanPointerDirection.UPSTREAM,
table_name=table_name,
primary_key=primary_key,
)
]

except Exception as e:
logger.warning(
"failed to generate DynamoDB span pointer: %s",
str(e),
)
return []


def _extract_table_name_from_dynamodb_stream_record(record) -> str:
# Example eventSourceARN:
# arn:aws:dynamodb:us-east-2:123456789012:table/my-table/stream/2024-06-10T19:26:16.525
event_source_arn = record["eventSourceARN"]

[_arn, _aws, _dynamodb, _region, _account, dynamodb_info] = event_source_arn.split(
":", maxsplit=5
)
if _arn != "arn" or _aws != "aws" or _dynamodb != "dynamodb":
raise ValueError(f"unexpected eventSourceARN format: {event_source_arn}")

[_table, table_name, _stream, _timestamp] = dynamodb_info.split("/")
if not _table == "table" or not _stream == "stream":
raise ValueError(f"unexpected eventSourceARN format: {event_source_arn}")

return table_name
387 changes: 201 additions & 186 deletions poetry.lock
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ classifiers = [
python = ">=3.8.0,<4"
datadog = ">=0.41.0,<1.0.0"
wrapt = "^1.11.2"
ddtrace = ">=2.14.1"
ddtrace = ">=2.15.0"
ujson = ">=5.9.0"
boto3 = { version = "^1.34.0", optional = true }
requests = { version ="^2.22.0", optional = true }
Expand Down
5 changes: 5 additions & 0 deletions tests/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,11 @@ FROM python:$python_version

ENV PYTHONDONTWRITEBYTECODE True

# Add Rust compiler which is needed to build dd-trace-py from source
RUN curl https://sh.rustup.rs -sSf | \
sh -s -- --default-toolchain stable -y
ENV PATH=/root/.cargo/bin:$PATH

RUN mkdir -p /test/datadog_lambda
WORKDIR /test

Expand Down
48 changes: 46 additions & 2 deletions tests/integration/snapshots/logs/async-metrics_python310.log
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -279,7 +279,39 @@ HTTP GET https://www.datadoghq.com/ Headers: ["Accept-Encoding:gzip, deflate","A
"metrics": {
"_dd.top_level": 1
},
"type": "serverless"
"type": "serverless",
"span_links": [
{
"trace_id": "XXXX",
"span_id": "XXXX",
"attributes": {
"ptr.kind": "aws.dynamodb.item",
"ptr.dir": "u",
"ptr.hash": "e2af34d333891f765c7f02d2da80895e",
"link.kind": "span-pointer"
}
},
{
"trace_id": "XXXX",
"span_id": "XXXX",
"attributes": {
"ptr.kind": "aws.dynamodb.item",
"ptr.dir": "u",
"ptr.hash": "e2af34d333891f765c7f02d2da80895e",
"link.kind": "span-pointer"
}
},
{
"trace_id": "XXXX",
"span_id": "XXXX",
"attributes": {
"ptr.kind": "aws.dynamodb.item",
"ptr.dir": "u",
"ptr.hash": "e2af34d333891f765c7f02d2da80895e",
"link.kind": "span-pointer"
}
}
]
},
{
"trace_id": "XXXX",
Expand Down Expand Up @@ -942,7 +974,19 @@ HTTP GET https://www.datadoghq.com/ Headers: ["Accept-Encoding:gzip, deflate","A
"metrics": {
"_dd.top_level": 1
},
"type": "serverless"
"type": "serverless",
"span_links": [
{
"trace_id": "XXXX",
"span_id": "XXXX",
"attributes": {
"ptr.kind": "aws.s3.object",
"ptr.dir": "u",
"ptr.hash": "1dc3e5d00dae48c1f07d95371a747788",
"link.kind": "span-pointer"
}
}
]
},
{
"trace_id": "XXXX",
Expand Down
48 changes: 46 additions & 2 deletions tests/integration/snapshots/logs/async-metrics_python311.log
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -279,7 +279,39 @@ HTTP GET https://www.datadoghq.com/ Headers: ["Accept-Encoding:gzip, deflate","A
"metrics": {
"_dd.top_level": 1
},
"type": "serverless"
"type": "serverless",
"span_links": [
{
"trace_id": "XXXX",
"span_id": "XXXX",
"attributes": {
"ptr.kind": "aws.dynamodb.item",
"ptr.dir": "u",
"ptr.hash": "e2af34d333891f765c7f02d2da80895e",
"link.kind": "span-pointer"
}
},
{
"trace_id": "XXXX",
"span_id": "XXXX",
"attributes": {
"ptr.kind": "aws.dynamodb.item",
"ptr.dir": "u",
"ptr.hash": "e2af34d333891f765c7f02d2da80895e",
"link.kind": "span-pointer"
}
},
{
"trace_id": "XXXX",
"span_id": "XXXX",
"attributes": {
"ptr.kind": "aws.dynamodb.item",
"ptr.dir": "u",
"ptr.hash": "e2af34d333891f765c7f02d2da80895e",
"link.kind": "span-pointer"
}
}
]
},
{
"trace_id": "XXXX",
Expand Down Expand Up @@ -942,7 +974,19 @@ HTTP GET https://www.datadoghq.com/ Headers: ["Accept-Encoding:gzip, deflate","A
"metrics": {
"_dd.top_level": 1
},
"type": "serverless"
"type": "serverless",
"span_links": [
{
"trace_id": "XXXX",
"span_id": "XXXX",
"attributes": {
"ptr.kind": "aws.s3.object",
"ptr.dir": "u",
"ptr.hash": "1dc3e5d00dae48c1f07d95371a747788",
"link.kind": "span-pointer"
}
}
]
},
{
"trace_id": "XXXX",
Expand Down
48 changes: 46 additions & 2 deletions tests/integration/snapshots/logs/async-metrics_python312.log
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -279,7 +279,39 @@ HTTP GET https://www.datadoghq.com/ Headers: ["Accept-Encoding:gzip, deflate","A
"metrics": {
"_dd.top_level": 1
},
"type": "serverless"
"type": "serverless",
"span_links": [
{
"trace_id": "XXXX",
"span_id": "XXXX",
"attributes": {
"ptr.kind": "aws.dynamodb.item",
"ptr.dir": "u",
"ptr.hash": "e2af34d333891f765c7f02d2da80895e",
"link.kind": "span-pointer"
}
},
{
"trace_id": "XXXX",
"span_id": "XXXX",
"attributes": {
"ptr.kind": "aws.dynamodb.item",
"ptr.dir": "u",
"ptr.hash": "e2af34d333891f765c7f02d2da80895e",
"link.kind": "span-pointer"
}
},
{
"trace_id": "XXXX",
"span_id": "XXXX",
"attributes": {
"ptr.kind": "aws.dynamodb.item",
"ptr.dir": "u",
"ptr.hash": "e2af34d333891f765c7f02d2da80895e",
"link.kind": "span-pointer"
}
}
]
},
{
"trace_id": "XXXX",
Expand Down Expand Up @@ -942,7 +974,19 @@ HTTP GET https://www.datadoghq.com/ Headers: ["Accept-Encoding:gzip, deflate","A
"metrics": {
"_dd.top_level": 1
},
"type": "serverless"
"type": "serverless",
"span_links": [
{
"trace_id": "XXXX",
"span_id": "XXXX",
"attributes": {
"ptr.kind": "aws.s3.object",
"ptr.dir": "u",
"ptr.hash": "1dc3e5d00dae48c1f07d95371a747788",
"link.kind": "span-pointer"
}
}
]
},
{
"trace_id": "XXXX",
Expand Down
48 changes: 46 additions & 2 deletions tests/integration/snapshots/logs/async-metrics_python38.log
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -279,7 +279,39 @@ HTTP GET https://www.datadoghq.com/ Headers: ["Accept-Encoding:gzip, deflate","A
"metrics": {
"_dd.top_level": 1
},
"type": "serverless"
"type": "serverless",
"span_links": [
{
"trace_id": "XXXX",
"span_id": "XXXX",
"attributes": {
"ptr.kind": "aws.dynamodb.item",
"ptr.dir": "u",
"ptr.hash": "e2af34d333891f765c7f02d2da80895e",
"link.kind": "span-pointer"
}
},
{
"trace_id": "XXXX",
"span_id": "XXXX",
"attributes": {
"ptr.kind": "aws.dynamodb.item",
"ptr.dir": "u",
"ptr.hash": "e2af34d333891f765c7f02d2da80895e",
"link.kind": "span-pointer"
}
},
{
"trace_id": "XXXX",
"span_id": "XXXX",
"attributes": {
"ptr.kind": "aws.dynamodb.item",
"ptr.dir": "u",
"ptr.hash": "e2af34d333891f765c7f02d2da80895e",
"link.kind": "span-pointer"
}
}
]
},
{
"trace_id": "XXXX",
Expand Down Expand Up @@ -942,7 +974,19 @@ HTTP GET https://www.datadoghq.com/ Headers: ["Accept-Encoding:gzip, deflate","A
"metrics": {
"_dd.top_level": 1
},
"type": "serverless"
"type": "serverless",
"span_links": [
{
"trace_id": "XXXX",
"span_id": "XXXX",
"attributes": {
"ptr.kind": "aws.s3.object",
"ptr.dir": "u",
"ptr.hash": "1dc3e5d00dae48c1f07d95371a747788",
"link.kind": "span-pointer"
}
}
]
},
{
"trace_id": "XXXX",
Expand Down
Loading
Loading