From ef81b2f52baac1ec1032f798b5a618efd7447214 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Thu, 27 Apr 2023 00:41:00 +0100 Subject: [PATCH 1/5] feat(jmespath): adding new envelopes + docs + small fix --- .../utilities/jmespath_utils/envelopes.py | 5 ++++ docs/utilities/jmespath_functions.md | 28 ++++++++++++------- docs/utilities/validation.md | 20 ++++++------- .../src/extract_data_from_builtin_envelope.py | 10 +++++-- 4 files changed, 40 insertions(+), 23 deletions(-) diff --git a/aws_lambda_powertools/utilities/jmespath_utils/envelopes.py b/aws_lambda_powertools/utilities/jmespath_utils/envelopes.py index df50e5f98d4..cabc6c265db 100644 --- a/aws_lambda_powertools/utilities/jmespath_utils/envelopes.py +++ b/aws_lambda_powertools/utilities/jmespath_utils/envelopes.py @@ -6,3 +6,8 @@ CLOUDWATCH_EVENTS_SCHEDULED = EVENTBRIDGE KINESIS_DATA_STREAM = "Records[*].kinesis.powertools_json(powertools_base64(data))" CLOUDWATCH_LOGS = "awslogs.powertools_base64_gzip(data) | powertools_json(@).logEvents[*]" +S3_SNS_SQS = "Records[*].powertools_json(body).powertools_json(Message).Records[0]" +S3_RAWSNS_SQS = "Records[*].powertools_json(body).Records[0]" +S3_SNS_KINESIS_FIREHOSE = "records[*].powertools_json(powertools_base64(data)).powertools_json(Message).Records[0]" +S3_RAWSNS_KINESIS_FIREHOSE = "records[*].powertools_json(powertools_base64(data)).Records[0]" +S3_EVENTBRIDGE_SQS = "Records[*].powertools_json(body).detail" diff --git a/docs/utilities/jmespath_functions.md b/docs/utilities/jmespath_functions.md index a01a72ced16..e2253901a65 100644 --- a/docs/utilities/jmespath_functions.md +++ b/docs/utilities/jmespath_functions.md @@ -64,16 +64,24 @@ We provide built-in envelopes for popular AWS Lambda event sources to easily dec These are all built-in envelopes you can use along with their expression as a reference: -| Envelope | JMESPath expression | -| --------------------------------- | ------------------------------------------------------------- | -| **`API_GATEWAY_REST`** | `powertools_json(body)` | -| **`API_GATEWAY_HTTP`** | `API_GATEWAY_REST` | -| **`SQS`** | `Records[*].powertools_json(body)` | -| **`SNS`** | `Records[0].Sns.Message | powertools_json(@)` | -| **`EVENTBRIDGE`** | `detail` | -| **`CLOUDWATCH_EVENTS_SCHEDULED`** | `EVENTBRIDGE` | -| **`KINESIS_DATA_STREAM`** | `Records[*].kinesis.powertools_json(powertools_base64(data))` | -| **`CLOUDWATCH_LOGS`** | `awslogs.powertools_base64_gzip(data) | powertools_json(@).logEvents[*]` | +| Envelope | JMESPath expression | +| --------------------------------- | ------------------------------------------------------------------------------------------ | +| **`API_GATEWAY_HTTP`** | `powertools_json(body)` | +| **`API_GATEWAY_REST`** | `powertools_json(body)` | +| **`CLOUDWATCH_EVENTS_SCHEDULED`** | `detail` | +| **`CLOUDWATCH_LOGS`** | `awslogs.powertools_base64_gzip(data) | powertools_json(@).logEvents[*]` | +| **`EVENTBRIDGE`** | `detail` | +| **`KINESIS_DATA_STREAM`** | `Records[*].kinesis.powertools_json(powertools_base64(data))` | +| **`S3_EVENTBRIDGE_SQS`** | `Records[*].powertools_json(body).detail` | +| **`S3_RAWSNS_KINESIS_FIREHOSE`** | `records[*].powertools_json(powertools_base64(data)).Records[0]` | +| **`S3_RAWSNS_SQS`** | `Records[*].powertools_json(body).Records[0]` | +| **`S3_SNS_KINESIS_FIREHOSE`** | `records[*].powertools_json(powertools_base64(data)).powertools_json(Message).Records[0]` | +| **`S3_SNS_SQS`** | `Records[*].powertools_json(body).powertools_json(Message).Records[0]` | +| **`SNS`** | `Records[0].Sns.Message | powertools_json(@)` | +| **`SQS`** | `Records[*].powertools_json(body)` | + +???+ tip + If you're using architectures that involve sending event notifications from S3 to SNS and then to SQS or Kinesis Data Firehose, consider enabling [Raw Delivery option](https://docs.aws.amazon.com/sns/latest/dg/sns-large-payload-raw-message-delivery.html){target="_blank"} in Amazon SNS. This can help reduce the payload by stripping the SNS metadata. ## Advanced diff --git a/docs/utilities/validation.md b/docs/utilities/validation.md index ca5907a5fe1..277a1f91f81 100644 --- a/docs/utilities/validation.md +++ b/docs/utilities/validation.md @@ -141,16 +141,16 @@ We provide built-in envelopes to easily extract the payload from popular event s Here is a handy table with built-in envelopes along with their JMESPath expressions in case you want to build your own. -| Envelope name | JMESPath expression | -| ------------------------------- | ------------------------------------------------------------- | -| **API_GATEWAY_REST** | "powertools_json(body)" | -| **API_GATEWAY_HTTP** | "powertools_json(body)" | -| **SQS** | "Records[*].powertools_json(body)" | -| **SNS** | "Records[0].Sns.Message | powertools_json(@)" | -| **EVENTBRIDGE** | "detail" | -| **CLOUDWATCH_EVENTS_SCHEDULED** | "detail" | -| **KINESIS_DATA_STREAM** | "Records[*].kinesis.powertools_json(powertools_base64(data))" | -| **CLOUDWATCH_LOGS** | "awslogs.powertools_base64_gzip(data) | powertools_json(@).logEvents[*]" | +| Envelope | JMESPath expression | +| --------------------------------- | ------------------------------------------------------------------------ | +| **`API_GATEWAY_HTTP`** | `powertools_json(body)` | +| **`API_GATEWAY_REST`** | `powertools_json(body)` | +| **`CLOUDWATCH_EVENTS_SCHEDULED`** | `detail` | +| **`CLOUDWATCH_LOGS`** | `awslogs.powertools_base64_gzip(data) | powertools_json(@).logEvents[*]` | +| **`EVENTBRIDGE`** | `detail` | +| **`KINESIS_DATA_STREAM`** | `Records[*].kinesis.powertools_json(powertools_base64(data))` | +| **`SNS`** | `Records[0].Sns.Message | powertools_json(@)` | +| **`SQS`** | `Records[*].powertools_json(body)` | ## Advanced diff --git a/examples/jmespath_functions/src/extract_data_from_builtin_envelope.py b/examples/jmespath_functions/src/extract_data_from_builtin_envelope.py index 31ae6cf268c..8d65355d97a 100644 --- a/examples/jmespath_functions/src/extract_data_from_builtin_envelope.py +++ b/examples/jmespath_functions/src/extract_data_from_builtin_envelope.py @@ -1,12 +1,16 @@ +from aws_lambda_powertools import Logger from aws_lambda_powertools.utilities.jmespath_utils import ( envelopes, extract_data_from_envelope, ) from aws_lambda_powertools.utilities.typing import LambdaContext +logger = Logger() + def handler(event: dict, context: LambdaContext) -> dict: - payload = extract_data_from_envelope(data=event, envelope=envelopes.SQS) - customer_id = payload.get("customerId") # now deserialized + records = extract_data_from_envelope(data=event, envelope=envelopes.SQS) + for record in records: # records is a list + logger.info(record.get("customerId")) # now deserialized - return {"customer_id": customer_id, "message": "success", "statusCode": 200} + return {"message": "success", "statusCode": 200} From 0de837079cfe1d830ebce3db3b51804b65097b13 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Thu, 27 Apr 2023 01:00:15 +0100 Subject: [PATCH 2/5] feat(jmespath): text --- docs/utilities/jmespath_functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/utilities/jmespath_functions.md b/docs/utilities/jmespath_functions.md index e2253901a65..fbd7b5be6b4 100644 --- a/docs/utilities/jmespath_functions.md +++ b/docs/utilities/jmespath_functions.md @@ -81,7 +81,7 @@ These are all built-in envelopes you can use along with their expression as a re | **`SQS`** | `Records[*].powertools_json(body)` | ???+ tip - If you're using architectures that involve sending event notifications from S3 to SNS and then to SQS or Kinesis Data Firehose, consider enabling [Raw Delivery option](https://docs.aws.amazon.com/sns/latest/dg/sns-large-payload-raw-message-delivery.html){target="_blank"} in Amazon SNS. This can help reduce the payload by stripping the SNS metadata. + If you're using architectures that involve sending event notifications from S3 to SNS and then to SQS or Kinesis Data Firehose, consider enabling [raw message delivery](https://docs.aws.amazon.com/sns/latest/dg/sns-large-payload-raw-message-delivery.html){target="_blank"} option in Amazon SNS. This can help reduce the payload by stripping the SNS metadata. ## Advanced From 7528d078c9013732b4e4514ab6f131d17272c678 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Thu, 27 Apr 2023 10:49:20 +0100 Subject: [PATCH 3/5] chore: addressing Ruben's feedback Co-authored-by: Ruben Fonseca Signed-off-by: Leandro Damascena --- docs/utilities/jmespath_functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/utilities/jmespath_functions.md b/docs/utilities/jmespath_functions.md index fbd7b5be6b4..03264a2f831 100644 --- a/docs/utilities/jmespath_functions.md +++ b/docs/utilities/jmespath_functions.md @@ -81,7 +81,7 @@ These are all built-in envelopes you can use along with their expression as a re | **`SQS`** | `Records[*].powertools_json(body)` | ???+ tip - If you're using architectures that involve sending event notifications from S3 to SNS and then to SQS or Kinesis Data Firehose, consider enabling [raw message delivery](https://docs.aws.amazon.com/sns/latest/dg/sns-large-payload-raw-message-delivery.html){target="_blank"} option in Amazon SNS. This can help reduce the payload by stripping the SNS metadata. + If you're using architectures that involve sending event notifications from S3 to SNS and then to SQS or Kinesis Data Firehose, consider enabling the [raw message delivery](https://docs.aws.amazon.com/sns/latest/dg/sns-large-payload-raw-message-delivery.html){target="_blank"} option in Amazon SNS. This can help reduce the payload by stripping the SNS metadata. ## Advanced From a5152dcca0aecacea9dc6e364a8862a1a045644e Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Fri, 28 Apr 2023 09:17:20 +0100 Subject: [PATCH 4/5] feat(jmespath): addressing Heitor's feedback --- .../utilities/jmespath_utils/envelopes.py | 4 ++-- docs/utilities/jmespath_functions.md | 8 ++++---- .../src/extract_data_from_builtin_envelope.py | 4 +++- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/aws_lambda_powertools/utilities/jmespath_utils/envelopes.py b/aws_lambda_powertools/utilities/jmespath_utils/envelopes.py index cabc6c265db..f4aecb24bae 100644 --- a/aws_lambda_powertools/utilities/jmespath_utils/envelopes.py +++ b/aws_lambda_powertools/utilities/jmespath_utils/envelopes.py @@ -7,7 +7,7 @@ KINESIS_DATA_STREAM = "Records[*].kinesis.powertools_json(powertools_base64(data))" CLOUDWATCH_LOGS = "awslogs.powertools_base64_gzip(data) | powertools_json(@).logEvents[*]" S3_SNS_SQS = "Records[*].powertools_json(body).powertools_json(Message).Records[0]" -S3_RAWSNS_SQS = "Records[*].powertools_json(body).Records[0]" +S3_SQS = "Records[*].powertools_json(body).Records[0]" S3_SNS_KINESIS_FIREHOSE = "records[*].powertools_json(powertools_base64(data)).powertools_json(Message).Records[0]" -S3_RAWSNS_KINESIS_FIREHOSE = "records[*].powertools_json(powertools_base64(data)).Records[0]" +S3_KINESIS_FIREHOSE = "records[*].powertools_json(powertools_base64(data)).Records[0]" S3_EVENTBRIDGE_SQS = "Records[*].powertools_json(body).detail" diff --git a/docs/utilities/jmespath_functions.md b/docs/utilities/jmespath_functions.md index 03264a2f831..33e0adcf6c6 100644 --- a/docs/utilities/jmespath_functions.md +++ b/docs/utilities/jmespath_functions.md @@ -73,15 +73,15 @@ These are all built-in envelopes you can use along with their expression as a re | **`EVENTBRIDGE`** | `detail` | | **`KINESIS_DATA_STREAM`** | `Records[*].kinesis.powertools_json(powertools_base64(data))` | | **`S3_EVENTBRIDGE_SQS`** | `Records[*].powertools_json(body).detail` | -| **`S3_RAWSNS_KINESIS_FIREHOSE`** | `records[*].powertools_json(powertools_base64(data)).Records[0]` | -| **`S3_RAWSNS_SQS`** | `Records[*].powertools_json(body).Records[0]` | +| **`S3_KINESIS_FIREHOSE`** | `records[*].powertools_json(powertools_base64(data)).Records[0]` | | **`S3_SNS_KINESIS_FIREHOSE`** | `records[*].powertools_json(powertools_base64(data)).powertools_json(Message).Records[0]` | | **`S3_SNS_SQS`** | `Records[*].powertools_json(body).powertools_json(Message).Records[0]` | +| **`S3_SQS`** | `Records[*].powertools_json(body).Records[0]` | | **`SNS`** | `Records[0].Sns.Message | powertools_json(@)` | | **`SQS`** | `Records[*].powertools_json(body)` | -???+ tip - If you're using architectures that involve sending event notifications from S3 to SNS and then to SQS or Kinesis Data Firehose, consider enabling the [raw message delivery](https://docs.aws.amazon.com/sns/latest/dg/sns-large-payload-raw-message-delivery.html){target="_blank"} option in Amazon SNS. This can help reduce the payload by stripping the SNS metadata. +???+ tip "Using SNS?" + Consider reducing the payload size by enabling the [raw message delivery](https://docs.aws.amazon.com/sns/latest/dg/sns-large-payload-raw-message-delivery.html){target="_blank"} option in Amazon SNS if you're utilizing architectures that require sending event notifications from S3 to SNS and then to SQS or Kinesis Data Firehose, but do not require the SNS metadata. ## Advanced diff --git a/examples/jmespath_functions/src/extract_data_from_builtin_envelope.py b/examples/jmespath_functions/src/extract_data_from_builtin_envelope.py index 8d65355d97a..d078e396519 100644 --- a/examples/jmespath_functions/src/extract_data_from_builtin_envelope.py +++ b/examples/jmespath_functions/src/extract_data_from_builtin_envelope.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from aws_lambda_powertools import Logger from aws_lambda_powertools.utilities.jmespath_utils import ( envelopes, @@ -9,7 +11,7 @@ def handler(event: dict, context: LambdaContext) -> dict: - records = extract_data_from_envelope(data=event, envelope=envelopes.SQS) + records: list = extract_data_from_envelope(data=event, envelope=envelopes.SQS) for record in records: # records is a list logger.info(record.get("customerId")) # now deserialized From ef541967a59cda753fd5859ecadac71bf7fdd454 Mon Sep 17 00:00:00 2001 From: Heitor Lessa Date: Mon, 1 May 2023 12:55:10 +0200 Subject: [PATCH 5/5] chore: break message in two sentences. Signed-off-by: Heitor Lessa --- docs/utilities/jmespath_functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/utilities/jmespath_functions.md b/docs/utilities/jmespath_functions.md index 33e0adcf6c6..41b8c3704ce 100644 --- a/docs/utilities/jmespath_functions.md +++ b/docs/utilities/jmespath_functions.md @@ -81,7 +81,7 @@ These are all built-in envelopes you can use along with their expression as a re | **`SQS`** | `Records[*].powertools_json(body)` | ???+ tip "Using SNS?" - Consider reducing the payload size by enabling the [raw message delivery](https://docs.aws.amazon.com/sns/latest/dg/sns-large-payload-raw-message-delivery.html){target="_blank"} option in Amazon SNS if you're utilizing architectures that require sending event notifications from S3 to SNS and then to SQS or Kinesis Data Firehose, but do not require the SNS metadata. + If you don't require SNS metadata, enable [raw message delivery](https://docs.aws.amazon.com/sns/latest/dg/sns-large-payload-raw-message-delivery.html){target="_blank"}. It will reduce multiple payload layers and size, when using SNS in combination with other services _(e.g., SQS, S3, etc)_. ## Advanced