From 0a1d01474e0195b41ab5e30f5f45845912ad4449 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Fri, 5 Aug 2022 02:05:15 +0100 Subject: [PATCH 01/29] docs(jmespath_function): snippets split, improved, fixed and lint --- docs/utilities/jmespath_functions.md | 180 +++++------------- .../extract_data_from_builtin_envelope.json | 20 ++ .../src/extract_data_from_builtin_envelope.py | 9 + .../src/extract_data_from_envelope.json | 3 + .../src/extract_data_from_envelope.py | 9 + ...owertools_base64_gzip_jmespath_function.py | 9 + .../powertools_base64_gzip_jmespath_schema.py | 39 ++++ .../powertools_base64_jmespath_function.py | 7 + .../src/powertools_base64_jmespath_schema.py | 39 ++++ .../powertools_custom_jmespath_function.json | 1 + .../powertools_custom_jmespath_function.py | 21 ++ .../powertools_json_idempotency_jmespath.json | 30 +++ .../powertools_json_idempotency_jmespath.py | 21 ++ .../src/powertools_json_jmespath_function.py | 7 + .../src/powertools_json_jmespath_schema.py | 39 ++++ 15 files changed, 297 insertions(+), 137 deletions(-) create mode 100644 examples/jmespath_functions/src/extract_data_from_builtin_envelope.json create mode 100644 examples/jmespath_functions/src/extract_data_from_builtin_envelope.py create mode 100644 examples/jmespath_functions/src/extract_data_from_envelope.json create mode 100644 examples/jmespath_functions/src/extract_data_from_envelope.py create mode 100644 examples/jmespath_functions/src/powertools_base64_gzip_jmespath_function.py create mode 100644 examples/jmespath_functions/src/powertools_base64_gzip_jmespath_schema.py create mode 100644 examples/jmespath_functions/src/powertools_base64_jmespath_function.py create mode 100644 examples/jmespath_functions/src/powertools_base64_jmespath_schema.py create mode 100644 examples/jmespath_functions/src/powertools_custom_jmespath_function.json create mode 100644 examples/jmespath_functions/src/powertools_custom_jmespath_function.py create mode 100644 examples/jmespath_functions/src/powertools_json_idempotency_jmespath.json create mode 100644 examples/jmespath_functions/src/powertools_json_idempotency_jmespath.py create mode 100644 examples/jmespath_functions/src/powertools_json_jmespath_function.py create mode 100644 examples/jmespath_functions/src/powertools_json_jmespath_schema.py diff --git a/docs/utilities/jmespath_functions.md b/docs/utilities/jmespath_functions.md index eee88c13cfb..3a87e9ff294 100644 --- a/docs/utilities/jmespath_functions.md +++ b/docs/utilities/jmespath_functions.md @@ -3,6 +3,8 @@ title: JMESPath Functions description: Utility --- + + ???+ tip JMESPath is a query language for JSON used by AWS CLI, AWS Python SDK, and AWS Lambda Powertools for Python. @@ -15,6 +17,9 @@ Built-in [JMESPath](https://jmespath.org/){target="_blank"} Functions to easily ## Getting started +???+ tip + All examples shared in this documentation are available within the [project repository](https://github.com/awslabs/aws-lambda-powertools-python/tree/develop/examples){target="_blank"}. + You might have events that contains encoded JSON payloads as string, base64, or even in compressed format. It is a common use case to decode and extract them partially or fully as part of your Lambda function invocation. Lambda Powertools also have utilities like [validation](validation.md), [idempotency](idempotency.md), or [feature flags](feature_flags.md) where you might need to extract a portion of your data before using them. @@ -26,69 +31,31 @@ Lambda Powertools also have utilities like [validation](validation.md), [idempot You can use the `extract_data_from_envelope` function along with any [JMESPath expression](https://jmespath.org/tutorial.html){target="_blank"}. -=== "app.py" - - ```python hl_lines="1 7" - from aws_lambda_powertools.utilities.jmespath_utils import extract_data_from_envelope - - from aws_lambda_powertools.utilities.typing import LambdaContext - - - def handler(event: dict, context: LambdaContext): - payload = extract_data_from_envelope(data=event, envelope="powertools_json(body)") - customer = payload.get("customerId") # now deserialized - ... - ``` +=== "extract_data_from_envelope.py" + ```python hl_lines="1 7" + --8<-- "examples/jmespath_functions/src/extract_data_from_envelope.py" + ``` -=== "event.json" +=== "extract_data_from_envelope.json" ```json - { - "body": "{\"customerId\":\"dd4649e6-2484-4993-acb8-0f9123103394\"}" - } + --8<-- "examples/jmespath_functions/src/extract_data_from_envelope.json" ``` ### Built-in envelopes We provide built-in envelopes for popular JMESPath expressions used when looking to decode/deserialize JSON objects within AWS Lambda Event Sources. -=== "app.py" +=== "extract_data_from_builtin_envelope.py" ```python hl_lines="1 7" - from aws_lambda_powertools.utilities.jmespath_utils import extract_data_from_envelope, envelopes - - from aws_lambda_powertools.utilities.typing import LambdaContext - - - def handler(event: dict, context: LambdaContext): - payload = extract_data_from_envelope(data=event, envelope=envelopes.SNS) - customer = payload.get("customerId") # now deserialized - ... + --8<-- "examples/jmespath_functions/src/extract_data_from_builtin_envelope.py" ``` -=== "event.json" - - ```json hl_lines="6" - { - "Records": [ - { - "messageId": "19dd0b57-b21e-4ac1-bd88-01bbb068cb78", - "receiptHandle": "MessageReceiptHandle", - "body": "{\"customerId\":\"dd4649e6-2484-4993-acb8-0f9123103394\",\"booking\":{\"id\":\"5b2c4803-330b-42b7-811a-c68689425de1\",\"reference\":\"ySz7oA\",\"outboundFlightId\":\"20c0d2f2-56a3-4068-bf20-ff7703db552d\"},\"payment\":{\"receipt\":\"https:\/\/pay.stripe.com\/receipts\/acct_1Dvn7pF4aIiftV70\/ch_3JTC14F4aIiftV700iFq2CHB\/rcpt_K7QsrFln9FgFnzUuBIiNdkkRYGxUL0X\",\"amount\":100}}", - "attributes": { - "ApproximateReceiveCount": "1", - "SentTimestamp": "1523232000000", - "SenderId": "123456789012", - "ApproximateFirstReceiveTimestamp": "1523232000001" - }, - "messageAttributes": {}, - "md5OfBody": "7b270e59b47ff90a553787216d55d91d", - "eventSource": "aws:sqs", - "eventSourceARN": "arn:aws:sqs:us-east-1:123456789012:MyQueue", - "awsRegion": "us-east-1" - } - ] - } +=== "extract_data_from_builtin_envelope.json" + + ```json hl_lines="6 15" + --8<-- "examples/jmespath_functions/src/extract_data_from_builtin_envelope.json" ``` These are all built-in envelopes you can use along with their expression as a reference: @@ -124,50 +91,30 @@ This sample will decode the value within the `data` key into a valid JSON before === "powertools_json_jmespath_function.py" ```python hl_lines="9" - from aws_lambda_powertools.utilities.validation import validate - - import schemas - - sample_event = { - 'data': '{"payload": {"message": "hello hello", "username": "blah blah"}}' - } - - validate(event=sample_event, schema=schemas.INPUT, envelope="powertools_json(data)") + --8<-- "examples/jmespath_functions/src/powertools_json_jmespath_function.py" ``` -=== "schemas.py" +=== "powertools_json_jmespath_schema.py" ```python hl_lines="7 14 16 23 39 45 47 52" - --8<-- "docs/shared/validation_basic_jsonschema.py" + --8<-- "examples/jmespath_functions/src/powertools_json_jmespath_schema.py" ``` > **Idempotency scenario** This sample will decode the value within the `body` key of an API Gateway event into a valid JSON object to ensure the Idempotency utility processes a JSON object instead of a string. -```python hl_lines="7" title="Deserializing JSON before using as idempotency key" -import json -from aws_lambda_powertools.utilities.idempotency import ( - IdempotencyConfig, DynamoDBPersistenceLayer, idempotent -) - -persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") -config = IdempotencyConfig(event_key_jmespath="powertools_json(body)") - -@idempotent(config=config, persistence_store=persistence_layer) -def handler(event:APIGatewayProxyEvent, context): - body = json.loads(event['body']) - payment = create_subscription_payment( - user=body['user'], - product=body['product_id'] - ) - ... - return { - "payment_id": payment.id, - "message": "success", - "statusCode": 200 - } -``` +=== "powertools_json_idempotency_jmespath.py" + + ```python hl_lines="7" + --8<-- "examples/jmespath_functions/src/powertools_json_idempotency_jmespath.py" + ``` + +=== "powertools_json_idempotency_jmespath.json" + + ```json hl_lines="28" + --8<-- "examples/jmespath_functions/src/powertools_json_idempotency_jmespath.json" + ``` #### powertools_base64 function @@ -175,28 +122,16 @@ Use `powertools_base64` function to decode any base64 data. This sample will decode the base64 value within the `data` key, and decode the JSON string into a valid JSON before we can validate it. -=== "powertools_json_jmespath_function.py" +=== "powertools_base64_jmespath_function.py" ```python hl_lines="12" - from aws_lambda_powertools.utilities.validation import validate - - import schemas - - sample_event = { - "data": "eyJtZXNzYWdlIjogImhlbGxvIGhlbGxvIiwgInVzZXJuYW1lIjogImJsYWggYmxhaCJ9=" - } - - validate( - event=sample_event, - schema=schemas.INPUT, - envelope="powertools_json(powertools_base64(data))" - ) + --8<-- "examples/jmespath_functions/src/powertools_base64_jmespath_function.py" ``` -=== "schemas.py" +=== "powertools_base64_jmespath_schema.py" ```python hl_lines="7 14 16 23 39 45 47 52" - --8<-- "docs/shared/validation_basic_jsonschema.py" + --8<-- "examples/jmespath_functions/src/powertools_base64_jmespath_schema.py" ``` #### powertools_base64_gzip function @@ -205,28 +140,16 @@ Use `powertools_base64_gzip` function to decompress and decode base64 data. This sample will decompress and decode base64 data, then use JMESPath pipeline expression to pass the result for decoding its JSON string. -=== "powertools_json_jmespath_function.py" +=== "powertools_base64_gzip_jmespath_function.py" ```python hl_lines="12" - from aws_lambda_powertools.utilities.validation import validate - - import schemas - - sample_event = { - "data": "H4sIACZAXl8C/52PzUrEMBhFX2UILpX8tPbHXWHqIOiq3Q1F0ubrWEiakqTWofTdTYYB0YWL2d5zvnuTFellBIOedoiyKH5M0iwnlKH7HZL6dDB6ngLDfLFYctUKjie9gHFaS/sAX1xNEq525QxwFXRGGMEkx4Th491rUZdV3YiIZ6Ljfd+lfSyAtZloacQgAkqSJCGhxM6t7cwwuUGPz4N0YKyvO6I9WDeMPMSo8Z4Ca/kJ6vMEYW5f1MX7W1lVxaG8vqX8hNFdjlc0iCBBSF4ERT/3Pl7RbMGMXF2KZMh/C+gDpNS7RRsp0OaRGzx0/t8e0jgmcczyLCWEePhni/23JWalzjdu0a3ZvgEaNLXeugEAAA==" - } - - validate( - event=sample_event, - schema=schemas.INPUT, - envelope="powertools_base64_gzip(data) | powertools_json(@)" - ) + --8<-- "examples/jmespath_functions/src/powertools_base64_gzip_jmespath_function.py" ``` -=== "schemas.py" +=== "powertools_base64_gzip_jmespath_schema.py" ```python hl_lines="7 14 16 23 39 45 47 52" - --8<-- "docs/shared/validation_basic_jsonschema.py" + --8<-- "examples/jmespath_functions/src/powertools_base64_gzip_jmespath_schema.py" ``` ### Bring your own JMESPath function @@ -238,31 +161,14 @@ For special binary formats that you want to decode before applying JSON Schema v In order to keep the built-in functions from Powertools, you can subclass from `PowertoolsFunctions`: -=== "custom_jmespath_function.py" +=== "powertools_custom_jmespath_function.py" ```python hl_lines="2-3 6-9 11 17" - from aws_lambda_powertools.utilities.jmespath_utils import ( - PowertoolsFunctions, extract_data_from_envelope) - from jmespath.functions import signature - - - class CustomFunctions(PowertoolsFunctions): - @signature({'types': ['string']}) # Only decode if value is a string - def _func_special_decoder(self, s): - return my_custom_decoder_logic(s) - - custom_jmespath_options = {"custom_functions": CustomFunctions()} - - def handler(event, context): - # use the custom name after `_func_` - extract_data_from_envelope(data=event, - envelope="special_decoder(body)", - jmespath_options=**custom_jmespath_options) - ... + --8<-- "examples/jmespath_functions/src/powertools_custom_jmespath_function.py" ``` -=== "event.json" +=== "powertools_custom_jmespath_function.json" ```json - {"body": "custom_encoded_data"} + --8<-- "examples/jmespath_functions/src/powertools_custom_jmespath_function.json" ``` diff --git a/examples/jmespath_functions/src/extract_data_from_builtin_envelope.json b/examples/jmespath_functions/src/extract_data_from_builtin_envelope.json new file mode 100644 index 00000000000..6fe1d1655ab --- /dev/null +++ b/examples/jmespath_functions/src/extract_data_from_builtin_envelope.json @@ -0,0 +1,20 @@ +{ + "Records": [ + { + "messageId": "19dd0b57-b21e-4ac1-bd88-01bbb068cb78", + "receiptHandle": "MessageReceiptHandle", + "body": "{\"customerId\":\"dd4649e6-2484-4993-acb8-0f9123103394\",\"booking\":{\"id\":\"5b2c4803-330b-42b7-811a-c68689425de1\",\"reference\":\"ySz7oA\",\"outboundFlightId\":\"20c0d2f2-56a3-4068-bf20-ff7703db552d\"},\"payment\":{\"receipt\":\"https:\/\/pay.stripe.com\/receipts\/acct_1Dvn7pF4aIiftV70\/ch_3JTC14F4aIiftV700iFq2CHB\/rcpt_K7QsrFln9FgFnzUuBIiNdkkRYGxUL0X\",\"amount\":100}}", + "attributes": { + "ApproximateReceiveCount": "1", + "SentTimestamp": "1523232000000", + "SenderId": "123456789012", + "ApproximateFirstReceiveTimestamp": "1523232000001" + }, + "messageAttributes": {}, + "md5OfBody": "7b270e59b47ff90a553787216d55d91d", + "eventSource": "aws:sqs", + "eventSourceARN": "arn:aws:sqs:us-east-1:123456789012:MyQueue", + "awsRegion": "us-east-1" + } + ] +} diff --git a/examples/jmespath_functions/src/extract_data_from_builtin_envelope.py b/examples/jmespath_functions/src/extract_data_from_builtin_envelope.py new file mode 100644 index 00000000000..5a84da9d604 --- /dev/null +++ b/examples/jmespath_functions/src/extract_data_from_builtin_envelope.py @@ -0,0 +1,9 @@ +from aws_lambda_powertools.utilities.jmespath_utils import envelopes, extract_data_from_envelope +from aws_lambda_powertools.utilities.typing import LambdaContext + + +def handler(event: dict, context: LambdaContext) -> str: + payload = extract_data_from_envelope(data=event, envelope=envelopes.SQS) + customer_id = payload.get("customerId") # now deserialized + ... + return {"customer_id": customer_id, "message": "success", "statusCode": 200} diff --git a/examples/jmespath_functions/src/extract_data_from_envelope.json b/examples/jmespath_functions/src/extract_data_from_envelope.json new file mode 100644 index 00000000000..f64410e74f0 --- /dev/null +++ b/examples/jmespath_functions/src/extract_data_from_envelope.json @@ -0,0 +1,3 @@ +{ + "body": "{\"customerId\":\"dd4649e6-2484-4993-acb8-0f9123103394\"}" +} diff --git a/examples/jmespath_functions/src/extract_data_from_envelope.py b/examples/jmespath_functions/src/extract_data_from_envelope.py new file mode 100644 index 00000000000..4bcfa303b3c --- /dev/null +++ b/examples/jmespath_functions/src/extract_data_from_envelope.py @@ -0,0 +1,9 @@ +from aws_lambda_powertools.utilities.jmespath_utils import extract_data_from_envelope +from aws_lambda_powertools.utilities.typing import LambdaContext + + +def handler(event: dict, context: LambdaContext): + payload = extract_data_from_envelope(data=event, envelope="powertools_json(body)") + customer_id = payload.get("customerId") # now deserialized + ... + return {"customer_id": customer_id, "message": "success", "statusCode": 200} diff --git a/examples/jmespath_functions/src/powertools_base64_gzip_jmespath_function.py b/examples/jmespath_functions/src/powertools_base64_gzip_jmespath_function.py new file mode 100644 index 00000000000..4edba2aa156 --- /dev/null +++ b/examples/jmespath_functions/src/powertools_base64_gzip_jmespath_function.py @@ -0,0 +1,9 @@ +import powertools_base64_gzip_jmespath_schema as schemas + +from aws_lambda_powertools.utilities.validation import validate + +sample_event = { + "data": "H4sIACZAXl8C/52PzUrEMBhFX2UILpX8tPbHXWHqIOiq3Q1F0ubrWEiakqTWofTdTYYB0YWL2d5zvnuTFellBIOedoiyKH5M0iwnlKH7HZL6dDB6ngLDfLFYctUKjie9gHFaS/sAX1xNEq525QxwFXRGGMEkx4Th491rUZdV3YiIZ6Ljfd+lfSyAtZloacQgAkqSJCGhxM6t7cwwuUGPz4N0YKyvO6I9WDeMPMSo8Z4Ca/kJ6vMEYW5f1MX7W1lVxaG8vqX8hNFdjlc0iCBBSF4ERT/3Pl7RbMGMXF2KZMh/C+gDpNS7RRsp0OaRGzx0/t8e0jgmcczyLCWEePhni/23JWalzjdu0a3ZvgEaNLXeugEAAA==" # noqa: E501 +} + +validate(event=sample_event, schema=schemas.INPUT, envelope="powertools_base64_gzip(data) | powertools_json(@)") diff --git a/examples/jmespath_functions/src/powertools_base64_gzip_jmespath_schema.py b/examples/jmespath_functions/src/powertools_base64_gzip_jmespath_schema.py new file mode 100644 index 00000000000..afb8a723d18 --- /dev/null +++ b/examples/jmespath_functions/src/powertools_base64_gzip_jmespath_schema.py @@ -0,0 +1,39 @@ +INPUT = { + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "http://example.com/example.json", + "type": "object", + "title": "Sample schema", + "description": "The root schema comprises the entire JSON document.", + "examples": [{"message": "hello world", "username": "lessa"}], + "required": ["message", "username"], + "properties": { + "message": { + "$id": "#/properties/message", + "type": "string", + "title": "The message", + "examples": ["hello world"], + "maxLength": 100, + }, + "username": { + "$id": "#/properties/username", + "type": "string", + "title": "The username", + "examples": ["lessa"], + "maxLength": 30, + }, + }, +} + +OUTPUT = { + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "http://example.com/example.json", + "type": "object", + "title": "Sample outgoing schema", + "description": "The root schema comprises the entire JSON document.", + "examples": [{"statusCode": 200, "body": "response"}], + "required": ["statusCode", "body"], + "properties": { + "statusCode": {"$id": "#/properties/statusCode", "type": "integer", "title": "The statusCode"}, + "body": {"$id": "#/properties/body", "type": "string", "title": "The response"}, + }, +} diff --git a/examples/jmespath_functions/src/powertools_base64_jmespath_function.py b/examples/jmespath_functions/src/powertools_base64_jmespath_function.py new file mode 100644 index 00000000000..435de5d2f74 --- /dev/null +++ b/examples/jmespath_functions/src/powertools_base64_jmespath_function.py @@ -0,0 +1,7 @@ +import powertools_base64_jmespath_schema as schemas + +from aws_lambda_powertools.utilities.validation import validate + +sample_event = {"data": "eyJtZXNzYWdlIjogImhlbGxvIGhlbGxvIiwgInVzZXJuYW1lIjogImJsYWggYmxhaCJ9="} + +validate(event=sample_event, schema=schemas.INPUT, envelope="powertools_json(powertools_base64(data))") diff --git a/examples/jmespath_functions/src/powertools_base64_jmespath_schema.py b/examples/jmespath_functions/src/powertools_base64_jmespath_schema.py new file mode 100644 index 00000000000..afb8a723d18 --- /dev/null +++ b/examples/jmespath_functions/src/powertools_base64_jmespath_schema.py @@ -0,0 +1,39 @@ +INPUT = { + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "http://example.com/example.json", + "type": "object", + "title": "Sample schema", + "description": "The root schema comprises the entire JSON document.", + "examples": [{"message": "hello world", "username": "lessa"}], + "required": ["message", "username"], + "properties": { + "message": { + "$id": "#/properties/message", + "type": "string", + "title": "The message", + "examples": ["hello world"], + "maxLength": 100, + }, + "username": { + "$id": "#/properties/username", + "type": "string", + "title": "The username", + "examples": ["lessa"], + "maxLength": 30, + }, + }, +} + +OUTPUT = { + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "http://example.com/example.json", + "type": "object", + "title": "Sample outgoing schema", + "description": "The root schema comprises the entire JSON document.", + "examples": [{"statusCode": 200, "body": "response"}], + "required": ["statusCode", "body"], + "properties": { + "statusCode": {"$id": "#/properties/statusCode", "type": "integer", "title": "The statusCode"}, + "body": {"$id": "#/properties/body", "type": "string", "title": "The response"}, + }, +} diff --git a/examples/jmespath_functions/src/powertools_custom_jmespath_function.json b/examples/jmespath_functions/src/powertools_custom_jmespath_function.json new file mode 100644 index 00000000000..f58b87e6553 --- /dev/null +++ b/examples/jmespath_functions/src/powertools_custom_jmespath_function.json @@ -0,0 +1 @@ +{"body": "custom_encoded_data"} diff --git a/examples/jmespath_functions/src/powertools_custom_jmespath_function.py b/examples/jmespath_functions/src/powertools_custom_jmespath_function.py new file mode 100644 index 00000000000..056378e03ce --- /dev/null +++ b/examples/jmespath_functions/src/powertools_custom_jmespath_function.py @@ -0,0 +1,21 @@ +import base64 + +from jmespath.functions import signature + +from aws_lambda_powertools.utilities.jmespath_utils import PowertoolsFunctions, extract_data_from_envelope + + +class CustomFunctions(PowertoolsFunctions): + @signature({"types": ["string"]}) # Only decode if value is a string + def _func_special_decoder(self, payload: str): + payload_decode = base64.b64decode(payload) + return payload_decode + + +custom_jmespath_options = {"custom_functions": CustomFunctions()} + + +def handler(event, context): + # use the custom name after `_func_` + extract_data_from_envelope(data=event, envelope="special_decoder(body)", jmespath_options=custom_jmespath_options) + ... diff --git a/examples/jmespath_functions/src/powertools_json_idempotency_jmespath.json b/examples/jmespath_functions/src/powertools_json_idempotency_jmespath.json new file mode 100644 index 00000000000..31d61c31839 --- /dev/null +++ b/examples/jmespath_functions/src/powertools_json_idempotency_jmespath.json @@ -0,0 +1,30 @@ +{ + "version":"2.0", + "routeKey":"ANY /createpayment", + "rawPath":"/createpayment", + "rawQueryString":"", + "headers": { + "Header1": "value1", + "Header2": "value2" + }, + "requestContext":{ + "accountId":"123456789012", + "apiId":"api-id", + "domainName":"id.execute-api.us-east-1.amazonaws.com", + "domainPrefix":"id", + "http":{ + "method":"POST", + "path":"/createpayment", + "protocol":"HTTP/1.1", + "sourceIp":"ip", + "userAgent":"agent" + }, + "requestId":"id", + "routeKey":"ANY /createpayment", + "stage":"$default", + "time":"10/Feb/2021:13:40:43 +0000", + "timeEpoch":1612964443723 + }, + "body":"{\"user\":\"xyz\",\"product_id\":\"123456789\"}", + "isBase64Encoded":false + } diff --git a/examples/jmespath_functions/src/powertools_json_idempotency_jmespath.py b/examples/jmespath_functions/src/powertools_json_idempotency_jmespath.py new file mode 100644 index 00000000000..f87d8e8dcbb --- /dev/null +++ b/examples/jmespath_functions/src/powertools_json_idempotency_jmespath.py @@ -0,0 +1,21 @@ +import json + +from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer, IdempotencyConfig, idempotent + +persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") + +# Treat everything under the "body" key +# in the event json object as our payload +config = IdempotencyConfig(event_key_jmespath="powertools_json(body)") + + +@idempotent(config=config, persistence_store=persistence_layer) +def handler(event, context): + body = json.loads(event["body"]) + payment = create_subscription_payment(user=body["user"], product_id=body["product_id"]) + ... + return {"payment_id": payment.id, "message": "success", "statusCode": 200} + + +def create_subscription_payment(user: str, product_id: str) -> dict: + return {"id": "1", "message": "paid"} diff --git a/examples/jmespath_functions/src/powertools_json_jmespath_function.py b/examples/jmespath_functions/src/powertools_json_jmespath_function.py new file mode 100644 index 00000000000..b3aaa760579 --- /dev/null +++ b/examples/jmespath_functions/src/powertools_json_jmespath_function.py @@ -0,0 +1,7 @@ +import powertools_json_jmespath_schema as schemas + +from aws_lambda_powertools.utilities.validation import validate + +sample_event = {"data": '{"payload": {"message": "hello hello", "username": "blah blah"}}'} + +validate(event=sample_event, schema=schemas.INPUT, envelope="powertools_json(data)") diff --git a/examples/jmespath_functions/src/powertools_json_jmespath_schema.py b/examples/jmespath_functions/src/powertools_json_jmespath_schema.py new file mode 100644 index 00000000000..afb8a723d18 --- /dev/null +++ b/examples/jmespath_functions/src/powertools_json_jmespath_schema.py @@ -0,0 +1,39 @@ +INPUT = { + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "http://example.com/example.json", + "type": "object", + "title": "Sample schema", + "description": "The root schema comprises the entire JSON document.", + "examples": [{"message": "hello world", "username": "lessa"}], + "required": ["message", "username"], + "properties": { + "message": { + "$id": "#/properties/message", + "type": "string", + "title": "The message", + "examples": ["hello world"], + "maxLength": 100, + }, + "username": { + "$id": "#/properties/username", + "type": "string", + "title": "The username", + "examples": ["lessa"], + "maxLength": 30, + }, + }, +} + +OUTPUT = { + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "http://example.com/example.json", + "type": "object", + "title": "Sample outgoing schema", + "description": "The root schema comprises the entire JSON document.", + "examples": [{"statusCode": 200, "body": "response"}], + "required": ["statusCode", "body"], + "properties": { + "statusCode": {"$id": "#/properties/statusCode", "type": "integer", "title": "The statusCode"}, + "body": {"$id": "#/properties/body", "type": "string", "title": "The response"}, + }, +} From 5a3a461b1dcd376a5c0867faf7e812834bb79c43 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Fri, 5 Aug 2022 02:13:34 +0100 Subject: [PATCH 02/29] docs(jmespath_function): code highlights --- docs/utilities/jmespath_functions.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/utilities/jmespath_functions.md b/docs/utilities/jmespath_functions.md index 3a87e9ff294..53550b9a31c 100644 --- a/docs/utilities/jmespath_functions.md +++ b/docs/utilities/jmespath_functions.md @@ -90,7 +90,7 @@ This sample will decode the value within the `data` key into a valid JSON before === "powertools_json_jmespath_function.py" - ```python hl_lines="9" + ```python hl_lines="7" --8<-- "examples/jmespath_functions/src/powertools_json_jmespath_function.py" ``` @@ -106,7 +106,7 @@ This sample will decode the value within the `body` key of an API Gateway event === "powertools_json_idempotency_jmespath.py" - ```python hl_lines="7" + ```python hl_lines="9" --8<-- "examples/jmespath_functions/src/powertools_json_idempotency_jmespath.py" ``` @@ -124,7 +124,7 @@ This sample will decode the base64 value within the `data` key, and decode the J === "powertools_base64_jmespath_function.py" - ```python hl_lines="12" + ```python hl_lines="7" --8<-- "examples/jmespath_functions/src/powertools_base64_jmespath_function.py" ``` @@ -142,7 +142,7 @@ This sample will decompress and decode base64 data, then use JMESPath pipeline e === "powertools_base64_gzip_jmespath_function.py" - ```python hl_lines="12" + ```python hl_lines="9" --8<-- "examples/jmespath_functions/src/powertools_base64_gzip_jmespath_function.py" ``` @@ -163,7 +163,7 @@ In order to keep the built-in functions from Powertools, you can subclass from ` === "powertools_custom_jmespath_function.py" - ```python hl_lines="2-3 6-9 11 17" + ```python hl_lines="3 5 8-12 15 20" --8<-- "examples/jmespath_functions/src/powertools_custom_jmespath_function.py" ``` From f9cd042c214d929cebb06c38a30135351db7f01d Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Fri, 5 Aug 2022 08:55:28 +0100 Subject: [PATCH 03/29] docs(jmespath_function): code highlights --- docs/utilities/jmespath_functions.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/utilities/jmespath_functions.md b/docs/utilities/jmespath_functions.md index 53550b9a31c..d42fd6f71ac 100644 --- a/docs/utilities/jmespath_functions.md +++ b/docs/utilities/jmespath_functions.md @@ -32,7 +32,7 @@ Lambda Powertools also have utilities like [validation](validation.md), [idempot You can use the `extract_data_from_envelope` function along with any [JMESPath expression](https://jmespath.org/tutorial.html){target="_blank"}. === "extract_data_from_envelope.py" - ```python hl_lines="1 7" + ```python hl_lines="1 6" --8<-- "examples/jmespath_functions/src/extract_data_from_envelope.py" ``` @@ -48,7 +48,7 @@ We provide built-in envelopes for popular JMESPath expressions used when looking === "extract_data_from_builtin_envelope.py" - ```python hl_lines="1 7" + ```python hl_lines="1 6" --8<-- "examples/jmespath_functions/src/extract_data_from_builtin_envelope.py" ``` From df9156b01e63fafd367ca20794c03721f91d20fe Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Fri, 5 Aug 2022 14:18:53 +0200 Subject: [PATCH 04/29] docs(jmespath_util): include example for nested data Signed-off-by: heitorlessa --- docs/utilities/jmespath_functions.md | 5 ++++- .../src/extract_data_from_envelope.json | 13 +++++++++++-- .../src/extract_data_from_envelope.py | 7 +++++-- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/docs/utilities/jmespath_functions.md b/docs/utilities/jmespath_functions.md index d42fd6f71ac..50a7cfcb06c 100644 --- a/docs/utilities/jmespath_functions.md +++ b/docs/utilities/jmespath_functions.md @@ -31,8 +31,11 @@ Lambda Powertools also have utilities like [validation](validation.md), [idempot You can use the `extract_data_from_envelope` function along with any [JMESPath expression](https://jmespath.org/tutorial.html){target="_blank"}. +???+ tip + Another common use case is to fetch deeply nested data, filter, flatten, and more. + === "extract_data_from_envelope.py" - ```python hl_lines="1 6" + ```python hl_lines="1 6 10" --8<-- "examples/jmespath_functions/src/extract_data_from_envelope.py" ``` diff --git a/examples/jmespath_functions/src/extract_data_from_envelope.json b/examples/jmespath_functions/src/extract_data_from_envelope.json index f64410e74f0..0a0f0763279 100644 --- a/examples/jmespath_functions/src/extract_data_from_envelope.json +++ b/examples/jmespath_functions/src/extract_data_from_envelope.json @@ -1,3 +1,12 @@ { - "body": "{\"customerId\":\"dd4649e6-2484-4993-acb8-0f9123103394\"}" -} + "body": "{\"customerId\":\"dd4649e6-2484-4993-acb8-0f9123103394\"}", + "deeply_nested": [ + { + "some_data": [ + 1, + 2, + 3 + ] + } + ] +} \ No newline at end of file diff --git a/examples/jmespath_functions/src/extract_data_from_envelope.py b/examples/jmespath_functions/src/extract_data_from_envelope.py index 4bcfa303b3c..cc4bb20c22d 100644 --- a/examples/jmespath_functions/src/extract_data_from_envelope.py +++ b/examples/jmespath_functions/src/extract_data_from_envelope.py @@ -5,5 +5,8 @@ def handler(event: dict, context: LambdaContext): payload = extract_data_from_envelope(data=event, envelope="powertools_json(body)") customer_id = payload.get("customerId") # now deserialized - ... - return {"customer_id": customer_id, "message": "success", "statusCode": 200} + + # also works for fetching and flattening deeply nested data + some_data = extract_data_from_envelope(data=event, envelope="deeply_nested[*].some_data[]") + + return {"customer_id": customer_id, "message": "success", "context": some_data, "statusCode": 200} From 76d7fe018168d784d7a9b637fdf0d5c6f1dc14ea Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Fri, 5 Aug 2022 14:20:17 +0200 Subject: [PATCH 05/29] docs(jmespath_util): fix return type Signed-off-by: heitorlessa --- .../src/extract_data_from_builtin_envelope.py | 4 ++-- examples/jmespath_functions/src/extract_data_from_envelope.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/jmespath_functions/src/extract_data_from_builtin_envelope.py b/examples/jmespath_functions/src/extract_data_from_builtin_envelope.py index 5a84da9d604..53c230e1b9b 100644 --- a/examples/jmespath_functions/src/extract_data_from_builtin_envelope.py +++ b/examples/jmespath_functions/src/extract_data_from_builtin_envelope.py @@ -2,8 +2,8 @@ from aws_lambda_powertools.utilities.typing import LambdaContext -def handler(event: dict, context: LambdaContext) -> str: +def handler(event: dict, context: LambdaContext) -> dict: payload = extract_data_from_envelope(data=event, envelope=envelopes.SQS) customer_id = payload.get("customerId") # now deserialized - ... + return {"customer_id": customer_id, "message": "success", "statusCode": 200} diff --git a/examples/jmespath_functions/src/extract_data_from_envelope.py b/examples/jmespath_functions/src/extract_data_from_envelope.py index cc4bb20c22d..5c35bc4348b 100644 --- a/examples/jmespath_functions/src/extract_data_from_envelope.py +++ b/examples/jmespath_functions/src/extract_data_from_envelope.py @@ -2,7 +2,7 @@ from aws_lambda_powertools.utilities.typing import LambdaContext -def handler(event: dict, context: LambdaContext): +def handler(event: dict, context: LambdaContext) -> dict: payload = extract_data_from_envelope(data=event, envelope="powertools_json(body)") customer_id = payload.get("customerId") # now deserialized From 9f96bd8ea4e790f0d236127ce894ace0cb857aab Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sat, 6 Aug 2022 18:50:27 +0100 Subject: [PATCH 06/29] docs(jmespath_function): Refactoring codes for demonstration purposes - generic envelope --- .../src/powertools_json_jmespath_function.py | 24 ++++++- .../src/powertools_json_jmespath_payload.json | 3 + .../src/powertools_json_jmespath_schema.py | 63 ++++++++++--------- 3 files changed, 59 insertions(+), 31 deletions(-) create mode 100644 examples/jmespath_functions/src/powertools_json_jmespath_payload.json diff --git a/examples/jmespath_functions/src/powertools_json_jmespath_function.py b/examples/jmespath_functions/src/powertools_json_jmespath_function.py index b3aaa760579..551a0c1e99e 100644 --- a/examples/jmespath_functions/src/powertools_json_jmespath_function.py +++ b/examples/jmespath_functions/src/powertools_json_jmespath_function.py @@ -1,7 +1,25 @@ +import random + import powertools_json_jmespath_schema as schemas -from aws_lambda_powertools.utilities.validation import validate +from aws_lambda_powertools.utilities.typing import LambdaContext +from aws_lambda_powertools.utilities.validation import SchemaValidationError, validate + + +def lambda_handler(event, context: LambdaContext) -> dict: + + # Let's validate the schema + try: + validate(event=event, schema=schemas.INPUT, envelope="powertools_json(payload)") + + order = create_order(event["payload"]) + return {"order_id": order.get("id"), "message": order.get("message"), "statusCode": 200} + except SchemaValidationError as exception: + + # if validation fails, a SchemaValidationError will be raised with the wrong fields + return {"order_id": None, "statusCode": 400, "message": str(exception)} -sample_event = {"data": '{"payload": {"message": "hello hello", "username": "blah blah"}}'} -validate(event=sample_event, schema=schemas.INPUT, envelope="powertools_json(data)") +def create_order(event: dict) -> dict: + # Add your logic here. For demostration purposes, this return a random order number + return {"id": random.randint(1, 100), "message": "order created"} diff --git a/examples/jmespath_functions/src/powertools_json_jmespath_payload.json b/examples/jmespath_functions/src/powertools_json_jmespath_payload.json new file mode 100644 index 00000000000..f212b53ae7d --- /dev/null +++ b/examples/jmespath_functions/src/powertools_json_jmespath_payload.json @@ -0,0 +1,3 @@ +{ + "payload":"{\"user_id\": 123, \"product_id\": 1, \"quantity\": 2, \"price\": 100, \"currency\": \"USD\"}" +} diff --git a/examples/jmespath_functions/src/powertools_json_jmespath_schema.py b/examples/jmespath_functions/src/powertools_json_jmespath_schema.py index afb8a723d18..424784f19c6 100644 --- a/examples/jmespath_functions/src/powertools_json_jmespath_schema.py +++ b/examples/jmespath_functions/src/powertools_json_jmespath_schema.py @@ -2,38 +2,45 @@ "$schema": "http://json-schema.org/draft-07/schema", "$id": "http://example.com/example.json", "type": "object", - "title": "Sample schema", + "title": "Sample order schema", "description": "The root schema comprises the entire JSON document.", - "examples": [{"message": "hello world", "username": "lessa"}], - "required": ["message", "username"], + "examples": [{"user_id": 123, "product_id": 1, "quantity": 2, "price": 10.40, "currency": "USD"}], + "required": ["user_id", "product_id", "quantity", "price", "currency"], "properties": { - "message": { - "$id": "#/properties/message", - "type": "string", - "title": "The message", - "examples": ["hello world"], - "maxLength": 100, + "user_id": { + "$id": "#/properties/user_id", + "type": "integer", + "title": "The unique identifier of the user", + "examples": [123], + "maxLength": 10, + }, + "product_id": { + "$id": "#/properties/product_id", + "type": "integer", + "title": "The unique identifier of the product", + "examples": [1], + "maxLength": 10, + }, + "quantity": { + "$id": "#/properties/quantity", + "type": "integer", + "title": "The quantity of the product", + "examples": [2], + "maxLength": 10, }, - "username": { - "$id": "#/properties/username", + "price": { + "$id": "#/properties/price", + "type": "integer", + "title": "The individual price of the product", + "examples": [10.40], + "maxLength": 10, + }, + "currency": { + "$id": "#/properties/currency", "type": "string", - "title": "The username", - "examples": ["lessa"], - "maxLength": 30, + "title": "The currency", + "examples": ["The currency of the order"], + "maxLength": 100, }, }, } - -OUTPUT = { - "$schema": "http://json-schema.org/draft-07/schema", - "$id": "http://example.com/example.json", - "type": "object", - "title": "Sample outgoing schema", - "description": "The root schema comprises the entire JSON document.", - "examples": [{"statusCode": 200, "body": "response"}], - "required": ["statusCode", "body"], - "properties": { - "statusCode": {"$id": "#/properties/statusCode", "type": "integer", "title": "The statusCode"}, - "body": {"$id": "#/properties/body", "type": "string", "title": "The response"}, - }, -} From 196f3c909b559b86fbd551c50b637e666efd432d Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sat, 6 Aug 2022 18:57:52 +0100 Subject: [PATCH 07/29] docs(jmespath_function): Refactoring codes for demonstration purposes - generic envelope (small fixes) --- .../src/powertools_json_jmespath_payload.json | 2 +- .../jmespath_functions/src/powertools_json_jmespath_schema.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/jmespath_functions/src/powertools_json_jmespath_payload.json b/examples/jmespath_functions/src/powertools_json_jmespath_payload.json index f212b53ae7d..647583bba82 100644 --- a/examples/jmespath_functions/src/powertools_json_jmespath_payload.json +++ b/examples/jmespath_functions/src/powertools_json_jmespath_payload.json @@ -1,3 +1,3 @@ { - "payload":"{\"user_id\": 123, \"product_id\": 1, \"quantity\": 2, \"price\": 100, \"currency\": \"USD\"}" + "payload":"{\"user_id\": 123, \"product_id\": 1, \"quantity\": 2, \"price\": 10.40, \"currency\": \"USD\"}" } diff --git a/examples/jmespath_functions/src/powertools_json_jmespath_schema.py b/examples/jmespath_functions/src/powertools_json_jmespath_schema.py index 424784f19c6..bd643a11c13 100644 --- a/examples/jmespath_functions/src/powertools_json_jmespath_schema.py +++ b/examples/jmespath_functions/src/powertools_json_jmespath_schema.py @@ -30,7 +30,7 @@ }, "price": { "$id": "#/properties/price", - "type": "integer", + "type": "number", "title": "The individual price of the product", "examples": [10.40], "maxLength": 10, From 848e660d7605958d26be0ef3dcb5cc99c15c691c Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sat, 6 Aug 2022 19:05:33 +0100 Subject: [PATCH 08/29] docs(jmespath_function): Refactoring codes for demonstration purposes - generic envelope (small fixes) --- .../jmespath_functions/src/powertools_json_jmespath_function.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/jmespath_functions/src/powertools_json_jmespath_function.py b/examples/jmespath_functions/src/powertools_json_jmespath_function.py index 551a0c1e99e..075d5715138 100644 --- a/examples/jmespath_functions/src/powertools_json_jmespath_function.py +++ b/examples/jmespath_functions/src/powertools_json_jmespath_function.py @@ -17,7 +17,7 @@ def lambda_handler(event, context: LambdaContext) -> dict: except SchemaValidationError as exception: # if validation fails, a SchemaValidationError will be raised with the wrong fields - return {"order_id": None, "statusCode": 400, "message": str(exception)} + return {"order_id": None, "message": str(exception), "statusCode": 400} def create_order(event: dict) -> dict: From 986f550ed02de0708b41e78eae62d0618596beef Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sat, 6 Aug 2022 19:40:09 +0100 Subject: [PATCH 09/29] docs(jmespath_function): Refactoring codes for demonstration purposes - generic envelope (small fixes) --- docs/utilities/jmespath_functions.md | 10 ++++++++-- .../src/powertools_json_jmespath_function.py | 9 ++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/docs/utilities/jmespath_functions.md b/docs/utilities/jmespath_functions.md index 50a7cfcb06c..c5890e549c1 100644 --- a/docs/utilities/jmespath_functions.md +++ b/docs/utilities/jmespath_functions.md @@ -93,16 +93,22 @@ This sample will decode the value within the `data` key into a valid JSON before === "powertools_json_jmespath_function.py" - ```python hl_lines="7" + ```python hl_lines="6 13 17 19" --8<-- "examples/jmespath_functions/src/powertools_json_jmespath_function.py" ``` === "powertools_json_jmespath_schema.py" - ```python hl_lines="7 14 16 23 39 45 47 52" + ```python hl_lines="7 8 10 12 17 19 24 26 31 33 38 40" --8<-- "examples/jmespath_functions/src/powertools_json_jmespath_schema.py" ``` +=== "powertools_json_jmespath_payload.json" + + ```python hl_lines="2" + --8<-- "examples/jmespath_functions/src/powertools_json_jmespath_payload.json" + ``` + > **Idempotency scenario** This sample will decode the value within the `body` key of an API Gateway event into a valid JSON object to ensure the Idempotency utility processes a JSON object instead of a string. diff --git a/examples/jmespath_functions/src/powertools_json_jmespath_function.py b/examples/jmespath_functions/src/powertools_json_jmespath_function.py index 075d5715138..e78061e39d4 100644 --- a/examples/jmespath_functions/src/powertools_json_jmespath_function.py +++ b/examples/jmespath_functions/src/powertools_json_jmespath_function.py @@ -1,4 +1,5 @@ import random +from json import JSONDecodeError import powertools_json_jmespath_schema as schemas @@ -14,12 +15,18 @@ def lambda_handler(event, context: LambdaContext) -> dict: order = create_order(event["payload"]) return {"order_id": order.get("id"), "message": order.get("message"), "statusCode": 200} + except JSONDecodeError: + return return_error_message("Payload must be valid JSON (plain text).") except SchemaValidationError as exception: # if validation fails, a SchemaValidationError will be raised with the wrong fields - return {"order_id": None, "message": str(exception), "statusCode": 400} + return return_error_message(str(exception)) def create_order(event: dict) -> dict: # Add your logic here. For demostration purposes, this return a random order number return {"id": random.randint(1, 100), "message": "order created"} + + +def return_error_message(message: str) -> dict: + return {"order_id": None, "message": message, "statusCode": 400} From 0578971722fde3bcdcd4a225fa5309bd8439ba2b Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sat, 6 Aug 2022 19:43:15 +0100 Subject: [PATCH 10/29] docs(jmespath_function): Refactoring codes for demonstration purposes - generic envelope (highlight code) --- docs/utilities/jmespath_functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/utilities/jmespath_functions.md b/docs/utilities/jmespath_functions.md index c5890e549c1..f59c00d91a9 100644 --- a/docs/utilities/jmespath_functions.md +++ b/docs/utilities/jmespath_functions.md @@ -93,7 +93,7 @@ This sample will decode the value within the `data` key into a valid JSON before === "powertools_json_jmespath_function.py" - ```python hl_lines="6 13 17 19" + ```python hl_lines="2 7 14 18 20 22" --8<-- "examples/jmespath_functions/src/powertools_json_jmespath_function.py" ``` From b9889dc021788dc56a14842499570cb66c19227a Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sat, 6 Aug 2022 20:40:22 +0100 Subject: [PATCH 11/29] docs(jmespath_function): Refactoring codes for demonstration purposes - generic envelope - changing to more clean and userful code --- .../src/powertools_json_jmespath_function.py | 51 +++++++++++++++---- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/examples/jmespath_functions/src/powertools_json_jmespath_function.py b/examples/jmespath_functions/src/powertools_json_jmespath_function.py index e78061e39d4..0f6d2387f04 100644 --- a/examples/jmespath_functions/src/powertools_json_jmespath_function.py +++ b/examples/jmespath_functions/src/powertools_json_jmespath_function.py @@ -1,5 +1,6 @@ -import random -from json import JSONDecodeError +import json +from dataclasses import asdict, dataclass, is_dataclass +from uuid import uuid4 import powertools_json_jmespath_schema as schemas @@ -7,16 +8,39 @@ from aws_lambda_powertools.utilities.validation import SchemaValidationError, validate +@dataclass +class Order: + order_id: str + user_id: int + product_id: int + quantity: int + price: float + currency: str + + +class DataclassCustomEncoder(json.JSONEncoder): + """A custom JSON encoder to serialize dataclass obj""" + + def default(self, obj): + # Only called for values that aren't JSON serializable + # where `obj` will be an instance of Todo in this example + return asdict(obj) if is_dataclass(obj) else super().default(obj) + + def lambda_handler(event, context: LambdaContext) -> dict: # Let's validate the schema try: validate(event=event, schema=schemas.INPUT, envelope="powertools_json(payload)") - order = create_order(event["payload"]) - return {"order_id": order.get("id"), "message": order.get("message"), "statusCode": 200} - except JSONDecodeError: - return return_error_message("Payload must be valid JSON (plain text).") + order = create_order(json.loads(event["payload"])) + return { + "order": json.dumps(order.get("order"), cls=DataclassCustomEncoder), + "message": order.get("message"), + "success": True, + } + except json.JSONDecodeError: + return return_error_message("Payload must be valid JSON (base64 encoded).") except SchemaValidationError as exception: # if validation fails, a SchemaValidationError will be raised with the wrong fields @@ -24,9 +48,18 @@ def lambda_handler(event, context: LambdaContext) -> dict: def create_order(event: dict) -> dict: - # Add your logic here. For demostration purposes, this return a random order number - return {"id": random.randint(1, 100), "message": "order created"} + # This return an Order dataclass + order_id = str(uuid4()) + order = Order( + order_id=order_id, + user_id=event.get("user_id"), + product_id=event.get("product_id"), + quantity=event.get("quantity"), + price=event.get("price"), + currency=event.get("currency"), + ) + return {"order": order, "message": "order created"} def return_error_message(message: str) -> dict: - return {"order_id": None, "message": message, "statusCode": 400} + return {"order": None, "message": message, "success": False} From b64984226decc09436df0d2dbc08b74e1081f711 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sat, 6 Aug 2022 20:43:02 +0100 Subject: [PATCH 12/29] docs(jmespath_function): Refactoring codes for demonstration purposes - generic envelope - changing to more clean and userful code --- docs/utilities/jmespath_functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/utilities/jmespath_functions.md b/docs/utilities/jmespath_functions.md index f59c00d91a9..c374106ec49 100644 --- a/docs/utilities/jmespath_functions.md +++ b/docs/utilities/jmespath_functions.md @@ -93,7 +93,7 @@ This sample will decode the value within the `data` key into a valid JSON before === "powertools_json_jmespath_function.py" - ```python hl_lines="2 7 14 18 20 22" + ```python hl_lines="5 8 34 42 44 46" --8<-- "examples/jmespath_functions/src/powertools_json_jmespath_function.py" ``` From 0b2505d61aca2379cb0d2aace92ddd58c05e9b57 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sun, 7 Aug 2022 01:36:17 +0100 Subject: [PATCH 13/29] docs(jmespath_function): Refactoring codes - base64 envelope --- docs/utilities/jmespath_functions.md | 10 ++- .../powertools_base64_jmespath_function.py | 69 ++++++++++++++++++- .../powertools_base64_jmespath_payload.json | 3 + .../src/powertools_base64_jmespath_schema.py | 63 +++++++++-------- 4 files changed, 112 insertions(+), 33 deletions(-) create mode 100644 examples/jmespath_functions/src/powertools_base64_jmespath_payload.json diff --git a/docs/utilities/jmespath_functions.md b/docs/utilities/jmespath_functions.md index c374106ec49..751eff5dc55 100644 --- a/docs/utilities/jmespath_functions.md +++ b/docs/utilities/jmespath_functions.md @@ -133,16 +133,22 @@ This sample will decode the base64 value within the `data` key, and decode the J === "powertools_base64_jmespath_function.py" - ```python hl_lines="7" + ```python hl_lines="7 10 36 46 48 50 51" --8<-- "examples/jmespath_functions/src/powertools_base64_jmespath_function.py" ``` === "powertools_base64_jmespath_schema.py" - ```python hl_lines="7 14 16 23 39 45 47 52" + ```python hl_lines="7 8 10 12 17 19 24 26 31 33 38 40" --8<-- "examples/jmespath_functions/src/powertools_base64_jmespath_schema.py" ``` +=== "powertools_base64_jmespath_payload.json" + + ```python hl_lines="2" + --8<-- "examples/jmespath_functions/src/powertools_base64_jmespath_payload.json" + ``` + #### powertools_base64_gzip function Use `powertools_base64_gzip` function to decompress and decode base64 data. diff --git a/examples/jmespath_functions/src/powertools_base64_jmespath_function.py b/examples/jmespath_functions/src/powertools_base64_jmespath_function.py index 435de5d2f74..b96e2c952b5 100644 --- a/examples/jmespath_functions/src/powertools_base64_jmespath_function.py +++ b/examples/jmespath_functions/src/powertools_base64_jmespath_function.py @@ -1,7 +1,70 @@ +import base64 +import binascii +import json +from dataclasses import asdict, dataclass, is_dataclass +from uuid import uuid4 + import powertools_base64_jmespath_schema as schemas -from aws_lambda_powertools.utilities.validation import validate +from aws_lambda_powertools.utilities.typing import LambdaContext +from aws_lambda_powertools.utilities.validation import SchemaValidationError, validate + + +@dataclass +class Order: + order_id: str + user_id: int + product_id: int + quantity: int + price: float + currency: str + + +class DataclassCustomEncoder(json.JSONEncoder): + """A custom JSON encoder to serialize dataclass obj""" + + def default(self, obj): + # Only called for values that aren't JSON serializable + # where `obj` will be an instance of Todo in this example + return asdict(obj) if is_dataclass(obj) else super().default(obj) + + +def lambda_handler(event, context: LambdaContext) -> dict: + + # Try to validate the schema + try: + validate(event=event, schema=schemas.INPUT, envelope="powertools_json(powertools_base64(payload))") + + payload_decoded = base64.b64decode(event["payload"]).decode() + + order = create_order(json.loads(payload_decoded)) + return { + "order": json.dumps(order.get("order"), cls=DataclassCustomEncoder), + "message": order.get("message"), + "success": True, + } + except binascii.Error: + return return_error_message("Payload must be valid base64 encoded string") + except json.JSONDecodeError: + return return_error_message("Payload must be valid JSON (base64 encoded).") + except SchemaValidationError as exception: + # if validation fails, a SchemaValidationError will be raised with the wrong fields + return return_error_message(str(exception)) + + +def create_order(event: dict) -> dict: + # This return an Order dataclass + order_id = str(uuid4()) + order = Order( + order_id=order_id, + user_id=event.get("user_id"), + product_id=event.get("product_id"), + quantity=event.get("quantity"), + price=event.get("price"), + currency=event.get("currency"), + ) + return {"order": order, "message": "order created"} -sample_event = {"data": "eyJtZXNzYWdlIjogImhlbGxvIGhlbGxvIiwgInVzZXJuYW1lIjogImJsYWggYmxhaCJ9="} -validate(event=sample_event, schema=schemas.INPUT, envelope="powertools_json(powertools_base64(data))") +def return_error_message(message: str) -> dict: + return {"order": None, "message": message, "success": False} diff --git a/examples/jmespath_functions/src/powertools_base64_jmespath_payload.json b/examples/jmespath_functions/src/powertools_base64_jmespath_payload.json new file mode 100644 index 00000000000..b4ea41d1d09 --- /dev/null +++ b/examples/jmespath_functions/src/powertools_base64_jmespath_payload.json @@ -0,0 +1,3 @@ +{ + "payload":"eyJ1c2VyX2lkIjogMTIzLCAicHJvZHVjdF9pZCI6IDEsICJxdWFudGl0eSI6IDIsICJwcmljZSI6IDEwLjQwLCAiY3VycmVuY3kiOiAiVVNEIn0=" +} diff --git a/examples/jmespath_functions/src/powertools_base64_jmespath_schema.py b/examples/jmespath_functions/src/powertools_base64_jmespath_schema.py index afb8a723d18..bd643a11c13 100644 --- a/examples/jmespath_functions/src/powertools_base64_jmespath_schema.py +++ b/examples/jmespath_functions/src/powertools_base64_jmespath_schema.py @@ -2,38 +2,45 @@ "$schema": "http://json-schema.org/draft-07/schema", "$id": "http://example.com/example.json", "type": "object", - "title": "Sample schema", + "title": "Sample order schema", "description": "The root schema comprises the entire JSON document.", - "examples": [{"message": "hello world", "username": "lessa"}], - "required": ["message", "username"], + "examples": [{"user_id": 123, "product_id": 1, "quantity": 2, "price": 10.40, "currency": "USD"}], + "required": ["user_id", "product_id", "quantity", "price", "currency"], "properties": { - "message": { - "$id": "#/properties/message", - "type": "string", - "title": "The message", - "examples": ["hello world"], - "maxLength": 100, + "user_id": { + "$id": "#/properties/user_id", + "type": "integer", + "title": "The unique identifier of the user", + "examples": [123], + "maxLength": 10, + }, + "product_id": { + "$id": "#/properties/product_id", + "type": "integer", + "title": "The unique identifier of the product", + "examples": [1], + "maxLength": 10, + }, + "quantity": { + "$id": "#/properties/quantity", + "type": "integer", + "title": "The quantity of the product", + "examples": [2], + "maxLength": 10, }, - "username": { - "$id": "#/properties/username", + "price": { + "$id": "#/properties/price", + "type": "number", + "title": "The individual price of the product", + "examples": [10.40], + "maxLength": 10, + }, + "currency": { + "$id": "#/properties/currency", "type": "string", - "title": "The username", - "examples": ["lessa"], - "maxLength": 30, + "title": "The currency", + "examples": ["The currency of the order"], + "maxLength": 100, }, }, } - -OUTPUT = { - "$schema": "http://json-schema.org/draft-07/schema", - "$id": "http://example.com/example.json", - "type": "object", - "title": "Sample outgoing schema", - "description": "The root schema comprises the entire JSON document.", - "examples": [{"statusCode": 200, "body": "response"}], - "required": ["statusCode", "body"], - "properties": { - "statusCode": {"$id": "#/properties/statusCode", "type": "integer", "title": "The statusCode"}, - "body": {"$id": "#/properties/body", "type": "string", "title": "The response"}, - }, -} From c4db9873653039c6a6d3076480dfb9a6c7c383d4 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sun, 7 Aug 2022 16:17:04 +0100 Subject: [PATCH 14/29] docs(jmespath_function): Refactoring examples - base64-gzip envelope --- docs/utilities/jmespath_functions.md | 16 +++-- ...owertools_base64_gzip_jmespath_function.py | 49 ++++++++++++++-- ...wertools_base64_gzip_jmespath_payload.json | 3 + .../powertools_base64_gzip_jmespath_schema.py | 58 +++++++++++-------- 4 files changed, 91 insertions(+), 35 deletions(-) create mode 100644 examples/jmespath_functions/src/powertools_base64_gzip_jmespath_payload.json diff --git a/docs/utilities/jmespath_functions.md b/docs/utilities/jmespath_functions.md index 751eff5dc55..b79086e8a70 100644 --- a/docs/utilities/jmespath_functions.md +++ b/docs/utilities/jmespath_functions.md @@ -105,7 +105,7 @@ This sample will decode the value within the `data` key into a valid JSON before === "powertools_json_jmespath_payload.json" - ```python hl_lines="2" + ```json --8<-- "examples/jmespath_functions/src/powertools_json_jmespath_payload.json" ``` @@ -145,7 +145,7 @@ This sample will decode the base64 value within the `data` key, and decode the J === "powertools_base64_jmespath_payload.json" - ```python hl_lines="2" + ```json --8<-- "examples/jmespath_functions/src/powertools_base64_jmespath_payload.json" ``` @@ -153,20 +153,26 @@ This sample will decode the base64 value within the `data` key, and decode the J Use `powertools_base64_gzip` function to decompress and decode base64 data. -This sample will decompress and decode base64 data, then use JMESPath pipeline expression to pass the result for decoding its JSON string. +This sample will decompress and decode base64 data from Cloudwatch Logs, then use JMESPath pipeline expression to pass the result for decoding its JSON string. === "powertools_base64_gzip_jmespath_function.py" - ```python hl_lines="9" + ```python hl_lines="6 10 17 28 30 32 34 35" --8<-- "examples/jmespath_functions/src/powertools_base64_gzip_jmespath_function.py" ``` === "powertools_base64_gzip_jmespath_schema.py" - ```python hl_lines="7 14 16 23 39 45 47 52" + ```python hl_lines="7-15 17 19 24 26 31 33 38 40" --8<-- "examples/jmespath_functions/src/powertools_base64_gzip_jmespath_schema.py" ``` +=== "powertools_base64_gzip_jmespath_payload.json" + + ```json + --8<-- "examples/jmespath_functions/src/powertools_base64_gzip_jmespath_payload.json" + ``` + ### Bring your own JMESPath function ???+ warning diff --git a/examples/jmespath_functions/src/powertools_base64_gzip_jmespath_function.py b/examples/jmespath_functions/src/powertools_base64_gzip_jmespath_function.py index 4edba2aa156..ecbb845e6c6 100644 --- a/examples/jmespath_functions/src/powertools_base64_gzip_jmespath_function.py +++ b/examples/jmespath_functions/src/powertools_base64_gzip_jmespath_function.py @@ -1,9 +1,48 @@ +import base64 +import binascii +import gzip +import json + import powertools_base64_gzip_jmespath_schema as schemas +from jmespath.exceptions import JMESPathTypeError + +from aws_lambda_powertools.utilities.typing import LambdaContext +from aws_lambda_powertools.utilities.validation import SchemaValidationError, validate + + +def lambda_handler(event, context: LambdaContext) -> dict: + + # Try to validate the schema + try: + validate(event=event, schema=schemas.INPUT, envelope="powertools_base64_gzip(payload) | powertools_json(@)") + + encoded_payload = base64.b64decode(event["payload"]) + uncompressed_payload = gzip.decompress(encoded_payload).decode() + + logs = process_cloudwatch_logs(json.loads(uncompressed_payload)) + return { + "message": logs, + "success": True, + } + + except JMESPathTypeError: + return return_error_message("Envelope powertools_base64_gzip function must receive a valid base64-gzip field") + except binascii.Error: + return return_error_message("Payload must be valid base64 encoded string") + except json.JSONDecodeError: + return return_error_message("Payload must be valid JSON (base64 encoded).") + except SchemaValidationError as exception: + # if validation fails, a SchemaValidationError will be raised with the wrong fields + return return_error_message(str(exception)) + -from aws_lambda_powertools.utilities.validation import validate +def return_error_message(message: str) -> dict: + return {"message": message, "success": False} -sample_event = { - "data": "H4sIACZAXl8C/52PzUrEMBhFX2UILpX8tPbHXWHqIOiq3Q1F0ubrWEiakqTWofTdTYYB0YWL2d5zvnuTFellBIOedoiyKH5M0iwnlKH7HZL6dDB6ngLDfLFYctUKjie9gHFaS/sAX1xNEq525QxwFXRGGMEkx4Th491rUZdV3YiIZ6Ljfd+lfSyAtZloacQgAkqSJCGhxM6t7cwwuUGPz4N0YKyvO6I9WDeMPMSo8Z4Ca/kJ6vMEYW5f1MX7W1lVxaG8vqX8hNFdjlc0iCBBSF4ERT/3Pl7RbMGMXF2KZMh/C+gDpNS7RRsp0OaRGzx0/t8e0jgmcczyLCWEePhni/23JWalzjdu0a3ZvgEaNLXeugEAAA==" # noqa: E501 -} -validate(event=sample_event, schema=schemas.INPUT, envelope="powertools_base64_gzip(data) | powertools_json(@)") +def process_cloudwatch_logs(event: dict) -> str: + owner = event.get("owner") # noqa F841 + log_group = event.get("logGroup") # noqa F841 + # Add your logic here + ... + return "Logs processed" diff --git a/examples/jmespath_functions/src/powertools_base64_gzip_jmespath_payload.json b/examples/jmespath_functions/src/powertools_base64_gzip_jmespath_payload.json new file mode 100644 index 00000000000..13995523099 --- /dev/null +++ b/examples/jmespath_functions/src/powertools_base64_gzip_jmespath_payload.json @@ -0,0 +1,3 @@ +{ + "payload": "H4sIACZAXl8C/52PzUrEMBhFX2UILpX8tPbHXWHqIOiq3Q1F0ubrWEiakqTWofTdTYYB0YWL2d5zvnuTFellBIOedoiyKH5M0iwnlKH7HZL6dDB6ngLDfLFYctUKjie9gHFaS/sAX1xNEq525QxwFXRGGMEkx4Th491rUZdV3YiIZ6Ljfd+lfSyAtZloacQgAkqSJCGhxM6t7cwwuUGPz4N0YKyvO6I9WDeMPMSo8Z4Ca/kJ6vMEYW5f1MX7W1lVxaG8vqX8hNFdjlc0iCBBSF4ERT/3Pl7RbMGMXF2KZMh/C+gDpNS7RRsp0OaRGzx0/t8e0jgmcczyLCWEePhni/23JWalzjdu0a3ZvgEaNLXeugEAAA==" +} diff --git a/examples/jmespath_functions/src/powertools_base64_gzip_jmespath_schema.py b/examples/jmespath_functions/src/powertools_base64_gzip_jmespath_schema.py index afb8a723d18..0ba02934928 100644 --- a/examples/jmespath_functions/src/powertools_base64_gzip_jmespath_schema.py +++ b/examples/jmespath_functions/src/powertools_base64_gzip_jmespath_schema.py @@ -4,36 +4,44 @@ "type": "object", "title": "Sample schema", "description": "The root schema comprises the entire JSON document.", - "examples": [{"message": "hello world", "username": "lessa"}], - "required": ["message", "username"], + "examples": [ + { + "owner": "123456789012", + "logGroup": "/aws/lambda/powertools-example", + "logStream": "2022/08/07/[$LATEST]d3a8dcaffc7f4de2b8db132e3e106660", + "logEvents": {}, + } + ], + "required": ["owner", "logGroup", "logStream", "logEvents"], "properties": { - "message": { - "$id": "#/properties/message", + "owner": { + "$id": "#/properties/owner", "type": "string", - "title": "The message", - "examples": ["hello world"], + "title": "The owner", + "examples": ["123456789012"], + "maxLength": 12, + }, + "logGroup": { + "$id": "#/properties/logGroup", + "type": "string", + "title": "The logGroup", + "examples": ["/aws/lambda/powertools-example"], "maxLength": 100, }, - "username": { - "$id": "#/properties/username", + "logStream": { + "$id": "#/properties/logStream", "type": "string", - "title": "The username", - "examples": ["lessa"], - "maxLength": 30, + "title": "The logGroup", + "examples": ["2022/08/07/[$LATEST]d3a8dcaffc7f4de2b8db132e3e106660"], + "maxLength": 100, + }, + "logEvents": { + "$id": "#/properties/logEvents", + "type": "array", + "title": "The logEvents", + "examples": [ + "{'id': 'eventId1', 'message': {'username': 'lessa', 'message': 'hello world'}, 'timestamp': 1440442987000}" # noqa E501 + ], }, - }, -} - -OUTPUT = { - "$schema": "http://json-schema.org/draft-07/schema", - "$id": "http://example.com/example.json", - "type": "object", - "title": "Sample outgoing schema", - "description": "The root schema comprises the entire JSON document.", - "examples": [{"statusCode": 200, "body": "response"}], - "required": ["statusCode", "body"], - "properties": { - "statusCode": {"$id": "#/properties/statusCode", "type": "integer", "title": "The statusCode"}, - "body": {"$id": "#/properties/body", "type": "string", "title": "The response"}, }, } From 8f05eb8afbdbdd16beb9833a3beacce35c9100fe Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sun, 7 Aug 2022 16:28:46 +0100 Subject: [PATCH 15/29] docs(jmespath_function): Refactoring examples - small fixes --- .../src/powertools_base64_gzip_jmespath_function.py | 2 +- .../src/powertools_base64_jmespath_function.py | 5 +++++ .../src/powertools_json_jmespath_function.py | 3 +++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/examples/jmespath_functions/src/powertools_base64_gzip_jmespath_function.py b/examples/jmespath_functions/src/powertools_base64_gzip_jmespath_function.py index ecbb845e6c6..cdd0752a0e8 100644 --- a/examples/jmespath_functions/src/powertools_base64_gzip_jmespath_function.py +++ b/examples/jmespath_functions/src/powertools_base64_gzip_jmespath_function.py @@ -26,7 +26,7 @@ def lambda_handler(event, context: LambdaContext) -> dict: } except JMESPathTypeError: - return return_error_message("Envelope powertools_base64_gzip function must receive a valid base64-gzip field") + return return_error_message("The powertools_base64_gzip() envelope function must match a valid path.") except binascii.Error: return return_error_message("Payload must be valid base64 encoded string") except json.JSONDecodeError: diff --git a/examples/jmespath_functions/src/powertools_base64_jmespath_function.py b/examples/jmespath_functions/src/powertools_base64_jmespath_function.py index b96e2c952b5..e2f24d97434 100644 --- a/examples/jmespath_functions/src/powertools_base64_jmespath_function.py +++ b/examples/jmespath_functions/src/powertools_base64_jmespath_function.py @@ -5,6 +5,7 @@ from uuid import uuid4 import powertools_base64_jmespath_schema as schemas +from jmespath.exceptions import JMESPathTypeError from aws_lambda_powertools.utilities.typing import LambdaContext from aws_lambda_powertools.utilities.validation import SchemaValidationError, validate @@ -43,6 +44,10 @@ def lambda_handler(event, context: LambdaContext) -> dict: "message": order.get("message"), "success": True, } + except JMESPathTypeError: + return return_error_message( + "The powertools_json(powertools_base64()) envelope function must match a valid path." + ) except binascii.Error: return return_error_message("Payload must be valid base64 encoded string") except json.JSONDecodeError: diff --git a/examples/jmespath_functions/src/powertools_json_jmespath_function.py b/examples/jmespath_functions/src/powertools_json_jmespath_function.py index 0f6d2387f04..f7bf628e2dd 100644 --- a/examples/jmespath_functions/src/powertools_json_jmespath_function.py +++ b/examples/jmespath_functions/src/powertools_json_jmespath_function.py @@ -3,6 +3,7 @@ from uuid import uuid4 import powertools_json_jmespath_schema as schemas +from jmespath.exceptions import JMESPathTypeError from aws_lambda_powertools.utilities.typing import LambdaContext from aws_lambda_powertools.utilities.validation import SchemaValidationError, validate @@ -39,6 +40,8 @@ def lambda_handler(event, context: LambdaContext) -> dict: "message": order.get("message"), "success": True, } + except JMESPathTypeError: + return return_error_message("The powertools_json() envelope function must match a valid path.") except json.JSONDecodeError: return return_error_message("Payload must be valid JSON (base64 encoded).") except SchemaValidationError as exception: From f73310491ad96802ada03c1e3586444084941648 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sun, 7 Aug 2022 16:57:35 +0100 Subject: [PATCH 16/29] docs(jmespath_function): Refactoring examples - small fixes --- .../src/powertools_json_idempotency_jmespath.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/jmespath_functions/src/powertools_json_idempotency_jmespath.py b/examples/jmespath_functions/src/powertools_json_idempotency_jmespath.py index f87d8e8dcbb..399dcb40fc6 100644 --- a/examples/jmespath_functions/src/powertools_json_idempotency_jmespath.py +++ b/examples/jmespath_functions/src/powertools_json_idempotency_jmespath.py @@ -10,7 +10,7 @@ @idempotent(config=config, persistence_store=persistence_layer) -def handler(event, context): +def handler(event, context) -> dict: body = json.loads(event["body"]) payment = create_subscription_payment(user=body["user"], product_id=body["product_id"]) ... From 4aab491abe883d0f51bdd2785f2802a30c6bbe39 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sun, 7 Aug 2022 20:30:30 +0100 Subject: [PATCH 17/29] docs(jmespath_function): Refactoring examples - custom function envelope --- docs/utilities/jmespath_functions.md | 4 +- .../powertools_custom_jmespath_function.json | 15 ++++++- .../powertools_custom_jmespath_function.py | 42 +++++++++++++++---- 3 files changed, 50 insertions(+), 11 deletions(-) diff --git a/docs/utilities/jmespath_functions.md b/docs/utilities/jmespath_functions.md index b79086e8a70..69e130ae334 100644 --- a/docs/utilities/jmespath_functions.md +++ b/docs/utilities/jmespath_functions.md @@ -180,11 +180,11 @@ This sample will decompress and decode base64 data from Cloudwatch Logs, then us For special binary formats that you want to decode before applying JSON Schema validation, you can bring your own [JMESPath function](https://github.com/jmespath/jmespath.py#custom-functions){target="_blank"} and any additional option via `jmespath_options` param. -In order to keep the built-in functions from Powertools, you can subclass from `PowertoolsFunctions`: +In order to keep the built-in functions from Powertools, you can subclass from `PowertoolsFunctions`. Here is an example of how to unzip compressed messages using snappy: === "powertools_custom_jmespath_function.py" - ```python hl_lines="3 5 8-12 15 20" + ```python hl_lines="4 5 8 11 15 18 22 31-35 38 40 42" --8<-- "examples/jmespath_functions/src/powertools_custom_jmespath_function.py" ``` diff --git a/examples/jmespath_functions/src/powertools_custom_jmespath_function.json b/examples/jmespath_functions/src/powertools_custom_jmespath_function.json index f58b87e6553..fa0c44a3060 100644 --- a/examples/jmespath_functions/src/powertools_custom_jmespath_function.json +++ b/examples/jmespath_functions/src/powertools_custom_jmespath_function.json @@ -1 +1,14 @@ -{"body": "custom_encoded_data"} +{ + "Records": [ + { + "user": "integration-kafka", + "datetime": "2022-01-01T00:00:00.000Z", + "log": "/QGIMjAyMi8wNi8xNiAxNjoyNTowMCBbY3JpdF0gMzA1MTg5MCMNCPBEOiAqMSBjb25uZWN0KCkg\ndG8gMTI3LjAuMC4xOjUwMDAgZmFpbGVkICgxMzogUGVybWlzc2lvbiBkZW5pZWQpIHdoaWxlEUEI\naW5nAUJAdXBzdHJlYW0sIGNsaWVudDoZVKgsIHNlcnZlcjogXywgcmVxdWVzdDogIk9QVElPTlMg\nLyBIVFRQLzEuMSIsFUckOiAiaHR0cDovLzabABQvIiwgaG8FQDAxMjcuMC4wLjE6ODEi\n" + }, + { + "user": "integration-kafka", + "datetime": "2022-01-01T00:00:01.000Z", + "log": "tQHwnDEyNy4wLjAuMSAtIC0gWzE2L0p1bi8yMDIyOjE2OjMwOjE5ICswMTAwXSAiT1BUSU9OUyAv\nIEhUVFAvMS4xIiAyMDQgMCAiLSIgIk1vemlsbGEvNS4wIChYMTE7IExpbnV4IHg4Nl82NCkgQXBw\nbGVXZWJLaXQvNTM3LjM2IChLSFRNTCwgbGlrZSBHZWNrbykgQ2hyb21lLzEwMi4BmUwwIFNhZmFy\naS81MzcuMzYiICItIg==\n" + } + ] +} diff --git a/examples/jmespath_functions/src/powertools_custom_jmespath_function.py b/examples/jmespath_functions/src/powertools_custom_jmespath_function.py index 056378e03ce..f805237db76 100644 --- a/examples/jmespath_functions/src/powertools_custom_jmespath_function.py +++ b/examples/jmespath_functions/src/powertools_custom_jmespath_function.py @@ -1,21 +1,47 @@ import base64 +import binascii +import snappy +from jmespath.exceptions import JMESPathTypeError from jmespath.functions import signature from aws_lambda_powertools.utilities.jmespath_utils import PowertoolsFunctions, extract_data_from_envelope class CustomFunctions(PowertoolsFunctions): - @signature({"types": ["string"]}) # Only decode if value is a string - def _func_special_decoder(self, payload: str): - payload_decode = base64.b64decode(payload) - return payload_decode + # only decode if value is a string + # see supported data types: https://jmespath.org/specification.html#built-in-functions + @signature({"types": ["string"]}) + def _func_decode_snappy_compression(self, payload: str): + decoded = base64.b64decode(payload) + # uncompressing snappy messages - very common compression in Kafka + uncompressed = snappy.uncompress(decoded) + return uncompressed custom_jmespath_options = {"custom_functions": CustomFunctions()} -def handler(event, context): - # use the custom name after `_func_` - extract_data_from_envelope(data=event, envelope="special_decoder(body)", jmespath_options=custom_jmespath_options) - ... +def lambda_handler(event, context) -> dict: + + try: + logs: list = [] + # use the prefix `_func_` before the name of the function + logs.append( + extract_data_from_envelope( + data=event, + envelope="Records[*].decode_snappy_compression(log)", + jmespath_options=custom_jmespath_options, + ) + ) + return {"logs": logs, "message": "Extracted messages", "success": True} + except JMESPathTypeError: + return return_error_message("The envelope function must match a valid path.") + except snappy.UncompressError: + return return_error_message("Log must be valid snappy compressed binary") + except binascii.Error: + return return_error_message("Log must be valid base64 encoded string") + + +def return_error_message(message: str) -> dict: + return {"logs": None, "message": message, "success": False} From c8cba954384d8fba4f564b6bcec7a22b9aacf71e Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 9 Aug 2022 14:29:33 +0200 Subject: [PATCH 18/29] docs(jmespath): fix highlighting --- docs/utilities/jmespath_functions.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/utilities/jmespath_functions.md b/docs/utilities/jmespath_functions.md index 69e130ae334..108c0ddf5d0 100644 --- a/docs/utilities/jmespath_functions.md +++ b/docs/utilities/jmespath_functions.md @@ -63,16 +63,16 @@ We provide built-in envelopes for popular JMESPath expressions used when looking These are all built-in envelopes you can use along with their expression as a reference: -Envelope | JMESPath expression -------------------------------------------------- | --------------------------------------------------------------------------------- -**`API_GATEWAY_REST`** | `powertools_json(body)` -**`API_GATEWAY_HTTP`** | `API_GATEWAY_REST` -**`SQS`** | `Records[*].powertools_json(body)` -**`SNS`** | `Records[0].Sns.Message | powertools_json(@)` -**`EVENTBRIDGE`** | `detail` -**`CLOUDWATCH_EVENTS_SCHEDULED`** | `EVENTBRIDGE` -**`KINESIS_DATA_STREAM`** | `Records[*].kinesis.powertools_json(powertools_base64(data))` -**`CLOUDWATCH_LOGS`** | `awslogs.powertools_base64_gzip(data) | powertools_json(@).logEvents[*]` +| Envelope | JMESPath expression | +| --------------------------------- | ------------------------------------------------------------- | +| **`API_GATEWAY_REST`** | `powertools_json(body)` | +| **`API_GATEWAY_HTTP`** | `API_GATEWAY_REST` | +| **`SQS`** | `Records[*].powertools_json(body)` | +| **`SNS`** | `Records[0].Sns.Message | powertools_json(@)` | +| **`EVENTBRIDGE`** | `detail` | +| **`CLOUDWATCH_EVENTS_SCHEDULED`** | `EVENTBRIDGE` | +| **`KINESIS_DATA_STREAM`** | `Records[*].kinesis.powertools_json(powertools_base64(data))` | +| **`CLOUDWATCH_LOGS`** | `awslogs.powertools_base64_gzip(data) | powertools_json(@).logEvents[*]` | ## Advanced @@ -93,7 +93,7 @@ This sample will decode the value within the `data` key into a valid JSON before === "powertools_json_jmespath_function.py" - ```python hl_lines="5 8 34 42 44 46" + ```python hl_lines="5 8 35 43 45 47" --8<-- "examples/jmespath_functions/src/powertools_json_jmespath_function.py" ``` From b49c3f47f4bfdc87f2dc9691d68da91fc47c0c02 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 9 Aug 2022 14:52:20 +0200 Subject: [PATCH 19/29] docs(jmespath): simplify order factory --- .../src/powertools_json_jmespath_function.py | 30 ++++++------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/examples/jmespath_functions/src/powertools_json_jmespath_function.py b/examples/jmespath_functions/src/powertools_json_jmespath_function.py index f7bf628e2dd..241981b6dad 100644 --- a/examples/jmespath_functions/src/powertools_json_jmespath_function.py +++ b/examples/jmespath_functions/src/powertools_json_jmespath_function.py @@ -1,5 +1,5 @@ import json -from dataclasses import asdict, dataclass, is_dataclass +from dataclasses import asdict, dataclass, field, is_dataclass from uuid import uuid4 import powertools_json_jmespath_schema as schemas @@ -11,12 +11,12 @@ @dataclass class Order: - order_id: str user_id: int product_id: int quantity: int price: float currency: str + order_id: str = field(default_factory=lambda: f"{uuid4()}") class DataclassCustomEncoder(json.JSONEncoder): @@ -29,15 +29,17 @@ def default(self, obj): def lambda_handler(event, context: LambdaContext) -> dict: - - # Let's validate the schema try: + # Validate order against our schema validate(event=event, schema=schemas.INPUT, envelope="powertools_json(payload)") - order = create_order(json.loads(event["payload"])) + # Deserialize JSON string order as dict + # alternatively, extract_data_from_envelope works here too + order_payload: dict = json.loads(event.get("payload")) + return { - "order": json.dumps(order.get("order"), cls=DataclassCustomEncoder), - "message": order.get("message"), + "order": json.dumps(Order(**order_payload), cls=DataclassCustomEncoder), + "message": "order created", "success": True, } except JMESPathTypeError: @@ -50,19 +52,5 @@ def lambda_handler(event, context: LambdaContext) -> dict: return return_error_message(str(exception)) -def create_order(event: dict) -> dict: - # This return an Order dataclass - order_id = str(uuid4()) - order = Order( - order_id=order_id, - user_id=event.get("user_id"), - product_id=event.get("product_id"), - quantity=event.get("quantity"), - price=event.get("price"), - currency=event.get("currency"), - ) - return {"order": order, "message": "order created"} - - def return_error_message(message: str) -> dict: return {"order": None, "message": message, "success": False} From 1a6a344749772b328e655355870f20903cc0477c Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 9 Aug 2022 14:54:55 +0200 Subject: [PATCH 20/29] docs(jmespath): simplify wording --- .../src/powertools_json_jmespath_function.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/jmespath_functions/src/powertools_json_jmespath_function.py b/examples/jmespath_functions/src/powertools_json_jmespath_function.py index 241981b6dad..879583c46ac 100644 --- a/examples/jmespath_functions/src/powertools_json_jmespath_function.py +++ b/examples/jmespath_functions/src/powertools_json_jmespath_function.py @@ -47,8 +47,7 @@ def lambda_handler(event, context: LambdaContext) -> dict: except json.JSONDecodeError: return return_error_message("Payload must be valid JSON (base64 encoded).") except SchemaValidationError as exception: - - # if validation fails, a SchemaValidationError will be raised with the wrong fields + # SchemaValidationError indicates where a data mismatch is return return_error_message(str(exception)) From da920a8a7eb77dd658e712f9785aed4d5952faa9 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 9 Aug 2022 14:55:43 +0200 Subject: [PATCH 21/29] docs(jmespath): correct json encoder wording --- .../jmespath_functions/src/powertools_json_jmespath_function.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/jmespath_functions/src/powertools_json_jmespath_function.py b/examples/jmespath_functions/src/powertools_json_jmespath_function.py index 879583c46ac..d582da7e659 100644 --- a/examples/jmespath_functions/src/powertools_json_jmespath_function.py +++ b/examples/jmespath_functions/src/powertools_json_jmespath_function.py @@ -24,7 +24,7 @@ class DataclassCustomEncoder(json.JSONEncoder): def default(self, obj): # Only called for values that aren't JSON serializable - # where `obj` will be an instance of Todo in this example + # where `obj` will be an instance of Order in this example return asdict(obj) if is_dataclass(obj) else super().default(obj) From 9b6ae91332b09e852aac065520a919b5fc54920d Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 9 Aug 2022 15:00:19 +0200 Subject: [PATCH 22/29] docs(jmespath): safer return --- .../src/powertools_json_jmespath_function.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/jmespath_functions/src/powertools_json_jmespath_function.py b/examples/jmespath_functions/src/powertools_json_jmespath_function.py index d582da7e659..8ddee03ca48 100644 --- a/examples/jmespath_functions/src/powertools_json_jmespath_function.py +++ b/examples/jmespath_functions/src/powertools_json_jmespath_function.py @@ -43,7 +43,8 @@ def lambda_handler(event, context: LambdaContext) -> dict: "success": True, } except JMESPathTypeError: - return return_error_message("The powertools_json() envelope function must match a valid path.") + # The powertools_json() envelope function must match a valid path + return return_error_message("Invalid request.") except json.JSONDecodeError: return return_error_message("Payload must be valid JSON (base64 encoded).") except SchemaValidationError as exception: From d77cd80629b52f711d1d397ece75523eec342cc0 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 9 Aug 2022 15:01:45 +0200 Subject: [PATCH 23/29] docs(jmespath): exception order --- .../src/powertools_json_jmespath_function.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/jmespath_functions/src/powertools_json_jmespath_function.py b/examples/jmespath_functions/src/powertools_json_jmespath_function.py index 8ddee03ca48..5eae585c0c1 100644 --- a/examples/jmespath_functions/src/powertools_json_jmespath_function.py +++ b/examples/jmespath_functions/src/powertools_json_jmespath_function.py @@ -45,11 +45,11 @@ def lambda_handler(event, context: LambdaContext) -> dict: except JMESPathTypeError: # The powertools_json() envelope function must match a valid path return return_error_message("Invalid request.") - except json.JSONDecodeError: - return return_error_message("Payload must be valid JSON (base64 encoded).") except SchemaValidationError as exception: # SchemaValidationError indicates where a data mismatch is return return_error_message(str(exception)) + except json.JSONDecodeError: + return return_error_message("Payload must be valid JSON (base64 encoded).") def return_error_message(message: str) -> dict: From 87d8b177db743c70ea06cb30da9334bb7e3aa31e Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 9 Aug 2022 15:03:04 +0200 Subject: [PATCH 24/29] docs(jmespath): adjust highlighting after refactor --- docs/utilities/jmespath_functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/utilities/jmespath_functions.md b/docs/utilities/jmespath_functions.md index 108c0ddf5d0..41c0991a1b8 100644 --- a/docs/utilities/jmespath_functions.md +++ b/docs/utilities/jmespath_functions.md @@ -93,7 +93,7 @@ This sample will decode the value within the `data` key into a valid JSON before === "powertools_json_jmespath_function.py" - ```python hl_lines="5 8 35 43 45 47" + ```python hl_lines="5 8 34 45 48 51" --8<-- "examples/jmespath_functions/src/powertools_json_jmespath_function.py" ``` From 9c4eb0490d223fa6c1ec1d50e270107fceab3308 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 9 Aug 2022 15:13:28 +0200 Subject: [PATCH 25/29] docs(jmespath): add exception and dummy endpoint to use fn arguments --- docs/utilities/jmespath_functions.md | 2 +- .../powertools_json_idempotency_jmespath.py | 21 +++++++++++++++---- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/docs/utilities/jmespath_functions.md b/docs/utilities/jmespath_functions.md index 41c0991a1b8..673095f84a1 100644 --- a/docs/utilities/jmespath_functions.md +++ b/docs/utilities/jmespath_functions.md @@ -115,7 +115,7 @@ This sample will decode the value within the `body` key of an API Gateway event === "powertools_json_idempotency_jmespath.py" - ```python hl_lines="9" + ```python hl_lines="12" --8<-- "examples/jmespath_functions/src/powertools_json_idempotency_jmespath.py" ``` diff --git a/examples/jmespath_functions/src/powertools_json_idempotency_jmespath.py b/examples/jmespath_functions/src/powertools_json_idempotency_jmespath.py index 399dcb40fc6..15880dedff3 100644 --- a/examples/jmespath_functions/src/powertools_json_idempotency_jmespath.py +++ b/examples/jmespath_functions/src/powertools_json_idempotency_jmespath.py @@ -1,4 +1,7 @@ import json +from uuid import uuid4 + +import requests from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer, IdempotencyConfig, idempotent @@ -9,13 +12,23 @@ config = IdempotencyConfig(event_key_jmespath="powertools_json(body)") +class PaymentError(Exception): + ... + + @idempotent(config=config, persistence_store=persistence_layer) def handler(event, context) -> dict: body = json.loads(event["body"]) - payment = create_subscription_payment(user=body["user"], product_id=body["product_id"]) - ... - return {"payment_id": payment.id, "message": "success", "statusCode": 200} + try: + payment = create_subscription_payment(user=body["user"], product_id=body["product_id"]) + return {"payment_id": payment.id, "message": "success", "statusCode": 200} + except requests.HTTPError as e: + raise PaymentError("Unable to create payment subscription") from e def create_subscription_payment(user: str, product_id: str) -> dict: - return {"id": "1", "message": "paid"} + payload = {"user": user, "product_id": product_id} + ret: requests.Response = requests.post(url="https://httpbin.org/anything", data=payload) + ret.raise_for_status() + + return {"id": f"{uuid4()}", "message": "paid"} From 05f4a4bf34716ab8b97a90986c67f8c63bc86f79 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 9 Aug 2022 15:20:59 +0200 Subject: [PATCH 26/29] docs(jmespath): sync base64 with powertools_json refactor --- docs/utilities/jmespath_functions.md | 2 +- .../powertools_base64_jmespath_function.py | 28 ++++++------------- 2 files changed, 9 insertions(+), 21 deletions(-) diff --git a/docs/utilities/jmespath_functions.md b/docs/utilities/jmespath_functions.md index 673095f84a1..f807818ad4a 100644 --- a/docs/utilities/jmespath_functions.md +++ b/docs/utilities/jmespath_functions.md @@ -133,7 +133,7 @@ This sample will decode the base64 value within the `data` key, and decode the J === "powertools_base64_jmespath_function.py" - ```python hl_lines="7 10 36 46 48 50 51" + ```python hl_lines="7 10 37 48 52 54 56" --8<-- "examples/jmespath_functions/src/powertools_base64_jmespath_function.py" ``` diff --git a/examples/jmespath_functions/src/powertools_base64_jmespath_function.py b/examples/jmespath_functions/src/powertools_base64_jmespath_function.py index e2f24d97434..9d86401ee6e 100644 --- a/examples/jmespath_functions/src/powertools_base64_jmespath_function.py +++ b/examples/jmespath_functions/src/powertools_base64_jmespath_function.py @@ -1,7 +1,7 @@ import base64 import binascii import json -from dataclasses import asdict, dataclass, is_dataclass +from dataclasses import asdict, dataclass, field, is_dataclass from uuid import uuid4 import powertools_base64_jmespath_schema as schemas @@ -13,12 +13,12 @@ @dataclass class Order: - order_id: str user_id: int product_id: int quantity: int price: float currency: str + order_id: str = field(default_factory=lambda: f"{uuid4()}") class DataclassCustomEncoder(json.JSONEncoder): @@ -36,12 +36,14 @@ def lambda_handler(event, context: LambdaContext) -> dict: try: validate(event=event, schema=schemas.INPUT, envelope="powertools_json(powertools_base64(payload))") + # alternatively, extract_data_from_envelope works here too payload_decoded = base64.b64decode(event["payload"]).decode() - order = create_order(json.loads(payload_decoded)) + order_payload: dict = json.loads(payload_decoded) + return { - "order": json.dumps(order.get("order"), cls=DataclassCustomEncoder), - "message": order.get("message"), + "order": json.dumps(Order(**order_payload), cls=DataclassCustomEncoder), + "message": "order created", "success": True, } except JMESPathTypeError: @@ -49,7 +51,7 @@ def lambda_handler(event, context: LambdaContext) -> dict: "The powertools_json(powertools_base64()) envelope function must match a valid path." ) except binascii.Error: - return return_error_message("Payload must be valid base64 encoded string") + return return_error_message("Payload must be a valid base64 encoded string") except json.JSONDecodeError: return return_error_message("Payload must be valid JSON (base64 encoded).") except SchemaValidationError as exception: @@ -57,19 +59,5 @@ def lambda_handler(event, context: LambdaContext) -> dict: return return_error_message(str(exception)) -def create_order(event: dict) -> dict: - # This return an Order dataclass - order_id = str(uuid4()) - order = Order( - order_id=order_id, - user_id=event.get("user_id"), - product_id=event.get("product_id"), - quantity=event.get("quantity"), - price=event.get("price"), - currency=event.get("currency"), - ) - return {"order": order, "message": "order created"} - - def return_error_message(message: str) -> dict: return {"order": None, "message": message, "success": False} From aec1e71d21b722fcb48a791cc62737c4eee83f38 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 9 Aug 2022 15:31:02 +0200 Subject: [PATCH 27/29] docs(jmespath): simplify base64 gzip example --- docs/utilities/jmespath_functions.md | 2 +- ...owertools_base64_gzip_jmespath_function.py | 21 +++++++------------ .../powertools_base64_jmespath_function.py | 2 +- 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/docs/utilities/jmespath_functions.md b/docs/utilities/jmespath_functions.md index f807818ad4a..a21b7f1ec09 100644 --- a/docs/utilities/jmespath_functions.md +++ b/docs/utilities/jmespath_functions.md @@ -157,7 +157,7 @@ This sample will decompress and decode base64 data from Cloudwatch Logs, then us === "powertools_base64_gzip_jmespath_function.py" - ```python hl_lines="6 10 17 28 30 32 34 35" + ```python hl_lines="6 10 15 29 31 33 35" --8<-- "examples/jmespath_functions/src/powertools_base64_gzip_jmespath_function.py" ``` diff --git a/examples/jmespath_functions/src/powertools_base64_gzip_jmespath_function.py b/examples/jmespath_functions/src/powertools_base64_gzip_jmespath_function.py index cdd0752a0e8..cff3424b487 100644 --- a/examples/jmespath_functions/src/powertools_base64_gzip_jmespath_function.py +++ b/examples/jmespath_functions/src/powertools_base64_gzip_jmespath_function.py @@ -11,38 +11,31 @@ def lambda_handler(event, context: LambdaContext) -> dict: - - # Try to validate the schema try: validate(event=event, schema=schemas.INPUT, envelope="powertools_base64_gzip(payload) | powertools_json(@)") + # Alternatively, extract_data_from_envelope works here too encoded_payload = base64.b64decode(event["payload"]) uncompressed_payload = gzip.decompress(encoded_payload).decode() + log: dict = json.loads(uncompressed_payload) - logs = process_cloudwatch_logs(json.loads(uncompressed_payload)) return { - "message": logs, + "message": "Logs processed", + "log_group": log.get("logGroup"), + "owner": log.get("owner"), "success": True, } except JMESPathTypeError: return return_error_message("The powertools_base64_gzip() envelope function must match a valid path.") except binascii.Error: - return return_error_message("Payload must be valid base64 encoded string") + return return_error_message("Payload must be a valid base64 encoded string") except json.JSONDecodeError: return return_error_message("Payload must be valid JSON (base64 encoded).") except SchemaValidationError as exception: - # if validation fails, a SchemaValidationError will be raised with the wrong fields + # SchemaValidationError indicates where a data mismatch is return return_error_message(str(exception)) def return_error_message(message: str) -> dict: return {"message": message, "success": False} - - -def process_cloudwatch_logs(event: dict) -> str: - owner = event.get("owner") # noqa F841 - log_group = event.get("logGroup") # noqa F841 - # Add your logic here - ... - return "Logs processed" diff --git a/examples/jmespath_functions/src/powertools_base64_jmespath_function.py b/examples/jmespath_functions/src/powertools_base64_jmespath_function.py index 9d86401ee6e..ba7208298a0 100644 --- a/examples/jmespath_functions/src/powertools_base64_jmespath_function.py +++ b/examples/jmespath_functions/src/powertools_base64_jmespath_function.py @@ -55,7 +55,7 @@ def lambda_handler(event, context: LambdaContext) -> dict: except json.JSONDecodeError: return return_error_message("Payload must be valid JSON (base64 encoded).") except SchemaValidationError as exception: - # if validation fails, a SchemaValidationError will be raised with the wrong fields + # SchemaValidationError indicates where a data mismatch is return return_error_message(str(exception)) From d673e58c18f3cdcfb698ab23d470c6e4c1fb5ef3 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 9 Aug 2022 15:41:46 +0200 Subject: [PATCH 28/29] docs(jmespath): emphasize important parts in jmespath example --- docs/utilities/jmespath_functions.md | 6 +++--- .../src/powertools_custom_jmespath_function.py | 14 ++++++-------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/docs/utilities/jmespath_functions.md b/docs/utilities/jmespath_functions.md index a21b7f1ec09..381bfb988db 100644 --- a/docs/utilities/jmespath_functions.md +++ b/docs/utilities/jmespath_functions.md @@ -178,13 +178,13 @@ This sample will decompress and decode base64 data from Cloudwatch Logs, then us ???+ warning This should only be used for advanced use cases where you have special formats not covered by the built-in functions. -For special binary formats that you want to decode before applying JSON Schema validation, you can bring your own [JMESPath function](https://github.com/jmespath/jmespath.py#custom-functions){target="_blank"} and any additional option via `jmespath_options` param. +For special binary formats that you want to decode before applying JSON Schema validation, you can bring your own [JMESPath function](https://github.com/jmespath/jmespath.py#custom-functions){target="_blank"} and any additional option via `jmespath_options` param. To keep Powertools built-in functions, you can subclass from `PowertoolsFunctions`. -In order to keep the built-in functions from Powertools, you can subclass from `PowertoolsFunctions`. Here is an example of how to unzip compressed messages using snappy: +Here is an example of how to decompress messages using [snappy](https://github.com/andrix/python-snappy){target="_blank"}: === "powertools_custom_jmespath_function.py" - ```python hl_lines="4 5 8 11 15 18 22 31-35 38 40 42" + ```python hl_lines="8 11 14-15 20 31 36 38 40" --8<-- "examples/jmespath_functions/src/powertools_custom_jmespath_function.py" ``` diff --git a/examples/jmespath_functions/src/powertools_custom_jmespath_function.py b/examples/jmespath_functions/src/powertools_custom_jmespath_function.py index f805237db76..71fdecd0db2 100644 --- a/examples/jmespath_functions/src/powertools_custom_jmespath_function.py +++ b/examples/jmespath_functions/src/powertools_custom_jmespath_function.py @@ -13,10 +13,8 @@ class CustomFunctions(PowertoolsFunctions): # see supported data types: https://jmespath.org/specification.html#built-in-functions @signature({"types": ["string"]}) def _func_decode_snappy_compression(self, payload: str): - decoded = base64.b64decode(payload) - # uncompressing snappy messages - very common compression in Kafka - uncompressed = snappy.uncompress(decoded) - return uncompressed + decoded: bytes = base64.b64decode(payload) + return snappy.uncompress(decoded) custom_jmespath_options = {"custom_functions": CustomFunctions()} @@ -25,11 +23,11 @@ def _func_decode_snappy_compression(self, payload: str): def lambda_handler(event, context) -> dict: try: - logs: list = [] - # use the prefix `_func_` before the name of the function + logs = [] logs.append( extract_data_from_envelope( data=event, + # NOTE: Use the prefix `_func_` before the name of the function envelope="Records[*].decode_snappy_compression(log)", jmespath_options=custom_jmespath_options, ) @@ -38,9 +36,9 @@ def lambda_handler(event, context) -> dict: except JMESPathTypeError: return return_error_message("The envelope function must match a valid path.") except snappy.UncompressError: - return return_error_message("Log must be valid snappy compressed binary") + return return_error_message("Log must be a valid snappy compressed binary") except binascii.Error: - return return_error_message("Log must be valid base64 encoded string") + return return_error_message("Log must be a valid base64 encoded string") def return_error_message(message: str) -> dict: From e38f2a08f54b9a44dc4a64fca6a361d3f9d39996 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 9 Aug 2022 16:03:52 +0200 Subject: [PATCH 29/29] docs(jmespath): address old typos and wording --- docs/utilities/jmespath_functions.md | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/docs/utilities/jmespath_functions.md b/docs/utilities/jmespath_functions.md index 381bfb988db..45250ea0fcd 100644 --- a/docs/utilities/jmespath_functions.md +++ b/docs/utilities/jmespath_functions.md @@ -14,6 +14,7 @@ Built-in [JMESPath](https://jmespath.org/){target="_blank"} Functions to easily * Deserialize JSON from JSON strings, base64, and compressed data * Use JMESPath to extract and combine data recursively +* Provides commonly used JMESPath expression with popular event sources ## Getting started @@ -22,14 +23,14 @@ Built-in [JMESPath](https://jmespath.org/){target="_blank"} Functions to easily You might have events that contains encoded JSON payloads as string, base64, or even in compressed format. It is a common use case to decode and extract them partially or fully as part of your Lambda function invocation. -Lambda Powertools also have utilities like [validation](validation.md), [idempotency](idempotency.md), or [feature flags](feature_flags.md) where you might need to extract a portion of your data before using them. +Powertools also have utilities like [validation](validation.md), [idempotency](idempotency.md), or [feature flags](feature_flags.md) where you might need to extract a portion of your data before using them. -???+ info - **Envelope** is the terminology we use for the JMESPath expression to extract your JSON object from your data input. +???+ info "Terminology" + **Envelope** is the terminology we use for the **JMESPath expression** to extract your JSON object from your data input. We might use those two terms interchangeably. ### Extracting data -You can use the `extract_data_from_envelope` function along with any [JMESPath expression](https://jmespath.org/tutorial.html){target="_blank"}. +You can use the `extract_data_from_envelope` function with any [JMESPath expression](https://jmespath.org/tutorial.html){target="_blank"}. ???+ tip Another common use case is to fetch deeply nested data, filter, flatten, and more. @@ -47,7 +48,7 @@ You can use the `extract_data_from_envelope` function along with any [JMESPath e ### Built-in envelopes -We provide built-in envelopes for popular JMESPath expressions used when looking to decode/deserialize JSON objects within AWS Lambda Event Sources. +We provide built-in envelopes for popular AWS Lambda event sources to easily decode and/or deserialize JSON objects. === "extract_data_from_builtin_envelope.py" @@ -78,18 +79,18 @@ These are all built-in envelopes you can use along with their expression as a re ### Built-in JMESPath functions -You can use our built-in JMESPath functions within your expressions to do exactly that to decode JSON Strings, base64, and uncompress gzip data. +You can use our built-in JMESPath functions within your envelope expression. They handle deserialization for common data formats found in AWS Lambda event sources such as JSON strings, base64, and uncompress gzip data. ???+ info - We use these for built-in envelopes to easily decode and unwrap events from sources like API Gateway, Kinesis, CloudWatch Logs, etc. + We use these everywhere in Powertools to easily decode and unwrap events from Amazon API Gateway, Amazon Kinesis, AWS CloudWatch Logs, etc. #### powertools_json function -Use `powertools_json` function to decode any JSON String anywhere a JMESPath expression is allowed. +Use `powertools_json` function to decode any JSON string anywhere a JMESPath expression is allowed. > **Validation scenario** -This sample will decode the value within the `data` key into a valid JSON before we can validate it. +This sample will deserialize the JSON string within the `data` key before validation. === "powertools_json_jmespath_function.py" @@ -111,7 +112,7 @@ This sample will decode the value within the `data` key into a valid JSON before > **Idempotency scenario** -This sample will decode the value within the `body` key of an API Gateway event into a valid JSON object to ensure the Idempotency utility processes a JSON object instead of a string. +This sample will deserialize the JSON string within the `body` key before [Idempotency](./idempotency.md){target="_blank"} processes it. === "powertools_json_idempotency_jmespath.py" @@ -129,7 +130,7 @@ This sample will decode the value within the `body` key of an API Gateway event Use `powertools_base64` function to decode any base64 data. -This sample will decode the base64 value within the `data` key, and decode the JSON string into a valid JSON before we can validate it. +This sample will decode the base64 value within the `data` key, and deserialize the JSON string before validation. === "powertools_base64_jmespath_function.py"