From fce12c1a1713b22b86fd8011b6a2caee409beb22 Mon Sep 17 00:00:00 2001 From: walmsles <2704782+walmsles@users.noreply.github.com> Date: Sat, 28 Aug 2021 11:09:29 +1000 Subject: [PATCH 1/4] Split out JMESPath Functions to speerate page under Utilities since relevant to multiple areas in Powertools --- docs/utilities/jmespath_functions.md | 166 +++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 docs/utilities/jmespath_functions.md diff --git a/docs/utilities/jmespath_functions.md b/docs/utilities/jmespath_functions.md new file mode 100644 index 00000000000..f20fd5c9d66 --- /dev/null +++ b/docs/utilities/jmespath_functions.md @@ -0,0 +1,166 @@ +--- +title: JMESPath Functions +description: Utility +--- + +You might have events or responses that contain non-encoded JSON, where you need to decode so that you can access portions of the object or ensure the Powertools utility recieves a JSON object + +## Built-in JMESPath functions +You can use our built-in JMESPath functions within your expressions to do exactly that to decode JSON Strings, base64, and uncompress gzip data. + +!!! info + We use these for built-in envelopes to easily decode and unwrap events from sources like API Gateway, Kinesis, CloudWatch Logs, etc. + +#### powertools_json function + +Use `powertools_json` function to decode any JSON String anywhere a JMESPath expression is allowed. + +> **Validation scenario** + +This sample will decode the value within the `data` key into a valid JSON before we can validate it. + +=== "powertools_json_jmespath_function.py" + + ```python hl_lines="9" + from aws_lambda_powertools.utilities.validation import validate + + import schemas + + sample_event = { + 'data': '{"payload": {"message": "hello hello", "username": "blah blah"}}' + } + + validate(event=sample_event, schema=schemas.INPUT, envelope="powertools_json(data)") + ``` + +=== "schemas.py" + + ```python hl_lines="7 14 16 23 39 45 47 52" + --8<-- "docs/shared/validation_basic_jsonschema.py" + ``` + +> **Idempotency scenario** + +This sample will decode the value within the `body` key of an API Gateway event into a valid JSON object to ensure the Idempotency utility processes a JSON object instead of a string. + +=== "powertools_json_jmespath_function.py" + + ```python hl_lines="8" + import json + from aws_lambda_powertools.utilities.idempotency import ( + IdempotencyConfig, DynamoDBPersistenceLayer, idempotent + ) + + persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") + + config = IdempotencyConfig(event_key_jmespath="powertools_json(body)") + @idempotent(config=config, persistence_store=persistence_layer) + def handler(event:APIGatewayProxyEvent, context): + body = json.loads(event['body']) + payment = create_subscription_payment( + user=body['user'], + product=body['product_id'] + ) + ... + return { + "payment_id": payment.id, + "message": "success", + "statusCode": 200 + } + ``` + +#### powertools_base64 function + +Use `powertools_base64` function to decode any base64 data. + +This sample will decode the base64 value within the `data` key, and decode the JSON string into a valid JSON before we can validate it. + +=== "powertools_json_jmespath_function.py" + + ```python hl_lines="12" + from aws_lambda_powertools.utilities.validation import validate + + import schemas + + sample_event = { + "data": "eyJtZXNzYWdlIjogImhlbGxvIGhlbGxvIiwgInVzZXJuYW1lIjogImJsYWggYmxhaCJ9=" + } + + validate( + event=sample_event, + schema=schemas.INPUT, + envelope="powertools_json(powertools_base64(data))" + ) + ``` + +=== "schemas.py" + + ```python hl_lines="7 14 16 23 39 45 47 52" + --8<-- "docs/shared/validation_basic_jsonschema.py" + ``` + +#### powertools_base64_gzip function + +Use `powertools_base64_gzip` function to decompress and decode base64 data. + +This sample will decompress and decode base64 data, then use JMESPath pipeline expression to pass the result for decoding its JSON string. + +=== "powertools_json_jmespath_function.py" + + ```python hl_lines="12" + from aws_lambda_powertools.utilities.validation import validate + + import schemas + + sample_event = { + "data": "H4sIACZAXl8C/52PzUrEMBhFX2UILpX8tPbHXWHqIOiq3Q1F0ubrWEiakqTWofTdTYYB0YWL2d5zvnuTFellBIOedoiyKH5M0iwnlKH7HZL6dDB6ngLDfLFYctUKjie9gHFaS/sAX1xNEq525QxwFXRGGMEkx4Th491rUZdV3YiIZ6Ljfd+lfSyAtZloacQgAkqSJCGhxM6t7cwwuUGPz4N0YKyvO6I9WDeMPMSo8Z4Ca/kJ6vMEYW5f1MX7W1lVxaG8vqX8hNFdjlc0iCBBSF4ERT/3Pl7RbMGMXF2KZMh/C+gDpNS7RRsp0OaRGzx0/t8e0jgmcczyLCWEePhni/23JWalzjdu0a3ZvgEaNLXeugEAAA==" + } + + validate( + event=sample_event, + schema=schemas.INPUT, + envelope="powertools_base64_gzip(data) | powertools_json(@)" + ) + ``` + +=== "schemas.py" + + ```python hl_lines="7 14 16 23 39 45 47 52" + --8<-- "docs/shared/validation_basic_jsonschema.py" + ``` + +### Bring your own JMESPath function + +!!! warning + This should only be used for advanced use cases where you have special formats not covered by the built-in functions. + + This will **replace all provided built-in functions such as `powertools_json`, so you will no longer be able to use them**. + +For special binary formats that you want to decode before applying JSON Schema validation, you can bring your own [JMESPath function](https://github.com/jmespath/jmespath.py#custom-functions){target="_blank"} and any additional option via `jmespath_options` param. + +=== "custom_jmespath_function.py" + + ```python hl_lines="2 6-10 14" + from aws_lambda_powertools.utilities.validation import validator + from jmespath import functions + + import schemas + + class CustomFunctions(functions.Functions): + + @functions.signature({'types': ['string']}) + def _func_special_decoder(self, s): + return my_custom_decoder_logic(s) + + custom_jmespath_options = {"custom_functions": CustomFunctions()} + + @validator(schema=schemas.INPUT, jmespath_options=**custom_jmespath_options) + def handler(event, context): + return event + ``` + +=== "schemas.py" + + ```python hl_lines="7 14 16 23 39 45 47 52" + --8<-- "docs/shared/validation_basic_jsonschema.py" + ``` From b947e1d467efddf55380d6a17b7c9578af3224d1 Mon Sep 17 00:00:00 2001 From: walmsles <2704782+walmsles@users.noreply.github.com> Date: Sat, 28 Aug 2021 11:14:19 +1000 Subject: [PATCH 2/4] Updated Idempotency examples and highlight importance of JMESPath for JSON APIs --- docs/utilities/idempotency.md | 8 ++- docs/utilities/validation.md | 124 +--------------------------------- mkdocs.yml | 1 + 3 files changed, 9 insertions(+), 124 deletions(-) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 495fe626d4f..0b601ebcba9 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -206,6 +206,11 @@ In this example, we have a Lambda handler that creates a payment for a user subs Imagine the function executes successfully, but the client never receives the response due to a connection issue. It is safe to retry in this instance, as the idempotent decorator will return a previously saved response. +!!! warning "Making JSON APIs Idempotent" + To ensure JSON structured APIs are fully idempotent we must ensure the idempotent key is a valid JSON object. + + To do this we can [customize the default behaviour](#customizing-the-default-behavior) with *event_key_jmespath* and the [JMESPath built-in function](/utilities/jmespath_functions) *powertools_json()*. + === "payment.py" ```python hl_lines="2-4 10 12 15 20" @@ -218,7 +223,7 @@ Imagine the function executes successfully, but the client never receives the re # Treat everything under the "body" key # in the event json object as our payload - config = IdempotencyConfig(event_key_jmespath="body") + config = IdempotencyConfig(event_key_jmespath="powertools_json(body)") @idempotent(config=config, persistence_store=persistence_layer) def handler(event, context): @@ -270,6 +275,7 @@ Imagine the function executes successfully, but the client never receives the re } ``` + ### Idempotency request flow This sequence diagram shows an example flow of what happens in the payment scenario: diff --git a/docs/utilities/validation.md b/docs/utilities/validation.md index 7df339b7503..73f1e085164 100644 --- a/docs/utilities/validation.md +++ b/docs/utilities/validation.md @@ -429,129 +429,7 @@ For each format defined in a dictionary key, you must use a regex, or a function You might have events or responses that contain non-encoded JSON, where you need to decode before validating them. -You can use our built-in JMESPath functions within your expressions to do exactly that to decode JSON Strings, base64, and uncompress gzip data. +You can use our built-in [JMESPath functions](/utilities/jmespath_functions) within your expressions to do exactly that to decode JSON Strings, base64, and uncompress gzip data. !!! info We use these for built-in envelopes to easily to decode and unwrap events from sources like Kinesis, CloudWatch Logs, etc. - -#### powertools_json function - -Use `powertools_json` function to decode any JSON String. - -This sample will decode the value within the `data` key into a valid JSON before we can validate it. - -=== "powertools_json_jmespath_function.py" - - ```python hl_lines="9" - from aws_lambda_powertools.utilities.validation import validate - - import schemas - - sample_event = { - 'data': '{"payload": {"message": "hello hello", "username": "blah blah"}}' - } - - validate(event=sample_event, schema=schemas.INPUT, envelope="powertools_json(data)") - ``` - -=== "schemas.py" - - ```python hl_lines="7 14 16 23 39 45 47 52" - --8<-- "docs/shared/validation_basic_jsonschema.py" - ``` - -#### powertools_base64 function - -Use `powertools_base64` function to decode any base64 data. - -This sample will decode the base64 value within the `data` key, and decode the JSON string into a valid JSON before we can validate it. - -=== "powertools_json_jmespath_function.py" - - ```python hl_lines="12" - from aws_lambda_powertools.utilities.validation import validate - - import schemas - - sample_event = { - "data": "eyJtZXNzYWdlIjogImhlbGxvIGhlbGxvIiwgInVzZXJuYW1lIjogImJsYWggYmxhaCJ9=" - } - - validate( - event=sample_event, - schema=schemas.INPUT, - envelope="powertools_json(powertools_base64(data))" - ) - ``` - -=== "schemas.py" - - ```python hl_lines="7 14 16 23 39 45 47 52" - --8<-- "docs/shared/validation_basic_jsonschema.py" - ``` - -#### powertools_base64_gzip function - -Use `powertools_base64_gzip` function to decompress and decode base64 data. - -This sample will decompress and decode base64 data, then use JMESPath pipeline expression to pass the result for decoding its JSON string. - -=== "powertools_json_jmespath_function.py" - - ```python hl_lines="12" - from aws_lambda_powertools.utilities.validation import validate - - import schemas - - sample_event = { - "data": "H4sIACZAXl8C/52PzUrEMBhFX2UILpX8tPbHXWHqIOiq3Q1F0ubrWEiakqTWofTdTYYB0YWL2d5zvnuTFellBIOedoiyKH5M0iwnlKH7HZL6dDB6ngLDfLFYctUKjie9gHFaS/sAX1xNEq525QxwFXRGGMEkx4Th491rUZdV3YiIZ6Ljfd+lfSyAtZloacQgAkqSJCGhxM6t7cwwuUGPz4N0YKyvO6I9WDeMPMSo8Z4Ca/kJ6vMEYW5f1MX7W1lVxaG8vqX8hNFdjlc0iCBBSF4ERT/3Pl7RbMGMXF2KZMh/C+gDpNS7RRsp0OaRGzx0/t8e0jgmcczyLCWEePhni/23JWalzjdu0a3ZvgEaNLXeugEAAA==" - } - - validate( - event=sample_event, - schema=schemas.INPUT, - envelope="powertools_base64_gzip(data) | powertools_json(@)" - ) - ``` - -=== "schemas.py" - - ```python hl_lines="7 14 16 23 39 45 47 52" - --8<-- "docs/shared/validation_basic_jsonschema.py" - ``` - -### Bring your own JMESPath function - -!!! warning - This should only be used for advanced use cases where you have special formats not covered by the built-in functions. - - This will **replace all provided built-in functions such as `powertools_json`, so you will no longer be able to use them**. - -For special binary formats that you want to decode before applying JSON Schema validation, you can bring your own [JMESPath function](https://github.com/jmespath/jmespath.py#custom-functions){target="_blank"} and any additional option via `jmespath_options` param. - -=== "custom_jmespath_function.py" - - ```python hl_lines="2 6-10 14" - from aws_lambda_powertools.utilities.validation import validator - from jmespath import functions - - import schemas - - class CustomFunctions(functions.Functions): - - @functions.signature({'types': ['string']}) - def _func_special_decoder(self, s): - return my_custom_decoder_logic(s) - - custom_jmespath_options = {"custom_functions": CustomFunctions()} - - @validator(schema=schemas.INPUT, jmespath_options=**custom_jmespath_options) - def handler(event, context): - return event - ``` - -=== "schemas.py" - - ```python hl_lines="7 14 16 23 39 45 47 52" - --8<-- "docs/shared/validation_basic_jsonschema.py" - ``` diff --git a/mkdocs.yml b/mkdocs.yml index 94dc9980cf1..b90ba4376de 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -26,6 +26,7 @@ nav: - utilities/parser.md - utilities/idempotency.md - utilities/feature_flags.md + - utilities/jmespath_functions.md theme: name: material From 4a1ed2326c7a3084c78bbd1ed547ea79025ca2c7 Mon Sep 17 00:00:00 2001 From: walmsles <2704782+walmsles@users.noreply.github.com> Date: Sat, 28 Aug 2021 11:18:02 +1000 Subject: [PATCH 3/4] Add Reference to JMESPath Functions page --- docs/utilities/idempotency.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 0b601ebcba9..8b773b8caa7 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -340,7 +340,7 @@ Idempotent decorator can be further configured with **`IdempotencyConfig`** as s Parameter | Default | Description ------------------------------------------------- | ------------------------------------------------- | --------------------------------------------------------------------------------- -**event_key_jmespath** | `""` | JMESPath expression to extract the idempotency key from the event record +**event_key_jmespath** | `""` | JMESPath expression to extract the idempotency key from the event record using [built-in functions](/utilities/jmespath_functions) **payload_validation_jmespath** | `""` | JMESPath expression to validate whether certain parameters have changed in the event while the event payload **raise_on_no_idempotency_key** | `False` | Raise exception if no idempotency key was found in the request **expires_after_seconds** | 3600 | The number of seconds to wait before a record is expired From da73191c88f0e972f7306199a2ce9399bd7f0d16 Mon Sep 17 00:00:00 2001 From: walmsles <2704782+walmsles@users.noreply.github.com> Date: Mon, 27 Sep 2021 19:50:45 +1000 Subject: [PATCH 4/4] Apply suggestions from code review Apply suggestions from code review Co-authored-by: Tom McCarthy --- docs/utilities/idempotency.md | 6 +++--- docs/utilities/jmespath_functions.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 8b773b8caa7..a9a5a129e63 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -206,10 +206,10 @@ In this example, we have a Lambda handler that creates a payment for a user subs Imagine the function executes successfully, but the client never receives the response due to a connection issue. It is safe to retry in this instance, as the idempotent decorator will return a previously saved response. -!!! warning "Making JSON APIs Idempotent" - To ensure JSON structured APIs are fully idempotent we must ensure the idempotent key is a valid JSON object. +!!! warning "Idempotency for JSON payloads" + The payload extracted by the `event_key_jmespath` is treated as a string by default, so will be sensitive to differences in whitespace even when the JSON payload itself is identical. - To do this we can [customize the default behaviour](#customizing-the-default-behavior) with *event_key_jmespath* and the [JMESPath built-in function](/utilities/jmespath_functions) *powertools_json()*. + To alter this behaviour, we can use the [JMESPath built-in function](/utilities/jmespath_functions) *powertools_json()* to treat the payload as a JSON object rather than a string. === "payment.py" diff --git a/docs/utilities/jmespath_functions.md b/docs/utilities/jmespath_functions.md index f20fd5c9d66..7ef6b2b32b2 100644 --- a/docs/utilities/jmespath_functions.md +++ b/docs/utilities/jmespath_functions.md @@ -3,7 +3,7 @@ title: JMESPath Functions description: Utility --- -You might have events or responses that contain non-encoded JSON, where you need to decode so that you can access portions of the object or ensure the Powertools utility recieves a JSON object +You might have events or responses that contain non-encoded JSON, where you need to decode so that you can access portions of the object or ensure the Powertools utility receives a JSON object. This is a common use case when using the [validation](/utilities/validation) or [idempotency](/utilities/idempotency) utilities. ## Built-in JMESPath functions You can use our built-in JMESPath functions within your expressions to do exactly that to decode JSON Strings, base64, and uncompress gzip data.