From d4270f1d39030875ef791886ac94029b0734fd6f Mon Sep 17 00:00:00 2001 From: Daniel Chen Date: Thu, 13 Jun 2024 17:46:26 +0000 Subject: [PATCH 01/10] feat: add support for fastjsonschema's handlers param in validation utilities --- .../utilities/validation/base.py | 9 +++- .../utilities/validation/validator.py | 19 +++++-- tests/functional/validator/conftest.py | 52 +++++++++++++++++++ tests/functional/validator/test_validator.py | 4 ++ 4 files changed, 79 insertions(+), 5 deletions(-) diff --git a/aws_lambda_powertools/utilities/validation/base.py b/aws_lambda_powertools/utilities/validation/base.py index 61d692d7f28..30880362c79 100644 --- a/aws_lambda_powertools/utilities/validation/base.py +++ b/aws_lambda_powertools/utilities/validation/base.py @@ -8,7 +8,9 @@ logger = logging.getLogger(__name__) -def validate_data_against_schema(data: Union[Dict, str], schema: Dict, formats: Optional[Dict] = None): +def validate_data_against_schema( + data: Union[Dict, str], schema: Dict, formats: Optional[Dict] = None, handlers: Optional[Dict] = None +): """Validate dict data against given JSON Schema Parameters @@ -19,6 +21,8 @@ def validate_data_against_schema(data: Union[Dict, str], schema: Dict, formats: JSON Schema to validate against formats: Dict Custom formats containing a key (e.g. int64) and a value expressed as regex or callback returning bool + handlers: Dict + Custom methods to retrieve remote schemes, keyed off of URI scheme Raises ------ @@ -29,7 +33,8 @@ def validate_data_against_schema(data: Union[Dict, str], schema: Dict, formats: """ try: formats = formats or {} - fastjsonschema.validate(definition=schema, data=data, formats=formats) + handlers = handlers or {} + fastjsonschema.validate(definition=schema, data=data, formats=formats, handlers=handlers) except (TypeError, AttributeError, fastjsonschema.JsonSchemaDefinitionException) as e: raise InvalidSchemaFormatError(f"Schema received: {schema}, Formats: {formats}. Error: {e}") except fastjsonschema.JsonSchemaValueException as e: diff --git a/aws_lambda_powertools/utilities/validation/validator.py b/aws_lambda_powertools/utilities/validation/validator.py index 968656ee49c..473f2684c7d 100644 --- a/aws_lambda_powertools/utilities/validation/validator.py +++ b/aws_lambda_powertools/utilities/validation/validator.py @@ -16,8 +16,10 @@ def validator( context: Any, inbound_schema: Optional[Dict] = None, inbound_formats: Optional[Dict] = None, + inbound_handlers: Optional[Dict] = None, outbound_schema: Optional[Dict] = None, outbound_formats: Optional[Dict] = None, + outbound_handlers: Optional[Dict] = None, envelope: str = "", jmespath_options: Optional[Dict] = None, **kwargs: Any, @@ -44,6 +46,10 @@ def validator( Custom formats containing a key (e.g. int64) and a value expressed as regex or callback returning bool outbound_formats: Dict Custom formats containing a key (e.g. int64) and a value expressed as regex or callback returning bool + inbound_handlers: Dict + Custom methods to retrieve remote schemes, keyed off of URI scheme + outbound_handlers: Dict + Custom methods to retrieve remote schemes, keyed off of URI scheme Example ------- @@ -127,13 +133,17 @@ def handler(event, context): if inbound_schema: logger.debug("Validating inbound event") - validate_data_against_schema(data=event, schema=inbound_schema, formats=inbound_formats) + validate_data_against_schema( + data=event, schema=inbound_schema, formats=inbound_formats, handlers=inbound_handlers + ) response = handler(event, context, **kwargs) if outbound_schema: logger.debug("Validating outbound event") - validate_data_against_schema(data=response, schema=outbound_schema, formats=outbound_formats) + validate_data_against_schema( + data=response, schema=outbound_schema, formats=outbound_formats, handlers=outbound_handlers + ) return response @@ -142,6 +152,7 @@ def validate( event: Any, schema: Dict, formats: Optional[Dict] = None, + handlers: Optional[Dict] = None, envelope: Optional[str] = None, jmespath_options: Optional[Dict] = None, ): @@ -161,6 +172,8 @@ def validate( Alternative JMESPath options to be included when filtering expr formats: Dict Custom formats containing a key (e.g. int64) and a value expressed as regex or callback returning bool + handlers: Dict + Custom methods to retrieve remote schemes, keyed off of URI scheme Example ------- @@ -229,4 +242,4 @@ def handler(event, context): jmespath_options=jmespath_options, ) - validate_data_against_schema(data=event, schema=schema, formats=formats) + validate_data_against_schema(data=event, schema=schema, formats=formats, handlers=handlers) diff --git a/tests/functional/validator/conftest.py b/tests/functional/validator/conftest.py index 750f7648d40..ce86283c7ec 100644 --- a/tests/functional/validator/conftest.py +++ b/tests/functional/validator/conftest.py @@ -85,6 +85,53 @@ def schema_response(): } +@pytest.fixture +def schema_refs(): + return { + "ParentSchema": { + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "testschema://ParentSchema", + "type": "object", + "title": "Sample schema", + "description": "Sample JSON Schema that references another schema", + "examples": [{"parent_object": {"child_string": "hello world"}}], + "required": "parent_object", + "properties": { + "parent_object": { + "$id": "#/properties/parent_object", + "$ref": "testschema://ChildSchema", + }, + }, + }, + "ChildSchema": { + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "testschema://ChildSchema", + "type": "object", + "title": "Sample schema", + "description": "Sample JSON Schema that is referenced by another schema", + "examples": [{"child_string": "hello world"}], + "required": "child_string", + "properties": { + "child_string": { + "$id": "#/properties/child_string", + "type": "string", + "title": "The child string", + "examples": ["hello world"], + }, + }, + }, + } + + +@pytest.fixture +def schema_ref_handlers(schema_refs): + def handle_test_schema(uri): + schema_key = uri.split("://")[1] + return schema_refs[schema_key] + + return {"testschema": handle_test_schema} + + @pytest.fixture def raw_event(): return {"message": "hello hello", "username": "blah blah"} @@ -105,6 +152,11 @@ def wrapped_event_base64_json_string(): return {"data": "eyJtZXNzYWdlIjogImhlbGxvIGhlbGxvIiwgInVzZXJuYW1lIjogImJsYWggYmxhaCJ9="} +@pytest.fixture +def parent_ref_event(): + return {"parent_object": {"child_string": "hello world"}} + + @pytest.fixture def raw_response(): return {"statusCode": 200, "body": "response"} diff --git a/tests/functional/validator/test_validator.py b/tests/functional/validator/test_validator.py index 23b4943223a..f18787990ff 100644 --- a/tests/functional/validator/test_validator.py +++ b/tests/functional/validator/test_validator.py @@ -83,6 +83,10 @@ def test_validate_invalid_custom_format( ) +def test_validate_custom_handlers(schema_refs, schema_ref_handlers, parent_ref_event): + validate(event=parent_ref_event, schema=schema_refs["ParentSchema"], handlers=schema_ref_handlers) + + def test_validate_invalid_envelope_expression(schema, wrapped_event): with pytest.raises(exceptions.InvalidEnvelopeExpressionError): validate(event=wrapped_event, schema=schema, envelope=True) From 8c3a84b0009164ad731b1aab1e849c252f482d87 Mon Sep 17 00:00:00 2001 From: Daniel Chen Date: Thu, 13 Jun 2024 17:47:05 +0000 Subject: [PATCH 02/10] docs: document new handlers param in validation utilities --- docs/utilities/validation.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docs/utilities/validation.md b/docs/utilities/validation.md index 1b569ddc14c..e17e5b4c066 100644 --- a/docs/utilities/validation.md +++ b/docs/utilities/validation.md @@ -199,3 +199,33 @@ You can use our built-in [JMESPath functions](./jmespath_functions.md){target="_ ???+ info We use these for [built-in envelopes](#built-in-envelopes) to easily to decode and unwrap events from sources like Kinesis, CloudWatch Logs, etc. + +### Validating with external references + +JSON schema [allows schemas to reference other schemas](https://json-schema.org/understanding-json-schema/structuring#dollarref) using the `$ref` keyword. The value of `$ref` is a URI reference, but you likely don't want to launch an HTTP request to resolve this URI. Instead, you can pass resolving functions through the `handlers` parameter: + +```python title="custom_reference_handlers.py" +from aws_lambda_powertools.utilities.validation import validate + +SCHEMA = { + "ParentSchema": { + "type": "object", + "properties": { + "child_object": {"$ref": "testschema://ChildSchema"}, + ... + }, + ... + }, + "ChildSchema": ..., +} + +def handle_test_schema(uri): + schema_key = uri.split("://")[1] + return SCHEMA[schema_key] + +test_schema_handlers = {"testschema": handle_test_schema} + +parent_event = {"child_object": {...}} + +validate(event=parent_event, schema=SCHEMA["ParentSchema"], handlers=test_schema_handlers) +``` From 650cf9b4150ddb2f1b412e94074c53a088955bf1 Mon Sep 17 00:00:00 2001 From: Daniel Chen Date: Sat, 15 Jun 2024 22:01:45 +0000 Subject: [PATCH 03/10] Add provider_options validation arg, fix test JSON schema --- .../utilities/validation/base.py | 12 +++++++--- .../utilities/validation/validator.py | 23 ++++++++++++++++--- tests/functional/validator/conftest.py | 4 ++-- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/aws_lambda_powertools/utilities/validation/base.py b/aws_lambda_powertools/utilities/validation/base.py index 30880362c79..d01eeac1b38 100644 --- a/aws_lambda_powertools/utilities/validation/base.py +++ b/aws_lambda_powertools/utilities/validation/base.py @@ -1,5 +1,5 @@ import logging -from typing import Dict, Optional, Union +from typing import Any, Dict, Optional, Union import fastjsonschema # type: ignore @@ -9,7 +9,11 @@ def validate_data_against_schema( - data: Union[Dict, str], schema: Dict, formats: Optional[Dict] = None, handlers: Optional[Dict] = None + data: Union[Dict, str], + schema: Dict, + formats: Optional[Dict] = None, + handlers: Optional[Dict] = None, + **provider_options: Any, ): """Validate dict data against given JSON Schema @@ -23,6 +27,8 @@ def validate_data_against_schema( Custom formats containing a key (e.g. int64) and a value expressed as regex or callback returning bool handlers: Dict Custom methods to retrieve remote schemes, keyed off of URI scheme + **provider options: Dict, optional + Arguments that will be passed directly to the underlying validate call Raises ------ @@ -34,7 +40,7 @@ def validate_data_against_schema( try: formats = formats or {} handlers = handlers or {} - fastjsonschema.validate(definition=schema, data=data, formats=formats, handlers=handlers) + fastjsonschema.validate(definition=schema, data=data, formats=formats, handlers=handlers, **provider_options) except (TypeError, AttributeError, fastjsonschema.JsonSchemaDefinitionException) as e: raise InvalidSchemaFormatError(f"Schema received: {schema}, Formats: {formats}. Error: {e}") except fastjsonschema.JsonSchemaValueException as e: diff --git a/aws_lambda_powertools/utilities/validation/validator.py b/aws_lambda_powertools/utilities/validation/validator.py index 473f2684c7d..e5553c3a808 100644 --- a/aws_lambda_powertools/utilities/validation/validator.py +++ b/aws_lambda_powertools/utilities/validation/validator.py @@ -17,9 +17,11 @@ def validator( inbound_schema: Optional[Dict] = None, inbound_formats: Optional[Dict] = None, inbound_handlers: Optional[Dict] = None, + inbound_provider_options: Dict = {}, outbound_schema: Optional[Dict] = None, outbound_formats: Optional[Dict] = None, outbound_handlers: Optional[Dict] = None, + outbound_provider_options: Dict = {}, envelope: str = "", jmespath_options: Optional[Dict] = None, **kwargs: Any, @@ -50,6 +52,10 @@ def validator( Custom methods to retrieve remote schemes, keyed off of URI scheme outbound_handlers: Dict Custom methods to retrieve remote schemes, keyed off of URI scheme + inbound_provider_options: Dict + Arguments that will be passed directly to the underlying validate call for the inbound event + outbound_provider_options: Dict + Arguments that will be passed directly to the underlying validate call for the outbound event Example ------- @@ -134,7 +140,11 @@ def handler(event, context): if inbound_schema: logger.debug("Validating inbound event") validate_data_against_schema( - data=event, schema=inbound_schema, formats=inbound_formats, handlers=inbound_handlers + data=event, + schema=inbound_schema, + formats=inbound_formats, + handlers=inbound_handlers, + **inbound_provider_options, ) response = handler(event, context, **kwargs) @@ -142,7 +152,11 @@ def handler(event, context): if outbound_schema: logger.debug("Validating outbound event") validate_data_against_schema( - data=response, schema=outbound_schema, formats=outbound_formats, handlers=outbound_handlers + data=response, + schema=outbound_schema, + formats=outbound_formats, + handlers=outbound_handlers, + **outbound_provider_options, ) return response @@ -153,6 +167,7 @@ def validate( schema: Dict, formats: Optional[Dict] = None, handlers: Optional[Dict] = None, + provider_options: Dict = {}, envelope: Optional[str] = None, jmespath_options: Optional[Dict] = None, ): @@ -174,6 +189,8 @@ def validate( Custom formats containing a key (e.g. int64) and a value expressed as regex or callback returning bool handlers: Dict Custom methods to retrieve remote schemes, keyed off of URI scheme + provider_options: Dict + Arguments that will be passed directly to the underlying validate call Example ------- @@ -242,4 +259,4 @@ def handler(event, context): jmespath_options=jmespath_options, ) - validate_data_against_schema(data=event, schema=schema, formats=formats, handlers=handlers) + validate_data_against_schema(data=event, schema=schema, formats=formats, handlers=handlers, **provider_options) diff --git a/tests/functional/validator/conftest.py b/tests/functional/validator/conftest.py index ce86283c7ec..3b9033c82d4 100644 --- a/tests/functional/validator/conftest.py +++ b/tests/functional/validator/conftest.py @@ -95,7 +95,7 @@ def schema_refs(): "title": "Sample schema", "description": "Sample JSON Schema that references another schema", "examples": [{"parent_object": {"child_string": "hello world"}}], - "required": "parent_object", + "required": ["parent_object"], "properties": { "parent_object": { "$id": "#/properties/parent_object", @@ -110,7 +110,7 @@ def schema_refs(): "title": "Sample schema", "description": "Sample JSON Schema that is referenced by another schema", "examples": [{"child_string": "hello world"}], - "required": "child_string", + "required": ["child_string"], "properties": { "child_string": { "$id": "#/properties/child_string", From e17370e3f3e907d2e6adf172e72c91ae997a1b59 Mon Sep 17 00:00:00 2001 From: Daniel Chen Date: Sun, 16 Jun 2024 05:03:32 +0000 Subject: [PATCH 04/10] Update provider_options default arg value --- aws_lambda_powertools/utilities/validation/base.py | 5 +++-- .../utilities/validation/validator.py | 14 ++++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/aws_lambda_powertools/utilities/validation/base.py b/aws_lambda_powertools/utilities/validation/base.py index d01eeac1b38..aef3d047fa3 100644 --- a/aws_lambda_powertools/utilities/validation/base.py +++ b/aws_lambda_powertools/utilities/validation/base.py @@ -13,7 +13,7 @@ def validate_data_against_schema( schema: Dict, formats: Optional[Dict] = None, handlers: Optional[Dict] = None, - **provider_options: Any, + provider_options: Optional[Dict] = None, ): """Validate dict data against given JSON Schema @@ -27,7 +27,7 @@ def validate_data_against_schema( Custom formats containing a key (e.g. int64) and a value expressed as regex or callback returning bool handlers: Dict Custom methods to retrieve remote schemes, keyed off of URI scheme - **provider options: Dict, optional + provider_options: Dict Arguments that will be passed directly to the underlying validate call Raises @@ -40,6 +40,7 @@ def validate_data_against_schema( try: formats = formats or {} handlers = handlers or {} + provider_options = provider_options or {} fastjsonschema.validate(definition=schema, data=data, formats=formats, handlers=handlers, **provider_options) except (TypeError, AttributeError, fastjsonschema.JsonSchemaDefinitionException) as e: raise InvalidSchemaFormatError(f"Schema received: {schema}, Formats: {formats}. Error: {e}") diff --git a/aws_lambda_powertools/utilities/validation/validator.py b/aws_lambda_powertools/utilities/validation/validator.py index e5553c3a808..a743f0d9fab 100644 --- a/aws_lambda_powertools/utilities/validation/validator.py +++ b/aws_lambda_powertools/utilities/validation/validator.py @@ -17,11 +17,11 @@ def validator( inbound_schema: Optional[Dict] = None, inbound_formats: Optional[Dict] = None, inbound_handlers: Optional[Dict] = None, - inbound_provider_options: Dict = {}, + inbound_provider_options: Optional[Dict] = None, outbound_schema: Optional[Dict] = None, outbound_formats: Optional[Dict] = None, outbound_handlers: Optional[Dict] = None, - outbound_provider_options: Dict = {}, + outbound_provider_options: Optional[Dict] = None, envelope: str = "", jmespath_options: Optional[Dict] = None, **kwargs: Any, @@ -144,7 +144,7 @@ def handler(event, context): schema=inbound_schema, formats=inbound_formats, handlers=inbound_handlers, - **inbound_provider_options, + provider_options=inbound_provider_options, ) response = handler(event, context, **kwargs) @@ -156,7 +156,7 @@ def handler(event, context): schema=outbound_schema, formats=outbound_formats, handlers=outbound_handlers, - **outbound_provider_options, + provider_options=outbound_provider_options, ) return response @@ -167,7 +167,7 @@ def validate( schema: Dict, formats: Optional[Dict] = None, handlers: Optional[Dict] = None, - provider_options: Dict = {}, + provider_options: Optional[Dict] = None, envelope: Optional[str] = None, jmespath_options: Optional[Dict] = None, ): @@ -259,4 +259,6 @@ def handler(event, context): jmespath_options=jmespath_options, ) - validate_data_against_schema(data=event, schema=schema, formats=formats, handlers=handlers, **provider_options) + validate_data_against_schema( + data=event, schema=schema, formats=formats, handlers=handlers, provider_options=provider_options + ) From 9e481eb41dca3e2ca68df10aac13d11bc6ab322a Mon Sep 17 00:00:00 2001 From: Daniel Chen Date: Mon, 17 Jun 2024 04:34:33 +0000 Subject: [PATCH 05/10] Remove unused import --- aws_lambda_powertools/utilities/validation/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_lambda_powertools/utilities/validation/base.py b/aws_lambda_powertools/utilities/validation/base.py index aef3d047fa3..ea7006ee252 100644 --- a/aws_lambda_powertools/utilities/validation/base.py +++ b/aws_lambda_powertools/utilities/validation/base.py @@ -1,5 +1,5 @@ import logging -from typing import Any, Dict, Optional, Union +from typing import Dict, Optional, Union import fastjsonschema # type: ignore From 5f3de5ca3939c59436de15613d5e3d12fdf53b71 Mon Sep 17 00:00:00 2001 From: Daniel Chen Date: Mon, 17 Jun 2024 20:31:10 +0000 Subject: [PATCH 06/10] Lint files --- aws_lambda_powertools/utilities/validation/validator.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/aws_lambda_powertools/utilities/validation/validator.py b/aws_lambda_powertools/utilities/validation/validator.py index a743f0d9fab..0a20b4e971f 100644 --- a/aws_lambda_powertools/utilities/validation/validator.py +++ b/aws_lambda_powertools/utilities/validation/validator.py @@ -260,5 +260,9 @@ def handler(event, context): ) validate_data_against_schema( - data=event, schema=schema, formats=formats, handlers=handlers, provider_options=provider_options + data=event, + schema=schema, + formats=formats, + handlers=handlers, + provider_options=provider_options, ) From d760ca6915e41520cf1a2dab80b380254ad091a5 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Wed, 19 Jun 2024 00:34:56 +0100 Subject: [PATCH 07/10] Small changes + example refactor --- aws_lambda_powertools/shared/functions.py | 17 +++++++ .../utilities/validation/base.py | 9 ++-- .../utilities/validation/exceptions.py | 2 +- .../utilities/validation/validator.py | 5 +-- docs/utilities/validation.md | 44 +++++++++---------- examples/validation/src/custom_handlers.py | 13 ++++++ .../src/custom_handlers_payload.json | 6 +++ .../validation/src/custom_handlers_schema.py | 22 ++++++++++ 8 files changed, 87 insertions(+), 31 deletions(-) create mode 100644 examples/validation/src/custom_handlers.py create mode 100644 examples/validation/src/custom_handlers_payload.json create mode 100644 examples/validation/src/custom_handlers_schema.py diff --git a/aws_lambda_powertools/shared/functions.py b/aws_lambda_powertools/shared/functions.py index 0f943f36d39..2559f6fa8f7 100644 --- a/aws_lambda_powertools/shared/functions.py +++ b/aws_lambda_powertools/shared/functions.py @@ -92,6 +92,23 @@ def resolve_env_var_choice( return choice if choice is not None else env +def get_field_or_empty_dict(field: Optional[Dict[str, Any]]) -> Dict[str, Any]: + """ + Returns the given dictionary field if it is not empty, otherwise returns an empty dictionary. + + Parameters + ---------- + field: Dict[str, Any] + The dictionary field to be checked. + + Returns + ------- + Dict[str, Any] + The input dictionary field if it is not empty, or an empty dictionary. + """ + return field or {} + + def base64_decode(value: str) -> bytes: try: logger.debug("Decoding base64 item to bytes") diff --git a/aws_lambda_powertools/utilities/validation/base.py b/aws_lambda_powertools/utilities/validation/base.py index ea7006ee252..629da2a62f7 100644 --- a/aws_lambda_powertools/utilities/validation/base.py +++ b/aws_lambda_powertools/utilities/validation/base.py @@ -3,7 +3,8 @@ import fastjsonschema # type: ignore -from .exceptions import InvalidSchemaFormatError, SchemaValidationError +from aws_lambda_powertools.shared.functions import get_field_or_empty_dict +from aws_lambda_powertools.utilities.validation.exceptions import InvalidSchemaFormatError, SchemaValidationError logger = logging.getLogger(__name__) @@ -38,9 +39,9 @@ def validate_data_against_schema( When JSON schema provided is invalid """ try: - formats = formats or {} - handlers = handlers or {} - provider_options = provider_options or {} + formats = get_field_or_empty_dict(formats) + handlers = get_field_or_empty_dict(handlers) + provider_options = get_field_or_empty_dict(provider_options) fastjsonschema.validate(definition=schema, data=data, formats=formats, handlers=handlers, **provider_options) except (TypeError, AttributeError, fastjsonschema.JsonSchemaDefinitionException) as e: raise InvalidSchemaFormatError(f"Schema received: {schema}, Formats: {formats}. Error: {e}") diff --git a/aws_lambda_powertools/utilities/validation/exceptions.py b/aws_lambda_powertools/utilities/validation/exceptions.py index 8789e3f2e80..c9ba0e2a75e 100644 --- a/aws_lambda_powertools/utilities/validation/exceptions.py +++ b/aws_lambda_powertools/utilities/validation/exceptions.py @@ -1,6 +1,6 @@ from typing import Any, List, Optional -from ...exceptions import InvalidEnvelopeExpressionError +from aws_lambda_powertools.exceptions import InvalidEnvelopeExpressionError class SchemaValidationError(Exception): diff --git a/aws_lambda_powertools/utilities/validation/validator.py b/aws_lambda_powertools/utilities/validation/validator.py index 0a20b4e971f..9840b9ede7e 100644 --- a/aws_lambda_powertools/utilities/validation/validator.py +++ b/aws_lambda_powertools/utilities/validation/validator.py @@ -1,10 +1,9 @@ import logging from typing import Any, Callable, Dict, Optional, Union +from aws_lambda_powertools.middleware_factory import lambda_handler_decorator from aws_lambda_powertools.utilities import jmespath_utils - -from ...middleware_factory import lambda_handler_decorator -from .base import validate_data_against_schema +from aws_lambda_powertools.utilities.validation.base import validate_data_against_schema logger = logging.getLogger(__name__) diff --git a/docs/utilities/validation.md b/docs/utilities/validation.md index e17e5b4c066..5878baf7f14 100644 --- a/docs/utilities/validation.md +++ b/docs/utilities/validation.md @@ -147,10 +147,10 @@ Here is a handy table with built-in envelopes along with their JMESPath expressi | **`API_GATEWAY_HTTP`** | `powertools_json(body)` | | **`API_GATEWAY_REST`** | `powertools_json(body)` | | **`CLOUDWATCH_EVENTS_SCHEDULED`** | `detail` | -| **`CLOUDWATCH_LOGS`** | `awslogs.powertools_base64_gzip(data) | powertools_json(@).logEvents[*]` | +| **`CLOUDWATCH_LOGS`** | `awslogs.powertools_base64_gzip(data)` or `powertools_json(@).logEvents[*]` | | **`EVENTBRIDGE`** | `detail` | | **`KINESIS_DATA_STREAM`** | `Records[*].kinesis.powertools_json(powertools_base64(data))` | -| **`SNS`** | `Records[0].Sns.Message | powertools_json(@)` | +| **`SNS`** | `Records[0].Sns.Message` or `powertools_json(@)` | | **`SQS`** | `Records[*].powertools_json(body)` | ## Advanced @@ -204,28 +204,26 @@ You can use our built-in [JMESPath functions](./jmespath_functions.md){target="_ JSON schema [allows schemas to reference other schemas](https://json-schema.org/understanding-json-schema/structuring#dollarref) using the `$ref` keyword. The value of `$ref` is a URI reference, but you likely don't want to launch an HTTP request to resolve this URI. Instead, you can pass resolving functions through the `handlers` parameter: -```python title="custom_reference_handlers.py" -from aws_lambda_powertools.utilities.validation import validate - -SCHEMA = { - "ParentSchema": { - "type": "object", - "properties": { - "child_object": {"$ref": "testschema://ChildSchema"}, - ... - }, - ... - }, - "ChildSchema": ..., -} +=== "custom_handlers.py" -def handle_test_schema(uri): - schema_key = uri.split("://")[1] - return SCHEMA[schema_key] + ```python hl_lines="1 7 8 11" + --8<-- "examples/validation/src/custom_handlers.py" + ``` -test_schema_handlers = {"testschema": handle_test_schema} +=== "custom_handlers_parent_schema" -parent_event = {"child_object": {...}} + ```python hl_lines="1 7" + --8<-- "examples/validation/src/custom_handlers_schema.py" + ``` -validate(event=parent_event, schema=SCHEMA["ParentSchema"], handlers=test_schema_handlers) -``` +=== "custom_handlers_child_schema" + + ```python hl_lines="12" + --8<-- "examples/validation/src/custom_handlers_schema.py" + ``` + +=== "custom_handlers_payload.json" + + ```json hl_lines="2" + --8<-- "examples/validation/src/custom_handlers_payload.json" + ``` diff --git a/examples/validation/src/custom_handlers.py b/examples/validation/src/custom_handlers.py new file mode 100644 index 00000000000..2b56cfc35ba --- /dev/null +++ b/examples/validation/src/custom_handlers.py @@ -0,0 +1,13 @@ +from custom_handlers_schema import CHILD_SCHEMA, PARENT_SCHEMA + +from aws_lambda_powertools.utilities.typing import LambdaContext +from aws_lambda_powertools.utilities.validation import validator + + +def get_child_schema(uri): + return CHILD_SCHEMA + + +@validator(inbound_schema=PARENT_SCHEMA, inbound_handlers={"https": get_child_schema}) +def lambda_handler(event, context: LambdaContext) -> dict: + return event diff --git a/examples/validation/src/custom_handlers_payload.json b/examples/validation/src/custom_handlers_payload.json new file mode 100644 index 00000000000..a44e194d7d6 --- /dev/null +++ b/examples/validation/src/custom_handlers_payload.json @@ -0,0 +1,6 @@ +{ + "ParentSchema": + { + "project": "powertools" + } +} \ No newline at end of file diff --git a/examples/validation/src/custom_handlers_schema.py b/examples/validation/src/custom_handlers_schema.py new file mode 100644 index 00000000000..ab911e3d63f --- /dev/null +++ b/examples/validation/src/custom_handlers_schema.py @@ -0,0 +1,22 @@ +PARENT_SCHEMA = { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://example.com/schemas/parent.json", + "type": "object", + "properties": { + "ParentSchema": { + "$ref": "https://SCHEMA", + }, + }, +} + +CHILD_SCHEMA = { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://example.com/schemas/child.json", + "type": "object", + "properties": { + "project": { + "type": "string", + }, + }, + "required": ["project"], +} From 868e94a17508a1c31d6a2b372b19bd9add246b4b Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Wed, 19 Jun 2024 00:44:22 +0100 Subject: [PATCH 08/10] Small changes + example refactor --- examples/validation/src/custom_handlers_payload.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/validation/src/custom_handlers_payload.json b/examples/validation/src/custom_handlers_payload.json index a44e194d7d6..09ab994f892 100644 --- a/examples/validation/src/custom_handlers_payload.json +++ b/examples/validation/src/custom_handlers_payload.json @@ -3,4 +3,4 @@ { "project": "powertools" } -} \ No newline at end of file +} From 189714e5b3a535cb7b981a02f2656fdbf11a1c61 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Wed, 19 Jun 2024 10:52:23 +0100 Subject: [PATCH 09/10] Addressing Heitor's feedback --- Makefile | 2 +- aws_lambda_powertools/shared/functions.py | 17 ----------------- .../utilities/validation/base.py | 10 +++++----- .../utilities/validation/validator.py | 7 +++++-- docs/utilities/validation.md | 4 +++- examples/validation/src/custom_handlers.py | 2 +- 6 files changed, 15 insertions(+), 27 deletions(-) diff --git a/Makefile b/Makefile index 55bed054c32..2a9970f1f67 100644 --- a/Makefile +++ b/Makefile @@ -83,7 +83,7 @@ complexity-baseline: $(info Maintenability index) poetry run radon mi aws_lambda_powertools $(info Cyclomatic complexity index) - poetry run xenon --max-absolute C --max-modules A --max-average A aws_lambda_powertools --exclude aws_lambda_powertools/shared/json_encoder.py + poetry run xenon --max-absolute C --max-modules A --max-average A aws_lambda_powertools --exclude aws_lambda_powertools/shared/json_encoder.py,aws_lambda_powertools/utilities/validation/base.py # # Use `poetry version /` for version bump diff --git a/aws_lambda_powertools/shared/functions.py b/aws_lambda_powertools/shared/functions.py index 2559f6fa8f7..0f943f36d39 100644 --- a/aws_lambda_powertools/shared/functions.py +++ b/aws_lambda_powertools/shared/functions.py @@ -92,23 +92,6 @@ def resolve_env_var_choice( return choice if choice is not None else env -def get_field_or_empty_dict(field: Optional[Dict[str, Any]]) -> Dict[str, Any]: - """ - Returns the given dictionary field if it is not empty, otherwise returns an empty dictionary. - - Parameters - ---------- - field: Dict[str, Any] - The dictionary field to be checked. - - Returns - ------- - Dict[str, Any] - The input dictionary field if it is not empty, or an empty dictionary. - """ - return field or {} - - def base64_decode(value: str) -> bytes: try: logger.debug("Decoding base64 item to bytes") diff --git a/aws_lambda_powertools/utilities/validation/base.py b/aws_lambda_powertools/utilities/validation/base.py index 629da2a62f7..9d7a36874aa 100644 --- a/aws_lambda_powertools/utilities/validation/base.py +++ b/aws_lambda_powertools/utilities/validation/base.py @@ -3,7 +3,6 @@ import fastjsonschema # type: ignore -from aws_lambda_powertools.shared.functions import get_field_or_empty_dict from aws_lambda_powertools.utilities.validation.exceptions import InvalidSchemaFormatError, SchemaValidationError logger = logging.getLogger(__name__) @@ -29,7 +28,8 @@ def validate_data_against_schema( handlers: Dict Custom methods to retrieve remote schemes, keyed off of URI scheme provider_options: Dict - Arguments that will be passed directly to the underlying validate call + Arguments that will be passed directly to the underlying validation call, in this case fastjsonchema.validate. + For all supported arguments see: https://horejsek.github.io/python-fastjsonschema/#fastjsonschema.validate Raises ------ @@ -39,9 +39,9 @@ def validate_data_against_schema( When JSON schema provided is invalid """ try: - formats = get_field_or_empty_dict(formats) - handlers = get_field_or_empty_dict(handlers) - provider_options = get_field_or_empty_dict(provider_options) + formats = formats or {} + handlers = handlers or {} + provider_options = provider_options or {} fastjsonschema.validate(definition=schema, data=data, formats=formats, handlers=handlers, **provider_options) except (TypeError, AttributeError, fastjsonschema.JsonSchemaDefinitionException) as e: raise InvalidSchemaFormatError(f"Schema received: {schema}, Formats: {formats}. Error: {e}") diff --git a/aws_lambda_powertools/utilities/validation/validator.py b/aws_lambda_powertools/utilities/validation/validator.py index 9840b9ede7e..74861f7de3f 100644 --- a/aws_lambda_powertools/utilities/validation/validator.py +++ b/aws_lambda_powertools/utilities/validation/validator.py @@ -52,9 +52,12 @@ def validator( outbound_handlers: Dict Custom methods to retrieve remote schemes, keyed off of URI scheme inbound_provider_options: Dict - Arguments that will be passed directly to the underlying validate call for the inbound event + Arguments that will be passed directly to the underlying validation call, in this case fastjsonchema.validate. + For all supported arguments see: https://horejsek.github.io/python-fastjsonschema/#fastjsonschema.validate outbound_provider_options: Dict - Arguments that will be passed directly to the underlying validate call for the outbound event + Arguments that will be passed directly to the underlying validation call, in this case fastjsonchema.validate. + For all supported arguments see: https://horejsek.github.io/python-fastjsonschema/#fastjsonschema.validate + Example ------- diff --git a/docs/utilities/validation.md b/docs/utilities/validation.md index 5878baf7f14..51085d417fa 100644 --- a/docs/utilities/validation.md +++ b/docs/utilities/validation.md @@ -202,7 +202,9 @@ You can use our built-in [JMESPath functions](./jmespath_functions.md){target="_ ### Validating with external references -JSON schema [allows schemas to reference other schemas](https://json-schema.org/understanding-json-schema/structuring#dollarref) using the `$ref` keyword. The value of `$ref` is a URI reference, but you likely don't want to launch an HTTP request to resolve this URI. Instead, you can pass resolving functions through the `handlers` parameter: +JSON Schema [allows schemas to reference other schemas](https://json-schema.org/understanding-json-schema/structuring#dollarref) using the `$ref` keyword with a URI value. By default, `fastjsonschema` will make a HTTP request to resolve this URI. + +You can use `handlers` parameter to have full control over how references schemas are fetched. This is useful when you might want to optimize caching, reducing HTTP calls, or fetching them from non-HTTP endpoints. === "custom_handlers.py" diff --git a/examples/validation/src/custom_handlers.py b/examples/validation/src/custom_handlers.py index 2b56cfc35ba..faf46cf67b3 100644 --- a/examples/validation/src/custom_handlers.py +++ b/examples/validation/src/custom_handlers.py @@ -4,7 +4,7 @@ from aws_lambda_powertools.utilities.validation import validator -def get_child_schema(uri): +def get_child_schema(uri: str): return CHILD_SCHEMA From 2191346a18f2e8f0d47c71e9f02e8c6d876cebb1 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Wed, 19 Jun 2024 11:17:29 +0100 Subject: [PATCH 10/10] Addressing Heitor's feedback --- examples/validation/src/custom_handlers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/validation/src/custom_handlers.py b/examples/validation/src/custom_handlers.py index faf46cf67b3..4cbc5d65b93 100644 --- a/examples/validation/src/custom_handlers.py +++ b/examples/validation/src/custom_handlers.py @@ -4,6 +4,7 @@ from aws_lambda_powertools.utilities.validation import validator +# Function to return the child schema def get_child_schema(uri: str): return CHILD_SCHEMA