From 2009dddf9f6c2c8ec76e48515f805a1030dab5ad Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Mon, 1 Jul 2024 09:55:01 +0200 Subject: [PATCH 1/7] fix(event_handler): custom serializer recursive values --- .../event_handler/openapi/encoders.py | 10 +++++++++- .../_pydantic/test_openapi_encoders.py | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/aws_lambda_powertools/event_handler/openapi/encoders.py b/aws_lambda_powertools/event_handler/openapi/encoders.py index 520c0d71509..605e0a334fc 100644 --- a/aws_lambda_powertools/event_handler/openapi/encoders.py +++ b/aws_lambda_powertools/event_handler/openapi/encoders.py @@ -115,8 +115,9 @@ def jsonable_encoder( # noqa: PLR0911 include=include, exclude=exclude, by_alias=by_alias, - exclude_none=exclude_none, exclude_unset=exclude_unset, + exclude_none=exclude_none, + custom_serializer=custom_serializer, ) # Sequences @@ -201,9 +202,14 @@ def _dump_dict( by_alias: bool = True, exclude_unset: bool = False, exclude_none: bool = False, + custom_serializer: Optional[Callable[[Any], str]] = None, ) -> Dict[str, Any]: """ Dump a dict to a dict, using the same parameters as jsonable_encoder + + Parameters + ---------- + custom_serializer """ encoded_dict = {} allowed_keys = set(obj.keys()) @@ -222,12 +228,14 @@ def _dump_dict( by_alias=by_alias, exclude_unset=exclude_unset, exclude_none=exclude_none, + custom_serializer=custom_serializer, ) encoded_value = jsonable_encoder( value, by_alias=by_alias, exclude_unset=exclude_unset, exclude_none=exclude_none, + custom_serializer=custom_serializer, ) encoded_dict[encoded_key] = encoded_value return encoded_dict diff --git a/tests/functional/event_handler/_pydantic/test_openapi_encoders.py b/tests/functional/event_handler/_pydantic/test_openapi_encoders.py index dee9c21b84c..b1de4a4d3f5 100644 --- a/tests/functional/event_handler/_pydantic/test_openapi_encoders.py +++ b/tests/functional/event_handler/_pydantic/test_openapi_encoders.py @@ -193,3 +193,21 @@ class MyClass: with pytest.raises(SerializationError, match="Unable to serialize the object*"): jsonable_encoder(MyClass()) + + +def test_openapi_encode_custom_serializer_nested_dict(): + # GIVEN a nested dictionary with a custom class + class CustomClass(object): + __slots__ = [] + + nested_dict = {"a": {"b": CustomClass()}} + + # AND a custom serializer + def serializer(value): + return "serialized" + + # WHEN we call jsonable_encoder with the nested dictionary and unserializable value + result = jsonable_encoder(nested_dict, custom_serializer=serializer) + + # THEN we should get the custom serializer output + assert result == {"a": {"b": "serialized"}} From 8bc85c6888e2e88b98c2d5fd0a7026eea6b75d69 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Mon, 1 Jul 2024 11:49:41 +0200 Subject: [PATCH 2/7] fix: use custom serializers for custom sequence values --- .../event_handler/openapi/encoders.py | 8 +++++-- .../_pydantic/test_openapi_encoders.py | 21 +++++++++++++++++-- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/aws_lambda_powertools/event_handler/openapi/encoders.py b/aws_lambda_powertools/event_handler/openapi/encoders.py index 605e0a334fc..e6848c7b885 100644 --- a/aws_lambda_powertools/event_handler/openapi/encoders.py +++ b/aws_lambda_powertools/event_handler/openapi/encoders.py @@ -130,6 +130,7 @@ def jsonable_encoder( # noqa: PLR0911 exclude_none=exclude_none, exclude_defaults=exclude_defaults, exclude_unset=exclude_unset, + custom_serializer=custom_serializer, ) # Other types @@ -209,7 +210,8 @@ def _dump_dict( Parameters ---------- - custom_serializer + custom_serializer : Callable, optional + A custom serializer to use for encoding the object, when everything else fails. """ encoded_dict = {} allowed_keys = set(obj.keys()) @@ -250,9 +252,10 @@ def _dump_sequence( exclude_unset: bool = False, exclude_none: bool = False, exclude_defaults: bool = False, + custom_serializer: Optional[Callable[[Any], str]] = None, ) -> List[Any]: """ - Dump a sequence to a list, using the same parameters as jsonable_encoder + Dump a sequence to a list, using the same parameters as jsonable_encoder. """ encoded_list = [] for item in obj: @@ -265,6 +268,7 @@ def _dump_sequence( exclude_unset=exclude_unset, exclude_defaults=exclude_defaults, exclude_none=exclude_none, + custom_serializer=custom_serializer, ), ) return encoded_list diff --git a/tests/functional/event_handler/_pydantic/test_openapi_encoders.py b/tests/functional/event_handler/_pydantic/test_openapi_encoders.py index b1de4a4d3f5..10a1db4bb71 100644 --- a/tests/functional/event_handler/_pydantic/test_openapi_encoders.py +++ b/tests/functional/event_handler/_pydantic/test_openapi_encoders.py @@ -197,8 +197,7 @@ class MyClass: def test_openapi_encode_custom_serializer_nested_dict(): # GIVEN a nested dictionary with a custom class - class CustomClass(object): - __slots__ = [] + class CustomClass: ... nested_dict = {"a": {"b": CustomClass()}} @@ -211,3 +210,21 @@ def serializer(value): # THEN we should get the custom serializer output assert result == {"a": {"b": "serialized"}} + + +def test_openapi_encode_custom_serializer_sequences(): + # GIVEN a sequence with a custom class + class CustomClass: + __slots__ = [] + + seq = [CustomClass()] + + # AND a custom serializer + def serializer(value): + return "serialized" + + # WHEN we call jsonable_encoder with the nested dictionary and unserializable value + result = jsonable_encoder(seq, custom_serializer=serializer) + + # THEN we should get the custom serializer output + assert result == ["serialized"] From ead7183708758d0b0202f2535e2a2386e5dcc34d Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Mon, 1 Jul 2024 11:58:19 +0200 Subject: [PATCH 3/7] fix: use custom serializers for custom base model values --- .../event_handler/openapi/encoders.py | 3 +++ .../_pydantic/test_openapi_encoders.py | 25 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/aws_lambda_powertools/event_handler/openapi/encoders.py b/aws_lambda_powertools/event_handler/openapi/encoders.py index e6848c7b885..8701dd90c18 100644 --- a/aws_lambda_powertools/event_handler/openapi/encoders.py +++ b/aws_lambda_powertools/event_handler/openapi/encoders.py @@ -81,6 +81,7 @@ def jsonable_encoder( # noqa: PLR0911 exclude_unset=exclude_unset, exclude_none=exclude_none, exclude_defaults=exclude_defaults, + custom_serializer=custom_serializer, ) # Dataclasses @@ -171,6 +172,7 @@ def _dump_base_model( exclude_unset: bool = False, exclude_none: bool = False, exclude_defaults: bool = False, + custom_serializer: Optional[Callable[[Any], str]] = None, ): """ Dump a BaseModel object to a dict, using the same parameters as jsonable_encoder @@ -192,6 +194,7 @@ def _dump_base_model( obj_dict, exclude_none=exclude_none, exclude_defaults=exclude_defaults, + custom_serializer=custom_serializer, ) diff --git a/tests/functional/event_handler/_pydantic/test_openapi_encoders.py b/tests/functional/event_handler/_pydantic/test_openapi_encoders.py index 10a1db4bb71..48572a93576 100644 --- a/tests/functional/event_handler/_pydantic/test_openapi_encoders.py +++ b/tests/functional/event_handler/_pydantic/test_openapi_encoders.py @@ -228,3 +228,28 @@ def serializer(value): # THEN we should get the custom serializer output assert result == ["serialized"] + + +def test_openapi_encode_custom_serializer_pydantic(): + # GIVEN a sequence with a custom class + class CustomClass: + __slots__ = [] + + class Order(BaseModel): + kind: CustomClass + + # maintenance: deprecate in V3; becomes model_config =ConfigDict(=True) + class Config: + arbitrary_types_allowed = True + + order = Order(kind=CustomClass()) + + # AND a custom serializer + def serializer(value): + return "serialized" + + # WHEN we call jsonable_encoder with the nested dictionary and unserializable value + result = jsonable_encoder(order, custom_serializer=serializer) + + # THEN we should get the custom serializer output + assert result == {"kind": "serialized"} From 31f97809823fcf8911296996758e4ae8cfc13a5d Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Mon, 1 Jul 2024 12:00:02 +0200 Subject: [PATCH 4/7] fix: use custom serializers for custom data classes values --- .../event_handler/openapi/encoders.py | 1 + .../_pydantic/test_openapi_encoders.py | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/aws_lambda_powertools/event_handler/openapi/encoders.py b/aws_lambda_powertools/event_handler/openapi/encoders.py index 8701dd90c18..b1ec88ba18e 100644 --- a/aws_lambda_powertools/event_handler/openapi/encoders.py +++ b/aws_lambda_powertools/event_handler/openapi/encoders.py @@ -95,6 +95,7 @@ def jsonable_encoder( # noqa: PLR0911 exclude_unset=exclude_unset, exclude_defaults=exclude_defaults, exclude_none=exclude_none, + custom_serializer=custom_serializer, ) # Enums diff --git a/tests/functional/event_handler/_pydantic/test_openapi_encoders.py b/tests/functional/event_handler/_pydantic/test_openapi_encoders.py index 48572a93576..8a43ac7901d 100644 --- a/tests/functional/event_handler/_pydantic/test_openapi_encoders.py +++ b/tests/functional/event_handler/_pydantic/test_openapi_encoders.py @@ -253,3 +253,25 @@ def serializer(value): # THEN we should get the custom serializer output assert result == {"kind": "serialized"} + + +def test_openapi_encode_custom_serializer_dataclasses(): + # GIVEN a sequence with a custom class + class CustomClass: + __slots__ = [] + + @dataclass + class Order: + kind: CustomClass + + order = Order(kind=CustomClass()) + + # AND a custom serializer + def serializer(value): + return "serialized" + + # WHEN we call jsonable_encoder with the nested dictionary and unserializable value + result = jsonable_encoder(order, custom_serializer=serializer) + + # THEN we should get the custom serializer output + assert result == {"kind": "serialized"} From 2bcd2484e096bbe8672af9104359d7bdb457edf0 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Mon, 1 Jul 2024 12:06:24 +0200 Subject: [PATCH 5/7] fix: propagate custom serializer for any unmatched type for safety --- aws_lambda_powertools/event_handler/openapi/encoders.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/aws_lambda_powertools/event_handler/openapi/encoders.py b/aws_lambda_powertools/event_handler/openapi/encoders.py index b1ec88ba18e..9846252ff5f 100644 --- a/aws_lambda_powertools/event_handler/openapi/encoders.py +++ b/aws_lambda_powertools/event_handler/openapi/encoders.py @@ -156,6 +156,7 @@ def jsonable_encoder( # noqa: PLR0911 exclude_none=exclude_none, exclude_unset=exclude_unset, exclude_defaults=exclude_defaults, + custom_serializer=custom_serializer, ) except ValueError as exc: raise SerializationError( @@ -287,6 +288,7 @@ def _dump_other( exclude_unset: bool = False, exclude_none: bool = False, exclude_defaults: bool = False, + custom_serializer: Optional[Callable[[Any], str]] = None, ) -> Any: """ Dump an object to a hashable object, using the same parameters as jsonable_encoder @@ -308,6 +310,7 @@ def _dump_other( exclude_unset=exclude_unset, exclude_defaults=exclude_defaults, exclude_none=exclude_none, + custom_serializer=custom_serializer, ) From bff9b81b75a3f32c3abd9684010cd01a6367be08 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Mon, 1 Jul 2024 13:25:34 +0200 Subject: [PATCH 6/7] fix(test): target pydantic v2 only in arbitrary type --- .../event_handler/_pydantic/test_openapi_encoders.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/functional/event_handler/_pydantic/test_openapi_encoders.py b/tests/functional/event_handler/_pydantic/test_openapi_encoders.py index 8a43ac7901d..7714b9efee6 100644 --- a/tests/functional/event_handler/_pydantic/test_openapi_encoders.py +++ b/tests/functional/event_handler/_pydantic/test_openapi_encoders.py @@ -3,7 +3,7 @@ from typing import List import pytest -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict from pydantic.color import Color from aws_lambda_powertools.event_handler.openapi.encoders import jsonable_encoder @@ -230,6 +230,7 @@ def serializer(value): assert result == ["serialized"] +@pytest.mark.usefixtures("pydanticv2_only") def test_openapi_encode_custom_serializer_pydantic(): # GIVEN a sequence with a custom class class CustomClass: @@ -238,9 +239,7 @@ class CustomClass: class Order(BaseModel): kind: CustomClass - # maintenance: deprecate in V3; becomes model_config =ConfigDict(=True) - class Config: - arbitrary_types_allowed = True + model_config = ConfigDict(arbitrary_types_allowed=True) order = Order(kind=CustomClass()) From 498e60a77621832fc9cf61c1ecd72cddc3029a84 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Tue, 2 Jul 2024 12:36:19 +0100 Subject: [PATCH 7/7] Customer must handle serialization on Pydantic models --- .../event_handler/openapi/encoders.py | 3 --- noxfile.py | 2 +- .../_pydantic/test_openapi_encoders.py | 26 +------------------ 3 files changed, 2 insertions(+), 29 deletions(-) diff --git a/aws_lambda_powertools/event_handler/openapi/encoders.py b/aws_lambda_powertools/event_handler/openapi/encoders.py index 9846252ff5f..bfa6c56b3b9 100644 --- a/aws_lambda_powertools/event_handler/openapi/encoders.py +++ b/aws_lambda_powertools/event_handler/openapi/encoders.py @@ -81,7 +81,6 @@ def jsonable_encoder( # noqa: PLR0911 exclude_unset=exclude_unset, exclude_none=exclude_none, exclude_defaults=exclude_defaults, - custom_serializer=custom_serializer, ) # Dataclasses @@ -174,7 +173,6 @@ def _dump_base_model( exclude_unset: bool = False, exclude_none: bool = False, exclude_defaults: bool = False, - custom_serializer: Optional[Callable[[Any], str]] = None, ): """ Dump a BaseModel object to a dict, using the same parameters as jsonable_encoder @@ -196,7 +194,6 @@ def _dump_base_model( obj_dict, exclude_none=exclude_none, exclude_defaults=exclude_defaults, - custom_serializer=custom_serializer, ) diff --git a/noxfile.py b/noxfile.py index 68882470de5..7023f45a2b7 100644 --- a/noxfile.py +++ b/noxfile.py @@ -148,7 +148,7 @@ def test_with_aws_encryption_sdk_as_required_package(session: nox.Session): @nox.session() -@nox.parametrize("pydantic", ["1.10", "2.0"]) +@nox.parametrize("pydantic", ["1.10,<2.0", "2.0"]) def test_with_pydantic_required_package(session: nox.Session, pydantic: str): """Tests that only depends for Pydantic library v1 and v2""" # Event Handler OpenAPI diff --git a/tests/functional/event_handler/_pydantic/test_openapi_encoders.py b/tests/functional/event_handler/_pydantic/test_openapi_encoders.py index 7714b9efee6..81f7299b04f 100644 --- a/tests/functional/event_handler/_pydantic/test_openapi_encoders.py +++ b/tests/functional/event_handler/_pydantic/test_openapi_encoders.py @@ -3,7 +3,7 @@ from typing import List import pytest -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel from pydantic.color import Color from aws_lambda_powertools.event_handler.openapi.encoders import jsonable_encoder @@ -230,30 +230,6 @@ def serializer(value): assert result == ["serialized"] -@pytest.mark.usefixtures("pydanticv2_only") -def test_openapi_encode_custom_serializer_pydantic(): - # GIVEN a sequence with a custom class - class CustomClass: - __slots__ = [] - - class Order(BaseModel): - kind: CustomClass - - model_config = ConfigDict(arbitrary_types_allowed=True) - - order = Order(kind=CustomClass()) - - # AND a custom serializer - def serializer(value): - return "serialized" - - # WHEN we call jsonable_encoder with the nested dictionary and unserializable value - result = jsonable_encoder(order, custom_serializer=serializer) - - # THEN we should get the custom serializer output - assert result == {"kind": "serialized"} - - def test_openapi_encode_custom_serializer_dataclasses(): # GIVEN a sequence with a custom class class CustomClass: