From a9ad45e16205f3e8abcb4dc4465ab2423327d8e0 Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Sat, 18 Dec 2021 19:17:35 -0800 Subject: [PATCH 1/7] feat(event-sources): cache parsed json in data class A micro optimization to cache the parsed json within the event source data class --- aws_lambda_powertools/utilities/data_classes/common.py | 10 +++++++++- tests/functional/test_data_classes.py | 4 +++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_classes/common.py b/aws_lambda_powertools/utilities/data_classes/common.py index f209fc8c192..22f14e17428 100644 --- a/aws_lambda_powertools/utilities/data_classes/common.py +++ b/aws_lambda_powertools/utilities/data_classes/common.py @@ -45,6 +45,10 @@ def get_header_value( class BaseProxyEvent(DictWrapper): + def __init__(self, data: Dict[str, Any]): + super().__init__(data) + self._parsed_json_body: Optional[Any] = None + @property def headers(self) -> Dict[str, str]: return self["headers"] @@ -65,7 +69,11 @@ def body(self) -> Optional[str]: @property def json_body(self) -> Any: """Parses the submitted body as json""" - return json.loads(self.decoded_body) + if self._parsed_json_body: + return self._parsed_json_body + + self._parsed_json_body = json.loads(self.decoded_body) + return self._parsed_json_body @property def decoded_body(self) -> str: diff --git a/tests/functional/test_data_classes.py b/tests/functional/test_data_classes.py index 7a211ec2e01..2e3be8ac16d 100644 --- a/tests/functional/test_data_classes.py +++ b/tests/functional/test_data_classes.py @@ -1053,7 +1053,9 @@ def test_base_proxy_event_json_body_key_error(): def test_base_proxy_event_json_body(): data = {"message": "Foo"} event = BaseProxyEvent({"body": json.dumps(data)}) + assert event._parsed_json_body is None assert event.json_body == data + assert event.json_body == event._parsed_json_body == data def test_base_proxy_event_decode_body_key_error(): @@ -1084,7 +1086,7 @@ def test_base_proxy_event_json_body_with_base64_encoded_data(): event = BaseProxyEvent({"body": encoded_data, "isBase64Encoded": True}) # WHEN calling json_body - # THEN then base64 decode and json load + # THEN base64 decode and json load assert event.json_body == data From f84c33e4807edfd99b002c1a9782d8414325eb0c Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Sun, 19 Dec 2021 22:30:12 -0800 Subject: [PATCH 2/7] refactor: allow for singleton caching where possible --- .../utilities/data_classes/active_mq_event.py | 4 +++- .../data_classes/code_pipeline_job_event.py | 4 +++- .../utilities/data_classes/common.py | 13 ++++--------- .../utilities/data_classes/kinesis_stream_event.py | 4 +++- .../utilities/data_classes/rabbit_mq_event.py | 4 +++- tests/functional/test_data_classes.py | 2 -- 6 files changed, 16 insertions(+), 15 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_classes/active_mq_event.py b/aws_lambda_powertools/utilities/data_classes/active_mq_event.py index 058a6a6ecf4..09981bdcdd2 100644 --- a/aws_lambda_powertools/utilities/data_classes/active_mq_event.py +++ b/aws_lambda_powertools/utilities/data_classes/active_mq_event.py @@ -27,7 +27,9 @@ def decoded_data(self) -> str: @property def json_data(self) -> Any: """Parses the data as json""" - return json.loads(self.decoded_data) + if self._json_data is None: + self._json_data = json.loads(self.decoded_data) + return self._json_data @property def connection_id(self) -> str: diff --git a/aws_lambda_powertools/utilities/data_classes/code_pipeline_job_event.py b/aws_lambda_powertools/utilities/data_classes/code_pipeline_job_event.py index e13d32fb169..e17bd13807c 100644 --- a/aws_lambda_powertools/utilities/data_classes/code_pipeline_job_event.py +++ b/aws_lambda_powertools/utilities/data_classes/code_pipeline_job_event.py @@ -23,7 +23,9 @@ def user_parameters(self) -> str: @property def decoded_user_parameters(self) -> Dict[str, Any]: """Json Decoded user parameters""" - return json.loads(self.user_parameters) + if self._json_data is None: + self._json_data = json.loads(self.user_parameters) + return self._json_data class CodePipelineActionConfiguration(DictWrapper): diff --git a/aws_lambda_powertools/utilities/data_classes/common.py b/aws_lambda_powertools/utilities/data_classes/common.py index 22f14e17428..88d1f1d9761 100644 --- a/aws_lambda_powertools/utilities/data_classes/common.py +++ b/aws_lambda_powertools/utilities/data_classes/common.py @@ -8,6 +8,7 @@ class DictWrapper: def __init__(self, data: Dict[str, Any]): self._data = data + self._json_data: Optional[Any] = None def __getitem__(self, key: str) -> Any: return self._data[key] @@ -45,10 +46,6 @@ def get_header_value( class BaseProxyEvent(DictWrapper): - def __init__(self, data: Dict[str, Any]): - super().__init__(data) - self._parsed_json_body: Optional[Any] = None - @property def headers(self) -> Dict[str, str]: return self["headers"] @@ -69,11 +66,9 @@ def body(self) -> Optional[str]: @property def json_body(self) -> Any: """Parses the submitted body as json""" - if self._parsed_json_body: - return self._parsed_json_body - - self._parsed_json_body = json.loads(self.decoded_body) - return self._parsed_json_body + if self._json_data is None: + self._json_data = json.loads(self.decoded_body) + return self._json_data @property def decoded_body(self) -> str: diff --git a/aws_lambda_powertools/utilities/data_classes/kinesis_stream_event.py b/aws_lambda_powertools/utilities/data_classes/kinesis_stream_event.py index ec45bfbd0b2..84b151856c7 100644 --- a/aws_lambda_powertools/utilities/data_classes/kinesis_stream_event.py +++ b/aws_lambda_powertools/utilities/data_classes/kinesis_stream_event.py @@ -41,7 +41,9 @@ def data_as_text(self) -> str: def data_as_json(self) -> dict: """Decode binary encoded data as json""" - return json.loads(self.data_as_text()) + if self._json_data is None: + self._json_data = json.loads(self.data_as_text()) + return self._json_data class KinesisStreamRecord(DictWrapper): diff --git a/aws_lambda_powertools/utilities/data_classes/rabbit_mq_event.py b/aws_lambda_powertools/utilities/data_classes/rabbit_mq_event.py index 7676e6ff9b5..0822a58da18 100644 --- a/aws_lambda_powertools/utilities/data_classes/rabbit_mq_event.py +++ b/aws_lambda_powertools/utilities/data_classes/rabbit_mq_event.py @@ -88,7 +88,9 @@ def decoded_data(self) -> str: @property def json_data(self) -> Any: """Parses the data as json""" - return json.loads(self.decoded_data) + if self._json_data is None: + self._json_data = json.loads(self.decoded_data) + return self._json_data class RabbitMQEvent(DictWrapper): diff --git a/tests/functional/test_data_classes.py b/tests/functional/test_data_classes.py index 2e3be8ac16d..9e3e2029c63 100644 --- a/tests/functional/test_data_classes.py +++ b/tests/functional/test_data_classes.py @@ -1053,9 +1053,7 @@ def test_base_proxy_event_json_body_key_error(): def test_base_proxy_event_json_body(): data = {"message": "Foo"} event = BaseProxyEvent({"body": json.dumps(data)}) - assert event._parsed_json_body is None assert event.json_body == data - assert event.json_body == event._parsed_json_body == data def test_base_proxy_event_decode_body_key_error(): From dd7bcd92b148dd10b0f06a45a48ca8a6a845e8f3 Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Sun, 19 Dec 2021 22:42:46 -0800 Subject: [PATCH 3/7] chore: add missing test cases --- tests/functional/data_classes/test_amazon_mq.py | 2 ++ tests/functional/test_data_classes.py | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/functional/data_classes/test_amazon_mq.py b/tests/functional/data_classes/test_amazon_mq.py index 0f4f5079565..24b96fc006a 100644 --- a/tests/functional/data_classes/test_amazon_mq.py +++ b/tests/functional/data_classes/test_amazon_mq.py @@ -34,6 +34,7 @@ def test_active_mq_event(): messages = list(event.messages) message = messages[1] assert message.json_data["timeout"] == 0 + assert message.json_data["timeout"] == 0 # cached lookup def test_rabbit_mq_event(): @@ -47,6 +48,7 @@ def test_rabbit_mq_event(): assert message.data is not None assert message.decoded_data is not None assert message.json_data["timeout"] == 0 + assert message.json_data["timeout"] == 0 # cached lookup assert isinstance(message, RabbitMessage) properties = message.basic_properties diff --git a/tests/functional/test_data_classes.py b/tests/functional/test_data_classes.py index 9e3e2029c63..7d4da000011 100644 --- a/tests/functional/test_data_classes.py +++ b/tests/functional/test_data_classes.py @@ -1120,7 +1120,9 @@ def test_kinesis_stream_event_json_data(): json_value = {"test": "value"} data = base64.b64encode(bytes(json.dumps(json_value), "utf-8")).decode("utf-8") event = KinesisStreamEvent({"Records": [{"kinesis": {"data": data}}]}) - assert next(event.records).kinesis.data_as_json() == json_value + record = next(event.records) + assert record.kinesis.data_as_json() == json_value + assert record.kinesis.data_as_json() == json_value # cached lookup def test_alb_event(): From bc0bf39e944bdedc386bd2817f2cd10691184241 Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Sun, 19 Dec 2021 23:08:33 -0800 Subject: [PATCH 4/7] chore: more coverage --- tests/functional/test_data_classes.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/functional/test_data_classes.py b/tests/functional/test_data_classes.py index 7d4da000011..94581d8ae02 100644 --- a/tests/functional/test_data_classes.py +++ b/tests/functional/test_data_classes.py @@ -1054,6 +1054,7 @@ def test_base_proxy_event_json_body(): data = {"message": "Foo"} event = BaseProxyEvent({"body": json.dumps(data)}) assert event.json_body == data + assert event.json_body == data # cached lookup def test_base_proxy_event_decode_body_key_error(): @@ -1394,10 +1395,14 @@ def test_code_pipeline_event_decoded_data(): event = CodePipelineJobEvent(load_event("codePipelineEventData.json")) assert event.data.continuation_token is None - decoded_params = event.data.action_configuration.configuration.decoded_user_parameters + configuration = event.data.action_configuration.configuration + decoded_params = configuration.decoded_user_parameters assert decoded_params == event.decoded_user_parameters assert "VALUE" == decoded_params["KEY"] + decoded_params = configuration.decoded_user_parameters # cached lookup + assert decoded_params is not None + assert "my-pipeline-SourceArtifact" == event.data.input_artifacts[0].name output_artifacts = event.data.output_artifacts From 57ba15c9203d8e8eb5862307129828a27e562bcf Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Sun, 19 Dec 2021 23:22:46 -0800 Subject: [PATCH 5/7] tests: more missing cached lookups --- tests/functional/test_data_classes.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/functional/test_data_classes.py b/tests/functional/test_data_classes.py index 94581d8ae02..050cbadbc17 100644 --- a/tests/functional/test_data_classes.py +++ b/tests/functional/test_data_classes.py @@ -272,6 +272,8 @@ def test_cognito_pre_token_generation_trigger_event(): claims_override_details.set_group_configuration_groups_to_override(expected_groups) assert claims_override_details.group_configuration.groups_to_override == expected_groups assert event["response"]["claimsOverrideDetails"]["groupOverrideDetails"]["groupsToOverride"] == expected_groups + claims_override_details = event.response.claims_override_details # cached lookups + assert claims_override_details["groupOverrideDetails"]["groupsToOverride"] == expected_groups claims_override_details.set_group_configuration_iam_roles_to_override(["role"]) assert claims_override_details.group_configuration.iam_roles_to_override == ["role"] From 396fdb2905478e7356b9d89fe51fcae32ff29aff Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Sun, 19 Dec 2021 23:42:01 -0800 Subject: [PATCH 6/7] chore: revert caching logic --- .../utilities/data_classes/kinesis_stream_event.py | 4 +--- tests/functional/test_data_classes.py | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_classes/kinesis_stream_event.py b/aws_lambda_powertools/utilities/data_classes/kinesis_stream_event.py index 84b151856c7..ec45bfbd0b2 100644 --- a/aws_lambda_powertools/utilities/data_classes/kinesis_stream_event.py +++ b/aws_lambda_powertools/utilities/data_classes/kinesis_stream_event.py @@ -41,9 +41,7 @@ def data_as_text(self) -> str: def data_as_json(self) -> dict: """Decode binary encoded data as json""" - if self._json_data is None: - self._json_data = json.loads(self.data_as_text()) - return self._json_data + return json.loads(self.data_as_text()) class KinesisStreamRecord(DictWrapper): diff --git a/tests/functional/test_data_classes.py b/tests/functional/test_data_classes.py index 050cbadbc17..54cb7759103 100644 --- a/tests/functional/test_data_classes.py +++ b/tests/functional/test_data_classes.py @@ -1125,7 +1125,6 @@ def test_kinesis_stream_event_json_data(): event = KinesisStreamEvent({"Records": [{"kinesis": {"data": data}}]}) record = next(event.records) assert record.kinesis.data_as_json() == json_value - assert record.kinesis.data_as_json() == json_value # cached lookup def test_alb_event(): From 7a623f262e3ee98b2ef1783a949bc6c820bbd424 Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Tue, 21 Dec 2021 00:25:18 -0800 Subject: [PATCH 7/7] chore: update tests --- tests/functional/data_classes/test_amazon_mq.py | 4 ++-- tests/functional/test_data_classes.py | 10 ++++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/functional/data_classes/test_amazon_mq.py b/tests/functional/data_classes/test_amazon_mq.py index 24b96fc006a..a88a962c17b 100644 --- a/tests/functional/data_classes/test_amazon_mq.py +++ b/tests/functional/data_classes/test_amazon_mq.py @@ -34,7 +34,7 @@ def test_active_mq_event(): messages = list(event.messages) message = messages[1] assert message.json_data["timeout"] == 0 - assert message.json_data["timeout"] == 0 # cached lookup + assert message.json_data["data"] == "CZrmf0Gw8Ov4bqLQxD4E" def test_rabbit_mq_event(): @@ -48,7 +48,7 @@ def test_rabbit_mq_event(): assert message.data is not None assert message.decoded_data is not None assert message.json_data["timeout"] == 0 - assert message.json_data["timeout"] == 0 # cached lookup + assert message.json_data["data"] == "CZrmf0Gw8Ov4bqLQxD4E" assert isinstance(message, RabbitMessage) properties = message.basic_properties diff --git a/tests/functional/test_data_classes.py b/tests/functional/test_data_classes.py index 54cb7759103..86d9344ca4d 100644 --- a/tests/functional/test_data_classes.py +++ b/tests/functional/test_data_classes.py @@ -272,7 +272,7 @@ def test_cognito_pre_token_generation_trigger_event(): claims_override_details.set_group_configuration_groups_to_override(expected_groups) assert claims_override_details.group_configuration.groups_to_override == expected_groups assert event["response"]["claimsOverrideDetails"]["groupOverrideDetails"]["groupsToOverride"] == expected_groups - claims_override_details = event.response.claims_override_details # cached lookups + claims_override_details = event.response.claims_override_details assert claims_override_details["groupOverrideDetails"]["groupsToOverride"] == expected_groups claims_override_details.set_group_configuration_iam_roles_to_override(["role"]) @@ -1056,7 +1056,7 @@ def test_base_proxy_event_json_body(): data = {"message": "Foo"} event = BaseProxyEvent({"body": json.dumps(data)}) assert event.json_body == data - assert event.json_body == data # cached lookup + assert event.json_body["message"] == "Foo" def test_base_proxy_event_decode_body_key_error(): @@ -1399,10 +1399,8 @@ def test_code_pipeline_event_decoded_data(): configuration = event.data.action_configuration.configuration decoded_params = configuration.decoded_user_parameters assert decoded_params == event.decoded_user_parameters - assert "VALUE" == decoded_params["KEY"] - - decoded_params = configuration.decoded_user_parameters # cached lookup - assert decoded_params is not None + assert decoded_params["KEY"] == "VALUE" + assert configuration.decoded_user_parameters["KEY"] == "VALUE" assert "my-pipeline-SourceArtifact" == event.data.input_artifacts[0].name