From db99580ebbd847b3728339e04a50c9993b79e84e Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Tue, 25 Aug 2020 22:21:32 -0700 Subject: [PATCH 1/4] feat(parameters): transform = "auto" Add a new tranform type called "auto", which looks at the key name to autodetect the encoding of the values in dynamodb or the parameter store. Keys ending with ".json" are assumed to be "json" string Keys ending with ".binary" are assumed to be "binary" base64 decode strings --- .../utilities/parameters/base.py | 22 ++++++++-- tests/functional/test_utilities_parameters.py | 42 +++++++++++++++++++ 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/aws_lambda_powertools/utilities/parameters/base.py b/aws_lambda_powertools/utilities/parameters/base.py index 274cd96aace..aad1bbd3363 100644 --- a/aws_lambda_powertools/utilities/parameters/base.py +++ b/aws_lambda_powertools/utilities/parameters/base.py @@ -115,8 +115,8 @@ def get_multiple( Maximum age of the cached value transform: str, optional Optional transformation of the parameter value. Supported values - are "json" for JSON strings and "binary" for base 64 encoded - values. + are "json" for JSON strings, "binary" for base 64 encoded + values or "auto" which looks at the attribute key to determine the type. raise_on_transform_error: bool, optional Raises an exception if any transform fails, otherwise this will return a None value for each transform that failed @@ -145,7 +145,11 @@ def get_multiple( if transform is not None: for (key, value) in values.items(): - values[key] = transform_value(value, transform, raise_on_transform_error) + _transform = get_transform_method(key, transform) + if _transform is None: + continue + + values[key] = transform_value(value, _transform, raise_on_transform_error) self.store[key] = ExpirableValue(values, datetime.now() + timedelta(seconds=max_age),) @@ -159,6 +163,18 @@ def _get_multiple(self, path: str, **sdk_options) -> Dict[str, str]: raise NotImplementedError() +def get_transform_method(key: str, transform: str) -> Union[str, None]: + if transform != "auto": + return transform + + if key.endswith(".json"): + return "json" + elif key.endswith(".binary"): + return "binary" + else: + return None + + def transform_value(value: str, transform: str, raise_on_transform_error: bool = True) -> Union[dict, bytes, None]: """ Apply a transform to a value diff --git a/tests/functional/test_utilities_parameters.py b/tests/functional/test_utilities_parameters.py index abd121540a6..a55f9da86ab 100644 --- a/tests/functional/test_utilities_parameters.py +++ b/tests/functional/test_utilities_parameters.py @@ -233,6 +233,48 @@ def test_dynamodb_provider_get_multiple(mock_name, mock_value, config): stubber.deactivate() +def test_dynamodb_provider_get_multiple_auto(mock_name, mock_value, config): + """ + Test DynamoDBProvider.get_multiple() with transform = "auto" + """ + mock_binary = mock_value.encode() + mock_binary_data = base64.b64encode(mock_binary).decode() + mock_json_data = json.dumps({mock_name: mock_value}) + mock_params = {"D.json": mock_json_data, "E.binary": mock_binary_data, "F": mock_value} + table_name = "TEST_TABLE_AUTO" + + # Create a new provider + provider = parameters.DynamoDBProvider(table_name, config=config) + + # Stub the boto3 client + stubber = stub.Stubber(provider.table.meta.client) + response = { + "Items": [ + {"id": {"S": mock_name}, "sk": {"S": name}, "value": {"S": value}} for (name, value) in mock_params.items() + ] + } + expected_params = {"TableName": table_name, "KeyConditionExpression": Key("id").eq(mock_name)} + stubber.add_response("query", response, expected_params) + stubber.activate() + + try: + values = provider.get_multiple(mock_name, transform="auto") + + stubber.assert_no_pending_responses() + + assert len(values) == len(mock_params) + for key in mock_params.keys(): + assert key in values + if key.endswith(".json"): + assert values[key][mock_name] == mock_value + elif key.endswith(".binary"): + assert values[key] == mock_binary + else: + assert values[key] == mock_value + finally: + stubber.deactivate() + + def test_dynamodb_provider_get_multiple_next_token(mock_name, mock_value, config): """ Test DynamoDBProvider.get_multiple() with a non-cached path From d8ea4e1a2ccd0801683c4f35b237be4df46d66e9 Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Wed, 26 Aug 2020 18:33:22 -0700 Subject: [PATCH 2/4] tests: include some additional tests and docs --- .../utilities/parameters/base.py | 29 ++++++++++++++++- tests/functional/test_utilities_parameters.py | 31 +++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/aws_lambda_powertools/utilities/parameters/base.py b/aws_lambda_powertools/utilities/parameters/base.py index aad1bbd3363..e1ba9ac25b1 100644 --- a/aws_lambda_powertools/utilities/parameters/base.py +++ b/aws_lambda_powertools/utilities/parameters/base.py @@ -163,7 +163,34 @@ def _get_multiple(self, path: str, **sdk_options) -> Dict[str, str]: raise NotImplementedError() -def get_transform_method(key: str, transform: str) -> Union[str, None]: +def get_transform_method(key: str, transform: str) -> Optional[str]: + """ + Determine the transform method + + Examples + ------- + >>> get_transform_method("key", "any_other_value") + 'any_other_value' + >>> get_transform_method("key.json", "auto") + 'json' + >>> get_transform_method("key.binary", "auto") + 'binary' + >>> get_transform_method("key", "auto") + None + + Parameters + --------- + key: str + Only used when the tranform is "auto". + transform: str + Original transform method, only "auto" will try to detect the transform method by the key + + Returns + ------ + Optional[str]: + The transform method either when transform is "auto" then None, "json" or "binary" is returned + or the original transform method + """ if transform != "auto": return transform diff --git a/tests/functional/test_utilities_parameters.py b/tests/functional/test_utilities_parameters.py index a55f9da86ab..744c7708185 100644 --- a/tests/functional/test_utilities_parameters.py +++ b/tests/functional/test_utilities_parameters.py @@ -1523,3 +1523,34 @@ def test_transform_value_ignore_error(mock_value): value = parameters.base.transform_value(mock_value, "INCORRECT", raise_on_transform_error=False) assert value is None + + +@pytest.mark.parametrize("original_transform", ["json", "binary", "other", "Auto"]) +def test_get_transform_method_preserve_original(original_transform): + """ + Check if original transform method is returned for anything other than "auto" + """ + transform = parameters.base.get_transform_method("key", original_transform) + + assert transform == original_transform + + +@pytest.mark.parametrize("extension", ["json", "binary"]) +def test_get_transform_method_preserve_auto(extension, mock_name): + """ + Check if we can auto detect the transform method by the support extensions json / binary + """ + transform = parameters.base.get_transform_method(f"{mock_name}.{extension}", "auto") + + assert transform == extension + + +@pytest.mark.parametrize("key", ["json", "binary", "example", "example.jsonp"]) +def test_get_transform_method_preserve_auto_unhandled(key): + """ + Check if any key that does not end with a supported extension returns None when + using the transform="auto" + """ + transform = parameters.base.get_transform_method(key, "auto") + + assert transform is None From b6116642c5f448faac856b53e30589697f80ff2a Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Tue, 1 Sep 2020 09:19:38 -0700 Subject: [PATCH 3/4] feat(parameters): add list of supported transforms Changes: * base.py - Add `SUPPORTED_TRANSFORM_METHODS` as a list of supported transform methods and add constants for the currently supported methods * base.py - Update `get_transform_method` to use `SUPPORTED_TRANSFORM_METHODS` to check for supported extensions --- .../utilities/parameters/base.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/aws_lambda_powertools/utilities/parameters/base.py b/aws_lambda_powertools/utilities/parameters/base.py index e1ba9ac25b1..126943204c5 100644 --- a/aws_lambda_powertools/utilities/parameters/base.py +++ b/aws_lambda_powertools/utilities/parameters/base.py @@ -15,6 +15,9 @@ ExpirableValue = namedtuple("ExpirableValue", ["value", "ttl"]) # These providers will be dynamically initialized on first use of the helper functions DEFAULT_PROVIDERS = {} +TRANSFORM_METHOD_JSON = "json" +TRANSFORM_METHOD_BINARY = "binary" +SUPPORTED_TRANSFORM_METHODS = [TRANSFORM_METHOD_JSON, TRANSFORM_METHOD_BINARY] class BaseProvider(ABC): @@ -194,12 +197,10 @@ def get_transform_method(key: str, transform: str) -> Optional[str]: if transform != "auto": return transform - if key.endswith(".json"): - return "json" - elif key.endswith(".binary"): - return "binary" - else: - return None + for transform_method in SUPPORTED_TRANSFORM_METHODS: + if key.endswith("." + transform_method): + return transform_method + return None def transform_value(value: str, transform: str, raise_on_transform_error: bool = True) -> Union[dict, bytes, None]: @@ -223,9 +224,9 @@ def transform_value(value: str, transform: str, raise_on_transform_error: bool = """ try: - if transform == "json": + if transform == TRANSFORM_METHOD_JSON: return json.loads(value) - elif transform == "binary": + elif transform == TRANSFORM_METHOD_BINARY: return base64.b64decode(value) else: raise ValueError(f"Invalid transform type '{transform}'") From 01e991ff0af9150135c4c0b0a451ab6034095181 Mon Sep 17 00:00:00 2001 From: Michael Brewer Date: Tue, 1 Sep 2020 11:31:48 -0700 Subject: [PATCH 4/4] feat: Make `transform` optional Changes: * Make `transform` optional in `get_transform_method` * Update tests to include this use case --- aws_lambda_powertools/utilities/parameters/base.py | 6 ++++-- tests/functional/test_utilities_parameters.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/aws_lambda_powertools/utilities/parameters/base.py b/aws_lambda_powertools/utilities/parameters/base.py index 126943204c5..7ce0c9e4d2e 100644 --- a/aws_lambda_powertools/utilities/parameters/base.py +++ b/aws_lambda_powertools/utilities/parameters/base.py @@ -166,7 +166,7 @@ def _get_multiple(self, path: str, **sdk_options) -> Dict[str, str]: raise NotImplementedError() -def get_transform_method(key: str, transform: str) -> Optional[str]: +def get_transform_method(key: str, transform: Optional[str] = None) -> Optional[str]: """ Determine the transform method @@ -180,12 +180,14 @@ def get_transform_method(key: str, transform: str) -> Optional[str]: 'binary' >>> get_transform_method("key", "auto") None + >>> get_transform_method("key", None) + None Parameters --------- key: str Only used when the tranform is "auto". - transform: str + transform: str, optional Original transform method, only "auto" will try to detect the transform method by the key Returns diff --git a/tests/functional/test_utilities_parameters.py b/tests/functional/test_utilities_parameters.py index 744c7708185..55f643924ad 100644 --- a/tests/functional/test_utilities_parameters.py +++ b/tests/functional/test_utilities_parameters.py @@ -1525,7 +1525,7 @@ def test_transform_value_ignore_error(mock_value): assert value is None -@pytest.mark.parametrize("original_transform", ["json", "binary", "other", "Auto"]) +@pytest.mark.parametrize("original_transform", ["json", "binary", "other", "Auto", None]) def test_get_transform_method_preserve_original(original_transform): """ Check if original transform method is returned for anything other than "auto"