feat: throw exception on failed transform for parameter utility

Author: nmoutschen

aws_lambda_powertools/utilities/parameters/__init__.py

@@ -6,7 +6,7 @@
 
 from .base import BaseProvider
 from .dynamodb import DynamoDBProvider
-from .exceptions import GetParameterError
+from .exceptions import GetParameterError, TransformParameterError
 from .secrets import SecretsProvider, get_secret
 from .ssm import SSMProvider, get_parameter, get_parameters
 
@@ -16,6 +16,7 @@
     "DynamoDBProvider",
     "SecretsProvider",
     "SSMProvider",
+    "TransformParameterError",
     "get_parameter",
     "get_parameters",
     "get_secret",

aws_lambda_powertools/utilities/parameters/base.py

@@ -9,7 +9,7 @@
 from datetime import datetime, timedelta
 from typing import Dict, Optional, Union
 
-from .exceptions import GetParameterError
+from .exceptions import GetParameterError, TransformParameterError
 
 DEFAULT_MAX_AGE_SECS = 5
 ExpirableValue = namedtuple("ExpirableValue", ["value", "ttl"])
@@ -32,14 +32,13 @@ def __init__(self):
         self.store = {}
 
     def get(
-        self, name: str, max_age: int = DEFAULT_MAX_AGE_SECS, transform: Optional[str] = None, **kwargs
+        self, name: str, max_age: int = DEFAULT_MAX_AGE_SECS, transform: Optional[str] = None, **sdk_options
     ) -> Union[str, list, dict, bytes]:
         """
         Retrieve a parameter value or return the cached value
 
         Parameters
         ----------
-
         name: str
             Parameter name
         max_age: int
@@ -51,10 +50,11 @@ def get(
 
         Raises
         ------
-
         GetParameterError
             When the parameter provider fails to retrieve a parameter value for
             a given name.
+        TransformParameterError
+            When the parameter provider fails to transform a parameter value.
         """
 
         # If there are multiple calls to the same parameter but in a different
@@ -70,54 +70,81 @@ def get(
 
         if key not in self.store or self.store[key].ttl < datetime.now():
             try:
-                value = self._get(name, **kwargs)
+                value = self._get(name, **sdk_options)
             # Encapsulate all errors into a generic GetParameterError
             except Exception as exc:
                 raise GetParameterError(str(exc))
 
-            if transform == "json":
-                value = json.loads(value)
-            elif transform == "binary":
-                value = base64.b64decode(value)
+            try:
+                if transform == "json":
+                    value = json.loads(value)
+                elif transform == "binary":
+                    value = base64.b64decode(value)
+            # Encapsulate transform exceptions into TransformParameterError
+            except Exception as exc:
+                raise TransformParameterError(str(exc))
 
             self.store[key] = ExpirableValue(value, datetime.now() + timedelta(seconds=max_age),)
 
         return self.store[key].value
 
     @abstractmethod
-    def _get(self, name: str, **kwargs) -> str:
+    def _get(self, name: str, **sdk_options) -> str:
         """
         Retrieve paramater value from the underlying parameter store
         """
         raise NotImplementedError()
 
     def get_multiple(
-        self, path: str, max_age: int = DEFAULT_MAX_AGE_SECS, transform: Optional[str] = None, **kwargs
+        self, path: str, max_age: int = DEFAULT_MAX_AGE_SECS, transform: Optional[str] = None, **sdk_options
     ) -> Union[Dict[str, str], Dict[str, dict], Dict[str, bytes]]:
         """
         Retrieve multiple parameters based on a path prefix
+
+        Parameters
+        ----------
+        path: str
+            Parameter path used to retrieve multiple parameters
+        max_age: int
+            Maximum age of the cached value
+        transform: str
+            Optional transformation of the parameter value. Supported values
+            are "json" for JSON strings and "binary" for base 64 encoded
+            values.
+
+        Raises
+        ------
+        GetParameterError
+            When the parameter provider fails to retrieve parameter values for
+            a given path.
+        TransformParameterError
+            When the parameter provider fails to transform a parameter value.
         """
 
         key = (path, transform)
 
         if key not in self.store or self.store[key].ttl < datetime.now():
             try:
-                values = self._get_multiple(path, **kwargs)
+                values = self._get_multiple(path, **sdk_options)
             # Encapsulate all errors into a generic GetParameterError
             except Exception as exc:
                 raise GetParameterError(str(exc))
 
-            if transform == "json":
-                values = {k: json.loads(v) for k, v in values.items()}
-            elif transform == "binary":
-                values = {k: base64.b64decode(v) for k, v in values.items()}
+            try:
+                if transform == "json":
+                    values = {k: json.loads(v) for k, v in values.items()}
+                elif transform == "binary":
+                    values = {k: base64.b64decode(v) for k, v in values.items()}
+            # Encapsulate transform exceptions into TransformParameterError
+            except Exception as exc:
+                raise TransformParameterError(str(exc))
 
             self.store[key] = ExpirableValue(values, datetime.now() + timedelta(seconds=max_age),)
 
         return self.store[key].value
 
     @abstractmethod
-    def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]:
+    def _get_multiple(self, path: str, **sdk_options) -> Dict[str, str]:
         """
         Retrieve multiple parameter values from the underlying parameter store
         """

aws_lambda_powertools/utilities/parameters/exceptions.py

@@ -5,3 +5,7 @@
 
 class GetParameterError(Exception):
     """When a provider raises an exception on parameter retrieval"""
+
+
+class TransformParameterError(Exception):
+    """When a provider fails to transform a parameter value"""

tests/functional/test_utilities_parameters.py

@@ -957,6 +957,48 @@ def test_secrets_provider_get_sdk_options_overwrite(mock_name, mock_value, confi
         stubber.deactivate()
 
 
+def test_base_provider_get_exception(mock_name):
+    """
+    Test BaseProvider.get() that raises an exception
+    """
+
+    class TestProvider(BaseProvider):
+        def _get(self, name: str, **kwargs) -> str:
+            assert name == mock_name
+            raise Exception("test exception raised")
+
+        def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]:
+            raise NotImplementedError()
+
+    provider = TestProvider()
+
+    with pytest.raises(parameters.GetParameterError) as excinfo:
+        provider.get(mock_name)
+
+    assert "test exception raised" in str(excinfo)
+
+
+def test_base_provider_get_multiple_exception(mock_name):
+    """
+    Test BaseProvider.get_multiple() that raises an exception
+    """
+
+    class TestProvider(BaseProvider):
+        def _get(self, name: str, **kwargs) -> str:
+            raise NotImplementedError()
+
+        def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]:
+            assert path == mock_name
+            raise Exception("test exception raised")
+
+    provider = TestProvider()
+
+    with pytest.raises(parameters.GetParameterError) as excinfo:
+        provider.get_multiple(mock_name)
+
+    assert "test exception raised" in str(excinfo)
+
+
 def test_base_provider_get_transform_json(mock_name, mock_value):
     """
     Test BaseProvider.get() with a json transform
@@ -981,55 +1023,60 @@ def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]:
     assert value[mock_name] == mock_value
 
 
-def test_base_provider_get_exception(mock_name):
+def test_base_provider_get_transform_json_exception(mock_name, mock_value):
     """
-    Test BaseProvider.get() that raises an exception
+    Test BaseProvider.get() with a json transform that raises an exception
     """
 
+    mock_data = json.dumps({mock_name: mock_value}) + "{"
+
     class TestProvider(BaseProvider):
         def _get(self, name: str, **kwargs) -> str:
             assert name == mock_name
-            raise Exception("test exception raised")
+            return mock_data
 
         def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]:
             raise NotImplementedError()
 
     provider = TestProvider()
 
-    with pytest.raises(parameters.GetParameterError) as excinfo:
-        provider.get(mock_name)
+    with pytest.raises(parameters.TransformParameterError) as excinfo:
+        provider.get(mock_name, transform="json")
 
-    assert "test exception raised" in str(excinfo)
+    assert "Extra data" in str(excinfo)
 
 
-def test_base_provider_get_multiple_exception(mock_name):
+def test_base_provider_get_transform_binary(mock_name, mock_value):
     """
-    Test BaseProvider.get_multiple() that raises an exception
+    Test BaseProvider.get() with a binary transform
     """
 
+    mock_binary = mock_value.encode()
+    mock_data = base64.b64encode(mock_binary).decode()
+
     class TestProvider(BaseProvider):
         def _get(self, name: str, **kwargs) -> str:
-            raise NotImplementedError()
+            assert name == mock_name
+            return mock_data
 
         def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]:
-            assert path == mock_name
-            raise Exception("test exception raised")
+            raise NotImplementedError()
 
     provider = TestProvider()
 
-    with pytest.raises(parameters.GetParameterError) as excinfo:
-        provider.get_multiple(mock_name)
+    value = provider.get(mock_name, transform="binary")
 
-    assert "test exception raised" in str(excinfo)
+    assert isinstance(value, bytes)
+    assert value == mock_binary
 
 
-def test_base_provider_get_transform_binary(mock_name, mock_value):
+def test_base_provider_get_transform_binary_exception(mock_name):
     """
-    Test BaseProvider.get() with a binary transform
+    Test BaseProvider.get() with a binary transform that raises an exception
     """
 
-    mock_binary = mock_value.encode()
-    mock_data = base64.b64encode(mock_binary).decode()
+    mock_data = "qw"
+    print(mock_data)
 
     class TestProvider(BaseProvider):
         def _get(self, name: str, **kwargs) -> str:
@@ -1041,10 +1088,10 @@ def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]:
 
     provider = TestProvider()
 
-    value = provider.get(mock_name, transform="binary")
+    with pytest.raises(parameters.TransformParameterError) as excinfo:
+        provider.get(mock_name, transform="binary")
 
-    assert isinstance(value, bytes)
-    assert value == mock_binary
+    assert "Incorrect padding" in str(excinfo)
 
 
 def test_base_provider_get_multiple_transform_json(mock_name, mock_value):
@@ -1070,6 +1117,29 @@ def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]:
     assert value["A"][mock_name] == mock_value
 
 
+def test_base_provider_get_multiple_transform_json_exception(mock_name, mock_value):
+    """
+    Test BaseProvider.get_multiple() with a json transform that raises an exception
+    """
+
+    mock_data = json.dumps({mock_name: mock_value}) + "{"
+
+    class TestProvider(BaseProvider):
+        def _get(self, name: str, **kwargs) -> str:
+            raise NotImplementedError()
+
+        def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]:
+            assert path == mock_name
+            return {"A": mock_data}
+
+    provider = TestProvider()
+
+    with pytest.raises(parameters.TransformParameterError) as excinfo:
+        provider.get_multiple(mock_name, transform="json")
+
+    assert "Extra data" in str(excinfo)
+
+
 def test_base_provider_get_multiple_transform_binary(mock_name, mock_value):
     """
     Test BaseProvider.get_multiple() with a binary transform
@@ -1094,6 +1164,29 @@ def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]:
     assert value["A"] == mock_binary
 
 
+def test_base_provider_get_multiple_transform_binary_exception(mock_name):
+    """
+    Test BaseProvider.get_multiple() with a binary transform that raises an exception
+    """
+
+    mock_data = "qw"
+
+    class TestProvider(BaseProvider):
+        def _get(self, name: str, **kwargs) -> str:
+            raise NotImplementedError()
+
+        def _get_multiple(self, path: str, **kwargs) -> Dict[str, str]:
+            assert path == mock_name
+            return {"A": mock_data}
+
+    provider = TestProvider()
+
+    with pytest.raises(parameters.TransformParameterError) as excinfo:
+        provider.get_multiple(mock_name, transform="binary")
+
+    assert "Incorrect padding" in str(excinfo)
+
+
 def test_base_provider_get_multiple_cached(mock_name, mock_value):
     """
     Test BaseProvider.get_multiple() with cached values