diff --git a/Makefile b/Makefile index 0ee0ee76fbd..4cee795ea30 100644 --- a/Makefile +++ b/Makefile @@ -90,3 +90,10 @@ changelog: mypy: poetry run mypy --pretty aws_lambda_powertools + +format-examples: + poetry run isort docs/examples + poetry run black docs/examples/*/*/*.py + +lint-examples: + poetry run python3 -m py_compile docs/examples/*/*/*.py diff --git a/docs/examples/utilities/parameters/app_config_provider.py b/docs/examples/utilities/parameters/app_config_provider.py new file mode 100644 index 00000000000..24a1eaea6f9 --- /dev/null +++ b/docs/examples/utilities/parameters/app_config_provider.py @@ -0,0 +1,15 @@ +from botocore.config import Config + +from aws_lambda_powertools.utilities import parameters + +config = Config(region_name="us-west-1") +appconf_provider = parameters.AppConfigProvider( + environment="my_env", + application="my_app", + config=config, +) + + +def handler(event, context): + # Retrieve a single secret + value: bytes = appconf_provider.get("my_conf") diff --git a/docs/examples/utilities/parameters/create_your_own_s3_provider.py b/docs/examples/utilities/parameters/create_your_own_s3_provider.py new file mode 100644 index 00000000000..28cc584c9ae --- /dev/null +++ b/docs/examples/utilities/parameters/create_your_own_s3_provider.py @@ -0,0 +1,53 @@ +import copy +from typing import Dict + +import boto3 + +from aws_lambda_powertools.utilities import BaseProvider + + +class S3Provider(BaseProvider): + bucket_name = None + client = None + + def __init__(self, bucket_name: str): + # Initialize the client to your custom parameter store + # E.g.: + + self.bucket_name = bucket_name + self.client = boto3.client("s3") + + def _get(self, name: str, **sdk_options) -> str: + # Retrieve a single value + # E.g.: + + sdk_options["Bucket"] = self.bucket_name + sdk_options["Key"] = name + + response = self.client.get_object(**sdk_options) + return + + def _get_multiple(self, path: str, **sdk_options) -> Dict[str, str]: + # Retrieve multiple values + # E.g.: + + list_sdk_options = copy.deepcopy(sdk_options) + + list_sdk_options["Bucket"] = self.bucket_name + list_sdk_options["Prefix"] = path + + list_response = self.client.list_objects_v2(**list_sdk_options) + + parameters = {} + + for obj in list_response.get("Contents", []): + get_sdk_options = copy.deepcopy(sdk_options) + + get_sdk_options["Bucket"] = self.bucket_name + get_sdk_options["Key"] = obj["Key"] + + get_response = self.client.get_object(**get_sdk_options) + + parameters[obj["Key"]] = get_response["Body"].read().decode() + + return parameters diff --git a/docs/examples/utilities/parameters/custom_caching_parameters.py b/docs/examples/utilities/parameters/custom_caching_parameters.py new file mode 100644 index 00000000000..d603484a02d --- /dev/null +++ b/docs/examples/utilities/parameters/custom_caching_parameters.py @@ -0,0 +1,16 @@ +from botocore.config import Config + +from aws_lambda_powertools.utilities import parameters + +config = Config(region_name="us-west-1") +ssm_provider = parameters.SSMProvider(config=config) + + +def handler(event, context): + # Retrieve a single parameter + value = ssm_provider.get("/my/parameter", max_age=60) # 1 minute + + # Retrieve multiple parameters from a path prefix + values = ssm_provider.get_multiple("/my/path/prefix", max_age=60) + for k, v in values.items(): + print(f"{k}: {v}") diff --git a/docs/examples/utilities/parameters/dynamodb_provider.py b/docs/examples/utilities/parameters/dynamodb_provider.py new file mode 100644 index 00000000000..f17d7456008 --- /dev/null +++ b/docs/examples/utilities/parameters/dynamodb_provider.py @@ -0,0 +1,8 @@ +from aws_lambda_powertools.utilities import parameters + +dynamodb_provider = parameters.DynamoDBProvider(table_name="my-table") + + +def handler(event, context): + # Retrieve a value from DynamoDB + value = dynamodb_provider.get("my-parameter") diff --git a/docs/examples/utilities/parameters/dynamodb_provider_customization.py b/docs/examples/utilities/parameters/dynamodb_provider_customization.py new file mode 100644 index 00000000000..9b67658d2cf --- /dev/null +++ b/docs/examples/utilities/parameters/dynamodb_provider_customization.py @@ -0,0 +1,12 @@ +from aws_lambda_powertools.utilities import parameters + +dynamodb_provider = parameters.DynamoDBProvider( + table_name="my-table", + key_attr="MyKeyAttr", + sort_attr="MySortAttr", + value_attr="MyvalueAttr", +) + + +def handler(event, context): + value = dynamodb_provider.get("my-parameter") diff --git a/docs/examples/utilities/parameters/dynamodb_provider_get_multiple.py b/docs/examples/utilities/parameters/dynamodb_provider_get_multiple.py new file mode 100644 index 00000000000..31e3d43981b --- /dev/null +++ b/docs/examples/utilities/parameters/dynamodb_provider_get_multiple.py @@ -0,0 +1,13 @@ +from aws_lambda_powertools.utilities import parameters + +dynamodb_provider = parameters.DynamoDBProvider(table_name="my-table") + + +def handler(event, context): + # Retrieve multiple values by performing a Query on the DynamoDB table + # This returns a dict with the sort key attribute as dict key. + parameters = dynamodb_provider.get_multiple("my-hash-key") + for k, v in parameters.items(): + # k: param-a + # v: "my-value-a" + print(f"{k}: {v}") diff --git a/docs/examples/utilities/parameters/dynamodb_provider_local.py b/docs/examples/utilities/parameters/dynamodb_provider_local.py new file mode 100644 index 00000000000..b7e7dca8829 --- /dev/null +++ b/docs/examples/utilities/parameters/dynamodb_provider_local.py @@ -0,0 +1,6 @@ +from aws_lambda_powertools.utilities import parameters + +dynamodb_provider = parameters.DynamoDBProvider( + table_name="my-table", + endpoint_url="http://localhost:8000", +) diff --git a/docs/examples/utilities/parameters/fetching_app_config.py b/docs/examples/utilities/parameters/fetching_app_config.py new file mode 100644 index 00000000000..b6ea5014e10 --- /dev/null +++ b/docs/examples/utilities/parameters/fetching_app_config.py @@ -0,0 +1,10 @@ +from aws_lambda_powertools.utilities import parameters + + +def handler(event, context): + # Retrieve a single configuration, latest version + value: bytes = parameters.get_app_config( + name="my_configuration", + environment="my_env", + application="my_app", + ) diff --git a/docs/examples/utilities/parameters/fetching_secrets.py b/docs/examples/utilities/parameters/fetching_secrets.py new file mode 100644 index 00000000000..a62ba094c65 --- /dev/null +++ b/docs/examples/utilities/parameters/fetching_secrets.py @@ -0,0 +1,6 @@ +from aws_lambda_powertools.utilities import parameters + + +def handler(event, context): + # Retrieve a single secret + value = parameters.get_secret("my-secret") diff --git a/docs/examples/utilities/parameters/force_fetch_parameters.py b/docs/examples/utilities/parameters/force_fetch_parameters.py new file mode 100644 index 00000000000..d049c1f0c41 --- /dev/null +++ b/docs/examples/utilities/parameters/force_fetch_parameters.py @@ -0,0 +1,6 @@ +from aws_lambda_powertools.utilities import parameters + + +def handler(event, context): + # Retrieve a single parameter + value = parameters.get_parameter("/my/parameter", force_fetch=True) diff --git a/docs/examples/utilities/parameters/parameters_boto3_client.py b/docs/examples/utilities/parameters/parameters_boto3_client.py new file mode 100644 index 00000000000..1acdcc194ed --- /dev/null +++ b/docs/examples/utilities/parameters/parameters_boto3_client.py @@ -0,0 +1,17 @@ +import boto3 +from botocore.config import Config + +from aws_lambda_powertools.utilities import parameters + +config = Config(region_name="us-west-1") + +# construct boto clients with any custom configuration +ssm = boto3.client("ssm", config=config) +secrets = boto3.client("secrets", config=config) +appconfig = boto3.client("appconfig", config=config) +dynamodb = boto3.resource("dynamodb", config=config) + +ssm_provider = parameters.SSMProvider(boto3_client=ssm) +secrets_provider = parameters.SecretsProvider(boto3_client=secrets) +appconf_provider = parameters.AppConfigProvider(boto3_client=appconfig, environment="my_env", application="my_app") +dynamodb_provider = parameters.DynamoDBProvider(boto3_client=dynamodb, table_name="my-table") diff --git a/docs/examples/utilities/parameters/parameters_custom_client.py b/docs/examples/utilities/parameters/parameters_custom_client.py new file mode 100644 index 00000000000..a062a81ca59 --- /dev/null +++ b/docs/examples/utilities/parameters/parameters_custom_client.py @@ -0,0 +1,12 @@ +import boto3 + +from aws_lambda_powertools.utilities import parameters + +boto3_client = boto3.client("ssm") +ssm_provider = parameters.SSMProvider(boto3_client=boto3_client) + + +def handler(event, context): + # Retrieve a single parameter + value = ssm_provider.get("/my/parameter") + ... diff --git a/docs/examples/utilities/parameters/parameters_custom_config.py b/docs/examples/utilities/parameters/parameters_custom_config.py new file mode 100644 index 00000000000..79510421617 --- /dev/null +++ b/docs/examples/utilities/parameters/parameters_custom_config.py @@ -0,0 +1,12 @@ +from botocore.config import Config + +from aws_lambda_powertools.utilities import parameters + +boto_config = Config() +ssm_provider = parameters.SSMProvider(config=boto_config) + + +def handler(event, context): + # Retrieve a single parameter + value = ssm_provider.get("/my/parameter") + ... diff --git a/docs/examples/utilities/parameters/parameters_custom_session.py b/docs/examples/utilities/parameters/parameters_custom_session.py new file mode 100644 index 00000000000..d76fffe00ac --- /dev/null +++ b/docs/examples/utilities/parameters/parameters_custom_session.py @@ -0,0 +1,12 @@ +import boto3 + +from aws_lambda_powertools.utilities import parameters + +boto3_session = boto3.session.Session() +ssm_provider = parameters.SSMProvider(boto3_session=boto3_session) + + +def handler(event, context): + # Retrieve a single parameter + value = ssm_provider.get("/my/parameter") + ... diff --git a/docs/examples/utilities/parameters/parameters_sdk_args.py b/docs/examples/utilities/parameters/parameters_sdk_args.py new file mode 100644 index 00000000000..571d070de8a --- /dev/null +++ b/docs/examples/utilities/parameters/parameters_sdk_args.py @@ -0,0 +1,8 @@ +from aws_lambda_powertools.utilities import parameters + +secrets_provider = parameters.SecretsProvider() + + +def handler(event, context): + # The 'VersionId' argument will be passed to the underlying get_secret_value() call. + value = secrets_provider.get("my-secret", VersionId="e62ec170-6b01-48c7-94f3-d7497851a8d2") diff --git a/docs/examples/utilities/parameters/parameters_transform.py b/docs/examples/utilities/parameters/parameters_transform.py new file mode 100644 index 00000000000..eb7c3be0d8a --- /dev/null +++ b/docs/examples/utilities/parameters/parameters_transform.py @@ -0,0 +1,5 @@ +from aws_lambda_powertools.utilities import parameters + + +def handler(event, context): + value_from_json = parameters.get_parameter("/my/json/parameter", transform="json") diff --git a/docs/examples/utilities/parameters/parameters_transform_auto.py b/docs/examples/utilities/parameters/parameters_transform_auto.py new file mode 100644 index 00000000000..f85e6aa1b13 --- /dev/null +++ b/docs/examples/utilities/parameters/parameters_transform_auto.py @@ -0,0 +1,7 @@ +from aws_lambda_powertools.utilities import parameters + +ssm_provider = parameters.SSMProvider() + + +def handler(event, context): + values = ssm_provider.get_multiple("/param", transform="auto") diff --git a/docs/examples/utilities/parameters/parameters_transform_providers.py b/docs/examples/utilities/parameters/parameters_transform_providers.py new file mode 100644 index 00000000000..2451330e9c7 --- /dev/null +++ b/docs/examples/utilities/parameters/parameters_transform_providers.py @@ -0,0 +1,11 @@ +from aws_lambda_powertools.utilities import parameters + +ssm_provider = parameters.SSMProvider() + + +def handler(event, context): + # Transform a JSON string + value_from_json = ssm_provider.get("/my/json/parameter", transform="json") + + # Transform a Base64 encoded string + value_from_binary = ssm_provider.get("/my/binary/parameter", transform="binary") diff --git a/docs/examples/utilities/parameters/parameters_transform_raise_on_transform_error.py b/docs/examples/utilities/parameters/parameters_transform_raise_on_transform_error.py new file mode 100644 index 00000000000..542ad754439 --- /dev/null +++ b/docs/examples/utilities/parameters/parameters_transform_raise_on_transform_error.py @@ -0,0 +1,19 @@ +from aws_lambda_powertools.utilities import parameters + +ssm_provider = parameters.SSMProvider() + + +def handler(event, context): + # This will display: + # /param/a: [some value] + # /param/b: [some value] + # /param/c: None + values = ssm_provider.get_multiple("/param", transform="json") + for k, v in values.items(): + print(f"{k}: {v}") + + try: + # This will raise a TransformParameterError exception + values = ssm_provider.get_multiple("/param", transform="json", raise_on_transform_error=True) + except parameters.exceptions.TransformParameterError: + ... diff --git a/docs/examples/utilities/parameters/recursively_parameters.py b/docs/examples/utilities/parameters/recursively_parameters.py new file mode 100644 index 00000000000..5ff2fcb7161 --- /dev/null +++ b/docs/examples/utilities/parameters/recursively_parameters.py @@ -0,0 +1,12 @@ +from aws_lambda_powertools.utilities import parameters + + +def handler(event, context): + # Retrieve a single parameter + value = parameters.get_parameter("/my/parameter") + + # Retrieve multiple parameters from a path prefix recursively + # This returns a dict with the parameter name as key + values = parameters.get_parameters("/my/path/prefix") + for k, v in values.items(): + print(f"{k}: {v}") diff --git a/docs/examples/utilities/parameters/secrets_provider.py b/docs/examples/utilities/parameters/secrets_provider.py new file mode 100644 index 00000000000..f775ff3a7b8 --- /dev/null +++ b/docs/examples/utilities/parameters/secrets_provider.py @@ -0,0 +1,11 @@ +from botocore.config import Config + +from aws_lambda_powertools.utilities import parameters + +config = Config(region_name="us-west-1") +secrets_provider = parameters.SecretsProvider(config=config) + + +def handler(event, context): + # Retrieve a single secret + value = secrets_provider.get("my-secret") diff --git a/docs/examples/utilities/parameters/ssm_provider.py b/docs/examples/utilities/parameters/ssm_provider.py new file mode 100644 index 00000000000..b83b2bafa16 --- /dev/null +++ b/docs/examples/utilities/parameters/ssm_provider.py @@ -0,0 +1,16 @@ +from botocore.config import Config + +from aws_lambda_powertools.utilities import parameters + +config = Config(region_name="us-west-1") +ssm_provider = parameters.SSMProvider(config=config) # or boto3_session=boto3.Session() + + +def handler(event, context): + # Retrieve a single parameter + value = ssm_provider.get("/my/parameter") + + # Retrieve multiple parameters from a path prefix + values = ssm_provider.get_multiple("/my/path/prefix") + for k, v in values.items(): + print(f"{k}: {v}") diff --git a/docs/examples/utilities/parameters/ssm_provider_get_options.py b/docs/examples/utilities/parameters/ssm_provider_get_options.py new file mode 100644 index 00000000000..0dbcc743411 --- /dev/null +++ b/docs/examples/utilities/parameters/ssm_provider_get_options.py @@ -0,0 +1,9 @@ +from aws_lambda_powertools.utilities import parameters + +ssm_provider = parameters.SSMProvider() + + +def handler(event, context): + decrypted_value = ssm_provider.get("/my/encrypted/parameter", decrypt=True) + + no_recursive_values = ssm_provider.get_multiple("/my/path/prefix", recursive=False) diff --git a/docs/examples/utilities/parameters/testing_parameters_app.py b/docs/examples/utilities/parameters/testing_parameters_app.py new file mode 100644 index 00000000000..b505749e31b --- /dev/null +++ b/docs/examples/utilities/parameters/testing_parameters_app.py @@ -0,0 +1,10 @@ +from botocore.config import Config + +from aws_lambda_powertools.utilities import parameters + +ssm_provider = parameters.SSMProvider(config=Config(region_name="us-west-1")) + + +def handler(event, context): + value = ssm_provider.get("/my/parameter") + return {"message": value} diff --git a/docs/examples/utilities/parameters/testing_parameters_clear_cache.py b/docs/examples/utilities/parameters/testing_parameters_clear_cache.py new file mode 100644 index 00000000000..0d590a3938c --- /dev/null +++ b/docs/examples/utilities/parameters/testing_parameters_clear_cache.py @@ -0,0 +1,22 @@ +import pytest +from src import app + + +@pytest.fixture(scope="function", autouse=True) +def clear_parameters_cache(): + yield + app.ssm_provider.clear_cache() # This will clear SSMProvider cache + + +@pytest.fixture +def mock_parameter_response(monkeypatch): + def mockreturn(name): + return "mock_value" + + monkeypatch.setattr(app.ssm_provider, "get", mockreturn) + + +# Pass our fixture as an argument to all tests where we want to mock the get_parameter response +def test_handler(mock_parameter_response): + return_val = app.handler({}, {}) + assert return_val.get("message") == "mock_value" diff --git a/docs/examples/utilities/parameters/testing_parameters_fixture.py b/docs/examples/utilities/parameters/testing_parameters_fixture.py new file mode 100644 index 00000000000..2a872172252 --- /dev/null +++ b/docs/examples/utilities/parameters/testing_parameters_fixture.py @@ -0,0 +1,16 @@ +import pytest +from src import index + + +@pytest.fixture +def mock_parameter_response(monkeypatch): + def mockreturn(name): + return "mock_value" + + monkeypatch.setattr(index.parameters, "get_parameter", mockreturn) + + +# Pass our fixture as an argument to all tests where we want to mock the get_parameter response +def test_handler(mock_parameter_response): + return_val = index.handler({}, {}) + assert return_val.get("message") == "mock_value" diff --git a/docs/examples/utilities/parameters/testing_parameters_global_clear_caches.py b/docs/examples/utilities/parameters/testing_parameters_global_clear_caches.py new file mode 100644 index 00000000000..27457889ee9 --- /dev/null +++ b/docs/examples/utilities/parameters/testing_parameters_global_clear_caches.py @@ -0,0 +1,24 @@ +import pytest +from src import app + +from aws_lambda_powertools.utilities import parameters + + +@pytest.fixture(scope="function", autouse=True) +def clear_parameters_cache(): + yield + parameters.clear_caches() # This will clear all providers cache + + +@pytest.fixture +def mock_parameter_response(monkeypatch): + def mockreturn(name): + return "mock_value" + + monkeypatch.setattr(app.ssm_provider, "get", mockreturn) + + +# Pass our fixture as an argument to all tests where we want to mock the get_parameter response +def test_handler(mock_parameter_response): + return_val = app.handler({}, {}) + assert return_val.get("message") == "mock_value" diff --git a/docs/examples/utilities/parameters/testing_parameters_index.py b/docs/examples/utilities/parameters/testing_parameters_index.py new file mode 100644 index 00000000000..ef94df54f25 --- /dev/null +++ b/docs/examples/utilities/parameters/testing_parameters_index.py @@ -0,0 +1,7 @@ +from aws_lambda_powertools.utilities import parameters + + +def handler(event, context): + # Retrieve a single parameter + value = parameters.get_parameter("my-parameter-name") + return {"message": value} diff --git a/docs/examples/utilities/parameters/testing_parameters_mock.py b/docs/examples/utilities/parameters/testing_parameters_mock.py new file mode 100644 index 00000000000..8ac789fd5a1 --- /dev/null +++ b/docs/examples/utilities/parameters/testing_parameters_mock.py @@ -0,0 +1,13 @@ +from unittest.mock import patch + +from src import index + + +# Replaces "aws_lambda_powertools.utilities.parameters.get_parameter" with a Mock object +@patch("aws_lambda_powertools.utilities.parameters.get_parameter") +def test_handler(get_parameter_mock): + get_parameter_mock.return_value = "mock_value" + + return_val = index.handler({}, {}) + get_parameter_mock.assert_called_with("my-parameter-name") + assert return_val.get("message") == "mock_value" diff --git a/docs/examples/utilities/parameters/testing_parameters_tests.py b/docs/examples/utilities/parameters/testing_parameters_tests.py new file mode 100644 index 00000000000..0a7679d990c --- /dev/null +++ b/docs/examples/utilities/parameters/testing_parameters_tests.py @@ -0,0 +1,10 @@ +from src import index + + +def test_handler(monkeypatch): + def mockreturn(name): + return "mock_value" + + monkeypatch.setattr(index.parameters, "get_parameter", mockreturn) + return_val = index.handler({}, {}) + assert return_val.get("message") == "mock_value" diff --git a/docs/utilities/parameters.md b/docs/utilities/parameters.md index 36990fdd2cb..8a3e8318761 100644 --- a/docs/utilities/parameters.md +++ b/docs/utilities/parameters.md @@ -39,30 +39,16 @@ You can retrieve a single parameter using `get_parameter` high-level function. For multiple parameters, you can use `get_parameters` and pass a path to retrieve them recursively. -```python hl_lines="1 5 9" title="Fetching multiple parameters recursively" -from aws_lambda_powertools.utilities import parameters - -def handler(event, context): - # Retrieve a single parameter - value = parameters.get_parameter("/my/parameter") - - # Retrieve multiple parameters from a path prefix recursively - # This returns a dict with the parameter name as key - values = parameters.get_parameters("/my/path/prefix") - for k, v in values.items(): - print(f"{k}: {v}") +```python hl_lines="1 6 10" title="Fetching multiple parameters recursively" +--8<-- "docs/examples/utilities/parameters/recursively_parameters.py" ``` ### Fetching secrets You can fetch secrets stored in Secrets Manager using `get_secrets`. -```python hl_lines="1 5" title="Fetching secrets" -from aws_lambda_powertools.utilities import parameters - -def handler(event, context): - # Retrieve a single secret - value = parameters.get_secret("my-secret") +```python hl_lines="1 6" title="Fetching secrets" +--8<-- "docs/examples/utilities/parameters/fetching_secrets.py" ``` ### Fetching app configurations @@ -71,12 +57,8 @@ You can fetch application configurations in AWS AppConfig using `get_app_config` The following will retrieve the latest version and store it in the cache. -```python hl_lines="1 5" title="Fetching latest config from AppConfig" -from aws_lambda_powertools.utilities import parameters - -def handler(event, context): - # Retrieve a single configuration, latest version - value: bytes = parameters.get_app_config(name="my_configuration", environment="my_env", application="my_app") +```python hl_lines="1 6-10" title="Fetching latest config from AppConfig" +--8<-- "docs/examples/utilities/parameters/fetching_app_config.py" ``` ## Advanced @@ -90,33 +72,16 @@ By default, we cache parameters retrieved in-memory for 5 seconds. You can adjust how long we should keep values in cache by using the param `max_age`, when using `get()` or `get_multiple()` methods across all providers. -```python hl_lines="9" title="Caching parameter(s) value in memory for longer than 5 seconds" -from aws_lambda_powertools.utilities import parameters -from botocore.config import Config - -config = Config(region_name="us-west-1") -ssm_provider = parameters.SSMProvider(config=config) - -def handler(event, context): - # Retrieve a single parameter - value = ssm_provider.get("/my/parameter", max_age=60) # 1 minute - - # Retrieve multiple parameters from a path prefix - values = ssm_provider.get_multiple("/my/path/prefix", max_age=60) - for k, v in values.items(): - print(f"{k}: {v}") +```python hl_lines="11 14" title="Caching parameter(s) value in memory for longer than 5 seconds" +--8<-- "docs/examples/utilities/parameters/custom_caching_parameters.py" ``` ### Always fetching the latest If you'd like to always ensure you fetch the latest parameter from the store regardless if already available in cache, use `force_fetch` param. -```python hl_lines="5" title="Forcefully fetching the latest parameter whether TTL has expired or not" -from aws_lambda_powertools.utilities import parameters - -def handler(event, context): - # Retrieve a single parameter - value = parameters.get_parameter("/my/parameter", force_fetch=True) +```python hl_lines="6" title="Forcefully fetching the latest parameter whether TTL has expired or not" +--8<-- "docs/examples/utilities/parameters/force_fetch_parameters.py" ``` ### Built-in provider class @@ -128,21 +93,8 @@ For greater flexibility such as configuring the underlying SDK client used by bu #### SSMProvider -```python hl_lines="5 9 12" title="Example with SSMProvider for further extensibility" -from aws_lambda_powertools.utilities import parameters -from botocore.config import Config - -config = Config(region_name="us-west-1") -ssm_provider = parameters.SSMProvider(config=config) # or boto3_session=boto3.Session() - -def handler(event, context): - # Retrieve a single parameter - value = ssm_provider.get("/my/parameter") - - # Retrieve multiple parameters from a path prefix - values = ssm_provider.get_multiple("/my/path/prefix") - for k, v in values.items(): - print(f"{k}: {v}") +```python hl_lines="6 11 14" title="Example with SSMProvider for further extensibility" +--8<-- "docs/examples/utilities/parameters/ssm_provider.py" ``` The AWS Systems Manager Parameter Store provider supports two additional arguments for the `get()` and `get_multiple()` methods: @@ -152,29 +104,14 @@ The AWS Systems Manager Parameter Store provider supports two additional argumen | **decrypt** | `False` | Will automatically decrypt the parameter. | | **recursive** | `True` | For `get_multiple()` only, will fetch all parameter values recursively based on a path prefix. | -```python hl_lines="6 8" title="Example with get() and get_multiple()" -from aws_lambda_powertools.utilities import parameters - -ssm_provider = parameters.SSMProvider() - -def handler(event, context): - decrypted_value = ssm_provider.get("/my/encrypted/parameter", decrypt=True) - - no_recursive_values = ssm_provider.get_multiple("/my/path/prefix", recursive=False) +```python hl_lines="7 9" title="Example with get() and get_multiple()" +--8<-- "docs/examples/utilities/parameters/ssm_provider_get_options.py" ``` #### SecretsProvider -```python hl_lines="5 9" title="Example with SecretsProvider for further extensibility" -from aws_lambda_powertools.utilities import parameters -from botocore.config import Config - -config = Config(region_name="us-west-1") -secrets_provider = parameters.SecretsProvider(config=config) - -def handler(event, context): - # Retrieve a single secret - value = secrets_provider.get("my-secret") +```python hl_lines="6 11" title="Example with SecretsProvider for further extensibility" +--8<-- "docs/examples/utilities/parameters/secrets_provider.py" ``` #### DynamoDBProvider @@ -196,23 +133,16 @@ For single parameters, you must use `id` as the [partition key](https://docs.aws With this table, `dynamodb_provider.get("my-param")` will return `my-value`. === "app.py" - ```python hl_lines="3 7" - from aws_lambda_powertools.utilities import parameters - - dynamodb_provider = parameters.DynamoDBProvider(table_name="my-table") - def handler(event, context): - # Retrieve a value from DynamoDB - value = dynamodb_provider.get("my-parameter") + ```python hl_lines="3 8" + --8<-- "docs/examples/utilities/parameters/dynamodb_provider.py" ``` === "DynamoDB Local example" You can initialize the DynamoDB provider pointing to [DynamoDB Local](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocal.html) using `endpoint_url` parameter: - ```python hl_lines="3" - from aws_lambda_powertools.utilities import parameters - - dynamodb_provider = parameters.DynamoDBProvider(table_name="my-table", endpoint_url="http://localhost:8000") + ```python hl_lines="5" + --8<-- "docs/examples/utilities/parameters/dynamodb_provider_local.py" ``` **DynamoDB table structure for multiple values parameters** @@ -232,19 +162,9 @@ You can retrieve multiple parameters sharing the same `id` by having a sort key With this table, `dynamodb_provider.get_multiple("my-hash-key")` will return a dictionary response in the shape of `sk:value`. === "app.py" - ```python hl_lines="3 8" - from aws_lambda_powertools.utilities import parameters - - dynamodb_provider = parameters.DynamoDBProvider(table_name="my-table") - - def handler(event, context): - # Retrieve multiple values by performing a Query on the DynamoDB table - # This returns a dict with the sort key attribute as dict key. - parameters = dynamodb_provider.get_multiple("my-hash-key") - for k, v in parameters.items(): - # k: param-a - # v: "my-value-a" - print(f"{k}: {v}") + + ```python hl_lines="3 9" + --8<-- "docs/examples/utilities/parameters/dynamodb_provider_get_multiple.py" ``` === "parameters dict response" @@ -269,31 +189,13 @@ DynamoDB provider can be customized at initialization to match your table struct | **value_attr** | No | `value` | Name of the attribute containing the parameter value. | ```python hl_lines="3-8" title="Customizing DynamoDBProvider to suit your table design" -from aws_lambda_powertools.utilities import parameters - -dynamodb_provider = parameters.DynamoDBProvider( - table_name="my-table", - key_attr="MyKeyAttr", - sort_attr="MySortAttr", - value_attr="MyvalueAttr" -) - -def handler(event, context): - value = dynamodb_provider.get("my-parameter") +--8<-- "docs/examples/utilities/parameters/dynamodb_provider_customization.py" ``` #### AppConfigProvider -```python hl_lines="5 9" title="Using AppConfigProvider" -from aws_lambda_powertools.utilities import parameters -from botocore.config import Config - -config = Config(region_name="us-west-1") -appconf_provider = parameters.AppConfigProvider(environment="my_env", application="my_app", config=config) - -def handler(event, context): - # Retrieve a single secret - value: bytes = appconf_provider.get("my_conf") +```python hl_lines="6-10 15" title="Using AppConfigProvider" +--8<-- "docs/examples/utilities/parameters/app_config_provider.py" ``` ### Create your own provider @@ -304,57 +206,8 @@ All transformation and caching logic is handled by the `get()` and `get_multiple Here is an example implementation using S3 as a custom parameter store: -```python hl_lines="3 6 17 27" title="Creating a S3 Provider to fetch parameters" -import copy - -from aws_lambda_powertools.utilities import BaseProvider -import boto3 - -class S3Provider(BaseProvider): - bucket_name = None - client = None - - def __init__(self, bucket_name: str): - # Initialize the client to your custom parameter store - # E.g.: - - self.bucket_name = bucket_name - self.client = boto3.client("s3") - - def _get(self, name: str, **sdk_options) -> str: - # Retrieve a single value - # E.g.: - - sdk_options["Bucket"] = self.bucket_name - sdk_options["Key"] = name - - response = self.client.get_object(**sdk_options) - return - - def _get_multiple(self, path: str, **sdk_options) -> Dict[str, str]: - # Retrieve multiple values - # E.g.: - - list_sdk_options = copy.deepcopy(sdk_options) - - list_sdk_options["Bucket"] = self.bucket_name - list_sdk_options["Prefix"] = path - - list_response = self.client.list_objects_v2(**list_sdk_options) - - parameters = {} - - for obj in list_response.get("Contents", []): - get_sdk_options = copy.deepcopy(sdk_options) - - get_sdk_options["Bucket"] = self.bucket_name - get_sdk_options["Key"] = obj["Key"] - - get_response = self.client.get_object(**get_sdk_options) - - parameters[obj["Key"]] = get_response["Body"].read().decode() - - return parameters +```python hl_lines="6 9 20 30" title="Creating a S3 Provider to fetch parameters" +--8<-- "docs/examples/utilities/parameters/create_your_own_s3_provider.py" ``` ### Deserializing values with transform parameter @@ -366,26 +219,14 @@ For parameters stored in JSON or Base64 format, you can use the `transform` argu === "High level functions" - ```python hl_lines="4" - from aws_lambda_powertools.utilities import parameters - - def handler(event, context): - value_from_json = parameters.get_parameter("/my/json/parameter", transform="json") + ```python hl_lines="5" + --8<-- "docs/examples/utilities/parameters/parameters_transform.py" ``` === "Providers" - ```python hl_lines="7 10" - from aws_lambda_powertools.utilities import parameters - - ssm_provider = parameters.SSMProvider() - - def handler(event, context): - # Transform a JSON string - value_from_json = ssm_provider.get("/my/json/parameter", transform="json") - - # Transform a Base64 encoded string - value_from_binary = ssm_provider.get("/my/binary/parameter", transform="binary") + ```python hl_lines="8 11" + --8<-- "docs/examples/utilities/parameters/parameters_transform_providers.py" ``` #### Partial transform failures with `get_multiple()` @@ -396,25 +237,8 @@ You can override this by setting the `raise_on_transform_error` argument to `Tru For example, if you have three parameters, */param/a*, */param/b* and */param/c*, but */param/c* is malformed: -```python hl_lines="9 16" title="Raising TransformParameterError at first malformed parameter" -from aws_lambda_powertools.utilities import parameters - -ssm_provider = parameters.SSMProvider() - -def handler(event, context): - # This will display: - # /param/a: [some value] - # /param/b: [some value] - # /param/c: None - values = ssm_provider.get_multiple("/param", transform="json") - for k, v in values.items(): - print(f"{k}: {v}") - - try: - # This will raise a TransformParameterError exception - values = ssm_provider.get_multiple("/param", transform="json", raise_on_transform_error=True) - except parameters.exceptions.TransformParameterError: - ... +```python hl_lines="11 17" title="Raising TransformParameterError at first malformed parameter" +--8<-- "docs/examples/utilities/parameters/parameters_transform_raise_on_transform_error.py" ``` #### Auto-transform values on suffix @@ -426,13 +250,8 @@ You can do this with a single request by using `transform="auto"`. This will ins ???+ info `transform="auto"` feature is available across all providers, including the high level functions. -```python hl_lines="6" title="Deserializing parameter values based on their suffix" -from aws_lambda_powertools.utilities import parameters - -ssm_provider = parameters.SSMProvider() - -def handler(event, context): - values = ssm_provider.get_multiple("/param", transform="auto") +```python hl_lines="7" title="Deserializing parameter values based on their suffix" +--8<-- "docs/examples/utilities/parameters/parameters_transform_auto.py" ``` For example, if you have two parameters with the following suffixes `.json` and `.binary`: @@ -455,14 +274,8 @@ The return of `ssm_provider.get_multiple("/param", transform="auto")` call will You can use arbitrary keyword arguments to pass it directly to the underlying SDK method. -```python hl_lines="8" title="" -from aws_lambda_powertools.utilities import parameters - -secrets_provider = parameters.SecretsProvider() - -def handler(event, context): - # The 'VersionId' argument will be passed to the underlying get_secret_value() call. - value = secrets_provider.get("my-secret", VersionId="e62ec170-6b01-48c7-94f3-d7497851a8d2") +```python hl_lines="7-8" title="" +--8<-- "docs/examples/utilities/parameters/parameters_sdk_args.py" ``` Here is the mapping between this utility's functions and methods and the underlying SDK: @@ -479,7 +292,6 @@ Here is the mapping between this utility's functions and methods and the underly | DynamoDB | `DynamoDBProvider.get_multiple` | `dynamodb` | ([Table resource](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#table)) | [query](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#DynamoDB.Table.query) | | App Config | `get_app_config` | `appconfig` | [get_configuration](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/appconfig.html#AppConfig.Client.get_configuration) | - ### Bring your own boto client You can use `boto3_client` parameter via any of the available [Provider Classes](#built-in-provider-class). Some providers expect a low level boto3 client while others expect a high level boto3 client, here is the mapping for each of them: @@ -491,28 +303,10 @@ You can use `boto3_client` parameter via any of the available [Provider Classes] | [AppConfigProvider](#appconfigprovider) | low level | `boto3.client("appconfig")` | | [DynamoDBProvider](#dynamodbprovider) | high level | `boto3.resource("dynamodb")` | - Bringing them together in a single code snippet would look like this: ```python title="Example: passing a custom boto3 client for each provider" -import boto3 -from botocore.config import Config - -from aws_lambda_powertools.utilities import parameters - -config = Config(region_name="us-west-1") - -# construct boto clients with any custom configuration -ssm = boto3.client("ssm", config=config) -secrets = boto3.client("secrets", config=config) -appconfig = boto3.client("appconfig", config=config) -dynamodb = boto3.resource("dynamodb", config=config) - -ssm_provider = parameters.SSMProvider(boto3_client=ssm) -secrets_provider = parameters.SecretsProvider(boto3_client=secrets) -appconf_provider = parameters.AppConfigProvider(boto3_client=appconfig, environment="my_env", application="my_app") -dynamodb_provider = parameters.DynamoDBProvider(boto3_client=dynamodb, table_name="my-table") - +--8<-- "docs/examples/utilities/parameters/parameters_boto3_client.py" ``` ???+ question "When is this useful?" @@ -529,49 +323,21 @@ The **`config`** , **`boto3_session`**, and **`boto3_client`** parameters enabl === "Custom session" - ```python hl_lines="2 4 5" - from aws_lambda_powertools.utilities import parameters - import boto3 - - boto3_session = boto3.session.Session() - ssm_provider = parameters.SSMProvider(boto3_session=boto3_session) - - def handler(event, context): - # Retrieve a single parameter - value = ssm_provider.get("/my/parameter") - ... + ```python hl_lines="1 5-6" + --8<-- "docs/examples/utilities/parameters/parameters_custom_session.py" ``` === "Custom config" - ```python hl_lines="2 4 5" - from aws_lambda_powertools.utilities import parameters - from botocore.config import Config - - boto_config = Config() - ssm_provider = parameters.SSMProvider(config=boto_config) - - def handler(event, context): - # Retrieve a single parameter - value = ssm_provider.get("/my/parameter") - ... + ```python hl_lines="1 5-6" + --8<-- "docs/examples/utilities/parameters/parameters_custom_config.py" ``` === "Custom client" - ```python hl_lines="2 4 5" - from aws_lambda_powertools.utilities import parameters - import boto3 - - boto3_client= boto3.client("ssm") - ssm_provider = parameters.SSMProvider(boto3_client=boto3_client) - - def handler(event, context): - # Retrieve a single parameter - value = ssm_provider.get("/my/parameter") - ... + ```python hl_lines="1 5-6" + --8<-- "docs/examples/utilities/parameters/parameters_custom_client.py" ``` - ## Testing your code ### Mocking parameter values @@ -579,49 +345,23 @@ The **`config`** , **`boto3_session`**, and **`boto3_client`** parameters enabl For unit testing your applications, you can mock the calls to the parameters utility to avoid calling AWS APIs. This can be achieved in a number of ways - in this example, we use the [pytest monkeypatch fixture](https://docs.pytest.org/en/latest/how-to/monkeypatch.html) to patch the `parameters.get_parameter` method: === "tests.py" - ```python - from src import index - - def test_handler(monkeypatch): - - def mockreturn(name): - return "mock_value" - monkeypatch.setattr(index.parameters, "get_parameter", mockreturn) - return_val = index.handler({}, {}) - assert return_val.get('message') == 'mock_value' + ```python + --8<-- "docs/examples/utilities/parameters/testing_parameters_tests.py" ``` === "src/index.py" - ```python - from aws_lambda_powertools.utilities import parameters - def handler(event, context): - # Retrieve a single parameter - value = parameters.get_parameter("my-parameter-name") - return {"message": value} + ```python + --8<-- "docs/examples/utilities/parameters/testing_parameters_index.py" ``` If we need to use this pattern across multiple tests, we can avoid repetition by refactoring to use our own pytest fixture: === "tests.py" - ```python - import pytest - - from src import index - - @pytest.fixture - def mock_parameter_response(monkeypatch): - def mockreturn(name): - return "mock_value" - - monkeypatch.setattr(index.parameters, "get_parameter", mockreturn) - - # Pass our fixture as an argument to all tests where we want to mock the get_parameter response - def test_handler(mock_parameter_response): - return_val = index.handler({}, {}) - assert return_val.get('message') == 'mock_value' + ```python + --8<-- "docs/examples/utilities/parameters/testing_parameters_fixture.py" ``` Alternatively, if we need more fully featured mocking (for example checking the arguments passed to `get_parameter`), we @@ -631,88 +371,26 @@ object named `get_parameter_mock`. === "tests.py" ```python - from unittest.mock import patch - from src import index - - # Replaces "aws_lambda_powertools.utilities.parameters.get_parameter" with a Mock object - @patch("aws_lambda_powertools.utilities.parameters.get_parameter") - def test_handler(get_parameter_mock): - get_parameter_mock.return_value = 'mock_value' - - return_val = index.handler({}, {}) - get_parameter_mock.assert_called_with("my-parameter-name") - assert return_val.get('message') == 'mock_value' - + --8<-- "docs/examples/utilities/parameters/testing_parameters_mock.py" ``` - ### Clearing cache Parameters utility caches all parameter values for performance and cost reasons. However, this can have unintended interference in tests using the same parameter name. Within your tests, you can use `clear_cache` method available in [every provider](#built-in-provider-class). When using multiple providers or higher level functions like `get_parameter`, use `clear_caches` standalone function to clear cache globally. - === "clear_cache method" - ```python hl_lines="9" - import pytest - - from src import app - - - @pytest.fixture(scope="function", autouse=True) - def clear_parameters_cache(): - yield - app.ssm_provider.clear_cache() # This will clear SSMProvider cache - - @pytest.fixture - def mock_parameter_response(monkeypatch): - def mockreturn(name): - return "mock_value" - - monkeypatch.setattr(app.ssm_provider, "get", mockreturn) - - # Pass our fixture as an argument to all tests where we want to mock the get_parameter response - def test_handler(mock_parameter_response): - return_val = app.handler({}, {}) - assert return_val.get('message') == 'mock_value' + ```python hl_lines="8" + --8<-- "docs/examples/utilities/parameters/testing_parameters_clear_cache.py" ``` === "global clear_caches" ```python hl_lines="10" - import pytest - - from aws_lambda_powertools.utilities import parameters - from src import app - - - @pytest.fixture(scope="function", autouse=True) - def clear_parameters_cache(): - yield - parameters.clear_caches() # This will clear all providers cache - - @pytest.fixture - def mock_parameter_response(monkeypatch): - def mockreturn(name): - return "mock_value" - - monkeypatch.setattr(app.ssm_provider, "get", mockreturn) - - # Pass our fixture as an argument to all tests where we want to mock the get_parameter response - def test_handler(mock_parameter_response): - return_val = app.handler({}, {}) - assert return_val.get('message') == 'mock_value' + --8<-- "docs/examples/utilities/parameters/testing_parameters_global_clear_caches.py" ``` === "app.py" ```python - from aws_lambda_powertools.utilities import parameters - from botocore.config import Config - - ssm_provider = parameters.SSMProvider(config=Config(region_name="us-west-1")) - - - def handler(event, context): - value = ssm_provider.get("/my/parameter") - return {"message": value} + --8<-- "docs/examples/utilities/parameters/testing_parameters_app.py" ```