Skip to content

feat(parameters): Configure max_age and decrypt parameters via environment variables #2088

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Apr 14, 2023
4 changes: 2 additions & 2 deletions aws_lambda_powertools/shared/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@

IDEMPOTENCY_DISABLED_ENV: str = "POWERTOOLS_IDEMPOTENCY_DISABLED"

PARAMETERS_DEFAULT_DECRYPT: str = "POWERTOOLS_PARAMETERS_SSM_DECRYPT"
PARAMETERS_MAX_AGE: str = "POWERTOOLS_PARAMETERS_MAX_AGE"
PARAMETERS_SSM_DECRYPT_ENV: str = "POWERTOOLS_PARAMETERS_SSM_DECRYPT"
PARAMETERS_MAX_AGE_ENV: str = "POWERTOOLS_PARAMETERS_MAX_AGE"

LOGGER_LAMBDA_CONTEXT_KEYS = [
"function_arn",
Expand Down
6 changes: 3 additions & 3 deletions aws_lambda_powertools/utilities/parameters/appconfig.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ def get_app_config(
Transforms the content from a JSON object ('json') or base64 binary string ('binary')
force_fetch: bool, optional
Force update even before a cached item has expired, defaults to False
max_age: int
max_age: int, optional
Maximum age of the cached value
sdk_options: dict, optional
SDK options to propagate to `start_configuration_session` API call
Expand Down Expand Up @@ -191,8 +191,8 @@ def get_app_config(
>>> print(value)
My configuration's JSON value
"""
# Resolving if will use the default value (5), the value passed by parameter or the environment variable
max_age = resolve_max_age(env=os.getenv(constants.PARAMETERS_MAX_AGE, DEFAULT_MAX_AGE_SECS), choice=max_age)
# If max_age is not set, resolve it from the environment variable, defaulting to DEFAULT_MAX_AGE_SECS
max_age = resolve_max_age(env=os.getenv(constants.PARAMETERS_MAX_AGE_ENV, DEFAULT_MAX_AGE_SECS), choice=max_age)

# Only create the provider if this function is called at least once
if "appconfig" not in DEFAULT_PROVIDERS:
Expand Down
9 changes: 4 additions & 5 deletions aws_lambda_powertools/utilities/parameters/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@
from mypy_boto3_ssm import SSMClient


# If the environment variable is not set, the default value is 5
DEFAULT_MAX_AGE_SECS = "5"

# These providers will be dynamically initialized on first use of the helper functions
Expand Down Expand Up @@ -126,8 +125,8 @@ def get(
value: Optional[Union[str, bytes, dict]] = None
key = (name, transform)

# Resolving if will use the default value (5), the value passed by parameter or the environment variable
max_age = resolve_max_age(env=os.getenv(constants.PARAMETERS_MAX_AGE, DEFAULT_MAX_AGE_SECS), choice=max_age)
# If max_age is not set, resolve it from the environment variable, defaulting to DEFAULT_MAX_AGE_SECS
max_age = resolve_max_age(env=os.getenv(constants.PARAMETERS_MAX_AGE_ENV, DEFAULT_MAX_AGE_SECS), choice=max_age)

if not force_fetch and self.has_not_expired_in_cache(key):
return self.store[key].value
Expand Down Expand Up @@ -194,8 +193,8 @@ def get_multiple(
"""
key = (path, transform)

# Resolving if will use the default value (5), the value passed by parameter or the environment variable
max_age = resolve_max_age(env=os.getenv(constants.PARAMETERS_MAX_AGE, DEFAULT_MAX_AGE_SECS), choice=max_age)
# If max_age is not set, resolve it from the environment variable, defaulting to DEFAULT_MAX_AGE_SECS
max_age = resolve_max_age(env=os.getenv(constants.PARAMETERS_MAX_AGE_ENV, DEFAULT_MAX_AGE_SECS), choice=max_age)

if not force_fetch and self.has_not_expired_in_cache(key):
return self.store[key].value # type: ignore # need to revisit entire typing here
Expand Down
6 changes: 3 additions & 3 deletions aws_lambda_powertools/utilities/parameters/secrets.py
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ def get_secret(
Transforms the content from a JSON object ('json') or base64 binary string ('binary')
force_fetch: bool, optional
Force update even before a cached item has expired, defaults to False
max_age: int
max_age: int, optional
Maximum age of the cached value
sdk_options: dict, optional
Dictionary of options that will be passed to the get_secret_value call
Expand Down Expand Up @@ -162,8 +162,8 @@ def get_secret(
>>> get_secret("my-secret", VersionId="f658cac0-98a5-41d9-b993-8a76a7799194")
"""

# Resolving if will use the default value (5), the value passed by parameter or the environment variable
max_age = resolve_max_age(env=os.getenv(constants.PARAMETERS_MAX_AGE, DEFAULT_MAX_AGE_SECS), choice=max_age)
# If max_age is not set, resolve it from the environment variable, defaulting to DEFAULT_MAX_AGE_SECS
max_age = resolve_max_age(env=os.getenv(constants.PARAMETERS_MAX_AGE_ENV, DEFAULT_MAX_AGE_SECS), choice=max_age)

# Only create the provider if this function is called at least once
if "secrets" not in DEFAULT_PROVIDERS:
Expand Down
50 changes: 25 additions & 25 deletions aws_lambda_powertools/utilities/parameters/ssm.py
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ def get( # type: ignore[override]
----------
name: str
Parameter name
max_age: int
max_age: int, optional
Maximum age of the cached value
transform: str
Optional transformation of the parameter value. Supported values
Expand All @@ -151,12 +151,12 @@ def get( # type: ignore[override]
When the parameter provider fails to transform a parameter value.
"""

# Resolving if will use the default value (5), the value passed by parameter or the environment variable
max_age = resolve_max_age(env=os.getenv(constants.PARAMETERS_MAX_AGE, DEFAULT_MAX_AGE_SECS), choice=max_age)
# If max_age is not set, resolve it from the environment variable, defaulting to DEFAULT_MAX_AGE_SECS
max_age = resolve_max_age(env=os.getenv(constants.PARAMETERS_MAX_AGE_ENV, DEFAULT_MAX_AGE_SECS), choice=max_age)

# Resolving if will use the default value (False), the value passed by parameter or the environment variable
# If decrypt is not set, resolve it from the environment variable, defaulting to False
decrypt = resolve_truthy_env_var_choice(
env=os.getenv(constants.PARAMETERS_DEFAULT_DECRYPT, "false"), choice=decrypt
env=os.getenv(constants.PARAMETERS_SSM_DECRYPT_ENV, "false"), choice=decrypt
)

# Add to `decrypt` sdk_options to we can have an explicit option for this
Expand Down Expand Up @@ -261,7 +261,7 @@ def get_parameters_by_name(
Transforms the content from a JSON object ('json') or base64 binary string ('binary')
decrypt: bool, optional
If the parameter values should be decrypted
max_age: int
max_age: int, optional
Maximum age of the cached value
raise_on_error: bool
Whether to fail-fast or fail gracefully by including "_errors" key in the response, by default True
Expand All @@ -274,12 +274,12 @@ def get_parameters_by_name(
When "_errors" reserved key is in parameters to be fetched from SSM.
"""

# Resolving if will use the default value (5), the value passed by parameter or the environment variable
max_age = resolve_max_age(env=os.getenv(constants.PARAMETERS_MAX_AGE, DEFAULT_MAX_AGE_SECS), choice=max_age)
# If max_age is not set, resolve it from the environment variable, defaulting to DEFAULT_MAX_AGE_SECS
max_age = resolve_max_age(env=os.getenv(constants.PARAMETERS_MAX_AGE_ENV, DEFAULT_MAX_AGE_SECS), choice=max_age)

# Resolving if will use the default value (False), the value passed by parameter or the environment variable
# If decrypt is not set, resolve it from the environment variable, defaulting to False
decrypt = resolve_truthy_env_var_choice(
env=os.getenv(constants.PARAMETERS_DEFAULT_DECRYPT, "false"), choice=decrypt
env=os.getenv(constants.PARAMETERS_SSM_DECRYPT_ENV, "false"), choice=decrypt
)

# Init potential batch/decrypt batch responses and errors
Expand Down Expand Up @@ -528,7 +528,7 @@ def get_parameter(
If the parameter values should be decrypted
force_fetch: bool, optional
Force update even before a cached item has expired, defaults to False
max_age: int
max_age: int, optional
Maximum age of the cached value
sdk_options: dict, optional
Dictionary of options that will be passed to the Parameter Store get_parameter API call
Expand Down Expand Up @@ -566,12 +566,12 @@ def get_parameter(
if "ssm" not in DEFAULT_PROVIDERS:
DEFAULT_PROVIDERS["ssm"] = SSMProvider()

# Resolving if will use the default value (5), the value passed by parameter or the environment variable
max_age = resolve_max_age(env=os.getenv(constants.PARAMETERS_MAX_AGE, DEFAULT_MAX_AGE_SECS), choice=max_age)
# If max_age is not set, resolve it from the environment variable, defaulting to DEFAULT_MAX_AGE_SECS
max_age = resolve_max_age(env=os.getenv(constants.PARAMETERS_MAX_AGE_ENV, DEFAULT_MAX_AGE_SECS), choice=max_age)

# Resolving if will use the default value (False), the value passed by parameter or the environment variable
# If decrypt is not set, resolve it from the environment variable, defaulting to False
decrypt = resolve_truthy_env_var_choice(
env=os.getenv(constants.PARAMETERS_DEFAULT_DECRYPT, "false"), choice=decrypt
env=os.getenv(constants.PARAMETERS_SSM_DECRYPT_ENV, "false"), choice=decrypt
)

# Add to `decrypt` sdk_options to we can have an explicit option for this
Expand Down Expand Up @@ -607,7 +607,7 @@ def get_parameters(
If the parameter values should be decrypted
force_fetch: bool, optional
Force update even before a cached item has expired, defaults to False
max_age: int
max_age: int, optional
Maximum age of the cached value
raise_on_transform_error: bool, optional
Raises an exception if any transform fails, otherwise this will
Expand Down Expand Up @@ -648,12 +648,12 @@ def get_parameters(
if "ssm" not in DEFAULT_PROVIDERS:
DEFAULT_PROVIDERS["ssm"] = SSMProvider()

# Resolving if will use the default value (5), the value passed by parameter or the environment variable
max_age = resolve_max_age(env=os.getenv(constants.PARAMETERS_MAX_AGE, DEFAULT_MAX_AGE_SECS), choice=max_age)
# If max_age is not set, resolve it from the environment variable, defaulting to DEFAULT_MAX_AGE_SECS
max_age = resolve_max_age(env=os.getenv(constants.PARAMETERS_MAX_AGE_ENV, DEFAULT_MAX_AGE_SECS), choice=max_age)

# Resolving if will use the default value (False), the value passed by parameter or the environment variable
# If decrypt is not set, resolve it from the environment variable, defaulting to False
decrypt = resolve_truthy_env_var_choice(
env=os.getenv(constants.PARAMETERS_DEFAULT_DECRYPT, "false"), choice=decrypt
env=os.getenv(constants.PARAMETERS_SSM_DECRYPT_ENV, "false"), choice=decrypt
)

sdk_options["recursive"] = recursive
Expand Down Expand Up @@ -731,7 +731,7 @@ def get_parameters_by_name(
Transforms the content from a JSON object ('json') or base64 binary string ('binary')
decrypt: bool, optional
If the parameter values should be decrypted
max_age: int
max_age: int, optional
Maximum age of the cached value
raise_on_error: bool, optional
Whether to fail-fast or fail gracefully by including "_errors" key in the response, by default True
Expand Down Expand Up @@ -771,12 +771,12 @@ def get_parameters_by_name(
# NOTE: Decided against using multi-thread due to single-thread outperforming in 128M and 1G + timeout risk
# see: https://github.com/awslabs/aws-lambda-powertools-python/issues/1040#issuecomment-1299954613

# Resolving if will use the default value (5), the value passed by parameter or the environment variable
max_age = resolve_max_age(env=os.getenv(constants.PARAMETERS_MAX_AGE, DEFAULT_MAX_AGE_SECS), choice=max_age)
# If max_age is not set, resolve it from the environment variable, defaulting to DEFAULT_MAX_AGE_SECS
max_age = resolve_max_age(env=os.getenv(constants.PARAMETERS_MAX_AGE_ENV, DEFAULT_MAX_AGE_SECS), choice=max_age)

# Resolving if will use the default value (False), the value passed by parameter or the environment variable
# If decrypt is not set, resolve it from the environment variable, defaulting to False
decrypt = resolve_truthy_env_var_choice(
env=os.getenv(constants.PARAMETERS_DEFAULT_DECRYPT, "false"), choice=decrypt
env=os.getenv(constants.PARAMETERS_SSM_DECRYPT_ENV, "false"), choice=decrypt
)

# Only create the provider if this function is called at least once
Expand Down
4 changes: 2 additions & 2 deletions docs/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -706,8 +706,8 @@ Core utilities such as Tracing, Logging, Metrics, and Event Handler will be avai
| **POWERTOOLS_LOGGER_LOG_EVENT** | Logs incoming event | [Logging](./core/logger) | `false` |
| **POWERTOOLS_LOGGER_SAMPLE_RATE** | Debug log sampling | [Logging](./core/logger) | `0` |
| **POWERTOOLS_LOG_DEDUPLICATION_DISABLED** | Disables log deduplication filter protection to use Pytest Live Log feature | [Logging](./core/logger) | `false` |
| **POWERTOOLS_PARAMETERS_MAX_AGE** | Adjust how long values are kept in cache (in seconds) | [Parameters](.//utilities/parameters/#adjusting-cache-ttl) | `5` |
| **POWERTOOLS_PARAMETERS_SSM_DECRYPT** | Sets whether to decrypt or not values retrieved from AWS SSM Parameters Store | [Parameters](.//utilities/parameters/#ssmprovider) | `false` |
| **POWERTOOLS_PARAMETERS_MAX_AGE** | Adjust how long values are kept in cache (in seconds) | [Parameters](./utilities/parameters/#adjusting-cache-ttl) | `5` |
| **POWERTOOLS_PARAMETERS_SSM_DECRYPT** | Sets whether to decrypt or not values retrieved from AWS SSM Parameters Store | [Parameters](./utilities/parameters/#ssmprovider) | `false` |
Comment on lines +709 to +710
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the environment variable names need to be updated too

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No no Ruben, we don't need to. In the documentation is the name of the environment variable that will be configured in Lambda, what we changed was the constant in the constant.py file.
We put the constants to make easy the replication of the same value throughout the project's code.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I need more coffee

| **POWERTOOLS_DEV** | Increases verbosity across utilities | Multiple; see [POWERTOOLS_DEV effect below](#increasing-verbosity-across-utilities) | `false` |
| **LOG_LEVEL** | Sets logging level | [Logging](./core/logger) | `INFO` |

Expand Down
4 changes: 2 additions & 2 deletions docs/utilities/parameters.md
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ The following will retrieve the latest version and store it in the cache.
???+ tip
`max_age` parameter is also available in underlying provider functions like `get()`, `get_multiple()`, etc.

By default, we cache parameters retrieved in-memory for 5 seconds. If you want to change this default value and set the same TTL for all parameters, you can set the `POWERTOOLS_PARAMETERS_MAX_AGE` environment variable. **This will override the default TTL of 5 seconds but can be overridden by the `maxAge` parameter**.
By default, we cache parameters retrieved in-memory for 5 seconds. If you want to change this default value and set the same TTL for all parameters, you can set the `POWERTOOLS_PARAMETERS_MAX_AGE` environment variable. **You can still set `max_age` for individual parameters**.

You can adjust how long we should keep values in cache by using the param `max_age`, when using `get_parameter()`, `get_parameters()` and `get_secret()` methods across all providers.

Expand Down Expand Up @@ -180,7 +180,7 @@ The AWS Systems Manager Parameter Store provider supports two additional argumen
You can create `SecureString` parameters, which are parameters that have a plaintext parameter name and an encrypted parameter value. If you don't use the `decrypt` argument, you will get an encrypted value. Read [here](https://docs.aws.amazon.com/kms/latest/developerguide/services-parameter-store.html) about best practices using KMS to secure your parameters.

???+ tip
If you want to always decrypt parameters, you can set the `POWERTOOLS_PARAMETERS_SSM_DECRYPT=true` environment variable. **This will override the default value of `false` but can be overridden by the `decrypt` parameter**.
If you want to always decrypt parameters, you can set the `POWERTOOLS_PARAMETERS_SSM_DECRYPT=true` environment variable. **This will override the default value of `false` but you can still set the `decrypt` option for individual parameters**.

=== "builtin_provider_ssm_with_decrypt.py"
```python hl_lines="6 10 16"
Expand Down
10 changes: 5 additions & 5 deletions tests/unit/test_shared_functions.py
Original file line number Diff line number Diff line change
Expand Up @@ -110,10 +110,10 @@ def test_extract_event_any(data):

def test_resolve_max_age_explicit_wins_over_env_var(monkeypatch: pytest.MonkeyPatch):
# GIVEN POWERTOOLS_PARAMETERS_MAX_AGE environment variable is set
monkeypatch.setenv(constants.PARAMETERS_MAX_AGE, "20")
monkeypatch.setenv(constants.PARAMETERS_MAX_AGE_ENV, "20")

# WHEN the choice is set explicitly
max_age = resolve_max_age(env=os.getenv(constants.PARAMETERS_MAX_AGE, DEFAULT_MAX_AGE_SECS), choice=10)
max_age = resolve_max_age(env=os.getenv(constants.PARAMETERS_MAX_AGE_ENV, DEFAULT_MAX_AGE_SECS), choice=10)

# THEN the result must be the choice
assert max_age == 10
Expand All @@ -123,18 +123,18 @@ def test_resolve_max_age_with_default_value():
# GIVEN POWERTOOLS_PARAMETERS_MAX_AGE is not set

# WHEN the choice is set to None
max_age = resolve_max_age(env=os.getenv(constants.PARAMETERS_MAX_AGE, DEFAULT_MAX_AGE_SECS), choice=None)
max_age = resolve_max_age(env=os.getenv(constants.PARAMETERS_MAX_AGE_ENV, DEFAULT_MAX_AGE_SECS), choice=None)

# THEN the result must be the default value (DEFAULT_MAX_AGE_SECS)
assert max_age == int(DEFAULT_MAX_AGE_SECS)


def test_resolve_max_age_env_var_wins_over_default_value(monkeypatch: pytest.MonkeyPatch):
# GIVEN POWERTOOLS_PARAMETERS_MAX_AGE environment variable is set
monkeypatch.setenv(constants.PARAMETERS_MAX_AGE, "20")
monkeypatch.setenv(constants.PARAMETERS_MAX_AGE_ENV, "20")

# WHEN the choice is set to None
max_age = resolve_max_age(env=os.getenv(constants.PARAMETERS_MAX_AGE, DEFAULT_MAX_AGE_SECS), choice=None)
max_age = resolve_max_age(env=os.getenv(constants.PARAMETERS_MAX_AGE_ENV, DEFAULT_MAX_AGE_SECS), choice=None)

# THEN the result must be the environment variable value
assert max_age == 20