feat(parameters): migrate AppConfig to new APIs due to API deprecation (#1553)

Co-authored-by: Heitor Lessa <[email protected]>

Author: 2 people

Diff for: aws_lambda_powertools/utilities/parameters/appconfig.py

@@ -5,20 +5,17 @@
 
 import os
 from typing import TYPE_CHECKING, Any, Dict, Optional, Union
-from uuid import uuid4
 
 import boto3
 from botocore.config import Config
 
 if TYPE_CHECKING:
-    from mypy_boto3_appconfig import AppConfigClient
+    from mypy_boto3_appconfigdata import AppConfigDataClient
 
 from ...shared import constants
 from ...shared.functions import resolve_env_var_choice
 from .base import DEFAULT_MAX_AGE_SECS, DEFAULT_PROVIDERS, BaseProvider
 
-CLIENT_ID = str(uuid4())
-
 
 class AppConfigProvider(BaseProvider):
     """
@@ -34,8 +31,8 @@ class AppConfigProvider(BaseProvider):
         Botocore configuration to pass during client initialization
     boto3_session : boto3.session.Session, optional
             Boto3 session to create a boto3_client from
-    boto3_client: AppConfigClient, optional
-            Boto3 AppConfig Client to use, boto3_session will be ignored if both are provided
+    boto3_client: AppConfigDataClient, optional
+            Boto3 AppConfigData Client to use, boto3_session will be ignored if both are provided
 
     Example
     -------
@@ -73,16 +70,16 @@ def __init__(
         application: Optional[str] = None,
         config: Optional[Config] = None,
         boto3_session: Optional[boto3.session.Session] = None,
-        boto3_client: Optional["AppConfigClient"] = None,
+        boto3_client: Optional["AppConfigDataClient"] = None,
     ):
         """
         Initialize the App Config client
         """
 
         super().__init__()
 
-        self.client: "AppConfigClient" = self._build_boto3_client(
-            service_name="appconfig", client=boto3_client, session=boto3_session, config=config
+        self.client: "AppConfigDataClient" = self._build_boto3_client(
+            service_name="appconfigdata", client=boto3_client, session=boto3_session, config=config
         )
 
         self.application = resolve_env_var_choice(
@@ -91,6 +88,9 @@ def __init__(
         self.environment = environment
         self.current_version = ""
 
+        self._next_token = ""  # nosec - token for get_latest_configuration executions
+        self.last_returned_value = ""
+
     def _get(self, name: str, **sdk_options) -> str:
         """
         Retrieve a parameter value from AWS App config.
@@ -100,16 +100,26 @@ def _get(self, name: str, **sdk_options) -> str:
         name: str
             Name of the configuration
         sdk_options: dict, optional
-            Dictionary of options that will be passed to the client's get_configuration API call
+            SDK options to propagate to `start_configuration_session` API call
         """
+        if not self._next_token:
+            sdk_options["ConfigurationProfileIdentifier"] = name
+            sdk_options["ApplicationIdentifier"] = self.application
+            sdk_options["EnvironmentIdentifier"] = self.environment
+            response_configuration = self.client.start_configuration_session(**sdk_options)
+            self._next_token = response_configuration["InitialConfigurationToken"]
 
-        sdk_options["Configuration"] = name
-        sdk_options["Application"] = self.application
-        sdk_options["Environment"] = self.environment
-        sdk_options["ClientId"] = CLIENT_ID
+        # The new AppConfig APIs require two API calls to return the configuration
+        # First we start the session and after that we retrieve the configuration
+        # We need to store the token to use in the next execution
+        response = self.client.get_latest_configuration(ConfigurationToken=self._next_token)
+        return_value = response["Configuration"].read()
+        self._next_token = response["NextPollConfigurationToken"]
 
-        response = self.client.get_configuration(**sdk_options)
-        return response["Content"].read()  # read() of botocore.response.StreamingBody
+        if return_value:
+            self.last_returned_value = return_value
+
+        return self.last_returned_value
 
     def _get_multiple(self, path: str, **sdk_options) -> Dict[str, str]:
         """
@@ -145,7 +155,7 @@ def get_app_config(
     max_age: int
         Maximum age of the cached value
     sdk_options: dict, optional
-        Dictionary of options that will be passed to the boto client get_configuration API call
+        SDK options to propagate to `start_configuration_session` API call
 
     Raises
     ------
@@ -180,8 +190,6 @@ def get_app_config(
     if "appconfig" not in DEFAULT_PROVIDERS:
         DEFAULT_PROVIDERS["appconfig"] = AppConfigProvider(environment=environment, application=application)
 
-    sdk_options["ClientId"] = CLIENT_ID
-
     return DEFAULT_PROVIDERS["appconfig"].get(
         name, max_age=max_age, transform=transform, force_fetch=force_fetch, **sdk_options
     )

Diff for: aws_lambda_powertools/utilities/parameters/base.py

@@ -15,7 +15,7 @@
 from .exceptions import GetParameterError, TransformParameterError
 
 if TYPE_CHECKING:
-    from mypy_boto3_appconfig import AppConfigClient
+    from mypy_boto3_appconfigdata import AppConfigDataClient
     from mypy_boto3_dynamodb import DynamoDBServiceResource
     from mypy_boto3_secretsmanager import SecretsManagerClient
     from mypy_boto3_ssm import SSMClient
@@ -28,7 +28,7 @@
 TRANSFORM_METHOD_JSON = "json"
 TRANSFORM_METHOD_BINARY = "binary"
 SUPPORTED_TRANSFORM_METHODS = [TRANSFORM_METHOD_JSON, TRANSFORM_METHOD_BINARY]
-ParameterClients = Union["AppConfigClient", "SecretsManagerClient", "SSMClient"]
+ParameterClients = Union["AppConfigDataClient", "SecretsManagerClient", "SSMClient"]
 
 
 class BaseProvider(ABC):

Diff for: docs/upgrade.md

@@ -13,6 +13,7 @@ Changes at a glance:
 * The API for **event handler's `Response`** has minor changes to support multi value headers and cookies.
 * The **legacy SQS batch processor** was removed.
 * The **Idempotency key** format changed slightly, invalidating all the existing cached results.
+* The **Feature Flags and AppConfig Parameter utility** API calls have changed and you must update your IAM permissions.
 
 ???+ important
     Powertools for Python v2 drops suport for Python 3.6, following the Python 3.6 End-Of-Life (EOL) reached on December 23, 2021.
@@ -154,3 +155,9 @@ Prior to this change, the Idempotency key was generated using only the caller fu
 After this change, the key is generated using the `module name` + `qualified function name` + `idempotency key` (e.g: `app.classExample.function#app.handler#282e83393862a613b612c00283fef4c8`).
 
 Using qualified names prevents distinct functions with the same name to contend for the same Idempotency key.
+
+## Feature Flags and AppConfig Parameter utility
+
+AWS AppConfig deprecated the current API (GetConfiguration) - [more details here](https://github.com/awslabs/aws-lambda-powertools-python/issues/1506#issuecomment-1266645884).
+
+You must update your IAM permissions to allow `appconfig:GetLatestConfiguration` and `appconfig:StartConfigurationSession`. There are no code changes required.

Diff for: docs/utilities/feature_flags.md

@@ -3,11 +3,11 @@ title: Feature flags
 description: Utility
 ---
 
-???+ note
-    This is currently in Beta, as we might change Store parameters in the next release.
-
 The feature flags utility provides a simple rule engine to define when one or multiple features should be enabled depending on the input.
 
+???+ info
+    We currently only support AppConfig using [freeform configuration profile](https://docs.aws.amazon.com/appconfig/latest/userguide/appconfig-creating-configuration-and-profile.html#appconfig-creating-configuration-and-profile-free-form-configurations).
+
 ## Terminology
 
 Feature flags are used to modify behaviour without changing the application's code. These flags can be **static** or **dynamic**.
@@ -28,6 +28,9 @@ If you want to learn more about feature flags, their variations and trade-offs,
 * [AWS Lambda Feature Toggles Made Simple - Ran Isenberg](https://isenberg-ran.medium.com/aws-lambda-feature-toggles-made-simple-580b0c444233)
 * [Feature Flags Getting Started - CloudBees](https://www.cloudbees.com/blog/ultimate-feature-flag-guide)
 
+???+ note
+    AWS AppConfig requires two API calls to fetch configuration for the first time. You can improve latency by consolidating your feature settings in a single [Configuration](https://docs.aws.amazon.com/appconfig/latest/userguide/appconfig-creating-configuration-and-profile.html).
+
 ## Key features
 
 * Define simple feature flags to dynamically decide when to enable a feature
@@ -38,7 +41,7 @@ If you want to learn more about feature flags, their variations and trade-offs,
 
 ### IAM Permissions
 
-Your Lambda function must have `appconfig:GetConfiguration` IAM permission in order to fetch configuration from AWS AppConfig.
+Your Lambda function IAM Role must have `appconfig:GetLatestConfiguration` and `appconfig:StartConfigurationSession` IAM permissions before using this feature.
 
 ### Required resources
 

Diff for: docs/utilities/parameters.md

@@ -24,14 +24,15 @@ This utility requires additional permissions to work as expected.
 ???+ note
     Different parameter providers require different permissions.
 
-| Provider            | Function/Method                                      | IAM Permission                  |
-| ------------------- | ---------------------------------------------------- | ------------------------------- |
-| SSM Parameter Store | `get_parameter`, `SSMProvider.get`                   | `ssm:GetParameter`              |
-| SSM Parameter Store | `get_parameters`, `SSMProvider.get_multiple`         | `ssm:GetParametersByPath`       |
-| Secrets Manager     | `get_secret`, `SecretsManager.get`                   | `secretsmanager:GetSecretValue` |
-| DynamoDB            | `DynamoDBProvider.get`                               | `dynamodb:GetItem`              |
-| DynamoDB            | `DynamoDBProvider.get_multiple`                      | `dynamodb:Query`                |
-| App Config          | `AppConfigProvider.get_app_config`, `get_app_config` | `appconfig:GetConfiguration`    |
+| Provider            | Function/Method                                                  | IAM Permission                                                               |
+| ------------------- | -----------------------------------------------------------------| -----------------------------------------------------------------------------|
+| SSM Parameter Store | `get_parameter`, `SSMProvider.get`                               | `ssm:GetParameter`                                                           |
+| SSM Parameter Store | `get_parameters`, `SSMProvider.get_multiple`                     | `ssm:GetParametersByPath`                                                    |
+| SSM Parameter Store | If using `decrypt=True`                                          | You must add an additional permission `kms:Decrypt`                          |
+| Secrets Manager     | `get_secret`, `SecretsManager.get`                               | `secretsmanager:GetSecretValue`                                              |
+| DynamoDB            | `DynamoDBProvider.get`                                           | `dynamodb:GetItem`                                                           |
+| DynamoDB            | `DynamoDBProvider.get_multiple`                                  | `dynamodb:Query`                                                             |
+| App Config          | `get_app_config`, `AppConfigProvider.get_app_config`             | `appconfig:GetLatestConfiguration` and `appconfig:StartConfigurationSession` |
 
 ### Fetching parameters
 

Diff for: poetry.lock

@@ -73,6 +73,7 @@ typing-extensions = "^4.4.0"
 mkdocs-material = "^8.5.4"
 filelock = "^3.8.0"
 checksumdir = "^1.2.0"
+mypy-boto3-appconfigdata = "^1.24.36"
 importlib-metadata = "^4.13"
 
 [tool.poetry.extras]

Diff for: pyproject.toml

@@ -0,0 +1,19 @@
+import pytest
+
+from tests.e2e.parameters.infrastructure import ParametersStack
+
+
+@pytest.fixture(autouse=True, scope="module")
+def infrastructure(tmp_path_factory, worker_id):
+    """Setup and teardown logic for E2E test infrastructure
+
+    Yields
+    ------
+    Dict[str, str]
+        CloudFormation Outputs from deployed infrastructure
+    """
+    stack = ParametersStack()
+    try:
+        yield stack.deploy()
+    finally:
+        stack.delete()

Diff for: tests/e2e/parameters/__init__.py

@@ -0,0 +1,11 @@
+from aws_lambda_powertools.utilities import parameters
+from aws_lambda_powertools.utilities.typing import LambdaContext
+
+
+def lambda_handler(event: dict, context: LambdaContext):
+    # Retrieve a single configuration, latest version
+    value: bytes = parameters.get_app_config(
+        name=event.get("name"), environment=event.get("environment"), application=event.get("application")
+    )
+
+    return value