From 50a0155f03cabd672da0d54bc19a139fbef11891 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Mon, 20 Mar 2023 16:17:45 +0100 Subject: [PATCH 01/20] Starting AppConfigProvider --- powertools-parameters/pom.xml | 5 + .../parameters/AppConfigProvider.java | 122 ++++++++++++++++++ .../powertools/parameters/ParamManager.java | 25 ++++ 3 files changed, 152 insertions(+) create mode 100644 powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java diff --git a/powertools-parameters/pom.xml b/powertools-parameters/pom.xml index befb57d09..f4b0006b0 100644 --- a/powertools-parameters/pom.xml +++ b/powertools-parameters/pom.xml @@ -77,6 +77,11 @@ software.amazon.awssdk dynamodb + + software.amazon.awssdk + appconfig + + com.fasterxml.jackson.core jackson-databind diff --git a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java new file mode 100644 index 000000000..f7d49c444 --- /dev/null +++ b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java @@ -0,0 +1,122 @@ +package software.amazon.lambda.powertools.parameters; + +import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider; +import software.amazon.awssdk.core.SdkSystemSetting; +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.appconfig.AppConfigClient; +import software.amazon.lambda.powertools.parameters.cache.CacheManager; +import software.amazon.lambda.powertools.parameters.transform.TransformationManager; + +import java.util.Map; + +public class AppConfigProvider extends BaseProvider{ + + private final AppConfigClient client; + + public AppConfigProvider(CacheManager cacheManager) { + this(cacheManager, AppConfigClient.builder() + .httpClientBuilder(UrlConnectionHttpClient.builder()) + .credentialsProvider(EnvironmentVariableCredentialsProvider.create()) + .region(Region.of(System.getenv(SdkSystemSetting.AWS_REGION.environmentVariable()))) + .build() + ); + + } + + AppConfigProvider(CacheManager cacheManager, AppConfigClient client) { + super(cacheManager); + this.client = client; + } + + + @Override + protected String getValue(String key) { + throw new RuntimeException("Not implemented"); + } + + @Override + protected Map getMultipleValues(String path) { + throw new RuntimeException("Not implemented"); + } + + /** + * Create a builder that can be used to configure and create a {@link AppConfigProvider}. + * + * @return a new instance of {@link AppConfigProvider.Builder} + */ + public static AppConfigProvider.Builder builder() { + return new AppConfigProvider.Builder(); + } + + static class Builder { + private AppConfigClient client; + private CacheManager cacheManager; + private TransformationManager transformationManager; + + /** + * Create a {@link AppConfigProvider} instance. + * + * @return a {@link AppConfigProvider} + */ + public AppConfigProvider build() { + if (cacheManager == null) { + throw new IllegalStateException("No CacheManager provided; please provide one"); + } + AppConfigProvider provider; + if (client != null) { + provider = new AppConfigProvider(cacheManager, client); + } else { + provider = new AppConfigProvider(cacheManager); + } + if (transformationManager != null) { + provider.setTransformationManager(transformationManager); + } + return provider; + } + + /** + * Set custom {@link AppConfigProvider} to pass to the {@link AppConfigClient}.
+ * Use it if you want to customize the region or any other part of the client. + * + * @param client Custom client + * @return the builder to chain calls (eg. 
builder.withClient().build()
) + */ + public AppConfigProvider.Builder withClient(AppConfigClient client) { + this.client = client; + return this; + } + + /** + * Mandatory. Provide a CacheManager to the {@link AppConfigProvider} + * + * @param cacheManager the manager that will handle the cache of parameters + * @return the builder to chain calls (eg. 
builder.withCacheManager().build()
) + */ + public AppConfigProvider.Builder withCacheManager(CacheManager cacheManager) { + this.cacheManager = cacheManager; + return this; + } + + /** + * Mandatory. Provide a DynamoDB table to the {@link AppConfigProvider} + * + * @param table the table that parameters will be retrieved from. + * @return the builder to chain calls (eg. 
builder.withTable().build()
) + */ + public AppConfigProvider.Builder withTable(String table) { + return this; + } + + /** + * Provide a transformationManager to the {@link AppConfigProvider} + * + * @param transformationManager the manager that will handle transformation of parameters + * @return the builder to chain calls (eg. 
builder.withTransformationManager().build()
) + */ + public AppConfigProvider.Builder withTransformationManager(TransformationManager transformationManager) { + this.transformationManager = transformationManager; + return this; + } + } +} diff --git a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/ParamManager.java b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/ParamManager.java index e4c70acb0..8860939f7 100644 --- a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/ParamManager.java +++ b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/ParamManager.java @@ -13,6 +13,7 @@ */ package software.amazon.lambda.powertools.parameters; +import software.amazon.awssdk.services.appconfig.AppConfigClient; import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; import software.amazon.awssdk.services.ssm.SsmClient; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; @@ -81,6 +82,16 @@ public static DynamoDbProvider getDynamoDbProvider(String tableName) { .build(); } + /** + * Get a {@link AppConfigProvider} with default {@link AppConfigClient}.
+ * If you need to customize the region, or other part of the client, use {@link ParamManager#getAppConfigProvider(appConfigClient)} instead. + * @return a {@link AppConfigProvider} + */ + public static AppConfigProvider getAppConfigProvider() { + return getProvider(AppConfigProvider.class); + } + + /** * Get a {@link SecretsProvider} with your custom {@link SecretsManagerClient}.
* Use this to configure region or other part of the client. Use {@link ParamManager#getSsmProvider()} if you don't need this customization. @@ -121,6 +132,20 @@ public static DynamoDbProvider getDynamoDbProvider(DynamoDbClient client, String .build()); } + /** + * Get a {@link AppConfigProvider} with your custom {@link AppConfigClient}.
+ * Use this to configure region or other part of the client. Use {@link ParamManager#getAppConfigProvider()} if you don't need this customization. + * @return a {@link AppConfigProvider} + */ + public static AppConfigProvider getSsmProvider(AppConfigClient client) { + return (AppConfigProvider) providers.computeIfAbsent(AppConfigProvider.class, (k) -> AppConfigProvider.builder() + .withClient(client) + .withCacheManager(cacheManager) + .withTransformationManager(transformationManager) + .build()); + } + + public static CacheManager getCacheManager() { return cacheManager; } From a273dc31630b0a256f182fb5fda29a45b81971af Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Mon, 20 Mar 2023 17:52:50 +0100 Subject: [PATCH 02/20] Fleshing it out --- .../parameters/AppConfigProvider.java | 60 ++++++++++++++++--- .../powertools/parameters/ParamManager.java | 22 +++++-- .../exception/MissingAppConfigException.java | 8 +++ 3 files changed, 76 insertions(+), 14 deletions(-) create mode 100644 powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/exception/MissingAppConfigException.java diff --git a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java index f7d49c444..126b0cd46 100644 --- a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java +++ b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java @@ -5,38 +5,51 @@ import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.appconfig.AppConfigClient; +import software.amazon.awssdk.services.appconfig.model.GetEnvironmentRequest; +import software.amazon.awssdk.services.appconfig.model.GetEnvironmentResponse; import software.amazon.lambda.powertools.parameters.cache.CacheManager; import software.amazon.lambda.powertools.parameters.transform.TransformationManager; import java.util.Map; +import java.util.Optional; public class AppConfigProvider extends BaseProvider{ private final AppConfigClient client; + private final String environment; + private final String application; - public AppConfigProvider(CacheManager cacheManager) { + public AppConfigProvider(CacheManager cacheManager, String environment, String application) { this(cacheManager, AppConfigClient.builder() .httpClientBuilder(UrlConnectionHttpClient.builder()) .credentialsProvider(EnvironmentVariableCredentialsProvider.create()) .region(Region.of(System.getenv(SdkSystemSetting.AWS_REGION.environmentVariable()))) - .build() - ); - + .build(), + environment, application); } - AppConfigProvider(CacheManager cacheManager, AppConfigClient client) { + AppConfigProvider(CacheManager cacheManager, AppConfigClient client, String environment, String application) { super(cacheManager); this.client = client; + this.environment = environment; + this.application = application; } @Override protected String getValue(String key) { - throw new RuntimeException("Not implemented"); + GetEnvironmentResponse env = client.getEnvironment(GetEnvironmentRequest.builder() + .environmentId(environment) + .applicationId(application) + .build()); + + Optional val = env.getValueForField(key, String.class); + return val.orElse(null); } @Override protected Map getMultipleValues(String path) { + // Retrieving multiple values is not supported with the AppConfig provider. throw new RuntimeException("Not implemented"); } @@ -53,6 +66,8 @@ static class Builder { private AppConfigClient client; private CacheManager cacheManager; private TransformationManager transformationManager; + private String environment; + private String application; /** * Create a {@link AppConfigProvider} instance. @@ -63,11 +78,18 @@ public AppConfigProvider build() { if (cacheManager == null) { throw new IllegalStateException("No CacheManager provided; please provide one"); } + if (environment == null) { + throw new IllegalStateException("No environment provided; please provide one"); + } + if (application == null) { + throw new IllegalStateException("No application provided; please provide one"); + } + AppConfigProvider provider; if (client != null) { - provider = new AppConfigProvider(cacheManager, client); + provider = new AppConfigProvider(cacheManager, client, environment, application); } else { - provider = new AppConfigProvider(cacheManager); + provider = new AppConfigProvider(cacheManager, environment, application); } if (transformationManager != null) { provider.setTransformationManager(transformationManager); @@ -87,6 +109,28 @@ public AppConfigProvider.Builder withClient(AppConfigClient client) { return this; } + /** + * Mandatory. Provide an environment to the {@link AppConfigProvider} + * + * @param environment the AppConfig environment + * @return the builder to chain calls (eg. 
builder.withCacheManager().build()
) + */ + public AppConfigProvider.Builder withEnvironment(String environment) { + this.environment = environment; + return this; + } + + /** + * Mandatory. Provide a CacheManager to the {@link AppConfigProvider} + * + * @param application the application to pull configuration from + * @return the builder to chain calls (eg. 
builder.withCacheManager().build()
) + */ + public AppConfigProvider.Builder withApplication(String application) { + this.application = application; + return this; + } + /** * Mandatory. Provide a CacheManager to the {@link AppConfigProvider} * diff --git a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/ParamManager.java b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/ParamManager.java index 8860939f7..8d52d3ed3 100644 --- a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/ParamManager.java +++ b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/ParamManager.java @@ -84,11 +84,19 @@ public static DynamoDbProvider getDynamoDbProvider(String tableName) { /** * Get a {@link AppConfigProvider} with default {@link AppConfigClient}.
- * If you need to customize the region, or other part of the client, use {@link ParamManager#getAppConfigProvider(appConfigClient)} instead. + * If you need to customize the region, or other part of the client, use {@link ParamManager#getAppConfigProvider(AppConfigClient, String, String)} instead. * @return a {@link AppConfigProvider} */ - public static AppConfigProvider getAppConfigProvider() { - return getProvider(AppConfigProvider.class); + public static AppConfigProvider getAppConfigProvider(String environment, String application) { + // Because we need a DDB table name to configure our client, we can't use + // ParamManager#getProvider. This means that we need to make sure we do the same stuff - + // set transformation manager and cache manager. + return AppConfigProvider.builder() + .withCacheManager(cacheManager) + .withTransformationManager(transformationManager) + .withEnvironment(environment) + .withApplication(application) + .build(); } @@ -131,17 +139,19 @@ public static DynamoDbProvider getDynamoDbProvider(DynamoDbClient client, String .withTransformationManager(transformationManager) .build()); } - + /** * Get a {@link AppConfigProvider} with your custom {@link AppConfigClient}.
- * Use this to configure region or other part of the client. Use {@link ParamManager#getAppConfigProvider()} if you don't need this customization. + * Use this to configure region or other part of the client. Use {@link ParamManager#getAppConfigProvider(String, String)} if you don't need this customization. * @return a {@link AppConfigProvider} */ - public static AppConfigProvider getSsmProvider(AppConfigClient client) { + public static AppConfigProvider getAppConfigProvider(AppConfigClient client, String environment, String application) { return (AppConfigProvider) providers.computeIfAbsent(AppConfigProvider.class, (k) -> AppConfigProvider.builder() .withClient(client) .withCacheManager(cacheManager) .withTransformationManager(transformationManager) + .withEnvironment(environment) + .withApplication(application) .build()); } diff --git a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/exception/MissingAppConfigException.java b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/exception/MissingAppConfigException.java new file mode 100644 index 000000000..67ab11147 --- /dev/null +++ b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/exception/MissingAppConfigException.java @@ -0,0 +1,8 @@ +package software.amazon.lambda.powertools.parameters.exception; + +public class MissingAppConfigException extends RuntimeException { + public MissingAppConfigException(String msg) { + super(msg); + } + +} From 78ba7beeaf1e210f194748f66f70d8e354904cce Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Wed, 31 May 2023 14:26:55 +0800 Subject: [PATCH 03/20] Adding tests --- .../parameters/AppConfigProvider.java | 17 ++-- .../parameters/AppConfigProviderTest.java | 83 +++++++++++++++++++ 2 files changed, 92 insertions(+), 8 deletions(-) create mode 100644 powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/AppConfigProviderTest.java diff --git a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java index 126b0cd46..e98ebf537 100644 --- a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java +++ b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java @@ -16,9 +16,11 @@ public class AppConfigProvider extends BaseProvider{ private final AppConfigClient client; - private final String environment; + private final String application; + private final GetEnvironmentResponse environment; + public AppConfigProvider(CacheManager cacheManager, String environment, String application) { this(cacheManager, AppConfigClient.builder() .httpClientBuilder(UrlConnectionHttpClient.builder()) @@ -31,19 +33,18 @@ public AppConfigProvider(CacheManager cacheManager, String environment, String a AppConfigProvider(CacheManager cacheManager, AppConfigClient client, String environment, String application) { super(cacheManager); this.client = client; - this.environment = environment; this.application = application; + + this.environment = client.getEnvironment(GetEnvironmentRequest.builder() + .environmentId(environment) + .applicationId(application) + .build()); } @Override protected String getValue(String key) { - GetEnvironmentResponse env = client.getEnvironment(GetEnvironmentRequest.builder() - .environmentId(environment) - .applicationId(application) - .build()); - - Optional val = env.getValueForField(key, String.class); + Optional val = environment.getValueForField(key, String.class); return val.orElse(null); } diff --git a/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/AppConfigProviderTest.java b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/AppConfigProviderTest.java new file mode 100644 index 000000000..81e198bb9 --- /dev/null +++ b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/AppConfigProviderTest.java @@ -0,0 +1,83 @@ +package software.amazon.lambda.powertools.parameters; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.Mockito; +import software.amazon.awssdk.services.appconfig.AppConfigClient; +import software.amazon.awssdk.services.appconfig.model.GetEnvironmentRequest; +import software.amazon.awssdk.services.appconfig.model.GetEnvironmentResponse; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; +import software.amazon.lambda.powertools.parameters.cache.CacheManager; +import org.junit.jupiter.api.Assertions; +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.HashMap; +import java.util.Optional; + +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.MockitoAnnotations.openMocks; + +public class AppConfigProviderTest { + + @Mock + AppConfigClient client; + + @Mock + GetEnvironmentResponse environment; + + private AppConfigProvider provider; + + @Captor + ArgumentCaptor getEnvironmentCaptor; + + @Captor + ArgumentCaptor getValueForFieldCaptor; + + private final String environmentName = "test"; + + private final String applicationName = "fakeApp"; + + @BeforeEach + public void init() { + openMocks(this); + + // getEnvironment gets called in the constructor, so we need to mock it now + Mockito.when(client.getEnvironment(getEnvironmentCaptor.capture())).thenReturn(environment); + + CacheManager cacheManager = new CacheManager(); + provider = new AppConfigProvider(cacheManager, client, environmentName, applicationName); + } + + @Test + public void getEnvironmentReturnsEnvironment() { + // getEnvironment gets called when we construct the AppConfigProvider. Let's make + // sure the request included the right environment name + + // Assert + assertThat(getEnvironmentCaptor.getValue().environmentId()).isEqualTo(environmentName); + } + + @Test + public void getValueReturnsValue() { + // Arrange + String key = "Key1"; + String expectedValue = "Value1"; + Mockito.when(environment.getValueForField(getValueForFieldCaptor.capture(), eq(String.class))) + .thenReturn(Optional.of(expectedValue)); + + // Act + String returnedValue = provider.getValue(key); + + // Assert + assertThat(returnedValue).isEqualTo(expectedValue); + assertThat(getValueForFieldCaptor.getValue()).isEqualTo(key); + } + + @Test + public void getValueWithMalformedKeyThrows() { + + } +} From 70db445bbf6f82a6fa59d6fb343811b87cd4cf8a Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Wed, 31 May 2023 17:19:09 +0800 Subject: [PATCH 04/20] Remove cruft --- .../powertools/parameters/AppConfigProvider.java | 10 ---------- .../powertools/parameters/AppConfigProviderTest.java | 4 ---- 2 files changed, 14 deletions(-) diff --git a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java index e98ebf537..96f9d19f3 100644 --- a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java +++ b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java @@ -143,16 +143,6 @@ public AppConfigProvider.Builder withCacheManager(CacheManager cacheManager) { return this; } - /** - * Mandatory. Provide a DynamoDB table to the {@link AppConfigProvider} - * - * @param table the table that parameters will be retrieved from. - * @return the builder to chain calls (eg. 
builder.withTable().build()
) - */ - public AppConfigProvider.Builder withTable(String table) { - return this; - } - /** * Provide a transformationManager to the {@link AppConfigProvider} * diff --git a/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/AppConfigProviderTest.java b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/AppConfigProviderTest.java index 81e198bb9..635e4ca09 100644 --- a/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/AppConfigProviderTest.java +++ b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/AppConfigProviderTest.java @@ -76,8 +76,4 @@ public void getValueReturnsValue() { assertThat(getValueForFieldCaptor.getValue()).isEqualTo(key); } - @Test - public void getValueWithMalformedKeyThrows() { - - } } From 0a5bef1e765364d35e4a3091589803370e871c8a Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Thu, 1 Jun 2023 12:37:15 +0800 Subject: [PATCH 05/20] Advancing the e2e tests and impl --- .../handlers/parameters/pom.xml | 57 +++++++++++ .../lambda/powertools/e2e/Function.java | 22 +++++ .../amazon/lambda/powertools/e2e/Input.java | 27 ++++++ .../parameters/src/main/resources/log4j2.xml | 16 ++++ .../lambda/powertools/ParametersE2ET.java | 61 ++++++++++++ .../powertools/testutils/AppConfig.java | 32 +++++++ .../powertools/testutils/Infrastructure.java | 68 ++++++++++++++ powertools-parameters/pom.xml | 2 +- .../parameters/AppConfigProvider.java | 81 ++++++++++++---- .../powertools/parameters/ParamManager.java | 10 +- .../parameters/AppConfigProviderTest.java | 94 +++++++++++++------ 11 files changed, 418 insertions(+), 52 deletions(-) create mode 100644 powertools-e2e-tests/handlers/parameters/pom.xml create mode 100644 powertools-e2e-tests/handlers/parameters/src/main/java/software/amazon/lambda/powertools/e2e/Function.java create mode 100644 powertools-e2e-tests/handlers/parameters/src/main/java/software/amazon/lambda/powertools/e2e/Input.java create mode 100644 powertools-e2e-tests/handlers/parameters/src/main/resources/log4j2.xml create mode 100644 powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/ParametersE2ET.java create mode 100644 powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/AppConfig.java diff --git a/powertools-e2e-tests/handlers/parameters/pom.xml b/powertools-e2e-tests/handlers/parameters/pom.xml new file mode 100644 index 000000000..a46702ca4 --- /dev/null +++ b/powertools-e2e-tests/handlers/parameters/pom.xml @@ -0,0 +1,57 @@ + + 4.0.0 + + + software.amazon.lambda + e2e-test-handlers-parent + 1.0.0 + + + e2e-test-handler-logging + jar + A Lambda function using powertools logging + + + + software.amazon.lambda + powertools-logging + + + com.amazonaws + aws-lambda-java-events + + + + + + + + org.codehaus.mojo + aspectj-maven-plugin + + ${maven.compiler.source} + ${maven.compiler.target} + ${maven.compiler.target} + + + software.amazon.lambda + powertools-logging + + + + + + + compile + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + diff --git a/powertools-e2e-tests/handlers/parameters/src/main/java/software/amazon/lambda/powertools/e2e/Function.java b/powertools-e2e-tests/handlers/parameters/src/main/java/software/amazon/lambda/powertools/e2e/Function.java new file mode 100644 index 000000000..5a9a87109 --- /dev/null +++ b/powertools-e2e-tests/handlers/parameters/src/main/java/software/amazon/lambda/powertools/e2e/Function.java @@ -0,0 +1,22 @@ +package software.amazon.lambda.powertools.e2e; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import software.amazon.lambda.powertools.logging.Logging; +import software.amazon.lambda.powertools.logging.LoggingUtils; + +public class Function implements RequestHandler { + + private static final Logger LOG = LogManager.getLogger(Function.class); + + @Logging + public String handleRequest(Input input, Context context) { + + LoggingUtils.appendKeys(input.getKeys()); + LOG.info(input.getMessage()); + + return "OK"; + } +} \ No newline at end of file diff --git a/powertools-e2e-tests/handlers/parameters/src/main/java/software/amazon/lambda/powertools/e2e/Input.java b/powertools-e2e-tests/handlers/parameters/src/main/java/software/amazon/lambda/powertools/e2e/Input.java new file mode 100644 index 000000000..83afbbd5a --- /dev/null +++ b/powertools-e2e-tests/handlers/parameters/src/main/java/software/amazon/lambda/powertools/e2e/Input.java @@ -0,0 +1,27 @@ +package software.amazon.lambda.powertools.e2e; + +import java.util.Map; + +public class Input { + private String message; + private Map keys; + + public Input() { + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public Map getKeys() { + return keys; + } + + public void setKeys(Map keys) { + this.keys = keys; + } +} diff --git a/powertools-e2e-tests/handlers/parameters/src/main/resources/log4j2.xml b/powertools-e2e-tests/handlers/parameters/src/main/resources/log4j2.xml new file mode 100644 index 000000000..8925f70b9 --- /dev/null +++ b/powertools-e2e-tests/handlers/parameters/src/main/resources/log4j2.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/ParametersE2ET.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/ParametersE2ET.java new file mode 100644 index 000000000..eca42b012 --- /dev/null +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/ParametersE2ET.java @@ -0,0 +1,61 @@ +package software.amazon.lambda.powertools; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.*; +import software.amazon.lambda.powertools.testutils.AppConfig; +import software.amazon.lambda.powertools.testutils.Infrastructure; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class ParametersE2ET { + + + private final ObjectMapper objectMapper = new ObjectMapper(); + + private Infrastructure infrastructure; + private String functionName; + + private String appConfigAppName = "powertools-e2e-test-app"; + private String appConfigEnvironment = "e2etest"; + private final AppConfig appConfig; + + public ParametersE2ET() { + Map params = new HashMap<>(); + params.put("key1", "value1"); + params.put("key2", "value2"); + appConfig = new AppConfig("e2eApp", "e2etest", params); + } + @BeforeAll + @Timeout(value = 5, unit = TimeUnit.MINUTES) + public void setup() { + infrastructure = Infrastructure.builder() + .testName(ParametersE2ET.class.getSimpleName()) + .pathToFunction("parameters") + .appConfig(appConfig) + .environmentVariables( + Stream.of(new String[][]{ + {"POWERTOOLS_LOG_LEVEL", "INFO"}, + {"POWERTOOLS_SERVICE_NAME", ParametersE2ET.class.getSimpleName()} + }) + .collect(Collectors.toMap(data -> data[0], data -> data[1]))) + .build(); + functionName = infrastructure.deploy(); + } + +// @AfterAll +// public static void tearDown() { +// if (infrastructure != null) +// infrastructure.destroy(); +// } + + @Test + public void test_getAppConfigValue() { + + } + +} diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/AppConfig.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/AppConfig.java new file mode 100644 index 000000000..295ca601c --- /dev/null +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/AppConfig.java @@ -0,0 +1,32 @@ +package software.amazon.lambda.powertools.testutils; + +import java.util.HashMap; +import java.util.Map; + +/** + * Defines configuration used to setup an AppConfig + * deployment when the infrastructure is rolled out. + */ +public class AppConfig { + private String application; + private String environment; + private Map configurationValues; + + public AppConfig(String application, String environment, Map configurationValues) { + this.application = application; + this.environment = environment; + this.configurationValues = configurationValues; + } + + public String getApplication() { + return application; + } + + public String getEnvironment() { + return environment; + } + + public Map getConfigurationValues() { + return configurationValues; + } +} diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java index f3659e5c7..6991e6d22 100644 --- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java @@ -7,10 +7,12 @@ import software.amazon.awscdk.Stack; import software.amazon.awscdk.*; import software.amazon.awscdk.cxapi.CloudAssembly; +import software.amazon.awscdk.services.appconfig.*; import software.amazon.awscdk.services.dynamodb.Attribute; import software.amazon.awscdk.services.dynamodb.AttributeType; import software.amazon.awscdk.services.dynamodb.BillingMode; import software.amazon.awscdk.services.dynamodb.Table; +import software.amazon.awscdk.services.groundstation.CfnConfig; import software.amazon.awscdk.services.lambda.Code; import software.amazon.awscdk.services.lambda.Function; import software.amazon.awscdk.services.lambda.Tracing; @@ -65,6 +67,9 @@ public class Infrastructure { private final Region region; private final String account; private final String idempotencyTable; + private final AppConfig appConfig; + + private String functionName; private Object cfnTemplate; private String cfnAssetDirectory; @@ -78,6 +83,7 @@ private Infrastructure(Builder builder) { this.timeout = builder.timeoutInSeconds; this.pathToFunction = builder.pathToFunction; this.idempotencyTable = builder.idemPotencyTable; + this.appConfig = builder.appConfig; this.app = new App(); this.stack = createStackWithLambda(); @@ -140,6 +146,7 @@ public static class Builder { public long timeoutInSeconds = 30; public String pathToFunction; public String testName; + public AppConfig appConfig; private String stackName; private boolean tracing = false; private JavaRuntime runtime; @@ -198,6 +205,11 @@ public Builder idempotencyTable(String tableName) { return this; } + public Builder appConfig(AppConfig app) { + this.appConfig = app; + return this; + } + public Builder environmentVariables(Map environmentVariables) { this.environmentVariables = environmentVariables; return this; @@ -275,9 +287,65 @@ private Stack createStackWithLambda() { .tableName(idempotencyTable) .timeToLiveAttribute("expiration") .build(); + table.grantReadWriteData(function); } + if (appConfig != null) { + CfnApplication app = CfnApplication.Builder + .create(stack, "AppConfigApp") + .name(appConfig.getApplication()) + .build(); + + CfnEnvironment environment = CfnEnvironment.Builder + .create(stack, "AppConfigEnvironment") + .applicationId(app.getRef()) + .name(appConfig.getEnvironment()) + .build(); + + // Create a fast deployment strategy so we don't have to wait ages + CfnDeploymentStrategy fastDeployment = CfnDeploymentStrategy.Builder + .create(stack, "AppConfigDeployment") + .name("FastDeploymentStrategy") + .deploymentDurationInMinutes(0) + .finalBakeTimeInMinutes(0) + .growthFactor(100) + .replicateTo("NONE") + .build(); + + CfnDeployment previousDeployment = null; + for (Map.Entry entry : appConfig.getConfigurationValues().entrySet()) { + CfnConfigurationProfile configProfile = CfnConfigurationProfile.Builder + .create(stack, "AppConfigProfileFor" + entry.getKey()) + .applicationId(app.getRef()) + .locationUri("hosted") + .name(entry.getKey()) + .build(); + + CfnHostedConfigurationVersion configVersion = CfnHostedConfigurationVersion.Builder + .create(stack, "AppConfigHostedVersionFor" + entry.getKey()) + .applicationId(app.getRef()) + .contentType("text/plain") + .configurationProfileId(configProfile.getRef()) + .content(entry.getValue()) + .build(); + + CfnDeployment deployment = CfnDeployment.Builder + .create(stack, "AppConfigDepoymentFor" + entry.getKey()) + .applicationId(app.getRef()) + .environmentId(environment.getRef()) + .deploymentStrategyId(fastDeployment.getRef()) + .configurationProfileId(configProfile.getRef()) + .configurationVersion(configVersion.getRef()) + .build(); + + // We need to chain the deployments to keep CFN happy + if (previousDeployment != null) { + deployment.addDependsOn(previousDeployment); + } + previousDeployment = deployment; + } + } return stack; } diff --git a/powertools-parameters/pom.xml b/powertools-parameters/pom.xml index f4d10647d..b5a4370b9 100644 --- a/powertools-parameters/pom.xml +++ b/powertools-parameters/pom.xml @@ -79,7 +79,7 @@ software.amazon.awssdk - appconfig + appconfigdata diff --git a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java index 96f9d19f3..15104c707 100644 --- a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java +++ b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java @@ -4,25 +4,46 @@ import software.amazon.awssdk.core.SdkSystemSetting; import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.appconfig.AppConfigClient; -import software.amazon.awssdk.services.appconfig.model.GetEnvironmentRequest; -import software.amazon.awssdk.services.appconfig.model.GetEnvironmentResponse; +import software.amazon.awssdk.services.appconfigdata.AppConfigDataClient; +import software.amazon.awssdk.services.appconfigdata.model.GetLatestConfigurationRequest; +import software.amazon.awssdk.services.appconfigdata.model.GetLatestConfigurationResponse; +import software.amazon.awssdk.services.appconfigdata.model.StartConfigurationSessionRequest; import software.amazon.lambda.powertools.parameters.cache.CacheManager; import software.amazon.lambda.powertools.parameters.transform.TransformationManager; +import java.util.HashMap; import java.util.Map; -import java.util.Optional; public class AppConfigProvider extends BaseProvider{ - private final AppConfigClient client; + private static class EstablishedSession { + private final String nextSessionToken; + private final String lastConfigurationValue; + + private EstablishedSession(String nextSessionToken, String value) { + this.nextSessionToken = nextSessionToken; + this.lastConfigurationValue = value; + } + + public String getNextSessionToken() { + return nextSessionToken; + } + + public String getLastConfigurationValue() { + return lastConfigurationValue; + } + } + + private final AppConfigDataClient client; private final String application; - private final GetEnvironmentResponse environment; + private final String environment; + + private final HashMap establishedSessions = new HashMap<>(); public AppConfigProvider(CacheManager cacheManager, String environment, String application) { - this(cacheManager, AppConfigClient.builder() + this(cacheManager, AppConfigDataClient.builder() .httpClientBuilder(UrlConnectionHttpClient.builder()) .credentialsProvider(EnvironmentVariableCredentialsProvider.create()) .region(Region.of(System.getenv(SdkSystemSetting.AWS_REGION.environmentVariable()))) @@ -30,22 +51,46 @@ public AppConfigProvider(CacheManager cacheManager, String environment, String a environment, application); } - AppConfigProvider(CacheManager cacheManager, AppConfigClient client, String environment, String application) { + AppConfigProvider(CacheManager cacheManager, AppConfigDataClient client, String environment, String application) { super(cacheManager); this.client = client; this.application = application; - - this.environment = client.getEnvironment(GetEnvironmentRequest.builder() - .environmentId(environment) - .applicationId(application) - .build()); + this.environment = environment; } @Override protected String getValue(String key) { - Optional val = environment.getValueForField(key, String.class); - return val.orElse(null); + // Start a configuration session if we don't already have one to get the initial token + EstablishedSession establishedSession = establishedSessions.getOrDefault(key, null); + String sessionToken = establishedSession != null? + establishedSession.nextSessionToken : + client.startConfigurationSession(StartConfigurationSessionRequest.builder() + .applicationIdentifier(this.application) + .environmentIdentifier(this.environment) + .configurationProfileIdentifier(key) + .build()) + .initialConfigurationToken(); + + // Get the configuration + GetLatestConfigurationResponse response = client.getLatestConfiguration(GetLatestConfigurationRequest.builder() + .configurationToken(sessionToken) + .build()); + + // Get the next token + String nextSessionToken = response.nextPollConfigurationToken(); + + // Get the value + String value = response.configuration() != null? + response.configuration().asUtf8String() : // if we have a new value, use it + establishedSession != null? + establishedSession.lastConfigurationValue : // if we don't but we have a previous value, use that + null; // otherwise we've got no value + + // Update the cache so we can get the next value later + establishedSessions.put(key, new EstablishedSession(nextSessionToken, value)); + + return value; } @Override @@ -64,7 +109,7 @@ public static AppConfigProvider.Builder builder() { } static class Builder { - private AppConfigClient client; + private AppConfigDataClient client; private CacheManager cacheManager; private TransformationManager transformationManager; private String environment; @@ -99,13 +144,13 @@ public AppConfigProvider build() { } /** - * Set custom {@link AppConfigProvider} to pass to the {@link AppConfigClient}.
+ * Set custom {@link AppConfigProvider} to pass to the {@link AppConfigDataClient}.
* Use it if you want to customize the region or any other part of the client. * * @param client Custom client * @return the builder to chain calls (eg. 
builder.withClient().build()
) */ - public AppConfigProvider.Builder withClient(AppConfigClient client) { + public AppConfigProvider.Builder withClient(AppConfigDataClient client) { this.client = client; return this; } diff --git a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/ParamManager.java b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/ParamManager.java index 8d52d3ed3..cc60a60fe 100644 --- a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/ParamManager.java +++ b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/ParamManager.java @@ -13,7 +13,7 @@ */ package software.amazon.lambda.powertools.parameters; -import software.amazon.awssdk.services.appconfig.AppConfigClient; +import software.amazon.awssdk.services.appconfigdata.AppConfigDataClient; import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; import software.amazon.awssdk.services.ssm.SsmClient; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; @@ -83,8 +83,8 @@ public static DynamoDbProvider getDynamoDbProvider(String tableName) { } /** - * Get a {@link AppConfigProvider} with default {@link AppConfigClient}.
- * If you need to customize the region, or other part of the client, use {@link ParamManager#getAppConfigProvider(AppConfigClient, String, String)} instead. + * Get a {@link AppConfigProvider} with default {@link AppConfigDataClient}.
+ * If you need to customize the region, or other part of the client, use {@link ParamManager#getAppConfigProvider(AppConfigDataClient, String, String)} instead. * @return a {@link AppConfigProvider} */ public static AppConfigProvider getAppConfigProvider(String environment, String application) { @@ -141,11 +141,11 @@ public static DynamoDbProvider getDynamoDbProvider(DynamoDbClient client, String } /** - * Get a {@link AppConfigProvider} with your custom {@link AppConfigClient}.
+ * Get a {@link AppConfigProvider} with your custom {@link AppConfigDataClient}.
* Use this to configure region or other part of the client. Use {@link ParamManager#getAppConfigProvider(String, String)} if you don't need this customization. * @return a {@link AppConfigProvider} */ - public static AppConfigProvider getAppConfigProvider(AppConfigClient client, String environment, String application) { + public static AppConfigProvider getAppConfigProvider(AppConfigDataClient client, String environment, String application) { return (AppConfigProvider) providers.computeIfAbsent(AppConfigProvider.class, (k) -> AppConfigProvider.builder() .withClient(client) .withCacheManager(cacheManager) diff --git a/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/AppConfigProviderTest.java b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/AppConfigProviderTest.java index 635e4ca09..0e6a424c3 100644 --- a/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/AppConfigProviderTest.java +++ b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/AppConfigProviderTest.java @@ -6,15 +6,16 @@ import org.mockito.Captor; import org.mockito.Mock; import org.mockito.Mockito; -import software.amazon.awssdk.services.appconfig.AppConfigClient; -import software.amazon.awssdk.services.appconfig.model.GetEnvironmentRequest; -import software.amazon.awssdk.services.appconfig.model.GetEnvironmentResponse; -import software.amazon.awssdk.services.dynamodb.model.AttributeValue; +import software.amazon.awssdk.core.SdkBytes; +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; +import software.amazon.awssdk.services.appconfigdata.AppConfigDataClient; +import software.amazon.awssdk.services.appconfigdata.model.GetLatestConfigurationRequest; +import software.amazon.awssdk.services.appconfigdata.model.GetLatestConfigurationResponse; +import software.amazon.awssdk.services.appconfigdata.model.StartConfigurationSessionRequest; +import software.amazon.awssdk.services.appconfigdata.model.StartConfigurationSessionResponse; import software.amazon.lambda.powertools.parameters.cache.CacheManager; -import org.junit.jupiter.api.Assertions; import static org.assertj.core.api.Assertions.assertThat; -import java.util.HashMap; import java.util.Optional; import static org.mockito.ArgumentMatchers.eq; @@ -23,15 +24,18 @@ public class AppConfigProviderTest { @Mock - AppConfigClient client; + AppConfigDataClient client; - @Mock - GetEnvironmentResponse environment; +// @Mock +// GetEnvironmentResponse environment; private AppConfigProvider provider; @Captor - ArgumentCaptor getEnvironmentCaptor; + ArgumentCaptor startSessionRequestCaptor; + + @Captor + ArgumentCaptor getLatestConfigurationRequestCaptor; @Captor ArgumentCaptor getValueForFieldCaptor; @@ -40,40 +44,74 @@ public class AppConfigProviderTest { private final String applicationName = "fakeApp"; + private final String defaultTestKey = "key1"; + @BeforeEach public void init() { openMocks(this); - // getEnvironment gets called in the constructor, so we need to mock it now - Mockito.when(client.getEnvironment(getEnvironmentCaptor.capture())).thenReturn(environment); - CacheManager cacheManager = new CacheManager(); provider = new AppConfigProvider(cacheManager, client, environmentName, applicationName); } - @Test - public void getEnvironmentReturnsEnvironment() { - // getEnvironment gets called when we construct the AppConfigProvider. Let's make - // sure the request included the right environment name - // Assert - assertThat(getEnvironmentCaptor.getValue().environmentId()).isEqualTo(environmentName); - } @Test - public void getValueReturnsValue() { + public void getValueRetrievesValue() { // Arrange - String key = "Key1"; - String expectedValue = "Value1"; - Mockito.when(environment.getValueForField(getValueForFieldCaptor.capture(), eq(String.class))) - .thenReturn(Optional.of(expectedValue)); + StartConfigurationSessionResponse firstSession = StartConfigurationSessionResponse.builder() + .initialConfigurationToken("token1") + .build(); + GetLatestConfigurationResponse firstResponse = GetLatestConfigurationResponse.builder() + .nextPollConfigurationToken("token2") + .configuration(SdkBytes.fromUtf8String("value1")) + .build(); + GetLatestConfigurationResponse secondResponse = GetLatestConfigurationResponse.builder() + .nextPollConfigurationToken("token3") + .configuration(SdkBytes.fromUtf8String("value2")) + .build(); + GetLatestConfigurationResponse thirdResponse = GetLatestConfigurationResponse.builder() + .nextPollConfigurationToken("token4") + .build(); + + Mockito.when(client.startConfigurationSession(startSessionRequestCaptor.capture())) + .thenReturn(firstSession); + Mockito.when(client.getLatestConfiguration(getLatestConfigurationRequestCaptor.capture())) + .thenReturn(firstResponse, secondResponse, thirdResponse); // Act - String returnedValue = provider.getValue(key); + String returnedValue1 = provider.getValue(defaultTestKey); + String returnedValue2 = provider.getValue(defaultTestKey); + String returnedValue3 = provider.getValue(defaultTestKey); // Assert - assertThat(returnedValue).isEqualTo(expectedValue); - assertThat(getValueForFieldCaptor.getValue()).isEqualTo(key); + assertThat(returnedValue1).isEqualTo(firstResponse.configuration().asUtf8String()); + assertThat(returnedValue2).isEqualTo(secondResponse.configuration().asUtf8String()); + assertThat(returnedValue3).isEqualTo(secondResponse.configuration().asUtf8String()); // Third response is mocked to return null and should re-use previous value + assertThat(startSessionRequestCaptor.getValue().applicationIdentifier()).isEqualTo(applicationName); + assertThat(startSessionRequestCaptor.getValue().environmentIdentifier()).isEqualTo(environmentName); + assertThat(startSessionRequestCaptor.getValue().configurationProfileIdentifier()).isEqualTo(defaultTestKey); + assertThat(getLatestConfigurationRequestCaptor.getAllValues().get(0).configurationToken()).isEqualTo(firstSession.initialConfigurationToken()); + assertThat(getLatestConfigurationRequestCaptor.getAllValues().get(1).configurationToken()).isEqualTo(firstResponse.nextPollConfigurationToken()); + + } + + + @Test + public void stubIntegrationTest() { + AppConfigDataClient appConfigClient = AppConfigDataClient.builder() + .httpClientBuilder(UrlConnectionHttpClient.builder()) + .build(); + + AppConfigProvider provider = AppConfigProvider.builder() + .withCacheManager(new CacheManager()) + .withClient(appConfigClient) + .withApplication("scottsapp") + .withEnvironment("dev") + .build(); + + String value = provider.get("myfield"); + assertThat(value).isEqualTo("myvalue"); } } From 3639a1e85a78345194be687c159f8db9e6a1bd63 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Thu, 1 Jun 2023 14:25:53 +0800 Subject: [PATCH 06/20] E2E test for params working --- .../handlers/parameters/pom.xml | 6 +++- .../lambda/powertools/e2e/Function.java | 13 +++++--- .../amazon/lambda/powertools/e2e/Input.java | 32 +++++++++++++------ powertools-e2e-tests/handlers/pom.xml | 8 ++++- .../lambda/powertools/ParametersE2ET.java | 23 ++++++++++--- .../powertools/testutils/Infrastructure.java | 11 +++++++ .../parameters/AppConfigProvider.java | 2 +- 7 files changed, 74 insertions(+), 21 deletions(-) diff --git a/powertools-e2e-tests/handlers/parameters/pom.xml b/powertools-e2e-tests/handlers/parameters/pom.xml index a46702ca4..baaedb324 100644 --- a/powertools-e2e-tests/handlers/parameters/pom.xml +++ b/powertools-e2e-tests/handlers/parameters/pom.xml @@ -8,7 +8,7 @@ 1.0.0 - e2e-test-handler-logging + e2e-test-handler-parameters jar A Lambda function using powertools logging @@ -17,6 +17,10 @@ software.amazon.lambda powertools-logging + + software.amazon.lambda + powertools-parameters + com.amazonaws aws-lambda-java-events diff --git a/powertools-e2e-tests/handlers/parameters/src/main/java/software/amazon/lambda/powertools/e2e/Function.java b/powertools-e2e-tests/handlers/parameters/src/main/java/software/amazon/lambda/powertools/e2e/Function.java index 5a9a87109..2ac7577c9 100644 --- a/powertools-e2e-tests/handlers/parameters/src/main/java/software/amazon/lambda/powertools/e2e/Function.java +++ b/powertools-e2e-tests/handlers/parameters/src/main/java/software/amazon/lambda/powertools/e2e/Function.java @@ -5,7 +5,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import software.amazon.lambda.powertools.logging.Logging; -import software.amazon.lambda.powertools.logging.LoggingUtils; +import software.amazon.lambda.powertools.parameters.cache.CacheManager; +import software.amazon.lambda.powertools.parameters.AppConfigProvider; public class Function implements RequestHandler { @@ -14,9 +15,13 @@ public class Function implements RequestHandler { @Logging public String handleRequest(Input input, Context context) { - LoggingUtils.appendKeys(input.getKeys()); - LOG.info(input.getMessage()); + AppConfigProvider provider = AppConfigProvider.builder() + .withCacheManager(new CacheManager()) + .withApplication(input.getApp()) + .withEnvironment(input.getEnvironment()) + .build(); + + return provider.get(input.getKey()); - return "OK"; } } \ No newline at end of file diff --git a/powertools-e2e-tests/handlers/parameters/src/main/java/software/amazon/lambda/powertools/e2e/Input.java b/powertools-e2e-tests/handlers/parameters/src/main/java/software/amazon/lambda/powertools/e2e/Input.java index 83afbbd5a..14e2e4c78 100644 --- a/powertools-e2e-tests/handlers/parameters/src/main/java/software/amazon/lambda/powertools/e2e/Input.java +++ b/powertools-e2e-tests/handlers/parameters/src/main/java/software/amazon/lambda/powertools/e2e/Input.java @@ -3,25 +3,37 @@ import java.util.Map; public class Input { - private String message; - private Map keys; + + private String app; + private String environment; + private String key; public Input() { + + } + + public void setApp(String app) { + this.app = app; } - public String getMessage() { - return message; + public void setEnvironment(String environment) { + this.environment = environment; } - public void setMessage(String message) { - this.message = message; + public void setKey(String key) { + this.key = key; } - public Map getKeys() { - return keys; + public String getApp() { + return app; } - public void setKeys(Map keys) { - this.keys = keys; + public String getEnvironment() { + return environment; } + + public String getKey() { + return key; + } + } diff --git a/powertools-e2e-tests/handlers/pom.xml b/powertools-e2e-tests/handlers/pom.xml index b82a9a0c9..82c30dd68 100644 --- a/powertools-e2e-tests/handlers/pom.xml +++ b/powertools-e2e-tests/handlers/pom.xml @@ -10,7 +10,7 @@ Fake handlers that use Lambda Powertools for Java. - 1.14.0 + 1.15.0 UTF-8 11 11 @@ -27,6 +27,7 @@ tracing metrics idempotency + parameters @@ -51,6 +52,11 @@ powertools-idempotency ${lambda.powertools.version} + + software.amazon.lambda + powertools-parameters + ${lambda.powertools.version} + com.amazonaws aws-lambda-java-core diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/ParametersE2ET.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/ParametersE2ET.java index eca42b012..703123ccc 100644 --- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/ParametersE2ET.java +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/ParametersE2ET.java @@ -4,6 +4,7 @@ import org.junit.jupiter.api.*; import software.amazon.lambda.powertools.testutils.AppConfig; import software.amazon.lambda.powertools.testutils.Infrastructure; +import software.amazon.lambda.powertools.testutils.lambda.InvocationResult; import java.util.HashMap; import java.util.Map; @@ -11,6 +12,9 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.assertj.core.api.Assertions.assertThat; +import static software.amazon.lambda.powertools.testutils.lambda.LambdaInvoker.invokeFunction; + @TestInstance(TestInstance.Lifecycle.PER_CLASS) public class ParametersE2ET { @@ -19,9 +23,6 @@ public class ParametersE2ET { private Infrastructure infrastructure; private String functionName; - - private String appConfigAppName = "powertools-e2e-test-app"; - private String appConfigEnvironment = "e2etest"; private final AppConfig appConfig; public ParametersE2ET() { @@ -48,14 +49,28 @@ public void setup() { } // @AfterAll -// public static void tearDown() { +// public void tearDown() { // if (infrastructure != null) // infrastructure.destroy(); // } @Test public void test_getAppConfigValue() { + for (Map.EntryconfigKey: appConfig.getConfigurationValues().entrySet()) { + + // Arrange + String event1 = "{" + + "\"app\": \"" + appConfig.getApplication() + "\", " + + "\"environment\": \"" + appConfig.getEnvironment() + "\", " + + "\"key\": \"" + configKey.getKey() + "\"" + + "}"; + + // Act + InvocationResult invocationResult = invokeFunction(functionName, event1); + // Assert + assertThat(invocationResult.getResult()).isEqualTo("\"" + configKey.getValue() + "\""); + } } } diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java index 6991e6d22..a678ef649 100644 --- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java @@ -13,8 +13,11 @@ import software.amazon.awscdk.services.dynamodb.BillingMode; import software.amazon.awscdk.services.dynamodb.Table; import software.amazon.awscdk.services.groundstation.CfnConfig; +import software.amazon.awscdk.services.iam.PolicyStatement; +import software.amazon.awscdk.services.iam.ServicePrincipal; import software.amazon.awscdk.services.lambda.Code; import software.amazon.awscdk.services.lambda.Function; +import software.amazon.awscdk.services.lambda.Permission; import software.amazon.awscdk.services.lambda.Tracing; import software.amazon.awscdk.services.logs.LogGroup; import software.amazon.awscdk.services.logs.RetentionDays; @@ -313,6 +316,14 @@ private Stack createStackWithLambda() { .replicateTo("NONE") .build(); + // Get the lambda permission to use AppConfig + function.addToRolePolicy(PolicyStatement.Builder + .create() + .actions(singletonList("appconfig:*")) + .resources(singletonList("*")) + .build() + ); + CfnDeployment previousDeployment = null; for (Map.Entry entry : appConfig.getConfigurationValues().entrySet()) { CfnConfigurationProfile configProfile = CfnConfigurationProfile.Builder diff --git a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java index 15104c707..6d45b7237 100644 --- a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java +++ b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java @@ -108,7 +108,7 @@ public static AppConfigProvider.Builder builder() { return new AppConfigProvider.Builder(); } - static class Builder { + public static class Builder { private AppConfigDataClient client; private CacheManager cacheManager; private TransformationManager transformationManager; From c71e4e09acdb0060218a0674cde9d4f343401e43 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Fri, 2 Jun 2023 11:57:02 +0800 Subject: [PATCH 07/20] More tests + inline doco --- .../parameters/AppConfigProvider.java | 26 ++++++-- .../parameters/AppConfigProviderTest.java | 61 ++++++++++++++----- 2 files changed, 68 insertions(+), 19 deletions(-) diff --git a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java index 6d45b7237..516d22c25 100644 --- a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java +++ b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java @@ -14,6 +14,20 @@ import java.util.HashMap; import java.util.Map; +/** + * Implements a {@link ParamProvider} on top of the AppConfig service. AppConfig provides + * a mechanism to retrieve and update configuration of applications over time. + * AppConfig requires the user to create an application, environment, and configuration profile. + * The configuration profile's value can then retrieved, by key name, through this provider. + * + * Because AppConfig is designed to handle rollouts of configuration over time, we must first + * establish a session for each key we wish to retrieve, and then poll the session for the latest + * value when the user re-requests it. This means we must hold a keyed set of session tokens + * and values. + * + * @see Parameters provider documentation + * @see AppConfig documentation + */ public class AppConfigProvider extends BaseProvider{ private static class EstablishedSession { @@ -61,7 +75,9 @@ public AppConfigProvider(CacheManager cacheManager, String environment, String a @Override protected String getValue(String key) { - // Start a configuration session if we don't already have one to get the initial token + // Start a configuration session if we don't already have one for the key requested + // so that we can the initial token. If we already have a session, we can take + // the next request token from there. EstablishedSession establishedSession = establishedSessions.getOrDefault(key, null); String sessionToken = establishedSession != null? establishedSession.nextSessionToken : @@ -72,15 +88,17 @@ protected String getValue(String key) { .build()) .initialConfigurationToken(); - // Get the configuration + // Get the configuration using the token GetLatestConfigurationResponse response = client.getLatestConfiguration(GetLatestConfigurationRequest.builder() .configurationToken(sessionToken) .build()); - // Get the next token + // Get the next session token we'll use next time we are asked for this key String nextSessionToken = response.nextPollConfigurationToken(); - // Get the value + // Get the value of the key. Note that AppConfig will return null if the value + // has not changed since we last asked for it in this session - in this case + // we return the value we stashed at last request. String value = response.configuration() != null? response.configuration().asUtf8String() : // if we have a new value, use it establishedSession != null? diff --git a/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/AppConfigProviderTest.java b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/AppConfigProviderTest.java index 0e6a424c3..5b9f5ad5d 100644 --- a/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/AppConfigProviderTest.java +++ b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/AppConfigProviderTest.java @@ -55,25 +55,32 @@ public void init() { } - + /** + * Tests repeated calls to the AppConfigProvider for the same key behave correctly. This is more complicated than + * it seems, as the service itself will return no-data if the value of a property remains unchanged since the + * start of a session. This means the provider must cache the result and return it again if it gets no data, but + * subsequent calls should once again return the new data. + */ @Test public void getValueRetrievesValue() { // Arrange StartConfigurationSessionResponse firstSession = StartConfigurationSessionResponse.builder() .initialConfigurationToken("token1") .build(); + // first response returns 'value1' GetLatestConfigurationResponse firstResponse = GetLatestConfigurationResponse.builder() .nextPollConfigurationToken("token2") .configuration(SdkBytes.fromUtf8String("value1")) .build(); + // Second response returns 'value2' GetLatestConfigurationResponse secondResponse = GetLatestConfigurationResponse.builder() .nextPollConfigurationToken("token3") .configuration(SdkBytes.fromUtf8String("value2")) .build(); + // Third response returns nothing, which means the provider should yield the previous value again GetLatestConfigurationResponse thirdResponse = GetLatestConfigurationResponse.builder() .nextPollConfigurationToken("token4") .build(); - Mockito.when(client.startConfigurationSession(startSessionRequestCaptor.capture())) .thenReturn(firstSession); Mockito.when(client.getLatestConfiguration(getLatestConfigurationRequestCaptor.capture())) @@ -93,25 +100,49 @@ public void getValueRetrievesValue() { assertThat(startSessionRequestCaptor.getValue().configurationProfileIdentifier()).isEqualTo(defaultTestKey); assertThat(getLatestConfigurationRequestCaptor.getAllValues().get(0).configurationToken()).isEqualTo(firstSession.initialConfigurationToken()); assertThat(getLatestConfigurationRequestCaptor.getAllValues().get(1).configurationToken()).isEqualTo(firstResponse.nextPollConfigurationToken()); - + assertThat(getLatestConfigurationRequestCaptor.getAllValues().get(2).configurationToken()).isEqualTo(secondResponse.nextPollConfigurationToken()); } - + /** + * If we mix requests for different keys together through the same provider, retrieval should + * work as expected. This means two separate configuration sessions should be established with AppConfig. + */ @Test - public void stubIntegrationTest() { - AppConfigDataClient appConfigClient = AppConfigDataClient.builder() - .httpClientBuilder(UrlConnectionHttpClient.builder()) + public void multipleKeysRetrievalWorks() { + // Arrange + String param1Key = "key1"; + StartConfigurationSessionResponse param1Session = StartConfigurationSessionResponse.builder() + .initialConfigurationToken("token1a") .build(); - - AppConfigProvider provider = AppConfigProvider.builder() - .withCacheManager(new CacheManager()) - .withClient(appConfigClient) - .withApplication("scottsapp") - .withEnvironment("dev") + GetLatestConfigurationResponse param1Response = GetLatestConfigurationResponse.builder() + .nextPollConfigurationToken("token1b") + .configuration(SdkBytes.fromUtf8String("value1")) + .build(); + String param2Key = "key2"; + StartConfigurationSessionResponse param2Session = StartConfigurationSessionResponse.builder() + .initialConfigurationToken("token2a") .build(); + GetLatestConfigurationResponse param2Response = GetLatestConfigurationResponse.builder() + .nextPollConfigurationToken("token2b") + .configuration(SdkBytes.fromUtf8String("value1")) + .build(); + Mockito.when(client.startConfigurationSession(startSessionRequestCaptor.capture())) + .thenReturn(param1Session, param2Session); + Mockito.when(client.getLatestConfiguration(getLatestConfigurationRequestCaptor.capture())) + .thenReturn(param1Response, param2Response); + + // Act + String firstKeyValue = provider.getValue(param1Key); + String secondKeyValue = provider.getValue(param2Key); + + // Assert + assertThat(firstKeyValue).isEqualTo(param1Response.configuration().asUtf8String()); + assertThat(secondKeyValue).isEqualTo(param2Response.configuration().asUtf8String()); + assertThat(startSessionRequestCaptor.getAllValues().get(0).configurationProfileIdentifier()).isEqualTo(param1Key); + assertThat(startSessionRequestCaptor.getAllValues().get(1).configurationProfileIdentifier()).isEqualTo(param2Key); + assertThat(getLatestConfigurationRequestCaptor.getAllValues().get(0).configurationToken()).isEqualTo(param1Session.initialConfigurationToken()); + assertThat(getLatestConfigurationRequestCaptor.getAllValues().get(1).configurationToken()).isEqualTo(param2Session.initialConfigurationToken()); - String value = provider.get("myfield"); - assertThat(value).isEqualTo("myvalue"); } } From dce177fa9e6315523284e36ce6a3f59ddab58feb Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Fri, 2 Jun 2023 12:12:44 +0800 Subject: [PATCH 08/20] Fix bug and docs --- docs/utilities/parameters.md | 43 +++++++++++++++++++ .../lambda/powertools/e2e/Function.java | 9 +--- .../parameters/AppConfigProvider.java | 2 +- 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/docs/utilities/parameters.md b/docs/utilities/parameters.md index e8df1f8d6..125eb518b 100644 --- a/docs/utilities/parameters.md +++ b/docs/utilities/parameters.md @@ -212,6 +212,49 @@ a `DynamoDbProvider` providing a client if you need to configure it yourself. } ``` +## AppConfig +To get parameters stored in AppConfig, use `getAppConfigProvider`, providing the application and environment +name to retrieve configuration from. As with the other providers, an overloaded methods allows you to retrieve +an `AppConfigProvider` providing a client if you need to configure it yourself. + +=== "AppConfigProvider" + + ```java hl_lines="6 9" + import software.amazon.lambda.powertools.parameters.AppConfigProvider; + import software.amazon.lambda.powertools.parameters.ParamManager; + + public class AppWitAppConfigParameters implements RequestHandler { + // Get an instance of the AppConfigProvider + AppConfigProvider appConfigProvider = ParamManager.getAppConfigProvider("my-environment", "my-app"); + + // Retrieve a single parameter + String value = appConfigProvider.get("my-key"); + } + ``` + +=== "AppConfigProvider with a custom client" + + ```java hl_lines="9 10 11 12 15 18" + import software.amazon.lambda.powertools.parameters.AppConfigProvider; + import software.amazon.lambda.powertools.parameters.ParamManager; + import software.amazon.awssdk.services.appconfigdata.AppConfigDataClient; + import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; + import software.amazon.awssdk.regions.Region; + + public class AppWithDynamoDbParameters implements RequestHandler { + // Get an AppConfig Client with an explicit region + AppConfigDataClient appConfigDataClient = AppConfigDataClient.builder() + .httpClientBuilder(UrlConnectionHttpClient.builder()) + .region(Region.EU_CENTRAL_2) + .build(); + + // Get an instance of the DynamoDbProvider + AppConfigProvider appConfigProvider = ParamManager.getAppConfigProvider(appConfigDataClient, "my-environment", "my-app"); + + // Retrieve a single parameter + String value = appConfigProvider.get("my-key"); + } + ``` ## Advanced configuration diff --git a/powertools-e2e-tests/handlers/parameters/src/main/java/software/amazon/lambda/powertools/e2e/Function.java b/powertools-e2e-tests/handlers/parameters/src/main/java/software/amazon/lambda/powertools/e2e/Function.java index 2ac7577c9..f7f784f78 100644 --- a/powertools-e2e-tests/handlers/parameters/src/main/java/software/amazon/lambda/powertools/e2e/Function.java +++ b/powertools-e2e-tests/handlers/parameters/src/main/java/software/amazon/lambda/powertools/e2e/Function.java @@ -5,6 +5,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import software.amazon.lambda.powertools.logging.Logging; +import software.amazon.lambda.powertools.parameters.ParamManager; import software.amazon.lambda.powertools.parameters.cache.CacheManager; import software.amazon.lambda.powertools.parameters.AppConfigProvider; @@ -14,13 +15,7 @@ public class Function implements RequestHandler { @Logging public String handleRequest(Input input, Context context) { - - AppConfigProvider provider = AppConfigProvider.builder() - .withCacheManager(new CacheManager()) - .withApplication(input.getApp()) - .withEnvironment(input.getEnvironment()) - .build(); - + AppConfigProvider provider = ParamManager.getAppConfigProvider(input.getEnvironment(), input.getApp()); return provider.get(input.getKey()); } diff --git a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java index 516d22c25..7a3d3667e 100644 --- a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java +++ b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java @@ -126,7 +126,7 @@ public static AppConfigProvider.Builder builder() { return new AppConfigProvider.Builder(); } - public static class Builder { + static class Builder { private AppConfigDataClient client; private CacheManager cacheManager; private TransformationManager transformationManager; From fcadb92cbd3a5fcf970b8d684c6e6857b067b4ac Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Fri, 2 Jun 2023 12:43:16 +0800 Subject: [PATCH 09/20] More testing --- .../amazon/lambda/powertools/ParametersE2ET.java | 10 +++++----- .../powertools/parameters/AppConfigProvider.java | 8 -------- .../parameters/AppConfigProviderTest.java | 16 ++++++++++------ 3 files changed, 15 insertions(+), 19 deletions(-) diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/ParametersE2ET.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/ParametersE2ET.java index 703123ccc..5abbb98df 100644 --- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/ParametersE2ET.java +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/ParametersE2ET.java @@ -48,11 +48,11 @@ public void setup() { functionName = infrastructure.deploy(); } -// @AfterAll -// public void tearDown() { -// if (infrastructure != null) -// infrastructure.destroy(); -// } + @AfterAll + public void tearDown() { + if (infrastructure != null) + infrastructure.destroy(); + } @Test public void test_getAppConfigValue() { diff --git a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java index 7a3d3667e..be06c63a0 100644 --- a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java +++ b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java @@ -38,14 +38,6 @@ private EstablishedSession(String nextSessionToken, String value) { this.nextSessionToken = nextSessionToken; this.lastConfigurationValue = value; } - - public String getNextSessionToken() { - return nextSessionToken; - } - - public String getLastConfigurationValue() { - return lastConfigurationValue; - } } private final AppConfigDataClient client; diff --git a/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/AppConfigProviderTest.java b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/AppConfigProviderTest.java index 5b9f5ad5d..4b0056812 100644 --- a/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/AppConfigProviderTest.java +++ b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/AppConfigProviderTest.java @@ -14,8 +14,11 @@ import software.amazon.awssdk.services.appconfigdata.model.StartConfigurationSessionRequest; import software.amazon.awssdk.services.appconfigdata.model.StartConfigurationSessionResponse; import software.amazon.lambda.powertools.parameters.cache.CacheManager; +import software.amazon.lambda.powertools.parameters.transform.TransformationManager; + import static org.assertj.core.api.Assertions.assertThat; +import java.time.Duration; import java.util.Optional; import static org.mockito.ArgumentMatchers.eq; @@ -26,9 +29,6 @@ public class AppConfigProviderTest { @Mock AppConfigDataClient client; -// @Mock -// GetEnvironmentResponse environment; - private AppConfigProvider provider; @Captor @@ -49,9 +49,13 @@ public class AppConfigProviderTest { @BeforeEach public void init() { openMocks(this); - - CacheManager cacheManager = new CacheManager(); - provider = new AppConfigProvider(cacheManager, client, environmentName, applicationName); + provider = AppConfigProvider.builder() + .withClient(client) + .withApplication(applicationName) + .withEnvironment(environmentName) + .withCacheManager(new CacheManager()) + .withTransformationManager(new TransformationManager()) + .build(); } From 713158c92e2e8805b669df9f9d5f4444432eeee7 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Fri, 2 Jun 2023 12:50:50 +0800 Subject: [PATCH 10/20] More test --- .../parameters/ParamManagerIntegrationTest.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/ParamManagerIntegrationTest.java b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/ParamManagerIntegrationTest.java index ec1672ead..8c5c6b91c 100644 --- a/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/ParamManagerIntegrationTest.java +++ b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/ParamManagerIntegrationTest.java @@ -19,6 +19,7 @@ import org.mockito.ArgumentCaptor; import org.mockito.Captor; import org.mockito.Mock; +import software.amazon.awssdk.services.appconfigdata.AppConfigDataClient; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest; @@ -45,6 +46,9 @@ public class ParamManagerIntegrationTest { @Mock DynamoDbClient ddbClient; + @Mock + private AppConfigDataClient appConfigDataClient; + @Captor ArgumentCaptor ssmParamCaptor; @@ -57,7 +61,6 @@ public class ParamManagerIntegrationTest { @Captor ArgumentCaptor secretsCaptor; - @BeforeEach public void setup() throws IllegalAccessException { openMocks(this); @@ -129,7 +132,16 @@ public void getDynamoDbProvider() { // Assert assertThat(provider).isNotNull(); + } + + @Test + public void getAppConfigProvier() { + + // Act + AppConfigProvider provider = ParamManager.getAppConfigProvider(appConfigDataClient, "test-env", "test-app"); + // Assert + assertThat(provider).isNotNull(); } } From a804069102a714686af909eeec0641000f1fa3f8 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Fri, 2 Jun 2023 12:54:00 +0800 Subject: [PATCH 11/20] Remove cruft --- .../parameters/exception/MissingAppConfigException.java | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/exception/MissingAppConfigException.java diff --git a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/exception/MissingAppConfigException.java b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/exception/MissingAppConfigException.java deleted file mode 100644 index 67ab11147..000000000 --- a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/exception/MissingAppConfigException.java +++ /dev/null @@ -1,8 +0,0 @@ -package software.amazon.lambda.powertools.parameters.exception; - -public class MissingAppConfigException extends RuntimeException { - public MissingAppConfigException(String msg) { - super(msg); - } - -} From 5cb36dd0afec42a1058dd23e0f0b00d73cf3df8e Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Fri, 2 Jun 2023 12:54:20 +0800 Subject: [PATCH 12/20] MOre cleanup --- .../powertools/parameters/ParamManagerIntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/ParamManagerIntegrationTest.java b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/ParamManagerIntegrationTest.java index 8c5c6b91c..fca0a9362 100644 --- a/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/ParamManagerIntegrationTest.java +++ b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/ParamManagerIntegrationTest.java @@ -135,7 +135,7 @@ public void getDynamoDbProvider() { } @Test - public void getAppConfigProvier() { + public void getAppConfigProvider() { // Act AppConfigProvider provider = ParamManager.getAppConfigProvider(appConfigDataClient, "test-env", "test-app"); From 5ff3e3e36e4b9fa02bcd534c9b55ff716d94925d Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Tue, 6 Jun 2023 08:15:02 +0800 Subject: [PATCH 13/20] Update docs/utilities/parameters.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jérôme Van Der Linden <117538+jeromevdl@users.noreply.github.com> --- docs/utilities/parameters.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/utilities/parameters.md b/docs/utilities/parameters.md index 125eb518b..b85301167 100644 --- a/docs/utilities/parameters.md +++ b/docs/utilities/parameters.md @@ -214,7 +214,7 @@ a `DynamoDbProvider` providing a client if you need to configure it yourself. ## AppConfig To get parameters stored in AppConfig, use `getAppConfigProvider`, providing the application and environment -name to retrieve configuration from. As with the other providers, an overloaded methods allows you to retrieve +name to retrieve configuration from. As with the other providers, an overloaded method allows you to retrieve an `AppConfigProvider` providing a client if you need to configure it yourself. === "AppConfigProvider" From 46a16dbea432bcf201e1db5afc9ae989f89372de Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Tue, 6 Jun 2023 08:15:33 +0800 Subject: [PATCH 14/20] Update powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jérôme Van Der Linden <117538+jeromevdl@users.noreply.github.com> --- .../amazon/lambda/powertools/parameters/AppConfigProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java index be06c63a0..5be850193 100644 --- a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java +++ b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java @@ -18,7 +18,7 @@ * Implements a {@link ParamProvider} on top of the AppConfig service. AppConfig provides * a mechanism to retrieve and update configuration of applications over time. * AppConfig requires the user to create an application, environment, and configuration profile. - * The configuration profile's value can then retrieved, by key name, through this provider. + * The configuration profile's value can then be retrieved, by key name, through this provider. * * Because AppConfig is designed to handle rollouts of configuration over time, we must first * establish a session for each key we wish to retrieve, and then poll the session for the latest From 105efceac2ec3e2678a6da03ca27e9082ddc47ec Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Tue, 6 Jun 2023 08:34:25 +0800 Subject: [PATCH 15/20] Fix comment --- .../amazon/lambda/powertools/parameters/AppConfigProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java index be06c63a0..bfc684156 100644 --- a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java +++ b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java @@ -177,7 +177,7 @@ public AppConfigProvider.Builder withEnvironment(String environment) { } /** - * Mandatory. Provide a CacheManager to the {@link AppConfigProvider} + * Mandatory. Provide an application to the {@link AppConfigProvider} * * @param application the application to pull configuration from * @return the builder to chain calls (eg. 
builder.withCacheManager().build()
) From ff701a8bb86cbd18065d7a12540354ad50fe0b66 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Tue, 6 Jun 2023 08:37:59 +0800 Subject: [PATCH 16/20] Address field name issue --- .../lambda/powertools/parameters/AppConfigProvider.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java index bfc684156..788fb6137 100644 --- a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java +++ b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java @@ -65,6 +65,12 @@ public AppConfigProvider(CacheManager cacheManager, String environment, String a } + /** + * Retrieve the parameter value from the AppConfig parameter store.
+ * + * @param key key of the parameter. This ties back to AppConfig's 'profile' concept + * @return the value of the parameter identified by the key + */ @Override protected String getValue(String key) { // Start a configuration session if we don't already have one for the key requested From e8a95e97fd6eac098635922c92b690697ce2b31b Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Tue, 6 Jun 2023 08:54:21 +0800 Subject: [PATCH 17/20] Use appropriate credentials provider depending on whether or not we're SnapStarting --- .../powertools/testutils/AppConfig.java | 2 ++ .../parameters/AppConfigProvider.java | 36 +++++++++++-------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/AppConfig.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/AppConfig.java index 295ca601c..c87f4ac48 100644 --- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/AppConfig.java +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/AppConfig.java @@ -6,6 +6,8 @@ /** * Defines configuration used to setup an AppConfig * deployment when the infrastructure is rolled out. + * + * All fields are non-nullable. */ public class AppConfig { private String application; diff --git a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java index 788fb6137..94daa72e2 100644 --- a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java +++ b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java @@ -5,15 +5,21 @@ import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.appconfigdata.AppConfigDataClient; +import software.amazon.awssdk.services.appconfigdata.AppConfigDataClientBuilder; import software.amazon.awssdk.services.appconfigdata.model.GetLatestConfigurationRequest; import software.amazon.awssdk.services.appconfigdata.model.GetLatestConfigurationResponse; import software.amazon.awssdk.services.appconfigdata.model.StartConfigurationSessionRequest; +import software.amazon.awssdk.services.ssm.SsmClient; +import software.amazon.awssdk.services.ssm.SsmClientBuilder; +import software.amazon.lambda.powertools.core.internal.LambdaConstants; import software.amazon.lambda.powertools.parameters.cache.CacheManager; import software.amazon.lambda.powertools.parameters.transform.TransformationManager; import java.util.HashMap; import java.util.Map; +import static software.amazon.lambda.powertools.core.internal.LambdaConstants.AWS_LAMBDA_INITIALIZATION_TYPE; + /** * Implements a {@link ParamProvider} on top of the AppConfig service. AppConfig provides * a mechanism to retrieve and update configuration of applications over time. @@ -48,15 +54,6 @@ private EstablishedSession(String nextSessionToken, String value) { private final HashMap establishedSessions = new HashMap<>(); - public AppConfigProvider(CacheManager cacheManager, String environment, String application) { - this(cacheManager, AppConfigDataClient.builder() - .httpClientBuilder(UrlConnectionHttpClient.builder()) - .credentialsProvider(EnvironmentVariableCredentialsProvider.create()) - .region(Region.of(System.getenv(SdkSystemSetting.AWS_REGION.environmentVariable()))) - .build(), - environment, application); - } - AppConfigProvider(CacheManager cacheManager, AppConfigDataClient client, String environment, String application) { super(cacheManager); this.client = client; @@ -147,12 +144,23 @@ public AppConfigProvider build() { throw new IllegalStateException("No application provided; please provide one"); } - AppConfigProvider provider; - if (client != null) { - provider = new AppConfigProvider(cacheManager, client, environment, application); - } else { - provider = new AppConfigProvider(cacheManager, environment, application); + // Create a AppConfigDataClient if we haven't been given one + if (client == null) { + AppConfigDataClientBuilder ssmClientBuilder = AppConfigDataClient.builder() + .httpClientBuilder(UrlConnectionHttpClient.builder()) + .region(Region.of(System.getenv(SdkSystemSetting.AWS_REGION.environmentVariable()))); + + // AWS_LAMBDA_INITIALIZATION_TYPE has two values on-demand and snap-start + // when using snap-start mode, the env var creds provider isn't used and causes a fatal error if set + // fall back to the default provider chain if the mode is anything other than on-demand. + String initializationType = System.getenv().get(AWS_LAMBDA_INITIALIZATION_TYPE); + if (initializationType != null && initializationType.equals(LambdaConstants.ON_DEMAND)) { + ssmClientBuilder.credentialsProvider(EnvironmentVariableCredentialsProvider.create()); + } } + + AppConfigProvider provider = new AppConfigProvider(cacheManager, client, environment, application); + if (transformationManager != null) { provider.setTransformationManager(transformationManager); } From c6eb2c9398fe0bdbe056bb475d9340ed8d611b7b Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Wed, 14 Jun 2023 08:51:29 +0800 Subject: [PATCH 18/20] Address review feedback --- .../lambda/powertools/parameters/AppConfigProvider.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java index 5a1d7780c..a131a94dd 100644 --- a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java +++ b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java @@ -146,7 +146,7 @@ public AppConfigProvider build() { // Create a AppConfigDataClient if we haven't been given one if (client == null) { - AppConfigDataClientBuilder ssmClientBuilder = AppConfigDataClient.builder() + AppConfigDataClientBuilder appConfigDataClientBuilder = AppConfigDataClient.builder() .httpClientBuilder(UrlConnectionHttpClient.builder()) .region(Region.of(System.getenv(SdkSystemSetting.AWS_REGION.environmentVariable()))); @@ -155,8 +155,10 @@ public AppConfigProvider build() { // fall back to the default provider chain if the mode is anything other than on-demand. String initializationType = System.getenv().get(AWS_LAMBDA_INITIALIZATION_TYPE); if (initializationType != null && initializationType.equals(LambdaConstants.ON_DEMAND)) { - ssmClientBuilder.credentialsProvider(EnvironmentVariableCredentialsProvider.create()); + appConfigDataClientBuilder.credentialsProvider(EnvironmentVariableCredentialsProvider.create()); } + + client = appConfigDataClientBuilder.build(); } AppConfigProvider provider = new AppConfigProvider(cacheManager, client, environment, application); From c607af384823de78d4902ab7545c78fe0ace10c7 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Wed, 21 Jun 2023 08:41:06 +0200 Subject: [PATCH 19/20] Address @kozub feedback --- powertools-e2e-tests/handlers/metrics/pom.xml | 2 +- .../java/software/amazon/lambda/powertools/e2e/Input.java | 5 ----- .../lambda/powertools/parameters/AppConfigProvider.java | 2 +- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/powertools-e2e-tests/handlers/metrics/pom.xml b/powertools-e2e-tests/handlers/metrics/pom.xml index 723ad75c5..35db53899 100644 --- a/powertools-e2e-tests/handlers/metrics/pom.xml +++ b/powertools-e2e-tests/handlers/metrics/pom.xml @@ -10,7 +10,7 @@ e2e-test-handler-metrics jar - A Lambda function using powertools metrics + A Lambda function using Powertools for AWS Lambda (Java) Parameters diff --git a/powertools-e2e-tests/handlers/parameters/src/main/java/software/amazon/lambda/powertools/e2e/Input.java b/powertools-e2e-tests/handlers/parameters/src/main/java/software/amazon/lambda/powertools/e2e/Input.java index 14e2e4c78..7ea22143f 100644 --- a/powertools-e2e-tests/handlers/parameters/src/main/java/software/amazon/lambda/powertools/e2e/Input.java +++ b/powertools-e2e-tests/handlers/parameters/src/main/java/software/amazon/lambda/powertools/e2e/Input.java @@ -7,11 +7,6 @@ public class Input { private String app; private String environment; private String key; - - public Input() { - - } - public void setApp(String app) { this.app = app; } diff --git a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java index a131a94dd..90f30cb0e 100644 --- a/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java +++ b/powertools-parameters/src/main/java/software/amazon/lambda/powertools/parameters/AppConfigProvider.java @@ -109,7 +109,7 @@ protected String getValue(String key) { @Override protected Map getMultipleValues(String path) { // Retrieving multiple values is not supported with the AppConfig provider. - throw new RuntimeException("Not implemented"); + throw new RuntimeException("Retrieving multiple parameter values is not supported with the AWS App Config Provider"); } /** From d03d5de76494f2408264460613081711b1be66be Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Wed, 21 Jun 2023 08:47:15 +0200 Subject: [PATCH 20/20] Remove unecessary captor --- .../lambda/powertools/parameters/AppConfigProviderTest.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/AppConfigProviderTest.java b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/AppConfigProviderTest.java index 4b0056812..d72a1f042 100644 --- a/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/AppConfigProviderTest.java +++ b/powertools-parameters/src/test/java/software/amazon/lambda/powertools/parameters/AppConfigProviderTest.java @@ -36,10 +36,6 @@ public class AppConfigProviderTest { @Captor ArgumentCaptor getLatestConfigurationRequestCaptor; - - @Captor - ArgumentCaptor getValueForFieldCaptor; - private final String environmentName = "test"; private final String applicationName = "fakeApp";