chore(v2): Split powertools-idempotency module to sub-modules and add redis implementation

.github/workflows/pr_artifacts_size.yml

@@ -11,7 +11,8 @@ on:
       - 'powertools-core/**'              # not in v2
       - 'powertools-common/**'            # v2 only
       - 'powertools-e2e-tests/**'
-      - 'powertools-idempotency/**'
+      - 'powertools-idempotency-core/**'
+      - 'powertools-idempotency-dynamodb/**'
       - 'powertools-large-messages/**'
       - 'powertools-logging/**'
       - 'powertools-metrics/**'

docs/utilities/idempotency.md

@@ -35,7 +35,7 @@ Depending on your version of Java (either Java 1.8 or 11+), the configuration sl
         ...
         <dependency>
             <groupId>software.amazon.lambda</groupId>
-            <artifactId>powertools-idempotency</artifactId>
+            <artifactId>powertools-idempotency-dynamodb</artifactId>
             <version>{{ powertools.version }}</version>
         </dependency>
         ...
@@ -56,7 +56,7 @@ Depending on your version of Java (either Java 1.8 or 11+), the configuration sl
                      <aspectLibraries>
                          <aspectLibrary>
                              <groupId>software.amazon.lambda</groupId>
-                             <artifactId>powertools-idempotency</artifactId>
+                             <artifactId>powertools-idempotency-dynamodb</artifactId>
                          </aspectLibrary>
                      </aspectLibraries>
                  </configuration>
@@ -80,7 +80,7 @@ Depending on your version of Java (either Java 1.8 or 11+), the configuration sl
         ...
         <dependency>
             <groupId>software.amazon.lambda</groupId>
-            <artifactId>powertools-idempotency</artifactId>
+            <artifactId>powertools-idempotency-dynamodb</artifactId>
             <version>{{ powertools.version }}</version>
         </dependency>
         ...
@@ -101,7 +101,7 @@ Depending on your version of Java (either Java 1.8 or 11+), the configuration sl
                      <aspectLibraries>
                          <aspectLibrary>
                              <groupId>software.amazon.lambda</groupId>
-                             <artifactId>powertools-idempotency</artifactId>
+                             <artifactId>powertools-idempotency-dynamodb</artifactId>
                          </aspectLibrary>
                      </aspectLibraries>
                  </configuration>
@@ -131,7 +131,7 @@ Depending on your version of Java (either Java 1.8 or 11+), the configuration sl
         }
 
         dependencies {
-            aspect 'software.amazon.lambda:powertools-idempotency:{{ powertools.version }}'
+            aspect 'software.amazon.lambda:powertools-idempotency-dynamodb:{{ powertools.version }}'
         }
 
         sourceCompatibility = 11 // or higher
@@ -151,18 +151,20 @@ Depending on your version of Java (either Java 1.8 or 11+), the configuration sl
         }
 
         dependencies {
-            aspect 'software.amazon.lambda:powertools-idempotency:{{ powertools.version }}'
+            aspect 'software.amazon.lambda:powertools-idempotency-dynamodb:{{ powertools.version }}'
         }
 
         sourceCompatibility = 1.8
         targetCompatibility = 1.8
     ```
 
 ### Required resources
-
 Before getting started, you need to create a persistent storage layer where the idempotency utility can store its state - your Lambda functions will need read and write access to it.
+As of now, [Amazon DynamoDB](https://aws.amazon.com/dynamodb/) and [Redis](https://redis.io/) are the supported persistnce layers.
+
+#### Using Amazon DynamoDB as persistent storage layer
 
-As of now, Amazon DynamoDB is the only supported persistent storage layer, so you'll need to create a table first.
+If you are using Amazon DynamoDB  you'll need to create a table.
 
 **Default table configuration**
 
@@ -215,6 +217,70 @@ Resources:
     see 1WCU and 1RCU. Review the [DynamoDB pricing documentation](https://aws.amazon.com/dynamodb/pricing/) to
     estimate the cost.
 
+#### Using Redis as persistent storage layer
+
+##### Redis resources
+
+You need an existing Redis service before setting up Redis as persistent storage layer provider. You can also use Redis compatible services like [Amazon ElastiCache for Redis](https://aws.amazon.com/elasticache/redis/) as persistent storage layer provider.
+!!! tip "Tip:No existing Redis service?"
+If you don't have an existing Redis service, we recommend using DynamoDB as persistent storage layer provider.
+
+If you are using Redis you'll need to provide the Redis host, port, user and password as AWS Lambda environment variables.
+If you want to connect to a Redis cluster instead of a Standalone server, you need to enable Redis cluster mode by setting an AWS Lambda
+environment variable `REDIS_CLUSTER_MODE` to `true`
+In the following example, you can see a SAM template for deploying an AWS Lambda function by specifying the required environment variables.
+If you provide a [custom Redis client](#Customizing Redis client) you can omit the environment variables declaration.
+
+!!! warning "Warning: Avoid including a plain text secret directly in your template"
+This can infer security implications
+
+!!! warning "Warning: Large responses with Redis persistence layer"
+When using this utility with Redis your function's responses must be smaller than 512MB.
+Persisting larger items cannot might cause exceptions.
+
+```yaml hl_lines="9-13" title="AWS Serverless Application Model (SAM) example"
+Resources:
+  IdempotencyFunction:
+    Type: AWS::Serverless::Function
+    Properties:
+      CodeUri: Function
+      Handler: helloworld.App::handleRequest
+      Environment:
+        Variables:
+          REDIS_HOST: %redis-host%
+          REDIS_PORT: %redis-port%
+          REDIS_USER: %redis-user%
+          REDIS_SECRET: %redis-secret%
+          REDIS_CLUSTER_MODE: "true"
+```
+##### VPC Access
+Your AWS Lambda Function must be able to reach the Redis endpoint before using it for idempotency persistent storage layer. In most cases you will need to [configure VPC access](https://docs.aws.amazon.com/lambda/latest/dg/configuration-vpc.html) for your AWS Lambda Function. Using a public accessible Redis is not recommended.
+
+!!! tip "Amazon ElastiCache for Redis as persistent storage layer provider"
+If you intend to use Amazon ElastiCache for Redis for idempotency persistent storage layer, you can also reference [This AWS Tutorial](https://docs.aws.amazon.com/lambda/latest/dg/services-elasticache-tutorial.html).
+
+!!! warning "Amazon ElastiCache Serverless not supported"
+[Amazon ElastiCache Serverless](https://aws.amazon.com/elasticache/features/#Serverless) is not supported for now.
+
+!!! warning "Check network connectivity to Redis server"
+Make sure that your AWS Lambda function can connect to your Redis server.
+
+```yaml hl_lines="9-12" title="AWS Serverless Application Model (SAM) example"
+Resources:
+  IdempotencyFunction:
+    Type: AWS::Serverless::Function
+    Properties:
+      CodeUri: Function
+      Handler: helloworld.App::handleRequest
+      VpcConfig: # (1)!
+        SecurityGroupIds:
+          - sg-{your_sg_id}
+        SubnetIds:
+          - subnet-{your_subnet_id_1}
+          - subnet-{your_subnet_id_2}
+```
+1. Replace the Security Group ID and Subnet ID to match your Redis' VPC setting.
+
 ### Idempotent annotation
 
 You can quickly start by initializing the `DynamoDBPersistenceStore` and using it with the `@Idempotent` annotation on your Lambda handler.
@@ -635,6 +701,29 @@ When using DynamoDB as a persistence layer, you can alter the attribute names by
 | **SortKeyAttr**    |          |                                      | Sort key of the table (if table is configured with a sort key).                                        |
 | **StaticPkValue**  |          | `idempotency#{LAMBDA_FUNCTION_NAME}` | Static value to use as the partition key. Only used when **SortKeyAttr** is set.                       |
 
+#### RedisPersistenceStore
+
+The redis persistence store has as a prerequisite to install a Redis datastore(https://redis.io/docs/about/) in either Standalone or Cluster mode.
+
+We are using [Redis hashes](https://redis.io/docs/data-types/hashes/) to store the idempotency fields and values.
+There are some predefined fields that you can see listed in the following table. The predefined fields have some default values.
+
+
+You can alter the field names by passing these parameters when initializing the persistence layer:
+
+| Parameter          | Required | Default                              | Description                                                                                            |
+|--------------------|----------|--------------------------------------|--------------------------------------------------------------------------------------------------------|
+| **KeyPrefixName**  | Y        | `idempotency`                        | The redis hash key prefix                                                                              |
+| **KeyAttr**        | Y        | `id`                                 | The redis hash key field name                                                                          |
+| **ExpiryAttr**     |          | `expiration`                         | Unix timestamp of when record expires                                                                  |
+| **StatusAttr**     |          | `status`                             | Stores status of the Lambda execution during and after invocation                                      |
+| **DataAttr**       |          | `data`                               | Stores results of successfully idempotent methods                                                      |
+| **ValidationAttr** |          | `validation`                         | Hashed representation of the parts of the event used for validation                                    |
+
+
+!!! Tip "Tip: You can share the same prefix and key for all functions"
+You can reuse the same prefix and key to store idempotency state. We add your function name in addition to the idempotency key as a hash key.
+
 ## Advanced
 
 ### Customizing the default behavior
@@ -884,6 +973,39 @@ When creating the `DynamoDBPersistenceStore`, you can set a custom [`DynamoDbCli
         .build();
     ```
 
+### Customizing Redis client
+
+The `RedisPersistenceStore` uses the JedisPooled java client to connect to the Redis Server.
+When creating the `RedisPersistenceStore`, you can set a custom [`JedisPooled`](https://www.javadoc.io/doc/redis.clients/jedis/latest/redis/clients/jedis/JedisPooled.html) client:
+
+=== "Custom JedisPooled with connection timeout"
+
+    ```java hl_lines="2-6 11"
+    public App() {
+        JedisPooled jedisPooled = new JedisPooled(new HostAndPort("host",6789), DefaultJedisClientConfig.builder()
+                .user("user")
+                .password("secret")
+                .connectionTimeoutMillis(3000)
+                .build())
+
+        Idempotency.config().withPersistenceStore(
+          RedisPersistenceStore.builder()
+                .withKeyPrefixName("items-idempotency")
+                .withJedisPooled(jedisPooled)
+                .build()
+      ).configure();
+    }
+    ```
+
+!!! info "Default configuration is the following:"
+
+    ```java
+    DefaultJedisClientConfig.builder()
+    .user(System.getenv(Constants.REDIS_USER))
+    .password(System.getenv(Constants.REDIS_SECRET))
+    .build();
+    ```
+
 ### Using a DynamoDB table with a composite primary key
 
 When using a composite primary key table (hash+range key), use `SortKeyAttr` parameter when initializing your persistence store.

examples/powertools-examples-idempotency/pom.xml

@@ -42,7 +42,7 @@
         </dependency>
         <dependency>
             <groupId>software.amazon.lambda</groupId>
-            <artifactId>powertools-idempotency</artifactId>
+            <artifactId>powertools-idempotency-dynamodb</artifactId>
             <version>${project.version}</version>
         </dependency>
         <dependency>
@@ -117,7 +117,7 @@
                         </aspectLibrary>
                         <aspectLibrary>
                             <groupId>software.amazon.lambda</groupId>
-                            <artifactId>powertools-idempotency</artifactId>
+                            <artifactId>powertools-idempotency-dynamodb</artifactId>
                         </aspectLibrary>
                     </aspectLibraries>
                 </configuration>

examples/powertools-examples-idempotency/src/main/java/helloworld/App.java

@@ -32,7 +32,7 @@
 import software.amazon.lambda.powertools.idempotency.Idempotency;
 import software.amazon.lambda.powertools.idempotency.IdempotencyConfig;
 import software.amazon.lambda.powertools.idempotency.Idempotent;
-import software.amazon.lambda.powertools.idempotency.persistence.DynamoDBPersistenceStore;
+import software.amazon.lambda.powertools.idempotency.persistence.dynamodb.DynamoDBPersistenceStore;
 import software.amazon.lambda.powertools.logging.Logging;
 import software.amazon.lambda.powertools.utilities.JsonConfig;
 

pom.xml

@@ -381,7 +381,6 @@
                 </configuration>
                 <executions>
                     <execution>
-                        <phase>process-sources</phase>
                         <goals>
                             <goal>compile</goal>
                             <goal>test-compile</goal>

powertools-batch/pom.xml

@@ -25,6 +25,7 @@
     </build>
 
     <artifactId>powertools-batch</artifactId>
+
     <dependencies>
         <dependency>
             <groupId>com.amazonaws</groupId>

powertools-e2e-tests/README.md

@@ -6,8 +6,16 @@ __Prerequisites__:
 ([credentials](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/credentials.html)).
 - [Java 11+](https://docs.aws.amazon.com/corretto/latest/corretto-11-ug/downloads-list.html)
 - [Docker](https://docs.docker.com/engine/install/)
+- [CDK](https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html#getting_started_install)
 
-To execute the E2E tests, use the following command: `export JAVA_VERSION=11 && mvn clean verify -Pe2e`
+### Execute test
+Before executing the tests in a new AWS account, [bootstrap CDK](https://docs.aws.amazon.com/cdk/v2/guide/bootstrapping.htmls) using the following command:
+
+`cdk bootstrap aws://<ACCOUNTID>/<REGION>`
+
+To execute the E2E tests, use the following command:
+
+`export JAVA_VERSION=11 && mvn clean verify -Pe2e`
 
 ### Under the hood
 This module leverages the following components:

powertools-e2e-tests/handlers/idempotency-dynamodb/pom.xml

@@ -0,0 +1,72 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>software.amazon.lambda</groupId>
+        <artifactId>e2e-test-handlers-parent</artifactId>
+        <version>1.0.0</version>
+    </parent>
+
+    <artifactId>e2e-test-handler-idempotency-dynamodb</artifactId>
+    <packaging>jar</packaging>
+    <name>A Lambda function using Powertools for AWS Lambda (Java) idempotency</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>software.amazon.lambda</groupId>
+            <artifactId>powertools-idempotency-dynamodb</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>software.amazon.lambda</groupId>
+            <artifactId>powertools-logging</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-slf4j2-impl</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-lambda-java-events</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.aspectj</groupId>
+            <artifactId>aspectjrt</artifactId>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>dev.aspectj</groupId>
+                <artifactId>aspectj-maven-plugin</artifactId>
+                <configuration>
+                    <source>${maven.compiler.source}</source>
+                    <target>${maven.compiler.target}</target>
+                    <complianceLevel>${maven.compiler.target}</complianceLevel>
+                    <aspectLibraries>
+                        <aspectLibrary>
+                            <groupId>software.amazon.lambda</groupId>
+                            <artifactId>powertools-idempotency-core</artifactId>
+                        </aspectLibrary>
+                        <aspectLibrary>
+                            <groupId>software.amazon.lambda</groupId>
+                            <artifactId>powertools-logging</artifactId>
+                        </aspectLibrary>
+                    </aspectLibraries>
+                </configuration>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>compile</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+</project>

powertools-e2e-tests/handlers/idempotency/src/main/java/software/amazon/lambda/powertools/e2e/Function.java → powertools-e2e-tests/handlers/idempotency-dynamodb/src/main/java/software/amazon/lambda/powertools/e2e/Function.java

@@ -27,7 +27,7 @@
 import software.amazon.lambda.powertools.idempotency.Idempotency;
 import software.amazon.lambda.powertools.idempotency.IdempotencyConfig;
 import software.amazon.lambda.powertools.idempotency.Idempotent;
-import software.amazon.lambda.powertools.idempotency.persistence.DynamoDBPersistenceStore;
+import software.amazon.lambda.powertools.idempotency.persistence.dynamodb.DynamoDBPersistenceStore;
 import software.amazon.lambda.powertools.logging.Logging;
 
 

powertools-e2e-tests/handlers/idempotency/pom.xml → powertools-e2e-tests/handlers/idempotency-redis/pom.xml

@@ -8,14 +8,13 @@
         <version>1.0.0</version>
     </parent>
 
-    <artifactId>e2e-test-handler-idempotency</artifactId>
+    <artifactId>e2e-test-handler-idempotency-redis</artifactId>
     <packaging>jar</packaging>
     <name>A Lambda function using Powertools for AWS Lambda (Java) idempotency</name>
-
     <dependencies>
         <dependency>
             <groupId>software.amazon.lambda</groupId>
-            <artifactId>powertools-idempotency</artifactId>
+            <artifactId>powertools-idempotency-redis</artifactId>
         </dependency>
         <dependency>
             <groupId>software.amazon.lambda</groupId>
@@ -34,7 +33,6 @@
             <artifactId>aspectjrt</artifactId>
         </dependency>
     </dependencies>
-
     <build>
         <plugins>
             <plugin>
@@ -47,7 +45,7 @@
                     <aspectLibraries>
                         <aspectLibrary>
                             <groupId>software.amazon.lambda</groupId>
-                            <artifactId>powertools-idempotency</artifactId>
+                            <artifactId>powertools-idempotency-core</artifactId>
                         </aspectLibrary>
                         <aspectLibrary>
                             <groupId>software.amazon.lambda</groupId>