item) {
+ // data and validation payload may be null
+ String data = item.get(this.dataAttr);
+ String validation = item.get(this.validationAttr);
+ return new DataRecord(item.get(keyAttr),
+ DataRecord.Status.valueOf(item.get(this.statusAttr)),
+ Long.parseLong(item.get(this.expiryAttr)),
+ data,
+ validation,
+ item.get(this.inProgressExpiryAttr) != null ?
+ OptionalLong.of(Long.parseLong(item.get(this.inProgressExpiryAttr))) :
+ OptionalLong.empty());
+ }
+
+ /**
+ * Use this builder to get an instance of {@link RedisPersistenceStore}.
+ * With this builder you can configure the characteristics of the Redis hash attributes.
+ * You can also set a custom {@link JedisPool}.
+ */
+ public static class Builder {
+ private String keyPrefixName = "idempotency";
+ private String keyAttr = "id";
+ private String expiryAttr = "expiration";
+ private String inProgressExpiryAttr = "in-progress-expiration";
+ private String statusAttr = "status";
+ private String dataAttr = "data";
+ private String validationAttr = "validation";
+ private JedisPooled jedisPool;
+
+ /**
+ * Initialize and return a new instance of {@link RedisPersistenceStore}.
+ * Example:
+ *
+ * RedisPersistenceStore.builder().withKeyAttr("uuid").build();
+ *
+ *
+ * @return an instance of the {@link RedisPersistenceStore}
+ */
+ public RedisPersistenceStore build() {
+ return new RedisPersistenceStore(keyPrefixName, keyAttr, expiryAttr,
+ inProgressExpiryAttr, statusAttr, dataAttr, validationAttr, jedisPool);
+ }
+
+ /**
+ * Redis prefix for the hash key (optional), by default "idempotency"
+ *
+ * @param keyPrefixName name of the key prefix
+ * @return the builder instance (to chain operations)
+ */
+ public Builder withKeyPrefixName(String keyPrefixName) {
+ this.keyPrefixName = keyPrefixName;
+ return this;
+ }
+
+ /**
+ * Redis name for hash key (optional), by default "id"
+ *
+ * @param keyAttr name of the key attribute of the hash
+ * @return the builder instance (to chain operations)
+ */
+ public Builder withKeyAttr(String keyAttr) {
+ this.keyAttr = keyAttr;
+ return this;
+ }
+
+ /**
+ * Redis attribute name for expiry timestamp (optional), by default "expiration"
+ *
+ * @param expiryAttr name of the expiry attribute in the hash
+ * @return the builder instance (to chain operations)
+ */
+ public Builder withExpiryAttr(String expiryAttr) {
+ this.expiryAttr = expiryAttr;
+ return this;
+ }
+
+ /**
+ * Redis attribute name for in progress expiry timestamp (optional), by default "in-progress-expiration"
+ *
+ * @param inProgressExpiryAttr name of the attribute in the hash
+ * @return the builder instance (to chain operations)
+ */
+ public Builder withInProgressExpiryAttr(String inProgressExpiryAttr) {
+ this.inProgressExpiryAttr = inProgressExpiryAttr;
+ return this;
+ }
+
+ /**
+ * Redis attribute name for status (optional), by default "status"
+ *
+ * @param statusAttr name of the status attribute in the hash
+ * @return the builder instance (to chain operations)
+ */
+ public Builder withStatusAttr(String statusAttr) {
+ this.statusAttr = statusAttr;
+ return this;
+ }
+
+ /**
+ * Redis attribute name for response data (optional), by default "data"
+ *
+ * @param dataAttr name of the data attribute in the hash
+ * @return the builder instance (to chain operations)
+ */
+ public Builder withDataAttr(String dataAttr) {
+ this.dataAttr = dataAttr;
+ return this;
+ }
+
+ /**
+ * Redis attribute name for validation (optional), by default "validation"
+ *
+ * @param validationAttr name of the validation attribute in the hash
+ * @return the builder instance (to chain operations)
+ */
+ public Builder withValidationAttr(String validationAttr) {
+ this.validationAttr = validationAttr;
+ return this;
+ }
+
+ /**
+ * Custom {@link JedisPool} used to query DynamoDB (optional).
+ *
+ * @param jedisPool the {@link JedisPool} instance to use
+ * @return the builder instance (to chain operations)
+ */
+ public Builder withJedisPooled(JedisPooled jedisPool) {
+ this.jedisPool = jedisPool;
+ return this;
+ }
+ }
+}
diff --git a/powertools-idempotency/powertool-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java b/powertools-idempotency/powertool-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
new file mode 100644
index 000000000..21fdd2652
--- /dev/null
+++ b/powertools-idempotency/powertool-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
@@ -0,0 +1,348 @@
+/*
+ * Copyright 2023 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package software.amazon.lambda.powertools.idempotency.redis;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static software.amazon.lambda.powertools.idempotency.redis.Constants.REDIS_HOST;
+import static software.amazon.lambda.powertools.idempotency.redis.Constants.REDIS_PORT;
+
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.HashMap;
+import java.util.Map;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junitpioneer.jupiter.SetEnvironmentVariable;
+import redis.clients.jedis.JedisPooled;
+import redis.embedded.RedisServer;
+import software.amazon.lambda.powertools.idempotency.IdempotencyConfig;
+import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyItemAlreadyExistsException;
+import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyItemNotFoundException;
+import software.amazon.lambda.powertools.idempotency.persistence.DataRecord;
+
+@SetEnvironmentVariable(key = REDIS_HOST, value = "localhost")
+@SetEnvironmentVariable(key = REDIS_PORT, value = "6379")
+public class RedisPersistenceStoreTest {
+ static RedisServer redisServer;
+ private final RedisPersistenceStore redisPersistenceStore = RedisPersistenceStore.builder().build();
+ private final JedisPooled jedisPool = new JedisPooled();
+
+ public RedisPersistenceStoreTest() {
+ }
+
+ @BeforeAll
+ public static void init() {
+ redisServer = new RedisServer(6379);
+ redisServer.start();
+ }
+
+ @AfterAll
+ public static void stop() {
+ redisServer.stop();
+ }
+
+ @Test
+ public void putRecord_shouldCreateItemInRedis() {
+ Instant now = Instant.now();
+ long ttl = 3600;
+ long expiry = now.plus(ttl, ChronoUnit.SECONDS).getEpochSecond();
+ redisPersistenceStore.putRecord(new DataRecord("key", DataRecord.Status.COMPLETED, expiry, null, null), now);
+
+ Map entry = jedisPool.hgetAll("idempotency:id:key");
+ long ttlInRedis = jedisPool.ttl("idempotency:id:key");
+
+ assertThat(entry).isNotNull();
+ assertThat(entry.get("status")).isEqualTo("COMPLETED");
+ assertThat(entry.get("expiration")).isEqualTo(String.valueOf(expiry));
+ assertThat(Math.round(ttlInRedis / 100.0) * 100).isEqualTo(ttl);
+ }
+
+ @Test
+ public void putRecord_shouldCreateItemInRedis_withExistingJedisClient() {
+ Instant now = Instant.now();
+ long expiry = now.plus(3600, ChronoUnit.SECONDS).getEpochSecond();
+ RedisPersistenceStore store = new RedisPersistenceStore.Builder().withJedisPooled(jedisPool).build();
+ store.putRecord(new DataRecord("key", DataRecord.Status.COMPLETED, expiry, null, null), now);
+
+ Map entry = jedisPool.hgetAll("idempotency:id:key");
+
+ assertThat(entry).isNotNull();
+ assertThat(entry.get("status")).isEqualTo("COMPLETED");
+ assertThat(entry.get("expiration")).isEqualTo(String.valueOf(expiry));
+ }
+
+ @Test
+ public void putRecord_shouldCreateItemInRedis_IfPreviousExpired() {
+
+ Map item = new HashMap<>();
+ Instant now = Instant.now();
+ long expiry = now.minus(30, ChronoUnit.SECONDS).getEpochSecond();
+ item.put("expiration", String.valueOf(expiry));
+ item.put("status", DataRecord.Status.COMPLETED.toString());
+ item.put("data", "Fake Data");
+
+ long ttl = 3600;
+ long expiry2 = now.plus(ttl, ChronoUnit.SECONDS).getEpochSecond();
+ jedisPool.hset("idempotency:id:key", item);
+ redisPersistenceStore.putRecord(
+ new DataRecord("key",
+ DataRecord.Status.INPROGRESS,
+ expiry2,
+ null,
+ null
+ ), now);
+
+ Map entry = jedisPool.hgetAll("idempotency:id:key");
+ long ttlInRedis = jedisPool.ttl("idempotency:id:key");
+
+ assertThat(entry).isNotNull();
+ assertThat(entry.get("status")).isEqualTo("INPROGRESS");
+ assertThat(entry.get("expiration")).isEqualTo(String.valueOf(expiry2));
+ assertThat(Math.round(ttlInRedis / 100.0) * 100).isEqualTo(ttl);
+ }
+
+ @Test
+ public void putRecord_shouldCreateItemInRedis_IfLambdaWasInProgressAndTimedOut() {
+
+ Map item = new HashMap<>();
+ Instant now = Instant.now();
+ long expiry = now.plus(30, ChronoUnit.SECONDS).getEpochSecond();
+ long progressExpiry = now.minus(30, ChronoUnit.SECONDS).toEpochMilli();
+ item.put("expiration", String.valueOf(expiry));
+ item.put("status", DataRecord.Status.INPROGRESS.toString());
+ item.put("data", "Fake Data");
+ item.put("in-progress-expiration", String.valueOf(progressExpiry));
+ jedisPool.hset("idempotency:id:key", item);
+
+ long expiry2 = now.plus(3600, ChronoUnit.SECONDS).getEpochSecond();
+ redisPersistenceStore.putRecord(
+ new DataRecord("key",
+ DataRecord.Status.INPROGRESS,
+ expiry2,
+ null,
+ null
+ ), now);
+
+ Map entry = jedisPool.hgetAll("idempotency:id:key");
+
+ assertThat(entry).isNotNull();
+ assertThat(entry.get("status")).isEqualTo("INPROGRESS");
+ assertThat(entry.get("expiration")).isEqualTo(String.valueOf(expiry2));
+ }
+
+ @Test
+ public void putRecord_shouldThrowIdempotencyItemAlreadyExistsException_IfRecordAlreadyExist() {
+
+ Map item = new HashMap<>();
+ Instant now = Instant.now();
+ long expiry = now.plus(30, ChronoUnit.SECONDS).getEpochSecond();
+ item.put("expiration", String.valueOf(expiry)); // not expired
+ item.put("status", DataRecord.Status.COMPLETED.toString());
+ item.put("data", "Fake Data");
+
+ jedisPool.hset("idempotency:id:key", item);
+
+ long expiry2 = now.plus(3600, ChronoUnit.SECONDS).getEpochSecond();
+ assertThatThrownBy(() -> redisPersistenceStore.putRecord(
+ new DataRecord("key",
+ DataRecord.Status.INPROGRESS,
+ expiry2,
+ null,
+ null
+ ), now)
+ ).isInstanceOf(IdempotencyItemAlreadyExistsException.class);
+
+ Map entry = jedisPool.hgetAll("idempotency:id:key");
+
+ assertThat(entry).isNotNull();
+ assertThat(entry.get("status")).isEqualTo("COMPLETED");
+ assertThat(entry.get("expiration")).isEqualTo(String.valueOf(expiry));
+ assertThat(entry.get("data")).isEqualTo("Fake Data");
+ }
+
+ @Test
+ public void putRecord_shouldBlockUpdate_IfRecordAlreadyExistAndProgressNotExpiredAfterLambdaTimedOut() {
+
+ Map item = new HashMap<>();
+ Instant now = Instant.now();
+ long expiry = now.plus(30, ChronoUnit.SECONDS).getEpochSecond(); // not expired
+ long progressExpiry = now.plus(30, ChronoUnit.SECONDS).toEpochMilli(); // not expired
+ item.put("expiration", String.valueOf(expiry));
+ item.put("status", DataRecord.Status.INPROGRESS.toString());
+ item.put("data", "Fake Data");
+ item.put("in-progress-expiration", String.valueOf(progressExpiry));
+ jedisPool.hset("idempotency:id:key", item);
+
+ long expiry2 = now.plus(3600, ChronoUnit.SECONDS).getEpochSecond();
+ assertThatThrownBy(() -> redisPersistenceStore.putRecord(
+ new DataRecord("key",
+ DataRecord.Status.INPROGRESS,
+ expiry2,
+ "Fake Data 2",
+ null
+ ), now))
+ .isInstanceOf(IdempotencyItemAlreadyExistsException.class);
+
+ Map entry = jedisPool.hgetAll("idempotency:id:key");
+
+ assertThat(entry).isNotNull();
+ assertThat(entry.get("status")).isEqualTo("INPROGRESS");
+ assertThat(entry.get("expiration")).isEqualTo(String.valueOf(expiry));
+ assertThat(entry.get("data")).isEqualTo("Fake Data");
+ }
+
+ @Test
+ public void getRecord_shouldReturnExistingRecord() throws IdempotencyItemNotFoundException {
+
+ Map item = new HashMap<>();
+ Instant now = Instant.now();
+ long expiry = now.plus(30, ChronoUnit.SECONDS).getEpochSecond();
+ item.put("expiration", String.valueOf(expiry));
+ item.put("status", DataRecord.Status.COMPLETED.toString());
+ item.put("data", ("Fake Data"));
+ jedisPool.hset("idempotency:id:key", item);
+
+ DataRecord record = redisPersistenceStore.getRecord("key");
+
+ assertThat(record.getIdempotencyKey()).isEqualTo("key");
+ assertThat(record.getStatus()).isEqualTo(DataRecord.Status.COMPLETED);
+ assertThat(record.getResponseData()).isEqualTo("Fake Data");
+ assertThat(record.getExpiryTimestamp()).isEqualTo(expiry);
+ }
+
+ @Test
+ public void getRecord_shouldThrowException_whenRecordIsAbsent() {
+ assertThatThrownBy(() -> redisPersistenceStore.getRecord("key")).isInstanceOf(
+ IdempotencyItemNotFoundException.class);
+ }
+
+ @Test
+ public void updateRecord_shouldUpdateRecord() {
+ Map item = new HashMap<>();
+ Instant now = Instant.now();
+ long expiry = now.plus(360, ChronoUnit.SECONDS).getEpochSecond();
+ item.put("expiration", String.valueOf(expiry));
+ item.put("status", DataRecord.Status.INPROGRESS.toString());
+ jedisPool.hset("idempotency:id:key", item);
+ // enable payload validation
+ redisPersistenceStore.configure(IdempotencyConfig.builder().withPayloadValidationJMESPath("path").build(),
+ null);
+
+ long ttl = 3600;
+ expiry = now.plus(ttl, ChronoUnit.SECONDS).getEpochSecond();
+ DataRecord record = new DataRecord("key", DataRecord.Status.COMPLETED, expiry, "Fake result", "hash");
+ redisPersistenceStore.updateRecord(record);
+
+ Map itemInDb = jedisPool.hgetAll("idempotency:id:key");
+ long ttlInRedis = jedisPool.ttl("idempotency:id:key");
+
+ assertThat(itemInDb.get("status")).isEqualTo("COMPLETED");
+ assertThat(itemInDb.get("expiration")).isEqualTo(String.valueOf(expiry));
+ assertThat(itemInDb.get("data")).isEqualTo("Fake result");
+ assertThat(itemInDb.get("validation")).isEqualTo("hash");
+ assertThat(Math.round(ttlInRedis / 100.0) * 100).isEqualTo(ttl);
+ }
+
+ @Test
+ public void deleteRecord_shouldDeleteRecord() {
+ Map item = new HashMap<>();
+ Instant now = Instant.now();
+ long expiry = now.plus(360, ChronoUnit.SECONDS).getEpochSecond();
+ item.put("expiration", String.valueOf(expiry));
+ item.put("status", DataRecord.Status.INPROGRESS.toString());
+ jedisPool.hset("idempotency:id:key", item);
+
+ redisPersistenceStore.deleteRecord("key");
+
+ Map items = jedisPool.hgetAll("idempotency:id:key");
+
+ assertThat(items.isEmpty()).isTrue();
+ }
+
+
+ @Test
+ public void endToEndWithCustomAttrNamesAndSortKey() throws IdempotencyItemNotFoundException {
+ try {
+ RedisPersistenceStore persistenceStore = RedisPersistenceStore.builder()
+ .withKeyPrefixName("items-idempotency")
+ .withJedisPooled(jedisPool)
+ .withDataAttr("result")
+ .withExpiryAttr("expiry")
+ .withKeyAttr("key")
+ .withStatusAttr("state")
+ .withValidationAttr("valid")
+ .build();
+
+ Instant now = Instant.now();
+ DataRecord record = new DataRecord(
+ "mykey",
+ DataRecord.Status.INPROGRESS,
+ now.plus(400, ChronoUnit.SECONDS).getEpochSecond(),
+ null,
+ null
+ );
+ // PUT
+ persistenceStore.putRecord(record, now);
+
+ Map itemInDb = jedisPool.hgetAll("items-idempotency:key:mykey");
+
+ // GET
+ DataRecord recordInDb = persistenceStore.getRecord("mykey");
+
+ assertThat(itemInDb).isNotNull();
+ assertThat(itemInDb.get("state")).isEqualTo(recordInDb.getStatus().toString());
+ assertThat(itemInDb.get("expiry")).isEqualTo(String.valueOf(recordInDb.getExpiryTimestamp()));
+
+ // UPDATE
+ DataRecord updatedRecord = new DataRecord(
+ "mykey",
+ DataRecord.Status.COMPLETED,
+ now.plus(500, ChronoUnit.SECONDS).getEpochSecond(),
+ "response",
+ null
+ );
+ persistenceStore.updateRecord(updatedRecord);
+ recordInDb = persistenceStore.getRecord("mykey");
+ assertThat(recordInDb).isEqualTo(updatedRecord);
+
+ // DELETE
+ persistenceStore.deleteRecord("mykey");
+ assertThat(jedisPool.hgetAll("items-idempotency:key:mykey").size()).isEqualTo(0);
+
+ } finally {
+ try {
+ jedisPool.del("items-idempotency:key:mykey");
+ } catch (Exception e) {
+ // OK
+ }
+ }
+ }
+
+ @Test
+ @SetEnvironmentVariable(key = software.amazon.lambda.powertools.idempotency.Constants.IDEMPOTENCY_DISABLED_ENV, value = "true")
+ public void idempotencyDisabled_noClientShouldBeCreated() {
+ RedisPersistenceStore store = RedisPersistenceStore.builder().build();
+ assertThatThrownBy(() -> store.getRecord("key")).isInstanceOf(NullPointerException.class);
+ }
+
+ @AfterEach
+ public void emptyDB() {
+ jedisPool.del("idempotency:id:key");
+ }
+
+}
diff --git a/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/exceptions/IdempotencyItemAlreadyExistsException.java b/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/exceptions/IdempotencyItemAlreadyExistsException.java
index ba7da69bf..42e17b5db 100644
--- a/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/exceptions/IdempotencyItemAlreadyExistsException.java
+++ b/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/exceptions/IdempotencyItemAlreadyExistsException.java
@@ -27,4 +27,9 @@ public IdempotencyItemAlreadyExistsException() {
public IdempotencyItemAlreadyExistsException(String msg, Throwable e) {
super(msg, e);
}
+
+ public IdempotencyItemAlreadyExistsException(String msg) {
+ super(msg);
+ }
+
}
From 30722baa98362168078ccaa10e2cc9813e1b2e3e Mon Sep 17 00:00:00 2001
From: Eleni Dimitropoulou <12170229+eldimi@users.noreply.github.com>
Date: Fri, 24 Nov 2023 16:42:47 +0200
Subject: [PATCH 14/31] Fix string replacement and add unit test
---
.../redis/RedisPersistenceStore.java | 5 +----
.../redis/RedisPersistenceStoreTest.java | 20 +++++++++++++++++++
2 files changed, 21 insertions(+), 4 deletions(-)
diff --git a/powertools-idempotency/powertool-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java b/powertools-idempotency/powertool-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java
index 1a1065e0b..a33e7182c 100644
--- a/powertools-idempotency/powertool-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java
+++ b/powertools-idempotency/powertool-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java
@@ -16,18 +16,15 @@
import java.time.Instant;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.OptionalLong;
-import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.DefaultJedisClientConfig;
import redis.clients.jedis.HostAndPort;
import redis.clients.jedis.JedisClientConfig;
-import redis.clients.jedis.JedisCluster;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPooled;
import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyItemAlreadyExistsException;
@@ -171,7 +168,7 @@ private Object putItemOnCondition(DataRecord record, Instant now, String inProgr
// only insert in-progress-expiry if it is set
if (inProgressExpiry != null) {
- insertItemExpression.replace(")", ", KEYS[4], ARGV[6])");
+ insertItemExpression = insertItemExpression.replace(")", ", KEYS[4], ARGV[6])");
}
// if redisHashExistsExpression or itemExpiredExpression or itemIsInProgressExpression then insertItemExpression
diff --git a/powertools-idempotency/powertool-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java b/powertools-idempotency/powertool-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
index 21fdd2652..b44db768f 100644
--- a/powertools-idempotency/powertool-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
+++ b/powertools-idempotency/powertool-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
@@ -23,6 +23,7 @@
import java.time.temporal.ChronoUnit;
import java.util.HashMap;
import java.util.Map;
+import java.util.OptionalLong;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
@@ -72,6 +73,25 @@ public void putRecord_shouldCreateItemInRedis() {
assertThat(Math.round(ttlInRedis / 100.0) * 100).isEqualTo(ttl);
}
+ @Test
+ public void putRecord_shouldCreateItemInRedisWithInProgressExpiration() {
+ Instant now = Instant.now();
+ long ttl = 3600;
+ long expiry = now.plus(ttl, ChronoUnit.SECONDS).getEpochSecond();
+ OptionalLong progressExpiry = OptionalLong.of(now.minus(30, ChronoUnit.SECONDS).toEpochMilli());
+ redisPersistenceStore.putRecord(
+ new DataRecord("key", DataRecord.Status.COMPLETED, expiry, null, null, progressExpiry), now);
+
+ Map entry = jedisPool.hgetAll("idempotency:id:key");
+ long ttlInRedis = jedisPool.ttl("idempotency:id:key");
+
+ assertThat(entry).isNotNull();
+ assertThat(entry.get("status")).isEqualTo("COMPLETED");
+ assertThat(entry.get("expiration")).isEqualTo(String.valueOf(expiry));
+ assertThat(entry.get("in-progress-expiration")).isEqualTo(String.valueOf(progressExpiry.getAsLong()));
+ assertThat(Math.round(ttlInRedis / 100.0) * 100).isEqualTo(ttl);
+ }
+
@Test
public void putRecord_shouldCreateItemInRedis_withExistingJedisClient() {
Instant now = Instant.now();
From 7ff57082aa774295c13dd239f404ab841bc7fba9 Mon Sep 17 00:00:00 2001
From: Eleni Dimitropoulou <12170229+eldimi@users.noreply.github.com>
Date: Fri, 24 Nov 2023 17:43:31 +0200
Subject: [PATCH 15/31] Address sonar findings
---
.../redis/RedisPersistenceStore.java | 102 +++++++-------
.../redis/RedisPersistenceStoreTest.java | 130 +++++++++---------
2 files changed, 115 insertions(+), 117 deletions(-)
diff --git a/powertools-idempotency/powertool-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java b/powertools-idempotency/powertool-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java
index a33e7182c..909953832 100644
--- a/powertools-idempotency/powertool-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java
+++ b/powertools-idempotency/powertool-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java
@@ -95,11 +95,10 @@ public static Builder builder() {
* @return
*/
private static JedisClientConfig getJedisClientConfig() {
- JedisClientConfig config = DefaultJedisClientConfig.builder()
+ return DefaultJedisClientConfig.builder()
.user(System.getenv().get(Constants.REDIS_USER))
.password(System.getenv().get(Constants.REDIS_SECRET))
.build();
- return config;
}
JedisClientConfig config = getJedisClientConfig();
@@ -117,53 +116,53 @@ public DataRecord getRecord(String idempotencyKey) throws IdempotencyItemNotFoun
}
/**
- * Store's the given idempotency record in the redis store. If there
- * is an existing record that has expired - either due to the
- * cache expiry or due to the in_progress_expiry - the record
+ * Store's the given idempotency dataRecord in the redis store. If there
+ * is an existing dataRecord that has expired - either due to the
+ * cache expiry or due to the in_progress_expiry - the dataRecord
* will be overwritten and the idempotent operation can continue.
*
* Note: This method writes only expiry and status information - not
* the results of the operation itself.
*
- * @param record DataRecord instance to store
+ * @param dataRecord DataRecord instance to store
* @param now
* @throws IdempotencyItemAlreadyExistsException
*/
@Override
- public void putRecord(DataRecord record, Instant now) {
+ public void putRecord(DataRecord dataRecord, Instant now) {
String inProgressExpiry = null;
- if (record.getInProgressExpiryTimestamp().isPresent()) {
- inProgressExpiry = String.valueOf(record.getInProgressExpiryTimestamp().getAsLong());
+ if (dataRecord.getInProgressExpiryTimestamp().isPresent()) {
+ inProgressExpiry = String.valueOf(dataRecord.getInProgressExpiryTimestamp().getAsLong());
}
- LOG.debug("Putting record for idempotency key: {}", record.getIdempotencyKey());
+ LOG.debug("Putting dataRecord for idempotency key: {}", dataRecord.getIdempotencyKey());
- Object execRes = putItemOnCondition(record, now, inProgressExpiry);
+ Object execRes = putItemOnCondition(dataRecord, now, inProgressExpiry);
if (execRes == null) {
- String msg = String.format("Failed to put record for already existing idempotency key: %s",
- getKey(record.getIdempotencyKey()));
+ String msg = String.format("Failed to put dataRecord for already existing idempotency key: %s",
+ getKey(dataRecord.getIdempotencyKey()));
LOG.debug(msg);
throw new IdempotencyItemAlreadyExistsException(msg);
} else {
- LOG.debug("Record for idempotency key is set: {}", record.getIdempotencyKey());
- jedisPool.expireAt(getKey(record.getIdempotencyKey()), record.getExpiryTimestamp());
+ LOG.debug("Record for idempotency key is set: {}", dataRecord.getIdempotencyKey());
+ jedisPool.expireAt(getKey(dataRecord.getIdempotencyKey()), dataRecord.getExpiryTimestamp());
}
}
- private Object putItemOnCondition(DataRecord record, Instant now, String inProgressExpiry) {
+ private Object putItemOnCondition(DataRecord dataRecord, Instant now, String inProgressExpiry) {
// if item with key exists
String redisHashExistsExpression = "redis.call('exists', KEYS[1]) == 0";
// if expiry timestamp is exceeded for existing item
String itemExpiredExpression = "redis.call('hget', KEYS[1], KEYS[2]) < ARGV[1]";
- // if item status attribute exists and has value is INPROGRESS
+ // if item status field exists and has value is INPROGRESS
// and the in-progress-expiry timestamp is still valid
String itemIsInProgressExpression = "(redis.call('hexists', KEYS[1], KEYS[4]) ~= 0" +
" and redis.call('hget', KEYS[1], KEYS[4]) < ARGV[2]" +
" and redis.call('hget', KEYS[1], KEYS[3]) == ARGV[3])";
- // insert item and attributes
+ // insert item and fields
String insertItemExpression = "return redis.call('hset', KEYS[1], KEYS[2], ARGV[4], KEYS[3], ARGV[5])";
// only insert in-progress-expiry if it is set
@@ -176,41 +175,40 @@ private Object putItemOnCondition(DataRecord record, Instant now, String inProgr
redisHashExistsExpression, itemExpiredExpression,
itemIsInProgressExpression, insertItemExpression);
- List params = new ArrayList<>();
- params.add(getKey(record.getIdempotencyKey()));
- params.add( this.expiryAttr);
- params.add(this.statusAttr);
- params.add(this.inProgressExpiryAttr);
- params.add(String.valueOf(now.getEpochSecond()));
- params.add(String.valueOf(now.toEpochMilli()));
- params.add(INPROGRESS.toString());
- params.add(String.valueOf(record.getExpiryTimestamp()));
- params.add(record.getStatus().toString());
+ List fields = new ArrayList<>();
+ fields.add(getKey(dataRecord.getIdempotencyKey()));
+ fields.add(this.expiryAttr);
+ fields.add(this.statusAttr);
+ fields.add(this.inProgressExpiryAttr);
+ fields.add(String.valueOf(now.getEpochSecond()));
+ fields.add(String.valueOf(now.toEpochMilli()));
+ fields.add(INPROGRESS.toString());
+ fields.add(String.valueOf(dataRecord.getExpiryTimestamp()));
+ fields.add(dataRecord.getStatus().toString());
if (inProgressExpiry != null) {
- params.add(inProgressExpiry);
+ fields.add(inProgressExpiry);
}
- String []arr = new String[params.size()];
- Object execRes = jedisPool.eval(luaScript, 4, (String[]) params.toArray(arr));
- return execRes;
+ String[] arr = new String[fields.size()];
+ return jedisPool.eval(luaScript, 4, (String[]) fields.toArray(arr));
}
@Override
- public void updateRecord(DataRecord record) {
- LOG.debug("Updating record for idempotency key: {}", record.getIdempotencyKey());
+ public void updateRecord(DataRecord dataRecord) {
+ LOG.debug("Updating dataRecord for idempotency key: {}", dataRecord.getIdempotencyKey());
Map item = new HashMap<>();
- item.put(this.dataAttr, record.getResponseData());
- item.put(this.expiryAttr, String.valueOf(record.getExpiryTimestamp()));
- item.put(this.statusAttr, String.valueOf(record.getStatus().toString()));
+ item.put(this.dataAttr, dataRecord.getResponseData());
+ item.put(this.expiryAttr, String.valueOf(dataRecord.getExpiryTimestamp()));
+ item.put(this.statusAttr, String.valueOf(dataRecord.getStatus().toString()));
if (payloadValidationEnabled) {
- item.put(this.validationAttr, record.getPayloadHash());
+ item.put(this.validationAttr, dataRecord.getPayloadHash());
}
- jedisPool.hset(getKey(record.getIdempotencyKey()), item);
- jedisPool.expireAt(getKey(record.getIdempotencyKey()), record.getExpiryTimestamp());
+ jedisPool.hset(getKey(dataRecord.getIdempotencyKey()), item);
+ jedisPool.expireAt(getKey(dataRecord.getIdempotencyKey()), dataRecord.getExpiryTimestamp());
}
@Override
@@ -252,7 +250,7 @@ private DataRecord itemToRecord(Map item) {
/**
* Use this builder to get an instance of {@link RedisPersistenceStore}.
- * With this builder you can configure the characteristics of the Redis hash attributes.
+ * With this builder you can configure the characteristics of the Redis hash fields.
* You can also set a custom {@link JedisPool}.
*/
public static class Builder {
@@ -293,7 +291,7 @@ public Builder withKeyPrefixName(String keyPrefixName) {
/**
* Redis name for hash key (optional), by default "id"
*
- * @param keyAttr name of the key attribute of the hash
+ * @param keyAttr name of the key field of the hash
* @return the builder instance (to chain operations)
*/
public Builder withKeyAttr(String keyAttr) {
@@ -302,9 +300,9 @@ public Builder withKeyAttr(String keyAttr) {
}
/**
- * Redis attribute name for expiry timestamp (optional), by default "expiration"
+ * Redis field name for expiry timestamp (optional), by default "expiration"
*
- * @param expiryAttr name of the expiry attribute in the hash
+ * @param expiryAttr name of the expiry field in the hash
* @return the builder instance (to chain operations)
*/
public Builder withExpiryAttr(String expiryAttr) {
@@ -313,9 +311,9 @@ public Builder withExpiryAttr(String expiryAttr) {
}
/**
- * Redis attribute name for in progress expiry timestamp (optional), by default "in-progress-expiration"
+ * Redis field name for in progress expiry timestamp (optional), by default "in-progress-expiration"
*
- * @param inProgressExpiryAttr name of the attribute in the hash
+ * @param inProgressExpiryAttr name of the field in the hash
* @return the builder instance (to chain operations)
*/
public Builder withInProgressExpiryAttr(String inProgressExpiryAttr) {
@@ -324,9 +322,9 @@ public Builder withInProgressExpiryAttr(String inProgressExpiryAttr) {
}
/**
- * Redis attribute name for status (optional), by default "status"
+ * Redis field name for status (optional), by default "status"
*
- * @param statusAttr name of the status attribute in the hash
+ * @param statusAttr name of the status field in the hash
* @return the builder instance (to chain operations)
*/
public Builder withStatusAttr(String statusAttr) {
@@ -335,9 +333,9 @@ public Builder withStatusAttr(String statusAttr) {
}
/**
- * Redis attribute name for response data (optional), by default "data"
+ * Redis field name for response data (optional), by default "data"
*
- * @param dataAttr name of the data attribute in the hash
+ * @param dataAttr name of the data field in the hash
* @return the builder instance (to chain operations)
*/
public Builder withDataAttr(String dataAttr) {
@@ -346,9 +344,9 @@ public Builder withDataAttr(String dataAttr) {
}
/**
- * Redis attribute name for validation (optional), by default "validation"
+ * Redis field name for validation (optional), by default "validation"
*
- * @param validationAttr name of the validation attribute in the hash
+ * @param validationAttr name of the validation field in the hash
* @return the builder instance (to chain operations)
*/
public Builder withValidationAttr(String validationAttr) {
diff --git a/powertools-idempotency/powertool-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java b/powertools-idempotency/powertool-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
index b44db768f..d6a438f3a 100644
--- a/powertools-idempotency/powertool-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
+++ b/powertools-idempotency/powertool-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
@@ -58,7 +58,7 @@ public static void stop() {
}
@Test
- public void putRecord_shouldCreateItemInRedis() {
+ void putRecord_shouldCreateItemInRedis() {
Instant now = Instant.now();
long ttl = 3600;
long expiry = now.plus(ttl, ChronoUnit.SECONDS).getEpochSecond();
@@ -74,7 +74,7 @@ public void putRecord_shouldCreateItemInRedis() {
}
@Test
- public void putRecord_shouldCreateItemInRedisWithInProgressExpiration() {
+ void putRecord_shouldCreateItemInRedisWithInProgressExpiration() {
Instant now = Instant.now();
long ttl = 3600;
long expiry = now.plus(ttl, ChronoUnit.SECONDS).getEpochSecond();
@@ -82,32 +82,32 @@ public void putRecord_shouldCreateItemInRedisWithInProgressExpiration() {
redisPersistenceStore.putRecord(
new DataRecord("key", DataRecord.Status.COMPLETED, expiry, null, null, progressExpiry), now);
- Map entry = jedisPool.hgetAll("idempotency:id:key");
+ Map redisItem = jedisPool.hgetAll("idempotency:id:key");
long ttlInRedis = jedisPool.ttl("idempotency:id:key");
- assertThat(entry).isNotNull();
- assertThat(entry.get("status")).isEqualTo("COMPLETED");
- assertThat(entry.get("expiration")).isEqualTo(String.valueOf(expiry));
- assertThat(entry.get("in-progress-expiration")).isEqualTo(String.valueOf(progressExpiry.getAsLong()));
+ assertThat(redisItem).isNotNull();
+ assertThat(redisItem).containsEntry("status", "COMPLETED");
+ assertThat(redisItem).containsEntry("expiration", String.valueOf(expiry));
+ assertThat(redisItem).containsEntry("in-progress-expiration", String.valueOf(progressExpiry.getAsLong()));
assertThat(Math.round(ttlInRedis / 100.0) * 100).isEqualTo(ttl);
}
@Test
- public void putRecord_shouldCreateItemInRedis_withExistingJedisClient() {
+ void putRecord_shouldCreateItemInRedis_withExistingJedisClient() {
Instant now = Instant.now();
long expiry = now.plus(3600, ChronoUnit.SECONDS).getEpochSecond();
RedisPersistenceStore store = new RedisPersistenceStore.Builder().withJedisPooled(jedisPool).build();
store.putRecord(new DataRecord("key", DataRecord.Status.COMPLETED, expiry, null, null), now);
- Map entry = jedisPool.hgetAll("idempotency:id:key");
+ Map redisItem = jedisPool.hgetAll("idempotency:id:key");
- assertThat(entry).isNotNull();
- assertThat(entry.get("status")).isEqualTo("COMPLETED");
- assertThat(entry.get("expiration")).isEqualTo(String.valueOf(expiry));
+ assertThat(redisItem).isNotNull();
+ assertThat(redisItem).containsEntry("status", "COMPLETED");
+ assertThat(redisItem).containsEntry("expiration", String.valueOf(expiry));
}
@Test
- public void putRecord_shouldCreateItemInRedis_IfPreviousExpired() {
+ void putRecord_shouldCreateItemInRedis_IfPreviousExpired() {
Map item = new HashMap<>();
Instant now = Instant.now();
@@ -127,17 +127,17 @@ public void putRecord_shouldCreateItemInRedis_IfPreviousExpired() {
null
), now);
- Map entry = jedisPool.hgetAll("idempotency:id:key");
+ Map redisItem = jedisPool.hgetAll("idempotency:id:key");
long ttlInRedis = jedisPool.ttl("idempotency:id:key");
- assertThat(entry).isNotNull();
- assertThat(entry.get("status")).isEqualTo("INPROGRESS");
- assertThat(entry.get("expiration")).isEqualTo(String.valueOf(expiry2));
+ assertThat(redisItem).isNotNull();
+ assertThat(redisItem).containsEntry("status", "INPROGRESS");
+ assertThat(redisItem).containsEntry("expiration", String.valueOf(expiry2));
assertThat(Math.round(ttlInRedis / 100.0) * 100).isEqualTo(ttl);
}
@Test
- public void putRecord_shouldCreateItemInRedis_IfLambdaWasInProgressAndTimedOut() {
+ void putRecord_shouldCreateItemInRedis_IfLambdaWasInProgressAndTimedOut() {
Map item = new HashMap<>();
Instant now = Instant.now();
@@ -158,15 +158,15 @@ public void putRecord_shouldCreateItemInRedis_IfLambdaWasInProgressAndTimedOut()
null
), now);
- Map entry = jedisPool.hgetAll("idempotency:id:key");
+ Map redisItem = jedisPool.hgetAll("idempotency:id:key");
- assertThat(entry).isNotNull();
- assertThat(entry.get("status")).isEqualTo("INPROGRESS");
- assertThat(entry.get("expiration")).isEqualTo(String.valueOf(expiry2));
+ assertThat(redisItem).isNotNull();
+ assertThat(redisItem).containsEntry("status", "INPROGRESS");
+ assertThat(redisItem).containsEntry("expiration", String.valueOf(expiry2));
}
@Test
- public void putRecord_shouldThrowIdempotencyItemAlreadyExistsException_IfRecordAlreadyExist() {
+ void putRecord_shouldThrowIdempotencyItemAlreadyExistsException_IfRecordAlreadyExist() {
Map item = new HashMap<>();
Instant now = Instant.now();
@@ -178,13 +178,16 @@ public void putRecord_shouldThrowIdempotencyItemAlreadyExistsException_IfRecordA
jedisPool.hset("idempotency:id:key", item);
long expiry2 = now.plus(3600, ChronoUnit.SECONDS).getEpochSecond();
- assertThatThrownBy(() -> redisPersistenceStore.putRecord(
- new DataRecord("key",
- DataRecord.Status.INPROGRESS,
- expiry2,
- null,
- null
- ), now)
+ DataRecord dataRecord = new DataRecord("key",
+ DataRecord.Status.INPROGRESS,
+ expiry2,
+ null,
+ null
+ );
+ assertThatThrownBy(() -> {
+ redisPersistenceStore.putRecord(
+ dataRecord, now);
+ }
).isInstanceOf(IdempotencyItemAlreadyExistsException.class);
Map entry = jedisPool.hgetAll("idempotency:id:key");
@@ -196,7 +199,7 @@ public void putRecord_shouldThrowIdempotencyItemAlreadyExistsException_IfRecordA
}
@Test
- public void putRecord_shouldBlockUpdate_IfRecordAlreadyExistAndProgressNotExpiredAfterLambdaTimedOut() {
+ void putRecord_shouldBlockUpdate_IfRecordAlreadyExistAndProgressNotExpiredAfterLambdaTimedOut() {
Map item = new HashMap<>();
Instant now = Instant.now();
@@ -209,25 +212,26 @@ public void putRecord_shouldBlockUpdate_IfRecordAlreadyExistAndProgressNotExpire
jedisPool.hset("idempotency:id:key", item);
long expiry2 = now.plus(3600, ChronoUnit.SECONDS).getEpochSecond();
+ DataRecord dataRecord = new DataRecord("key",
+ DataRecord.Status.INPROGRESS,
+ expiry2,
+ "Fake Data 2",
+ null
+ );
assertThatThrownBy(() -> redisPersistenceStore.putRecord(
- new DataRecord("key",
- DataRecord.Status.INPROGRESS,
- expiry2,
- "Fake Data 2",
- null
- ), now))
+ dataRecord, now))
.isInstanceOf(IdempotencyItemAlreadyExistsException.class);
- Map entry = jedisPool.hgetAll("idempotency:id:key");
+ Map redisItem = jedisPool.hgetAll("idempotency:id:key");
- assertThat(entry).isNotNull();
- assertThat(entry.get("status")).isEqualTo("INPROGRESS");
- assertThat(entry.get("expiration")).isEqualTo(String.valueOf(expiry));
- assertThat(entry.get("data")).isEqualTo("Fake Data");
+ assertThat(redisItem).isNotNull();
+ assertThat(redisItem).containsEntry("status", "INPROGRESS");
+ assertThat(redisItem).containsEntry("expiration", String.valueOf(expiry));
+ assertThat(redisItem).containsEntry("data", "Fake Data");
}
@Test
- public void getRecord_shouldReturnExistingRecord() throws IdempotencyItemNotFoundException {
+ void getRecord_shouldReturnExistingRecord() throws IdempotencyItemNotFoundException {
Map item = new HashMap<>();
Instant now = Instant.now();
@@ -246,13 +250,13 @@ public void getRecord_shouldReturnExistingRecord() throws IdempotencyItemNotFoun
}
@Test
- public void getRecord_shouldThrowException_whenRecordIsAbsent() {
+ void getRecord_shouldThrowException_whenRecordIsAbsent() {
assertThatThrownBy(() -> redisPersistenceStore.getRecord("key")).isInstanceOf(
IdempotencyItemNotFoundException.class);
}
@Test
- public void updateRecord_shouldUpdateRecord() {
+ void updateRecord_shouldUpdateRecord() {
Map item = new HashMap<>();
Instant now = Instant.now();
long expiry = now.plus(360, ChronoUnit.SECONDS).getEpochSecond();
@@ -268,18 +272,18 @@ public void updateRecord_shouldUpdateRecord() {
DataRecord record = new DataRecord("key", DataRecord.Status.COMPLETED, expiry, "Fake result", "hash");
redisPersistenceStore.updateRecord(record);
- Map itemInDb = jedisPool.hgetAll("idempotency:id:key");
+ Map redisItem = jedisPool.hgetAll("idempotency:id:key");
long ttlInRedis = jedisPool.ttl("idempotency:id:key");
- assertThat(itemInDb.get("status")).isEqualTo("COMPLETED");
- assertThat(itemInDb.get("expiration")).isEqualTo(String.valueOf(expiry));
- assertThat(itemInDb.get("data")).isEqualTo("Fake result");
- assertThat(itemInDb.get("validation")).isEqualTo("hash");
+ assertThat(redisItem).containsEntry("status", "COMPLETED");
+ assertThat(redisItem).containsEntry("expiration", String.valueOf(expiry));
+ assertThat(redisItem).containsEntry("data", "Fake result");
+ assertThat(redisItem).containsEntry("validation", "hash");
assertThat(Math.round(ttlInRedis / 100.0) * 100).isEqualTo(ttl);
}
@Test
- public void deleteRecord_shouldDeleteRecord() {
+ void deleteRecord_shouldDeleteRecord() {
Map item = new HashMap<>();
Instant now = Instant.now();
long expiry = now.plus(360, ChronoUnit.SECONDS).getEpochSecond();
@@ -291,12 +295,12 @@ public void deleteRecord_shouldDeleteRecord() {
Map items = jedisPool.hgetAll("idempotency:id:key");
- assertThat(items.isEmpty()).isTrue();
+ assertThat(items).isEmpty();
}
@Test
- public void endToEndWithCustomAttrNamesAndSortKey() throws IdempotencyItemNotFoundException {
+ void endToEndWithCustomAttrNamesAndSortKey() throws IdempotencyItemNotFoundException {
try {
RedisPersistenceStore persistenceStore = RedisPersistenceStore.builder()
.withKeyPrefixName("items-idempotency")
@@ -319,14 +323,14 @@ public void endToEndWithCustomAttrNamesAndSortKey() throws IdempotencyItemNotFou
// PUT
persistenceStore.putRecord(record, now);
- Map itemInDb = jedisPool.hgetAll("items-idempotency:key:mykey");
+ Map redisItem = jedisPool.hgetAll("items-idempotency:key:mykey");
// GET
DataRecord recordInDb = persistenceStore.getRecord("mykey");
- assertThat(itemInDb).isNotNull();
- assertThat(itemInDb.get("state")).isEqualTo(recordInDb.getStatus().toString());
- assertThat(itemInDb.get("expiry")).isEqualTo(String.valueOf(recordInDb.getExpiryTimestamp()));
+ assertThat(redisItem).isNotNull();
+ assertThat(redisItem).containsEntry("state", recordInDb.getStatus().toString());
+ assertThat(redisItem).containsEntry("expiry", String.valueOf(recordInDb.getExpiryTimestamp()));
// UPDATE
DataRecord updatedRecord = new DataRecord(
@@ -342,26 +346,22 @@ public void endToEndWithCustomAttrNamesAndSortKey() throws IdempotencyItemNotFou
// DELETE
persistenceStore.deleteRecord("mykey");
- assertThat(jedisPool.hgetAll("items-idempotency:key:mykey").size()).isEqualTo(0);
+ assertThat(jedisPool.hgetAll("items-idempotency:key:mykey").size()).isZero();
} finally {
- try {
- jedisPool.del("items-idempotency:key:mykey");
- } catch (Exception e) {
- // OK
- }
+ jedisPool.del("items-idempotency:key:mykey");
}
}
@Test
@SetEnvironmentVariable(key = software.amazon.lambda.powertools.idempotency.Constants.IDEMPOTENCY_DISABLED_ENV, value = "true")
- public void idempotencyDisabled_noClientShouldBeCreated() {
+ void idempotencyDisabled_noClientShouldBeCreated() {
RedisPersistenceStore store = RedisPersistenceStore.builder().build();
assertThatThrownBy(() -> store.getRecord("key")).isInstanceOf(NullPointerException.class);
}
@AfterEach
- public void emptyDB() {
+ void emptyDB() {
jedisPool.del("idempotency:id:key");
}
From d2e4efa53a4ec8356315c50ce82f3b3ee5760e09 Mon Sep 17 00:00:00 2001
From: Eleni Dimitropoulou <12170229+eldimi@users.noreply.github.com>
Date: Tue, 28 Nov 2023 09:52:27 +0200
Subject: [PATCH 16/31] E2E test for idempotency redis implementation
---
.../handlers/idempotency-dynamodb/pom.xml | 72 +++++++++++++++++
.../lambda/powertools/e2e/Function.java | 2 +-
.../amazon/lambda/powertools/e2e/Input.java | 0
.../src/main/resources/log4j2.xml | 0
.../pom.xml | 8 +-
.../lambda/powertools/e2e/Function.java | 56 +++++++++++++
.../amazon/lambda/powertools/e2e/Input.java | 34 ++++++++
.../src/main/resources/log4j2.xml | 16 ++++
.../handlers/largemessage_idempotent/pom.xml | 4 +-
powertools-e2e-tests/handlers/pom.xml | 10 ++-
...E2ET.java => IdempotencyDynamoDBE2ET.java} | 6 +-
.../powertools/IdempotencyRedisE2ET.java | 80 +++++++++++++++++++
.../powertools/testutils/Infrastructure.java | 45 ++++++++++-
powertools-idempotency/pom.xml | 2 +-
.../pom.xml | 2 +-
.../idempotency/redis/Constants.java | 0
.../redis/RedisPersistenceStore.java | 0
.../redis/RedisPersistenceStoreTest.java | 0
18 files changed, 321 insertions(+), 16 deletions(-)
create mode 100644 powertools-e2e-tests/handlers/idempotency-dynamodb/pom.xml
rename powertools-e2e-tests/handlers/{idempotency => idempotency-dynamodb}/src/main/java/software/amazon/lambda/powertools/e2e/Function.java (98%)
rename powertools-e2e-tests/handlers/{idempotency => idempotency-dynamodb}/src/main/java/software/amazon/lambda/powertools/e2e/Input.java (100%)
rename powertools-e2e-tests/handlers/{idempotency => idempotency-dynamodb}/src/main/resources/log4j2.xml (100%)
rename powertools-e2e-tests/handlers/{idempotency => idempotency-redis}/pom.xml (92%)
create mode 100644 powertools-e2e-tests/handlers/idempotency-redis/src/main/java/software/amazon/lambda/powertools/e2e/Function.java
create mode 100644 powertools-e2e-tests/handlers/idempotency-redis/src/main/java/software/amazon/lambda/powertools/e2e/Input.java
create mode 100644 powertools-e2e-tests/handlers/idempotency-redis/src/main/resources/log4j2.xml
rename powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/{IdempotencyE2ET.java => IdempotencyDynamoDBE2ET.java} (96%)
create mode 100644 powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/IdempotencyRedisE2ET.java
rename powertools-idempotency/{powertool-idempotency-redis => powertools-idempotency-redis}/pom.xml (98%)
rename powertools-idempotency/{powertool-idempotency-redis => powertools-idempotency-redis}/src/main/java/software/amazon/lambda/powertools/idempotency/redis/Constants.java (100%)
rename powertools-idempotency/{powertool-idempotency-redis => powertools-idempotency-redis}/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java (100%)
rename powertools-idempotency/{powertool-idempotency-redis => powertools-idempotency-redis}/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java (100%)
diff --git a/powertools-e2e-tests/handlers/idempotency-dynamodb/pom.xml b/powertools-e2e-tests/handlers/idempotency-dynamodb/pom.xml
new file mode 100644
index 000000000..b9d9fdb03
--- /dev/null
+++ b/powertools-e2e-tests/handlers/idempotency-dynamodb/pom.xml
@@ -0,0 +1,72 @@
+
+ 4.0.0
+
+
+ software.amazon.lambda
+ e2e-test-handlers-parent
+ 1.0.0
+
+
+ e2e-test-handler-idempotency-dynamodb
+ jar
+ A Lambda function using Powertools for AWS Lambda (Java) idempotency
+
+
+
+ software.amazon.lambda
+ powertools-idempotency-dynamodb
+
+
+ software.amazon.lambda
+ powertools-logging
+
+
+ org.apache.logging.log4j
+ log4j-slf4j2-impl
+
+
+ com.amazonaws
+ aws-lambda-java-events
+
+
+ org.aspectj
+ aspectjrt
+
+
+
+
+
+
+ dev.aspectj
+ aspectj-maven-plugin
+
+ ${maven.compiler.source}
+ ${maven.compiler.target}
+ ${maven.compiler.target}
+
+
+ software.amazon.lambda
+ powertools-idempotency-core
+
+
+ software.amazon.lambda
+ powertools-logging
+
+
+
+
+
+
+ compile
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+
+
+
+
diff --git a/powertools-e2e-tests/handlers/idempotency/src/main/java/software/amazon/lambda/powertools/e2e/Function.java b/powertools-e2e-tests/handlers/idempotency-dynamodb/src/main/java/software/amazon/lambda/powertools/e2e/Function.java
similarity index 98%
rename from powertools-e2e-tests/handlers/idempotency/src/main/java/software/amazon/lambda/powertools/e2e/Function.java
rename to powertools-e2e-tests/handlers/idempotency-dynamodb/src/main/java/software/amazon/lambda/powertools/e2e/Function.java
index e4c2f2b9a..16109778d 100644
--- a/powertools-e2e-tests/handlers/idempotency/src/main/java/software/amazon/lambda/powertools/e2e/Function.java
+++ b/powertools-e2e-tests/handlers/idempotency-dynamodb/src/main/java/software/amazon/lambda/powertools/e2e/Function.java
@@ -27,7 +27,7 @@
import software.amazon.lambda.powertools.idempotency.Idempotency;
import software.amazon.lambda.powertools.idempotency.IdempotencyConfig;
import software.amazon.lambda.powertools.idempotency.Idempotent;
-import software.amazon.lambda.powertools.idempotency.persistence.DynamoDBPersistenceStore;
+import software.amazon.lambda.powertools.idempotency.persistence.dynamodb.DynamoDBPersistenceStore;
import software.amazon.lambda.powertools.logging.Logging;
diff --git a/powertools-e2e-tests/handlers/idempotency/src/main/java/software/amazon/lambda/powertools/e2e/Input.java b/powertools-e2e-tests/handlers/idempotency-dynamodb/src/main/java/software/amazon/lambda/powertools/e2e/Input.java
similarity index 100%
rename from powertools-e2e-tests/handlers/idempotency/src/main/java/software/amazon/lambda/powertools/e2e/Input.java
rename to powertools-e2e-tests/handlers/idempotency-dynamodb/src/main/java/software/amazon/lambda/powertools/e2e/Input.java
diff --git a/powertools-e2e-tests/handlers/idempotency/src/main/resources/log4j2.xml b/powertools-e2e-tests/handlers/idempotency-dynamodb/src/main/resources/log4j2.xml
similarity index 100%
rename from powertools-e2e-tests/handlers/idempotency/src/main/resources/log4j2.xml
rename to powertools-e2e-tests/handlers/idempotency-dynamodb/src/main/resources/log4j2.xml
diff --git a/powertools-e2e-tests/handlers/idempotency/pom.xml b/powertools-e2e-tests/handlers/idempotency-redis/pom.xml
similarity index 92%
rename from powertools-e2e-tests/handlers/idempotency/pom.xml
rename to powertools-e2e-tests/handlers/idempotency-redis/pom.xml
index 22b6a1c53..26ced1c39 100644
--- a/powertools-e2e-tests/handlers/idempotency/pom.xml
+++ b/powertools-e2e-tests/handlers/idempotency-redis/pom.xml
@@ -8,14 +8,13 @@
1.0.0
- e2e-test-handler-idempotency
+ e2e-test-handler-idempotency-redis
jar
A Lambda function using Powertools for AWS Lambda (Java) idempotency
-
software.amazon.lambda
- powertools-idempotency
+ powertools-idempotency-redis
software.amazon.lambda
@@ -34,7 +33,6 @@
aspectjrt
-
@@ -47,7 +45,7 @@
software.amazon.lambda
- powertools-idempotency
+ powertools-idempotency-core
software.amazon.lambda
diff --git a/powertools-e2e-tests/handlers/idempotency-redis/src/main/java/software/amazon/lambda/powertools/e2e/Function.java b/powertools-e2e-tests/handlers/idempotency-redis/src/main/java/software/amazon/lambda/powertools/e2e/Function.java
new file mode 100644
index 000000000..5ca0f316e
--- /dev/null
+++ b/powertools-e2e-tests/handlers/idempotency-redis/src/main/java/software/amazon/lambda/powertools/e2e/Function.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2023 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package software.amazon.lambda.powertools.e2e;
+
+import com.amazonaws.services.lambda.runtime.Context;
+import com.amazonaws.services.lambda.runtime.RequestHandler;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.format.DateTimeFormatter;
+import java.time.temporal.ChronoUnit;
+import java.util.TimeZone;
+import redis.clients.jedis.JedisPooled;
+import software.amazon.lambda.powertools.idempotency.Idempotency;
+import software.amazon.lambda.powertools.idempotency.IdempotencyConfig;
+import software.amazon.lambda.powertools.idempotency.Idempotent;
+import software.amazon.lambda.powertools.idempotency.redis.RedisPersistenceStore;
+import software.amazon.lambda.powertools.logging.Logging;
+
+
+public class Function implements RequestHandler {
+
+ public Function() {
+ this(new JedisPooled(System.getenv().get("REDIS_HOST"), Integer.parseInt(System.getenv().get("REDIS_PORT")), System.getenv().get("REDIS_USER"), System.getenv().get("REDIS_SECRET")));
+ }
+
+ public Function(JedisPooled client) {
+ Idempotency.config().withConfig(
+ IdempotencyConfig.builder()
+ .withExpiration(Duration.of(10, ChronoUnit.SECONDS))
+ .build())
+ .withPersistenceStore(
+ RedisPersistenceStore.builder()
+ .withJedisPooled(client)
+ .build()
+ ).configure();
+ }
+
+ @Logging(logEvent = true)
+ @Idempotent
+ public String handleRequest(Input input, Context context) {
+ DateTimeFormatter dtf = DateTimeFormatter.ISO_DATE_TIME.withZone(TimeZone.getTimeZone("UTC").toZoneId());
+ return dtf.format(Instant.now());
+ }
+}
\ No newline at end of file
diff --git a/powertools-e2e-tests/handlers/idempotency-redis/src/main/java/software/amazon/lambda/powertools/e2e/Input.java b/powertools-e2e-tests/handlers/idempotency-redis/src/main/java/software/amazon/lambda/powertools/e2e/Input.java
new file mode 100644
index 000000000..e0e4c27c9
--- /dev/null
+++ b/powertools-e2e-tests/handlers/idempotency-redis/src/main/java/software/amazon/lambda/powertools/e2e/Input.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2023 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package software.amazon.lambda.powertools.e2e;
+
+public class Input {
+ private String message;
+
+ public Input(String message) {
+ this.message = message;
+ }
+
+ public Input() {
+ }
+
+ public String getMessage() {
+ return message;
+ }
+
+ public void setMessage(String message) {
+ this.message = message;
+ }
+}
diff --git a/powertools-e2e-tests/handlers/idempotency-redis/src/main/resources/log4j2.xml b/powertools-e2e-tests/handlers/idempotency-redis/src/main/resources/log4j2.xml
new file mode 100644
index 000000000..8925f70b9
--- /dev/null
+++ b/powertools-e2e-tests/handlers/idempotency-redis/src/main/resources/log4j2.xml
@@ -0,0 +1,16 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/powertools-e2e-tests/handlers/largemessage_idempotent/pom.xml b/powertools-e2e-tests/handlers/largemessage_idempotent/pom.xml
index 1fe9092ef..8fb2d8b75 100644
--- a/powertools-e2e-tests/handlers/largemessage_idempotent/pom.xml
+++ b/powertools-e2e-tests/handlers/largemessage_idempotent/pom.xml
@@ -15,7 +15,7 @@
software.amazon.lambda
- powertools-idempotency
+ powertools-idempotency-dynamodb
software.amazon.lambda
@@ -51,7 +51,7 @@
software.amazon.lambda
- powertools-idempotency
+ powertools-idempotency-dynamodb
software.amazon.lambda
diff --git a/powertools-e2e-tests/handlers/pom.xml b/powertools-e2e-tests/handlers/pom.xml
index fbe2e6d8b..888231965 100644
--- a/powertools-e2e-tests/handlers/pom.xml
+++ b/powertools-e2e-tests/handlers/pom.xml
@@ -30,7 +30,8 @@
logging
tracing
metrics
- idempotency
+ idempotency-dynamodb
+ idempotency-redis
parameters
@@ -61,7 +62,12 @@
software.amazon.lambda
- powertools-idempotency
+ powertools-idempotency-dynamodb
+ ${lambda.powertools.version}
+
+
+ software.amazon.lambda
+ powertools-idempotency-redis
${lambda.powertools.version}
diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/IdempotencyE2ET.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/IdempotencyDynamoDBE2ET.java
similarity index 96%
rename from powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/IdempotencyE2ET.java
rename to powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/IdempotencyDynamoDBE2ET.java
index 242d1a2db..1c9ac30b6 100644
--- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/IdempotencyE2ET.java
+++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/IdempotencyDynamoDBE2ET.java
@@ -29,7 +29,7 @@
import software.amazon.lambda.powertools.testutils.Infrastructure;
import software.amazon.lambda.powertools.testutils.lambda.InvocationResult;
-public class IdempotencyE2ET {
+public class IdempotencyDynamoDBE2ET {
private static Infrastructure infrastructure;
private static String functionName;
@@ -38,7 +38,7 @@ public class IdempotencyE2ET {
public static void setup() {
String random = UUID.randomUUID().toString().substring(0, 6);
infrastructure = Infrastructure.builder()
- .testName(IdempotencyE2ET.class.getSimpleName())
+ .testName(IdempotencyDynamoDBE2ET.class.getSimpleName())
.pathToFunction("idempotency")
.idempotencyTable("idempo" + random)
.build();
@@ -75,4 +75,4 @@ public void test_ttlNotExpired_sameResult_ttlExpired_differentResult() throws In
Assertions.assertThat(result2.getResult()).isEqualTo(result1.getResult());
Assertions.assertThat(result3.getResult()).isNotEqualTo(result2.getResult());
}
-}
+}
\ No newline at end of file
diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/IdempotencyRedisE2ET.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/IdempotencyRedisE2ET.java
new file mode 100644
index 000000000..e01189207
--- /dev/null
+++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/IdempotencyRedisE2ET.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2023 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package software.amazon.lambda.powertools;
+
+import static software.amazon.lambda.powertools.testutils.Infrastructure.FUNCTION_NAME_OUTPUT;
+import static software.amazon.lambda.powertools.testutils.lambda.LambdaInvoker.invokeFunction;
+
+import java.time.Year;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+import software.amazon.lambda.powertools.testutils.Infrastructure;
+import software.amazon.lambda.powertools.testutils.lambda.InvocationResult;
+
+public class IdempotencyRedisE2ET {
+ private static Infrastructure infrastructure;
+ private static String functionName;
+
+ @BeforeAll
+ @Timeout(value = 5, unit = TimeUnit.MINUTES)
+ public static void setup() {
+ infrastructure = Infrastructure.builder()
+ .testName(IdempotencyRedisE2ET.class.getSimpleName())
+ .redisHost(System.getenv("REDIS_HOST"))
+ .redisPort(System.getenv("REDIS_PORT"))
+ .redisUser(System.getenv("REDIS_USER"))
+ .redisSecret(System.getenv("REDIS_SECRET"))
+ .pathToFunction("idempotency-redis")
+ .build();
+ Map outputs = infrastructure.deploy();
+ functionName = outputs.get(FUNCTION_NAME_OUTPUT);
+ }
+
+ @AfterAll
+ public static void tearDown() {
+ if (infrastructure != null) {
+ infrastructure.destroy();
+ }
+ }
+
+ @Test
+ public void test_ttlNotExpired_sameResult_ttlExpired_differentResult() throws InterruptedException {
+ // GIVEN
+ String event = "{\"message\":\"TTL 10sec\"}";
+
+ // WHEN
+ // First invocation
+ InvocationResult result1 = invokeFunction(functionName, event);
+
+ // Second invocation (should get same result)
+ InvocationResult result2 = invokeFunction(functionName, event);
+
+ Thread.sleep(12000);
+
+ // Third invocation (should get different result)
+ InvocationResult result3 = invokeFunction(functionName, event);
+
+ // THEN
+ Assertions.assertThat(result1.getResult()).contains(Year.now().toString());
+ Assertions.assertThat(result2.getResult()).isEqualTo(result1.getResult());
+ Assertions.assertThat(result3.getResult()).isNotEqualTo(result2.getResult());
+ }
+}
diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java
index b1fab2883..ab0094935 100644
--- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java
+++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java
@@ -45,7 +45,11 @@
import software.amazon.awscdk.services.appconfig.CfnDeploymentStrategy;
import software.amazon.awscdk.services.appconfig.CfnEnvironment;
import software.amazon.awscdk.services.appconfig.CfnHostedConfigurationVersion;
-import software.amazon.awscdk.services.dynamodb.*;
+import software.amazon.awscdk.services.dynamodb.Attribute;
+import software.amazon.awscdk.services.dynamodb.AttributeType;
+import software.amazon.awscdk.services.dynamodb.BillingMode;
+import software.amazon.awscdk.services.dynamodb.StreamViewType;
+import software.amazon.awscdk.services.dynamodb.Table;
import software.amazon.awscdk.services.iam.PolicyStatement;
import software.amazon.awscdk.services.kinesis.Stream;
import software.amazon.awscdk.services.kinesis.StreamMode;
@@ -114,6 +118,10 @@ public class Infrastructure {
private final String queue;
private final String kinesisStream;
private final String largeMessagesBucket;
+ private final String redisHost;
+ private final String redisPort;
+ private final String redisUser;
+ private final String redisSecret;
private String ddbStreamsTableName;
private String functionName;
private Object cfnTemplate;
@@ -127,6 +135,10 @@ private Infrastructure(Builder builder) {
this.timeout = builder.timeoutInSeconds;
this.pathToFunction = builder.pathToFunction;
this.idempotencyTable = builder.idemPotencyTable;
+ this.redisHost = builder.redisHost;
+ this.redisPort = builder.redisPort;
+ this.redisUser = builder.redisUser;
+ this.redisSecret = builder.redisSecret;
this.appConfig = builder.appConfig;
this.queue = builder.queue;
this.kinesisStream = builder.kinesisStream;
@@ -273,6 +285,13 @@ private Stack createStackWithLambda() {
table.grantReadWriteData(function);
}
+ if (!StringUtils.isEmpty(redisHost)) {
+ function.addEnvironment("REDIS_HOST", redisHost);
+ function.addEnvironment("REDIS_PORT", redisPort);
+ function.addEnvironment("REDIS_USER", redisUser);
+ function.addEnvironment("REDIS_SECRET", redisSecret);
+ }
+
if (!StringUtils.isEmpty(queue)) {
Queue sqsQueue = Queue.Builder
.create(stack, "SQSQueue")
@@ -504,6 +523,10 @@ public static class Builder {
private String queue;
private String kinesisStream;
private String ddbStreamsTableName;
+ private String redisHost;
+ private String redisPort;
+ private String redisUser;
+ private String redisSecret;
private Builder() {
getJavaRuntime();
@@ -559,6 +582,26 @@ public Builder idempotencyTable(String tableName) {
return this;
}
+ public Builder redisHost(String redisHost) {
+ this.redisHost = redisHost;
+ return this;
+ }
+
+ public Builder redisPort(String redisPort) {
+ this.redisPort = redisPort;
+ return this;
+ }
+
+ public Builder redisUser(String redisUser) {
+ this.redisUser = redisUser;
+ return this;
+ }
+
+ public Builder redisSecret(String redisSecret) {
+ this.redisSecret = redisSecret;
+ return this;
+ }
+
public Builder appConfig(AppConfig app) {
this.appConfig = app;
return this;
diff --git a/powertools-idempotency/pom.xml b/powertools-idempotency/pom.xml
index 0ffc47dd1..dd98576c4 100644
--- a/powertools-idempotency/pom.xml
+++ b/powertools-idempotency/pom.xml
@@ -36,7 +36,7 @@
powertools-idempotency-core
powertools-idempotency-dynamodb
- powertool-idempotency-redis
+ powertools-idempotency-redis
diff --git a/powertools-idempotency/powertool-idempotency-redis/pom.xml b/powertools-idempotency/powertools-idempotency-redis/pom.xml
similarity index 98%
rename from powertools-idempotency/powertool-idempotency-redis/pom.xml
rename to powertools-idempotency/powertools-idempotency-redis/pom.xml
index 2b9487aa6..1ead9407f 100644
--- a/powertools-idempotency/powertool-idempotency-redis/pom.xml
+++ b/powertools-idempotency/powertools-idempotency-redis/pom.xml
@@ -23,7 +23,7 @@
2.0.0-SNAPSHOT
- powertool-idempotency-redis
+ powertools-idempotency-redis
Powertools for AWS Lambda (Java) library Idempotency - Redis
Redis implementation for the idempotency module
diff --git a/powertools-idempotency/powertool-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/Constants.java b/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/Constants.java
similarity index 100%
rename from powertools-idempotency/powertool-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/Constants.java
rename to powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/Constants.java
diff --git a/powertools-idempotency/powertool-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java b/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java
similarity index 100%
rename from powertools-idempotency/powertool-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java
rename to powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java
diff --git a/powertools-idempotency/powertool-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java b/powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
similarity index 100%
rename from powertools-idempotency/powertool-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
rename to powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
From d00b5e2afc3550089973b844677e6ea8ae7c5c89 Mon Sep 17 00:00:00 2001
From: Eleni Dimitropoulou <12170229+eldimi@users.noreply.github.com>
Date: Tue, 28 Nov 2023 10:12:52 +0200
Subject: [PATCH 17/31] Adding instructions to bootstrap cdk in e2e README file
---
powertools-e2e-tests/README.md | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/powertools-e2e-tests/README.md b/powertools-e2e-tests/README.md
index 61799e6f7..f41e16cd8 100644
--- a/powertools-e2e-tests/README.md
+++ b/powertools-e2e-tests/README.md
@@ -6,8 +6,16 @@ __Prerequisites__:
([credentials](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/credentials.html)).
- [Java 11+](https://docs.aws.amazon.com/corretto/latest/corretto-11-ug/downloads-list.html)
- [Docker](https://docs.docker.com/engine/install/)
+- [CDK](https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html#getting_started_install)
-To execute the E2E tests, use the following command: `export JAVA_VERSION=11 && mvn clean verify -Pe2e`
+### Execute test
+Before executing the tests in a new AWS account, [bootstrap CDK](https://docs.aws.amazon.com/cdk/v2/guide/bootstrapping.htmls) using the following command:
+
+`cdk bootstrap aws:///`
+
+To execute the E2E tests, use the following command:
+
+`export JAVA_VERSION=11 && mvn clean verify -Pe2e`
### Under the hood
This module leverages the following components:
From b3e3d7df6956e88b796c665564f8d57897d78cc5 Mon Sep 17 00:00:00 2001
From: Eleni Dimitropoulou <12170229+eldimi@users.noreply.github.com>
Date: Tue, 28 Nov 2023 17:12:44 +0200
Subject: [PATCH 18/31] Add documentation for redis idempotency
---
docs/utilities/idempotency.md | 90 ++++++++++++++++++++++++++++++++++-
1 file changed, 88 insertions(+), 2 deletions(-)
diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md
index f4defbdfd..3bf1a938e 100644
--- a/docs/utilities/idempotency.md
+++ b/docs/utilities/idempotency.md
@@ -159,10 +159,12 @@ Depending on your version of Java (either Java 1.8 or 11+), the configuration sl
```
### Required resources
-
Before getting started, you need to create a persistent storage layer where the idempotency utility can store its state - your Lambda functions will need read and write access to it.
+As of now, [Amazon DynamoDB](https://aws.amazon.com/dynamodb/) and [Redis](https://redis.io/) are the supported persistnce layers.
+
+#### Using Amazon DynamoDB as persistent storage layer
-As of now, Amazon DynamoDB is the only supported persistent storage layer, so you'll need to create a table first.
+If you are using Amazon DynamoDB you'll need to create a table.
**Default table configuration**
@@ -215,6 +217,34 @@ Resources:
see 1WCU and 1RCU. Review the [DynamoDB pricing documentation](https://aws.amazon.com/dynamodb/pricing/) to
estimate the cost.
+#### Using Redis as persistent storage layer
+
+If you are using Redis you'll need to provide the Redis host, port, user and password as AWS Lambda environment variables.
+In the following example, you can see a SAM template for deploying an AWS Lambda function by specifying the required environment variables.
+If you don't provide a custom [Redis client](# Customizing Redis client) you can omit the environment variables declaration.
+
+!!! warning "Warning: Avoid including a plain text secret in your template"
+This can infer security implications
+
+!!! warning "Warning: Large responses with Redis persistence layer"
+When using this utility with Redis your function's responses must be [smaller than 512MB].
+Persisting larger items cannot might cause exceptions.
+
+```yaml hl_lines="9-12" title="AWS Serverless Application Model (SAM) example"
+Resources:
+ IdempotencyFunction:
+ Type: AWS::Serverless::Function
+ Properties:
+ CodeUri: Function
+ Handler: helloworld.App::handleRequest
+ Environment:
+ Variables:
+ REDIS_HOST: %redis-host%
+ REDIS_PORT: %redis-port%
+ REDIS_USER: %redis-user%
+ REDIS_SECRET: %redis-secret%
+```
+
### Idempotent annotation
You can quickly start by initializing the `DynamoDBPersistenceStore` and using it with the `@Idempotent` annotation on your Lambda handler.
@@ -635,6 +665,29 @@ When using DynamoDB as a persistence layer, you can alter the attribute names by
| **SortKeyAttr** | | | Sort key of the table (if table is configured with a sort key). |
| **StaticPkValue** | | `idempotency#{LAMBDA_FUNCTION_NAME}` | Static value to use as the partition key. Only used when **SortKeyAttr** is set. |
+#### RedisPersistenceStore
+
+The redis persistence store has as a prerequisite to install a Redis datastore(https://redis.io/docs/about/) in Standalone mode.
+
+We are using [Redis hashes](https://redis.io/docs/data-types/hashes/) to store the idempotency fields and values.
+There are some predefined fields that you can see listed in the following table. The predefined fields have some default values.
+
+
+You can alter the field names by passing these parameters when initializing the persistence layer:
+
+| Parameter | Required | Default | Description |
+|--------------------|----------|--------------------------------------|--------------------------------------------------------------------------------------------------------|
+| **KeyPrefixName** | Y | `idempotency` | The redis hash key prefix |
+| **KeyAttr** | Y | `id` | The redis hash key field name |
+| **ExpiryAttr** | | `expiration` | Unix timestamp of when record expires |
+| **StatusAttr** | | `status` | Stores status of the Lambda execution during and after invocation |
+| **DataAttr** | | `data` | Stores results of successfully idempotent methods |
+| **ValidationAttr** | | `validation` | Hashed representation of the parts of the event used for validation |
+
+
+!!! Tip "Tip: You can share the same prefix and key for all functions"
+You can reuse the same prefix and key to store idempotency state. We add your function name in addition to the idempotency key as a hash key.
+
## Advanced
### Customizing the default behavior
@@ -884,6 +937,39 @@ When creating the `DynamoDBPersistenceStore`, you can set a custom [`DynamoDbCli
.build();
```
+### Customizing Redis client
+
+The `RedisPersistenceStore` uses the JedisPooled java client to connect to the Redis Server.
+When creating the `RedisPersistenceStore`, you can set a custom [`JedisPooled`](https://www.javadoc.io/doc/redis.clients/jedis/latest/redis/clients/jedis/JedisPooled.html) client:
+
+=== "Custom JedisPooled with connection timeout"
+
+ ```java hl_lines="2-6 11"
+ public App() {
+ JedisPooled jedisPooled = new JedisPooled(new HostAndPort("host",6789), DefaultJedisClientConfig.builder()
+ .user("user")
+ .password("secret")
+ .connectionTimeoutMillis(3000)
+ .build())
+
+ Idempotency.config().withPersistenceStore(
+ RedisPersistenceStore.builder()
+ .withKeyPrefixName("items-idempotency")
+ .withJedisPooled(jedisPooled)
+ .build()
+ ).configure();
+ }
+ ```
+
+!!! info "Default configuration is the following:"
+
+ ```java
+ DefaultJedisClientConfig.builder()
+ .user(System.getenv(Constants.REDIS_USER))
+ .password(System.getenv(Constants.REDIS_SECRET))
+ .build();
+ ```
+
### Using a DynamoDB table with a composite primary key
When using a composite primary key table (hash+range key), use `SortKeyAttr` parameter when initializing your persistence store.
From 0501b8fc9420e43252a9ca155e3906863c3d7af6 Mon Sep 17 00:00:00 2001
From: Eleni Dimitropoulou <12170229+eldimi@users.noreply.github.com>
Date: Fri, 8 Dec 2023 10:33:21 +0200
Subject: [PATCH 19/31] Add support for Redis Cluster, improve documentation
and e2e tests
---
docs/utilities/idempotency.md | 46 ++++-
.../lambda/powertools/e2e/Function.java | 7 -
.../powertools/IdempotencyRedisE2ET.java | 8 +-
.../powertools/testutils/Infrastructure.java | 179 +++++++++++------
.../powertools-idempotency-redis/pom.xml | 7 +-
.../idempotency/redis/Constants.java | 1 +
.../redis/RedisPersistenceStore.java | 146 ++++++++------
.../redis/RedisPersistenceStoreTest.java | 185 +++++++++++-------
8 files changed, 372 insertions(+), 207 deletions(-)
diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md
index 3bf1a938e..8d6077d68 100644
--- a/docs/utilities/idempotency.md
+++ b/docs/utilities/idempotency.md
@@ -219,18 +219,26 @@ Resources:
#### Using Redis as persistent storage layer
+##### Redis resources
+
+You need an existing Redis service before setting up Redis as persistent storage layer provider. You can also use Redis compatible services like [Amazon ElastiCache for Redis](https://aws.amazon.com/elasticache/redis/) as persistent storage layer provider.
+!!! tip "Tip:No existing Redis service?"
+If you don't have an existing Redis service, we recommend using DynamoDB as persistent storage layer provider.
+
If you are using Redis you'll need to provide the Redis host, port, user and password as AWS Lambda environment variables.
+If you want to connect to a Redis cluster instead of a Standalone server, you need to enable Redis cluster mode by setting an AWS Lambda
+environment variable `REDIS_CLUSTER_MODE` to `true`
In the following example, you can see a SAM template for deploying an AWS Lambda function by specifying the required environment variables.
-If you don't provide a custom [Redis client](# Customizing Redis client) you can omit the environment variables declaration.
+If you provide a [custom Redis client](#Customizing Redis client) you can omit the environment variables declaration.
-!!! warning "Warning: Avoid including a plain text secret in your template"
+!!! warning "Warning: Avoid including a plain text secret directly in your template"
This can infer security implications
!!! warning "Warning: Large responses with Redis persistence layer"
-When using this utility with Redis your function's responses must be [smaller than 512MB].
+When using this utility with Redis your function's responses must be smaller than 512MB.
Persisting larger items cannot might cause exceptions.
-```yaml hl_lines="9-12" title="AWS Serverless Application Model (SAM) example"
+```yaml hl_lines="9-13" title="AWS Serverless Application Model (SAM) example"
Resources:
IdempotencyFunction:
Type: AWS::Serverless::Function
@@ -243,7 +251,35 @@ Resources:
REDIS_PORT: %redis-port%
REDIS_USER: %redis-user%
REDIS_SECRET: %redis-secret%
+ REDIS_CLUSTER_MODE: "true"
+```
+##### VPC Access
+Your AWS Lambda Function must be able to reach the Redis endpoint before using it for idempotency persistent storage layer. In most cases you will need to [configure VPC access](https://docs.aws.amazon.com/lambda/latest/dg/configuration-vpc.html) for your AWS Lambda Function. Using a public accessible Redis is not recommended.
+
+!!! tip "Amazon ElastiCache for Redis as persistent storage layer provider"
+If you intend to use Amazon ElastiCache for Redis for idempotency persistent storage layer, you can also reference [This AWS Tutorial](https://docs.aws.amazon.com/lambda/latest/dg/services-elasticache-tutorial.html).
+
+!!! warning "Amazon ElastiCache Serverless not supported"
+[Amazon ElastiCache Serverless](https://aws.amazon.com/elasticache/features/#Serverless) is not supported for now.
+
+!!! warning "Check network connectivity to Redis server"
+Make sure that your AWS Lambda function can connect to your Redis server.
+
+```yaml hl_lines="9-12" title="AWS Serverless Application Model (SAM) example"
+Resources:
+ IdempotencyFunction:
+ Type: AWS::Serverless::Function
+ Properties:
+ CodeUri: Function
+ Handler: helloworld.App::handleRequest
+ VpcConfig: # (1)!
+ SecurityGroupIds:
+ - sg-{your_sg_id}
+ SubnetIds:
+ - subnet-{your_subnet_id_1}
+ - subnet-{your_subnet_id_2}
```
+1. Replace the Security Group ID and Subnet ID to match your Redis' VPC setting.
### Idempotent annotation
@@ -667,7 +703,7 @@ When using DynamoDB as a persistence layer, you can alter the attribute names by
#### RedisPersistenceStore
-The redis persistence store has as a prerequisite to install a Redis datastore(https://redis.io/docs/about/) in Standalone mode.
+The redis persistence store has as a prerequisite to install a Redis datastore(https://redis.io/docs/about/) in either Standalone or Cluster mode.
We are using [Redis hashes](https://redis.io/docs/data-types/hashes/) to store the idempotency fields and values.
There are some predefined fields that you can see listed in the following table. The predefined fields have some default values.
diff --git a/powertools-e2e-tests/handlers/idempotency-redis/src/main/java/software/amazon/lambda/powertools/e2e/Function.java b/powertools-e2e-tests/handlers/idempotency-redis/src/main/java/software/amazon/lambda/powertools/e2e/Function.java
index 5ca0f316e..994f14d0c 100644
--- a/powertools-e2e-tests/handlers/idempotency-redis/src/main/java/software/amazon/lambda/powertools/e2e/Function.java
+++ b/powertools-e2e-tests/handlers/idempotency-redis/src/main/java/software/amazon/lambda/powertools/e2e/Function.java
@@ -21,7 +21,6 @@
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.util.TimeZone;
-import redis.clients.jedis.JedisPooled;
import software.amazon.lambda.powertools.idempotency.Idempotency;
import software.amazon.lambda.powertools.idempotency.IdempotencyConfig;
import software.amazon.lambda.powertools.idempotency.Idempotent;
@@ -30,19 +29,13 @@
public class Function implements RequestHandler {
-
public Function() {
- this(new JedisPooled(System.getenv().get("REDIS_HOST"), Integer.parseInt(System.getenv().get("REDIS_PORT")), System.getenv().get("REDIS_USER"), System.getenv().get("REDIS_SECRET")));
- }
-
- public Function(JedisPooled client) {
Idempotency.config().withConfig(
IdempotencyConfig.builder()
.withExpiration(Duration.of(10, ChronoUnit.SECONDS))
.build())
.withPersistenceStore(
RedisPersistenceStore.builder()
- .withJedisPooled(client)
.build()
).configure();
}
diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/IdempotencyRedisE2ET.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/IdempotencyRedisE2ET.java
index e01189207..412389741 100644
--- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/IdempotencyRedisE2ET.java
+++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/IdempotencyRedisE2ET.java
@@ -19,7 +19,6 @@
import java.time.Year;
import java.util.Map;
-import java.util.UUID;
import java.util.concurrent.TimeUnit;
import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.AfterAll;
@@ -34,14 +33,11 @@ public class IdempotencyRedisE2ET {
private static String functionName;
@BeforeAll
- @Timeout(value = 5, unit = TimeUnit.MINUTES)
+ @Timeout(value = 15, unit = TimeUnit.MINUTES)
public static void setup() {
infrastructure = Infrastructure.builder()
.testName(IdempotencyRedisE2ET.class.getSimpleName())
- .redisHost(System.getenv("REDIS_HOST"))
- .redisPort(System.getenv("REDIS_PORT"))
- .redisUser(System.getenv("REDIS_USER"))
- .redisSecret(System.getenv("REDIS_SECRET"))
+ .redisDeployment(true)
.pathToFunction("idempotency-redis")
.build();
Map outputs = infrastructure.deploy();
diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java
index ab0094935..1bfd32605 100644
--- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java
+++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java
@@ -27,6 +27,7 @@
import java.util.Objects;
import java.util.UUID;
import java.util.stream.Collectors;
+import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
@@ -36,8 +37,10 @@
import software.amazon.awscdk.CfnOutput;
import software.amazon.awscdk.DockerVolume;
import software.amazon.awscdk.Duration;
+import software.amazon.awscdk.Environment;
import software.amazon.awscdk.RemovalPolicy;
import software.amazon.awscdk.Stack;
+import software.amazon.awscdk.StackProps;
import software.amazon.awscdk.cxapi.CloudAssembly;
import software.amazon.awscdk.services.appconfig.CfnApplication;
import software.amazon.awscdk.services.appconfig.CfnConfigurationProfile;
@@ -50,6 +53,14 @@
import software.amazon.awscdk.services.dynamodb.BillingMode;
import software.amazon.awscdk.services.dynamodb.StreamViewType;
import software.amazon.awscdk.services.dynamodb.Table;
+import software.amazon.awscdk.services.ec2.IVpc;
+import software.amazon.awscdk.services.ec2.Peer;
+import software.amazon.awscdk.services.ec2.Port;
+import software.amazon.awscdk.services.ec2.SecurityGroup;
+import software.amazon.awscdk.services.ec2.SubnetSelection;
+import software.amazon.awscdk.services.ec2.Vpc;
+import software.amazon.awscdk.services.elasticache.CfnCacheCluster;
+import software.amazon.awscdk.services.elasticache.CfnSubnetGroup;
import software.amazon.awscdk.services.iam.PolicyStatement;
import software.amazon.awscdk.services.kinesis.Stream;
import software.amazon.awscdk.services.kinesis.StreamMode;
@@ -118,14 +129,15 @@ public class Infrastructure {
private final String queue;
private final String kinesisStream;
private final String largeMessagesBucket;
- private final String redisHost;
- private final String redisPort;
- private final String redisUser;
- private final String redisSecret;
+ private IVpc vpc;
private String ddbStreamsTableName;
private String functionName;
private Object cfnTemplate;
private String cfnAssetDirectory;
+ private SubnetSelection subnetSelection;
+ private CfnSubnetGroup cfnSubnetGroup;
+ private SecurityGroup securityGroup;
+ private boolean isRedisDeployment = false;
private Infrastructure(Builder builder) {
this.stackName = builder.stackName;
@@ -135,28 +147,52 @@ private Infrastructure(Builder builder) {
this.timeout = builder.timeoutInSeconds;
this.pathToFunction = builder.pathToFunction;
this.idempotencyTable = builder.idemPotencyTable;
- this.redisHost = builder.redisHost;
- this.redisPort = builder.redisPort;
- this.redisUser = builder.redisUser;
- this.redisSecret = builder.redisSecret;
+ this.isRedisDeployment = builder.redisDeployment;
this.appConfig = builder.appConfig;
this.queue = builder.queue;
this.kinesisStream = builder.kinesisStream;
this.largeMessagesBucket = builder.largeMessagesBucket;
this.ddbStreamsTableName = builder.ddbStreamsTableName;
+ this.region = Region.of(System.getProperty("AWS_DEFAULT_REGION", "eu-west-1"));
this.app = new App();
- this.stack = createStackWithLambda();
- this.synthesize();
+ this.stack = createStack();
this.httpClient = UrlConnectionHttpClient.builder().build();
- this.region = Region.of(System.getProperty("AWS_DEFAULT_REGION", "eu-west-1"));
+
this.account = StsClient.builder()
.httpClient(httpClient)
.region(region)
.build().getCallerIdentity().account();
+ if (isRedisDeployment) {
+ this.vpc = Vpc.Builder.create(this.stack, "MyVPC-" + stackName)
+ .availabilityZones(List.of(region.toString() + "a", region.toString() + "b"))
+ .build();
+
+ List subnets = vpc.getPublicSubnets().stream().map(subnet ->
+ subnet.getSubnetId()).collect(Collectors.toList());
+
+ securityGroup = SecurityGroup.Builder.create(stack, "ElastiCache-SG-" + stackName)
+ .vpc(vpc)
+ .allowAllOutbound(true)
+ .description("ElastiCache SecurityGroup")
+ .build();
+
+ cfnSubnetGroup = CfnSubnetGroup.Builder.create(stack, "Redis Subnet-" + stackName)
+ .description("A subnet for the ElastiCache cluster")
+ .subnetIds(subnets).cacheSubnetGroupName("redis-SG-" + stackName).build();
+
+ subnetSelection = SubnetSelection.builder().subnets(vpc.getPublicSubnets()).build();
+ }
+
+
+ createStackWithLambda();
+
+ this.synthesize();
+
+
s3 = S3Client.builder()
.httpClient(httpClient)
.region(region)
@@ -213,9 +249,9 @@ public void destroy() {
*
* @return the CDK stack
*/
- private Stack createStackWithLambda() {
+ private void createStackWithLambda() {
boolean createTableForAsyncTests = false;
- Stack stack = new Stack(app, stackName);
+
List packagingInstruction = Arrays.asList(
"/bin/sh",
"-c",
@@ -242,14 +278,23 @@ private Stack createStackWithLambda() {
.outputType(BundlingOutput.ARCHIVED);
functionName = stackName + "-function";
- CfnOutput.Builder.create(stack, FUNCTION_NAME_OUTPUT)
+ CfnOutput.Builder.create(this.stack, FUNCTION_NAME_OUTPUT)
.value(functionName)
.build();
LOG.debug("Building Lambda function with command " +
packagingInstruction.stream().collect(Collectors.joining(" ", "[", "]")));
- Function function = Function.Builder
- .create(stack, functionName)
+
+ final SecurityGroup lambdaSecurityGroup = SecurityGroup.Builder.create(this.stack, "Lambda-SG")
+ .vpc(vpc)
+ .allowAllOutbound(true)
+ .description("Lambda SecurityGroup")
+ .build();
+ securityGroup.addIngressRule(Peer.securityGroupId(lambdaSecurityGroup.getSecurityGroupId()), Port.tcp(6379),
+ "Allow ElastiCache Server");
+
+ Function.Builder functionBuilder = Function.Builder
+ .create(this.stack, functionName)
.code(Code.fromAsset("handlers/", AssetOptions.builder()
.bundling(builderOptions
.command(packagingInstruction)
@@ -259,13 +304,22 @@ private Stack createStackWithLambda() {
.handler("software.amazon.lambda.powertools.e2e.Function::handleRequest")
.memorySize(1024)
.timeout(Duration.seconds(timeout))
+ .allowPublicSubnet(true)
.runtime(runtime.getCdkRuntime())
.environment(envVar)
- .tracing(tracing ? Tracing.ACTIVE : Tracing.DISABLED)
- .build();
+ .tracing(tracing ? Tracing.ACTIVE : Tracing.DISABLED);
+
+ if (isRedisDeployment) {
+ functionBuilder.vpc(vpc)
+ .vpcSubnets(subnetSelection)
+ .securityGroups(List.of(lambdaSecurityGroup));
+ }
+
+ Function function = functionBuilder.build();
+
LogGroup.Builder
- .create(stack, functionName + "-logs")
+ .create(this.stack, functionName + "-logs")
.logGroupName("/aws/lambda/" + functionName)
.retention(RetentionDays.ONE_DAY)
.removalPolicy(RemovalPolicy.DESTROY)
@@ -273,7 +327,7 @@ private Stack createStackWithLambda() {
if (!StringUtils.isEmpty(idempotencyTable)) {
Table table = Table.Builder
- .create(stack, "IdempotencyTable")
+ .create(this.stack, "IdempotencyTable")
.billingMode(BillingMode.PAY_PER_REQUEST)
.removalPolicy(RemovalPolicy.DESTROY)
.partitionKey(Attribute.builder().name("id").type(AttributeType.STRING).build())
@@ -285,16 +339,24 @@ private Stack createStackWithLambda() {
table.grantReadWriteData(function);
}
- if (!StringUtils.isEmpty(redisHost)) {
- function.addEnvironment("REDIS_HOST", redisHost);
- function.addEnvironment("REDIS_PORT", redisPort);
- function.addEnvironment("REDIS_USER", redisUser);
- function.addEnvironment("REDIS_SECRET", redisSecret);
+ if (isRedisDeployment) {
+ CfnCacheCluster redisServer = CfnCacheCluster.Builder.create(this.stack, "ElastiCacheCluster-" + stackName)
+ .clusterName("redis-cluster-" + stackName)
+ .engine("redis")
+ .cacheNodeType("cache.t2.micro")
+ .cacheSubnetGroupName(cfnSubnetGroup.getCacheSubnetGroupName())
+ .vpcSecurityGroupIds(List.of(securityGroup.getSecurityGroupId()))
+ .numCacheNodes(1)
+ .build();
+ redisServer.addDependency(cfnSubnetGroup);
+ function.addEnvironment("REDIS_HOST", redisServer.getAttrRedisEndpointAddress());
+ function.addEnvironment("REDIS_PORT", redisServer.getAttrRedisEndpointPort());
+ function.addEnvironment("REDIS_USER", "default");
}
if (!StringUtils.isEmpty(queue)) {
Queue sqsQueue = Queue.Builder
- .create(stack, "SQSQueue")
+ .create(this.stack, "SQSQueue")
.queueName(queue)
.visibilityTimeout(Duration.seconds(timeout * 6))
.retentionPeriod(Duration.seconds(timeout * 6))
@@ -312,14 +374,14 @@ private Stack createStackWithLambda() {
.build();
function.addEventSource(sqsEventSource);
CfnOutput.Builder
- .create(stack, "QueueURL")
+ .create(this.stack, "QueueURL")
.value(sqsQueue.getQueueUrl())
.build();
createTableForAsyncTests = true;
}
if (!StringUtils.isEmpty(kinesisStream)) {
Stream stream = Stream.Builder
- .create(stack, "KinesisStream")
+ .create(this.stack, "KinesisStream")
.streamMode(StreamMode.ON_DEMAND)
.streamName(kinesisStream)
.build();
@@ -335,13 +397,13 @@ private Stack createStackWithLambda() {
.build();
function.addEventSource(kinesisEventSource);
CfnOutput.Builder
- .create(stack, "KinesisStreamName")
+ .create(this.stack, "KinesisStreamName")
.value(stream.getStreamName())
.build();
}
if (!StringUtils.isEmpty(ddbStreamsTableName)) {
- Table ddbStreamsTable = Table.Builder.create(stack, "DDBStreamsTable")
+ Table ddbStreamsTable = Table.Builder.create(this.stack, "DDBStreamsTable")
.tableName(ddbStreamsTableName)
.stream(StreamViewType.KEYS_ONLY)
.removalPolicy(RemovalPolicy.DESTROY)
@@ -355,12 +417,12 @@ private Stack createStackWithLambda() {
.reportBatchItemFailures(true)
.build();
function.addEventSource(ddbEventSource);
- CfnOutput.Builder.create(stack, "DdbStreamsTestTable").value(ddbStreamsTable.getTableName()).build();
+ CfnOutput.Builder.create(this.stack, "DdbStreamsTestTable").value(ddbStreamsTable.getTableName()).build();
}
if (!StringUtils.isEmpty(largeMessagesBucket)) {
Bucket offloadBucket = Bucket.Builder
- .create(stack, "LargeMessagesOffloadBucket")
+ .create(this.stack, "LargeMessagesOffloadBucket")
.removalPolicy(RemovalPolicy.RETAIN) // autodelete does not work without cdk deploy
.bucketName(largeMessagesBucket)
.build();
@@ -371,19 +433,19 @@ private Stack createStackWithLambda() {
if (appConfig != null) {
CfnApplication app = CfnApplication.Builder
- .create(stack, "AppConfigApp")
+ .create(this.stack, "AppConfigApp")
.name(appConfig.getApplication())
.build();
CfnEnvironment environment = CfnEnvironment.Builder
- .create(stack, "AppConfigEnvironment")
+ .create(this.stack, "AppConfigEnvironment")
.applicationId(app.getRef())
.name(appConfig.getEnvironment())
.build();
// Create a fast deployment strategy, so we don't have to wait ages
CfnDeploymentStrategy fastDeployment = CfnDeploymentStrategy.Builder
- .create(stack, "AppConfigDeployment")
+ .create(this.stack, "AppConfigDeployment")
.name("FastDeploymentStrategy")
.deploymentDurationInMinutes(0)
.finalBakeTimeInMinutes(0)
@@ -402,14 +464,14 @@ private Stack createStackWithLambda() {
CfnDeployment previousDeployment = null;
for (Map.Entry entry : appConfig.getConfigurationValues().entrySet()) {
CfnConfigurationProfile configProfile = CfnConfigurationProfile.Builder
- .create(stack, "AppConfigProfileFor" + entry.getKey())
+ .create(this.stack, "AppConfigProfileFor" + entry.getKey())
.applicationId(app.getRef())
.locationUri("hosted")
.name(entry.getKey())
.build();
CfnHostedConfigurationVersion configVersion = CfnHostedConfigurationVersion.Builder
- .create(stack, "AppConfigHostedVersionFor" + entry.getKey())
+ .create(this.stack, "AppConfigHostedVersionFor" + entry.getKey())
.applicationId(app.getRef())
.contentType("text/plain")
.configurationProfileId(configProfile.getRef())
@@ -417,7 +479,7 @@ private Stack createStackWithLambda() {
.build();
CfnDeployment deployment = CfnDeployment.Builder
- .create(stack, "AppConfigDepoymentFor" + entry.getKey())
+ .create(this.stack, "AppConfigDepoymentFor" + entry.getKey())
.applicationId(app.getRef())
.environmentId(environment.getRef())
.deploymentStrategyId(fastDeployment.getRef())
@@ -434,7 +496,7 @@ private Stack createStackWithLambda() {
}
if (createTableForAsyncTests) {
Table table = Table.Builder
- .create(stack, "TableForAsyncTests")
+ .create(this.stack, "TableForAsyncTests")
.billingMode(BillingMode.PAY_PER_REQUEST)
.removalPolicy(RemovalPolicy.DESTROY)
.partitionKey(Attribute.builder().name("functionName").type(AttributeType.STRING).build())
@@ -443,9 +505,17 @@ private Stack createStackWithLambda() {
table.grantReadWriteData(function);
function.addEnvironment("TABLE_FOR_ASYNC_TESTS", table.getTableName());
- CfnOutput.Builder.create(stack, "TableNameForAsyncTests").value(table.getTableName()).build();
+ CfnOutput.Builder.create(this.stack, "TableNameForAsyncTests").value(table.getTableName()).build();
}
+ }
+ @NotNull
+ private Stack createStack() {
+ Stack stack = new Stack(app, stackName, StackProps.builder()
+ .env(Environment.builder()
+ .account(account)
+ .region(region.id())
+ .build()).build());
return stack;
}
@@ -456,6 +526,7 @@ private void synthesize() {
CloudAssembly synth = app.synth();
cfnTemplate = synth.getStackByName(stack.getStackName()).getTemplate();
cfnAssetDirectory = synth.getDirectory();
+
}
/**
@@ -496,9 +567,9 @@ private Map findAssets() {
String assetPath = file.get("source").get("path").asText();
String assetPackaging = file.get("source").get("packaging").asText();
String bucketName =
- file.get("destinations").get("current_account-current_region").get("bucketName").asText();
+ file.get("destinations").get("current_account-" + region.id()).get("bucketName").asText();
String objectKey =
- file.get("destinations").get("current_account-current_region").get("objectKey").asText();
+ file.get("destinations").get("current_account-" + region.id()).get("objectKey").asText();
Asset asset = new Asset(assetPath, assetPackaging, bucketName.replace("${AWS::AccountId}", account)
.replace("${AWS::Region}", region.toString()));
assets.put(objectKey, asset);
@@ -523,10 +594,7 @@ public static class Builder {
private String queue;
private String kinesisStream;
private String ddbStreamsTableName;
- private String redisHost;
- private String redisPort;
- private String redisUser;
- private String redisSecret;
+ private boolean redisDeployment = false;
private Builder() {
getJavaRuntime();
@@ -582,23 +650,8 @@ public Builder idempotencyTable(String tableName) {
return this;
}
- public Builder redisHost(String redisHost) {
- this.redisHost = redisHost;
- return this;
- }
-
- public Builder redisPort(String redisPort) {
- this.redisPort = redisPort;
- return this;
- }
-
- public Builder redisUser(String redisUser) {
- this.redisUser = redisUser;
- return this;
- }
-
- public Builder redisSecret(String redisSecret) {
- this.redisSecret = redisSecret;
+ public Builder redisDeployment(boolean isRedisDeployment) {
+ this.redisDeployment = isRedisDeployment;
return this;
}
diff --git a/powertools-idempotency/powertools-idempotency-redis/pom.xml b/powertools-idempotency/powertools-idempotency-redis/pom.xml
index 1ead9407f..69b89dd3b 100644
--- a/powertools-idempotency/powertools-idempotency-redis/pom.xml
+++ b/powertools-idempotency/powertools-idempotency-redis/pom.xml
@@ -38,7 +38,7 @@
redis.clients
jedis
- 4.3.1
+ 5.1.0
org.signal
@@ -46,6 +46,11 @@
0.8.3
test
+
+ com.github.fppt
+ jedis-mock
+ 1.0.11
+
diff --git a/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/Constants.java b/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/Constants.java
index ea8bd8695..55b37f999 100644
--- a/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/Constants.java
+++ b/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/Constants.java
@@ -19,4 +19,5 @@ public class Constants {
public static final String REDIS_PORT = "REDIS_PORT";
public static final String REDIS_USER = "REDIS_USER";
public static final String REDIS_SECRET = "REDIS_SECRET";
+ public static final String REDIS_CLUSTER_MODE = "REDIS_CLUSTER_MODE";
}
diff --git a/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java b/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java
index 909953832..efd139d82 100644
--- a/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java
+++ b/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java
@@ -20,13 +20,16 @@
import java.util.List;
import java.util.Map;
import java.util.OptionalLong;
+import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.DefaultJedisClientConfig;
import redis.clients.jedis.HostAndPort;
import redis.clients.jedis.JedisClientConfig;
+import redis.clients.jedis.JedisCluster;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPooled;
+import redis.clients.jedis.UnifiedJedis;
import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyItemAlreadyExistsException;
import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyItemNotFoundException;
import software.amazon.lambda.powertools.idempotency.persistence.BasePersistenceStore;
@@ -34,7 +37,7 @@
import software.amazon.lambda.powertools.idempotency.persistence.PersistenceStore;
/**
- * Redis version of the {@link PersistenceStore}. Will store idempotency data in Redis.
+ * Redis version of the {@link PersistenceStore}. Stores idempotency data in Redis standalone or cluster mode.
* Use the {@link Builder} to create a new instance.
*/
public class RedisPersistenceStore extends BasePersistenceStore implements PersistenceStore {
@@ -47,7 +50,7 @@ public class RedisPersistenceStore extends BasePersistenceStore implements Persi
private final String statusAttr;
private final String dataAttr;
private final String validationAttr;
- private final JedisPooled jedisPool;
+ private final UnifiedJedis jedisClient;
/**
* Private: use the {@link Builder} to instantiate a new {@link RedisPersistenceStore}
@@ -59,7 +62,7 @@ private RedisPersistenceStore(String keyPrefixName,
String statusAttr,
String dataAttr,
String validationAttr,
- JedisPooled jedisPool) {
+ UnifiedJedis jedisClient) {
this.keyPrefixName = keyPrefixName;
this.keyAttr = keyAttr;
this.expiryAttr = expiryAttr;
@@ -68,51 +71,63 @@ private RedisPersistenceStore(String keyPrefixName,
this.dataAttr = dataAttr;
this.validationAttr = validationAttr;
- if (jedisPool != null) {
- this.jedisPool = jedisPool;
+ if (jedisClient != null) {
+ this.jedisClient = jedisClient;
} else {
- String idempotencyDisabledEnv = System.getenv().get(software.amazon.lambda.powertools.idempotency.Constants.IDEMPOTENCY_DISABLED_ENV);
+ String idempotencyDisabledEnv =
+ System.getenv(software.amazon.lambda.powertools.idempotency.Constants.IDEMPOTENCY_DISABLED_ENV);
if (idempotencyDisabledEnv == null || "false".equalsIgnoreCase(idempotencyDisabledEnv)) {
- HostAndPort address = new HostAndPort(System.getenv().get(Constants.REDIS_HOST),
- Integer.parseInt(System.getenv().get(Constants.REDIS_PORT)));
- JedisClientConfig config = getJedisClientConfig();
- this.jedisPool = new JedisPooled(address, config);
+ this.jedisClient = getJedisClient(System.getenv(Constants.REDIS_HOST), Integer.parseInt(System.getenv(Constants.REDIS_PORT)));
} else {
// we do not want to create a Jedis connection pool if idempotency is disabled
// null is ok as idempotency won't be called
- this.jedisPool = null;
+ this.jedisClient = null;
}
}
}
- public static Builder builder() {
- return new Builder();
- }
-
/**
* Set redis user and secret to connect to the redis server
*
* @return
*/
private static JedisClientConfig getJedisClientConfig() {
+ String redisSecret = "";
+ String redisSecretEnv = System.getenv(Constants.REDIS_SECRET);
+ if (redisSecretEnv != null) {
+ redisSecret = redisSecretEnv;
+ }
return DefaultJedisClientConfig.builder()
- .user(System.getenv().get(Constants.REDIS_USER))
- .password(System.getenv().get(Constants.REDIS_SECRET))
+ .user(System.getenv(Constants.REDIS_USER))
+ .password(System.getenv(redisSecret))
.build();
}
- JedisClientConfig config = getJedisClientConfig();
+ public static Builder builder() {
+ return new Builder();
+ }
+ UnifiedJedis getJedisClient(String redisHost, Integer redisPort) {
+ HostAndPort address = new HostAndPort(redisHost, redisPort);
+ JedisClientConfig config = getJedisClientConfig();
+ String isClusterMode = System.getenv(Constants.REDIS_CLUSTER_MODE);
+ if (isClusterMode != null && "true".equalsIgnoreCase(isClusterMode)) {
+ return new JedisCluster(address, getJedisClientConfig(), 5, new GenericObjectPoolConfig<>());
+ } else {
+ return new JedisPooled(address, config);
+ }
+ }
@Override
public DataRecord getRecord(String idempotencyKey) throws IdempotencyItemNotFoundException {
- Map item = jedisPool.hgetAll(getKey(idempotencyKey));
+ String hashKey = getKey(idempotencyKey);
+ Map item = jedisClient.hgetAll(hashKey);
if (item.isEmpty()) {
throw new IdempotencyItemNotFoundException(idempotencyKey);
}
- item.put(this.keyAttr, idempotencyKey);
- return itemToRecord(item);
+ item.put(hashKey, idempotencyKey);
+ return itemToRecord(item, idempotencyKey);
}
/**
@@ -136,18 +151,18 @@ public void putRecord(DataRecord dataRecord, Instant now) {
inProgressExpiry = String.valueOf(dataRecord.getInProgressExpiryTimestamp().getAsLong());
}
- LOG.debug("Putting dataRecord for idempotency key: {}", dataRecord.getIdempotencyKey());
+ LOG.info("Putting dataRecord for idempotency key: {}", dataRecord.getIdempotencyKey());
Object execRes = putItemOnCondition(dataRecord, now, inProgressExpiry);
if (execRes == null) {
String msg = String.format("Failed to put dataRecord for already existing idempotency key: %s",
getKey(dataRecord.getIdempotencyKey()));
- LOG.debug(msg);
+ LOG.info(msg);
throw new IdempotencyItemAlreadyExistsException(msg);
} else {
- LOG.debug("Record for idempotency key is set: {}", dataRecord.getIdempotencyKey());
- jedisPool.expireAt(getKey(dataRecord.getIdempotencyKey()), dataRecord.getExpiryTimestamp());
+ LOG.info("Record for idempotency key is set: {}", dataRecord.getIdempotencyKey());
+ jedisClient.expireAt(getKey(dataRecord.getIdempotencyKey()), dataRecord.getExpiryTimestamp());
}
}
@@ -176,10 +191,11 @@ private Object putItemOnCondition(DataRecord dataRecord, Instant now, String inP
itemIsInProgressExpression, insertItemExpression);
List fields = new ArrayList<>();
- fields.add(getKey(dataRecord.getIdempotencyKey()));
- fields.add(this.expiryAttr);
- fields.add(this.statusAttr);
- fields.add(this.inProgressExpiryAttr);
+ String hashKey = getKey(dataRecord.getIdempotencyKey());
+ fields.add(hashKey);
+ fields.add(prependField(hashKey, this.expiryAttr));
+ fields.add(prependField(hashKey, this.statusAttr));
+ fields.add(prependField(hashKey, this.inProgressExpiryAttr));
fields.add(String.valueOf(now.getEpochSecond()));
fields.add(String.valueOf(now.toEpochMilli()));
fields.add(INPROGRESS.toString());
@@ -191,41 +207,60 @@ private Object putItemOnCondition(DataRecord dataRecord, Instant now, String inP
}
String[] arr = new String[fields.size()];
- return jedisPool.eval(luaScript, 4, (String[]) fields.toArray(arr));
+ return jedisClient.eval(luaScript, 4, fields.toArray(arr));
}
@Override
public void updateRecord(DataRecord dataRecord) {
LOG.debug("Updating dataRecord for idempotency key: {}", dataRecord.getIdempotencyKey());
+ String hashKey = getKey(dataRecord.getIdempotencyKey());
Map item = new HashMap<>();
- item.put(this.dataAttr, dataRecord.getResponseData());
- item.put(this.expiryAttr, String.valueOf(dataRecord.getExpiryTimestamp()));
- item.put(this.statusAttr, String.valueOf(dataRecord.getStatus().toString()));
+ item.put(prependField(hashKey, this.dataAttr), dataRecord.getResponseData());
+ item.put(prependField(hashKey, this.expiryAttr), String.valueOf(dataRecord.getExpiryTimestamp()));
+ item.put(prependField(hashKey, this.statusAttr), String.valueOf(dataRecord.getStatus().toString()));
if (payloadValidationEnabled) {
- item.put(this.validationAttr, dataRecord.getPayloadHash());
+ item.put(prependField(hashKey, this.validationAttr), dataRecord.getPayloadHash());
}
- jedisPool.hset(getKey(dataRecord.getIdempotencyKey()), item);
- jedisPool.expireAt(getKey(dataRecord.getIdempotencyKey()), dataRecord.getExpiryTimestamp());
+ jedisClient.hset(hashKey, item);
+ jedisClient.expireAt(hashKey, dataRecord.getExpiryTimestamp());
}
+
@Override
public void deleteRecord(String idempotencyKey) {
LOG.debug("Deleting record for idempotency key: {}", idempotencyKey);
- jedisPool.del(getKey(idempotencyKey));
+ jedisClient.del(getKey(idempotencyKey));
}
/**
* Get the key to use for requests
* Sets a keyPrefixName for hash name and a keyAttr for hash key
+ * The key will be used in multi-key operations, therefore we need to
+ * include it into curly braces to instruct the redis cluster which part
+ * of the key will be used for hash and should be stored and looked-up in the same slot.
*
* @param idempotencyKey
* @return
+ * @see Redis Key distribution model
*/
private String getKey(String idempotencyKey) {
- return this.keyPrefixName + ":" + this.keyAttr + ":" + idempotencyKey;
+ return "{" + this.keyPrefixName + ":" + this.keyAttr + ":" + idempotencyKey + "}";
+ }
+
+ /**
+ * Prepend each field key with the unique prefix that will be used for calculating the hash slot
+ * it will be stored in case of cluster mode Redis deployement
+ *
+ * @param hashKey
+ * @param field
+ * @return
+ * @see Redis Key distribution model
+ */
+ private String prependField(String hashKey, String field) {
+ return hashKey + ":" + field;
}
/**
@@ -234,24 +269,22 @@ private String getKey(String idempotencyKey) {
* @param item Item from redis response
* @return DataRecord instance
*/
- private DataRecord itemToRecord(Map item) {
- // data and validation payload may be null
- String data = item.get(this.dataAttr);
- String validation = item.get(this.validationAttr);
- return new DataRecord(item.get(keyAttr),
- DataRecord.Status.valueOf(item.get(this.statusAttr)),
- Long.parseLong(item.get(this.expiryAttr)),
- data,
- validation,
- item.get(this.inProgressExpiryAttr) != null ?
- OptionalLong.of(Long.parseLong(item.get(this.inProgressExpiryAttr))) :
+ private DataRecord itemToRecord(Map item, String idempotencyKey) {
+ String hashKey = getKey(idempotencyKey);
+ return new DataRecord(item.get(getKey(idempotencyKey)),
+ DataRecord.Status.valueOf(item.get(prependField(hashKey, this.statusAttr))),
+ Long.parseLong(item.get(prependField(hashKey, this.expiryAttr))),
+ item.get(prependField(hashKey, this.dataAttr)),
+ item.get(prependField(hashKey, this.validationAttr)),
+ item.get(prependField(hashKey, this.inProgressExpiryAttr)) != null ?
+ OptionalLong.of(Long.parseLong(item.get(prependField(hashKey, this.inProgressExpiryAttr)))) :
OptionalLong.empty());
}
/**
* Use this builder to get an instance of {@link RedisPersistenceStore}.
* With this builder you can configure the characteristics of the Redis hash fields.
- * You can also set a custom {@link JedisPool}.
+ * You can also set a custom {@link UnifiedJedis} client.
*/
public static class Builder {
private String keyPrefixName = "idempotency";
@@ -261,7 +294,7 @@ public static class Builder {
private String statusAttr = "status";
private String dataAttr = "data";
private String validationAttr = "validation";
- private JedisPooled jedisPool;
+ private UnifiedJedis jedisPool;
/**
* Initialize and return a new instance of {@link RedisPersistenceStore}.
@@ -355,13 +388,16 @@ public Builder withValidationAttr(String validationAttr) {
}
/**
- * Custom {@link JedisPool} used to query DynamoDB (optional).
+ * Custom {@link UnifiedJedis} used to query Redis (optional).
+ * This will be cast to either {@link JedisPool} or {@link JedisCluster}
+ * depending on the mode of the Redis deployment and instructed by
+ * the value of {@link Constants#REDIS_CLUSTER_MODE} environment variable.
*
- * @param jedisPool the {@link JedisPool} instance to use
+ * @param jedisClient the {@link UnifiedJedis} instance to use
* @return the builder instance (to chain operations)
*/
- public Builder withJedisPooled(JedisPooled jedisPool) {
- this.jedisPool = jedisPool;
+ public Builder withJedisClient(UnifiedJedis jedisClient) {
+ this.jedisPool = jedisClient;
return this;
}
}
diff --git a/powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java b/powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
index d6a438f3a..b2f6dd2ee 100644
--- a/powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
+++ b/powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
@@ -19,6 +19,8 @@
import static software.amazon.lambda.powertools.idempotency.redis.Constants.REDIS_HOST;
import static software.amazon.lambda.powertools.idempotency.redis.Constants.REDIS_PORT;
+import com.github.fppt.jedismock.server.ServiceOptions;
+import java.io.IOException;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.HashMap;
@@ -29,6 +31,7 @@
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junitpioneer.jupiter.SetEnvironmentVariable;
+import redis.clients.jedis.JedisCluster;
import redis.clients.jedis.JedisPooled;
import redis.embedded.RedisServer;
import software.amazon.lambda.powertools.idempotency.IdempotencyConfig;
@@ -64,15 +67,54 @@ void putRecord_shouldCreateItemInRedis() {
long expiry = now.plus(ttl, ChronoUnit.SECONDS).getEpochSecond();
redisPersistenceStore.putRecord(new DataRecord("key", DataRecord.Status.COMPLETED, expiry, null, null), now);
- Map entry = jedisPool.hgetAll("idempotency:id:key");
- long ttlInRedis = jedisPool.ttl("idempotency:id:key");
+ Map entry = jedisPool.hgetAll("{idempotency:id:key}");
+ long ttlInRedis = jedisPool.ttl("{idempotency:id:key}");
assertThat(entry).isNotNull();
- assertThat(entry.get("status")).isEqualTo("COMPLETED");
- assertThat(entry.get("expiration")).isEqualTo(String.valueOf(expiry));
+ assertThat(entry.get("{idempotency:id:key}:status")).isEqualTo("COMPLETED");
+ assertThat(entry.get("{idempotency:id:key}:expiration")).isEqualTo(String.valueOf(expiry));
assertThat(Math.round(ttlInRedis / 100.0) * 100).isEqualTo(ttl);
}
+ @Test
+ void putRecord_shouldCreateItemInRedisClusterMode() throws IOException {
+ com.github.fppt.jedismock.RedisServer redisCluster = com.github.fppt.jedismock.RedisServer
+ .newRedisServer()
+ .setOptions(ServiceOptions.defaultOptions().withClusterModeEnabled())
+ .start();
+ Instant now = Instant.now();
+ long ttl = 3600;
+ long expiry = now.plus(ttl, ChronoUnit.SECONDS).getEpochSecond();
+ JedisPooled jp = new JedisPooled(redisCluster.getHost(), redisCluster.getBindPort());
+ RedisPersistenceStore store = new RedisPersistenceStore.Builder().withJedisClient(jp).build();
+
+ store.putRecord(new DataRecord("key", DataRecord.Status.COMPLETED, expiry, null, null), now);
+
+ Map entry = jp.hgetAll("{idempotency:id:key}");
+ long ttlInRedis = jp.ttl("{idempotency:id:key}");
+
+ assertThat(entry).isNotNull();
+ assertThat(entry.get("{idempotency:id:key}:status")).isEqualTo("COMPLETED");
+ assertThat(entry.get("{idempotency:id:key}:expiration")).isEqualTo(String.valueOf(expiry));
+ assertThat(Math.round(ttlInRedis / 100.0) * 100).isEqualTo(ttl);
+ }
+
+ @SetEnvironmentVariable(key = Constants.REDIS_CLUSTER_MODE, value = "true")
+ @Test
+ void putRecord_JedisClientInstanceOfJedisCluster() throws IOException {
+ com.github.fppt.jedismock.RedisServer redisCluster = com.github.fppt.jedismock.RedisServer
+ .newRedisServer()
+ .setOptions(ServiceOptions.defaultOptions().withClusterModeEnabled())
+ .start();
+ assertThat(redisPersistenceStore.getJedisClient(redisCluster.getHost(), redisCluster.getBindPort()) instanceof JedisCluster).isTrue();
+ redisCluster.stop();
+ }
+
+ @SetEnvironmentVariable(key = Constants.REDIS_CLUSTER_MODE, value = "false")
+ @Test
+ void putRecord_JedisClientInstanceOfJedisPooled() {
+ assertThat(redisPersistenceStore.getJedisClient(System.getenv(REDIS_HOST), Integer.parseInt(System.getenv(REDIS_PORT))) instanceof JedisCluster).isFalse();
+ }
@Test
void putRecord_shouldCreateItemInRedisWithInProgressExpiration() {
Instant now = Instant.now();
@@ -82,13 +124,14 @@ void putRecord_shouldCreateItemInRedisWithInProgressExpiration() {
redisPersistenceStore.putRecord(
new DataRecord("key", DataRecord.Status.COMPLETED, expiry, null, null, progressExpiry), now);
- Map redisItem = jedisPool.hgetAll("idempotency:id:key");
- long ttlInRedis = jedisPool.ttl("idempotency:id:key");
+ Map redisItem = jedisPool.hgetAll("{idempotency:id:key}");
+ long ttlInRedis = jedisPool.ttl("{idempotency:id:key}");
assertThat(redisItem).isNotNull();
- assertThat(redisItem).containsEntry("status", "COMPLETED");
- assertThat(redisItem).containsEntry("expiration", String.valueOf(expiry));
- assertThat(redisItem).containsEntry("in-progress-expiration", String.valueOf(progressExpiry.getAsLong()));
+ assertThat(redisItem).containsEntry("{idempotency:id:key}:status", "COMPLETED");
+ assertThat(redisItem).containsEntry("{idempotency:id:key}:expiration", String.valueOf(expiry));
+ assertThat(redisItem).containsEntry("{idempotency:id:key}:in-progress-expiration",
+ String.valueOf(progressExpiry.getAsLong()));
assertThat(Math.round(ttlInRedis / 100.0) * 100).isEqualTo(ttl);
}
@@ -96,14 +139,14 @@ void putRecord_shouldCreateItemInRedisWithInProgressExpiration() {
void putRecord_shouldCreateItemInRedis_withExistingJedisClient() {
Instant now = Instant.now();
long expiry = now.plus(3600, ChronoUnit.SECONDS).getEpochSecond();
- RedisPersistenceStore store = new RedisPersistenceStore.Builder().withJedisPooled(jedisPool).build();
+ RedisPersistenceStore store = new RedisPersistenceStore.Builder().withJedisClient(jedisPool).build();
store.putRecord(new DataRecord("key", DataRecord.Status.COMPLETED, expiry, null, null), now);
- Map redisItem = jedisPool.hgetAll("idempotency:id:key");
+ Map redisItem = jedisPool.hgetAll("{idempotency:id:key}");
assertThat(redisItem).isNotNull();
- assertThat(redisItem).containsEntry("status", "COMPLETED");
- assertThat(redisItem).containsEntry("expiration", String.valueOf(expiry));
+ assertThat(redisItem).containsEntry("{idempotency:id:key}:status", "COMPLETED");
+ assertThat(redisItem).containsEntry("{idempotency:id:key}:expiration", String.valueOf(expiry));
}
@Test
@@ -112,13 +155,13 @@ void putRecord_shouldCreateItemInRedis_IfPreviousExpired() {
Map item = new HashMap<>();
Instant now = Instant.now();
long expiry = now.minus(30, ChronoUnit.SECONDS).getEpochSecond();
- item.put("expiration", String.valueOf(expiry));
- item.put("status", DataRecord.Status.COMPLETED.toString());
- item.put("data", "Fake Data");
+ item.put("{idempotency:id:key}:expiration", String.valueOf(expiry));
+ item.put("{idempotency:id:key}:status", DataRecord.Status.COMPLETED.toString());
+ item.put("{idempotency:id:key}:data", "Fake Data");
long ttl = 3600;
long expiry2 = now.plus(ttl, ChronoUnit.SECONDS).getEpochSecond();
- jedisPool.hset("idempotency:id:key", item);
+ jedisPool.hset("{idempotency:id:key}", item);
redisPersistenceStore.putRecord(
new DataRecord("key",
DataRecord.Status.INPROGRESS,
@@ -127,12 +170,12 @@ void putRecord_shouldCreateItemInRedis_IfPreviousExpired() {
null
), now);
- Map redisItem = jedisPool.hgetAll("idempotency:id:key");
- long ttlInRedis = jedisPool.ttl("idempotency:id:key");
+ Map redisItem = jedisPool.hgetAll("{idempotency:id:key}");
+ long ttlInRedis = jedisPool.ttl("{idempotency:id:key}");
assertThat(redisItem).isNotNull();
- assertThat(redisItem).containsEntry("status", "INPROGRESS");
- assertThat(redisItem).containsEntry("expiration", String.valueOf(expiry2));
+ assertThat(redisItem).containsEntry("{idempotency:id:key}:status", "INPROGRESS");
+ assertThat(redisItem).containsEntry("{idempotency:id:key}:expiration", String.valueOf(expiry2));
assertThat(Math.round(ttlInRedis / 100.0) * 100).isEqualTo(ttl);
}
@@ -143,11 +186,11 @@ void putRecord_shouldCreateItemInRedis_IfLambdaWasInProgressAndTimedOut() {
Instant now = Instant.now();
long expiry = now.plus(30, ChronoUnit.SECONDS).getEpochSecond();
long progressExpiry = now.minus(30, ChronoUnit.SECONDS).toEpochMilli();
- item.put("expiration", String.valueOf(expiry));
- item.put("status", DataRecord.Status.INPROGRESS.toString());
- item.put("data", "Fake Data");
- item.put("in-progress-expiration", String.valueOf(progressExpiry));
- jedisPool.hset("idempotency:id:key", item);
+ item.put("{idempotency:id:key}:expiration", String.valueOf(expiry));
+ item.put("{idempotency:id:key}:status", DataRecord.Status.INPROGRESS.toString());
+ item.put("{idempotency:id:key}:data", "Fake Data");
+ item.put("{idempotency:id:key}:in-progress-expiration", String.valueOf(progressExpiry));
+ jedisPool.hset("{idempotency:id:key}", item);
long expiry2 = now.plus(3600, ChronoUnit.SECONDS).getEpochSecond();
redisPersistenceStore.putRecord(
@@ -158,11 +201,11 @@ void putRecord_shouldCreateItemInRedis_IfLambdaWasInProgressAndTimedOut() {
null
), now);
- Map redisItem = jedisPool.hgetAll("idempotency:id:key");
+ Map redisItem = jedisPool.hgetAll("{idempotency:id:key}");
assertThat(redisItem).isNotNull();
- assertThat(redisItem).containsEntry("status", "INPROGRESS");
- assertThat(redisItem).containsEntry("expiration", String.valueOf(expiry2));
+ assertThat(redisItem).containsEntry("{idempotency:id:key}:status", "INPROGRESS");
+ assertThat(redisItem).containsEntry("{idempotency:id:key}:expiration", String.valueOf(expiry2));
}
@Test
@@ -171,11 +214,11 @@ void putRecord_shouldThrowIdempotencyItemAlreadyExistsException_IfRecordAlreadyE
Map item = new HashMap<>();
Instant now = Instant.now();
long expiry = now.plus(30, ChronoUnit.SECONDS).getEpochSecond();
- item.put("expiration", String.valueOf(expiry)); // not expired
- item.put("status", DataRecord.Status.COMPLETED.toString());
- item.put("data", "Fake Data");
+ item.put("{idempotency:id:key}:expiration", String.valueOf(expiry)); // not expired
+ item.put("{idempotency:id:key}:status", DataRecord.Status.COMPLETED.toString());
+ item.put("{idempotency:id:key}:data", "Fake Data");
- jedisPool.hset("idempotency:id:key", item);
+ jedisPool.hset("{idempotency:id:key}", item);
long expiry2 = now.plus(3600, ChronoUnit.SECONDS).getEpochSecond();
DataRecord dataRecord = new DataRecord("key",
@@ -190,12 +233,12 @@ void putRecord_shouldThrowIdempotencyItemAlreadyExistsException_IfRecordAlreadyE
}
).isInstanceOf(IdempotencyItemAlreadyExistsException.class);
- Map entry = jedisPool.hgetAll("idempotency:id:key");
+ Map entry = jedisPool.hgetAll("{idempotency:id:key}");
assertThat(entry).isNotNull();
- assertThat(entry.get("status")).isEqualTo("COMPLETED");
- assertThat(entry.get("expiration")).isEqualTo(String.valueOf(expiry));
- assertThat(entry.get("data")).isEqualTo("Fake Data");
+ assertThat(entry.get("{idempotency:id:key}:status")).isEqualTo("COMPLETED");
+ assertThat(entry.get("{idempotency:id:key}:expiration")).isEqualTo(String.valueOf(expiry));
+ assertThat(entry.get("{idempotency:id:key}:data")).isEqualTo("Fake Data");
}
@Test
@@ -205,11 +248,11 @@ void putRecord_shouldBlockUpdate_IfRecordAlreadyExistAndProgressNotExpiredAfterL
Instant now = Instant.now();
long expiry = now.plus(30, ChronoUnit.SECONDS).getEpochSecond(); // not expired
long progressExpiry = now.plus(30, ChronoUnit.SECONDS).toEpochMilli(); // not expired
- item.put("expiration", String.valueOf(expiry));
- item.put("status", DataRecord.Status.INPROGRESS.toString());
- item.put("data", "Fake Data");
- item.put("in-progress-expiration", String.valueOf(progressExpiry));
- jedisPool.hset("idempotency:id:key", item);
+ item.put("{idempotency:id:key}:expiration", String.valueOf(expiry));
+ item.put("{idempotency:id:key}:status", DataRecord.Status.INPROGRESS.toString());
+ item.put("{idempotency:id:key}:data", "Fake Data");
+ item.put("{idempotency:id:key}:in-progress-expiration", String.valueOf(progressExpiry));
+ jedisPool.hset("{idempotency:id:key}", item);
long expiry2 = now.plus(3600, ChronoUnit.SECONDS).getEpochSecond();
DataRecord dataRecord = new DataRecord("key",
@@ -222,12 +265,12 @@ void putRecord_shouldBlockUpdate_IfRecordAlreadyExistAndProgressNotExpiredAfterL
dataRecord, now))
.isInstanceOf(IdempotencyItemAlreadyExistsException.class);
- Map redisItem = jedisPool.hgetAll("idempotency:id:key");
+ Map redisItem = jedisPool.hgetAll("{idempotency:id:key}");
assertThat(redisItem).isNotNull();
- assertThat(redisItem).containsEntry("status", "INPROGRESS");
- assertThat(redisItem).containsEntry("expiration", String.valueOf(expiry));
- assertThat(redisItem).containsEntry("data", "Fake Data");
+ assertThat(redisItem).containsEntry("{idempotency:id:key}:status", "INPROGRESS");
+ assertThat(redisItem).containsEntry("{idempotency:id:key}:expiration", String.valueOf(expiry));
+ assertThat(redisItem).containsEntry("{idempotency:id:key}:data", "Fake Data");
}
@Test
@@ -236,10 +279,10 @@ void getRecord_shouldReturnExistingRecord() throws IdempotencyItemNotFoundExcept
Map item = new HashMap<>();
Instant now = Instant.now();
long expiry = now.plus(30, ChronoUnit.SECONDS).getEpochSecond();
- item.put("expiration", String.valueOf(expiry));
- item.put("status", DataRecord.Status.COMPLETED.toString());
- item.put("data", ("Fake Data"));
- jedisPool.hset("idempotency:id:key", item);
+ item.put("{idempotency:id:key}:expiration", String.valueOf(expiry));
+ item.put("{idempotency:id:key}:status", DataRecord.Status.COMPLETED.toString());
+ item.put("{idempotency:id:key}:data", ("Fake Data"));
+ jedisPool.hset("{idempotency:id:key}", item);
DataRecord record = redisPersistenceStore.getRecord("key");
@@ -260,9 +303,9 @@ void updateRecord_shouldUpdateRecord() {
Map item = new HashMap<>();
Instant now = Instant.now();
long expiry = now.plus(360, ChronoUnit.SECONDS).getEpochSecond();
- item.put("expiration", String.valueOf(expiry));
- item.put("status", DataRecord.Status.INPROGRESS.toString());
- jedisPool.hset("idempotency:id:key", item);
+ item.put("{idempotency:id:key}:expiration", String.valueOf(expiry));
+ item.put("{idempotency:id:key}:status", DataRecord.Status.INPROGRESS.toString());
+ jedisPool.hset("{idempotency:id:key}", item);
// enable payload validation
redisPersistenceStore.configure(IdempotencyConfig.builder().withPayloadValidationJMESPath("path").build(),
null);
@@ -272,13 +315,13 @@ void updateRecord_shouldUpdateRecord() {
DataRecord record = new DataRecord("key", DataRecord.Status.COMPLETED, expiry, "Fake result", "hash");
redisPersistenceStore.updateRecord(record);
- Map redisItem = jedisPool.hgetAll("idempotency:id:key");
- long ttlInRedis = jedisPool.ttl("idempotency:id:key");
+ Map redisItem = jedisPool.hgetAll("{idempotency:id:key}");
+ long ttlInRedis = jedisPool.ttl("{idempotency:id:key}");
- assertThat(redisItem).containsEntry("status", "COMPLETED");
- assertThat(redisItem).containsEntry("expiration", String.valueOf(expiry));
- assertThat(redisItem).containsEntry("data", "Fake result");
- assertThat(redisItem).containsEntry("validation", "hash");
+ assertThat(redisItem).containsEntry("{idempotency:id:key}:status", "COMPLETED");
+ assertThat(redisItem).containsEntry("{idempotency:id:key}:expiration", String.valueOf(expiry));
+ assertThat(redisItem).containsEntry("{idempotency:id:key}:data", "Fake result");
+ assertThat(redisItem).containsEntry("{idempotency:id:key}:validation", "hash");
assertThat(Math.round(ttlInRedis / 100.0) * 100).isEqualTo(ttl);
}
@@ -287,13 +330,13 @@ void deleteRecord_shouldDeleteRecord() {
Map item = new HashMap<>();
Instant now = Instant.now();
long expiry = now.plus(360, ChronoUnit.SECONDS).getEpochSecond();
- item.put("expiration", String.valueOf(expiry));
- item.put("status", DataRecord.Status.INPROGRESS.toString());
- jedisPool.hset("idempotency:id:key", item);
+ item.put("{idempotency:id:key}:expiration", String.valueOf(expiry));
+ item.put("{idempotency:id:key}:status", DataRecord.Status.INPROGRESS.toString());
+ jedisPool.hset("{idempotency:id:key}", item);
redisPersistenceStore.deleteRecord("key");
- Map items = jedisPool.hgetAll("idempotency:id:key");
+ Map items = jedisPool.hgetAll("{idempotency:id:key}");
assertThat(items).isEmpty();
}
@@ -304,7 +347,7 @@ void endToEndWithCustomAttrNamesAndSortKey() throws IdempotencyItemNotFoundExcep
try {
RedisPersistenceStore persistenceStore = RedisPersistenceStore.builder()
.withKeyPrefixName("items-idempotency")
- .withJedisPooled(jedisPool)
+ .withJedisClient(jedisPool)
.withDataAttr("result")
.withExpiryAttr("expiry")
.withKeyAttr("key")
@@ -323,14 +366,16 @@ void endToEndWithCustomAttrNamesAndSortKey() throws IdempotencyItemNotFoundExcep
// PUT
persistenceStore.putRecord(record, now);
- Map redisItem = jedisPool.hgetAll("items-idempotency:key:mykey");
+ Map redisItem = jedisPool.hgetAll("{items-idempotency:key:mykey}");
// GET
DataRecord recordInDb = persistenceStore.getRecord("mykey");
assertThat(redisItem).isNotNull();
- assertThat(redisItem).containsEntry("state", recordInDb.getStatus().toString());
- assertThat(redisItem).containsEntry("expiry", String.valueOf(recordInDb.getExpiryTimestamp()));
+ assertThat(redisItem).containsEntry("{items-idempotency:key:mykey}:state",
+ recordInDb.getStatus().toString());
+ assertThat(redisItem).containsEntry("{items-idempotency:key:mykey}:expiry",
+ String.valueOf(recordInDb.getExpiryTimestamp()));
// UPDATE
DataRecord updatedRecord = new DataRecord(
@@ -346,10 +391,10 @@ void endToEndWithCustomAttrNamesAndSortKey() throws IdempotencyItemNotFoundExcep
// DELETE
persistenceStore.deleteRecord("mykey");
- assertThat(jedisPool.hgetAll("items-idempotency:key:mykey").size()).isZero();
+ assertThat(jedisPool.hgetAll("{items-idempotency:key:mykey}").size()).isZero();
} finally {
- jedisPool.del("items-idempotency:key:mykey");
+ jedisPool.del("{items-idempotency:key:mykey}");
}
}
@@ -362,7 +407,7 @@ void idempotencyDisabled_noClientShouldBeCreated() {
@AfterEach
void emptyDB() {
- jedisPool.del("idempotency:id:key");
+ jedisPool.del("{idempotency:id:key}");
}
}
From 0582f2483614fcc422db30dc603c9fd003e6b4a4 Mon Sep 17 00:00:00 2001
From: Eleni Dimitropoulou <12170229+eldimi@users.noreply.github.com>
Date: Wed, 13 Dec 2023 10:19:06 +0200
Subject: [PATCH 20/31] docs improvements - Apply suggestions from code review
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Co-authored-by: Jérôme Van Der Linden <117538+jeromevdl@users.noreply.github.com>
---
docs/utilities/idempotency.md | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md
index 8d6077d68..6810628e8 100644
--- a/docs/utilities/idempotency.md
+++ b/docs/utilities/idempotency.md
@@ -164,7 +164,7 @@ As of now, [Amazon DynamoDB](https://aws.amazon.com/dynamodb/) and [Redis](https
#### Using Amazon DynamoDB as persistent storage layer
-If you are using Amazon DynamoDB you'll need to create a table.
+If you are using Amazon DynamoDB, you'll need to create a table.
**Default table configuration**
@@ -221,9 +221,9 @@ Resources:
##### Redis resources
-You need an existing Redis service before setting up Redis as persistent storage layer provider. You can also use Redis compatible services like [Amazon ElastiCache for Redis](https://aws.amazon.com/elasticache/redis/) as persistent storage layer provider.
+You need an existing Redis service before setting up Redis as persistent storage layer provider. You can also use Redis compatible services like [Amazon ElastiCache for Redis](https://aws.amazon.com/elasticache/redis/).
!!! tip "Tip:No existing Redis service?"
-If you don't have an existing Redis service, we recommend using DynamoDB as persistent storage layer provider.
+ If you don't have an existing Redis service, we recommend using DynamoDB as persistent storage layer provider.
If you are using Redis you'll need to provide the Redis host, port, user and password as AWS Lambda environment variables.
If you want to connect to a Redis cluster instead of a Standalone server, you need to enable Redis cluster mode by setting an AWS Lambda
@@ -257,10 +257,10 @@ Resources:
Your AWS Lambda Function must be able to reach the Redis endpoint before using it for idempotency persistent storage layer. In most cases you will need to [configure VPC access](https://docs.aws.amazon.com/lambda/latest/dg/configuration-vpc.html) for your AWS Lambda Function. Using a public accessible Redis is not recommended.
!!! tip "Amazon ElastiCache for Redis as persistent storage layer provider"
-If you intend to use Amazon ElastiCache for Redis for idempotency persistent storage layer, you can also reference [This AWS Tutorial](https://docs.aws.amazon.com/lambda/latest/dg/services-elasticache-tutorial.html).
+ If you intend to use Amazon ElastiCache for Redis for idempotency persistent storage layer, you can also reference [This AWS Tutorial](https://docs.aws.amazon.com/lambda/latest/dg/services-elasticache-tutorial.html).
!!! warning "Amazon ElastiCache Serverless not supported"
-[Amazon ElastiCache Serverless](https://aws.amazon.com/elasticache/features/#Serverless) is not supported for now.
+ [Amazon ElastiCache Serverless](https://aws.amazon.com/elasticache/features/#Serverless) is not supported for now.
!!! warning "Check network connectivity to Redis server"
Make sure that your AWS Lambda function can connect to your Redis server.
@@ -722,7 +722,7 @@ You can alter the field names by passing these parameters when initializing the
!!! Tip "Tip: You can share the same prefix and key for all functions"
-You can reuse the same prefix and key to store idempotency state. We add your function name in addition to the idempotency key as a hash key.
+ You can reuse the same prefix and key to store idempotency state. We add your function name in addition to the idempotency key as a hash key.
## Advanced
From fce92df657690b91f2a5ef180d642890bc43986f Mon Sep 17 00:00:00 2001
From: Eleni Dimitropoulou <12170229+eldimi@users.noreply.github.com>
Date: Wed, 13 Dec 2023 11:14:26 +0200
Subject: [PATCH 21/31] Apply checkstyle to fix import order
---
.../idempotency/IdempotencyConfig.java | 3 +-
.../powertools/idempotency/Idempotent.java | 1 -
.../internal/IdempotencyHandler.java | 11 +++---
.../internal/IdempotentAspect.java | 9 ++---
.../persistence/BasePersistenceStore.java | 39 +++++++++----------
.../idempotency/persistence/DataRecord.java | 3 +-
.../persistence/PersistenceStore.java | 3 +-
.../internal/IdempotencyAspectTest.java | 29 +++++++-------
.../internal/cache/LRUCacheTest.java | 4 +-
.../persistence/BasePersistenceStoreTest.java | 15 ++++---
.../dynamodb/DynamoDBPersistenceStore.java | 23 ++++++-----
.../handlers/IdempotencyFunction.java | 15 ++++---
.../DynamoDBPersistenceStoreTest.java | 17 ++++----
.../redis/RedisPersistenceStore.java | 11 ++++--
.../redis/RedisPersistenceStoreTest.java | 7 +++-
15 files changed, 92 insertions(+), 98 deletions(-)
diff --git a/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/IdempotencyConfig.java b/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/IdempotencyConfig.java
index 2b22cac51..baf939d11 100644
--- a/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/IdempotencyConfig.java
+++ b/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/IdempotencyConfig.java
@@ -15,9 +15,8 @@
package software.amazon.lambda.powertools.idempotency;
import com.amazonaws.services.lambda.runtime.Context;
-import software.amazon.lambda.powertools.idempotency.internal.cache.LRUCache;
-
import java.time.Duration;
+import software.amazon.lambda.powertools.idempotency.internal.cache.LRUCache;
/**
* Configuration of the idempotency feature. Use the {@link Builder} to create an instance.
diff --git a/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/Idempotent.java b/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/Idempotent.java
index d08874492..6ca40a0e1 100644
--- a/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/Idempotent.java
+++ b/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/Idempotent.java
@@ -15,7 +15,6 @@
package software.amazon.lambda.powertools.idempotency;
import com.amazonaws.services.lambda.runtime.Context;
-
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
diff --git a/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/internal/IdempotencyHandler.java b/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/internal/IdempotencyHandler.java
index 7982d911a..2875ab3d1 100644
--- a/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/internal/IdempotencyHandler.java
+++ b/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/internal/IdempotencyHandler.java
@@ -14,8 +14,13 @@
package software.amazon.lambda.powertools.idempotency.internal;
+import static software.amazon.lambda.powertools.idempotency.persistence.DataRecord.Status.EXPIRED;
+import static software.amazon.lambda.powertools.idempotency.persistence.DataRecord.Status.INPROGRESS;
+
import com.amazonaws.services.lambda.runtime.Context;
import com.fasterxml.jackson.databind.JsonNode;
+import java.time.Instant;
+import java.util.OptionalInt;
import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.reflect.MethodSignature;
import org.slf4j.Logger;
@@ -32,12 +37,6 @@
import software.amazon.lambda.powertools.idempotency.persistence.DataRecord;
import software.amazon.lambda.powertools.utilities.JsonConfig;
-import java.time.Instant;
-import java.util.OptionalInt;
-
-import static software.amazon.lambda.powertools.idempotency.persistence.DataRecord.Status.EXPIRED;
-import static software.amazon.lambda.powertools.idempotency.persistence.DataRecord.Status.INPROGRESS;
-
/**
* Internal class that will handle the Idempotency, and use the {@link software.amazon.lambda.powertools.idempotency.persistence.PersistenceStore}
* to store the result of previous calls.
diff --git a/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/internal/IdempotentAspect.java b/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/internal/IdempotentAspect.java
index ea6d743f0..0b9d729f4 100644
--- a/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/internal/IdempotentAspect.java
+++ b/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/internal/IdempotentAspect.java
@@ -14,8 +14,12 @@
package software.amazon.lambda.powertools.idempotency.internal;
+import static software.amazon.lambda.powertools.common.internal.LambdaHandlerProcessor.placedOnRequestHandler;
+
import com.amazonaws.services.lambda.runtime.Context;
import com.fasterxml.jackson.databind.JsonNode;
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Method;
import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.annotation.Around;
import org.aspectj.lang.annotation.Aspect;
@@ -29,11 +33,6 @@
import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyConfigurationException;
import software.amazon.lambda.powertools.utilities.JsonConfig;
-import java.lang.annotation.Annotation;
-import java.lang.reflect.Method;
-
-import static software.amazon.lambda.powertools.common.internal.LambdaHandlerProcessor.placedOnRequestHandler;
-
/**
* Aspect that handles the {@link Idempotent} annotation.
* It uses the {@link IdempotencyHandler} to actually do the job.
diff --git a/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/persistence/BasePersistenceStore.java b/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/persistence/BasePersistenceStore.java
index bafbcbd42..0a1acdf5c 100644
--- a/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/persistence/BasePersistenceStore.java
+++ b/powertools-idempotency/powertools-idempotency-core/src/main/java/software/amazon/lambda/powertools/idempotency/persistence/BasePersistenceStore.java
@@ -14,20 +14,12 @@
package software.amazon.lambda.powertools.idempotency.persistence;
+import static software.amazon.lambda.powertools.common.internal.LambdaConstants.LAMBDA_FUNCTION_NAME_ENV;
+
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectWriter;
import io.burt.jmespath.Expression;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import software.amazon.lambda.powertools.idempotency.IdempotencyConfig;
-import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyItemAlreadyExistsException;
-import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyItemNotFoundException;
-import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyKeyException;
-import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyValidationException;
-import software.amazon.lambda.powertools.idempotency.internal.cache.LRUCache;
-import software.amazon.lambda.powertools.utilities.JsonConfig;
-
import java.math.BigInteger;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
@@ -41,8 +33,15 @@
import java.util.Spliterators;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
-
-import static software.amazon.lambda.powertools.common.internal.LambdaConstants.LAMBDA_FUNCTION_NAME_ENV;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.lambda.powertools.idempotency.IdempotencyConfig;
+import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyItemAlreadyExistsException;
+import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyItemNotFoundException;
+import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyKeyException;
+import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyValidationException;
+import software.amazon.lambda.powertools.idempotency.internal.cache.LRUCache;
+import software.amazon.lambda.powertools.utilities.JsonConfig;
/**
* Persistence layer that will store the idempotency result.
@@ -64,6 +63,14 @@ public abstract class BasePersistenceStore implements PersistenceStore {
private boolean throwOnNoIdempotencyKey = false;
private String hashFunctionName;
+ private static boolean isEqual(String dataRecordPayload, String dataHash) {
+ if (dataHash != null && dataRecordPayload != null) {
+ return dataHash.length() != dataRecordPayload.length() ? false : dataHash.equals(dataRecordPayload);
+ } else {
+ return false;
+ }
+ }
+
/**
* Initialize the base persistence layer from the configuration settings
*
@@ -402,12 +409,4 @@ void configure(IdempotencyConfig config, String functionName, LRUCache
* Use the {@link Builder} to create a new instance.
diff --git a/powertools-idempotency/powertools-idempotency-dynamodb/src/test/java/software.amazon.lambda.powertools.idempotency.dynamodb/handlers/IdempotencyFunction.java b/powertools-idempotency/powertools-idempotency-dynamodb/src/test/java/software.amazon.lambda.powertools.idempotency.dynamodb/handlers/IdempotencyFunction.java
index 1296a75c7..76a012930 100644
--- a/powertools-idempotency/powertools-idempotency-dynamodb/src/test/java/software.amazon.lambda.powertools.idempotency.dynamodb/handlers/IdempotencyFunction.java
+++ b/powertools-idempotency/powertools-idempotency-dynamodb/src/test/java/software.amazon.lambda.powertools.idempotency.dynamodb/handlers/IdempotencyFunction.java
@@ -18,6 +18,13 @@
import com.amazonaws.services.lambda.runtime.RequestHandler;
import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyRequestEvent;
import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyResponseEvent;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.stream.Collectors;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
@@ -27,14 +34,6 @@
import software.amazon.lambda.powertools.idempotency.persistence.dynamodb.DynamoDBPersistenceStore;
import software.amazon.lambda.powertools.utilities.JsonConfig;
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.net.URL;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.stream.Collectors;
-
public class IdempotencyFunction implements RequestHandler {
private final static Logger LOG = LogManager.getLogger(IdempotencyFunction.class);
diff --git a/powertools-idempotency/powertools-idempotency-dynamodb/src/test/java/software/amazon/lambda/powertools/idempotency/persistence/dynamodb/DynamoDBPersistenceStoreTest.java b/powertools-idempotency/powertools-idempotency-dynamodb/src/test/java/software/amazon/lambda/powertools/idempotency/persistence/dynamodb/DynamoDBPersistenceStoreTest.java
index cc682a81f..e67420def 100644
--- a/powertools-idempotency/powertools-idempotency-dynamodb/src/test/java/software/amazon/lambda/powertools/idempotency/persistence/dynamodb/DynamoDBPersistenceStoreTest.java
+++ b/powertools-idempotency/powertools-idempotency-dynamodb/src/test/java/software/amazon/lambda/powertools/idempotency/persistence/dynamodb/DynamoDBPersistenceStoreTest.java
@@ -14,6 +14,14 @@
package software.amazon.lambda.powertools.idempotency.persistence.dynamodb;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -37,15 +45,6 @@
import software.amazon.lambda.powertools.idempotency.exceptions.IdempotencyItemNotFoundException;
import software.amazon.lambda.powertools.idempotency.persistence.DataRecord;
-import java.time.Instant;
-import java.time.temporal.ChronoUnit;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-
/**
* These test are using DynamoDBLocal and sqlite, see https://nickolasfisher.com/blog/Configuring-an-In-Memory-DynamoDB-instance-with-Java-for-Integration-Testing
* NOTE: on a Mac with Apple Chipset, you need to use the Oracle JDK x86 64-bit
diff --git a/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java b/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java
index efd139d82..0ca687e6e 100644
--- a/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java
+++ b/powertools-idempotency/powertools-idempotency-redis/src/main/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStore.java
@@ -1,4 +1,4 @@
-package software.amazon.lambda.powertools.idempotency.redis;/*
+/*
* Copyright 2023 Amazon.com, Inc. or its affiliates.
* Licensed under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
@@ -12,6 +12,8 @@
*
*/
+package software.amazon.lambda.powertools.idempotency.redis;
+
import static software.amazon.lambda.powertools.idempotency.persistence.DataRecord.Status.INPROGRESS;
import java.time.Instant;
@@ -77,7 +79,8 @@ private RedisPersistenceStore(String keyPrefixName,
String idempotencyDisabledEnv =
System.getenv(software.amazon.lambda.powertools.idempotency.Constants.IDEMPOTENCY_DISABLED_ENV);
if (idempotencyDisabledEnv == null || "false".equalsIgnoreCase(idempotencyDisabledEnv)) {
- this.jedisClient = getJedisClient(System.getenv(Constants.REDIS_HOST), Integer.parseInt(System.getenv(Constants.REDIS_PORT)));
+ this.jedisClient = getJedisClient(System.getenv(Constants.REDIS_HOST),
+ Integer.parseInt(System.getenv(Constants.REDIS_PORT)));
} else {
// we do not want to create a Jedis connection pool if idempotency is disabled
// null is ok as idempotency won't be called
@@ -107,11 +110,11 @@ public static Builder builder() {
return new Builder();
}
- UnifiedJedis getJedisClient(String redisHost, Integer redisPort) {
+ UnifiedJedis getJedisClient(String redisHost, Integer redisPort) {
HostAndPort address = new HostAndPort(redisHost, redisPort);
JedisClientConfig config = getJedisClientConfig();
String isClusterMode = System.getenv(Constants.REDIS_CLUSTER_MODE);
- if (isClusterMode != null && "true".equalsIgnoreCase(isClusterMode)) {
+ if ("true".equalsIgnoreCase(isClusterMode)) {
return new JedisCluster(address, getJedisClientConfig(), 5, new GenericObjectPoolConfig<>());
} else {
return new JedisPooled(address, config);
diff --git a/powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java b/powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
index b2f6dd2ee..adacb1a2b 100644
--- a/powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
+++ b/powertools-idempotency/powertools-idempotency-redis/src/test/java/software/amazon/lambda/powertools/idempotency/redis/RedisPersistenceStoreTest.java
@@ -106,15 +106,18 @@ void putRecord_JedisClientInstanceOfJedisCluster() throws IOException {
.newRedisServer()
.setOptions(ServiceOptions.defaultOptions().withClusterModeEnabled())
.start();
- assertThat(redisPersistenceStore.getJedisClient(redisCluster.getHost(), redisCluster.getBindPort()) instanceof JedisCluster).isTrue();
+ assertThat(redisPersistenceStore.getJedisClient(redisCluster.getHost(),
+ redisCluster.getBindPort()) instanceof JedisCluster).isTrue();
redisCluster.stop();
}
@SetEnvironmentVariable(key = Constants.REDIS_CLUSTER_MODE, value = "false")
@Test
void putRecord_JedisClientInstanceOfJedisPooled() {
- assertThat(redisPersistenceStore.getJedisClient(System.getenv(REDIS_HOST), Integer.parseInt(System.getenv(REDIS_PORT))) instanceof JedisCluster).isFalse();
+ assertThat(redisPersistenceStore.getJedisClient(System.getenv(REDIS_HOST),
+ Integer.parseInt(System.getenv(REDIS_PORT))) instanceof JedisCluster).isFalse();
}
+
@Test
void putRecord_shouldCreateItemInRedisWithInProgressExpiration() {
Instant now = Instant.now();
From df7452f17ec630265caa207c1c6a96eadd32eb1b Mon Sep 17 00:00:00 2001
From: Eleni Dimitropoulou <12170229+eldimi@users.noreply.github.com>
Date: Tue, 19 Dec 2023 12:15:50 +0200
Subject: [PATCH 22/31] Fix build
---
powertools-large-messages/pom.xml | 5 +++++
powertools-logging/powertools-logging-log4j/pom.xml | 4 ++++
powertools-validation/pom.xml | 4 ++++
3 files changed, 13 insertions(+)
diff --git a/powertools-large-messages/pom.xml b/powertools-large-messages/pom.xml
index 4206183de..1bd670054 100644
--- a/powertools-large-messages/pom.xml
+++ b/powertools-large-messages/pom.xml
@@ -117,6 +117,11 @@
log4j-slf4j2-impl
test
+
+ org.apache.logging.log4j
+ log4j-api
+ test
+
diff --git a/powertools-logging/powertools-logging-log4j/pom.xml b/powertools-logging/powertools-logging-log4j/pom.xml
index df6154560..752f8014c 100644
--- a/powertools-logging/powertools-logging-log4j/pom.xml
+++ b/powertools-logging/powertools-logging-log4j/pom.xml
@@ -35,6 +35,10 @@
org.apache.logging.log4j
log4j-core