From 2c2002328825f601037a952429d261234db3dc8b Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Wed, 21 Jun 2023 16:01:19 +0200 Subject: [PATCH 01/79] Starting to sketch out shape of API for batch processor --- pom.xml | 1 + powertools-batch/.gitignore | 38 +++++++++++++++ powertools-batch/pom.xml | 24 ++++++++++ .../batch/BatchMessageHandlerBuilder.java | 47 +++++++++++++++++++ .../batch/BatchMessageProcessor.java | 18 +++++++ .../batch/examples/ExampleMessageHandler.java | 27 +++++++++++ .../batch/message/BatchProcessorMessage.java | 18 +++++++ .../message/BatchProcessorMessageHandler.java | 7 +++ .../message/BatchProcessorMessageType.java | 7 +++ .../batch/message/DynamoDbStreamMessage.java | 5 ++ .../message/KinesisDataStreamsMessage.java | 15 ++++++ .../powertools/batch/message/SqsMessage.java | 14 ++++++ 12 files changed, 221 insertions(+) create mode 100644 powertools-batch/.gitignore create mode 100644 powertools-batch/pom.xml create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageProcessor.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/examples/ExampleMessageHandler.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/BatchProcessorMessage.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/BatchProcessorMessageHandler.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/BatchProcessorMessageType.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/DynamoDbStreamMessage.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/KinesisDataStreamsMessage.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/SqsMessage.java diff --git a/pom.xml b/pom.xml index 53c9762f9..9ae4f3f65 100644 --- a/pom.xml +++ b/pom.xml @@ -40,6 +40,7 @@ powertools-idempotency powertools-e2e-tests examples + powertools-batch diff --git a/powertools-batch/.gitignore b/powertools-batch/.gitignore new file mode 100644 index 000000000..5ff6309b7 --- /dev/null +++ b/powertools-batch/.gitignore @@ -0,0 +1,38 @@ +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/**/target/ +!**/src/test/**/target/ + +### IntelliJ IDEA ### +.idea/modules.xml +.idea/jarRepositories.xml +.idea/compiler.xml +.idea/libraries/ +*.iws +*.iml +*.ipr + +### Eclipse ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ +!**/src/main/**/build/ +!**/src/test/**/build/ + +### VS Code ### +.vscode/ + +### Mac OS ### +.DS_Store \ No newline at end of file diff --git a/powertools-batch/pom.xml b/powertools-batch/pom.xml new file mode 100644 index 000000000..569558756 --- /dev/null +++ b/powertools-batch/pom.xml @@ -0,0 +1,24 @@ + + + 4.0.0 + + software.amazon.lambda + powertools-parent + 1.15.0 + + + powertools-batch + + + com.amazonaws + aws-lambda-java-events + + + com.amazonaws + aws-lambda-java-core + + + + \ No newline at end of file diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java new file mode 100644 index 000000000..6aaf7ce4a --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java @@ -0,0 +1,47 @@ +package software.amazon.lambda.powertools.batch; + +import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; +import com.amazonaws.services.lambda.runtime.events.KinesisEvent; +import com.amazonaws.services.lambda.runtime.events.SQSEvent; +import software.amazon.lambda.powertools.batch.BatchMessageProcessor; +import software.amazon.lambda.powertools.batch.message.*; +import sun.reflect.generics.reflectiveObjects.NotImplementedException; + +import java.time.Duration; +import java.util.List; +import java.util.function.Consumer; + +/** + * Let's make this builder-style so that we can extend the interface + * if we need to with extra fields without breaking clients + */ +public class BatchMessageHandlerBuilder { + + private Duration timeout; + + public BatchMessageHandlerBuilder() { + } + + /** + * An example parameter we might want to configure here to show + * the shape of the interface. + */ + public BatchMessageHandlerBuilder withTimeout(Duration timeout) { + this.timeout = timeout; + return this; + } + + // TODO - can we put a meaningful type on the message, or do we need to do it, per-message-type? + public List process(SQSEvent message, Consumer messageHandler) { + throw new NotImplementedException(); + }; + + public List process(KinesisEvent message, Consumer messageHandler) { + throw new NotImplementedException(); + } + + public List process(DynamodbEvent message, Consumer messageHandler) { + throw new NotImplementedException(); + } + +} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageProcessor.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageProcessor.java new file mode 100644 index 000000000..5a8a7edfa --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageProcessor.java @@ -0,0 +1,18 @@ +package software.amazon.lambda.powertools.batch; + +import software.amazon.lambda.powertools.batch.message.BatchProcessorMessageHandler; +import software.amazon.lambda.powertools.batch.message.BatchProcessorMessageType; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.METHOD) +public @interface BatchMessageProcessor { + Class> value(); + + BatchProcessorMessageType MessageType(); + +} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/examples/ExampleMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/examples/ExampleMessageHandler.java new file mode 100644 index 000000000..0d5fd7383 --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/examples/ExampleMessageHandler.java @@ -0,0 +1,27 @@ +package software.amazon.lambda.powertools.batch.examples; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.amazonaws.services.lambda.runtime.events.SQSEvent; +import software.amazon.lambda.powertools.batch.BatchMessageHandlerBuilder; +import software.amazon.lambda.powertools.batch.BatchMessageProcessor; +import software.amazon.lambda.powertools.batch.message.BatchProcessorMessageType; +import software.amazon.lambda.powertools.batch.message.SqsMessage; + +import java.util.List; + +/** + * This is just here for illustrative purposes, and won't + * be released with this code! + */ +public class ExampleMessageHandler implements RequestHandler> { + + @Override + public List handleRequest(SQSEvent sqsEvent, Context context) { + return new BatchMessageHandlerBuilder() + .process(sqsEvent, (SqsMessage message) -> { + // Process the message without throwing an exception + }); + } + +} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/BatchProcessorMessage.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/BatchProcessorMessage.java new file mode 100644 index 000000000..eed62daba --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/BatchProcessorMessage.java @@ -0,0 +1,18 @@ +package software.amazon.lambda.powertools.batch.message; + +/** + * Common base class for all batch message processing message types. + */ +public abstract class BatchProcessorMessage { + + private String eventSourceArn; + private String eventSource; + private String awsRegion; + + // Called messageId or eventId depending on source + private String messageId; + + private String body; + + private String md5OfBody; +} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/BatchProcessorMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/BatchProcessorMessageHandler.java new file mode 100644 index 000000000..e46918d85 --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/BatchProcessorMessageHandler.java @@ -0,0 +1,7 @@ +package software.amazon.lambda.powertools.batch.message; + +@FunctionalInterface +public interface BatchProcessorMessageHandler { + + R process(BatchProcessorMessage message); +} \ No newline at end of file diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/BatchProcessorMessageType.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/BatchProcessorMessageType.java new file mode 100644 index 000000000..f4c6bd6f1 --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/BatchProcessorMessageType.java @@ -0,0 +1,7 @@ +package software.amazon.lambda.powertools.batch.message; + +public enum BatchProcessorMessageType { + Sqs, + DynamoDbStreams, + KinesisDataStreams +} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/DynamoDbStreamMessage.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/DynamoDbStreamMessage.java new file mode 100644 index 000000000..364f6f95b --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/DynamoDbStreamMessage.java @@ -0,0 +1,5 @@ +package software.amazon.lambda.powertools.batch.message; + +public class DynamoDbStreamMessage extends BatchProcessorMessage { + +} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/KinesisDataStreamsMessage.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/KinesisDataStreamsMessage.java new file mode 100644 index 000000000..46fa8c40b --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/KinesisDataStreamsMessage.java @@ -0,0 +1,15 @@ +package software.amazon.lambda.powertools.batch.message; + +/** + * A message from a kinesis data streams batch. + * + * @see Java API Documentation + */ +public class KinesisDataStreamsMessage extends BatchProcessorMessage { + private String eventVersion; + private String invokeIdentityArn; + private String kinesisSchemaVersion; + private String partitionKey; + private String sequenceNumber; + private float approximateArrivalTimestamp; +} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/SqsMessage.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/SqsMessage.java new file mode 100644 index 000000000..89bd69d94 --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/SqsMessage.java @@ -0,0 +1,14 @@ +package software.amazon.lambda.powertools.batch.message; + +import java.util.Map; +import com.amazonaws.services.lambda.runtime.events.SQSEvent.MessageAttribute; + +public class SqsMessage extends BatchProcessorMessage { + + private String receiptHandle; + private String body; + private String md5OfBody; + private String md5OfMessageAttributes; + private Map attributes; + private Map messageAttributes; +} From 870ec4711c78f938cea872ecb6cc6e3a3e843418 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Sun, 25 Jun 2023 14:47:01 +0200 Subject: [PATCH 02/79] Variant 1 --- .../batch/BatchMessageHandlerBuilder.java | 12 ++--- .../batch/BatchMessageProcessor.java | 18 ------- .../powertools/batch/BatchRequestHandler.java | 53 +++++++++++++++++++ .../batch/KinesisBatchRequestHandler.java | 23 ++++++++ .../batch/SQSBatchRequestHandler.java | 24 +++++++++ .../examples/ExampleBatchRequestHandler.java | 13 +++++ ...java => ExampleMessageHandlerBuilder.java} | 11 ++-- .../batch/message/BatchProcessorMessage.java | 18 ------- .../message/BatchProcessorMessageHandler.java | 7 --- .../message/BatchProcessorMessageType.java | 7 --- .../batch/message/DynamoDbStreamMessage.java | 5 -- .../message/KinesisDataStreamsMessage.java | 15 ------ .../powertools/batch/message/SqsMessage.java | 14 ----- 13 files changed, 120 insertions(+), 100 deletions(-) delete mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageProcessor.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchRequestHandler.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/KinesisBatchRequestHandler.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/SQSBatchRequestHandler.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/examples/ExampleBatchRequestHandler.java rename powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/examples/{ExampleMessageHandler.java => ExampleMessageHandlerBuilder.java} (55%) delete mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/BatchProcessorMessage.java delete mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/BatchProcessorMessageHandler.java delete mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/BatchProcessorMessageType.java delete mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/DynamoDbStreamMessage.java delete mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/KinesisDataStreamsMessage.java delete mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/SqsMessage.java diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java index 6aaf7ce4a..193d1260a 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java @@ -1,10 +1,6 @@ package software.amazon.lambda.powertools.batch; -import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; -import com.amazonaws.services.lambda.runtime.events.KinesisEvent; -import com.amazonaws.services.lambda.runtime.events.SQSEvent; -import software.amazon.lambda.powertools.batch.BatchMessageProcessor; -import software.amazon.lambda.powertools.batch.message.*; +import com.amazonaws.services.lambda.runtime.events.*; import sun.reflect.generics.reflectiveObjects.NotImplementedException; import java.time.Duration; @@ -32,15 +28,15 @@ public BatchMessageHandlerBuilder withTimeout(Duration timeout) { } // TODO - can we put a meaningful type on the message, or do we need to do it, per-message-type? - public List process(SQSEvent message, Consumer messageHandler) { + public SQSBatchResponse process(SQSEvent message, Consumer messageHandler) { throw new NotImplementedException(); }; - public List process(KinesisEvent message, Consumer messageHandler) { + public StreamsEventResponse process(KinesisEvent message, Consumer messageHandler) { throw new NotImplementedException(); } - public List process(DynamodbEvent message, Consumer messageHandler) { + public StreamsEventResponse process(DynamodbEvent message, Consumer messageHandler) { throw new NotImplementedException(); } diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageProcessor.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageProcessor.java deleted file mode 100644 index 5a8a7edfa..000000000 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageProcessor.java +++ /dev/null @@ -1,18 +0,0 @@ -package software.amazon.lambda.powertools.batch; - -import software.amazon.lambda.powertools.batch.message.BatchProcessorMessageHandler; -import software.amazon.lambda.powertools.batch.message.BatchProcessorMessageType; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -@Retention(RetentionPolicy.RUNTIME) -@Target(ElementType.METHOD) -public @interface BatchMessageProcessor { - Class> value(); - - BatchProcessorMessageType MessageType(); - -} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchRequestHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchRequestHandler.java new file mode 100644 index 000000000..3269b4646 --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchRequestHandler.java @@ -0,0 +1,53 @@ +package software.amazon.lambda.powertools.batch; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.sun.xml.internal.ws.api.message.Message; + +import java.util.List; +import java.util.stream.Collectors; + +public abstract class BatchRequestHandler implements RequestHandler { + + protected class MessageProcessingResult { + private final U message; + private final Exception exception; + + public MessageProcessingResult(U message, Exception exception) { + this.message = message; + this.exception = exception; + } + } + + @Override + public V handleRequest(T input, Context context) { + // Extract messages + List messages = extractMessages(input); + + // Try process them + List> results = messages.stream().map(m -> { + try { + processItem(m, context); + return new MessageProcessingResult<>(m, null); + } catch (Exception e) { + return new MessageProcessingResult<>(m, e); + } + }).collect(Collectors.toList()); + + // Generate the response + return writeResponse(results); + } + + /** + * Provided by the event-specific child to extract the individual records + * from the batch request + * + * @param input + * @return + */ + protected abstract List extractMessages(T input); + + protected abstract V writeResponse(Iterable> results); + + public abstract void processItem(U message, Context context); +} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/KinesisBatchRequestHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/KinesisBatchRequestHandler.java new file mode 100644 index 000000000..65ac7b4c6 --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/KinesisBatchRequestHandler.java @@ -0,0 +1,23 @@ +package software.amazon.lambda.powertools.batch; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.events.KinesisEvent; +import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; +import sun.reflect.generics.reflectiveObjects.NotImplementedException; + +import java.util.List; + +public abstract class KinesisBatchRequestHandler extends BatchRequestHandler { + + @Override + protected List extractMessages(KinesisEvent input) { + return input.getRecords(); + } + + @Override + protected StreamsEventResponse writeResponse(Iterable> results) { + // Here we map up the kinesis-specific response for the batch based on the success of the individual messages + throw new NotImplementedException(); + } + +} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/SQSBatchRequestHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/SQSBatchRequestHandler.java new file mode 100644 index 000000000..92e2f5731 --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/SQSBatchRequestHandler.java @@ -0,0 +1,24 @@ +package software.amazon.lambda.powertools.batch; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.events.KinesisEvent; +import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; +import com.amazonaws.services.lambda.runtime.events.SQSEvent; +import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; +import sun.reflect.generics.reflectiveObjects.NotImplementedException; + +import java.util.List; + +public abstract class SQSBatchRequestHandler extends BatchRequestHandler { + + @Override + protected List extractMessages(SQSEvent input) { + return input.getRecords(); + } + + @Override + protected SQSBatchResponse writeResponse(Iterable> results) { + // Here we map up the SQS-specific response for the batch based on the success of the individual messages + throw new NotImplementedException(); + } +} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/examples/ExampleBatchRequestHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/examples/ExampleBatchRequestHandler.java new file mode 100644 index 000000000..b26da8f71 --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/examples/ExampleBatchRequestHandler.java @@ -0,0 +1,13 @@ +package software.amazon.lambda.powertools.batch.examples; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.events.SQSEvent; +import software.amazon.lambda.powertools.batch.SQSBatchRequestHandler; + +public class ExampleBatchRequestHandler extends SQSBatchRequestHandler { + + @Override + public void processItem(SQSEvent.SQSMessage message, Context context) { + // Process an SQS message without throwing + } +} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/examples/ExampleMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/examples/ExampleMessageHandlerBuilder.java similarity index 55% rename from powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/examples/ExampleMessageHandler.java rename to powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/examples/ExampleMessageHandlerBuilder.java index 0d5fd7383..51bc6a66d 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/examples/ExampleMessageHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/examples/ExampleMessageHandlerBuilder.java @@ -4,22 +4,17 @@ import com.amazonaws.services.lambda.runtime.RequestHandler; import com.amazonaws.services.lambda.runtime.events.SQSEvent; import software.amazon.lambda.powertools.batch.BatchMessageHandlerBuilder; -import software.amazon.lambda.powertools.batch.BatchMessageProcessor; -import software.amazon.lambda.powertools.batch.message.BatchProcessorMessageType; -import software.amazon.lambda.powertools.batch.message.SqsMessage; - -import java.util.List; /** * This is just here for illustrative purposes, and won't * be released with this code! */ -public class ExampleMessageHandler implements RequestHandler> { +public class ExampleMessageHandlerBuilder implements RequestHandler { @Override - public List handleRequest(SQSEvent sqsEvent, Context context) { + public Object handleRequest(SQSEvent sqsEvent, Context context) { return new BatchMessageHandlerBuilder() - .process(sqsEvent, (SqsMessage message) -> { + .process(sqsEvent, (SQSEvent.SQSMessage message) -> { // Process the message without throwing an exception }); } diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/BatchProcessorMessage.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/BatchProcessorMessage.java deleted file mode 100644 index eed62daba..000000000 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/BatchProcessorMessage.java +++ /dev/null @@ -1,18 +0,0 @@ -package software.amazon.lambda.powertools.batch.message; - -/** - * Common base class for all batch message processing message types. - */ -public abstract class BatchProcessorMessage { - - private String eventSourceArn; - private String eventSource; - private String awsRegion; - - // Called messageId or eventId depending on source - private String messageId; - - private String body; - - private String md5OfBody; -} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/BatchProcessorMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/BatchProcessorMessageHandler.java deleted file mode 100644 index e46918d85..000000000 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/BatchProcessorMessageHandler.java +++ /dev/null @@ -1,7 +0,0 @@ -package software.amazon.lambda.powertools.batch.message; - -@FunctionalInterface -public interface BatchProcessorMessageHandler { - - R process(BatchProcessorMessage message); -} \ No newline at end of file diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/BatchProcessorMessageType.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/BatchProcessorMessageType.java deleted file mode 100644 index f4c6bd6f1..000000000 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/BatchProcessorMessageType.java +++ /dev/null @@ -1,7 +0,0 @@ -package software.amazon.lambda.powertools.batch.message; - -public enum BatchProcessorMessageType { - Sqs, - DynamoDbStreams, - KinesisDataStreams -} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/DynamoDbStreamMessage.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/DynamoDbStreamMessage.java deleted file mode 100644 index 364f6f95b..000000000 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/DynamoDbStreamMessage.java +++ /dev/null @@ -1,5 +0,0 @@ -package software.amazon.lambda.powertools.batch.message; - -public class DynamoDbStreamMessage extends BatchProcessorMessage { - -} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/KinesisDataStreamsMessage.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/KinesisDataStreamsMessage.java deleted file mode 100644 index 46fa8c40b..000000000 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/KinesisDataStreamsMessage.java +++ /dev/null @@ -1,15 +0,0 @@ -package software.amazon.lambda.powertools.batch.message; - -/** - * A message from a kinesis data streams batch. - * - * @see Java API Documentation - */ -public class KinesisDataStreamsMessage extends BatchProcessorMessage { - private String eventVersion; - private String invokeIdentityArn; - private String kinesisSchemaVersion; - private String partitionKey; - private String sequenceNumber; - private float approximateArrivalTimestamp; -} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/SqsMessage.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/SqsMessage.java deleted file mode 100644 index 89bd69d94..000000000 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/message/SqsMessage.java +++ /dev/null @@ -1,14 +0,0 @@ -package software.amazon.lambda.powertools.batch.message; - -import java.util.Map; -import com.amazonaws.services.lambda.runtime.events.SQSEvent.MessageAttribute; - -public class SqsMessage extends BatchProcessorMessage { - - private String receiptHandle; - private String body; - private String md5OfBody; - private String md5OfMessageAttributes; - private Map attributes; - private Map messageAttributes; -} From 54692871c706c5dd36b575dd80a0eab6687234d9 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Sun, 25 Jun 2023 15:09:04 +0200 Subject: [PATCH 03/79] Some more examples --- .../batch/BatchMessageHandlerBuilder.java | 7 ++- .../powertools/batch/BatchRequestHandler.java | 47 +++++++++++++++++-- .../examples/ExampleBatchRequestHandler.java | 4 +- .../ExampleMessageHandlerBuilder.java | 4 +- .../powertools/batch2/BatchProcessor.java | 47 +++++++++++++++++++ .../batch2/ProcessSQSMessageBatch.java | 29 ++++++++++++ .../examples/ExampleBatchRequestHandler.java | 13 +++++ 7 files changed, 141 insertions(+), 10 deletions(-) create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/BatchProcessor.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/ProcessSQSMessageBatch.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/ExampleBatchRequestHandler.java diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java index 193d1260a..83825fae1 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java @@ -7,9 +7,12 @@ import java.util.List; import java.util.function.Consumer; + /** - * Let's make this builder-style so that we can extend the interface - * if we need to with extra fields without breaking clients + * A builder-style interface we can use within an existing Lambda RequestHandler to + * deal with our batch responses. + * + * @see software.amazon.lambda.powertools.batch.examples.ExampleMessageHandlerBuilder */ public class BatchMessageHandlerBuilder { diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchRequestHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchRequestHandler.java index 3269b4646..530b41791 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchRequestHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchRequestHandler.java @@ -7,8 +7,26 @@ import java.util.List; import java.util.stream.Collectors; +/** + * An abstract base class that will be extended per-event-source to supply event-source + * specific functionality. As much as possible is pulled into the base class, and + * event-source specific logic is delegated downwards (e.g., mapping errors to response, + * extracting messages from batch). + * + * IRL, this would be implemented leaning on the {@link BatchMessageHandlerBuilder}, here + * I have provided implementation inline for illustrative purposes. + * + * @param The batch event type + * @param The individual message type for each message within teh batch + * @param The batch result type + */ public abstract class BatchRequestHandler implements RequestHandler { + /** + * Used to wrap the result of processing a single message. We wrap the message itself, + * and optionally an exception that was raised during the processing. A lack of + * exception indicates success. + */ protected class MessageProcessingResult { private final U message; private final Exception exception; @@ -19,12 +37,19 @@ public MessageProcessingResult(U message, Exception exception) { } } + /** + * The batch processing logic goes here. This can be generic across all message types. + * + * @param input The batch message to process + * @param context The Lambda execution environment context object. + * @return + */ @Override public V handleRequest(T input, Context context) { - // Extract messages + // Extract messages from batch List messages = extractMessages(input); - // Try process them + // For each message, map it to either 1/ a successful result, or 2/ an exception List> results = messages.stream().map(m -> { try { processItem(m, context); @@ -34,7 +59,7 @@ public V handleRequest(T input, Context context) { } }).collect(Collectors.toList()); - // Generate the response + // Generate the response from the list of results return writeResponse(results); } @@ -42,12 +67,24 @@ public V handleRequest(T input, Context context) { * Provided by the event-specific child to extract the individual records * from the batch request * - * @param input - * @return + * @param input The batch + * @return the messages within the batch */ protected abstract List extractMessages(T input); + /** + * Given the set of message processing results, generates the appropriate batch + * processing result to return to Lambda. + * + * @param results the result of processing each message, and the messages themselves + * @return the batch processing result to return to lambda + */ protected abstract V writeResponse(Iterable> results); + /** + * To be provided by the user. Processes an individual message within the batch + * @param message + * @param context + */ public abstract void processItem(U message, Context context); } diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/examples/ExampleBatchRequestHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/examples/ExampleBatchRequestHandler.java index b26da8f71..9a1e74bf8 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/examples/ExampleBatchRequestHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/examples/ExampleBatchRequestHandler.java @@ -4,8 +4,10 @@ import com.amazonaws.services.lambda.runtime.events.SQSEvent; import software.amazon.lambda.powertools.batch.SQSBatchRequestHandler; +/** + * An example handler that is implemented by extending our {@link SQSBatchRequestHandler} + */ public class ExampleBatchRequestHandler extends SQSBatchRequestHandler { - @Override public void processItem(SQSEvent.SQSMessage message, Context context) { // Process an SQS message without throwing diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/examples/ExampleMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/examples/ExampleMessageHandlerBuilder.java index 51bc6a66d..f22bad9b5 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/examples/ExampleMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/examples/ExampleMessageHandlerBuilder.java @@ -6,8 +6,8 @@ import software.amazon.lambda.powertools.batch.BatchMessageHandlerBuilder; /** - * This is just here for illustrative purposes, and won't - * be released with this code! + * An example handler that is implemented by using the {@link BatchMessageHandlerBuilder} inline + * in an existing RequestHandler. */ public class ExampleMessageHandlerBuilder implements RequestHandler { diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/BatchProcessor.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/BatchProcessor.java new file mode 100644 index 000000000..b3a800475 --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/BatchProcessor.java @@ -0,0 +1,47 @@ +package software.amazon.lambda.powertools.batch2; + +import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; +import com.amazonaws.services.lambda.runtime.events.KinesisEvent; +import com.amazonaws.services.lambda.runtime.events.SQSEvent; +import sun.reflect.generics.reflectiveObjects.NotImplementedException; + +public interface BatchProcessor { + + default O processBatch(I input) { + // depending on the input type (SQS/Kinesis/Dynamo/...), create the appropriate response + + // TODO - need a way of extracting items from input. Defer to interface? + + // browse the list of items + // for each item + try { + // Need to typematch to get the processItem calls to bind + if (input instanceof SQSEvent.SQSMessage) { + processItem((SQSEvent.SQSMessage) input); + } else if (input instanceof KinesisEvent.KinesisEventRecord) { + processItem((KinesisEvent.KinesisEventRecord) input); + } else if (input instanceof DynamodbEvent.DynamodbStreamRecord) { + processItem((DynamodbEvent.DynamodbStreamRecord) input); + } + } + catch(Throwable t) { + // put item in item failure list + } + + // TODO - need a way of converting resultset back to return type. Defer to interface? + + throw new NotImplementedException(); + } + + default void processItem(SQSEvent.SQSMessage message) { + System.out.println(message.getMessageId()); + } + + default void processItem(KinesisEvent.KinesisEventRecord record) { + System.out.println(record.getEventID()); + } + + default void processItem(DynamodbEvent.DynamodbStreamRecord record) { + System.out.println(record.getEventID()); + } +} \ No newline at end of file diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/ProcessSQSMessageBatch.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/ProcessSQSMessageBatch.java new file mode 100644 index 000000000..0c03977ae --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/ProcessSQSMessageBatch.java @@ -0,0 +1,29 @@ +package software.amazon.lambda.powertools.batch2; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; +import com.amazonaws.services.lambda.runtime.events.SQSEvent; + +public class ProcessSQSMessageBatch implements RequestHandler, BatchProcessor { + + @Override + public SQSBatchResponse handleRequest(SQSEvent sqsEvent, Context context) { + // processBatch is implemented as default in the interface and handle exceptions in the processElement function to add the item to the BatchItemFailure list + + // TODO - as we're getting this on the outside, we could pass in `sqsEvent.getMessages()` which would + // fix the "how do I extract event-specific message" problem in the interface default. + return this.processBatch(sqsEvent); // we may need to pass context too... ? + + // TODO - as we're still on the outside, we could take the list of results from `this.processBatch()` and + // map it back to the SQS-specific response, again solving the "how do I do this event-specific thing" + // problem in the base class. + } + + // this method comes from the BatchProcessor interface, developers need to override the appropriate one + @Override + public void processItem(SQSEvent.SQSMessage message) { + // do some stuff with this item + } + +} \ No newline at end of file diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/ExampleBatchRequestHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/ExampleBatchRequestHandler.java new file mode 100644 index 000000000..be7a10a84 --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/ExampleBatchRequestHandler.java @@ -0,0 +1,13 @@ +package software.amazon.lambda.powertools.batch2.examples; + +import com.amazonaws.services.lambda.runtime.events.SQSEvent; +import software.amazon.lambda.powertools.batch2.ProcessSQSMessageBatch; + +public class ExampleBatchRequestHandler extends ProcessSQSMessageBatch { + + // this method comes from the BatchProcessor interface, developers need to override the appropriate one + @Override + public void processItem(SQSEvent.SQSMessage message) { + // do some stuff with this item + } +} From 0435d188bf1381574d6c45126cac53cc44732a74 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Sun, 25 Jun 2023 15:11:50 +0200 Subject: [PATCH 04/79] Add extra bit for handling message-specific mutation --- .../lambda/powertools/batch/BatchRequestHandler.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchRequestHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchRequestHandler.java index 530b41791..1acd070bd 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchRequestHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchRequestHandler.java @@ -52,6 +52,7 @@ public V handleRequest(T input, Context context) { // For each message, map it to either 1/ a successful result, or 2/ an exception List> results = messages.stream().map(m -> { try { + enhanceMessage(m); processItem(m, context); return new MessageProcessingResult<>(m, null); } catch (Exception e) { @@ -87,4 +88,13 @@ public V handleRequest(T input, Context context) { * @param context */ public abstract void processItem(U message, Context context); + + /** + * This could be overriden by event-specific children to implement things like large + * message processing. + * @param message + */ + protected void enhanceMessage(U message) { + + } } From 6241014095e45801d23501398868e76d5ad01245 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Sun, 25 Jun 2023 18:45:52 +0200 Subject: [PATCH 05/79] Make clear what's not public --- .../amazon/lambda/powertools/batch/BatchRequestHandler.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchRequestHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchRequestHandler.java index 1acd070bd..9d74517ae 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchRequestHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchRequestHandler.java @@ -16,11 +16,15 @@ * IRL, this would be implemented leaning on the {@link BatchMessageHandlerBuilder}, here * I have provided implementation inline for illustrative purposes. * + * Note - this is package private. We don't let users extend this, which gives us + * much more flexibility to add extra functionality later by minimising our API surface + * area. + * * @param The batch event type * @param The individual message type for each message within teh batch * @param The batch result type */ -public abstract class BatchRequestHandler implements RequestHandler { +abstract class BatchRequestHandler implements RequestHandler { /** * Used to wrap the result of processing a single message. We wrap the message itself, From 8168719022856c59c39a3f6991c8d6155392e9a2 Mon Sep 17 00:00:00 2001 From: Jerome Van Der Linden Date: Wed, 28 Jun 2023 15:18:09 +0200 Subject: [PATCH 06/79] test with interfaces --- powertools-batch/pom.xml | 32 ++++++++ .../powertools/batch2/BatchProcessor.java | 43 +--------- .../batch2/DynamoDBBatchProcessor.java | 29 +++++++ .../batch2/KinesisBatchProcessor.java | 34 ++++++++ .../batch2/ProcessSQSMessageBatch.java | 29 ------- .../powertools/batch2/SQSBatchProcessor.java | 34 ++++++++ .../batch2/examples/DynamoDBBatchHandler.java | 19 +++++ .../examples/ExampleBatchRequestHandler.java | 13 --- .../batch2/examples/KinesisBatchHandler.java | 21 +++++ .../batch2/examples/SQSBatchHandler.java | 21 +++++ .../batch2/examples/model/Basket.java | 62 +++++++++++++++ .../batch2/examples/model/Product.java | 79 +++++++++++++++++++ .../batch2/SQSBatchProcessorTest.java | 38 +++++++++ .../src/main/test/resources/sqs_event.json | 40 ++++++++++ .../utilities/EventDeserializer.java | 20 ++++- 15 files changed, 432 insertions(+), 82 deletions(-) create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/DynamoDBBatchProcessor.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/KinesisBatchProcessor.java delete mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/ProcessSQSMessageBatch.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessor.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/DynamoDBBatchHandler.java delete mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/ExampleBatchRequestHandler.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/KinesisBatchHandler.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/SQSBatchHandler.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/model/Basket.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/model/Product.java create mode 100644 powertools-batch/src/main/test/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessorTest.java create mode 100644 powertools-batch/src/main/test/resources/sqs_event.json diff --git a/powertools-batch/pom.xml b/powertools-batch/pom.xml index 569558756..c8f952b38 100644 --- a/powertools-batch/pom.xml +++ b/powertools-batch/pom.xml @@ -19,6 +19,38 @@ com.amazonaws aws-lambda-java-core + + software.amazon.lambda + powertools-serialization + ${project.version} + + + + + org.junit.jupiter + junit-jupiter-api + test + + + org.assertj + assertj-core + test + + + com.amazonaws + aws-lambda-java-tests + test + + + org.mockito + mockito-core + test + + + org.mockito + mockito-inline + test + \ No newline at end of file diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/BatchProcessor.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/BatchProcessor.java index b3a800475..1ab55a3fe 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/BatchProcessor.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/BatchProcessor.java @@ -1,47 +1,12 @@ package software.amazon.lambda.powertools.batch2; -import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; -import com.amazonaws.services.lambda.runtime.events.KinesisEvent; -import com.amazonaws.services.lambda.runtime.events.SQSEvent; -import sun.reflect.generics.reflectiveObjects.NotImplementedException; -public interface BatchProcessor { +import com.amazonaws.services.lambda.runtime.Context; - default O processBatch(I input) { - // depending on the input type (SQS/Kinesis/Dynamo/...), create the appropriate response +public interface BatchProcessor { - // TODO - need a way of extracting items from input. Defer to interface? + RESPONSE processBatch(EVENT event, Context context); - // browse the list of items - // for each item - try { - // Need to typematch to get the processItem calls to bind - if (input instanceof SQSEvent.SQSMessage) { - processItem((SQSEvent.SQSMessage) input); - } else if (input instanceof KinesisEvent.KinesisEventRecord) { - processItem((KinesisEvent.KinesisEventRecord) input); - } else if (input instanceof DynamodbEvent.DynamodbStreamRecord) { - processItem((DynamodbEvent.DynamodbStreamRecord) input); - } - } - catch(Throwable t) { - // put item in item failure list - } + void processItem(ITEM item, Context context); - // TODO - need a way of converting resultset back to return type. Defer to interface? - - throw new NotImplementedException(); - } - - default void processItem(SQSEvent.SQSMessage message) { - System.out.println(message.getMessageId()); - } - - default void processItem(KinesisEvent.KinesisEventRecord record) { - System.out.println(record.getEventID()); - } - - default void processItem(DynamodbEvent.DynamodbStreamRecord record) { - System.out.println(record.getEventID()); - } } \ No newline at end of file diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/DynamoDBBatchProcessor.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/DynamoDBBatchProcessor.java new file mode 100644 index 000000000..aff5c3a60 --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/DynamoDBBatchProcessor.java @@ -0,0 +1,29 @@ +package software.amazon.lambda.powertools.batch2; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; +import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; + +public interface DynamoDBBatchProcessor extends BatchProcessor { + + @Override + default StreamsEventResponse processBatch(DynamodbEvent event, Context context) { + StreamsEventResponse response = new StreamsEventResponse(); + for (DynamodbEvent.DynamodbStreamRecord record : event.getRecords()) { + try { + processItem(record, context); + } catch (Throwable t) { + response.getBatchItemFailures().add(new StreamsEventResponse.BatchItemFailure(record.getEventID())); + } + } + return response; + } + + @Override + default void processItem(Void message, Context context) { + System.out.println("Nothing to do here"); + } + + void processItem(DynamodbEvent.DynamodbStreamRecord record, Context context); + +} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/KinesisBatchProcessor.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/KinesisBatchProcessor.java new file mode 100644 index 000000000..3417d6560 --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/KinesisBatchProcessor.java @@ -0,0 +1,34 @@ +package software.amazon.lambda.powertools.batch2; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.events.KinesisEvent; +import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; +import software.amazon.lambda.powertools.utilities.EventDeserializer; +import sun.reflect.generics.reflectiveObjects.ParameterizedTypeImpl; + +public interface KinesisBatchProcessor extends BatchProcessor { + + @Override + default StreamsEventResponse processBatch(KinesisEvent event, Context context) { + Class bodyClass = (Class) ((ParameterizedTypeImpl) getClass().getGenericInterfaces()[0]).getActualTypeArguments()[0]; + + StreamsEventResponse response = new StreamsEventResponse(); + for (KinesisEvent.KinesisEventRecord record : event.getRecords()) { + try { + if (bodyClass.equals(KinesisEvent.KinesisEventRecord.class)) { + processItem(record, context); + } else { + processItem(EventDeserializer.extractDataFrom(record).as(bodyClass), context); + } + } catch (Throwable t) { + response.getBatchItemFailures().add(new StreamsEventResponse.BatchItemFailure(record.getEventID())); + } + } + return response; + } + + default void processItem(KinesisEvent.KinesisEventRecord record, Context context) { + System.out.println("Processing message " + record.getEventID()); + } + +} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/ProcessSQSMessageBatch.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/ProcessSQSMessageBatch.java deleted file mode 100644 index 0c03977ae..000000000 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/ProcessSQSMessageBatch.java +++ /dev/null @@ -1,29 +0,0 @@ -package software.amazon.lambda.powertools.batch2; - -import com.amazonaws.services.lambda.runtime.Context; -import com.amazonaws.services.lambda.runtime.RequestHandler; -import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; -import com.amazonaws.services.lambda.runtime.events.SQSEvent; - -public class ProcessSQSMessageBatch implements RequestHandler, BatchProcessor { - - @Override - public SQSBatchResponse handleRequest(SQSEvent sqsEvent, Context context) { - // processBatch is implemented as default in the interface and handle exceptions in the processElement function to add the item to the BatchItemFailure list - - // TODO - as we're getting this on the outside, we could pass in `sqsEvent.getMessages()` which would - // fix the "how do I extract event-specific message" problem in the interface default. - return this.processBatch(sqsEvent); // we may need to pass context too... ? - - // TODO - as we're still on the outside, we could take the list of results from `this.processBatch()` and - // map it back to the SQS-specific response, again solving the "how do I do this event-specific thing" - // problem in the base class. - } - - // this method comes from the BatchProcessor interface, developers need to override the appropriate one - @Override - public void processItem(SQSEvent.SQSMessage message) { - // do some stuff with this item - } - -} \ No newline at end of file diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessor.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessor.java new file mode 100644 index 000000000..66e15ff5d --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessor.java @@ -0,0 +1,34 @@ +package software.amazon.lambda.powertools.batch2; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; +import com.amazonaws.services.lambda.runtime.events.SQSEvent; +import software.amazon.lambda.powertools.utilities.EventDeserializer; +import sun.reflect.generics.reflectiveObjects.ParameterizedTypeImpl; + +public interface SQSBatchProcessor extends BatchProcessor { + + @Override + default SQSBatchResponse processBatch(SQSEvent event, Context context) { + Class bodyClass = (Class) ((ParameterizedTypeImpl) getClass().getGenericInterfaces()[0]).getActualTypeArguments()[0]; + + SQSBatchResponse response = new SQSBatchResponse(); + for (SQSEvent.SQSMessage message : event.getRecords()) { + try { + if (bodyClass.equals(SQSEvent.SQSMessage.class)) { + processItem(message, context); + } else { + processItem(EventDeserializer.extractDataFrom(message).as(bodyClass), context); + } + } catch (Throwable t) { + response.getBatchItemFailures().add(new SQSBatchResponse.BatchItemFailure(message.getMessageId())); + } + } + return response; + } + + default void processItem(SQSEvent.SQSMessage message, Context context) { + System.out.println("Processing message " + message.getMessageId()); + } + +} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/DynamoDBBatchHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/DynamoDBBatchHandler.java new file mode 100644 index 000000000..7c4d35673 --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/DynamoDBBatchHandler.java @@ -0,0 +1,19 @@ +package software.amazon.lambda.powertools.batch2.examples; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; +import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; +import software.amazon.lambda.powertools.batch2.DynamoDBBatchProcessor; + +public class DynamoDBBatchHandler implements RequestHandler, DynamoDBBatchProcessor { + @Override + public StreamsEventResponse handleRequest(DynamodbEvent input, Context context) { + return processBatch(input, context); + } + + @Override + public void processItem(DynamodbEvent.DynamodbStreamRecord record, Context context) { + + } +} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/ExampleBatchRequestHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/ExampleBatchRequestHandler.java deleted file mode 100644 index be7a10a84..000000000 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/ExampleBatchRequestHandler.java +++ /dev/null @@ -1,13 +0,0 @@ -package software.amazon.lambda.powertools.batch2.examples; - -import com.amazonaws.services.lambda.runtime.events.SQSEvent; -import software.amazon.lambda.powertools.batch2.ProcessSQSMessageBatch; - -public class ExampleBatchRequestHandler extends ProcessSQSMessageBatch { - - // this method comes from the BatchProcessor interface, developers need to override the appropriate one - @Override - public void processItem(SQSEvent.SQSMessage message) { - // do some stuff with this item - } -} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/KinesisBatchHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/KinesisBatchHandler.java new file mode 100644 index 000000000..2c4c5824c --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/KinesisBatchHandler.java @@ -0,0 +1,21 @@ +package software.amazon.lambda.powertools.batch2.examples; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.amazonaws.services.lambda.runtime.events.KinesisEvent; +import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; +import software.amazon.lambda.powertools.batch2.KinesisBatchProcessor; +import software.amazon.lambda.powertools.batch2.examples.model.Basket; + +public class KinesisBatchHandler implements RequestHandler, KinesisBatchProcessor { + @Override + public StreamsEventResponse handleRequest(KinesisEvent input, Context context) { + return processBatch(input, context); + } + + @Override + public void processItem(Basket basket, Context context) { + // do some stuff with the item + System.out.println(basket.toString()); + } +} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/SQSBatchHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/SQSBatchHandler.java new file mode 100644 index 000000000..9738f40ae --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/SQSBatchHandler.java @@ -0,0 +1,21 @@ +package software.amazon.lambda.powertools.batch2.examples; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; +import com.amazonaws.services.lambda.runtime.events.SQSEvent; +import software.amazon.lambda.powertools.batch2.SQSBatchProcessor; +import software.amazon.lambda.powertools.batch2.examples.model.Basket; + +public class SQSBatchHandler implements RequestHandler, SQSBatchProcessor { + @Override + public SQSBatchResponse handleRequest(SQSEvent input, Context context) { + return processBatch(input, context); + } + + @Override + public void processItem(Basket basket, Context context) { + // do some stuff with the item + System.out.println(basket.toString()); + } +} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/model/Basket.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/model/Basket.java new file mode 100644 index 000000000..d9c387aca --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/model/Basket.java @@ -0,0 +1,62 @@ +/* + * Copyright 2022 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package software.amazon.lambda.powertools.batch2.examples.model; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +public class Basket { + private List products = new ArrayList<>(); + + public List getProducts() { + return products; + } + + public void setProducts(List products) { + this.products = products; + } + + public Basket() { + } + + public Basket( Product ...p){ + products.addAll(Arrays.asList(p)); + } + + public void add(Product product) { + products.add(product); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Basket basket = (Basket) o; + return products.equals(basket.products); + } + + @Override + public String toString() { + return "Basket{" + + "products=" + products + + '}'; + } + + @Override + public int hashCode() { + return Objects.hash(products); + } +} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/model/Product.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/model/Product.java new file mode 100644 index 000000000..0311dca66 --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/model/Product.java @@ -0,0 +1,79 @@ +/* + * Copyright 2022 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package software.amazon.lambda.powertools.batch2.examples.model; + +import java.util.Objects; + +public class Product { + private long id; + + private String name; + + private double price; + + public Product() { + } + + public Product(long id, String name, double price) { + this.id = id; + this.name = name; + this.price = price; + } + + public long getId() { + return id; + } + + public void setId(long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public double getPrice() { + return price; + } + + public void setPrice(double price) { + this.price = price; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Product product = (Product) o; + return id == product.id && Double.compare(product.price, price) == 0 && Objects.equals(name, product.name); + } + + @Override + public int hashCode() { + return Objects.hash(id, name, price); + } + + @Override + public String toString() { + return "Product{" + + "id=" + id + + ", name='" + name + '\'' + + ", price=" + price + + '}'; + } +} diff --git a/powertools-batch/src/main/test/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessorTest.java b/powertools-batch/src/main/test/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessorTest.java new file mode 100644 index 000000000..9615a7860 --- /dev/null +++ b/powertools-batch/src/main/test/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessorTest.java @@ -0,0 +1,38 @@ +package software.amazon.lambda.powertools.batch2; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.events.SQSEvent; +import com.amazonaws.services.lambda.runtime.tests.annotations.Event; +import org.junit.jupiter.params.ParameterizedTest; +import org.mockito.Mock; +import software.amazon.lambda.powertools.batch2.examples.model.Product; + +public class SQSBatchProcessorTest { + @Mock + private Context context; + static class SQSBProduct implements SQSBatchProcessor { + public void processItem(Product product, Context context) { + System.out.println(product); + } + } + + static class SQSBP implements SQSBatchProcessor { + public void processItem(SQSEvent.SQSMessage message, Context context) { + System.out.println(message.toString()); + } + } + + @ParameterizedTest + @Event(value = "sqs_event.json", type = SQSEvent.class) + public void testBasic(SQSEvent event) { + SQSBP sqsbp = new SQSBP(); + sqsbp.processBatch(event, context); + } + + @ParameterizedTest + @Event(value = "sqs_event.json", type = SQSEvent.class) + public void testCustom(SQSEvent event) { + SQSBProduct sqsbproduct = new SQSBProduct(); + sqsbproduct.processBatch(event, context); + } +} diff --git a/powertools-batch/src/main/test/resources/sqs_event.json b/powertools-batch/src/main/test/resources/sqs_event.json new file mode 100644 index 000000000..d33db4b53 --- /dev/null +++ b/powertools-batch/src/main/test/resources/sqs_event.json @@ -0,0 +1,40 @@ +{ + "Records": [ + { + "messageId": "d9144555-9a4f-4ec3-99a0-34ce359b4b54", + "receiptHandle": "13e7f7851d2eaa5c01f208ebadbf1e72==", + "body": "{\n \"id\": 1234,\n \"name\": \"product\",\n \"price\": 42\n}", + "attributes": { + "ApproximateReceiveCount": "1", + "SentTimestamp": "1601975706495", + "SenderId": "AROAIFU437PVZ5L2J53F5", + "ApproximateFirstReceiveTimestamp": "1601975706499" + }, + "messageAttributes": { + + }, + "md5OfBody": "13e7f7851d2eaa5c01f208ebadbf1e72", + "eventSource": "aws:sqs", + "eventSourceARN": "arn:aws:sqs:eu-central-1:123456789012:TestLambda", + "awsRegion": "eu-central-1" + }, + { + "messageId": "d9144555-9a4f-4ec3-99a0-34ce359b4b54", + "receiptHandle": "13e7f7851d2eaa5c01f208ebadbf1e72==", + "body": "{\n \"id\": 12345,\n \"name\": \"product5\",\n \"price\": 45\n}", + "attributes": { + "ApproximateReceiveCount": "1", + "SentTimestamp": "1601975706495", + "SenderId": "AROAIFU437PVZ5L2J53F5", + "ApproximateFirstReceiveTimestamp": "1601975706499" + }, + "messageAttributes": { + + }, + "md5OfBody": "13e7f7851d2eaa5c01f208ebadbf1e72", + "eventSource": "aws:sqs", + "eventSourceARN": "arn:aws:sqs:eu-central-1:123456789012:TestLambda", + "awsRegion": "eu-central-1" + } + ] +} \ No newline at end of file diff --git a/powertools-serialization/src/main/java/software/amazon/lambda/powertools/utilities/EventDeserializer.java b/powertools-serialization/src/main/java/software/amazon/lambda/powertools/utilities/EventDeserializer.java index 9742299ee..2a05f26c6 100644 --- a/powertools-serialization/src/main/java/software/amazon/lambda/powertools/utilities/EventDeserializer.java +++ b/powertools-serialization/src/main/java/software/amazon/lambda/powertools/utilities/EventDeserializer.java @@ -13,7 +13,21 @@ */ package software.amazon.lambda.powertools.utilities; -import com.amazonaws.services.lambda.runtime.events.*; +import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyRequestEvent; +import com.amazonaws.services.lambda.runtime.events.APIGatewayV2HTTPEvent; +import com.amazonaws.services.lambda.runtime.events.ActiveMQEvent; +import com.amazonaws.services.lambda.runtime.events.ApplicationLoadBalancerRequestEvent; +import com.amazonaws.services.lambda.runtime.events.CloudFormationCustomResourceEvent; +import com.amazonaws.services.lambda.runtime.events.CloudWatchLogsEvent; +import com.amazonaws.services.lambda.runtime.events.KafkaEvent; +import com.amazonaws.services.lambda.runtime.events.KinesisAnalyticsFirehoseInputPreprocessingEvent; +import com.amazonaws.services.lambda.runtime.events.KinesisAnalyticsStreamsInputPreprocessingEvent; +import com.amazonaws.services.lambda.runtime.events.KinesisEvent; +import com.amazonaws.services.lambda.runtime.events.KinesisFirehoseEvent; +import com.amazonaws.services.lambda.runtime.events.RabbitMQEvent; +import com.amazonaws.services.lambda.runtime.events.SNSEvent; +import com.amazonaws.services.lambda.runtime.events.SQSEvent; +import com.amazonaws.services.lambda.runtime.events.ScheduledEvent; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectReader; import org.slf4j.Logger; @@ -82,6 +96,8 @@ public static EventPart extractDataFrom(Object object) { return new EventPart(event.getRecords().stream() .map(SQSEvent.SQSMessage::getBody) .collect(Collectors.toList())); + } else if (object instanceof SQSEvent.SQSMessage) { + return new EventPart(((SQSEvent.SQSMessage) object).getBody()); } else if (object instanceof ScheduledEvent) { ScheduledEvent event = (ScheduledEvent) object; return new EventPart(event.getDetail()); @@ -99,6 +115,8 @@ public static EventPart extractDataFrom(Object object) { return new EventPart(event.getRecords().stream() .map(r -> decode(r.getKinesis().getData())) .collect(Collectors.toList())); + } else if (object instanceof KinesisEvent.KinesisEventRecord) { + return new EventPart(decode(((KinesisEvent.KinesisEventRecord)object).getKinesis().getData())); } else if (object instanceof KinesisFirehoseEvent) { KinesisFirehoseEvent event = (KinesisFirehoseEvent) object; return new EventPart(event.getRecords().stream() From b0675b78819824be627baf176eb9790698142ad9 Mon Sep 17 00:00:00 2001 From: Jerome Van Der Linden Date: Wed, 28 Jun 2023 15:20:52 +0200 Subject: [PATCH 07/79] move tests --- .../amazon/lambda/powertools/batch2/SQSBatchProcessorTest.java | 0 powertools-batch/src/{main => }/test/resources/sqs_event.json | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename powertools-batch/src/{main => }/test/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessorTest.java (100%) rename powertools-batch/src/{main => }/test/resources/sqs_event.json (100%) diff --git a/powertools-batch/src/main/test/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessorTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessorTest.java similarity index 100% rename from powertools-batch/src/main/test/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessorTest.java rename to powertools-batch/src/test/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessorTest.java diff --git a/powertools-batch/src/main/test/resources/sqs_event.json b/powertools-batch/src/test/resources/sqs_event.json similarity index 100% rename from powertools-batch/src/main/test/resources/sqs_event.json rename to powertools-batch/src/test/resources/sqs_event.json From 5ae16f987eabcdb5c227a991d4891b0f1529eba7 Mon Sep 17 00:00:00 2001 From: Jerome Van Der Linden Date: Thu, 29 Jun 2023 09:56:55 +0200 Subject: [PATCH 08/79] refactoring a bit --- powertools-batch/pom.xml | 5 +++++ .../powertools/batch2/BatchProcessor.java | 4 +++- .../batch2/DynamoDBBatchProcessor.java | 10 +++++---- .../batch2/KinesisBatchProcessor.java | 8 ++++--- .../powertools/batch2/SQSBatchProcessor.java | 10 +++++---- .../batch2/examples/DynamoDBBatchHandler.java | 2 +- .../batch2/SQSBatchProcessorTest.java | 22 +++++++++++-------- 7 files changed, 39 insertions(+), 22 deletions(-) diff --git a/powertools-batch/pom.xml b/powertools-batch/pom.xml index c8f952b38..c165064f1 100644 --- a/powertools-batch/pom.xml +++ b/powertools-batch/pom.xml @@ -24,6 +24,11 @@ powertools-serialization ${project.version} + + org.aspectj + aspectjrt + compile + diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/BatchProcessor.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/BatchProcessor.java index 1ab55a3fe..6a71ff002 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/BatchProcessor.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/BatchProcessor.java @@ -7,6 +7,8 @@ public interface BatchProcessor { RESPONSE processBatch(EVENT event, Context context); - void processItem(ITEM item, Context context); + default void processItem(ITEM item, Context context) { + System.out.println("Processing custom item"); + } } \ No newline at end of file diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/DynamoDBBatchProcessor.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/DynamoDBBatchProcessor.java index aff5c3a60..00a756eb2 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/DynamoDBBatchProcessor.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/DynamoDBBatchProcessor.java @@ -4,14 +4,16 @@ import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; +import java.util.ArrayList; + public interface DynamoDBBatchProcessor extends BatchProcessor { @Override default StreamsEventResponse processBatch(DynamodbEvent event, Context context) { - StreamsEventResponse response = new StreamsEventResponse(); + StreamsEventResponse response = StreamsEventResponse.builder().withBatchItemFailures(new ArrayList<>()).build(); for (DynamodbEvent.DynamodbStreamRecord record : event.getRecords()) { try { - processItem(record, context); + processRecord(record, context); } catch (Throwable t) { response.getBatchItemFailures().add(new StreamsEventResponse.BatchItemFailure(record.getEventID())); } @@ -21,9 +23,9 @@ default StreamsEventResponse processBatch(DynamodbEvent event, Context context) @Override default void processItem(Void message, Context context) { - System.out.println("Nothing to do here"); + System.out.println("This method is not used in the case of DynamoDB"); } - void processItem(DynamodbEvent.DynamodbStreamRecord record, Context context); + void processRecord(DynamodbEvent.DynamodbStreamRecord record, Context context); } diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/KinesisBatchProcessor.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/KinesisBatchProcessor.java index 3417d6560..fd364e09d 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/KinesisBatchProcessor.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/KinesisBatchProcessor.java @@ -6,17 +6,19 @@ import software.amazon.lambda.powertools.utilities.EventDeserializer; import sun.reflect.generics.reflectiveObjects.ParameterizedTypeImpl; +import java.util.ArrayList; + public interface KinesisBatchProcessor extends BatchProcessor { @Override default StreamsEventResponse processBatch(KinesisEvent event, Context context) { Class bodyClass = (Class) ((ParameterizedTypeImpl) getClass().getGenericInterfaces()[0]).getActualTypeArguments()[0]; - StreamsEventResponse response = new StreamsEventResponse(); + StreamsEventResponse response = StreamsEventResponse.builder().withBatchItemFailures(new ArrayList<>()).build(); for (KinesisEvent.KinesisEventRecord record : event.getRecords()) { try { if (bodyClass.equals(KinesisEvent.KinesisEventRecord.class)) { - processItem(record, context); + processRecord(record, context); } else { processItem(EventDeserializer.extractDataFrom(record).as(bodyClass), context); } @@ -27,7 +29,7 @@ default StreamsEventResponse processBatch(KinesisEvent event, Context context) { return response; } - default void processItem(KinesisEvent.KinesisEventRecord record, Context context) { + default void processRecord(KinesisEvent.KinesisEventRecord record, Context context) { System.out.println("Processing message " + record.getEventID()); } diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessor.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessor.java index 66e15ff5d..3d08933b9 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessor.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessor.java @@ -6,17 +6,19 @@ import software.amazon.lambda.powertools.utilities.EventDeserializer; import sun.reflect.generics.reflectiveObjects.ParameterizedTypeImpl; +import java.util.ArrayList; + public interface SQSBatchProcessor extends BatchProcessor { @Override default SQSBatchResponse processBatch(SQSEvent event, Context context) { Class bodyClass = (Class) ((ParameterizedTypeImpl) getClass().getGenericInterfaces()[0]).getActualTypeArguments()[0]; - SQSBatchResponse response = new SQSBatchResponse(); + SQSBatchResponse response = SQSBatchResponse.builder().withBatchItemFailures(new ArrayList<>()).build(); for (SQSEvent.SQSMessage message : event.getRecords()) { try { - if (bodyClass.equals(SQSEvent.SQSMessage.class)) { - processItem(message, context); + if (bodyClass.getCanonicalName().equals(SQSEvent.SQSMessage.class.getCanonicalName())) { + processMessage(message, context); } else { processItem(EventDeserializer.extractDataFrom(message).as(bodyClass), context); } @@ -27,7 +29,7 @@ default SQSBatchResponse processBatch(SQSEvent event, Context context) { return response; } - default void processItem(SQSEvent.SQSMessage message, Context context) { + default void processMessage(SQSEvent.SQSMessage message, Context context) { System.out.println("Processing message " + message.getMessageId()); } diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/DynamoDBBatchHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/DynamoDBBatchHandler.java index 7c4d35673..42a2a6afe 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/DynamoDBBatchHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/DynamoDBBatchHandler.java @@ -13,7 +13,7 @@ public StreamsEventResponse handleRequest(DynamodbEvent input, Context context) } @Override - public void processItem(DynamodbEvent.DynamodbStreamRecord record, Context context) { + public void processRecord(DynamodbEvent.DynamodbStreamRecord record, Context context) { } } diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessorTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessorTest.java index 9615a7860..5eb8d6664 100644 --- a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessorTest.java +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessorTest.java @@ -1,8 +1,10 @@ package software.amazon.lambda.powertools.batch2; import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; import com.amazonaws.services.lambda.runtime.events.SQSEvent; import com.amazonaws.services.lambda.runtime.tests.annotations.Event; +import org.assertj.core.api.Assertions; import org.junit.jupiter.params.ParameterizedTest; import org.mockito.Mock; import software.amazon.lambda.powertools.batch2.examples.model.Product; @@ -16,8 +18,16 @@ public void processItem(Product product, Context context) { } } + @ParameterizedTest + @Event(value = "sqs_event.json", type = SQSEvent.class) + public void testCustom(SQSEvent event) { + SQSBProduct sqsbproduct = new SQSBProduct(); + SQSBatchResponse sqsBatchResponse = sqsbproduct.processBatch(event, context); + Assertions.assertThat(sqsBatchResponse.getBatchItemFailures()).isEmpty(); + } + static class SQSBP implements SQSBatchProcessor { - public void processItem(SQSEvent.SQSMessage message, Context context) { + public void processMessage(SQSEvent.SQSMessage message, Context context) { System.out.println(message.toString()); } } @@ -26,13 +36,7 @@ public void processItem(SQSEvent.SQSMessage message, Context context) { @Event(value = "sqs_event.json", type = SQSEvent.class) public void testBasic(SQSEvent event) { SQSBP sqsbp = new SQSBP(); - sqsbp.processBatch(event, context); - } - - @ParameterizedTest - @Event(value = "sqs_event.json", type = SQSEvent.class) - public void testCustom(SQSEvent event) { - SQSBProduct sqsbproduct = new SQSBProduct(); - sqsbproduct.processBatch(event, context); + SQSBatchResponse sqsBatchResponse = sqsbp.processBatch(event, context); + Assertions.assertThat(sqsBatchResponse.getBatchItemFailures()).isEmpty(); } } From 312709601b94783b24b87b40f8f59a460b527001 Mon Sep 17 00:00:00 2001 From: Jerome Van Der Linden Date: Fri, 30 Jun 2023 09:15:32 +0200 Subject: [PATCH 09/79] refactoring and adding FIFO --- .../powertools/batch2/BatchProcessor.java | 6 ++- .../batch2/DynamoDBBatchProcessor.java | 7 ++- .../batch2/KinesisBatchProcessor.java | 10 ++++- .../powertools/batch2/SQSBatchProcessor.java | 43 +++++++++++++++++-- 4 files changed, 58 insertions(+), 8 deletions(-) diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/BatchProcessor.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/BatchProcessor.java index 6a71ff002..57bf793e9 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/BatchProcessor.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/BatchProcessor.java @@ -2,13 +2,17 @@ import com.amazonaws.services.lambda.runtime.Context; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public interface BatchProcessor { + Logger BATCH_LOGGER = LoggerFactory.getLogger(BatchProcessor.class); + RESPONSE processBatch(EVENT event, Context context); default void processItem(ITEM item, Context context) { - System.out.println("Processing custom item"); + BATCH_LOGGER.debug("[DEFAULT IMPLEMENTATION] Processing custom item"); } } \ No newline at end of file diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/DynamoDBBatchProcessor.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/DynamoDBBatchProcessor.java index 00a756eb2..0677f1007 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/DynamoDBBatchProcessor.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/DynamoDBBatchProcessor.java @@ -3,11 +3,15 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.ArrayList; public interface DynamoDBBatchProcessor extends BatchProcessor { + Logger DDB_BATCH_LOGGER = LoggerFactory.getLogger(SQSBatchProcessor.class); + @Override default StreamsEventResponse processBatch(DynamodbEvent event, Context context) { StreamsEventResponse response = StreamsEventResponse.builder().withBatchItemFailures(new ArrayList<>()).build(); @@ -15,6 +19,7 @@ default StreamsEventResponse processBatch(DynamodbEvent event, Context context) try { processRecord(record, context); } catch (Throwable t) { + DDB_BATCH_LOGGER.error("Error while processing record with id {}: {}, adding it to batch item failures", record.getEventID(), t.getMessage()); response.getBatchItemFailures().add(new StreamsEventResponse.BatchItemFailure(record.getEventID())); } } @@ -23,7 +28,7 @@ default StreamsEventResponse processBatch(DynamodbEvent event, Context context) @Override default void processItem(Void message, Context context) { - System.out.println("This method is not used in the case of DynamoDB"); + DDB_BATCH_LOGGER.warn("[DEFAULT IMPLEMENTATION] This method should not be used in the case of DynamoDB"); } void processRecord(DynamodbEvent.DynamodbStreamRecord record, Context context); diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/KinesisBatchProcessor.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/KinesisBatchProcessor.java index fd364e09d..af21a8fb1 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/KinesisBatchProcessor.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/KinesisBatchProcessor.java @@ -3,6 +3,8 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.KinesisEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import software.amazon.lambda.powertools.utilities.EventDeserializer; import sun.reflect.generics.reflectiveObjects.ParameterizedTypeImpl; @@ -10,19 +12,23 @@ public interface KinesisBatchProcessor extends BatchProcessor { + Logger KINESIS_BATCH_LOGGER = LoggerFactory.getLogger(KinesisBatchProcessor.class); + @Override default StreamsEventResponse processBatch(KinesisEvent event, Context context) { Class bodyClass = (Class) ((ParameterizedTypeImpl) getClass().getGenericInterfaces()[0]).getActualTypeArguments()[0]; + boolean isKinesisRecord = bodyClass.equals(KinesisEvent.KinesisEventRecord.class); StreamsEventResponse response = StreamsEventResponse.builder().withBatchItemFailures(new ArrayList<>()).build(); for (KinesisEvent.KinesisEventRecord record : event.getRecords()) { try { - if (bodyClass.equals(KinesisEvent.KinesisEventRecord.class)) { + if (isKinesisRecord) { processRecord(record, context); } else { processItem(EventDeserializer.extractDataFrom(record).as(bodyClass), context); } } catch (Throwable t) { + KINESIS_BATCH_LOGGER.error("Error while processing record with id {}: {}, adding it to batch item failures", record.getEventID(), t.getMessage()); response.getBatchItemFailures().add(new StreamsEventResponse.BatchItemFailure(record.getEventID())); } } @@ -30,7 +36,7 @@ default StreamsEventResponse processBatch(KinesisEvent event, Context context) { } default void processRecord(KinesisEvent.KinesisEventRecord record, Context context) { - System.out.println("Processing message " + record.getEventID()); + KINESIS_BATCH_LOGGER.debug("[DEFAULT IMPLEMENTATION] Processing record {}", record.getEventID()); } } diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessor.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessor.java index 3d08933b9..fb68d877a 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessor.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessor.java @@ -2,7 +2,10 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; +import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse.BatchItemFailure; import com.amazonaws.services.lambda.runtime.events.SQSEvent; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import software.amazon.lambda.powertools.utilities.EventDeserializer; import sun.reflect.generics.reflectiveObjects.ParameterizedTypeImpl; @@ -10,27 +13,59 @@ public interface SQSBatchProcessor extends BatchProcessor { + Logger SQS_BATCH_LOGGER = LoggerFactory.getLogger(SQSBatchProcessor.class); + + // The attribute on an SQS-FIFO message used to record the message group ID + // https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#sample-fifo-queues-message-event + String MESSAGE_GROUP_ID_KEY = "MessageGroupId"; + @Override default SQSBatchResponse processBatch(SQSEvent event, Context context) { Class bodyClass = (Class) ((ParameterizedTypeImpl) getClass().getGenericInterfaces()[0]).getActualTypeArguments()[0]; + boolean isSQSMessage = bodyClass.getCanonicalName().equals(SQSEvent.SQSMessage.class.getCanonicalName()); SQSBatchResponse response = SQSBatchResponse.builder().withBatchItemFailures(new ArrayList<>()).build(); - for (SQSEvent.SQSMessage message : event.getRecords()) { + + // If we are working on a FIFO queue, when any message fails we should stop processing and return the + // rest of the batch as failed too. We use this variable to track when that has happened. + // https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#services-sqs-batchfailurereporting + boolean failWholeBatch = false; + + int messageCursor = 0; + for (; messageCursor < event.getRecords().size() && !failWholeBatch; messageCursor++) { + SQSEvent.SQSMessage message = event.getRecords().get(messageCursor); + + String messageGroupId = message.getAttributes() != null ? + message.getAttributes().get(MESSAGE_GROUP_ID_KEY) : null; + try { - if (bodyClass.getCanonicalName().equals(SQSEvent.SQSMessage.class.getCanonicalName())) { + if (isSQSMessage) { processMessage(message, context); } else { processItem(EventDeserializer.extractDataFrom(message).as(bodyClass), context); } } catch (Throwable t) { - response.getBatchItemFailures().add(new SQSBatchResponse.BatchItemFailure(message.getMessageId())); + SQS_BATCH_LOGGER.error("Error while processing message with messageId {}: {}, adding it to batch item failures", message.getMessageId(), t.getMessage()); + response.getBatchItemFailures().add(BatchItemFailure.builder().withItemIdentifier(message.getMessageId()).build()); + if (messageGroupId != null) { + failWholeBatch = true; + SQS_BATCH_LOGGER.info("A message in a batch with messageGroupId {} and messageId {} failed; failing the rest of the batch too" + , messageGroupId, message.getMessageId()); + } } } + + if (failWholeBatch) { + // Add the remaining messages to the batch item failures + event.getRecords() + .subList(messageCursor, event.getRecords().size()) // TODO test the cursors... + .forEach(message -> response.getBatchItemFailures().add(BatchItemFailure.builder().withItemIdentifier(message.getMessageId()).build())); + } return response; } default void processMessage(SQSEvent.SQSMessage message, Context context) { - System.out.println("Processing message " + message.getMessageId()); + SQS_BATCH_LOGGER.debug("[DEFAULT IMPLEMENTATION] Processing message {}", message.getMessageId()); } } From 9b8a31023be1b1f5ae19decac8bff30fbbf76594 Mon Sep 17 00:00:00 2001 From: Jerome Van Der Linden Date: Fri, 30 Jun 2023 09:15:32 +0200 Subject: [PATCH 10/79] refactoring and adding FIFO --- powertools-batch/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/powertools-batch/pom.xml b/powertools-batch/pom.xml index c165064f1..aae0ef07a 100644 --- a/powertools-batch/pom.xml +++ b/powertools-batch/pom.xml @@ -24,6 +24,11 @@ powertools-serialization ${project.version} + + software.amazon.lambda + powertools-idempotency + ${project.version} + org.aspectj aspectjrt From 12cb97ff68c020ec3aee476ff77420fcc250fcb3 Mon Sep 17 00:00:00 2001 From: Jerome Van Der Linden Date: Sat, 1 Jul 2023 23:04:16 +0200 Subject: [PATCH 11/79] adding FIFO management --- .../powertools/batch2/SQSBatchProcessor.java | 2 +- .../batch2/SQSBatchProcessorTest.java | 54 +++++++++++----- .../src/test/resources/sqs_event.json | 20 +++++- .../src/test/resources/sqs_fifo_event.json | 61 +++++++++++++++++++ 4 files changed, 121 insertions(+), 16 deletions(-) create mode 100644 powertools-batch/src/test/resources/sqs_fifo_event.json diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessor.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessor.java index fb68d877a..ad37a598d 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessor.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessor.java @@ -58,7 +58,7 @@ default SQSBatchResponse processBatch(SQSEvent event, Context context) { if (failWholeBatch) { // Add the remaining messages to the batch item failures event.getRecords() - .subList(messageCursor, event.getRecords().size()) // TODO test the cursors... + .subList(messageCursor, event.getRecords().size()) .forEach(message -> response.getBatchItemFailures().add(BatchItemFailure.builder().withItemIdentifier(message.getMessageId()).build())); } return response; diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessorTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessorTest.java index 5eb8d6664..1a69b336f 100644 --- a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessorTest.java +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessorTest.java @@ -12,31 +12,57 @@ public class SQSBatchProcessorTest { @Mock private Context context; - static class SQSBProduct implements SQSBatchProcessor { - public void processItem(Product product, Context context) { - System.out.println(product); + + + static class SQSBP implements SQSBatchProcessor { + public void processMessage(SQSEvent.SQSMessage message, Context context) { + if (message.getMessageId().equals("e9144555-9a4f-4ec3-99a0-34ce359b4b54")) { + throw new RuntimeException("fake exception"); + } } } @ParameterizedTest @Event(value = "sqs_event.json", type = SQSEvent.class) - public void testCustom(SQSEvent event) { - SQSBProduct sqsbproduct = new SQSBProduct(); - SQSBatchResponse sqsBatchResponse = sqsbproduct.processBatch(event, context); - Assertions.assertThat(sqsBatchResponse.getBatchItemFailures()).isEmpty(); + public void shouldAddMessageToBatchFailure_whenException_withMessage(SQSEvent event) { + SQSBP sqsbp = new SQSBP(); + SQSBatchResponse sqsBatchResponse = sqsbp.processBatch(event, context); + Assertions.assertThat(sqsBatchResponse.getBatchItemFailures()).hasSize(1); + + SQSBatchResponse.BatchItemFailure batchItemFailure = sqsBatchResponse.getBatchItemFailures().get(0); + Assertions.assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("e9144555-9a4f-4ec3-99a0-34ce359b4b54"); } - static class SQSBP implements SQSBatchProcessor { - public void processMessage(SQSEvent.SQSMessage message, Context context) { - System.out.println(message.toString()); + @ParameterizedTest + @Event(value = "sqs_fifo_event.json", type = SQSEvent.class) + public void shouldAddMessageToBatchFailure_whenException_withSQSFIFO(SQSEvent event) { + SQSBP sqsbp = new SQSBP(); + SQSBatchResponse sqsBatchResponse = sqsbp.processBatch(event, context); + Assertions.assertThat(sqsBatchResponse.getBatchItemFailures()).hasSize(2); + + SQSBatchResponse.BatchItemFailure batchItemFailure = sqsBatchResponse.getBatchItemFailures().get(0); + Assertions.assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("e9144555-9a4f-4ec3-99a0-34ce359b4b54"); + batchItemFailure = sqsBatchResponse.getBatchItemFailures().get(1); + Assertions.assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("f9144555-9a4f-4ec3-99a0-34ce359b4b54"); + } + + static class SQSBProduct implements SQSBatchProcessor { + public void processItem(Product product, Context context) { + if (product.getId() == 12345) { + throw new RuntimeException("fake exception"); + } } } @ParameterizedTest @Event(value = "sqs_event.json", type = SQSEvent.class) - public void testBasic(SQSEvent event) { - SQSBP sqsbp = new SQSBP(); - SQSBatchResponse sqsBatchResponse = sqsbp.processBatch(event, context); - Assertions.assertThat(sqsBatchResponse.getBatchItemFailures()).isEmpty(); + public void shouldAddMessageToBatchFailure_whenException_withProduct(SQSEvent event) { + SQSBProduct sqsbproduct = new SQSBProduct(); + SQSBatchResponse sqsBatchResponse = sqsbproduct.processBatch(event, context); + Assertions.assertThat(sqsBatchResponse.getBatchItemFailures()).hasSize(1); + + SQSBatchResponse.BatchItemFailure batchItemFailure = sqsBatchResponse.getBatchItemFailures().get(0); + Assertions.assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("e9144555-9a4f-4ec3-99a0-34ce359b4b54"); } + } diff --git a/powertools-batch/src/test/resources/sqs_event.json b/powertools-batch/src/test/resources/sqs_event.json index d33db4b53..1366029d3 100644 --- a/powertools-batch/src/test/resources/sqs_event.json +++ b/powertools-batch/src/test/resources/sqs_event.json @@ -19,7 +19,7 @@ "awsRegion": "eu-central-1" }, { - "messageId": "d9144555-9a4f-4ec3-99a0-34ce359b4b54", + "messageId": "e9144555-9a4f-4ec3-99a0-34ce359b4b54", "receiptHandle": "13e7f7851d2eaa5c01f208ebadbf1e72==", "body": "{\n \"id\": 12345,\n \"name\": \"product5\",\n \"price\": 45\n}", "attributes": { @@ -30,6 +30,24 @@ }, "messageAttributes": { + }, + "md5OfBody": "13e7f7851d2eaa5c01f208ebadbf1e72", + "eventSource": "aws:sqs", + "eventSourceARN": "arn:aws:sqs:eu-central-1:123456789012:TestLambda", + "awsRegion": "eu-central-1" + }, + { + "messageId": "f9144555-9a4f-4ec3-99a0-34ce359b4b54", + "receiptHandle": "13e7f7851d2eaa5c01f208ebadbf1e72==", + "body": "{\n \"id\": 123456,\n \"name\": \"product6\",\n \"price\": 46\n}", + "attributes": { + "ApproximateReceiveCount": "1", + "SentTimestamp": "1601975706495", + "SenderId": "AROAIFU437PVZ5L2J53F5", + "ApproximateFirstReceiveTimestamp": "1601975706499" + }, + "messageAttributes": { + }, "md5OfBody": "13e7f7851d2eaa5c01f208ebadbf1e72", "eventSource": "aws:sqs", diff --git a/powertools-batch/src/test/resources/sqs_fifo_event.json b/powertools-batch/src/test/resources/sqs_fifo_event.json new file mode 100644 index 000000000..8ac6a4b8f --- /dev/null +++ b/powertools-batch/src/test/resources/sqs_fifo_event.json @@ -0,0 +1,61 @@ +{ + "Records": [ + { + "messageId": "d9144555-9a4f-4ec3-99a0-34ce359b4b54", + "receiptHandle": "13e7f7851d2eaa5c01f208ebadbf1e72==", + "body": "{\n \"id\": 1234,\n \"name\": \"product\",\n \"price\": 42\n}", + "attributes": { + "MessageGroupId": "groupA", + "ApproximateReceiveCount": "1", + "SentTimestamp": "1601975706495", + "SenderId": "AROAIFU437PVZ5L2J53F5", + "ApproximateFirstReceiveTimestamp": "1601975706499" + }, + "messageAttributes": { + + }, + "md5OfBody": "13e7f7851d2eaa5c01f208ebadbf1e72", + "eventSource": "aws:sqs", + "eventSourceARN": "arn:aws:sqs:eu-central-1:123456789012:TestLambda", + "awsRegion": "eu-central-1" + }, + { + "messageId": "e9144555-9a4f-4ec3-99a0-34ce359b4b54", + "receiptHandle": "13e7f7851d2eaa5c01f208ebadbf1e72==", + "body": "{\n \"id\": 12345,\n \"name\": \"product5\",\n \"price\": 45\n}", + "attributes": { + "MessageGroupId": "groupA", + "ApproximateReceiveCount": "1", + "SentTimestamp": "1601975706495", + "SenderId": "AROAIFU437PVZ5L2J53F5", + "ApproximateFirstReceiveTimestamp": "1601975706499" + }, + "messageAttributes": { + + }, + "md5OfBody": "13e7f7851d2eaa5c01f208ebadbf1e72", + "eventSource": "aws:sqs", + "eventSourceARN": "arn:aws:sqs:eu-central-1:123456789012:TestLambda", + "awsRegion": "eu-central-1" + }, + { + "messageId": "f9144555-9a4f-4ec3-99a0-34ce359b4b54", + "receiptHandle": "13e7f7851d2eaa5c01f208ebadbf1e72==", + "body": "{\n \"id\": 123456,\n \"name\": \"product6\",\n \"price\": 46\n}", + "attributes": { + "MessageGroupId": "groupA", + "ApproximateReceiveCount": "1", + "SentTimestamp": "1601975706495", + "SenderId": "AROAIFU437PVZ5L2J53F5", + "ApproximateFirstReceiveTimestamp": "1601975706499" + }, + "messageAttributes": { + + }, + "md5OfBody": "13e7f7851d2eaa5c01f208ebadbf1e72", + "eventSource": "aws:sqs", + "eventSourceARN": "arn:aws:sqs:eu-central-1:123456789012:TestLambda", + "awsRegion": "eu-central-1" + } + ] +} \ No newline at end of file From 061bfb1741378ad3dc08f06dd24ec768f0820835 Mon Sep 17 00:00:00 2001 From: Jerome Van Der Linden Date: Sat, 1 Jul 2023 23:04:38 +0200 Subject: [PATCH 12/79] cleanup --- .../lambda/powertools/batch2/DynamoDBBatchProcessor.java | 3 ++- .../amazon/lambda/powertools/batch2/KinesisBatchProcessor.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/DynamoDBBatchProcessor.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/DynamoDBBatchProcessor.java index 0677f1007..d44656706 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/DynamoDBBatchProcessor.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/DynamoDBBatchProcessor.java @@ -3,6 +3,7 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; +import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse.BatchItemFailure; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,7 +21,7 @@ default StreamsEventResponse processBatch(DynamodbEvent event, Context context) processRecord(record, context); } catch (Throwable t) { DDB_BATCH_LOGGER.error("Error while processing record with id {}: {}, adding it to batch item failures", record.getEventID(), t.getMessage()); - response.getBatchItemFailures().add(new StreamsEventResponse.BatchItemFailure(record.getEventID())); + response.getBatchItemFailures().add(BatchItemFailure.builder().withItemIdentifier(record.getEventID()).build()); } } return response; diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/KinesisBatchProcessor.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/KinesisBatchProcessor.java index af21a8fb1..26b318b2b 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/KinesisBatchProcessor.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/KinesisBatchProcessor.java @@ -3,6 +3,7 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.KinesisEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; +import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse.BatchItemFailure; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.lambda.powertools.utilities.EventDeserializer; @@ -29,7 +30,7 @@ default StreamsEventResponse processBatch(KinesisEvent event, Context context) { } } catch (Throwable t) { KINESIS_BATCH_LOGGER.error("Error while processing record with id {}: {}, adding it to batch item failures", record.getEventID(), t.getMessage()); - response.getBatchItemFailures().add(new StreamsEventResponse.BatchItemFailure(record.getEventID())); + response.getBatchItemFailures().add(BatchItemFailure.builder().withItemIdentifier(record.getEventID()).build()); } } return response; From 6b07f784a21350761303f9d8a9b03a8585542788 Mon Sep 17 00:00:00 2001 From: Jerome Van Der Linden Date: Mon, 3 Jul 2023 09:14:48 +0200 Subject: [PATCH 13/79] add javadoc --- .../powertools/batch2/BatchProcessor.java | 49 ++++++++++++++++++- .../powertools/batch2/SQSBatchProcessor.java | 46 +++++++++++++++-- 2 files changed, 91 insertions(+), 4 deletions(-) diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/BatchProcessor.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/BatchProcessor.java index 57bf793e9..33529dfa3 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/BatchProcessor.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/BatchProcessor.java @@ -1,16 +1,63 @@ package software.amazon.lambda.powertools.batch2; - import com.amazonaws.services.lambda.runtime.Context; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Generic interface for batch processing with Lambda. This interface provides 2 main methods: + *
    + *
  • {@link #processBatch(Object, Context)} that will take a batch of items (messages, records), + * loop over it, and trigger the processItem function for each item. + * This method is also in charge to handle partial batch failures.
    • + *
  • {@link #processItem(Object, Context)} that will handle each item of the batch. This method is generally implemented by the handler or any other class in charge of processing items. + * If there are exceptions within this method, you can let them go and the processBatch method will handle them and add the messages to the partial batch failures.
    • + *
+ * + * @param Type of the incoming event. Natively supported: + *
    + *
  • {@link com.amazonaws.services.lambda.runtime.events.SQSEvent}
    • + *
  • {@link com.amazonaws.services.lambda.runtime.events.KinesisEvent}
    • + *
  • {@link com.amazonaws.services.lambda.runtime.events.DynamodbEvent}
    • + *
+ * @param Type of the message or inner element in the message (e.g. body). Natively supported: + *
    + *
  • {@link com.amazonaws.services.lambda.runtime.events.SQSEvent.SQSMessage}
    • + *
  • {@link com.amazonaws.services.lambda.runtime.events.KinesisEvent.KinesisEventRecord}
    • + *
  • {@link com.amazonaws.services.lambda.runtime.events.DynamodbEvent.DynamodbStreamRecord}
    • + *
+ * If you choose your own model type, the content of the body within the message will be deserialized. + *
    + *
  • For SQS, it will be {@link com.amazonaws.services.lambda.runtime.events.SQSEvent.SQSMessage#getBody()}
    • + *
  • For Kinesis, it will be {@link com.amazonaws.services.lambda.runtime.events.models.kinesis.Record#getData()} (base 64 decoded)
    • + *
  • Not supported for DynamoDB
    • + *
+ * @param Type of the handler response. Natively supported: + *
    + *
  • {@link com.amazonaws.services.lambda.runtime.events.SQSBatchResponse} for SQS.
    • + *
  • {@link com.amazonaws.services.lambda.runtime.events.StreamsEventResponse} for Kinesis and DynamoDB.
    • + *
+ */ public interface BatchProcessor { Logger BATCH_LOGGER = LoggerFactory.getLogger(BatchProcessor.class); + /** + * This method takes a batch of items (messages, records) from an event, + * loop over it, and trigger the processItem function for each item. + * @param event handler event (e.g. {@link com.amazonaws.services.lambda.runtime.events.SQSEvent}) + * @param context handler context + * @return the handler response containing partial batch failures (e.g. {@link com.amazonaws.services.lambda.runtime.events.SQSBatchResponse}) + */ RESPONSE processBatch(EVENT event, Context context); + /** + * This method handles each item of the batch. This method is generally implemented by the handler or any other class in charge of processing items. + * If there are exceptions within this method, you can let them go and the processBatch method will handle them and add the messages to the partial batch failures. + * This method has a default implementation because it's not available for DynamoDB. + * @param item the item to process (e.g. {@link com.amazonaws.services.lambda.runtime.events.SQSEvent.SQSMessage}) + * @param context handler context + */ default void processItem(ITEM item, Context context) { BATCH_LOGGER.debug("[DEFAULT IMPLEMENTATION] Processing custom item"); } diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessor.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessor.java index ad37a598d..c66139a92 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessor.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessor.java @@ -4,6 +4,7 @@ import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse.BatchItemFailure; import com.amazonaws.services.lambda.runtime.events.SQSEvent; +import com.amazonaws.services.lambda.runtime.events.SQSEvent.SQSMessage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.lambda.powertools.utilities.EventDeserializer; @@ -11,6 +12,45 @@ import java.util.ArrayList; +/** + * SQSBatchProcessor is an interface that can be implemented by your handler or any other class + * to process batches of SQS messages. It handles the partial batch failures. + * It extends the {@link BatchProcessor} interface and provide + * an implementation for the @link {@link BatchProcessor#processBatch(Object, Context)} method.
+ * When you implement this interface you have the choice to implement the {@link #processMessage(SQSMessage, Context)} + * method OR {@link #processItem(Object, Context)} method (do not implement both, only one will be used):
    + *
  • If you need access to the whole {@link SQSMessage} + * (with the attributes, messageAttributes, ...), use the processMessage method.
    • + *
  • If you simply want to access the body of the message and have it deserialized as ITEM type, + * use the processItem method.
    • + *
+ * The choice is made by typing the SQSBatchProcessor either with {@link SQSMessage}: + * 
+ *     public class SQSBatchHandler implements SQSBatchProcessor<SQSMessage> {
+ *         public void processMessage(SQSEvent.SQSMessage message, Context context) {
+ *             // access message information (id, attributes, messageAttributes, ...)
+ *             MessageAttribute attribute = message.getMessageAttributes().get("myAttribute");
+ *
+ *             // access the body manually (message.getBody())
+ *             // using powertools-serialization module, you can easily deserialize the message body
+ *             MyModel myModel = EventDeserializer.extractDataFrom(message).as(MyModel.class)
+ *         }
+ *     }
+ *
+ * + * Or with the type of your choice: + * 
+ *     public class SQSBatchHandler implements SQSBatchProcessor<MyModel> {
+ *         public void processItem(MyModel myModel, Context context) {
+ *             // ... do some stuff with your model
+ *         }
+ *     }
+ *
+ * + * In each case, you have access to the Lambda context if needed.
+ * + * @param Type of the messages, can be either {@link SQSMessage} for the whole message or any type for the message body only. + */ public interface SQSBatchProcessor extends BatchProcessor { Logger SQS_BATCH_LOGGER = LoggerFactory.getLogger(SQSBatchProcessor.class); @@ -22,7 +62,7 @@ public interface SQSBatchProcessor extends BatchProcessor bodyClass = (Class) ((ParameterizedTypeImpl) getClass().getGenericInterfaces()[0]).getActualTypeArguments()[0]; - boolean isSQSMessage = bodyClass.getCanonicalName().equals(SQSEvent.SQSMessage.class.getCanonicalName()); + boolean isSQSMessage = bodyClass.getCanonicalName().equals(SQSMessage.class.getCanonicalName()); SQSBatchResponse response = SQSBatchResponse.builder().withBatchItemFailures(new ArrayList<>()).build(); @@ -33,7 +73,7 @@ default SQSBatchResponse processBatch(SQSEvent event, Context context) { int messageCursor = 0; for (; messageCursor < event.getRecords().size() && !failWholeBatch; messageCursor++) { - SQSEvent.SQSMessage message = event.getRecords().get(messageCursor); + SQSMessage message = event.getRecords().get(messageCursor); String messageGroupId = message.getAttributes() != null ? message.getAttributes().get(MESSAGE_GROUP_ID_KEY) : null; @@ -64,7 +104,7 @@ default SQSBatchResponse processBatch(SQSEvent event, Context context) { return response; } - default void processMessage(SQSEvent.SQSMessage message, Context context) { + default void processMessage(SQSMessage message, Context context) { SQS_BATCH_LOGGER.debug("[DEFAULT IMPLEMENTATION] Processing message {}", message.getMessageId()); } From cc28ce5d75e35086159d0fe4f57970e1ba0f4677 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Tue, 11 Jul 2023 07:51:35 +0200 Subject: [PATCH 14/79] Flesh out builder option a bit --- powertools-batch/pom.xml | 4 ++ .../batch/BatchMessageHandlerBuilder.java | 46 ------------------- .../powertools/batch/BatchRequestHandler.java | 2 +- .../ExampleMessageHandlerBuilder.java | 22 --------- .../batch2/examples/KinesisBatchHandler.java | 2 +- .../batch2/examples/SQSBatchHandler.java | 2 +- .../batch3/BatchMessageHandlerBuilder.java | 22 +++++++++ .../batch3/DdbMessageHandlerBuilder.java | 14 ++++++ .../batch3/SqsBatchMessageHandlerBuilder.java | 40 ++++++++++++++++ .../batch3/examples/DynamodbExample.java | 21 +++++++++ .../SqsExampleWithDeserialization.java | 26 +++++++++++ .../examples/SqsExampleWithIdempotency.java | 39 ++++++++++++++++ .../{batch2/examples => }/model/Basket.java | 2 +- .../{batch2/examples => }/model/Product.java | 2 +- .../batch2/SQSBatchProcessorTest.java | 2 +- 15 files changed, 172 insertions(+), 74 deletions(-) delete mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java delete mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/examples/ExampleMessageHandlerBuilder.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/BatchMessageHandlerBuilder.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/DdbMessageHandlerBuilder.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/SqsBatchMessageHandlerBuilder.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/DynamodbExample.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/SqsExampleWithDeserialization.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/SqsExampleWithIdempotency.java rename powertools-batch/src/main/java/software/amazon/lambda/powertools/{batch2/examples => }/model/Basket.java (96%) rename powertools-batch/src/main/java/software/amazon/lambda/powertools/{batch2/examples => }/model/Product.java (96%) diff --git a/powertools-batch/pom.xml b/powertools-batch/pom.xml index aae0ef07a..f359ccd95 100644 --- a/powertools-batch/pom.xml +++ b/powertools-batch/pom.xml @@ -61,6 +61,10 @@ mockito-inline test + + software.amazon.lambda + powertools-sqs + \ No newline at end of file diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java deleted file mode 100644 index 83825fae1..000000000 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java +++ /dev/null @@ -1,46 +0,0 @@ -package software.amazon.lambda.powertools.batch; - -import com.amazonaws.services.lambda.runtime.events.*; -import sun.reflect.generics.reflectiveObjects.NotImplementedException; - -import java.time.Duration; -import java.util.List; -import java.util.function.Consumer; - - -/** - * A builder-style interface we can use within an existing Lambda RequestHandler to - * deal with our batch responses. - * - * @see software.amazon.lambda.powertools.batch.examples.ExampleMessageHandlerBuilder - */ -public class BatchMessageHandlerBuilder { - - private Duration timeout; - - public BatchMessageHandlerBuilder() { - } - - /** - * An example parameter we might want to configure here to show - * the shape of the interface. - */ - public BatchMessageHandlerBuilder withTimeout(Duration timeout) { - this.timeout = timeout; - return this; - } - - // TODO - can we put a meaningful type on the message, or do we need to do it, per-message-type? - public SQSBatchResponse process(SQSEvent message, Consumer messageHandler) { - throw new NotImplementedException(); - }; - - public StreamsEventResponse process(KinesisEvent message, Consumer messageHandler) { - throw new NotImplementedException(); - } - - public StreamsEventResponse process(DynamodbEvent message, Consumer messageHandler) { - throw new NotImplementedException(); - } - -} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchRequestHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchRequestHandler.java index 9d74517ae..7ab54173e 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchRequestHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchRequestHandler.java @@ -2,7 +2,7 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.RequestHandler; -import com.sun.xml.internal.ws.api.message.Message; +import software.amazon.lambda.powertools.batch3.BatchMessageHandlerBuilder; import java.util.List; import java.util.stream.Collectors; diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/examples/ExampleMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/examples/ExampleMessageHandlerBuilder.java deleted file mode 100644 index f22bad9b5..000000000 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/examples/ExampleMessageHandlerBuilder.java +++ /dev/null @@ -1,22 +0,0 @@ -package software.amazon.lambda.powertools.batch.examples; - -import com.amazonaws.services.lambda.runtime.Context; -import com.amazonaws.services.lambda.runtime.RequestHandler; -import com.amazonaws.services.lambda.runtime.events.SQSEvent; -import software.amazon.lambda.powertools.batch.BatchMessageHandlerBuilder; - -/** - * An example handler that is implemented by using the {@link BatchMessageHandlerBuilder} inline - * in an existing RequestHandler. - */ -public class ExampleMessageHandlerBuilder implements RequestHandler { - - @Override - public Object handleRequest(SQSEvent sqsEvent, Context context) { - return new BatchMessageHandlerBuilder() - .process(sqsEvent, (SQSEvent.SQSMessage message) -> { - // Process the message without throwing an exception - }); - } - -} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/KinesisBatchHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/KinesisBatchHandler.java index 2c4c5824c..fd70f8394 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/KinesisBatchHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/KinesisBatchHandler.java @@ -5,7 +5,7 @@ import com.amazonaws.services.lambda.runtime.events.KinesisEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; import software.amazon.lambda.powertools.batch2.KinesisBatchProcessor; -import software.amazon.lambda.powertools.batch2.examples.model.Basket; +import software.amazon.lambda.powertools.model.Basket; public class KinesisBatchHandler implements RequestHandler, KinesisBatchProcessor { @Override diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/SQSBatchHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/SQSBatchHandler.java index 9738f40ae..717787d27 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/SQSBatchHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/SQSBatchHandler.java @@ -5,7 +5,7 @@ import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; import com.amazonaws.services.lambda.runtime.events.SQSEvent; import software.amazon.lambda.powertools.batch2.SQSBatchProcessor; -import software.amazon.lambda.powertools.batch2.examples.model.Basket; +import software.amazon.lambda.powertools.model.Basket; public class SQSBatchHandler implements RequestHandler, SQSBatchProcessor { @Override diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/BatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/BatchMessageHandlerBuilder.java new file mode 100644 index 000000000..015a37b44 --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/BatchMessageHandlerBuilder.java @@ -0,0 +1,22 @@ +package software.amazon.lambda.powertools.batch3; + + +import software.amazon.lambda.powertools.batch3.examples.SqsExampleWithIdempotency; + +/** + * A builder-style interface we can use within an existing Lambda RequestHandler to + * deal with our batch responses. + * + * @see SqsExampleWithIdempotency + */ +public class BatchMessageHandlerBuilder { + + public SqsBatchMessageHandlerBuilder withSqsBatchHandler() { + return new SqsBatchMessageHandlerBuilder(); + } + + public DdbMessageHandlerBuilder withDynamoDbBatchHandler() { + return new DdbMessageHandlerBuilder(); + } + +} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/DdbMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/DdbMessageHandlerBuilder.java new file mode 100644 index 000000000..b61c29273 --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/DdbMessageHandlerBuilder.java @@ -0,0 +1,14 @@ +package software.amazon.lambda.powertools.batch3; + +import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; +import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; +import sun.reflect.generics.reflectiveObjects.NotImplementedException; + +import java.util.function.Consumer; + +public class DdbMessageHandlerBuilder { + + public StreamsEventResponse processBatch(DynamodbEvent batch, Consumer handler) { + throw new NotImplementedException(); + } +} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/SqsBatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/SqsBatchMessageHandlerBuilder.java new file mode 100644 index 000000000..6e13cf7cc --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/SqsBatchMessageHandlerBuilder.java @@ -0,0 +1,40 @@ +package software.amazon.lambda.powertools.batch3; + +import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; +import com.amazonaws.services.lambda.runtime.events.SQSEvent; +import sun.reflect.generics.reflectiveObjects.NotImplementedException; + +import java.util.function.Consumer; + +public class SqsBatchMessageHandlerBuilder { + + private Consumer failureHandler; + private Consumer successHandler; + + /** + * success and failure handler hooks are used as an example of a tuneable from the python + * powertools. + * + * https://docs.powertools.aws.dev/lambda/python/2.15.0/utilities/batch/#extending-batchprocessor + * + * @param handler + * @return + */ + public SqsBatchMessageHandlerBuilder withSuccessHandler(Consumer handler) { + this.successHandler = handler; + return this; + } + + public SqsBatchMessageHandlerBuilder withFailureHandler(Consumer handler) { + this.failureHandler = handler; + return this; + } + + public SQSBatchResponse processMessage(SQSEvent event, Consumer handler) { + throw new NotImplementedException(); + } + + public SQSBatchResponse processRawMessage(SQSEvent event, Consumer handler) { + throw new NotImplementedException(); + } +} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/DynamodbExample.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/DynamodbExample.java new file mode 100644 index 000000000..433b9ba84 --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/DynamodbExample.java @@ -0,0 +1,21 @@ +package software.amazon.lambda.powertools.batch3.examples; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; +import com.amazonaws.services.lambda.runtime.events.SQSEvent; +import software.amazon.lambda.powertools.batch3.BatchMessageHandlerBuilder; +import software.amazon.lambda.powertools.model.Basket; + +public class DynamodbExample implements RequestHandler { + + @Override + public Object handleRequest(DynamodbEvent ddbEvent, Context context) { + // Example 2 - process a deserialized message + return new BatchMessageHandlerBuilder() + .withDynamoDbBatchHandler() + .processBatch(ddbEvent, basket -> { + + }); + } +} \ No newline at end of file diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/SqsExampleWithDeserialization.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/SqsExampleWithDeserialization.java new file mode 100644 index 000000000..814d225de --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/SqsExampleWithDeserialization.java @@ -0,0 +1,26 @@ +package software.amazon.lambda.powertools.batch3.examples; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.amazonaws.services.lambda.runtime.events.SQSEvent; +import software.amazon.lambda.powertools.batch3.BatchMessageHandlerBuilder; +import software.amazon.lambda.powertools.idempotency.Idempotent; +import software.amazon.lambda.powertools.model.Basket; + +/** + * An example handler that is implemented by using the {@link BatchMessageHandlerBuilder} inline + * in an existing RequestHandler. + */ +public class SqsExampleWithDeserialization implements RequestHandler { + + @Override + public Object handleRequest(SQSEvent sqsEvent, Context context) { + // Example 2 - process a deserialized message + return new BatchMessageHandlerBuilder() + .withSqsBatchHandler() + .processMessage(sqsEvent, basket -> { + + }); + } +} + diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/SqsExampleWithIdempotency.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/SqsExampleWithIdempotency.java new file mode 100644 index 000000000..08bacdbce --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/SqsExampleWithIdempotency.java @@ -0,0 +1,39 @@ +package software.amazon.lambda.powertools.batch3.examples; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.amazonaws.services.lambda.runtime.events.SQSEvent; +import software.amazon.lambda.powertools.batch3.BatchMessageHandlerBuilder; +import software.amazon.lambda.powertools.idempotency.Idempotent; +import software.amazon.lambda.powertools.model.Basket; +import software.amazon.lambda.powertools.sqs.SqsLargeMessage; + +/** + * An example handler that is implemented by using the {@link BatchMessageHandlerBuilder} inline + * in an existing RequestHandler. + */ +public class SqsExampleWithIdempotency implements RequestHandler { + + @Override + public Object handleRequest(SQSEvent sqsEvent, Context context) { + // Example 1 - process a raw SQS message in an idempotent fashion + // return ... + new BatchMessageHandlerBuilder() + .withSqsBatchHandler() + .processRawMessage(sqsEvent, this::processWithIdempotency); + + // Example 2 - process a deserialized message + return new BatchMessageHandlerBuilder() + .withSqsBatchHandler() + .processMessage(sqsEvent, basket -> { + + }); + } + + @Idempotent + @SqsLargeMessage + private void processWithIdempotency(final SQSEvent.SQSMessage message) { + + } + +} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/model/Basket.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/model/Basket.java similarity index 96% rename from powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/model/Basket.java rename to powertools-batch/src/main/java/software/amazon/lambda/powertools/model/Basket.java index d9c387aca..ee08523ac 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/model/Basket.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/model/Basket.java @@ -11,7 +11,7 @@ * limitations under the License. * */ -package software.amazon.lambda.powertools.batch2.examples.model; +package software.amazon.lambda.powertools.model; import java.util.ArrayList; import java.util.Arrays; diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/model/Product.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/model/Product.java similarity index 96% rename from powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/model/Product.java rename to powertools-batch/src/main/java/software/amazon/lambda/powertools/model/Product.java index 0311dca66..97b046189 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/model/Product.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/model/Product.java @@ -11,7 +11,7 @@ * limitations under the License. * */ -package software.amazon.lambda.powertools.batch2.examples.model; +package software.amazon.lambda.powertools.model; import java.util.Objects; diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessorTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessorTest.java index 1a69b336f..35d259437 100644 --- a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessorTest.java +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessorTest.java @@ -7,7 +7,7 @@ import org.assertj.core.api.Assertions; import org.junit.jupiter.params.ParameterizedTest; import org.mockito.Mock; -import software.amazon.lambda.powertools.batch2.examples.model.Product; +import software.amazon.lambda.powertools.model.Product; public class SQSBatchProcessorTest { @Mock From 916c26fc7ed56819abb260187adf82b3d8d0c867 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Tue, 11 Jul 2023 07:56:18 +0200 Subject: [PATCH 15/79] Flesh out a bit more --- .../batch3/BatchMessageHandlerBuilder.java | 3 ++- .../batch3/SqsBatchMessageHandlerBuilder.java | 15 +++++++++++++-- .../examples/SqsExampleWithDeserialization.java | 2 +- .../examples/SqsExampleWithIdempotency.java | 12 +++--------- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/BatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/BatchMessageHandlerBuilder.java index 015a37b44..f1b1fe6d3 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/BatchMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/BatchMessageHandlerBuilder.java @@ -5,7 +5,8 @@ /** * A builder-style interface we can use within an existing Lambda RequestHandler to - * deal with our batch responses. + * deal with our batch responses. A second tier of builders is returned per-event-source + * to bind the appropriate message types and provider source-specific logic and tuneables. * * @see SqsExampleWithIdempotency */ diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/SqsBatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/SqsBatchMessageHandlerBuilder.java index 6e13cf7cc..46cee8da3 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/SqsBatchMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/SqsBatchMessageHandlerBuilder.java @@ -3,9 +3,14 @@ import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; import com.amazonaws.services.lambda.runtime.events.SQSEvent; import sun.reflect.generics.reflectiveObjects.NotImplementedException; +import com.amazonaws.services.lambda.runtime.Context; +import java.util.function.BiConsumer; import java.util.function.Consumer; +/** + * Builds a batch processor for the SQS event source. + */ public class SqsBatchMessageHandlerBuilder { private Consumer failureHandler; @@ -30,11 +35,17 @@ public SqsBatchMessageHandlerBuilder withFailureHandler(Consumer SQSBatchResponse processMessage(SQSEvent event, Consumer handler) { + /** + * The user can consume either raw messages .... + */ + public SQSBatchResponse processMessage(SQSEvent event, BiConsumer handler) { throw new NotImplementedException(); } - public SQSBatchResponse processRawMessage(SQSEvent event, Consumer handler) { + /** + * ... or deserialized messages + */ + public SQSBatchResponse processRawMessage(SQSEvent event, BiConsumer handler) { throw new NotImplementedException(); } } diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/SqsExampleWithDeserialization.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/SqsExampleWithDeserialization.java index 814d225de..222eda4c7 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/SqsExampleWithDeserialization.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/SqsExampleWithDeserialization.java @@ -18,7 +18,7 @@ public Object handleRequest(SQSEvent sqsEvent, Context context) { // Example 2 - process a deserialized message return new BatchMessageHandlerBuilder() .withSqsBatchHandler() - .processMessage(sqsEvent, basket -> { + .processMessage(sqsEvent, (basket, ctx) -> { }); } diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/SqsExampleWithIdempotency.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/SqsExampleWithIdempotency.java index 08bacdbce..8d112e88d 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/SqsExampleWithIdempotency.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/SqsExampleWithIdempotency.java @@ -4,6 +4,7 @@ import com.amazonaws.services.lambda.runtime.RequestHandler; import com.amazonaws.services.lambda.runtime.events.SQSEvent; import software.amazon.lambda.powertools.batch3.BatchMessageHandlerBuilder; +import software.amazon.lambda.powertools.idempotency.IdempotencyKey; import software.amazon.lambda.powertools.idempotency.Idempotent; import software.amazon.lambda.powertools.model.Basket; import software.amazon.lambda.powertools.sqs.SqsLargeMessage; @@ -18,22 +19,15 @@ public class SqsExampleWithIdempotency implements RequestHandlerprocessMessage(sqsEvent, basket -> { + .processRawMessage(sqsEvent, this::processWithIdempotency); - }); } @Idempotent @SqsLargeMessage - private void processWithIdempotency(final SQSEvent.SQSMessage message) { - + private void processWithIdempotency(@IdempotencyKey SQSEvent.SQSMessage sqsMessage, Context context) { } } From 96c30ff39a1685c13cf192dcce2050c2f7768786 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Tue, 11 Jul 2023 09:53:11 +0200 Subject: [PATCH 16/79] more changes --- .../powertools/batch3/examples/SqsExampleWithIdempotency.java | 1 + 1 file changed, 1 insertion(+) diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/SqsExampleWithIdempotency.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/SqsExampleWithIdempotency.java index 8d112e88d..6d5684149 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/SqsExampleWithIdempotency.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/SqsExampleWithIdempotency.java @@ -21,6 +21,7 @@ public Object handleRequest(SQSEvent sqsEvent, Context context) { // return ... return new BatchMessageHandlerBuilder() .withSqsBatchHandler() + .withFailureHandler(msg -> System.out.println("Whoops: " + msg.getMessageId())) .processRawMessage(sqsEvent, this::processWithIdempotency); } From 26d8da58fe5108edb787ed34601bbe7c79dbb5f5 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Wed, 19 Jul 2023 16:50:24 +0200 Subject: [PATCH 17/79] Leaning into the builder style. needs some more thought --- .../batch/AbstractMessageHandlerBuilder.java | 59 ++++++++++ .../BatchMessageHandlerBuilder.java | 9 +- .../powertools/batch/BatchRequestHandler.java | 104 ---------------- .../DdbMessageHandlerBuilder.java | 5 +- .../batch/KinesisBatchRequestHandler.java | 23 ---- .../batch/SQSBatchRequestHandler.java | 24 ---- .../batch/SqsBatchMessageHandlerBuilder.java | 31 +++++ .../TooManyMessageHandlersException.java | 9 ++ .../examples/ExampleBatchRequestHandler.java | 15 --- .../batch/handler/BatchMessageHandler.java | 9 ++ .../batch/handler/SqsBatchMessageHandler.java | 91 ++++++++++++++ .../powertools/batch2/BatchProcessor.java | 65 ---------- .../batch2/DynamoDBBatchProcessor.java | 37 ------ .../batch2/KinesisBatchProcessor.java | 43 ------- .../powertools/batch2/SQSBatchProcessor.java | 111 ------------------ .../batch2/examples/DynamoDBBatchHandler.java | 19 --- .../batch2/examples/KinesisBatchHandler.java | 21 ---- .../batch2/examples/SQSBatchHandler.java | 21 ---- .../batch3/SqsBatchMessageHandlerBuilder.java | 51 -------- .../batch3/examples/DynamodbExample.java | 21 ---- .../SqsExampleWithDeserialization.java | 26 ---- .../examples/SqsExampleWithIdempotency.java | 34 ------ .../batch/SQSBatchProcessorTest.java | 68 +++++++++++ .../powertools/batch/SimpleBuilderTest.java | 25 ++++ .../batch2/SQSBatchProcessorTest.java | 68 ----------- 25 files changed, 297 insertions(+), 692 deletions(-) create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/AbstractMessageHandlerBuilder.java rename powertools-batch/src/main/java/software/amazon/lambda/powertools/{batch3 => batch}/BatchMessageHandlerBuilder.java (76%) delete mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchRequestHandler.java rename powertools-batch/src/main/java/software/amazon/lambda/powertools/{batch3 => batch}/DdbMessageHandlerBuilder.java (67%) delete mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/KinesisBatchRequestHandler.java delete mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/SQSBatchRequestHandler.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/SqsBatchMessageHandlerBuilder.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/TooManyMessageHandlersException.java delete mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/examples/ExampleBatchRequestHandler.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/BatchMessageHandler.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java delete mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/BatchProcessor.java delete mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/DynamoDBBatchProcessor.java delete mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/KinesisBatchProcessor.java delete mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessor.java delete mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/DynamoDBBatchHandler.java delete mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/KinesisBatchHandler.java delete mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/SQSBatchHandler.java delete mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/SqsBatchMessageHandlerBuilder.java delete mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/DynamodbExample.java delete mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/SqsExampleWithDeserialization.java delete mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/SqsExampleWithIdempotency.java create mode 100644 powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java create mode 100644 powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SimpleBuilderTest.java delete mode 100644 powertools-batch/src/test/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessorTest.java diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/AbstractMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/AbstractMessageHandlerBuilder.java new file mode 100644 index 000000000..b98f144e8 --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/AbstractMessageHandlerBuilder.java @@ -0,0 +1,59 @@ +package software.amazon.lambda.powertools.batch; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.events.SQSEvent; +import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; + +import java.util.function.BiConsumer; +import java.util.function.Consumer; + +/** + * + * An abstract class to capture common arguments used + * across all the message-binding-specific builders. + * + * @param The type of a single message in the batch + * @param The type of the child builder + * @param The type of the Lambda event + * @param The type of the batch response + */ +abstract class AbstractMessageHandlerBuilder { + protected BiConsumer failureHandler; + protected Consumer successHandler; + protected BiConsumer messageHandler; + protected BiConsumer rawMessageHandler; + + + public C withSuccessHandler(Consumer handler) { + this.successHandler = handler; + return getThis(); + } + + public C withFailureHandler(BiConsumer handler) { + this.failureHandler = handler; + return getThis(); + } + + public C withMessageHandler(BiConsumer handler) { + if (this.rawMessageHandler != null) { + throw new TooManyMessageHandlersException(); + } + + this.messageHandler = handler; + return getThis(); + } + + public C withRawMessageHandler(BiConsumer handler) { + if (this.messageHandler != null) { + throw new TooManyMessageHandlersException(); + } + + this.rawMessageHandler = handler; + return getThis(); + } + + public abstract BatchMessageHandler build(); + + + protected abstract C getThis(); +} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/BatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java similarity index 76% rename from powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/BatchMessageHandlerBuilder.java rename to powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java index f1b1fe6d3..cf31d78e6 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/BatchMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java @@ -1,15 +1,12 @@ -package software.amazon.lambda.powertools.batch3; +package software.amazon.lambda.powertools.batch; - -import software.amazon.lambda.powertools.batch3.examples.SqsExampleWithIdempotency; +import software.amazon.lambda.powertools.batch.SqsBatchMessageHandlerBuilder; /** * A builder-style interface we can use within an existing Lambda RequestHandler to * deal with our batch responses. A second tier of builders is returned per-event-source * to bind the appropriate message types and provider source-specific logic and tuneables. - * - * @see SqsExampleWithIdempotency - */ + **/ public class BatchMessageHandlerBuilder { public SqsBatchMessageHandlerBuilder withSqsBatchHandler() { diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchRequestHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchRequestHandler.java deleted file mode 100644 index 7ab54173e..000000000 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchRequestHandler.java +++ /dev/null @@ -1,104 +0,0 @@ -package software.amazon.lambda.powertools.batch; - -import com.amazonaws.services.lambda.runtime.Context; -import com.amazonaws.services.lambda.runtime.RequestHandler; -import software.amazon.lambda.powertools.batch3.BatchMessageHandlerBuilder; - -import java.util.List; -import java.util.stream.Collectors; - -/** - * An abstract base class that will be extended per-event-source to supply event-source - * specific functionality. As much as possible is pulled into the base class, and - * event-source specific logic is delegated downwards (e.g., mapping errors to response, - * extracting messages from batch). - * - * IRL, this would be implemented leaning on the {@link BatchMessageHandlerBuilder}, here - * I have provided implementation inline for illustrative purposes. - * - * Note - this is package private. We don't let users extend this, which gives us - * much more flexibility to add extra functionality later by minimising our API surface - * area. - * - * @param The batch event type - * @param The individual message type for each message within teh batch - * @param The batch result type - */ -abstract class BatchRequestHandler implements RequestHandler { - - /** - * Used to wrap the result of processing a single message. We wrap the message itself, - * and optionally an exception that was raised during the processing. A lack of - * exception indicates success. - */ - protected class MessageProcessingResult { - private final U message; - private final Exception exception; - - public MessageProcessingResult(U message, Exception exception) { - this.message = message; - this.exception = exception; - } - } - - /** - * The batch processing logic goes here. This can be generic across all message types. - * - * @param input The batch message to process - * @param context The Lambda execution environment context object. - * @return - */ - @Override - public V handleRequest(T input, Context context) { - // Extract messages from batch - List messages = extractMessages(input); - - // For each message, map it to either 1/ a successful result, or 2/ an exception - List> results = messages.stream().map(m -> { - try { - enhanceMessage(m); - processItem(m, context); - return new MessageProcessingResult<>(m, null); - } catch (Exception e) { - return new MessageProcessingResult<>(m, e); - } - }).collect(Collectors.toList()); - - // Generate the response from the list of results - return writeResponse(results); - } - - /** - * Provided by the event-specific child to extract the individual records - * from the batch request - * - * @param input The batch - * @return the messages within the batch - */ - protected abstract List extractMessages(T input); - - /** - * Given the set of message processing results, generates the appropriate batch - * processing result to return to Lambda. - * - * @param results the result of processing each message, and the messages themselves - * @return the batch processing result to return to lambda - */ - protected abstract V writeResponse(Iterable> results); - - /** - * To be provided by the user. Processes an individual message within the batch - * @param message - * @param context - */ - public abstract void processItem(U message, Context context); - - /** - * This could be overriden by event-specific children to implement things like large - * message processing. - * @param message - */ - protected void enhanceMessage(U message) { - - } -} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/DdbMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/DdbMessageHandlerBuilder.java similarity index 67% rename from powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/DdbMessageHandlerBuilder.java rename to powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/DdbMessageHandlerBuilder.java index b61c29273..3dcb98add 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/DdbMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/DdbMessageHandlerBuilder.java @@ -1,14 +1,13 @@ -package software.amazon.lambda.powertools.batch3; +package software.amazon.lambda.powertools.batch; import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; -import sun.reflect.generics.reflectiveObjects.NotImplementedException; import java.util.function.Consumer; public class DdbMessageHandlerBuilder { public StreamsEventResponse processBatch(DynamodbEvent batch, Consumer handler) { - throw new NotImplementedException(); + throw new RuntimeException("Not implemented"); } } diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/KinesisBatchRequestHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/KinesisBatchRequestHandler.java deleted file mode 100644 index 65ac7b4c6..000000000 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/KinesisBatchRequestHandler.java +++ /dev/null @@ -1,23 +0,0 @@ -package software.amazon.lambda.powertools.batch; - -import com.amazonaws.services.lambda.runtime.Context; -import com.amazonaws.services.lambda.runtime.events.KinesisEvent; -import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; -import sun.reflect.generics.reflectiveObjects.NotImplementedException; - -import java.util.List; - -public abstract class KinesisBatchRequestHandler extends BatchRequestHandler { - - @Override - protected List extractMessages(KinesisEvent input) { - return input.getRecords(); - } - - @Override - protected StreamsEventResponse writeResponse(Iterable> results) { - // Here we map up the kinesis-specific response for the batch based on the success of the individual messages - throw new NotImplementedException(); - } - -} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/SQSBatchRequestHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/SQSBatchRequestHandler.java deleted file mode 100644 index 92e2f5731..000000000 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/SQSBatchRequestHandler.java +++ /dev/null @@ -1,24 +0,0 @@ -package software.amazon.lambda.powertools.batch; - -import com.amazonaws.services.lambda.runtime.Context; -import com.amazonaws.services.lambda.runtime.events.KinesisEvent; -import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; -import com.amazonaws.services.lambda.runtime.events.SQSEvent; -import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; -import sun.reflect.generics.reflectiveObjects.NotImplementedException; - -import java.util.List; - -public abstract class SQSBatchRequestHandler extends BatchRequestHandler { - - @Override - protected List extractMessages(SQSEvent input) { - return input.getRecords(); - } - - @Override - protected SQSBatchResponse writeResponse(Iterable> results) { - // Here we map up the SQS-specific response for the batch based on the success of the individual messages - throw new NotImplementedException(); - } -} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/SqsBatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/SqsBatchMessageHandlerBuilder.java new file mode 100644 index 000000000..0b6c0fefe --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/SqsBatchMessageHandlerBuilder.java @@ -0,0 +1,31 @@ +package software.amazon.lambda.powertools.batch; + +import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; +import com.amazonaws.services.lambda.runtime.events.SQSEvent; +import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; +import software.amazon.lambda.powertools.batch.handler.SqsBatchMessageHandler; + +/** + * Builds a batch processor for the SQS event source. + */ +public class SqsBatchMessageHandlerBuilder extends AbstractMessageHandlerBuilder { + + @Override + protected SqsBatchMessageHandlerBuilder getThis() { + return this; + } + + @Override + public BatchMessageHandler build() { + + return new SqsBatchMessageHandler( + messageHandler, + rawMessageHandler, + successHandler, + failureHandler + ); + } +} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/TooManyMessageHandlersException.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/TooManyMessageHandlersException.java new file mode 100644 index 000000000..ec3d70f9d --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/TooManyMessageHandlersException.java @@ -0,0 +1,9 @@ +package software.amazon.lambda.powertools.batch; + +public class TooManyMessageHandlersException extends RuntimeException { + + public TooManyMessageHandlersException() { + super("You must configure either a rawMessageHandler or a messageHandler - not both"); + } + +} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/examples/ExampleBatchRequestHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/examples/ExampleBatchRequestHandler.java deleted file mode 100644 index 9a1e74bf8..000000000 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/examples/ExampleBatchRequestHandler.java +++ /dev/null @@ -1,15 +0,0 @@ -package software.amazon.lambda.powertools.batch.examples; - -import com.amazonaws.services.lambda.runtime.Context; -import com.amazonaws.services.lambda.runtime.events.SQSEvent; -import software.amazon.lambda.powertools.batch.SQSBatchRequestHandler; - -/** - * An example handler that is implemented by extending our {@link SQSBatchRequestHandler} - */ -public class ExampleBatchRequestHandler extends SQSBatchRequestHandler { - @Override - public void processItem(SQSEvent.SQSMessage message, Context context) { - // Process an SQS message without throwing - } -} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/BatchMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/BatchMessageHandler.java new file mode 100644 index 000000000..94280f1eb --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/BatchMessageHandler.java @@ -0,0 +1,9 @@ +package software.amazon.lambda.powertools.batch.handler; + +import com.amazonaws.services.lambda.runtime.Context; + +public interface BatchMessageHandler { + + public abstract R processBatch(E event, Context context); + +} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java new file mode 100644 index 000000000..671c25430 --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java @@ -0,0 +1,91 @@ +package software.amazon.lambda.powertools.batch.handler; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; +import com.amazonaws.services.lambda.runtime.events.SQSEvent; +import software.amazon.lambda.powertools.utilities.EventDeserializer; +import sun.reflect.generics.reflectiveObjects.ParameterizedTypeImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.function.BiConsumer; +import java.util.function.Consumer; + +public class SqsBatchMessageHandler implements BatchMessageHandler { + Logger SQS_BATCH_LOGGER = LoggerFactory.getLogger(SqsBatchMessageHandler.class); + + // The attribute on an SQS-FIFO message used to record the message group ID + // https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#sample-fifo-queues-message-event + String MESSAGE_GROUP_ID_KEY = "MessageGroupId"; + + private final BiConsumer messageHandler; + private final BiConsumer rawMessageHandler; + private final Consumer successHandler; + private final BiConsumer failureHandler; + private final Class bodyClass; + + public SqsBatchMessageHandler(BiConsumer messageHandler, BiConsumer rawMessageHandler, Consumer successHandler, BiConsumer failureHandler) { + this.messageHandler = messageHandler; + this.rawMessageHandler = rawMessageHandler; + this.successHandler = successHandler; + this.failureHandler = failureHandler; + + // If we've got a message handler, work out once now what type we have to deserialize + if (this.messageHandler != null) { + this.bodyClass = (Class) ((ParameterizedTypeImpl) messageHandler.getClass() + .getGenericInterfaces()[0]) + .getActualTypeArguments()[0]; + } else { + bodyClass = null; + } + } + + @Override + public SQSBatchResponse processBatch(SQSEvent event, Context context) { + SQSBatchResponse response = SQSBatchResponse.builder().withBatchItemFailures(new ArrayList<>()).build(); + + // If we are working on a FIFO queue, when any message fails we should stop processing and return the + // rest of the batch as failed too. We use this variable to track when that has happened. + // https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#services-sqs-batchfailurereporting + boolean failWholeBatch = false; + + int messageCursor = 0; + for (; messageCursor < event.getRecords().size() && !failWholeBatch; messageCursor++) { + SQSEvent.SQSMessage message = event.getRecords().get(messageCursor); + + String messageGroupId = message.getAttributes() != null ? + message.getAttributes().get(MESSAGE_GROUP_ID_KEY) : null; + + try { + if (this.rawMessageHandler != null) { + rawMessageHandler.accept(message, context); + } else { + // TODO this is bad bad not good + // TODO fix + // TODO either with type bounds for the concrete consumer type on the builder (and buildWithHandler(..)) + // TODO .... or by making this cast in the constructor + Object messageDeserialized = EventDeserializer.extractDataFrom(message).as(bodyClass); + BiConsumer consumerCast = (BiConsumer) messageHandler; + consumerCast.accept(messageDeserialized, context); + } + } catch (Throwable t) { + SQS_BATCH_LOGGER.error("Error while processing message with messageId {}: {}, adding it to batch item failures", message.getMessageId(), t.getMessage()); + response.getBatchItemFailures().add(SQSBatchResponse.BatchItemFailure.builder().withItemIdentifier(message.getMessageId()).build()); + if (messageGroupId != null) { + failWholeBatch = true; + SQS_BATCH_LOGGER.info("A message in a batch with messageGroupId {} and messageId {} failed; failing the rest of the batch too" + , messageGroupId, message.getMessageId()); + } + } + } + + if (failWholeBatch) { + // Add the remaining messages to the batch item failures + event.getRecords() + .subList(messageCursor, event.getRecords().size()) + .forEach(message -> response.getBatchItemFailures().add(SQSBatchResponse.BatchItemFailure.builder().withItemIdentifier(message.getMessageId()).build())); + } + return response; + } +} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/BatchProcessor.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/BatchProcessor.java deleted file mode 100644 index 33529dfa3..000000000 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/BatchProcessor.java +++ /dev/null @@ -1,65 +0,0 @@ -package software.amazon.lambda.powertools.batch2; - -import com.amazonaws.services.lambda.runtime.Context; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Generic interface for batch processing with Lambda. This interface provides 2 main methods: - *
    - *
  • {@link #processBatch(Object, Context)} that will take a batch of items (messages, records), - * loop over it, and trigger the processItem function for each item. - * This method is also in charge to handle partial batch failures.
    • - *
  • {@link #processItem(Object, Context)} that will handle each item of the batch. This method is generally implemented by the handler or any other class in charge of processing items. - * If there are exceptions within this method, you can let them go and the processBatch method will handle them and add the messages to the partial batch failures.
    • - *
- * - * @param Type of the incoming event. Natively supported: - *
    - *
  • {@link com.amazonaws.services.lambda.runtime.events.SQSEvent}
    • - *
  • {@link com.amazonaws.services.lambda.runtime.events.KinesisEvent}
    • - *
  • {@link com.amazonaws.services.lambda.runtime.events.DynamodbEvent}
    • - *
- * @param Type of the message or inner element in the message (e.g. body). Natively supported: - *
    - *
  • {@link com.amazonaws.services.lambda.runtime.events.SQSEvent.SQSMessage}
    • - *
  • {@link com.amazonaws.services.lambda.runtime.events.KinesisEvent.KinesisEventRecord}
    • - *
  • {@link com.amazonaws.services.lambda.runtime.events.DynamodbEvent.DynamodbStreamRecord}
    • - *
- * If you choose your own model type, the content of the body within the message will be deserialized. - *
    - *
  • For SQS, it will be {@link com.amazonaws.services.lambda.runtime.events.SQSEvent.SQSMessage#getBody()}
    • - *
  • For Kinesis, it will be {@link com.amazonaws.services.lambda.runtime.events.models.kinesis.Record#getData()} (base 64 decoded)
    • - *
  • Not supported for DynamoDB
    • - *
- * @param Type of the handler response. Natively supported: - *
    - *
  • {@link com.amazonaws.services.lambda.runtime.events.SQSBatchResponse} for SQS.
    • - *
  • {@link com.amazonaws.services.lambda.runtime.events.StreamsEventResponse} for Kinesis and DynamoDB.
    • - *
- */ -public interface BatchProcessor { - - Logger BATCH_LOGGER = LoggerFactory.getLogger(BatchProcessor.class); - - /** - * This method takes a batch of items (messages, records) from an event, - * loop over it, and trigger the processItem function for each item. - * @param event handler event (e.g. {@link com.amazonaws.services.lambda.runtime.events.SQSEvent}) - * @param context handler context - * @return the handler response containing partial batch failures (e.g. {@link com.amazonaws.services.lambda.runtime.events.SQSBatchResponse}) - */ - RESPONSE processBatch(EVENT event, Context context); - - /** - * This method handles each item of the batch. This method is generally implemented by the handler or any other class in charge of processing items. - * If there are exceptions within this method, you can let them go and the processBatch method will handle them and add the messages to the partial batch failures. - * This method has a default implementation because it's not available for DynamoDB. - * @param item the item to process (e.g. {@link com.amazonaws.services.lambda.runtime.events.SQSEvent.SQSMessage}) - * @param context handler context - */ - default void processItem(ITEM item, Context context) { - BATCH_LOGGER.debug("[DEFAULT IMPLEMENTATION] Processing custom item"); - } - -} \ No newline at end of file diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/DynamoDBBatchProcessor.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/DynamoDBBatchProcessor.java deleted file mode 100644 index d44656706..000000000 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/DynamoDBBatchProcessor.java +++ /dev/null @@ -1,37 +0,0 @@ -package software.amazon.lambda.powertools.batch2; - -import com.amazonaws.services.lambda.runtime.Context; -import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; -import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; -import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse.BatchItemFailure; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; - -public interface DynamoDBBatchProcessor extends BatchProcessor { - - Logger DDB_BATCH_LOGGER = LoggerFactory.getLogger(SQSBatchProcessor.class); - - @Override - default StreamsEventResponse processBatch(DynamodbEvent event, Context context) { - StreamsEventResponse response = StreamsEventResponse.builder().withBatchItemFailures(new ArrayList<>()).build(); - for (DynamodbEvent.DynamodbStreamRecord record : event.getRecords()) { - try { - processRecord(record, context); - } catch (Throwable t) { - DDB_BATCH_LOGGER.error("Error while processing record with id {}: {}, adding it to batch item failures", record.getEventID(), t.getMessage()); - response.getBatchItemFailures().add(BatchItemFailure.builder().withItemIdentifier(record.getEventID()).build()); - } - } - return response; - } - - @Override - default void processItem(Void message, Context context) { - DDB_BATCH_LOGGER.warn("[DEFAULT IMPLEMENTATION] This method should not be used in the case of DynamoDB"); - } - - void processRecord(DynamodbEvent.DynamodbStreamRecord record, Context context); - -} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/KinesisBatchProcessor.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/KinesisBatchProcessor.java deleted file mode 100644 index 26b318b2b..000000000 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/KinesisBatchProcessor.java +++ /dev/null @@ -1,43 +0,0 @@ -package software.amazon.lambda.powertools.batch2; - -import com.amazonaws.services.lambda.runtime.Context; -import com.amazonaws.services.lambda.runtime.events.KinesisEvent; -import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; -import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse.BatchItemFailure; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import software.amazon.lambda.powertools.utilities.EventDeserializer; -import sun.reflect.generics.reflectiveObjects.ParameterizedTypeImpl; - -import java.util.ArrayList; - -public interface KinesisBatchProcessor extends BatchProcessor { - - Logger KINESIS_BATCH_LOGGER = LoggerFactory.getLogger(KinesisBatchProcessor.class); - - @Override - default StreamsEventResponse processBatch(KinesisEvent event, Context context) { - Class bodyClass = (Class) ((ParameterizedTypeImpl) getClass().getGenericInterfaces()[0]).getActualTypeArguments()[0]; - boolean isKinesisRecord = bodyClass.equals(KinesisEvent.KinesisEventRecord.class); - - StreamsEventResponse response = StreamsEventResponse.builder().withBatchItemFailures(new ArrayList<>()).build(); - for (KinesisEvent.KinesisEventRecord record : event.getRecords()) { - try { - if (isKinesisRecord) { - processRecord(record, context); - } else { - processItem(EventDeserializer.extractDataFrom(record).as(bodyClass), context); - } - } catch (Throwable t) { - KINESIS_BATCH_LOGGER.error("Error while processing record with id {}: {}, adding it to batch item failures", record.getEventID(), t.getMessage()); - response.getBatchItemFailures().add(BatchItemFailure.builder().withItemIdentifier(record.getEventID()).build()); - } - } - return response; - } - - default void processRecord(KinesisEvent.KinesisEventRecord record, Context context) { - KINESIS_BATCH_LOGGER.debug("[DEFAULT IMPLEMENTATION] Processing record {}", record.getEventID()); - } - -} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessor.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessor.java deleted file mode 100644 index c66139a92..000000000 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessor.java +++ /dev/null @@ -1,111 +0,0 @@ -package software.amazon.lambda.powertools.batch2; - -import com.amazonaws.services.lambda.runtime.Context; -import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; -import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse.BatchItemFailure; -import com.amazonaws.services.lambda.runtime.events.SQSEvent; -import com.amazonaws.services.lambda.runtime.events.SQSEvent.SQSMessage; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import software.amazon.lambda.powertools.utilities.EventDeserializer; -import sun.reflect.generics.reflectiveObjects.ParameterizedTypeImpl; - -import java.util.ArrayList; - -/** - * SQSBatchProcessor is an interface that can be implemented by your handler or any other class - * to process batches of SQS messages. It handles the partial batch failures. - * It extends the {@link BatchProcessor} interface and provide - * an implementation for the @link {@link BatchProcessor#processBatch(Object, Context)} method.
- * When you implement this interface you have the choice to implement the {@link #processMessage(SQSMessage, Context)} - * method OR {@link #processItem(Object, Context)} method (do not implement both, only one will be used):
    - *
  • If you need access to the whole {@link SQSMessage} - * (with the attributes, messageAttributes, ...), use the processMessage method.
    • - *
  • If you simply want to access the body of the message and have it deserialized as ITEM type, - * use the processItem method.
    • - *
- * The choice is made by typing the SQSBatchProcessor either with {@link SQSMessage}: - * 
- *     public class SQSBatchHandler implements SQSBatchProcessor<SQSMessage> {
- *         public void processMessage(SQSEvent.SQSMessage message, Context context) {
- *             // access message information (id, attributes, messageAttributes, ...)
- *             MessageAttribute attribute = message.getMessageAttributes().get("myAttribute");
- *
- *             // access the body manually (message.getBody())
- *             // using powertools-serialization module, you can easily deserialize the message body
- *             MyModel myModel = EventDeserializer.extractDataFrom(message).as(MyModel.class)
- *         }
- *     }
- *
- * - * Or with the type of your choice: - * 
- *     public class SQSBatchHandler implements SQSBatchProcessor<MyModel> {
- *         public void processItem(MyModel myModel, Context context) {
- *             // ... do some stuff with your model
- *         }
- *     }
- *
- * - * In each case, you have access to the Lambda context if needed.
- * - * @param Type of the messages, can be either {@link SQSMessage} for the whole message or any type for the message body only. - */ -public interface SQSBatchProcessor extends BatchProcessor { - - Logger SQS_BATCH_LOGGER = LoggerFactory.getLogger(SQSBatchProcessor.class); - - // The attribute on an SQS-FIFO message used to record the message group ID - // https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#sample-fifo-queues-message-event - String MESSAGE_GROUP_ID_KEY = "MessageGroupId"; - - @Override - default SQSBatchResponse processBatch(SQSEvent event, Context context) { - Class bodyClass = (Class) ((ParameterizedTypeImpl) getClass().getGenericInterfaces()[0]).getActualTypeArguments()[0]; - boolean isSQSMessage = bodyClass.getCanonicalName().equals(SQSMessage.class.getCanonicalName()); - - SQSBatchResponse response = SQSBatchResponse.builder().withBatchItemFailures(new ArrayList<>()).build(); - - // If we are working on a FIFO queue, when any message fails we should stop processing and return the - // rest of the batch as failed too. We use this variable to track when that has happened. - // https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#services-sqs-batchfailurereporting - boolean failWholeBatch = false; - - int messageCursor = 0; - for (; messageCursor < event.getRecords().size() && !failWholeBatch; messageCursor++) { - SQSMessage message = event.getRecords().get(messageCursor); - - String messageGroupId = message.getAttributes() != null ? - message.getAttributes().get(MESSAGE_GROUP_ID_KEY) : null; - - try { - if (isSQSMessage) { - processMessage(message, context); - } else { - processItem(EventDeserializer.extractDataFrom(message).as(bodyClass), context); - } - } catch (Throwable t) { - SQS_BATCH_LOGGER.error("Error while processing message with messageId {}: {}, adding it to batch item failures", message.getMessageId(), t.getMessage()); - response.getBatchItemFailures().add(BatchItemFailure.builder().withItemIdentifier(message.getMessageId()).build()); - if (messageGroupId != null) { - failWholeBatch = true; - SQS_BATCH_LOGGER.info("A message in a batch with messageGroupId {} and messageId {} failed; failing the rest of the batch too" - , messageGroupId, message.getMessageId()); - } - } - } - - if (failWholeBatch) { - // Add the remaining messages to the batch item failures - event.getRecords() - .subList(messageCursor, event.getRecords().size()) - .forEach(message -> response.getBatchItemFailures().add(BatchItemFailure.builder().withItemIdentifier(message.getMessageId()).build())); - } - return response; - } - - default void processMessage(SQSMessage message, Context context) { - SQS_BATCH_LOGGER.debug("[DEFAULT IMPLEMENTATION] Processing message {}", message.getMessageId()); - } - -} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/DynamoDBBatchHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/DynamoDBBatchHandler.java deleted file mode 100644 index 42a2a6afe..000000000 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/DynamoDBBatchHandler.java +++ /dev/null @@ -1,19 +0,0 @@ -package software.amazon.lambda.powertools.batch2.examples; - -import com.amazonaws.services.lambda.runtime.Context; -import com.amazonaws.services.lambda.runtime.RequestHandler; -import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; -import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; -import software.amazon.lambda.powertools.batch2.DynamoDBBatchProcessor; - -public class DynamoDBBatchHandler implements RequestHandler, DynamoDBBatchProcessor { - @Override - public StreamsEventResponse handleRequest(DynamodbEvent input, Context context) { - return processBatch(input, context); - } - - @Override - public void processRecord(DynamodbEvent.DynamodbStreamRecord record, Context context) { - - } -} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/KinesisBatchHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/KinesisBatchHandler.java deleted file mode 100644 index fd70f8394..000000000 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/KinesisBatchHandler.java +++ /dev/null @@ -1,21 +0,0 @@ -package software.amazon.lambda.powertools.batch2.examples; - -import com.amazonaws.services.lambda.runtime.Context; -import com.amazonaws.services.lambda.runtime.RequestHandler; -import com.amazonaws.services.lambda.runtime.events.KinesisEvent; -import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; -import software.amazon.lambda.powertools.batch2.KinesisBatchProcessor; -import software.amazon.lambda.powertools.model.Basket; - -public class KinesisBatchHandler implements RequestHandler, KinesisBatchProcessor { - @Override - public StreamsEventResponse handleRequest(KinesisEvent input, Context context) { - return processBatch(input, context); - } - - @Override - public void processItem(Basket basket, Context context) { - // do some stuff with the item - System.out.println(basket.toString()); - } -} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/SQSBatchHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/SQSBatchHandler.java deleted file mode 100644 index 717787d27..000000000 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/examples/SQSBatchHandler.java +++ /dev/null @@ -1,21 +0,0 @@ -package software.amazon.lambda.powertools.batch2.examples; - -import com.amazonaws.services.lambda.runtime.Context; -import com.amazonaws.services.lambda.runtime.RequestHandler; -import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; -import com.amazonaws.services.lambda.runtime.events.SQSEvent; -import software.amazon.lambda.powertools.batch2.SQSBatchProcessor; -import software.amazon.lambda.powertools.model.Basket; - -public class SQSBatchHandler implements RequestHandler, SQSBatchProcessor { - @Override - public SQSBatchResponse handleRequest(SQSEvent input, Context context) { - return processBatch(input, context); - } - - @Override - public void processItem(Basket basket, Context context) { - // do some stuff with the item - System.out.println(basket.toString()); - } -} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/SqsBatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/SqsBatchMessageHandlerBuilder.java deleted file mode 100644 index 46cee8da3..000000000 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/SqsBatchMessageHandlerBuilder.java +++ /dev/null @@ -1,51 +0,0 @@ -package software.amazon.lambda.powertools.batch3; - -import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; -import com.amazonaws.services.lambda.runtime.events.SQSEvent; -import sun.reflect.generics.reflectiveObjects.NotImplementedException; -import com.amazonaws.services.lambda.runtime.Context; - -import java.util.function.BiConsumer; -import java.util.function.Consumer; - -/** - * Builds a batch processor for the SQS event source. - */ -public class SqsBatchMessageHandlerBuilder { - - private Consumer failureHandler; - private Consumer successHandler; - - /** - * success and failure handler hooks are used as an example of a tuneable from the python - * powertools. - * - * https://docs.powertools.aws.dev/lambda/python/2.15.0/utilities/batch/#extending-batchprocessor - * - * @param handler - * @return - */ - public SqsBatchMessageHandlerBuilder withSuccessHandler(Consumer handler) { - this.successHandler = handler; - return this; - } - - public SqsBatchMessageHandlerBuilder withFailureHandler(Consumer handler) { - this.failureHandler = handler; - return this; - } - - /** - * The user can consume either raw messages .... - */ - public SQSBatchResponse processMessage(SQSEvent event, BiConsumer handler) { - throw new NotImplementedException(); - } - - /** - * ... or deserialized messages - */ - public SQSBatchResponse processRawMessage(SQSEvent event, BiConsumer handler) { - throw new NotImplementedException(); - } -} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/DynamodbExample.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/DynamodbExample.java deleted file mode 100644 index 433b9ba84..000000000 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/DynamodbExample.java +++ /dev/null @@ -1,21 +0,0 @@ -package software.amazon.lambda.powertools.batch3.examples; - -import com.amazonaws.services.lambda.runtime.Context; -import com.amazonaws.services.lambda.runtime.RequestHandler; -import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; -import com.amazonaws.services.lambda.runtime.events.SQSEvent; -import software.amazon.lambda.powertools.batch3.BatchMessageHandlerBuilder; -import software.amazon.lambda.powertools.model.Basket; - -public class DynamodbExample implements RequestHandler { - - @Override - public Object handleRequest(DynamodbEvent ddbEvent, Context context) { - // Example 2 - process a deserialized message - return new BatchMessageHandlerBuilder() - .withDynamoDbBatchHandler() - .processBatch(ddbEvent, basket -> { - - }); - } -} \ No newline at end of file diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/SqsExampleWithDeserialization.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/SqsExampleWithDeserialization.java deleted file mode 100644 index 222eda4c7..000000000 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/SqsExampleWithDeserialization.java +++ /dev/null @@ -1,26 +0,0 @@ -package software.amazon.lambda.powertools.batch3.examples; - -import com.amazonaws.services.lambda.runtime.Context; -import com.amazonaws.services.lambda.runtime.RequestHandler; -import com.amazonaws.services.lambda.runtime.events.SQSEvent; -import software.amazon.lambda.powertools.batch3.BatchMessageHandlerBuilder; -import software.amazon.lambda.powertools.idempotency.Idempotent; -import software.amazon.lambda.powertools.model.Basket; - -/** - * An example handler that is implemented by using the {@link BatchMessageHandlerBuilder} inline - * in an existing RequestHandler. - */ -public class SqsExampleWithDeserialization implements RequestHandler { - - @Override - public Object handleRequest(SQSEvent sqsEvent, Context context) { - // Example 2 - process a deserialized message - return new BatchMessageHandlerBuilder() - .withSqsBatchHandler() - .processMessage(sqsEvent, (basket, ctx) -> { - - }); - } -} - diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/SqsExampleWithIdempotency.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/SqsExampleWithIdempotency.java deleted file mode 100644 index 6d5684149..000000000 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch3/examples/SqsExampleWithIdempotency.java +++ /dev/null @@ -1,34 +0,0 @@ -package software.amazon.lambda.powertools.batch3.examples; - -import com.amazonaws.services.lambda.runtime.Context; -import com.amazonaws.services.lambda.runtime.RequestHandler; -import com.amazonaws.services.lambda.runtime.events.SQSEvent; -import software.amazon.lambda.powertools.batch3.BatchMessageHandlerBuilder; -import software.amazon.lambda.powertools.idempotency.IdempotencyKey; -import software.amazon.lambda.powertools.idempotency.Idempotent; -import software.amazon.lambda.powertools.model.Basket; -import software.amazon.lambda.powertools.sqs.SqsLargeMessage; - -/** - * An example handler that is implemented by using the {@link BatchMessageHandlerBuilder} inline - * in an existing RequestHandler. - */ -public class SqsExampleWithIdempotency implements RequestHandler { - - @Override - public Object handleRequest(SQSEvent sqsEvent, Context context) { - // Example 1 - process a raw SQS message in an idempotent fashion - // return ... - return new BatchMessageHandlerBuilder() - .withSqsBatchHandler() - .withFailureHandler(msg -> System.out.println("Whoops: " + msg.getMessageId())) - .processRawMessage(sqsEvent, this::processWithIdempotency); - - } - - @Idempotent - @SqsLargeMessage - private void processWithIdempotency(@IdempotencyKey SQSEvent.SQSMessage sqsMessage, Context context) { - } - -} diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java new file mode 100644 index 000000000..4934512e3 --- /dev/null +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java @@ -0,0 +1,68 @@ +package software.amazon.lambda.powertools.batch; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; +import com.amazonaws.services.lambda.runtime.events.SQSEvent; +import com.amazonaws.services.lambda.runtime.tests.annotations.Event; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.mockito.Mock; +import software.amazon.lambda.powertools.model.Product; + +public class SQSBatchProcessorTest { + @Mock + private Context context; + + +// static class SQSBP implements SQSBatchProcessor { +// public void processMessage(SQSEvent.SQSMessage message, Context context) { +// if (message.getMessageId().equals("e9144555-9a4f-4ec3-99a0-34ce359b4b54")) { +// throw new RuntimeException("fake exception"); +// } +// } +// } +// +// @ParameterizedTest +// @Event(value = "sqs_event.json", type = SQSEvent.class) +// public void shouldAddMessageToBatchFailure_whenException_withMessage(SQSEvent event) { +// SQSBP sqsbp = new SQSBP(); +// SQSBatchResponse sqsBatchResponse = sqsbp.processBatch(event, context); +// Assertions.assertThat(sqsBatchResponse.getBatchItemFailures()).hasSize(1); +// +// SQSBatchResponse.BatchItemFailure batchItemFailure = sqsBatchResponse.getBatchItemFailures().get(0); +// Assertions.assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("e9144555-9a4f-4ec3-99a0-34ce359b4b54"); +// } +// +// @ParameterizedTest +// @Event(value = "sqs_fifo_event.json", type = SQSEvent.class) +// public void shouldAddMessageToBatchFailure_whenException_withSQSFIFO(SQSEvent event) { +// SQSBP sqsbp = new SQSBP(); +// SQSBatchResponse sqsBatchResponse = sqsbp.processBatch(event, context); +// Assertions.assertThat(sqsBatchResponse.getBatchItemFailures()).hasSize(2); +// +// SQSBatchResponse.BatchItemFailure batchItemFailure = sqsBatchResponse.getBatchItemFailures().get(0); +// Assertions.assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("e9144555-9a4f-4ec3-99a0-34ce359b4b54"); +// batchItemFailure = sqsBatchResponse.getBatchItemFailures().get(1); +// Assertions.assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("f9144555-9a4f-4ec3-99a0-34ce359b4b54"); +// } +// +// static class SQSBProduct implements SQSBatchProcessor { +// public void processItem(Product product, Context context) { +// if (product.getId() == 12345) { +// throw new RuntimeException("fake exception"); +// } +// } +// } +// +// @ParameterizedTest +// @Event(value = "sqs_event.json", type = SQSEvent.class) +// public void shouldAddMessageToBatchFailure_whenException_withProduct(SQSEvent event) { +// SQSBProduct sqsbproduct = new SQSBProduct(); +// SQSBatchResponse sqsBatchResponse = sqsbproduct.processBatch(event, context); +// Assertions.assertThat(sqsBatchResponse.getBatchItemFailures()).hasSize(1); +// +// SQSBatchResponse.BatchItemFailure batchItemFailure = sqsBatchResponse.getBatchItemFailures().get(0); +// Assertions.assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("e9144555-9a4f-4ec3-99a0-34ce359b4b54"); +// } + +} diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SimpleBuilderTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SimpleBuilderTest.java new file mode 100644 index 000000000..9a46d51c5 --- /dev/null +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SimpleBuilderTest.java @@ -0,0 +1,25 @@ +package software.amazon.lambda.powertools.batch; + +import org.junit.jupiter.api.Test; +import software.amazon.lambda.powertools.batch.SqsBatchMessageHandlerBuilder; +import software.amazon.lambda.powertools.model.Basket; +import com.amazonaws.services.lambda.runtime.Context; + +import static org.assertj.core.api.Assertions.assertThat; + +public class SimpleBuilderTest { + + @Test + public void sqsBuilderWorks() { + SqsBatchMessageHandlerBuilder builder = new BatchMessageHandlerBuilder() + .withSqsBatchHandler() + .withFailureHandler((c, e) -> System.out.println("Whoops!")) + .withMessageHandler(this::processMessage); + + assertThat(builder).isNotNull(); + } + + public void processMessage(Basket basket, Context context) { + + } +} diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessorTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessorTest.java deleted file mode 100644 index 35d259437..000000000 --- a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch2/SQSBatchProcessorTest.java +++ /dev/null @@ -1,68 +0,0 @@ -package software.amazon.lambda.powertools.batch2; - -import com.amazonaws.services.lambda.runtime.Context; -import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; -import com.amazonaws.services.lambda.runtime.events.SQSEvent; -import com.amazonaws.services.lambda.runtime.tests.annotations.Event; -import org.assertj.core.api.Assertions; -import org.junit.jupiter.params.ParameterizedTest; -import org.mockito.Mock; -import software.amazon.lambda.powertools.model.Product; - -public class SQSBatchProcessorTest { - @Mock - private Context context; - - - static class SQSBP implements SQSBatchProcessor { - public void processMessage(SQSEvent.SQSMessage message, Context context) { - if (message.getMessageId().equals("e9144555-9a4f-4ec3-99a0-34ce359b4b54")) { - throw new RuntimeException("fake exception"); - } - } - } - - @ParameterizedTest - @Event(value = "sqs_event.json", type = SQSEvent.class) - public void shouldAddMessageToBatchFailure_whenException_withMessage(SQSEvent event) { - SQSBP sqsbp = new SQSBP(); - SQSBatchResponse sqsBatchResponse = sqsbp.processBatch(event, context); - Assertions.assertThat(sqsBatchResponse.getBatchItemFailures()).hasSize(1); - - SQSBatchResponse.BatchItemFailure batchItemFailure = sqsBatchResponse.getBatchItemFailures().get(0); - Assertions.assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("e9144555-9a4f-4ec3-99a0-34ce359b4b54"); - } - - @ParameterizedTest - @Event(value = "sqs_fifo_event.json", type = SQSEvent.class) - public void shouldAddMessageToBatchFailure_whenException_withSQSFIFO(SQSEvent event) { - SQSBP sqsbp = new SQSBP(); - SQSBatchResponse sqsBatchResponse = sqsbp.processBatch(event, context); - Assertions.assertThat(sqsBatchResponse.getBatchItemFailures()).hasSize(2); - - SQSBatchResponse.BatchItemFailure batchItemFailure = sqsBatchResponse.getBatchItemFailures().get(0); - Assertions.assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("e9144555-9a4f-4ec3-99a0-34ce359b4b54"); - batchItemFailure = sqsBatchResponse.getBatchItemFailures().get(1); - Assertions.assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("f9144555-9a4f-4ec3-99a0-34ce359b4b54"); - } - - static class SQSBProduct implements SQSBatchProcessor { - public void processItem(Product product, Context context) { - if (product.getId() == 12345) { - throw new RuntimeException("fake exception"); - } - } - } - - @ParameterizedTest - @Event(value = "sqs_event.json", type = SQSEvent.class) - public void shouldAddMessageToBatchFailure_whenException_withProduct(SQSEvent event) { - SQSBProduct sqsbproduct = new SQSBProduct(); - SQSBatchResponse sqsBatchResponse = sqsbproduct.processBatch(event, context); - Assertions.assertThat(sqsBatchResponse.getBatchItemFailures()).hasSize(1); - - SQSBatchResponse.BatchItemFailure batchItemFailure = sqsBatchResponse.getBatchItemFailures().get(0); - Assertions.assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("e9144555-9a4f-4ec3-99a0-34ce359b4b54"); - } - -} From a9517f68d228ec25a79b53255a8ed38676c4f8e0 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Thu, 20 Jul 2023 08:35:44 +0200 Subject: [PATCH 18/79] The shape of it is rightish --- powertools-batch/pom.xml | 2 +- .../batch/AbstractMessageHandlerBuilder.java | 26 ++-------------- .../batch/SqsBatchMessageHandlerBuilder.java | 30 +++++++++++++++---- .../batch/handler/BatchMessageHandler.java | 5 ++++ .../batch/handler/SqsBatchMessageHandler.java | 27 ++++------------- .../powertools/batch/SimpleBuilderTest.java | 9 ++++-- .../utilities/EventDeserializer.java | 13 ++++++++ 7 files changed, 58 insertions(+), 54 deletions(-) diff --git a/powertools-batch/pom.xml b/powertools-batch/pom.xml index f359ccd95..aa43def63 100644 --- a/powertools-batch/pom.xml +++ b/powertools-batch/pom.xml @@ -6,7 +6,7 @@ software.amazon.lambda powertools-parent - 1.15.0 + 1.17.0-SNAPSHOT powertools-batch diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/AbstractMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/AbstractMessageHandlerBuilder.java index b98f144e8..2931fe0a1 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/AbstractMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/AbstractMessageHandlerBuilder.java @@ -20,9 +20,6 @@ abstract class AbstractMessageHandlerBuilder { protected BiConsumer failureHandler; protected Consumer successHandler; - protected BiConsumer messageHandler; - protected BiConsumer rawMessageHandler; - public C withSuccessHandler(Consumer handler) { this.successHandler = handler; @@ -34,26 +31,9 @@ public C withFailureHandler(BiConsumer handler) return getThis(); } - public C withMessageHandler(BiConsumer handler) { - if (this.rawMessageHandler != null) { - throw new TooManyMessageHandlersException(); - } - - this.messageHandler = handler; - return getThis(); - } - - public C withRawMessageHandler(BiConsumer handler) { - if (this.messageHandler != null) { - throw new TooManyMessageHandlersException(); - } - - this.rawMessageHandler = handler; - return getThis(); - } - - public abstract BatchMessageHandler build(); + public abstract BatchMessageHandler buildWithRawMessageHandler(BiConsumer handler); + public abstract BatchMessageHandler buildWithMessageHandler(BiConsumer handler); - protected abstract C getThis(); + protected abstract C getThis(); } diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/SqsBatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/SqsBatchMessageHandlerBuilder.java index 0b6c0fefe..2dedb36c1 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/SqsBatchMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/SqsBatchMessageHandlerBuilder.java @@ -1,10 +1,13 @@ package software.amazon.lambda.powertools.batch; +import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; import com.amazonaws.services.lambda.runtime.events.SQSEvent; import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; import software.amazon.lambda.powertools.batch.handler.SqsBatchMessageHandler; +import java.util.function.BiConsumer; + /** * Builds a batch processor for the SQS event source. */ @@ -13,19 +16,34 @@ public class SqsBatchMessageHandlerBuilder extends AbstractMessageHandlerBuilder SQSEvent, SQSBatchResponse> { + @Override - protected SqsBatchMessageHandlerBuilder getThis() { - return this; + public BatchMessageHandler buildWithRawMessageHandler(BiConsumer rawMessageHandler) { + return new SqsBatchMessageHandler( + null, + rawMessageHandler, + successHandler, + failureHandler + ); } @Override - public BatchMessageHandler build() { - - return new SqsBatchMessageHandler( + public BatchMessageHandler buildWithMessageHandler(BiConsumer messageHandler) { + return new SqsBatchMessageHandler<>( messageHandler, - rawMessageHandler, + null, successHandler, failureHandler ); } + + + @Override + protected SqsBatchMessageHandlerBuilder getThis() { + return this; + } + + + + } diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/BatchMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/BatchMessageHandler.java index 94280f1eb..29038b10d 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/BatchMessageHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/BatchMessageHandler.java @@ -2,6 +2,11 @@ import com.amazonaws.services.lambda.runtime.Context; +/** + * + * @param The type of the Lambda event + * @param The type of the lambda response + */ public interface BatchMessageHandler { public abstract R processBatch(E event, Context context); diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java index 671c25430..402e4f71f 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java @@ -4,41 +4,31 @@ import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; import com.amazonaws.services.lambda.runtime.events.SQSEvent; import software.amazon.lambda.powertools.utilities.EventDeserializer; -import sun.reflect.generics.reflectiveObjects.ParameterizedTypeImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.lang.model.type.DeclaredType; import java.util.ArrayList; import java.util.function.BiConsumer; import java.util.function.Consumer; -public class SqsBatchMessageHandler implements BatchMessageHandler { +public class SqsBatchMessageHandler implements BatchMessageHandler { Logger SQS_BATCH_LOGGER = LoggerFactory.getLogger(SqsBatchMessageHandler.class); // The attribute on an SQS-FIFO message used to record the message group ID // https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#sample-fifo-queues-message-event String MESSAGE_GROUP_ID_KEY = "MessageGroupId"; - private final BiConsumer messageHandler; + private final BiConsumer messageHandler; private final BiConsumer rawMessageHandler; private final Consumer successHandler; private final BiConsumer failureHandler; - private final Class bodyClass; - public SqsBatchMessageHandler(BiConsumer messageHandler, BiConsumer rawMessageHandler, Consumer successHandler, BiConsumer failureHandler) { + public SqsBatchMessageHandler(BiConsumer messageHandler, BiConsumer rawMessageHandler, Consumer successHandler, BiConsumer failureHandler) { this.messageHandler = messageHandler; this.rawMessageHandler = rawMessageHandler; this.successHandler = successHandler; this.failureHandler = failureHandler; - - // If we've got a message handler, work out once now what type we have to deserialize - if (this.messageHandler != null) { - this.bodyClass = (Class) ((ParameterizedTypeImpl) messageHandler.getClass() - .getGenericInterfaces()[0]) - .getActualTypeArguments()[0]; - } else { - bodyClass = null; - } } @Override @@ -61,13 +51,8 @@ public SQSBatchResponse processBatch(SQSEvent event, Context context) { if (this.rawMessageHandler != null) { rawMessageHandler.accept(message, context); } else { - // TODO this is bad bad not good - // TODO fix - // TODO either with type bounds for the concrete consumer type on the builder (and buildWithHandler(..)) - // TODO .... or by making this cast in the constructor - Object messageDeserialized = EventDeserializer.extractDataFrom(message).as(bodyClass); - BiConsumer consumerCast = (BiConsumer) messageHandler; - consumerCast.accept(messageDeserialized, context); + M messageDeserialized = EventDeserializer.extractDataFrom(message).as(); + messageHandler.accept(messageDeserialized, context); } } catch (Throwable t) { SQS_BATCH_LOGGER.error("Error while processing message with messageId {}: {}, adding it to batch item failures", message.getMessageId(), t.getMessage()); diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SimpleBuilderTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SimpleBuilderTest.java index 9a46d51c5..34e3163eb 100644 --- a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SimpleBuilderTest.java +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SimpleBuilderTest.java @@ -1,7 +1,10 @@ package software.amazon.lambda.powertools.batch; +import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; +import com.amazonaws.services.lambda.runtime.events.SQSEvent; import org.junit.jupiter.api.Test; import software.amazon.lambda.powertools.batch.SqsBatchMessageHandlerBuilder; +import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; import software.amazon.lambda.powertools.model.Basket; import com.amazonaws.services.lambda.runtime.Context; @@ -11,12 +14,12 @@ public class SimpleBuilderTest { @Test public void sqsBuilderWorks() { - SqsBatchMessageHandlerBuilder builder = new BatchMessageHandlerBuilder() + BatchMessageHandler messageHandler = new BatchMessageHandlerBuilder() .withSqsBatchHandler() .withFailureHandler((c, e) -> System.out.println("Whoops!")) - .withMessageHandler(this::processMessage); + .buildWithMessageHandler(this::processMessage); - assertThat(builder).isNotNull(); + assertThat(messageHandler).isNotNull(); } public void processMessage(Basket basket, Context context) { diff --git a/powertools-serialization/src/main/java/software/amazon/lambda/powertools/utilities/EventDeserializer.java b/powertools-serialization/src/main/java/software/amazon/lambda/powertools/utilities/EventDeserializer.java index 2a05f26c6..9ed3f5ed8 100644 --- a/powertools-serialization/src/main/java/software/amazon/lambda/powertools/utilities/EventDeserializer.java +++ b/powertools-serialization/src/main/java/software/amazon/lambda/powertools/utilities/EventDeserializer.java @@ -28,7 +28,9 @@ import com.amazonaws.services.lambda.runtime.events.SNSEvent; import com.amazonaws.services.lambda.runtime.events.SQSEvent; import com.amazonaws.services.lambda.runtime.events.ScheduledEvent; +import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -214,6 +216,17 @@ public T as(Class clazz) { } } + public M as() { + TypeReference typeRef = new TypeReference() {}; + + try { + JsonParser parser = JsonConfig.get().getObjectMapper().createParser(content); + return JsonConfig.get().getObjectMapper().reader().readValue(parser, typeRef); + } catch (IOException e) { + throw new EventDeserializationException("Cannot load the event as " + typeRef, e); + } + }; + /** * Deserialize this part of event from JSON to a list of objects of type T * @param clazz the target type for deserialization From d3ad219fb2b38c7560007e6bb2092c633a1a9420 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Thu, 20 Jul 2023 11:09:06 +0200 Subject: [PATCH 19/79] Working working --- .../batch/AbstractMessageHandlerBuilder.java | 15 ++- .../batch/SqsBatchMessageHandlerBuilder.java | 4 +- .../batch/handler/SqsBatchMessageHandler.java | 21 +++- .../batch/SQSBatchProcessorTest.java | 111 ++++++++++-------- .../powertools/batch/SimpleBuilderTest.java | 2 +- 5 files changed, 94 insertions(+), 59 deletions(-) diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/AbstractMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/AbstractMessageHandlerBuilder.java index 2931fe0a1..f8f0097a9 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/AbstractMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/AbstractMessageHandlerBuilder.java @@ -18,7 +18,7 @@ * @param The type of the batch response */ abstract class AbstractMessageHandlerBuilder { - protected BiConsumer failureHandler; + protected BiConsumer failureHandler; protected Consumer successHandler; public C withSuccessHandler(Consumer handler) { @@ -26,14 +26,23 @@ public C withSuccessHandler(Consumer handler) { return getThis(); } - public C withFailureHandler(BiConsumer handler) { + public C withFailureHandler(BiConsumer handler) { this.failureHandler = handler; return getThis(); } public abstract BatchMessageHandler buildWithRawMessageHandler(BiConsumer handler); - public abstract BatchMessageHandler buildWithMessageHandler(BiConsumer handler); + public BatchMessageHandler buildWithRawMessageHandler(Consumer handler) { + return buildWithRawMessageHandler((f, c) -> handler.accept(f)); + } + + public abstract BatchMessageHandler buildWithMessageHandler(BiConsumer handler, Class messageClass); + + public BatchMessageHandler buildWithMessageHandler(Consumer handler, Class messageClass) { + return buildWithMessageHandler((f, c) -> handler.accept(f), messageClass); + } + protected abstract C getThis(); } diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/SqsBatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/SqsBatchMessageHandlerBuilder.java index 2dedb36c1..abf3a62b8 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/SqsBatchMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/SqsBatchMessageHandlerBuilder.java @@ -20,6 +20,7 @@ public class SqsBatchMessageHandlerBuilder extends AbstractMessageHandlerBuilder @Override public BatchMessageHandler buildWithRawMessageHandler(BiConsumer rawMessageHandler) { return new SqsBatchMessageHandler( + null, null, rawMessageHandler, successHandler, @@ -28,9 +29,10 @@ public BatchMessageHandler buildWithRawMessageHandle } @Override - public BatchMessageHandler buildWithMessageHandler(BiConsumer messageHandler) { + public BatchMessageHandler buildWithMessageHandler(BiConsumer messageHandler, Class messageClass) { return new SqsBatchMessageHandler<>( messageHandler, + messageClass, null, successHandler, failureHandler diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java index 402e4f71f..d42cffac4 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java @@ -7,12 +7,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.lang.model.type.DeclaredType; import java.util.ArrayList; import java.util.function.BiConsumer; import java.util.function.Consumer; public class SqsBatchMessageHandler implements BatchMessageHandler { + private final Class messageClass; Logger SQS_BATCH_LOGGER = LoggerFactory.getLogger(SqsBatchMessageHandler.class); // The attribute on an SQS-FIFO message used to record the message group ID @@ -22,13 +22,15 @@ public class SqsBatchMessageHandler implements BatchMessageHandler messageHandler; private final BiConsumer rawMessageHandler; private final Consumer successHandler; - private final BiConsumer failureHandler; + private final BiConsumer failureHandler; - public SqsBatchMessageHandler(BiConsumer messageHandler, BiConsumer rawMessageHandler, Consumer successHandler, BiConsumer failureHandler) { + public SqsBatchMessageHandler(BiConsumer messageHandler, Class messageClass, BiConsumer rawMessageHandler, Consumer successHandler, BiConsumer failureHandler) { this.messageHandler = messageHandler; + this.messageClass = messageClass; this.rawMessageHandler = rawMessageHandler; this.successHandler = successHandler; this.failureHandler = failureHandler; + } @Override @@ -51,9 +53,15 @@ public SQSBatchResponse processBatch(SQSEvent event, Context context) { if (this.rawMessageHandler != null) { rawMessageHandler.accept(message, context); } else { - M messageDeserialized = EventDeserializer.extractDataFrom(message).as(); + M messageDeserialized = EventDeserializer.extractDataFrom(message).as(messageClass); messageHandler.accept(messageDeserialized, context); } + + // Report success if we have a handler + if (this.successHandler != null) { + this.successHandler.accept(message); + } + } catch (Throwable t) { SQS_BATCH_LOGGER.error("Error while processing message with messageId {}: {}, adding it to batch item failures", message.getMessageId(), t.getMessage()); response.getBatchItemFailures().add(SQSBatchResponse.BatchItemFailure.builder().withItemIdentifier(message.getMessageId()).build()); @@ -62,6 +70,11 @@ public SQSBatchResponse processBatch(SQSEvent event, Context context) { SQS_BATCH_LOGGER.info("A message in a batch with messageGroupId {} and messageId {} failed; failing the rest of the batch too" , messageGroupId, message.getMessageId()); } + + // Report failure if we have a handler + if (this.failureHandler != null) { + this.failureHandler.accept(message, t); + } } } diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java index 4934512e3..bb4667b03 100644 --- a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java @@ -7,6 +7,7 @@ import org.assertj.core.api.Assertions; import org.junit.jupiter.params.ParameterizedTest; import org.mockito.Mock; +import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; import software.amazon.lambda.powertools.model.Product; public class SQSBatchProcessorTest { @@ -14,55 +15,65 @@ public class SQSBatchProcessorTest { private Context context; -// static class SQSBP implements SQSBatchProcessor { -// public void processMessage(SQSEvent.SQSMessage message, Context context) { -// if (message.getMessageId().equals("e9144555-9a4f-4ec3-99a0-34ce359b4b54")) { -// throw new RuntimeException("fake exception"); -// } -// } -// } -// -// @ParameterizedTest -// @Event(value = "sqs_event.json", type = SQSEvent.class) -// public void shouldAddMessageToBatchFailure_whenException_withMessage(SQSEvent event) { -// SQSBP sqsbp = new SQSBP(); -// SQSBatchResponse sqsBatchResponse = sqsbp.processBatch(event, context); -// Assertions.assertThat(sqsBatchResponse.getBatchItemFailures()).hasSize(1); -// -// SQSBatchResponse.BatchItemFailure batchItemFailure = sqsBatchResponse.getBatchItemFailures().get(0); -// Assertions.assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("e9144555-9a4f-4ec3-99a0-34ce359b4b54"); -// } -// -// @ParameterizedTest -// @Event(value = "sqs_fifo_event.json", type = SQSEvent.class) -// public void shouldAddMessageToBatchFailure_whenException_withSQSFIFO(SQSEvent event) { -// SQSBP sqsbp = new SQSBP(); -// SQSBatchResponse sqsBatchResponse = sqsbp.processBatch(event, context); -// Assertions.assertThat(sqsBatchResponse.getBatchItemFailures()).hasSize(2); -// -// SQSBatchResponse.BatchItemFailure batchItemFailure = sqsBatchResponse.getBatchItemFailures().get(0); -// Assertions.assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("e9144555-9a4f-4ec3-99a0-34ce359b4b54"); -// batchItemFailure = sqsBatchResponse.getBatchItemFailures().get(1); -// Assertions.assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("f9144555-9a4f-4ec3-99a0-34ce359b4b54"); -// } -// -// static class SQSBProduct implements SQSBatchProcessor { -// public void processItem(Product product, Context context) { -// if (product.getId() == 12345) { -// throw new RuntimeException("fake exception"); -// } -// } -// } -// -// @ParameterizedTest -// @Event(value = "sqs_event.json", type = SQSEvent.class) -// public void shouldAddMessageToBatchFailure_whenException_withProduct(SQSEvent event) { -// SQSBProduct sqsbproduct = new SQSBProduct(); -// SQSBatchResponse sqsBatchResponse = sqsbproduct.processBatch(event, context); -// Assertions.assertThat(sqsBatchResponse.getBatchItemFailures()).hasSize(1); -// -// SQSBatchResponse.BatchItemFailure batchItemFailure = sqsBatchResponse.getBatchItemFailures().get(0); -// Assertions.assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("e9144555-9a4f-4ec3-99a0-34ce359b4b54"); -// } + private void processMessageFailSometimes(SQSEvent.SQSMessage message, Context context) { + if (message.getMessageId().equals("e9144555-9a4f-4ec3-99a0-34ce359b4b54")) { + throw new RuntimeException("fake exception"); + } + } + + public void processMessageFailsForFixedProduct(Product product, Context context) { + if (product.getId() == 12345) { + throw new RuntimeException("fake exception"); + } + } + + @ParameterizedTest + @Event(value = "sqs_event.json", type = SQSEvent.class) + public void shouldAddMessageToBatchFailure_whenException_withMessage(SQSEvent event) { + BatchMessageHandler handler = new BatchMessageHandlerBuilder() + .withSqsBatchHandler() + .buildWithRawMessageHandler(this::processMessageFailSometimes); + + SQSBatchResponse sqsBatchResponse = handler.processBatch(event, context); + Assertions.assertThat(sqsBatchResponse.getBatchItemFailures()).hasSize(1); + + SQSBatchResponse.BatchItemFailure batchItemFailure = sqsBatchResponse.getBatchItemFailures().get(0); + Assertions.assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("e9144555-9a4f-4ec3-99a0-34ce359b4b54"); + } + + @ParameterizedTest + @Event(value = "sqs_fifo_event.json", type = SQSEvent.class) + public void shouldAddMessageToBatchFailure_whenException_withSQSFIFO(SQSEvent event) { + BatchMessageHandler handler = new BatchMessageHandlerBuilder() + .withSqsBatchHandler() + .buildWithRawMessageHandler(this::processMessageFailSometimes); + + SQSBatchResponse sqsBatchResponse = handler.processBatch(event, context); + + Assertions.assertThat(sqsBatchResponse.getBatchItemFailures()).hasSize(2); + + SQSBatchResponse.BatchItemFailure batchItemFailure = sqsBatchResponse.getBatchItemFailures().get(0); + Assertions.assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("e9144555-9a4f-4ec3-99a0-34ce359b4b54"); + batchItemFailure = sqsBatchResponse.getBatchItemFailures().get(1); + Assertions.assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("f9144555-9a4f-4ec3-99a0-34ce359b4b54"); + } + + + + + @ParameterizedTest + @Event(value = "sqs_event.json", type = SQSEvent.class) + public void shouldAddMessageToBatchFailure_whenException_withProduct(SQSEvent event) { + + BatchMessageHandler handler = new BatchMessageHandlerBuilder() + .withSqsBatchHandler() + .buildWithMessageHandler(this::processMessageFailsForFixedProduct, Product.class); + + SQSBatchResponse sqsBatchResponse = handler.processBatch(event, context); + Assertions.assertThat(sqsBatchResponse.getBatchItemFailures()).hasSize(1); + + SQSBatchResponse.BatchItemFailure batchItemFailure = sqsBatchResponse.getBatchItemFailures().get(0); + Assertions.assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("e9144555-9a4f-4ec3-99a0-34ce359b4b54"); + } } diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SimpleBuilderTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SimpleBuilderTest.java index 34e3163eb..7dc37cf47 100644 --- a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SimpleBuilderTest.java +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SimpleBuilderTest.java @@ -17,7 +17,7 @@ public void sqsBuilderWorks() { BatchMessageHandler messageHandler = new BatchMessageHandlerBuilder() .withSqsBatchHandler() .withFailureHandler((c, e) -> System.out.println("Whoops!")) - .buildWithMessageHandler(this::processMessage); + .buildWithMessageHandler(this::processMessage, Basket.class); assertThat(messageHandler).isNotNull(); } From 418e32b3600f572df785f9d86bbfda069e133451 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Thu, 20 Jul 2023 13:26:12 +0200 Subject: [PATCH 20/79] Work --- .../batch/AbstractMessageHandlerBuilder.java | 48 -------- .../batch/BatchMessageHandlerBuilder.java | 16 ++- .../batch/DdbMessageHandlerBuilder.java | 13 -- .../AbstractBatchMessageHandlerBuilder.java | 114 ++++++++++++++++++ .../DdbBatchMessageHandlerBuilder.java | 36 ++++++ .../KinesisStreamsBatchProcessorBuilder.java | 40 ++++++ .../SqsBatchMessageHandlerBuilder.java | 4 +- .../DeserializationNotSupportedException.java | 9 ++ .../TooManyMessageHandlersException.java | 2 +- .../batch/handler/BatchMessageHandler.java | 15 ++- .../batch/handler/DdbBatchMessageHandler.java | 28 +++++ .../KinesisStreamsBatchMessageHandler.java | 35 ++++++ .../batch/SQSBatchProcessorTest.java | 2 - .../powertools/batch/SimpleBuilderTest.java | 1 - 14 files changed, 291 insertions(+), 72 deletions(-) delete mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/AbstractMessageHandlerBuilder.java delete mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/DdbMessageHandlerBuilder.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/AbstractBatchMessageHandlerBuilder.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/DdbBatchMessageHandlerBuilder.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/KinesisStreamsBatchProcessorBuilder.java rename powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/{ => builder}/SqsBatchMessageHandlerBuilder.java (89%) create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/exception/DeserializationNotSupportedException.java rename powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/{ => exception}/TooManyMessageHandlersException.java (79%) create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/DdbBatchMessageHandler.java create mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/KinesisStreamsBatchMessageHandler.java diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/AbstractMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/AbstractMessageHandlerBuilder.java deleted file mode 100644 index f8f0097a9..000000000 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/AbstractMessageHandlerBuilder.java +++ /dev/null @@ -1,48 +0,0 @@ -package software.amazon.lambda.powertools.batch; - -import com.amazonaws.services.lambda.runtime.Context; -import com.amazonaws.services.lambda.runtime.events.SQSEvent; -import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; - -import java.util.function.BiConsumer; -import java.util.function.Consumer; - -/** - * - * An abstract class to capture common arguments used - * across all the message-binding-specific builders. - * - * @param The type of a single message in the batch - * @param The type of the child builder - * @param The type of the Lambda event - * @param The type of the batch response - */ -abstract class AbstractMessageHandlerBuilder { - protected BiConsumer failureHandler; - protected Consumer successHandler; - - public C withSuccessHandler(Consumer handler) { - this.successHandler = handler; - return getThis(); - } - - public C withFailureHandler(BiConsumer handler) { - this.failureHandler = handler; - return getThis(); - } - - public abstract BatchMessageHandler buildWithRawMessageHandler(BiConsumer handler); - - public BatchMessageHandler buildWithRawMessageHandler(Consumer handler) { - return buildWithRawMessageHandler((f, c) -> handler.accept(f)); - } - - public abstract BatchMessageHandler buildWithMessageHandler(BiConsumer handler, Class messageClass); - - public BatchMessageHandler buildWithMessageHandler(Consumer handler, Class messageClass) { - return buildWithMessageHandler((f, c) -> handler.accept(f), messageClass); - } - - - protected abstract C getThis(); -} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java index cf31d78e6..2b09f5ab5 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java @@ -1,6 +1,6 @@ package software.amazon.lambda.powertools.batch; -import software.amazon.lambda.powertools.batch.SqsBatchMessageHandlerBuilder; +import software.amazon.lambda.powertools.batch.builder.*; /** * A builder-style interface we can use within an existing Lambda RequestHandler to @@ -9,12 +9,22 @@ **/ public class BatchMessageHandlerBuilder { + /** + * Build an SQS-batch message handler. + * + * @return A fluent builder interface to continue the building + */ public SqsBatchMessageHandlerBuilder withSqsBatchHandler() { return new SqsBatchMessageHandlerBuilder(); } - public DdbMessageHandlerBuilder withDynamoDbBatchHandler() { - return new DdbMessageHandlerBuilder(); + /** + * Build a DynamoDB streams batch message handler. + * + * @return A fluent builder interface to continue the building + */ + public DdbBatchMessageHandlerBuilder withDynamoDbBatchHandler() { + return new DdbBatchMessageHandlerBuilder(); } } diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/DdbMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/DdbMessageHandlerBuilder.java deleted file mode 100644 index 3dcb98add..000000000 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/DdbMessageHandlerBuilder.java +++ /dev/null @@ -1,13 +0,0 @@ -package software.amazon.lambda.powertools.batch; - -import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; -import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; - -import java.util.function.Consumer; - -public class DdbMessageHandlerBuilder { - - public StreamsEventResponse processBatch(DynamodbEvent batch, Consumer handler) { - throw new RuntimeException("Not implemented"); - } -} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/AbstractBatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/AbstractBatchMessageHandlerBuilder.java new file mode 100644 index 000000000..d73e6b842 --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/AbstractBatchMessageHandlerBuilder.java @@ -0,0 +1,114 @@ +package software.amazon.lambda.powertools.batch.builder; + +import com.amazonaws.services.lambda.runtime.Context; +import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; + +import java.util.function.BiConsumer; +import java.util.function.Consumer; + +/** + * + * An abstract class to capture common arguments used across all the message-binding-specific batch processing + * builders. The builders provide a fluent interface to configure the batch processors. Any arguments specific + * to a particular batch binding can be added to the child builder. + * + * We capture types for the various messages involved, so that we can provide an interface that makes + * sense for the concrete child. + * + * @param The type of a single message in the batch + * @param The type of the child builder. We need this to provide a fluent interface - see also getThis() + * @param The type of the Lambda batch event + * @param The type of the batch response we return to Lambda + */ +abstract class AbstractBatchMessageHandlerBuilder { + protected BiConsumer failureHandler; + protected Consumer successHandler; + + /** + * Provides a success handler. A success handler is invoked once for + * each message after it has been processed by the user-provided + * handler. + * If the success handler throws, the item in the batch will be + * marked failed. + * + * @param handler The handler to invoke + */ + public C withSuccessHandler(Consumer handler) { + this.successHandler = handler; + return getThis(); + } + + /** + * Provides a failure handler. A failure handler is invoked once + * for each message after it has failed to be processed by the + * user-provided handler. + * + * @param handler The handler to invoke on failure + */ + public C withFailureHandler(BiConsumer handler) { + this.failureHandler = handler; + return getThis(); + } + + /** + * Builds a BatchMessageHandler that can be used to process batches, given + * a user-defined handler to process each item in the batch. This variant + * takes a function that consumes a raw message and the Lambda context. + * Note: If you don't need the Lambda context, use the variant of this function + * that does not require it. + * + * @param handler Takes a raw message - the AWS-SDK-shaped inner type of the batch - to process + * @return A BatchMessageHandler for processing the batch + */ + public abstract BatchMessageHandler buildWithRawMessageHandler(BiConsumer handler); + + /** + * Builds a BatchMessageHandler that can be used to process batches, given + * a user-defined handler to process each item in the batch. This variant + * takes a function that consumes a raw message. + * + * @param handler Takes a raw message - the AWS-SDK-shaped inner type of the batch - to process + * @return A BatchMessageHandler for processing the batch + */ + public BatchMessageHandler buildWithRawMessageHandler(Consumer handler) { + return buildWithRawMessageHandler((f, c) -> handler.accept(f)); + } + + /** + * Builds a BatchMessageHandler that can be used to process batches, given + * a user-defined handler to process each item in the batch. This variant + * takes a function that consumes the deserialized body of the given message + * and the lambda context. If deserialization fails, it will be treated as + * failure of the processing of that item in the batch. + * Note: If you don't need the Lambda context, use the variant of this function + * that does not require it. + * + * @param handler Processes the deserialized body of the message + * @return A BatchMessageHandler for processing the batch + */ + public abstract BatchMessageHandler buildWithMessageHandler(BiConsumer handler, Class messageClass); + + /** + * Builds a BatchMessageHandler that can be used to process batches, given + * a user-defined handler to process each item in the batch. This variant + * takes a function that consumes the deserialized body of the given message + * If deserialization fails, it will be treated as + * failure of the processing of that item in the batch. + * Note: If you don't need the Lambda context, use the variant of this function + * that does not require it. + * + * @param handler Processes the deserialized body of the message + * @return A BatchMessageHandler for processing the batch + */ + public BatchMessageHandler buildWithMessageHandler(Consumer handler, Class messageClass) { + return buildWithMessageHandler((f, c) -> handler.accept(f), messageClass); + } + + + /** + * Used to chain the fluent builder interface through the child classes. + * + * @return This + */ + protected abstract C getThis(); +} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/DdbBatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/DdbBatchMessageHandlerBuilder.java new file mode 100644 index 000000000..25d382d28 --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/DdbBatchMessageHandlerBuilder.java @@ -0,0 +1,36 @@ +package software.amazon.lambda.powertools.batch.builder; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; +import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; +import software.amazon.lambda.powertools.batch.exception.DeserializationNotSupportedException; +import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; +import software.amazon.lambda.powertools.batch.handler.DdbBatchMessageHandler; + +import java.util.function.BiConsumer; + +public class DdbBatchMessageHandlerBuilder extends AbstractBatchMessageHandlerBuilder { + + + @Override + public BatchMessageHandler buildWithRawMessageHandler(BiConsumer rawMessageHandler) { + return new DdbBatchMessageHandler( + this.successHandler, + this.failureHandler, + rawMessageHandler); + } + + @Override + public BatchMessageHandler buildWithMessageHandler(BiConsumer handler, Class messageClass) { + // The DDB provider streams DynamoDB changes, and therefore does not have a customizable payload + throw new DeserializationNotSupportedException(); + } + + @Override + protected DdbBatchMessageHandlerBuilder getThis() { + return this; + } +} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/KinesisStreamsBatchProcessorBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/KinesisStreamsBatchProcessorBuilder.java new file mode 100644 index 000000000..f1c084e8c --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/KinesisStreamsBatchProcessorBuilder.java @@ -0,0 +1,40 @@ +package software.amazon.lambda.powertools.batch.builder; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.events.KinesisEvent; +import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; +import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; +import software.amazon.lambda.powertools.batch.handler.KinesisStreamsBatchMessageHandler; + +import java.util.function.BiConsumer; + +public class KinesisStreamsBatchProcessorBuilder extends AbstractBatchMessageHandlerBuilder +{ + @Override + public BatchMessageHandler buildWithRawMessageHandler(BiConsumer rawMessageHandler) { + return new KinesisStreamsBatchMessageHandler( + rawMessageHandler, + null, + null, + successHandler, + failureHandler); + } + + @Override + public BatchMessageHandler buildWithMessageHandler(BiConsumer messageHandler, Class messageClass) { + return new KinesisStreamsBatchMessageHandler<>( + null, + messageHandler, + messageClass, + successHandler, + failureHandler); + } + + @Override + protected KinesisStreamsBatchProcessorBuilder getThis() { + return null; + } +} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/SqsBatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/SqsBatchMessageHandlerBuilder.java similarity index 89% rename from powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/SqsBatchMessageHandlerBuilder.java rename to powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/SqsBatchMessageHandlerBuilder.java index abf3a62b8..51b5eef88 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/SqsBatchMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/SqsBatchMessageHandlerBuilder.java @@ -1,4 +1,4 @@ -package software.amazon.lambda.powertools.batch; +package software.amazon.lambda.powertools.batch.builder; import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; @@ -11,7 +11,7 @@ /** * Builds a batch processor for the SQS event source. */ -public class SqsBatchMessageHandlerBuilder extends AbstractMessageHandlerBuilder { diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/exception/DeserializationNotSupportedException.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/exception/DeserializationNotSupportedException.java new file mode 100644 index 000000000..f1be500ed --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/exception/DeserializationNotSupportedException.java @@ -0,0 +1,9 @@ +package software.amazon.lambda.powertools.batch.exception; + +public class DeserializationNotSupportedException extends RuntimeException { + + public DeserializationNotSupportedException() { + super("This BatchMessageHandler has a fixed schema and does not support user-defined types"); + } + +} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/TooManyMessageHandlersException.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/exception/TooManyMessageHandlersException.java similarity index 79% rename from powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/TooManyMessageHandlersException.java rename to powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/exception/TooManyMessageHandlersException.java index ec3d70f9d..fc1981b8b 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/TooManyMessageHandlersException.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/exception/TooManyMessageHandlersException.java @@ -1,4 +1,4 @@ -package software.amazon.lambda.powertools.batch; +package software.amazon.lambda.powertools.batch.exception; public class TooManyMessageHandlersException extends RuntimeException { diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/BatchMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/BatchMessageHandler.java index 29038b10d..14fe52508 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/BatchMessageHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/BatchMessageHandler.java @@ -4,11 +4,22 @@ /** * - * @param The type of the Lambda event - * @param The type of the lambda response + * The basic interface a batch message handler must meet. + * + * @param The type of the Lambda batch event + * @param The type of the lambda batch response */ public interface BatchMessageHandler { + /** + * Processes the given batch returning a partial batch + * response indicating the success and failure of individual + * messages within the batch. + * + * @param event The Lambda event containing the batch to process + * @param context The lambda context + * @return A partial batch response + */ public abstract R processBatch(E event, Context context); } diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/DdbBatchMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/DdbBatchMessageHandler.java new file mode 100644 index 000000000..cbf4960d0 --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/DdbBatchMessageHandler.java @@ -0,0 +1,28 @@ +package software.amazon.lambda.powertools.batch.handler; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; +import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; + +import java.util.function.BiConsumer; +import java.util.function.Consumer; + +public class DdbBatchMessageHandler implements BatchMessageHandler{ + + private final Consumer successHandler; + private final BiConsumer failureHandler; + private final BiConsumer rawMessageHandler; + + public DdbBatchMessageHandler(Consumer successHandler, BiConsumer failureHandler, BiConsumer rawMessageHandler) { + this.successHandler = successHandler; + this.failureHandler = failureHandler; + this.rawMessageHandler = rawMessageHandler; + } + + @Override + public StreamsEventResponse processBatch(DynamodbEvent event, Context context) { + // TODO Probably helpful https://github.com/aws-powertools/powertools-lambda-java/blob/ee64d620aff083b1a9734d880af2e077d73692a8/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/DynamoDBBatchProcessor.java + // TODO -success/failure handlers too + throw new RuntimeException("Not implemented"); + } +} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/KinesisStreamsBatchMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/KinesisStreamsBatchMessageHandler.java new file mode 100644 index 000000000..1aefe7977 --- /dev/null +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/KinesisStreamsBatchMessageHandler.java @@ -0,0 +1,35 @@ +package software.amazon.lambda.powertools.batch.handler; + + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.events.KinesisEvent; +import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; + +import java.util.function.BiConsumer; +import java.util.function.Consumer; + +public class KinesisStreamsBatchMessageHandler implements BatchMessageHandler { + + private final BiConsumer rawMessageHandler; + private final BiConsumer messageHandler; + private final Class messageClass; + private final Consumer successHandler; + private final BiConsumer failureHandler; + + public KinesisStreamsBatchMessageHandler(BiConsumer rawMessageHandler, + BiConsumer messageHandler, + Class messageClass, Consumer successHandler, + BiConsumer failureHandler) { + + this.rawMessageHandler = rawMessageHandler; + this.messageHandler = messageHandler; + this.messageClass = messageClass; + this.successHandler = successHandler; + this.failureHandler = failureHandler; + } + + @Override + public StreamsEventResponse processBatch(KinesisEvent event, Context context) { + throw new RuntimeException("Not implemented"); + } +} diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java index bb4667b03..599ec6587 100644 --- a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java @@ -59,8 +59,6 @@ public void shouldAddMessageToBatchFailure_whenException_withSQSFIFO(SQSEvent ev } - - @ParameterizedTest @Event(value = "sqs_event.json", type = SQSEvent.class) public void shouldAddMessageToBatchFailure_whenException_withProduct(SQSEvent event) { diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SimpleBuilderTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SimpleBuilderTest.java index 7dc37cf47..187363f59 100644 --- a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SimpleBuilderTest.java +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SimpleBuilderTest.java @@ -3,7 +3,6 @@ import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; import com.amazonaws.services.lambda.runtime.events.SQSEvent; import org.junit.jupiter.api.Test; -import software.amazon.lambda.powertools.batch.SqsBatchMessageHandlerBuilder; import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; import software.amazon.lambda.powertools.model.Basket; import com.amazonaws.services.lambda.runtime.Context; From a1e441c776c9e7991767badb43fe1e88a0a08173 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Mon, 24 Jul 2023 11:15:14 +0200 Subject: [PATCH 21/79] Work on kinesis batch handler --- .../batch/BatchMessageHandlerBuilder.java | 8 ++ ...=> KinesisBatchMessageHandlerBuilder.java} | 8 +- .../KinesisStreamsBatchMessageHandler.java | 46 ++++++- .../batch/handler/SqsBatchMessageHandler.java | 15 ++- .../batch/AllBatchProcessorTests.java | 46 +++++++ .../batch/KinesisBatchProcessorTest.java | 124 ++++++++++++++++++ .../batch/SQSBatchProcessorTest.java | 60 +++++++-- .../src/test/resources/dynamo_event.json | 97 ++++++++++++++ .../src/test/resources/kinesis_event.json | 38 ++++++ 9 files changed, 421 insertions(+), 21 deletions(-) rename powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/{KinesisStreamsBatchProcessorBuilder.java => KinesisBatchMessageHandlerBuilder.java} (83%) create mode 100644 powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/AllBatchProcessorTests.java create mode 100644 powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/KinesisBatchProcessorTest.java create mode 100644 powertools-batch/src/test/resources/dynamo_event.json create mode 100644 powertools-batch/src/test/resources/kinesis_event.json diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java index 2b09f5ab5..134cb206d 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java @@ -27,4 +27,12 @@ public DdbBatchMessageHandlerBuilder withDynamoDbBatchHandler() { return new DdbBatchMessageHandlerBuilder(); } + /** + * Builds a Kinesis streams batch message handler. + * + * @return a fluent builder interface to continue the building + */ + public KinesisBatchMessageHandlerBuilder withKinesisBatchHandler() { + return new KinesisBatchMessageHandlerBuilder(); + } } diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/KinesisStreamsBatchProcessorBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/KinesisBatchMessageHandlerBuilder.java similarity index 83% rename from powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/KinesisStreamsBatchProcessorBuilder.java rename to powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/KinesisBatchMessageHandlerBuilder.java index f1c084e8c..56d7898ad 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/KinesisStreamsBatchProcessorBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/KinesisBatchMessageHandlerBuilder.java @@ -8,8 +8,8 @@ import java.util.function.BiConsumer; -public class KinesisStreamsBatchProcessorBuilder extends AbstractBatchMessageHandlerBuilder { @@ -34,7 +34,7 @@ public BatchMessageHandler buildWithMess } @Override - protected KinesisStreamsBatchProcessorBuilder getThis() { - return null; + protected KinesisBatchMessageHandlerBuilder getThis() { + return this; } } diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/KinesisStreamsBatchMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/KinesisStreamsBatchMessageHandler.java index 1aefe7977..9f9675d0d 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/KinesisStreamsBatchMessageHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/KinesisStreamsBatchMessageHandler.java @@ -4,11 +4,23 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.KinesisEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.lambda.powertools.utilities.EventDeserializer; +import java.util.ArrayList; +import java.util.List; import java.util.function.BiConsumer; import java.util.function.Consumer; +/** + * A batch message handler for Kinesis Streams batch processing. + * + * Refer to The kinesis batch processing document + * @param + */ public class KinesisStreamsBatchMessageHandler implements BatchMessageHandler { + Logger LOGGER = LoggerFactory.getLogger(KinesisStreamsBatchMessageHandler.class); private final BiConsumer rawMessageHandler; private final BiConsumer messageHandler; @@ -30,6 +42,38 @@ public KinesisStreamsBatchMessageHandler(BiConsumer batchFailures = new ArrayList<>(); + + for (KinesisEvent.KinesisEventRecord record : event.getRecords()) { + try { + if (this.rawMessageHandler != null) { + rawMessageHandler.accept(record, context); + } else { + M messageDeserialized = EventDeserializer.extractDataFrom(record).as(messageClass); + messageHandler.accept(messageDeserialized, context); + } + + // Report success if we have a handler + if (this.successHandler != null) { + this.successHandler.accept(record); + } + } catch (Throwable t) { + batchFailures.add(new StreamsEventResponse.BatchItemFailure(record.getKinesis().getSequenceNumber())); + + // Report failure if we have a handler + if (this.failureHandler != null) { + // A failing failure handler is no reason to fail the batch + try { + this.failureHandler.accept(record, t); + } + catch (Throwable t2) { + LOGGER.warn("failureHandler threw handling failure", t2); + } + } + } + } + + return new StreamsEventResponse(batchFailures); } } + diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java index d42cffac4..e44a62b9b 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java @@ -13,7 +13,7 @@ public class SqsBatchMessageHandler implements BatchMessageHandler { private final Class messageClass; - Logger SQS_BATCH_LOGGER = LoggerFactory.getLogger(SqsBatchMessageHandler.class); + Logger LOGGER = LoggerFactory.getLogger(SqsBatchMessageHandler.class); // The attribute on an SQS-FIFO message used to record the message group ID // https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#sample-fifo-queues-message-event @@ -63,18 +63,25 @@ public SQSBatchResponse processBatch(SQSEvent event, Context context) { } } catch (Throwable t) { - SQS_BATCH_LOGGER.error("Error while processing message with messageId {}: {}, adding it to batch item failures", message.getMessageId(), t.getMessage()); + LOGGER.error("Error while processing message with messageId {}: {}, adding it to batch item failures", message.getMessageId(), t.getMessage()); response.getBatchItemFailures().add(SQSBatchResponse.BatchItemFailure.builder().withItemIdentifier(message.getMessageId()).build()); if (messageGroupId != null) { failWholeBatch = true; - SQS_BATCH_LOGGER.info("A message in a batch with messageGroupId {} and messageId {} failed; failing the rest of the batch too" + LOGGER.info("A message in a batch with messageGroupId {} and messageId {} failed; failing the rest of the batch too" , messageGroupId, message.getMessageId()); } // Report failure if we have a handler if (this.failureHandler != null) { - this.failureHandler.accept(message, t); + // A failing failure handler is no reason to fail the batch + try { + this.failureHandler.accept(message, t); + } + catch (Throwable t2) { + LOGGER.warn("failureHandler threw handling failure", t2); + } } + } } diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/AllBatchProcessorTests.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/AllBatchProcessorTests.java new file mode 100644 index 000000000..bad4e9d0c --- /dev/null +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/AllBatchProcessorTests.java @@ -0,0 +1,46 @@ +package software.amazon.lambda.powertools.batch; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.events.KinesisEvent; +import com.amazonaws.services.lambda.runtime.events.SQSEvent; +import com.amazonaws.services.lambda.runtime.tests.annotations.Event; +import com.amazonaws.services.lambda.runtime.tests.annotations.Events; +import org.junit.jupiter.params.ParameterizedTest; +import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; + +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Test functionality that should be common to all the batch processors + * + * TODO probably remove. Trying to think of a way of factoring + * TODO common tests for all the message handlers into one place + */ +public class AllBatchProcessorTests { +// +// AtomicBoolean failureHandlerWasCalled = new AtomicBoolean(false); +// public void failingFailureHandler(T evt, Throwable t) { +// failureHandlerWasCalled.set(true); +// throw new RuntimeException("Well, this doesn't look great"); +// } +// +// @ParameterizedTest +// @Events( +// events = { +// @Event(value = "sqs_event.json", type = SQSEvent.class), +// @Event(value = "kinesis_event.json", type = KinesisEvent.class) +// } +// ) +// public void failingFailureHandlerShouldntFailBatch(Object event) { +// BatchMessageHandler handler; +// if (event instanceof SQSEvent) { +// handler = new BatchMessageHandlerBuilder() +// .withSqsBatchHandler() +// .withFailureHandler(this::failingFailureHandler) +// .build +// } +// } + +} diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/KinesisBatchProcessorTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/KinesisBatchProcessorTest.java new file mode 100644 index 000000000..528e39bcf --- /dev/null +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/KinesisBatchProcessorTest.java @@ -0,0 +1,124 @@ +package software.amazon.lambda.powertools.batch; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.events.KinesisEvent; +import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; +import com.amazonaws.services.lambda.runtime.events.SQSEvent; +import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; +import com.amazonaws.services.lambda.runtime.tests.annotations.Event; +import org.junit.jupiter.params.ParameterizedTest; +import org.mockito.Mock; +import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; +import software.amazon.lambda.powertools.model.Product; + +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.assertj.core.api.Assertions.assertThat; + +public class KinesisBatchProcessorTest { + + @Mock + private Context context; + + private void processMessageSucceeds(KinesisEvent.KinesisEventRecord record, Context context) { + // Great success + } + + private void processMessageFailsForFixedMessage(KinesisEvent.KinesisEventRecord record, Context context) { + if (record.getKinesis().getSequenceNumber().equals("49545115243490985018280067714973144582180062593244200961")) { + throw new RuntimeException("fake exception"); + } + } + + // A handler that throws an exception for _one_ of the deserialized products in the same messages + public void processMessageFailsForFixedProduct(Product product, Context context) { + if (product.getId() == 1234) { + throw new RuntimeException("fake exception"); + } + } + + @ParameterizedTest + @Event(value = "kinesis_event.json", type = KinesisEvent.class) + public void shouldAddMessageToBatchFailure_whenException_withMessage(KinesisEvent event) { + // Arrange + BatchMessageHandler handler = new BatchMessageHandlerBuilder() + .withKinesisBatchHandler() + .buildWithRawMessageHandler(this::processMessageFailsForFixedMessage); + + // Act + StreamsEventResponse kinesisBatchResponse = handler.processBatch(event, context); + + // Assert + assertThat(kinesisBatchResponse.getBatchItemFailures()).hasSize(1); + StreamsEventResponse.BatchItemFailure batchItemFailure = kinesisBatchResponse.getBatchItemFailures().get(0); + assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("49545115243490985018280067714973144582180062593244200961"); + } + + @ParameterizedTest + @Event(value = "kinesis_event.json", type = KinesisEvent.class) + public void shouldAddMessageToBatchFailure_whenException_withProduct(KinesisEvent event) { + // Arrange + BatchMessageHandler handler = new BatchMessageHandlerBuilder() + .withKinesisBatchHandler() + .buildWithMessageHandler(this::processMessageFailsForFixedProduct, Product.class); + + // Act + StreamsEventResponse kinesisBatchResponse = handler.processBatch(event, context); + + // Assert + assertThat(kinesisBatchResponse.getBatchItemFailures()).hasSize(1); + StreamsEventResponse.BatchItemFailure batchItemFailure = kinesisBatchResponse.getBatchItemFailures().get(0); + assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("49545115243490985018280067714973144582180062593244200961"); + } + + @ParameterizedTest + @Event(value = "kinesis_event.json", type = KinesisEvent.class) + public void failingFailureHandlerShouldntFailBatch(KinesisEvent event) { + // Arrange + AtomicBoolean wasCalled = new AtomicBoolean(false); + BatchMessageHandler handler = new BatchMessageHandlerBuilder() + .withKinesisBatchHandler() + .withFailureHandler((e, ex) -> { + wasCalled.set(true); + throw new RuntimeException("Well, this doesn't look great"); + }) + .buildWithMessageHandler(this::processMessageFailsForFixedProduct, Product.class); + + // Act + StreamsEventResponse kinesisBatchResponse = handler.processBatch(event, context); + + // Assert + assertThat(kinesisBatchResponse).isNotNull(); + assertThat(kinesisBatchResponse.getBatchItemFailures().size()).isEqualTo(1); + assertThat(wasCalled.get()).isTrue(); + StreamsEventResponse.BatchItemFailure batchItemFailure = kinesisBatchResponse.getBatchItemFailures().get(0); + assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("49545115243490985018280067714973144582180062593244200961"); + } + + @ParameterizedTest + @Event(value = "kinesis_event.json", type = KinesisEvent.class) + public void failingSuccessHandlerShouldntFailBatchButShouldFailMessage(KinesisEvent event) { + // Arrange + AtomicBoolean wasCalled = new AtomicBoolean(false); + BatchMessageHandler handler = new BatchMessageHandlerBuilder() + .withKinesisBatchHandler() + .withSuccessHandler((e) -> { + wasCalled.set(true); + if (e.getKinesis().getSequenceNumber().equals("49545115243490985018280067714973144582180062593244200961")) { + throw new RuntimeException("Success handler throws"); + } + }) + .buildWithRawMessageHandler(this::processMessageSucceeds); + + // Act + StreamsEventResponse kinesisBatchResponse = handler.processBatch(event, context); + + // Assert + assertThat(kinesisBatchResponse).isNotNull(); + assertThat(kinesisBatchResponse.getBatchItemFailures().size()).isEqualTo(1); + assertThat(wasCalled.get()).isTrue(); + StreamsEventResponse.BatchItemFailure batchItemFailure = kinesisBatchResponse.getBatchItemFailures().get(0); + assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("49545115243490985018280067714973144582180062593244200961"); + } + +} diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java index 599ec6587..4c9858724 100644 --- a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java @@ -4,23 +4,28 @@ import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; import com.amazonaws.services.lambda.runtime.events.SQSEvent; import com.amazonaws.services.lambda.runtime.tests.annotations.Event; -import org.assertj.core.api.Assertions; import org.junit.jupiter.params.ParameterizedTest; import org.mockito.Mock; import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; import software.amazon.lambda.powertools.model.Product; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.assertj.core.api.Assertions.assertThat; + public class SQSBatchProcessorTest { @Mock private Context context; - private void processMessageFailSometimes(SQSEvent.SQSMessage message, Context context) { + // A handler that throws an exception for _one_ of the sample messages + private void processMessageFailsForFixedMessage(SQSEvent.SQSMessage message, Context context) { if (message.getMessageId().equals("e9144555-9a4f-4ec3-99a0-34ce359b4b54")) { throw new RuntimeException("fake exception"); } } + // A handler that throws an exception for _one_ of the deserialized products in the same messages public void processMessageFailsForFixedProduct(Product product, Context context) { if (product.getId() == 12345) { throw new RuntimeException("fake exception"); @@ -30,32 +35,37 @@ public void processMessageFailsForFixedProduct(Product product, Context context) @ParameterizedTest @Event(value = "sqs_event.json", type = SQSEvent.class) public void shouldAddMessageToBatchFailure_whenException_withMessage(SQSEvent event) { + // Arrange BatchMessageHandler handler = new BatchMessageHandlerBuilder() .withSqsBatchHandler() - .buildWithRawMessageHandler(this::processMessageFailSometimes); + .buildWithRawMessageHandler(this::processMessageFailsForFixedMessage); + // Act SQSBatchResponse sqsBatchResponse = handler.processBatch(event, context); - Assertions.assertThat(sqsBatchResponse.getBatchItemFailures()).hasSize(1); + // Assert + assertThat(sqsBatchResponse.getBatchItemFailures()).hasSize(1); SQSBatchResponse.BatchItemFailure batchItemFailure = sqsBatchResponse.getBatchItemFailures().get(0); - Assertions.assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("e9144555-9a4f-4ec3-99a0-34ce359b4b54"); + assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("e9144555-9a4f-4ec3-99a0-34ce359b4b54"); } @ParameterizedTest @Event(value = "sqs_fifo_event.json", type = SQSEvent.class) public void shouldAddMessageToBatchFailure_whenException_withSQSFIFO(SQSEvent event) { + // Arrange BatchMessageHandler handler = new BatchMessageHandlerBuilder() .withSqsBatchHandler() - .buildWithRawMessageHandler(this::processMessageFailSometimes); + .buildWithRawMessageHandler(this::processMessageFailsForFixedMessage); + // Act SQSBatchResponse sqsBatchResponse = handler.processBatch(event, context); - Assertions.assertThat(sqsBatchResponse.getBatchItemFailures()).hasSize(2); - + // Assert + assertThat(sqsBatchResponse.getBatchItemFailures()).hasSize(2); SQSBatchResponse.BatchItemFailure batchItemFailure = sqsBatchResponse.getBatchItemFailures().get(0); - Assertions.assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("e9144555-9a4f-4ec3-99a0-34ce359b4b54"); + assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("e9144555-9a4f-4ec3-99a0-34ce359b4b54"); batchItemFailure = sqsBatchResponse.getBatchItemFailures().get(1); - Assertions.assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("f9144555-9a4f-4ec3-99a0-34ce359b4b54"); + assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("f9144555-9a4f-4ec3-99a0-34ce359b4b54"); } @@ -63,15 +73,41 @@ public void shouldAddMessageToBatchFailure_whenException_withSQSFIFO(SQSEvent ev @Event(value = "sqs_event.json", type = SQSEvent.class) public void shouldAddMessageToBatchFailure_whenException_withProduct(SQSEvent event) { + // Arrange + BatchMessageHandler handler = new BatchMessageHandlerBuilder() + .withSqsBatchHandler() + .buildWithMessageHandler(this::processMessageFailsForFixedProduct, Product.class); + + // Act + SQSBatchResponse sqsBatchResponse = handler.processBatch(event, context); + assertThat(sqsBatchResponse.getBatchItemFailures()).hasSize(1); + + // Assert + SQSBatchResponse.BatchItemFailure batchItemFailure = sqsBatchResponse.getBatchItemFailures().get(0); + assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("e9144555-9a4f-4ec3-99a0-34ce359b4b54"); + } + + @ParameterizedTest + @Event(value = "sqs_event.json", type = SQSEvent.class) + public void failingFailureHandlerShouldntFailBatch(SQSEvent event) { + // Arrange + AtomicBoolean wasCalled = new AtomicBoolean(false); BatchMessageHandler handler = new BatchMessageHandlerBuilder() .withSqsBatchHandler() + .withFailureHandler((e, ex) -> { + wasCalled.set(true); + throw new RuntimeException("Well, this doesn't look great"); + }) .buildWithMessageHandler(this::processMessageFailsForFixedProduct, Product.class); + // Act SQSBatchResponse sqsBatchResponse = handler.processBatch(event, context); - Assertions.assertThat(sqsBatchResponse.getBatchItemFailures()).hasSize(1); + // Assert + assertThat(sqsBatchResponse).isNotNull(); + assertThat(wasCalled.get()).isTrue(); SQSBatchResponse.BatchItemFailure batchItemFailure = sqsBatchResponse.getBatchItemFailures().get(0); - Assertions.assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("e9144555-9a4f-4ec3-99a0-34ce359b4b54"); + assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("e9144555-9a4f-4ec3-99a0-34ce359b4b54"); } } diff --git a/powertools-batch/src/test/resources/dynamo_event.json b/powertools-batch/src/test/resources/dynamo_event.json new file mode 100644 index 000000000..f28ce0e6e --- /dev/null +++ b/powertools-batch/src/test/resources/dynamo_event.json @@ -0,0 +1,97 @@ +{ + "Records": [ + { + "eventID": "c4ca4238a0b923820dcc509a6f75849b", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "eu-central-1", + "dynamodb": { + "Keys": { + "Id": { + "N": "101" + } + }, + "NewImage": { + "Message": { + "S": "New item!" + }, + "Id": { + "N": "101" + } + }, + "ApproximateCreationDateTime": 1428537600, + "SequenceNumber": "4421584500000000017450439091", + "SizeBytes": 26, + "StreamViewType": "NEW_AND_OLD_IMAGES" + }, + "eventSourceARN": "arn:aws:dynamodb:eu-central-1:123456789012:table/ExampleTableWithStream/stream/2015-06-27T00:48:05.899", + "userIdentity": { + "principalId": "dynamodb.amazonaws.com", + "type": "Service" + } + }, + { + "eventID": "c81e728d9d4c2f636f067f89cc14862c", + "eventName": "MODIFY", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "eu-central-1", + "dynamodb": { + "Keys": { + "Id": { + "N": "101" + } + }, + "NewImage": { + "Message": { + "S": "This item has changed" + }, + "Id": { + "N": "101" + } + }, + "OldImage": { + "Message": { + "S": "New item!" + }, + "Id": { + "N": "101" + } + }, + "ApproximateCreationDateTime": 1428537600, + "SequenceNumber": "4421584500000000017450439092", + "SizeBytes": 59, + "StreamViewType": "NEW_AND_OLD_IMAGES" + }, + "eventSourceARN": "arn:aws:dynamodb:eu-central-1:123456789012:table/ExampleTableWithStream/stream/2015-06-27T00:48:05.899" + }, + { + "eventID": "eccbc87e4b5ce2fe28308fd9f2a7baf3", + "eventName": "REMOVE", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "eu-central-1", + "dynamodb": { + "Keys": { + "Id": { + "N": "101" + } + }, + "OldImage": { + "Message": { + "S": "This item has changed" + }, + "Id": { + "N": "101" + } + }, + "ApproximateCreationDateTime": 1428537600, + "SequenceNumber": "4421584500000000017450439093", + "SizeBytes": 38, + "StreamViewType": "NEW_AND_OLD_IMAGES" + }, + "eventSourceARN": "arn:aws:dynamodb:eu-central-1:123456789012:table/ExampleTableWithStream/stream/2015-06-27T00:48:05.899" + } + ] +} \ No newline at end of file diff --git a/powertools-batch/src/test/resources/kinesis_event.json b/powertools-batch/src/test/resources/kinesis_event.json new file mode 100644 index 000000000..c9068da9b --- /dev/null +++ b/powertools-batch/src/test/resources/kinesis_event.json @@ -0,0 +1,38 @@ +{ + "Records": [ + { + "kinesis": { + "partitionKey": "partitionKey-03", + "kinesisSchemaVersion": "1.0", + "data": "eyJpZCI6MTIzNCwgIm5hbWUiOiJwcm9kdWN0IiwgInByaWNlIjo0Mn0=", + "sequenceNumber": "49545115243490985018280067714973144582180062593244200961", + "approximateArrivalTimestamp": 1428537600, + "encryptionType": "NONE" + }, + "eventSource": "aws:kinesis", + "eventID": "shardId-000000000000:49545115243490985018280067714973144582180062593244200961", + "invokeIdentityArn": "arn:aws:iam::EXAMPLE", + "eventVersion": "1.0", + "eventName": "aws:kinesis:record", + "eventSourceARN": "arn:aws:kinesis:EXAMPLE", + "awsRegion": "eu-central-1" + }, + { + "kinesis": { + "partitionKey": "partitionKey-03", + "kinesisSchemaVersion": "1.0", + "data": "eyJpZCI6MTIzNDUsICJuYW1lIjoicHJvZHVjdDUiLCAicHJpY2UiOjQ1fQ==", + "sequenceNumber": "49545115243490985018280067714973144582180062593244200962", + "approximateArrivalTimestamp": 1428537600, + "encryptionType": "NONE" + }, + "eventSource": "aws:kinesis", + "eventID": "shardId-000000000000:49545115243490985018280067714973144582180062593244200961", + "invokeIdentityArn": "arn:aws:iam::EXAMPLE", + "eventVersion": "1.0", + "eventName": "aws:kinesis:record", + "eventSourceARN": "arn:aws:kinesis:EXAMPLE", + "awsRegion": "eu-central-1" + } + ] +} \ No newline at end of file From 28d1f8dc767cceea84ed63c2586a3a58cf44ac1d Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Mon, 24 Jul 2023 11:26:03 +0200 Subject: [PATCH 22/79] More tests --- .../batch/KinesisBatchProcessorTest.java | 8 ++--- .../batch/SQSBatchProcessorTest.java | 30 +++++++++++++++++++ .../powertools/batch/SimpleBuilderTest.java | 27 ----------------- 3 files changed, 33 insertions(+), 32 deletions(-) delete mode 100644 powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SimpleBuilderTest.java diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/KinesisBatchProcessorTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/KinesisBatchProcessorTest.java index 528e39bcf..bf2263f0a 100644 --- a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/KinesisBatchProcessorTest.java +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/KinesisBatchProcessorTest.java @@ -2,8 +2,6 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.KinesisEvent; -import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; -import com.amazonaws.services.lambda.runtime.events.SQSEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; import com.amazonaws.services.lambda.runtime.tests.annotations.Event; import org.junit.jupiter.params.ParameterizedTest; @@ -99,12 +97,12 @@ public void failingFailureHandlerShouldntFailBatch(KinesisEvent event) { @Event(value = "kinesis_event.json", type = KinesisEvent.class) public void failingSuccessHandlerShouldntFailBatchButShouldFailMessage(KinesisEvent event) { // Arrange - AtomicBoolean wasCalled = new AtomicBoolean(false); + AtomicBoolean wasCalledAndFailed = new AtomicBoolean(false); BatchMessageHandler handler = new BatchMessageHandlerBuilder() .withKinesisBatchHandler() .withSuccessHandler((e) -> { - wasCalled.set(true); if (e.getKinesis().getSequenceNumber().equals("49545115243490985018280067714973144582180062593244200961")) { + wasCalledAndFailed.set(true); throw new RuntimeException("Success handler throws"); } }) @@ -116,7 +114,7 @@ public void failingSuccessHandlerShouldntFailBatchButShouldFailMessage(KinesisEv // Assert assertThat(kinesisBatchResponse).isNotNull(); assertThat(kinesisBatchResponse.getBatchItemFailures().size()).isEqualTo(1); - assertThat(wasCalled.get()).isTrue(); + assertThat(wasCalledAndFailed.get()).isTrue(); StreamsEventResponse.BatchItemFailure batchItemFailure = kinesisBatchResponse.getBatchItemFailures().get(0); assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("49545115243490985018280067714973144582180062593244200961"); } diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java index 4c9858724..8a269e93c 100644 --- a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java @@ -17,6 +17,9 @@ public class SQSBatchProcessorTest { @Mock private Context context; + // A handler that works + private void processMessageSucceeds(SQSEvent.SQSMessage sqsMessage) { + } // A handler that throws an exception for _one_ of the sample messages private void processMessageFailsForFixedMessage(SQSEvent.SQSMessage message, Context context) { @@ -110,4 +113,31 @@ public void failingFailureHandlerShouldntFailBatch(SQSEvent event) { assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("e9144555-9a4f-4ec3-99a0-34ce359b4b54"); } + @ParameterizedTest + @Event(value = "sqs_event.json", type = SQSEvent.class) + public void failingSuccessHandlerShouldntFailBatchButShouldFailMessage(SQSEvent event) { + // Arrange + AtomicBoolean wasCalledAndFailed = new AtomicBoolean(false); + BatchMessageHandler handler = new BatchMessageHandlerBuilder() + .withSqsBatchHandler() + .withSuccessHandler((e) -> { + if (e.getMessageId().equals("e9144555-9a4f-4ec3-99a0-34ce359b4b54")) { + wasCalledAndFailed.set(true); + throw new RuntimeException("Success handler throws"); + } + }) + .buildWithRawMessageHandler(this::processMessageSucceeds); + + // Act + SQSBatchResponse sqsBatchResponse = handler.processBatch(event, context); + + // Assert + assertThat(sqsBatchResponse).isNotNull(); + assertThat(wasCalledAndFailed.get()).isTrue(); + SQSBatchResponse.BatchItemFailure batchItemFailure = sqsBatchResponse.getBatchItemFailures().get(0); + assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("e9144555-9a4f-4ec3-99a0-34ce359b4b54"); + } + + + } diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SimpleBuilderTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SimpleBuilderTest.java deleted file mode 100644 index 187363f59..000000000 --- a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SimpleBuilderTest.java +++ /dev/null @@ -1,27 +0,0 @@ -package software.amazon.lambda.powertools.batch; - -import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; -import com.amazonaws.services.lambda.runtime.events.SQSEvent; -import org.junit.jupiter.api.Test; -import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; -import software.amazon.lambda.powertools.model.Basket; -import com.amazonaws.services.lambda.runtime.Context; - -import static org.assertj.core.api.Assertions.assertThat; - -public class SimpleBuilderTest { - - @Test - public void sqsBuilderWorks() { - BatchMessageHandler messageHandler = new BatchMessageHandlerBuilder() - .withSqsBatchHandler() - .withFailureHandler((c, e) -> System.out.println("Whoops!")) - .buildWithMessageHandler(this::processMessage, Basket.class); - - assertThat(messageHandler).isNotNull(); - } - - public void processMessage(Basket basket, Context context) { - - } -} From a660a8b07488c3f260125bca0a27f4831301efeb Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Mon, 24 Jul 2023 17:59:09 +0200 Subject: [PATCH 23/79] More tests and starting to add an example --- examples/powertools-examples-batch/.gitignore | 38 +++++++++ examples/powertools-examples-batch/pom.xml | 43 ++++++++++ .../src/main/java/SqsApp.java | 31 +++++++ .../src/main/java}/model/Product.java | 2 +- pom.xml | 1 + .../batch/KinesisBatchProcessorTest.java | 2 +- .../batch/SQSBatchProcessorTest.java | 2 +- .../powertools/batch}/model/Basket.java | 2 +- .../powertools/batch/model/Product.java | 80 +++++++++++++++++++ 9 files changed, 197 insertions(+), 4 deletions(-) create mode 100644 examples/powertools-examples-batch/.gitignore create mode 100644 examples/powertools-examples-batch/pom.xml create mode 100644 examples/powertools-examples-batch/src/main/java/SqsApp.java rename {powertools-batch/src/main/java/software/amazon/lambda/powertools => examples/powertools-examples-batch/src/main/java}/model/Product.java (97%) rename powertools-batch/src/{main/java/software/amazon/lambda/powertools => test/java/software/amazon/lambda/powertools/batch}/model/Basket.java (96%) create mode 100644 powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/model/Product.java diff --git a/examples/powertools-examples-batch/.gitignore b/examples/powertools-examples-batch/.gitignore new file mode 100644 index 000000000..5ff6309b7 --- /dev/null +++ b/examples/powertools-examples-batch/.gitignore @@ -0,0 +1,38 @@ +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/**/target/ +!**/src/test/**/target/ + +### IntelliJ IDEA ### +.idea/modules.xml +.idea/jarRepositories.xml +.idea/compiler.xml +.idea/libraries/ +*.iws +*.iml +*.ipr + +### Eclipse ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ +!**/src/main/**/build/ +!**/src/test/**/build/ + +### VS Code ### +.vscode/ + +### Mac OS ### +.DS_Store \ No newline at end of file diff --git a/examples/powertools-examples-batch/pom.xml b/examples/powertools-examples-batch/pom.xml new file mode 100644 index 000000000..24ab10b2a --- /dev/null +++ b/examples/powertools-examples-batch/pom.xml @@ -0,0 +1,43 @@ + + + 4.0.0 + + software.amazon.lambda.examples + 1.17.0-SNAPSHOT + powertools-examples-batch + jar + Powertools for AWS Lambda (Java) library Examples - Batch + + + 2.20.0 + 1.8 + 1.8 + true + + + + + software.amazon.lambda + powertools-tracing + ${project.version} + + + software.amazon.lambda + powertools-logging + ${project.version} + + + software.amazon.lambda + powertools-batch + ${project.version} + + + com.amazonaws + aws-lambda-java-core + 1.2.2 + + + + \ No newline at end of file diff --git a/examples/powertools-examples-batch/src/main/java/SqsApp.java b/examples/powertools-examples-batch/src/main/java/SqsApp.java new file mode 100644 index 000000000..b091ea3ad --- /dev/null +++ b/examples/powertools-examples-batch/src/main/java/SqsApp.java @@ -0,0 +1,31 @@ +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; +import com.amazonaws.services.lambda.runtime.events.SQSEvent; +import model.Product; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import software.amazon.lambda.powertools.batch.BatchMessageHandlerBuilder; +import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; + +public class SqsApp implements RequestHandler { + private final static Logger LOGGER = LogManager.getLogger(SqsApp.class); + private final BatchMessageHandler handler; + + public SqsApp() { + handler = new BatchMessageHandlerBuilder() + .withSqsBatchHandler() + .buildWithMessageHandler(this::processMessage, Product.class); + } + + @Override + public SQSBatchResponse handleRequest(SQSEvent sqsEvent, Context context) { + return handler.processBatch(sqsEvent, context); + } + + + private void processMessage(Product p, Context c) { + LOGGER.info("Processing product " + p); + } + +} diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/model/Product.java b/examples/powertools-examples-batch/src/main/java/model/Product.java similarity index 97% rename from powertools-batch/src/main/java/software/amazon/lambda/powertools/model/Product.java rename to examples/powertools-examples-batch/src/main/java/model/Product.java index 97b046189..fcb862923 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/model/Product.java +++ b/examples/powertools-examples-batch/src/main/java/model/Product.java @@ -11,7 +11,7 @@ * limitations under the License. * */ -package software.amazon.lambda.powertools.model; +package model; import java.util.Objects; diff --git a/pom.xml b/pom.xml index 3239bbf3a..6ccea24ef 100644 --- a/pom.xml +++ b/pom.xml @@ -41,6 +41,7 @@ powertools-e2e-tests examples powertools-batch + examples/powertools-examples-batch diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/KinesisBatchProcessorTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/KinesisBatchProcessorTest.java index bf2263f0a..811ddbe0b 100644 --- a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/KinesisBatchProcessorTest.java +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/KinesisBatchProcessorTest.java @@ -7,7 +7,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.mockito.Mock; import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; -import software.amazon.lambda.powertools.model.Product; +import software.amazon.lambda.powertools.batch.model.Product; import java.util.concurrent.atomic.AtomicBoolean; diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java index 8a269e93c..5206f99e9 100644 --- a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java @@ -7,7 +7,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.mockito.Mock; import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; -import software.amazon.lambda.powertools.model.Product; +import software.amazon.lambda.powertools.batch.model.Product; import java.util.concurrent.atomic.AtomicBoolean; diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/model/Basket.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/model/Basket.java similarity index 96% rename from powertools-batch/src/main/java/software/amazon/lambda/powertools/model/Basket.java rename to powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/model/Basket.java index ee08523ac..fd1c9fcc8 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/model/Basket.java +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/model/Basket.java @@ -11,7 +11,7 @@ * limitations under the License. * */ -package software.amazon.lambda.powertools.model; +package software.amazon.lambda.powertools.batch.model; import java.util.ArrayList; import java.util.Arrays; diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/model/Product.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/model/Product.java new file mode 100644 index 000000000..20f9da9a4 --- /dev/null +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/model/Product.java @@ -0,0 +1,80 @@ +package software.amazon.lambda.powertools.batch.model; + +/* + * Copyright 2022 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import java.util.Objects; + +public class Product { + private long id; + + private String name; + + private double price; + + public Product() { + } + + public Product(long id, String name, double price) { + this.id = id; + this.name = name; + this.price = price; + } + + public long getId() { + return id; + } + + public void setId(long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public double getPrice() { + return price; + } + + public void setPrice(double price) { + this.price = price; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Product product = (Product) o; + return id == product.id && Double.compare(product.price, price) == 0 && Objects.equals(name, product.name); + } + + @Override + public int hashCode() { + return Objects.hash(id, name, price); + } + + @Override + public String toString() { + return "Product{" + + "id=" + id + + ", name='" + name + '\'' + + ", price=" + price + + '}'; + } +} From 2332e2d250970ff26c9887411b33605fcf72c073 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Tue, 25 Jul 2023 16:04:53 +0200 Subject: [PATCH 24/79] Working on batch --- examples/powertools-examples-batch/pom.xml | 12 +- .../batch/kinesis/KinesisBatchHandler.java | 33 ++++ .../batch/kinesis/KinesisBatchSender.java | 77 +++++++++ .../{ => org/demo/batch}/model/Product.java | 2 +- .../demo/batch/sqs/SqsBatchHandler.java} | 10 +- .../org/demo/batch/sqs/SqsBatchSender.java | 72 +++++++++ .../template-sqs.yaml | 147 ++++++++++++++++++ 7 files changed, 347 insertions(+), 6 deletions(-) create mode 100644 examples/powertools-examples-batch/src/main/java/org/demo/batch/kinesis/KinesisBatchHandler.java create mode 100644 examples/powertools-examples-batch/src/main/java/org/demo/batch/kinesis/KinesisBatchSender.java rename examples/powertools-examples-batch/src/main/java/{ => org/demo/batch}/model/Product.java (98%) rename examples/powertools-examples-batch/src/main/java/{SqsApp.java => org/demo/batch/sqs/SqsBatchHandler.java} (83%) create mode 100644 examples/powertools-examples-batch/src/main/java/org/demo/batch/sqs/SqsBatchSender.java create mode 100644 examples/powertools-examples-batch/template-sqs.yaml diff --git a/examples/powertools-examples-batch/pom.xml b/examples/powertools-examples-batch/pom.xml index 24ab10b2a..20ead1e9f 100644 --- a/examples/powertools-examples-batch/pom.xml +++ b/examples/powertools-examples-batch/pom.xml @@ -15,6 +15,7 @@ 1.8 1.8 true + 2.20.109 @@ -38,6 +39,15 @@ aws-lambda-java-core 1.2.2 - + + software.amazon.awssdk + sdk-core + ${sdk.version} + + + software.amazon.awssdk + kinesis + ${sdk.version} + \ No newline at end of file diff --git a/examples/powertools-examples-batch/src/main/java/org/demo/batch/kinesis/KinesisBatchHandler.java b/examples/powertools-examples-batch/src/main/java/org/demo/batch/kinesis/KinesisBatchHandler.java new file mode 100644 index 000000000..d9339549b --- /dev/null +++ b/examples/powertools-examples-batch/src/main/java/org/demo/batch/kinesis/KinesisBatchHandler.java @@ -0,0 +1,33 @@ +package org.demo.batch.kinesis; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.amazonaws.services.lambda.runtime.events.KinesisEvent; +import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.demo.batch.model.Product; +import software.amazon.lambda.powertools.batch.BatchMessageHandlerBuilder; +import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; + +public class KinesisBatchHandler implements RequestHandler { + + private final static Logger LOGGER = LogManager.getLogger(org.demo.batch.sqs.SqsBatchHandler.class); + private final BatchMessageHandler handler; + + public KinesisBatchHandler() { + handler = new BatchMessageHandlerBuilder() + .withKinesisBatchHandler() + .buildWithMessageHandler(this::processMessage, Product.class); + } + + @Override + public StreamsEventResponse handleRequest(KinesisEvent kinesisEvent, Context context) { + return handler.processBatch(kinesisEvent, context); + } + + private void processMessage(Product p, Context c) { + LOGGER.info("Processing product " + p); + } + +} diff --git a/examples/powertools-examples-batch/src/main/java/org/demo/batch/kinesis/KinesisBatchSender.java b/examples/powertools-examples-batch/src/main/java/org/demo/batch/kinesis/KinesisBatchSender.java new file mode 100644 index 000000000..132d0f5c2 --- /dev/null +++ b/examples/powertools-examples-batch/src/main/java/org/demo/batch/kinesis/KinesisBatchSender.java @@ -0,0 +1,77 @@ +package org.demo.batch.kinesis; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.amazonaws.services.lambda.runtime.events.ScheduledEvent; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.demo.batch.model.Product; +import software.amazon.awssdk.core.BytesWrapper; +import software.amazon.awssdk.core.SdkBytes; +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; +import software.amazon.awssdk.services.kinesis.KinesisClient; +import software.amazon.awssdk.services.kinesis.model.PutRecordsRequest; +import software.amazon.awssdk.services.kinesis.model.PutRecordsRequestEntry; +import software.amazon.awssdk.services.kinesis.model.PutRecordsResponse; +import software.amazon.awssdk.services.sqs.SqsClient; +import software.amazon.awssdk.services.sqs.model.SendMessageBatchRequest; +import software.amazon.awssdk.services.sqs.model.SendMessageBatchRequestEntry; +import software.amazon.awssdk.services.sqs.model.SendMessageBatchResponse; + +import java.security.SecureRandom; +import java.util.List; +import java.util.stream.IntStream; + +import static java.util.stream.Collectors.toList; + +public class KinesisBatchSender implements RequestHandler { + + private static final Logger LOGGER = LogManager.getLogger(KinesisBatchSender.class); + + private final KinesisClient kinesisClient; + private final SecureRandom random; + private final ObjectMapper objectMapper; + + public KinesisBatchSender() { + kinesisClient = KinesisClient.builder() + .httpClient(UrlConnectionHttpClient.create()) + .build(); + random = new SecureRandom(); + objectMapper = new ObjectMapper(); + } + + @Override + public String handleRequest(ScheduledEvent scheduledEvent, Context context) { + String streamName = System.getenv("STREAM_NAME"); + + LOGGER.info("handleRequest"); + + // Push 5 messages on each invoke. + List records = IntStream.range(0, 5) + .mapToObj(value -> { + long id = random.nextLong(); + float price = random.nextFloat(); + Product product = new Product(id, "product-" + id, price); + try { + SdkBytes data = SdkBytes.fromUtf8String(objectMapper.writeValueAsString(product)); + return PutRecordsRequestEntry.builder() + .data(data) + .build(); + } catch (JsonProcessingException e) { + LOGGER.error("Failed serializing body", e); + throw new RuntimeException(e); + } + }).collect(toList()); + + PutRecordsResponse putRecordsResponse = kinesisClient.putRecords(PutRecordsRequest.builder() + .streamName(streamName) + .records(records) + .build()); + + LOGGER.info("Sent Message {}", putRecordsResponse); + + return "Success"; + } +} diff --git a/examples/powertools-examples-batch/src/main/java/model/Product.java b/examples/powertools-examples-batch/src/main/java/org/demo/batch/model/Product.java similarity index 98% rename from examples/powertools-examples-batch/src/main/java/model/Product.java rename to examples/powertools-examples-batch/src/main/java/org/demo/batch/model/Product.java index fcb862923..b513226b0 100644 --- a/examples/powertools-examples-batch/src/main/java/model/Product.java +++ b/examples/powertools-examples-batch/src/main/java/org/demo/batch/model/Product.java @@ -11,7 +11,7 @@ * limitations under the License. * */ -package model; +package org.demo.batch.model; import java.util.Objects; diff --git a/examples/powertools-examples-batch/src/main/java/SqsApp.java b/examples/powertools-examples-batch/src/main/java/org/demo/batch/sqs/SqsBatchHandler.java similarity index 83% rename from examples/powertools-examples-batch/src/main/java/SqsApp.java rename to examples/powertools-examples-batch/src/main/java/org/demo/batch/sqs/SqsBatchHandler.java index b091ea3ad..5d508b5bf 100644 --- a/examples/powertools-examples-batch/src/main/java/SqsApp.java +++ b/examples/powertools-examples-batch/src/main/java/org/demo/batch/sqs/SqsBatchHandler.java @@ -1,18 +1,20 @@ +package org.demo.batch.sqs; + import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.RequestHandler; import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; import com.amazonaws.services.lambda.runtime.events.SQSEvent; -import model.Product; +import org.demo.batch.model.Product; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import software.amazon.lambda.powertools.batch.BatchMessageHandlerBuilder; import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; -public class SqsApp implements RequestHandler { - private final static Logger LOGGER = LogManager.getLogger(SqsApp.class); +public class SqsBatchHandler implements RequestHandler { + private final static Logger LOGGER = LogManager.getLogger(SqsBatchHandler.class); private final BatchMessageHandler handler; - public SqsApp() { + public SqsBatchHandler() { handler = new BatchMessageHandlerBuilder() .withSqsBatchHandler() .buildWithMessageHandler(this::processMessage, Product.class); diff --git a/examples/powertools-examples-batch/src/main/java/org/demo/batch/sqs/SqsBatchSender.java b/examples/powertools-examples-batch/src/main/java/org/demo/batch/sqs/SqsBatchSender.java new file mode 100644 index 000000000..246b7ad99 --- /dev/null +++ b/examples/powertools-examples-batch/src/main/java/org/demo/batch/sqs/SqsBatchSender.java @@ -0,0 +1,72 @@ +package org.demo.batch.sqs; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.amazonaws.services.lambda.runtime.events.ScheduledEvent; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.demo.batch.model.Product; +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; +import software.amazon.awssdk.services.sqs.SqsClient; +import software.amazon.awssdk.services.sqs.model.SendMessageBatchRequest; +import software.amazon.awssdk.services.sqs.model.SendMessageBatchRequestEntry; +import software.amazon.awssdk.services.sqs.model.SendMessageBatchResponse; + +import java.security.SecureRandom; +import java.util.List; +import java.util.stream.IntStream; + +import static java.util.stream.Collectors.toList; + +public class SqsBatchSender implements RequestHandler { + + private static final Logger LOGGER = LogManager.getLogger(SqsBatchSender.class); + + private final SqsClient sqsClient; + private final SecureRandom random; + private final ObjectMapper objectMapper; + + public SqsBatchSender() { + sqsClient = SqsClient.builder() + .httpClient(UrlConnectionHttpClient.create()) + .build(); + random = new SecureRandom(); + objectMapper = new ObjectMapper(); + } + + @Override + public String handleRequest(ScheduledEvent scheduledEvent, Context context) { + String queueUrl = System.getenv("QUEUE_URL"); + + LOGGER.info("handleRequest"); + + // Push 5 messages on each invoke. + List batchRequestEntries = IntStream.range(0, 5) + .mapToObj(value -> { + long id = random.nextLong(); + float price = random.nextFloat(); + Product product = new Product(id, "product-" + id, price); + try { + + return SendMessageBatchRequestEntry.builder() + .id(scheduledEvent.getId() + value) + .messageBody(objectMapper.writeValueAsString(product)) + .build(); + } catch (JsonProcessingException e) { + LOGGER.error("Failed serializing body", e); + throw new RuntimeException(e); + } + }).collect(toList()); + + SendMessageBatchResponse sendMessageBatchResponse = sqsClient.sendMessageBatch(SendMessageBatchRequest.builder() + .queueUrl(queueUrl) + .entries(batchRequestEntries) + .build()); + + LOGGER.info("Sent Message {}", sendMessageBatchResponse); + + return "Success"; + } +} diff --git a/examples/powertools-examples-batch/template-sqs.yaml b/examples/powertools-examples-batch/template-sqs.yaml new file mode 100644 index 000000000..c56d7544b --- /dev/null +++ b/examples/powertools-examples-batch/template-sqs.yaml @@ -0,0 +1,147 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: > + sqs batch processing demo + +Globals: + Function: + Timeout: 20 + Runtime: java11 + MemorySize: 512 + Tracing: Active + Environment: + Variables: + POWERTOOLS_LOG_LEVEL: INFO + POWERTOOLS_LOGGER_SAMPLE_RATE: 1.0 + POWERTOOLS_LOGGER_LOG_EVENT: true + +Resources: + CustomerKey: + Type: AWS::KMS::Key + Properties: + Description: KMS key for encrypted queues + Enabled: true + KeyPolicy: + Version: '2012-10-17' + Statement: + - Sid: Enable IAM User Permissions + Effect: Allow + Principal: + AWS: !Sub 'arn:aws:iam::${AWS::AccountId}:root' + Action: 'kms:*' + Resource: '*' + - Sid: Allow use of the key + Effect: Allow + Principal: + Service: lambda.amazonaws.com + Action: + - kms:Decrypt + - kms:GenerateDataKey + Resource: '*' + + CustomerKeyAlias: + Type: AWS::KMS::Alias + Properties: + AliasName: alias/powertools-batch-sqs-demo + TargetKeyId: !Ref CustomerKey + + DemoDlqSqsQueue: + Type: AWS::SQS::Queue + Properties: + KmsMasterKeyId: !Ref CustomerKey + + DemoSqsQueue: + Type: AWS::SQS::Queue + Properties: + RedrivePolicy: + deadLetterTargetArn: + Fn::GetAtt: + - "DemoDlqSqsQueue" + - "Arn" + maxReceiveCount: 2 + KmsMasterKeyId: !Ref CustomerKey + + DemoSQSSenderFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: . + Handler: org.demo.batch.sqs.SqsBatchSender::handleRequest + Environment: + Variables: + POWERTOOLS_SERVICE_NAME: sqs-batch-demo + QUEUE_URL: !Ref DemoSqsQueue + Policies: + - Statement: + - Sid: SQSSendMessageBatch + Effect: Allow + Action: + - sqs:SendMessageBatch + - sqs:SendMessage + Resource: !GetAtt DemoSqsQueue.Arn + - Sid: SQSKMSKey + Effect: Allow + Action: + - kms:GenerateDataKey + - kms:Decrypt + Resource: !GetAtt CustomerKey.Arn + Events: + CWSchedule: + Type: Schedule + Properties: + Schedule: 'rate(5 minutes)' + Name: !Join ["-", ["message-producer-schedule", !Select [0, !Split [-, !Select [2, !Split [/, !Ref AWS::StackId ]]]]]] + Description: Produce message to SQS via a Lambda function + Enabled: true + + DemoSQSConsumerFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: . + Handler: org.demo.batch.sqs.SqsBatchHandler::handleRequest + Environment: + Variables: + POWERTOOLS_SERVICE_NAME: sqs-demo + Policies: + - Statement: + - Sid: SQSDeleteGetAttribute + Effect: Allow + Action: + - sqs:DeleteMessageBatch + - sqs:GetQueueAttributes + Resource: !GetAtt DemoSqsQueue.Arn + - Sid: SQSSendMessageBatch + Effect: Allow + Action: + - sqs:SendMessageBatch + - sqs:SendMessage + Resource: !GetAtt DemoDlqSqsQueue.Arn + - Sid: SQSKMSKey + Effect: Allow + Action: + - kms:GenerateDataKey + - kms:Decrypt + Resource: !GetAtt CustomerKey.Arn + Events: + MySQSEvent: + Type: SQS + Properties: + Queue: !GetAtt DemoSqsQueue.Arn + BatchSize: 2 + MaximumBatchingWindowInSeconds: 300 + +Outputs: + DemoSqsQueue: + Description: "ARN for main SQS queue" + Value: !GetAtt DemoSqsQueue.Arn + DemoDlqSqsQueue: + Description: "ARN for DLQ" + Value: !GetAtt DemoDlqSqsQueue.Arn + DemoSQSSenderFunction: + Description: "SQS Batch Sender - Lambda Function ARN" + Value: !GetAtt DemoSQSSenderFunction.Arn + DemoSQSConsumerFunction: + Description: "SQS Batch Handler - Lambda Function ARN" + Value: !GetAtt DemoSQSConsumerFunction.Arn + DemoSQSConsumerFunctionRole: + Description: "Implicit IAM Role created for SQS Lambda Function ARN" + Value: !GetAtt DemoSQSConsumerFunctionRole.Arn From 4af594cabf07c2b41c4ad203b4f592d1e672cfab Mon Sep 17 00:00:00 2001 From: Michele Ricciardi Date: Tue, 25 Jul 2023 16:55:07 +0200 Subject: [PATCH 25/79] feat(batch): initial DdbBatchMessageHandler implementation --- .../batch/handler/DdbBatchMessageHandler.java | 37 +++++++++- .../batch/DdbBatchProcessorTest.java | 73 +++++++++++++++++++ 2 files changed, 107 insertions(+), 3 deletions(-) create mode 100644 powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/DdbBatchProcessorTest.java diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/DdbBatchMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/DdbBatchMessageHandler.java index cbf4960d0..ddf30aa52 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/DdbBatchMessageHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/DdbBatchMessageHandler.java @@ -3,11 +3,16 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.List; import java.util.function.BiConsumer; import java.util.function.Consumer; public class DdbBatchMessageHandler implements BatchMessageHandler{ + Logger LOGGER = LoggerFactory.getLogger(DdbBatchMessageHandler.class); private final Consumer successHandler; private final BiConsumer failureHandler; @@ -21,8 +26,34 @@ public DdbBatchMessageHandler(Consumer succe @Override public StreamsEventResponse processBatch(DynamodbEvent event, Context context) { - // TODO Probably helpful https://github.com/aws-powertools/powertools-lambda-java/blob/ee64d620aff083b1a9734d880af2e077d73692a8/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch2/DynamoDBBatchProcessor.java - // TODO -success/failure handlers too - throw new RuntimeException("Not implemented"); + List batchFailures = new ArrayList<>(); + + for (DynamodbEvent.DynamodbStreamRecord record : event.getRecords()) { + try { + + rawMessageHandler.accept(record, context); + // Report success if we have a handler + if (this.successHandler != null) { + this.successHandler.accept(record); + } + } catch (Throwable t) { + String sequenceNumber = record.getDynamodb().getSequenceNumber(); + LOGGER.error("Error while processing record with id {}: {}, adding it to batch item failures", sequenceNumber, t.getMessage()); + batchFailures.add(new StreamsEventResponse.BatchItemFailure(sequenceNumber)); + + // Report failure if we have a handler + if (this.failureHandler != null) { + // A failing failure handler is no reason to fail the batch + try { + this.failureHandler.accept(record, t); + } + catch (Throwable t2) { + LOGGER.warn("failureHandler threw handling failure", t2); + } + } + } + } + + return new StreamsEventResponse(batchFailures); } } diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/DdbBatchProcessorTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/DdbBatchProcessorTest.java new file mode 100644 index 000000000..c3b8a1cbb --- /dev/null +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/DdbBatchProcessorTest.java @@ -0,0 +1,73 @@ +package software.amazon.lambda.powertools.batch; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; +import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; +import com.amazonaws.services.lambda.runtime.tests.annotations.Event; +import org.junit.jupiter.params.ParameterizedTest; +import org.mockito.Mock; +import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; + +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.assertj.core.api.Assertions.assertThat; + +public class DdbBatchProcessorTest { + + @Mock + private Context context; + + private void processMessageSucceeds(DynamodbEvent.DynamodbStreamRecord record, Context context) { + // Great success + } + + private void processMessageFailsForFixedMessage(DynamodbEvent.DynamodbStreamRecord record, Context context) { + if (record.getDynamodb().getSequenceNumber().equals("4421584500000000017450439091")) { + throw new RuntimeException("fake exception"); + } + } + + @ParameterizedTest + @Event(value = "dynamo_event.json", type = DynamodbEvent.class) + public void shouldAddMessageToBatchFailure_whenException_withMessage(DynamodbEvent event) { + // Arrange + BatchMessageHandler handler = new BatchMessageHandlerBuilder() + .withDynamoDbBatchHandler() + .buildWithRawMessageHandler(this::processMessageFailsForFixedMessage); + + // Act + StreamsEventResponse dynamodbBatchResponse = handler.processBatch(event, context); + + // Assert + assertThat(dynamodbBatchResponse.getBatchItemFailures()).hasSize(1); + StreamsEventResponse.BatchItemFailure batchItemFailure = dynamodbBatchResponse.getBatchItemFailures().get(0); + assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("4421584500000000017450439091"); + } + + @ParameterizedTest + @Event(value = "dynamo_event.json", type = DynamodbEvent.class) + public void failingSuccessHandlerShouldntFailBatchButShouldFailMessage(DynamodbEvent event) { + // Arrange + AtomicBoolean wasCalledAndFailed = new AtomicBoolean(false); + BatchMessageHandler handler = new BatchMessageHandlerBuilder() + .withDynamoDbBatchHandler() + .withSuccessHandler((e) -> { + if (e.getDynamodb().getSequenceNumber().equals("4421584500000000017450439091")) { + wasCalledAndFailed.set(true); + throw new RuntimeException("Success handler throws"); + } + }) + .buildWithRawMessageHandler(this::processMessageSucceeds); + + // Act + StreamsEventResponse dynamodbBatchResponse = handler.processBatch(event, context); + + // Assert + assertThat(dynamodbBatchResponse).isNotNull(); + assertThat(dynamodbBatchResponse.getBatchItemFailures().size()).isEqualTo(1); + assertThat(wasCalledAndFailed.get()).isTrue(); + StreamsEventResponse.BatchItemFailure batchItemFailure = dynamodbBatchResponse.getBatchItemFailures().get(0); + assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("4421584500000000017450439091"); + } + +} From 34a58e6eed356e715a7e6accf1b590d9d519371b Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Wed, 26 Jul 2023 08:39:16 +0200 Subject: [PATCH 26/79] more --- .../src/main/java/org/demo/batch/sqs/SqsBatchSender.java | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/powertools-examples-batch/src/main/java/org/demo/batch/sqs/SqsBatchSender.java b/examples/powertools-examples-batch/src/main/java/org/demo/batch/sqs/SqsBatchSender.java index 246b7ad99..a817fc6a9 100644 --- a/examples/powertools-examples-batch/src/main/java/org/demo/batch/sqs/SqsBatchSender.java +++ b/examples/powertools-examples-batch/src/main/java/org/demo/batch/sqs/SqsBatchSender.java @@ -20,6 +20,7 @@ import static java.util.stream.Collectors.toList; + public class SqsBatchSender implements RequestHandler { private static final Logger LOGGER = LogManager.getLogger(SqsBatchSender.class); From ed161e91edf7a7af9f11224bb7914d86b096fa60 Mon Sep 17 00:00:00 2001 From: Michele Ricciardi Date: Wed, 26 Jul 2023 12:21:18 +0200 Subject: [PATCH 27/79] fix pom.xml for powertools-examples-batch --- examples/powertools-examples-batch/pom.xml | 135 +++++++++++++++++++++ 1 file changed, 135 insertions(+) diff --git a/examples/powertools-examples-batch/pom.xml b/examples/powertools-examples-batch/pom.xml index 20ead1e9f..82e15b9a2 100644 --- a/examples/powertools-examples-batch/pom.xml +++ b/examples/powertools-examples-batch/pom.xml @@ -44,10 +44,145 @@ sdk-core ${sdk.version} + + software.amazon.awssdk + dynamodb-enhanced + ${sdk.version} + software.amazon.awssdk kinesis ${sdk.version} + + + + + dev.aspectj + aspectj-maven-plugin + 1.13.1 + + ${maven.compiler.source} + ${maven.compiler.target} + ${maven.compiler.target} + + + software.amazon.lambda + powertools-tracing + + + software.amazon.lambda + powertools-logging + + + + + + + compile + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.5.0 + + + package + + shade + + + + + + + + + + + + com.github.edwgiz + maven-shade-plugin.log4j2-cachefile-transformer + 2.15 + + + + + + + + + jdk8 + + (,11) + + + 1.9.7 + + + + + org.aspectj + aspectjtools + ${aspectj.version} + + + + + + + + dev.aspectj + aspectj-maven-plugin + ${aspectj.plugin.version} + + ${maven.compiler.source} + ${maven.compiler.target} + ${maven.compiler.target} + + + software.amazon.lambda + powertools-tracing + + + software.amazon.lambda + powertools-logging + + + + + + + compile + test-compile + + + + + + + org.aspectj + aspectjtools + ${aspectj.version} + + + + + + + + \ No newline at end of file From f8812bec0f3e8980df04a0a36dc857bb9a5cb54d Mon Sep 17 00:00:00 2001 From: Michele Ricciardi Date: Wed, 26 Jul 2023 12:23:06 +0200 Subject: [PATCH 28/79] Add dynamodb example --- .../dynamo/DynamoDBStreamBatchHandler.java | 37 ++++++++ .../org/demo/batch/dynamo/DynamoDBWriter.java | 68 +++++++++++++++ .../java/org/demo/batch/model/DdbProduct.java | 84 +++++++++++++++++++ .../src/main/resources/log4j2.xml | 16 ++++ .../template-dynamodb.yaml | 72 ++++++++++++++++ 5 files changed, 277 insertions(+) create mode 100644 examples/powertools-examples-batch/src/main/java/org/demo/batch/dynamo/DynamoDBStreamBatchHandler.java create mode 100644 examples/powertools-examples-batch/src/main/java/org/demo/batch/dynamo/DynamoDBWriter.java create mode 100644 examples/powertools-examples-batch/src/main/java/org/demo/batch/model/DdbProduct.java create mode 100644 examples/powertools-examples-batch/src/main/resources/log4j2.xml create mode 100644 examples/powertools-examples-batch/template-dynamodb.yaml diff --git a/examples/powertools-examples-batch/src/main/java/org/demo/batch/dynamo/DynamoDBStreamBatchHandler.java b/examples/powertools-examples-batch/src/main/java/org/demo/batch/dynamo/DynamoDBStreamBatchHandler.java new file mode 100644 index 000000000..087501011 --- /dev/null +++ b/examples/powertools-examples-batch/src/main/java/org/demo/batch/dynamo/DynamoDBStreamBatchHandler.java @@ -0,0 +1,37 @@ +package org.demo.batch.dynamo; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; +import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.demo.batch.model.Product; +import software.amazon.lambda.powertools.batch.BatchMessageHandlerBuilder; +import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; + +public class DynamoDBStreamBatchHandler implements RequestHandler { + + private final static Logger LOGGER = LogManager.getLogger(DynamoDBStreamBatchHandler.class); + private final BatchMessageHandler handler; + + public DynamoDBStreamBatchHandler() { + handler = new BatchMessageHandlerBuilder() + .withDynamoDbBatchHandler() + .buildWithRawMessageHandler(this::processMessage); + } + + private void processMessage(DynamodbEvent.DynamodbStreamRecord dynamodbStreamRecord, Context context) { + LOGGER.info("Processing DynamoDB Stream Record" + dynamodbStreamRecord); + } + + @Override + public StreamsEventResponse handleRequest(DynamodbEvent kinesisEvent, Context context) { + return handler.processBatch(kinesisEvent, context); + } + + private void processMessage(Product p, Context c) { + LOGGER.info("Processing product " + p); + } + +} diff --git a/examples/powertools-examples-batch/src/main/java/org/demo/batch/dynamo/DynamoDBWriter.java b/examples/powertools-examples-batch/src/main/java/org/demo/batch/dynamo/DynamoDBWriter.java new file mode 100644 index 000000000..183931474 --- /dev/null +++ b/examples/powertools-examples-batch/src/main/java/org/demo/batch/dynamo/DynamoDBWriter.java @@ -0,0 +1,68 @@ +package org.demo.batch.dynamo; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.amazonaws.services.lambda.runtime.events.ScheduledEvent; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.demo.batch.model.DdbProduct; +import software.amazon.awssdk.enhanced.dynamodb.DynamoDbEnhancedClient; +import software.amazon.awssdk.enhanced.dynamodb.TableSchema; +import software.amazon.awssdk.enhanced.dynamodb.model.BatchWriteItemEnhancedRequest; +import software.amazon.awssdk.enhanced.dynamodb.model.BatchWriteResult; +import software.amazon.awssdk.enhanced.dynamodb.model.WriteBatch; +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; + +import java.security.SecureRandom; +import java.util.UUID; + +public class DynamoDBWriter implements RequestHandler { + + private static final Logger LOGGER = LogManager.getLogger(DynamoDBWriter.class); + + private final DynamoDbEnhancedClient enhancedClient; + + private final SecureRandom random; + + public DynamoDBWriter() { + random = new SecureRandom(); + DynamoDbClient dynamoDbClient = DynamoDbClient.builder() + .httpClientBuilder(UrlConnectionHttpClient.builder()) + .build(); + + enhancedClient = DynamoDbEnhancedClient.builder() + .dynamoDbClient(dynamoDbClient) + .build(); + } + + @Override + public String handleRequest(ScheduledEvent scheduledEvent, Context context) { + String tableName = System.getenv("TABLE_NAME"); + + LOGGER.info("handleRequest"); + + WriteBatch.Builder productBuilder = WriteBatch.builder(DdbProduct.class) + .mappedTableResource(enhancedClient.table(tableName, TableSchema.fromBean(DdbProduct.class))); + + + for (int i = 0; i < 5; i++) { + String id = UUID.randomUUID().toString(); + + float price = random.nextFloat(); + DdbProduct product = new DdbProduct(id, "product-" + id, price); + productBuilder.addPutItem(product); + } + + BatchWriteItemEnhancedRequest batchWriteItemEnhancedRequest = BatchWriteItemEnhancedRequest.builder().writeBatches( + productBuilder.build()) + .build(); + + BatchWriteResult batchWriteResult = enhancedClient.batchWriteItem(batchWriteItemEnhancedRequest); + + + LOGGER.info("Sent Message {}", batchWriteResult); + + return "Success"; + } +} diff --git a/examples/powertools-examples-batch/src/main/java/org/demo/batch/model/DdbProduct.java b/examples/powertools-examples-batch/src/main/java/org/demo/batch/model/DdbProduct.java new file mode 100644 index 000000000..91e9943d2 --- /dev/null +++ b/examples/powertools-examples-batch/src/main/java/org/demo/batch/model/DdbProduct.java @@ -0,0 +1,84 @@ +/* + * Copyright 2022 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.demo.batch.model; + +import software.amazon.awssdk.enhanced.dynamodb.mapper.annotations.DynamoDbBean; +import software.amazon.awssdk.enhanced.dynamodb.mapper.annotations.DynamoDbPartitionKey; + +import java.util.Objects; + +@DynamoDbBean +public class DdbProduct { + private String id; + + private String name; + + private double price; + + public DdbProduct() { + } + + public DdbProduct(String id, String name, double price) { + this.id = id; + this.name = name; + this.price = price; + } + + @DynamoDbPartitionKey + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public double getPrice() { + return price; + } + + public void setPrice(double price) { + this.price = price; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DdbProduct that = (DdbProduct) o; + return Double.compare(that.price, price) == 0 && Objects.equals(id, that.id) && Objects.equals(name, that.name); + } + + @Override + public int hashCode() { + return Objects.hash(id, name, price); + } + + @Override + public String toString() { + return "Product{" + + "id=" + id + + ", name='" + name + '\'' + + ", price=" + price + + '}'; + } +} diff --git a/examples/powertools-examples-batch/src/main/resources/log4j2.xml b/examples/powertools-examples-batch/src/main/resources/log4j2.xml new file mode 100644 index 000000000..e1fd14cea --- /dev/null +++ b/examples/powertools-examples-batch/src/main/resources/log4j2.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/examples/powertools-examples-batch/template-dynamodb.yaml b/examples/powertools-examples-batch/template-dynamodb.yaml new file mode 100644 index 000000000..7c32514bc --- /dev/null +++ b/examples/powertools-examples-batch/template-dynamodb.yaml @@ -0,0 +1,72 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: > + powertools-examples-batch-ddb + + Sample SAM Template for powertools-examples-batch-ddb + +# More info about Globals: https://github.com/awslabs/serverless-application-model/blob/master/docs/globals.rst +Globals: + Function: + Timeout: 20 + Runtime: java17 + MemorySize: 512 + Tracing: Active + Architectures: + - x86_64 + Environment: + Variables: + POWERTOOLS_LOG_LEVEL: INFO + POWERTOOLS_LOGGER_SAMPLE_RATE: 1.0 + POWERTOOLS_LOGGER_LOG_EVENT: true + +Resources: + DynamoDBTable: + Type: AWS::DynamoDB::Table + Properties: + AttributeDefinitions: + - AttributeName: id + AttributeType: S + KeySchema: + - AttributeName: id + KeyType: HASH + ProvisionedThroughput: + ReadCapacityUnits: 5 + WriteCapacityUnits: 5 + StreamSpecification: + StreamViewType: NEW_IMAGE + + DemoDynamoDBStreamsConsumerFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: . + Handler: org.demo.batch.dynamo.DynamoDBStreamBatchHandler::handleRequest + Policies: AWSLambdaDynamoDBExecutionRole + Events: + Stream: + Type: DynamoDB + Properties: + Stream: !GetAtt DynamoDBTable.StreamArn + BatchSize: 100 + StartingPosition: TRIM_HORIZON + + DemoDynamoDBWriter: + Type: AWS::Serverless::Function # More info about Function Resource: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#awsserverlessfunction + Properties: + CodeUri: . + Handler: org.demo.batch.dynamo.DynamoDBWriter::handleRequest + Policies: + - DynamoDBCrudPolicy: + TableName: !Ref DynamoDBTable + Environment: # More info about Env Vars: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#environment-object + Variables: + TABLE_NAME: !Ref DynamoDBTable + Events: + CWSchedule: + Type: Schedule + Properties: + Schedule: 'rate(5 minutes)' + Name: !Join ["-", ["ddb-writer-schedule", !Select [0, !Split [-, !Select [2, !Split [/, !Ref AWS::StackId ]]]]]] + Description: Produce message to SQS via a Lambda function + Enabled: true + From 53a5abeb7e8f0e87f148c47c455510019d5c222d Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Wed, 26 Jul 2023 12:41:15 +0200 Subject: [PATCH 29/79] Move template into subdir --- .../{template-sqs.yaml => sqstemplate/template.yml} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename examples/powertools-examples-batch/{template-sqs.yaml => sqstemplate/template.yml} (99%) diff --git a/examples/powertools-examples-batch/template-sqs.yaml b/examples/powertools-examples-batch/sqstemplate/template.yml similarity index 99% rename from examples/powertools-examples-batch/template-sqs.yaml rename to examples/powertools-examples-batch/sqstemplate/template.yml index c56d7544b..1b59aca55 100644 --- a/examples/powertools-examples-batch/template-sqs.yaml +++ b/examples/powertools-examples-batch/sqstemplate/template.yml @@ -64,7 +64,7 @@ Resources: DemoSQSSenderFunction: Type: AWS::Serverless::Function Properties: - CodeUri: . + CodeUri: .. Handler: org.demo.batch.sqs.SqsBatchSender::handleRequest Environment: Variables: @@ -96,7 +96,7 @@ Resources: DemoSQSConsumerFunction: Type: AWS::Serverless::Function Properties: - CodeUri: . + CodeUri: .. Handler: org.demo.batch.sqs.SqsBatchHandler::handleRequest Environment: Variables: From 98724108c41f4ec3f38d074cbb352a2d4e64f7be Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Wed, 26 Jul 2023 13:02:39 +0200 Subject: [PATCH 30/79] Better structure --- .../{sqstemplate => deploy/sqs}/template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename examples/powertools-examples-batch/{sqstemplate => deploy/sqs}/template.yml (99%) diff --git a/examples/powertools-examples-batch/sqstemplate/template.yml b/examples/powertools-examples-batch/deploy/sqs/template.yml similarity index 99% rename from examples/powertools-examples-batch/sqstemplate/template.yml rename to examples/powertools-examples-batch/deploy/sqs/template.yml index 1b59aca55..d038362b0 100644 --- a/examples/powertools-examples-batch/sqstemplate/template.yml +++ b/examples/powertools-examples-batch/deploy/sqs/template.yml @@ -64,7 +64,7 @@ Resources: DemoSQSSenderFunction: Type: AWS::Serverless::Function Properties: - CodeUri: .. + CodeUri: ../.. Handler: org.demo.batch.sqs.SqsBatchSender::handleRequest Environment: Variables: @@ -96,7 +96,7 @@ Resources: DemoSQSConsumerFunction: Type: AWS::Serverless::Function Properties: - CodeUri: .. + CodeUri: ../.. Handler: org.demo.batch.sqs.SqsBatchHandler::handleRequest Environment: Variables: From e53c1e91c332e4ba7301d69801ac7ebc5bd58531 Mon Sep 17 00:00:00 2001 From: Michele Ricciardi Date: Wed, 26 Jul 2023 13:28:57 +0200 Subject: [PATCH 31/79] tidy up --- .../main/java/org/demo/batch/dynamo/DynamoDBWriter.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/powertools-examples-batch/src/main/java/org/demo/batch/dynamo/DynamoDBWriter.java b/examples/powertools-examples-batch/src/main/java/org/demo/batch/dynamo/DynamoDBWriter.java index 183931474..4f2b9f9da 100644 --- a/examples/powertools-examples-batch/src/main/java/org/demo/batch/dynamo/DynamoDBWriter.java +++ b/examples/powertools-examples-batch/src/main/java/org/demo/batch/dynamo/DynamoDBWriter.java @@ -16,6 +16,7 @@ import java.security.SecureRandom; import java.util.UUID; +import java.util.stream.IntStream; public class DynamoDBWriter implements RequestHandler { @@ -45,14 +46,13 @@ public String handleRequest(ScheduledEvent scheduledEvent, Context context) { WriteBatch.Builder productBuilder = WriteBatch.builder(DdbProduct.class) .mappedTableResource(enhancedClient.table(tableName, TableSchema.fromBean(DdbProduct.class))); - - for (int i = 0; i < 5; i++) { + IntStream.range(0, 5).forEach(i -> { String id = UUID.randomUUID().toString(); float price = random.nextFloat(); DdbProduct product = new DdbProduct(id, "product-" + id, price); productBuilder.addPutItem(product); - } + }); BatchWriteItemEnhancedRequest batchWriteItemEnhancedRequest = BatchWriteItemEnhancedRequest.builder().writeBatches( productBuilder.build()) @@ -61,7 +61,7 @@ public String handleRequest(ScheduledEvent scheduledEvent, Context context) { BatchWriteResult batchWriteResult = enhancedClient.batchWriteItem(batchWriteItemEnhancedRequest); - LOGGER.info("Sent Message {}", batchWriteResult); + LOGGER.info("Wrote batch of messages to DynamoDB: {}", batchWriteResult); return "Success"; } From 4ca1726a7803bfb8b296b6cbd5cb72ca5dad1517 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Wed, 26 Jul 2023 14:56:25 +0200 Subject: [PATCH 32/79] Trying to get kinesis going --- examples/powertools-examples-batch/.gitignore | 38 --------- .../deploy/kinesis/template.yml | 83 +++++++++++++++++++ .../TooManyMessageHandlersException.java | 9 -- 3 files changed, 83 insertions(+), 47 deletions(-) delete mode 100644 examples/powertools-examples-batch/.gitignore create mode 100644 examples/powertools-examples-batch/deploy/kinesis/template.yml delete mode 100644 powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/exception/TooManyMessageHandlersException.java diff --git a/examples/powertools-examples-batch/.gitignore b/examples/powertools-examples-batch/.gitignore deleted file mode 100644 index 5ff6309b7..000000000 --- a/examples/powertools-examples-batch/.gitignore +++ /dev/null @@ -1,38 +0,0 @@ -target/ -!.mvn/wrapper/maven-wrapper.jar -!**/src/main/**/target/ -!**/src/test/**/target/ - -### IntelliJ IDEA ### -.idea/modules.xml -.idea/jarRepositories.xml -.idea/compiler.xml -.idea/libraries/ -*.iws -*.iml -*.ipr - -### Eclipse ### -.apt_generated -.classpath -.factorypath -.project -.settings -.springBeans -.sts4-cache - -### NetBeans ### -/nbproject/private/ -/nbbuild/ -/dist/ -/nbdist/ -/.nb-gradle/ -build/ -!**/src/main/**/build/ -!**/src/test/**/build/ - -### VS Code ### -.vscode/ - -### Mac OS ### -.DS_Store \ No newline at end of file diff --git a/examples/powertools-examples-batch/deploy/kinesis/template.yml b/examples/powertools-examples-batch/deploy/kinesis/template.yml new file mode 100644 index 000000000..beb1246db --- /dev/null +++ b/examples/powertools-examples-batch/deploy/kinesis/template.yml @@ -0,0 +1,83 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: > + Kinesis batch processing demo + +Globals: + Function: + Timeout: 20 + Runtime: java11 + MemorySize: 512 + Tracing: Active + Environment: + Variables: + POWERTOOLS_LOG_LEVEL: INFO + POWERTOOLS_LOGGER_SAMPLE_RATE: 1.0 + POWERTOOLS_LOGGER_LOG_EVENT: true + +Resources: + + DemoKinesisStream: + Type: AWS::Kinesis::Stream + Properties: + ShardCount: 1 + + StreamConsumer: + Type: "AWS::Kinesis::StreamConsumer" + Properties: + StreamARN: !GetAtt DemoKinesisStream.Arn + ConsumerName: KinesisBatchHandlerConsumer + + DemoKinesisSenderFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: ../.. + Handler: org.demo.batch.sqs.KinesisBatchSender::handleRequest + Environment: + Variables: + POWERTOOLS_SERVICE_NAME: sqs-batch-demo + STREAM_NAME: !Ref DemoKinesisStream + Policies: + - Statement: + - Sid: WriteToKinesis + Effect: Allow + Action: + - kinesis:PutRecords + - kinesis:DescribeStream + Resource: !GetAtt DemoKinesisStream.Arn + Events: + CWSchedule: + Type: Schedule + Properties: + Schedule: 'rate(5 minutes)' + Name: !Join ["-", ["message-producer-schedule", !Select [0, !Split [-, !Select [2, !Split [/, !Ref AWS::StackId ]]]]]] + Description: Produce message to SQS via a Lambda function + Enabled: true + + DemoKinesisConsumerFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: ../.. + Handler: org.demo.batch.kinesis.KinesisBatchHandler::handleRequest + Environment: + Variables: + POWERTOOLS_SERVICE_NAME: kinesis-demo + Events: + Kinesis: + Type: Kinesis + Properties: + Stream: !GetAtt StreamConsumer.ConsumerARN + StartingPosition: LATEST + BatchSize: 2 + +Outputs: + DemoSqsQueue: + Description: "ARN for Kinesis Stream" + Value: !GetAtt DemoKinesisStream.Arn + DemoSQSSenderFunction: + Description: "Kinesis Batch Sender - Lambda Function ARN" + Value: !GetAtt DemoKinesisSenderFunction.Arn + DemoSQSConsumerFunction: + Description: "SQS Batch Handler - Lambda Function ARN" + Value: !GetAtt DemoKinesisConsumerFunction.Arn + diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/exception/TooManyMessageHandlersException.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/exception/TooManyMessageHandlersException.java deleted file mode 100644 index fc1981b8b..000000000 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/exception/TooManyMessageHandlersException.java +++ /dev/null @@ -1,9 +0,0 @@ -package software.amazon.lambda.powertools.batch.exception; - -public class TooManyMessageHandlersException extends RuntimeException { - - public TooManyMessageHandlersException() { - super("You must configure either a rawMessageHandler or a messageHandler - not both"); - } - -} From fb422dad2dab2ac1a951377226ebc2569c660e35 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Wed, 26 Jul 2023 15:22:20 +0200 Subject: [PATCH 33/79] Kinesis demo working --- .../deploy/kinesis/template.yml | 10 +++++----- .../org/demo/batch/kinesis/KinesisBatchSender.java | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/examples/powertools-examples-batch/deploy/kinesis/template.yml b/examples/powertools-examples-batch/deploy/kinesis/template.yml index beb1246db..dbe017558 100644 --- a/examples/powertools-examples-batch/deploy/kinesis/template.yml +++ b/examples/powertools-examples-batch/deploy/kinesis/template.yml @@ -32,10 +32,10 @@ Resources: Type: AWS::Serverless::Function Properties: CodeUri: ../.. - Handler: org.demo.batch.sqs.KinesisBatchSender::handleRequest + Handler: org.demo.batch.kinesis.KinesisBatchSender::handleRequest Environment: Variables: - POWERTOOLS_SERVICE_NAME: sqs-batch-demo + POWERTOOLS_SERVICE_NAME: kinesis-batch-demo STREAM_NAME: !Ref DemoKinesisStream Policies: - Statement: @@ -51,7 +51,7 @@ Resources: Properties: Schedule: 'rate(5 minutes)' Name: !Join ["-", ["message-producer-schedule", !Select [0, !Split [-, !Select [2, !Split [/, !Ref AWS::StackId ]]]]]] - Description: Produce message to SQS via a Lambda function + Description: Produce message to Kinesis via a Lambda function Enabled: true DemoKinesisConsumerFunction: @@ -71,10 +71,10 @@ Resources: BatchSize: 2 Outputs: - DemoSqsQueue: + DemoKinesisQueue: Description: "ARN for Kinesis Stream" Value: !GetAtt DemoKinesisStream.Arn - DemoSQSSenderFunction: + DemoKinesisSenderFunction: Description: "Kinesis Batch Sender - Lambda Function ARN" Value: !GetAtt DemoKinesisSenderFunction.Arn DemoSQSConsumerFunction: diff --git a/examples/powertools-examples-batch/src/main/java/org/demo/batch/kinesis/KinesisBatchSender.java b/examples/powertools-examples-batch/src/main/java/org/demo/batch/kinesis/KinesisBatchSender.java index 132d0f5c2..ecfe941a9 100644 --- a/examples/powertools-examples-batch/src/main/java/org/demo/batch/kinesis/KinesisBatchSender.java +++ b/examples/powertools-examples-batch/src/main/java/org/demo/batch/kinesis/KinesisBatchSender.java @@ -57,6 +57,7 @@ public String handleRequest(ScheduledEvent scheduledEvent, Context context) { try { SdkBytes data = SdkBytes.fromUtf8String(objectMapper.writeValueAsString(product)); return PutRecordsRequestEntry.builder() + .partitionKey(String.format("%d", id)) .data(data) .build(); } catch (JsonProcessingException e) { From 4118768ab5e6a922216bda5a25ce4b4801dc03f4 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Wed, 26 Jul 2023 15:35:03 +0200 Subject: [PATCH 34/79] Updated readme --- examples/README.md | 3 +- examples/powertools-examples-batch/README.md | 33 ++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 examples/powertools-examples-batch/README.md diff --git a/examples/README.md b/examples/README.md index 1869b4e8f..fb99811dc 100644 --- a/examples/README.md +++ b/examples/README.md @@ -9,9 +9,10 @@ Each example can be copied from its subdirectory and used independently of the r * [powertools-examples-idempotency](powertools-examples-idempotency) - An idempotent HTTP API * [powertools-examples-parameters](powertools-examples-parameters) - Uses the parameters module to provide runtime parameters to a function * [powertools-examples-serialization](powertools-examples-serialization) - Uses the serialization module to serialize and deserialize API Gateway & SQS payloads -* [powertools-examples-sqs](powertools-examples-sqs) - Processes SQS batch requests +* [powertools-examples-sqs](powertools-examples-sqs) - Processes SQS batch requests (**Deprecated** - will be replaced by `powertools-examples-batch` in version 2 of this library) * [powertools-examples-validation](powertools-examples-validation) - Uses the validation module to validate user requests received via API Gateway * [powertools-examples-cloudformation](powertools-examples-cloudformation) - Deploys a Cloudformation custom resource +* [powertools-examples-batch](powertools-examples-batch) - Examples for each of the different batch processing deployments ## Working with AWS Serverless Application Model (SAM) Examples Many of the examples use [AWS Serverless Application Model](https://aws.amazon.com/serverless/sam/) (SAM). To get started diff --git a/examples/powertools-examples-batch/README.md b/examples/powertools-examples-batch/README.md new file mode 100644 index 000000000..ef832e035 --- /dev/null +++ b/examples/powertools-examples-batch/README.md @@ -0,0 +1,33 @@ + # Powertools for AWS Lambda (Java) - Batch Example + +This project contains examples of Lambda function using the batch processing module of Powertools for AWS Lambda (Java). For more information on this module, please refer to the +[documentation](https://docs.powertools.aws.dev/lambda-java/utilities/batch/). + +Three different examples and SAM deployments are included, covering each of the batch sources: + +* [SQS](src/main/java/org/demo/batch/sqs) - SQS batch processing +* [Kinesis Streams](src/main/java/org/demo/batch/kinesis) - Kinesis Streams batch processing +* [DynamoDB Streams](src/main/java/org/demo/batch/dynamo) - DynamoDB Streams batch processing + +## Deploy the sample application + +This sample is based on Serverless Application Model (SAM). To deploy it, check out the instructions for getting +started with SAM in [the examples directory](../README.md) + +This sample contains three different deployments, depending on which batch processor you'd like to use, you can +change to the subdirectory containing the example SAM template, and deploy. For instance, for the SQS batch +deployment: +```bash +cd deploy/sqs +sam build +sam deploy --guided +``` + +## Test the application + +Each of the examples uses a Lambda scheduled every 5 minutes to push a batch, and a separate lambda to read it. To +see this in action, we can simply tail the logs of our stack: + +```bash +sam logs --tail $STACK_NAME +``` \ No newline at end of file From e75de66263a4cca34265336b82d347c2e76dbc65 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Wed, 26 Jul 2023 16:42:08 +0200 Subject: [PATCH 35/79] Deprecated everywhere --- .../lambda/powertools/sqs/SqsBatch.java | 5 ++ .../lambda/powertools/sqs/SqsUtils.java | 47 +++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/powertools-sqs/src/main/java/software/amazon/lambda/powertools/sqs/SqsBatch.java b/powertools-sqs/src/main/java/software/amazon/lambda/powertools/sqs/SqsBatch.java index cd529ff22..65586ef08 100644 --- a/powertools-sqs/src/main/java/software/amazon/lambda/powertools/sqs/SqsBatch.java +++ b/powertools-sqs/src/main/java/software/amazon/lambda/powertools/sqs/SqsBatch.java @@ -10,6 +10,10 @@ import static com.amazonaws.services.lambda.runtime.events.SQSEvent.*; /** + * @deprecated + * @see software.amazon.lambda.powertools.batch in powertools-batch module. + * Will be removed in V2. + * * {@link SqsBatch} is used to process batch messages in {@link SQSEvent} * *

@@ -73,6 +77,7 @@ */ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.METHOD) +@Deprecated public @interface SqsBatch { Class> value(); diff --git a/powertools-sqs/src/main/java/software/amazon/lambda/powertools/sqs/SqsUtils.java b/powertools-sqs/src/main/java/software/amazon/lambda/powertools/sqs/SqsUtils.java index 9fff4dc6f..e1bd9f1d9 100644 --- a/powertools-sqs/src/main/java/software/amazon/lambda/powertools/sqs/SqsUtils.java +++ b/powertools-sqs/src/main/java/software/amazon/lambda/powertools/sqs/SqsUtils.java @@ -115,6 +115,11 @@ public static void overrideS3Client(S3Client s3Client) { } /** + * + * @deprecated + * @see software.amazon.lambda.powertools.batch in powertools-batch module. + * Will be removed in V2. + * * This utility method is used to process each {@link SQSMessage} inside the received {@link SQSEvent} * *

@@ -142,12 +147,17 @@ public static void overrideS3Client(S3Client s3Client) { * @return List of values returned by {@link SqsMessageHandler#process(SQSMessage)} while processing each message. * @throws SQSBatchProcessingException if some messages fail during processing. */ + @Deprecated public static List batchProcessor(final SQSEvent event, final Class> handler) { return batchProcessor(event, false, handler); } /** + * @deprecated + * @see software.amazon.lambda.powertools.batch in powertools-batch module. + * Will be removed in V2. + * * This utility method is used to process each {@link SQSMessage} inside the received {@link SQSEvent} * *

@@ -195,6 +205,7 @@ public static List batchProcessor(final SQSEvent event, * @throws SQSBatchProcessingException if some messages fail during processing. */ @SafeVarargs + @Deprecated public static List batchProcessor(final SQSEvent event, final Class> handler, final Class... nonRetryableExceptions) { @@ -202,6 +213,10 @@ public static List batchProcessor(final SQSEvent event, } /** + * @deprecated + * @see software.amazon.lambda.powertools.batch in powertools-batch module. + * Will be removed in V2. + * * This utility method is used to process each {@link SQSMessage} inside the received {@link SQSEvent} * *

@@ -227,6 +242,7 @@ public static List batchProcessor(final SQSEvent event, * @return List of values returned by {@link SqsMessageHandler#process(SQSMessage)} while processing each message. * @throws SQSBatchProcessingException if some messages fail during processing and no suppression enabled. */ + @Deprecated public static List batchProcessor(final SQSEvent event, final boolean suppressException, final Class> handler) { @@ -236,6 +252,10 @@ public static List batchProcessor(final SQSEvent event, } /** + * @deprecated + * @see software.amazon.lambda.powertools.batch in powertools-batch module. + * Will be removed in V2. + * * This utility method is used to process each {@link SQSMessage} inside the received {@link SQSEvent} * *

@@ -286,6 +306,7 @@ public static List batchProcessor(final SQSEvent event, * @throws SQSBatchProcessingException if some messages fail during processing. */ @SafeVarargs + @Deprecated public static List batchProcessor(final SQSEvent event, final boolean suppressException, final Class> handler, @@ -296,6 +317,10 @@ public static List batchProcessor(final SQSEvent event, } /** + * @deprecated + * @see software.amazon.lambda.powertools.batch in powertools-batch module. + * Will be removed in V2. + * * This utility method is used to process each {@link SQSMessage} inside the received {@link SQSEvent} * *

@@ -349,6 +374,7 @@ public static List batchProcessor(final SQSEvent event, * @throws SQSBatchProcessingException if some messages fail during processing. */ @SafeVarargs + @Deprecated public static List batchProcessor(final SQSEvent event, final boolean suppressException, final Class> handler, @@ -360,6 +386,10 @@ public static List batchProcessor(final SQSEvent event, } /** + * @deprecated + * @see software.amazon.lambda.powertools.batch in powertools-batch module. + * Will be removed in V2. + * * This utility method is used to process each {@link SQSMessage} inside the received {@link SQSEvent} * *

@@ -387,6 +417,7 @@ public static List batchProcessor(final SQSEvent event, * @return List of values returned by {@link SqsMessageHandler#process(SQSMessage)} while processing each message- * @throws SQSBatchProcessingException if some messages fail during processing. */ + @Deprecated public static List batchProcessor(final SQSEvent event, final SqsMessageHandler handler) { return batchProcessor(event, false, handler); @@ -394,6 +425,10 @@ public static List batchProcessor(final SQSEvent event, /** + * @deprecated + * @see software.amazon.lambda.powertools.batch in powertools-batch module. + * Will be removed in V2. + * * This utility method is used to process each {@link SQSMessage} inside the received {@link SQSEvent} * *

@@ -442,6 +477,7 @@ public static List batchProcessor(final SQSEvent event, * @throws SQSBatchProcessingException if some messages fail during processing. */ @SafeVarargs + @Deprecated public static List batchProcessor(final SQSEvent event, final SqsMessageHandler handler, final Class... nonRetryableExceptions) { @@ -450,6 +486,10 @@ public static List batchProcessor(final SQSEvent event, /** + * @deprecated + * @see software.amazon.lambda.powertools.batch in powertools-batch module. + * Will be removed in V2. + * * This utility method is used to process each {@link SQSMessage} inside the received {@link SQSEvent} * *

@@ -476,6 +516,7 @@ public static List batchProcessor(final SQSEvent event, * @return List of values returned by {@link SqsMessageHandler#process(SQSMessage)} while processing each message. * @throws SQSBatchProcessingException if some messages fail during processing and no suppression enabled. */ + @Deprecated public static List batchProcessor(final SQSEvent event, final boolean suppressException, final SqsMessageHandler handler) { @@ -483,7 +524,13 @@ public static List batchProcessor(final SQSEvent event, } + /** + * @deprecated + * @see software.amazon.lambda.powertools.batch in powertools-batch module. + * Will be removed in V2. + */ @SafeVarargs + @Deprecated public static List batchProcessor(final SQSEvent event, final boolean suppressException, final SqsMessageHandler handler, From 1e7d305b2ea152c30ec9c8a6ec57ff9b871e38a8 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Thu, 27 Jul 2023 08:54:21 +0200 Subject: [PATCH 36/79] Address initial review comments --- examples/pom.xml | 1 + examples/powertools-examples-batch/pom.xml | 10 ++++ .../dynamo/DynamoDBStreamBatchHandler.java | 7 +-- .../batch/kinesis/KinesisBatchSender.java | 6 +++ .../org/demo/batch/sqs/SqsBatchSender.java | 5 ++ pom.xml | 3 +- powertools-batch/.gitignore | 38 --------------- powertools-batch/pom.xml | 9 ---- .../batch/BatchMessageHandlerBuilder.java | 14 ++++-- .../AbstractBatchMessageHandlerBuilder.java | 33 +++++++++---- ...> DynamoDbBatchMessageHandlerBuilder.java} | 13 ++++-- .../KinesisBatchMessageHandlerBuilder.java | 3 ++ .../DeserializationNotSupportedException.java | 5 ++ ....java => DynamoDbBatchMessageHandler.java} | 12 +++-- .../KinesisStreamsBatchMessageHandler.java | 8 ++-- .../batch/handler/SqsBatchMessageHandler.java | 13 ++++-- .../batch/AllBatchProcessorTests.java | 46 ------------------- 17 files changed, 96 insertions(+), 130 deletions(-) delete mode 100644 powertools-batch/.gitignore rename powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/{DdbBatchMessageHandlerBuilder.java => DynamoDbBatchMessageHandlerBuilder.java} (72%) rename powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/{DdbBatchMessageHandler.java => DynamoDbBatchMessageHandler.java} (76%) delete mode 100644 powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/AllBatchProcessorTests.java diff --git a/examples/pom.xml b/examples/pom.xml index c9b8ea8ae..50caf9296 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -20,6 +20,7 @@ powertools-examples-parameters powertools-examples-serialization powertools-examples-sqs + powertools-examples-batch powertools-examples-validation powertools-examples-cloudformation diff --git a/examples/powertools-examples-batch/pom.xml b/examples/powertools-examples-batch/pom.xml index 82e15b9a2..efb2b565d 100644 --- a/examples/powertools-examples-batch/pom.xml +++ b/examples/powertools-examples-batch/pom.xml @@ -44,6 +44,16 @@ sdk-core ${sdk.version} + + software.amazon.awssdk + sqs + ${sdk.version} + + + software.amazon.awssdk + url-connection-client + ${sdk.version} + software.amazon.awssdk dynamodb-enhanced diff --git a/examples/powertools-examples-batch/src/main/java/org/demo/batch/dynamo/DynamoDBStreamBatchHandler.java b/examples/powertools-examples-batch/src/main/java/org/demo/batch/dynamo/DynamoDBStreamBatchHandler.java index 087501011..c8cb81bc5 100644 --- a/examples/powertools-examples-batch/src/main/java/org/demo/batch/dynamo/DynamoDBStreamBatchHandler.java +++ b/examples/powertools-examples-batch/src/main/java/org/demo/batch/dynamo/DynamoDBStreamBatchHandler.java @@ -26,12 +26,9 @@ private void processMessage(DynamodbEvent.DynamodbStreamRecord dynamodbStreamRec } @Override - public StreamsEventResponse handleRequest(DynamodbEvent kinesisEvent, Context context) { - return handler.processBatch(kinesisEvent, context); + public StreamsEventResponse handleRequest(DynamodbEvent ddbEvent, Context context) { + return handler.processBatch(ddbEvent, context); } - private void processMessage(Product p, Context c) { - LOGGER.info("Processing product " + p); - } } diff --git a/examples/powertools-examples-batch/src/main/java/org/demo/batch/kinesis/KinesisBatchSender.java b/examples/powertools-examples-batch/src/main/java/org/demo/batch/kinesis/KinesisBatchSender.java index ecfe941a9..84b79436e 100644 --- a/examples/powertools-examples-batch/src/main/java/org/demo/batch/kinesis/KinesisBatchSender.java +++ b/examples/powertools-examples-batch/src/main/java/org/demo/batch/kinesis/KinesisBatchSender.java @@ -26,6 +26,12 @@ import static java.util.stream.Collectors.toList; + +/** + * A Lambda handler used to send message batches to Kinesis Streams. This is only here + * to produce an end-to-end demo, so that the {{@link org.demo.batch.kinesis.KinesisBatchHandler}} + * has some data to consume. + */ public class KinesisBatchSender implements RequestHandler { private static final Logger LOGGER = LogManager.getLogger(KinesisBatchSender.class); diff --git a/examples/powertools-examples-batch/src/main/java/org/demo/batch/sqs/SqsBatchSender.java b/examples/powertools-examples-batch/src/main/java/org/demo/batch/sqs/SqsBatchSender.java index a817fc6a9..1938414ae 100644 --- a/examples/powertools-examples-batch/src/main/java/org/demo/batch/sqs/SqsBatchSender.java +++ b/examples/powertools-examples-batch/src/main/java/org/demo/batch/sqs/SqsBatchSender.java @@ -21,6 +21,11 @@ import static java.util.stream.Collectors.toList; +/** + * A Lambda handler used to send message batches to SQS. This is only here + * to produce an end-to-end demo, so that the {{@link org.demo.batch.sqs.SqsBatchHandler}} + * has some data to consume. + */ public class SqsBatchSender implements RequestHandler { private static final Logger LOGGER = LogManager.getLogger(SqsBatchSender.class); diff --git a/pom.xml b/pom.xml index cd45a09f8..424b22161 100644 --- a/pom.xml +++ b/pom.xml @@ -39,9 +39,8 @@ powertools-cloudformation powertools-idempotency powertools-e2e-tests - examples powertools-batch - examples/powertools-examples-batch + examples diff --git a/powertools-batch/.gitignore b/powertools-batch/.gitignore deleted file mode 100644 index 5ff6309b7..000000000 --- a/powertools-batch/.gitignore +++ /dev/null @@ -1,38 +0,0 @@ -target/ -!.mvn/wrapper/maven-wrapper.jar -!**/src/main/**/target/ -!**/src/test/**/target/ - -### IntelliJ IDEA ### -.idea/modules.xml -.idea/jarRepositories.xml -.idea/compiler.xml -.idea/libraries/ -*.iws -*.iml -*.ipr - -### Eclipse ### -.apt_generated -.classpath -.factorypath -.project -.settings -.springBeans -.sts4-cache - -### NetBeans ### -/nbproject/private/ -/nbbuild/ -/dist/ -/nbdist/ -/.nb-gradle/ -build/ -!**/src/main/**/build/ -!**/src/test/**/build/ - -### VS Code ### -.vscode/ - -### Mac OS ### -.DS_Store \ No newline at end of file diff --git a/powertools-batch/pom.xml b/powertools-batch/pom.xml index aa43def63..4008cc36e 100644 --- a/powertools-batch/pom.xml +++ b/powertools-batch/pom.xml @@ -24,11 +24,6 @@ powertools-serialization ${project.version} - - software.amazon.lambda - powertools-idempotency - ${project.version} - org.aspectj aspectjrt @@ -61,10 +56,6 @@ mockito-inline test - - software.amazon.lambda - powertools-sqs - \ No newline at end of file diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java index 134cb206d..97c7c235b 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java @@ -3,9 +3,13 @@ import software.amazon.lambda.powertools.batch.builder.*; /** - * A builder-style interface we can use within an existing Lambda RequestHandler to - * deal with our batch responses. A second tier of builders is returned per-event-source - * to bind the appropriate message types and provider source-specific logic and tuneables. + * A builder-style interface we can use to build batch processing handlers for SQS, Kinesis Streams, + * and DynamoDB Streams batches. The batch processing handlers that are returned allow + * the user to easily process batches of messages, one-by-one, while offloading + * the common issues - failure handling, partial responses, deserialization - + * to the library. + * + * @see Powertools for AWS Lambda (Java) Batch Documentation **/ public class BatchMessageHandlerBuilder { @@ -23,8 +27,8 @@ public SqsBatchMessageHandlerBuilder withSqsBatchHandler() { * * @return A fluent builder interface to continue the building */ - public DdbBatchMessageHandlerBuilder withDynamoDbBatchHandler() { - return new DdbBatchMessageHandlerBuilder(); + public DynamoDbBatchMessageHandlerBuilder withDynamoDbBatchHandler() { + return new DynamoDbBatchMessageHandlerBuilder(); } /** diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/AbstractBatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/AbstractBatchMessageHandlerBuilder.java index d73e6b842..c6019a626 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/AbstractBatchMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/AbstractBatchMessageHandlerBuilder.java @@ -25,9 +25,10 @@ abstract class AbstractBatchMessageHandlerBuilder { protected Consumer successHandler; /** - * Provides a success handler. A success handler is invoked once for - * each message after it has been processed by the user-provided + * Provides an (Optional!) success handler. A success handler is invoked + * once for each message after it has been processed by the user-provided * handler. + * * If the success handler throws, the item in the batch will be * marked failed. * @@ -39,9 +40,16 @@ public C withSuccessHandler(Consumer handler) { } /** - * Provides a failure handler. A failure handler is invoked once - * for each message after it has failed to be processed by the - * user-provided handler. + * Provides an (Optional!) failure handler. A failure handler is invoked + * once for each message after it has failed to be processed by the + * user-provided handler. This gives the user's code a useful hook to do + * anything else that might have to be done in response to a failure - for + * instance, updating a metric, or writing a detailed log. + * + * Please note that this method has nothing to do with the partial batch + * failure mechanism. Regardless of whether a failure handler is + * specified, partial batch failures and responses to the Lambda environment + * are handled by the batch utility separately. * * @param handler The handler to invoke on failure */ @@ -53,11 +61,15 @@ public C withFailureHandler(BiConsumer handler) { /** * Builds a BatchMessageHandler that can be used to process batches, given * a user-defined handler to process each item in the batch. This variant - * takes a function that consumes a raw message and the Lambda context. + * takes a function that consumes a raw message and the Lambda context. This + * is useful for handlers that need access to the entire message object, not + * just the deserialized contents of the body. + * * Note: If you don't need the Lambda context, use the variant of this function * that does not require it. * - * @param handler Takes a raw message - the AWS-SDK-shaped inner type of the batch - to process + * @param handler Takes a raw message - the underlying AWS Events Library event - to process. + * For instance for SQS this would be an SQSMessage. * @return A BatchMessageHandler for processing the batch */ public abstract BatchMessageHandler buildWithRawMessageHandler(BiConsumer handler); @@ -65,9 +77,12 @@ public C withFailureHandler(BiConsumer handler) { /** * Builds a BatchMessageHandler that can be used to process batches, given * a user-defined handler to process each item in the batch. This variant - * takes a function that consumes a raw message. + * takes a function that consumes a raw message and the Lambda context. This + * is useful for handlers that need access to the entire message object, not + * just the deserialized contents of the body. * - * @param handler Takes a raw message - the AWS-SDK-shaped inner type of the batch - to process + * @param handler Takes a raw message - the underlying AWS Events Library event - to process. + * For instance for SQS this would be an SQSMessage. * @return A BatchMessageHandler for processing the batch */ public BatchMessageHandler buildWithRawMessageHandler(Consumer handler) { diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/DdbBatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/DynamoDbBatchMessageHandlerBuilder.java similarity index 72% rename from powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/DdbBatchMessageHandlerBuilder.java rename to powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/DynamoDbBatchMessageHandlerBuilder.java index 25d382d28..b1d90fd99 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/DdbBatchMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/DynamoDbBatchMessageHandlerBuilder.java @@ -5,19 +5,22 @@ import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; import software.amazon.lambda.powertools.batch.exception.DeserializationNotSupportedException; import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; -import software.amazon.lambda.powertools.batch.handler.DdbBatchMessageHandler; +import software.amazon.lambda.powertools.batch.handler.DynamoDbBatchMessageHandler; import java.util.function.BiConsumer; -public class DdbBatchMessageHandlerBuilder extends AbstractBatchMessageHandlerBuilder { @Override public BatchMessageHandler buildWithRawMessageHandler(BiConsumer rawMessageHandler) { - return new DdbBatchMessageHandler( + return new DynamoDbBatchMessageHandler( this.successHandler, this.failureHandler, rawMessageHandler); @@ -30,7 +33,7 @@ public BatchMessageHandler buildWithMes } @Override - protected DdbBatchMessageHandlerBuilder getThis() { + protected DynamoDbBatchMessageHandlerBuilder getThis() { return this; } } diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/KinesisBatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/KinesisBatchMessageHandlerBuilder.java index 56d7898ad..96e9705dc 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/KinesisBatchMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/KinesisBatchMessageHandlerBuilder.java @@ -8,6 +8,9 @@ import java.util.function.BiConsumer; +/** + * Builds a batch processor for processing Kinesis Streams batch events + */ public class KinesisBatchMessageHandlerBuilder extends AbstractBatchMessageHandlerBuilder{ - Logger LOGGER = LoggerFactory.getLogger(DdbBatchMessageHandler.class); +/** + * A batch message processor for DynamoDB Streams batches. + * + * @see DynamoDB Streams batch failure reporting + * + */ +public class DynamoDbBatchMessageHandler implements BatchMessageHandler{ + private final static Logger LOGGER = LoggerFactory.getLogger(DynamoDbBatchMessageHandler.class); private final Consumer successHandler; private final BiConsumer failureHandler; private final BiConsumer rawMessageHandler; - public DdbBatchMessageHandler(Consumer successHandler, BiConsumer failureHandler, BiConsumer rawMessageHandler) { + public DynamoDbBatchMessageHandler(Consumer successHandler, BiConsumer failureHandler, BiConsumer rawMessageHandler) { this.successHandler = successHandler; this.failureHandler = failureHandler; this.rawMessageHandler = rawMessageHandler; diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/KinesisStreamsBatchMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/KinesisStreamsBatchMessageHandler.java index 9f9675d0d..17fda7860 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/KinesisStreamsBatchMessageHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/KinesisStreamsBatchMessageHandler.java @@ -14,13 +14,13 @@ import java.util.function.Consumer; /** - * A batch message handler for Kinesis Streams batch processing. + * A batch message processor for Kinesis Streams batch processing. * - * Refer to The kinesis batch processing document - * @param + * Refer to Kinesis Batch failure reporting + * @param The user-defined type of the Kinesis record payload */ public class KinesisStreamsBatchMessageHandler implements BatchMessageHandler { - Logger LOGGER = LoggerFactory.getLogger(KinesisStreamsBatchMessageHandler.class); + private final static Logger LOGGER = LoggerFactory.getLogger(KinesisStreamsBatchMessageHandler.class); private final BiConsumer rawMessageHandler; private final BiConsumer messageHandler; diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java index e44a62b9b..e6f489b37 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java @@ -11,14 +11,20 @@ import java.util.function.BiConsumer; import java.util.function.Consumer; +/** + * A batch message processor for SQS batches. + * + * @see SQS Batch failure reporting + * @param The user-defined type of the message payload + */ public class SqsBatchMessageHandler implements BatchMessageHandler { - private final Class messageClass; - Logger LOGGER = LoggerFactory.getLogger(SqsBatchMessageHandler.class); + private final static Logger LOGGER = LoggerFactory.getLogger(SqsBatchMessageHandler.class); // The attribute on an SQS-FIFO message used to record the message group ID // https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#sample-fifo-queues-message-event - String MESSAGE_GROUP_ID_KEY = "MessageGroupId"; + private final static String MESSAGE_GROUP_ID_KEY = "MessageGroupId"; + private final Class messageClass; private final BiConsumer messageHandler; private final BiConsumer rawMessageHandler; private final Consumer successHandler; @@ -30,7 +36,6 @@ public SqsBatchMessageHandler(BiConsumer messageHandler, Class me this.rawMessageHandler = rawMessageHandler; this.successHandler = successHandler; this.failureHandler = failureHandler; - } @Override diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/AllBatchProcessorTests.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/AllBatchProcessorTests.java deleted file mode 100644 index bad4e9d0c..000000000 --- a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/AllBatchProcessorTests.java +++ /dev/null @@ -1,46 +0,0 @@ -package software.amazon.lambda.powertools.batch; - -import com.amazonaws.services.lambda.runtime.Context; -import com.amazonaws.services.lambda.runtime.events.KinesisEvent; -import com.amazonaws.services.lambda.runtime.events.SQSEvent; -import com.amazonaws.services.lambda.runtime.tests.annotations.Event; -import com.amazonaws.services.lambda.runtime.tests.annotations.Events; -import org.junit.jupiter.params.ParameterizedTest; -import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; - -import java.util.concurrent.atomic.AtomicBoolean; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * Test functionality that should be common to all the batch processors - * - * TODO probably remove. Trying to think of a way of factoring - * TODO common tests for all the message handlers into one place - */ -public class AllBatchProcessorTests { -// -// AtomicBoolean failureHandlerWasCalled = new AtomicBoolean(false); -// public void failingFailureHandler(T evt, Throwable t) { -// failureHandlerWasCalled.set(true); -// throw new RuntimeException("Well, this doesn't look great"); -// } -// -// @ParameterizedTest -// @Events( -// events = { -// @Event(value = "sqs_event.json", type = SQSEvent.class), -// @Event(value = "kinesis_event.json", type = KinesisEvent.class) -// } -// ) -// public void failingFailureHandlerShouldntFailBatch(Object event) { -// BatchMessageHandler handler; -// if (event instanceof SQSEvent) { -// handler = new BatchMessageHandlerBuilder() -// .withSqsBatchHandler() -// .withFailureHandler(this::failingFailureHandler) -// .build -// } -// } - -} From 6ad7e7d2c5f51bb2250a56f0ba9bc7b990937ac3 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Thu, 27 Jul 2023 09:02:02 +0200 Subject: [PATCH 37/79] Add success tests for Kinesis/S3 --- .../batch/KinesisBatchProcessorTest.java | 15 +++++++++++++++ .../powertools/batch/SQSBatchProcessorTest.java | 16 ++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/KinesisBatchProcessorTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/KinesisBatchProcessorTest.java index 811ddbe0b..405091db8 100644 --- a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/KinesisBatchProcessorTest.java +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/KinesisBatchProcessorTest.java @@ -35,6 +35,21 @@ public void processMessageFailsForFixedProduct(Product product, Context context) } } + @ParameterizedTest + @Event(value = "kinesis_event.json", type = KinesisEvent.class) + public void batchProcessingSucceedsAndReturns(KinesisEvent event) { + // Arrange + BatchMessageHandler handler = new BatchMessageHandlerBuilder() + .withKinesisBatchHandler() + .buildWithRawMessageHandler(this::processMessageSucceeds); + + // Act + StreamsEventResponse kinesisBatchResponse = handler.processBatch(event, context); + + // Assert + assertThat(kinesisBatchResponse.getBatchItemFailures()).hasSize(0); + } + @ParameterizedTest @Event(value = "kinesis_event.json", type = KinesisEvent.class) public void shouldAddMessageToBatchFailure_whenException_withMessage(KinesisEvent event) { diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java index 5206f99e9..97e3596c3 100644 --- a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java @@ -35,6 +35,22 @@ public void processMessageFailsForFixedProduct(Product product, Context context) } } + @ParameterizedTest + @Event(value = "sqs_event.json", type = SQSEvent.class) + public void batchProcessingSucceedsAndReturns(SQSEvent event) { + // Arrange + BatchMessageHandler handler = new BatchMessageHandlerBuilder() + .withSqsBatchHandler() + .buildWithRawMessageHandler(this::processMessageSucceeds); + + // Act + SQSBatchResponse sqsBatchResponse = handler.processBatch(event, context); + + // Assert + assertThat(sqsBatchResponse.getBatchItemFailures()).hasSize(0); + } + + @ParameterizedTest @Event(value = "sqs_event.json", type = SQSEvent.class) public void shouldAddMessageToBatchFailure_whenException_withMessage(SQSEvent event) { From 8f69551b8a01e28aaffebb442f9c68a95cd76587 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Thu, 27 Jul 2023 09:07:35 +0200 Subject: [PATCH 38/79] Increase DDB coverage --- .../batch/DdbBatchProcessorTest.java | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/DdbBatchProcessorTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/DdbBatchProcessorTest.java index c3b8a1cbb..8800eb25f 100644 --- a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/DdbBatchProcessorTest.java +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/DdbBatchProcessorTest.java @@ -27,6 +27,21 @@ private void processMessageFailsForFixedMessage(DynamodbEvent.DynamodbStreamReco } } + @ParameterizedTest + @Event(value = "dynamo_event.json", type = DynamodbEvent.class) + public void batchProcessingSucceedsAndReturns(DynamodbEvent event) { + // Arrange + BatchMessageHandler handler = new BatchMessageHandlerBuilder() + .withDynamoDbBatchHandler() + .buildWithRawMessageHandler(this::processMessageSucceeds); + + // Act + StreamsEventResponse dynamodbBatchResponse = handler.processBatch(event, context); + + // Assert + assertThat(dynamodbBatchResponse.getBatchItemFailures()).hasSize(0); + } + @ParameterizedTest @Event(value = "dynamo_event.json", type = DynamodbEvent.class) public void shouldAddMessageToBatchFailure_whenException_withMessage(DynamodbEvent event) { @@ -44,6 +59,32 @@ public void shouldAddMessageToBatchFailure_whenException_withMessage(DynamodbEve assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("4421584500000000017450439091"); } + @ParameterizedTest + @Event(value = "dynamo_event.json", type = DynamodbEvent.class) + public void failingFailureHandlerShouldntFailBatch(DynamodbEvent event) { + // Arrange + AtomicBoolean wasCalledAndFailed = new AtomicBoolean(false); + BatchMessageHandler handler = new BatchMessageHandlerBuilder() + .withDynamoDbBatchHandler() + .withFailureHandler((m, e) -> { + if (m.getDynamodb().getSequenceNumber().equals("4421584500000000017450439091")) { + wasCalledAndFailed.set(true); + throw new RuntimeException("Success handler throws"); + } + }) + .buildWithRawMessageHandler(this::processMessageFailsForFixedMessage); + + // Act + StreamsEventResponse dynamodbBatchResponse = handler.processBatch(event, context); + + // Assert + assertThat(dynamodbBatchResponse).isNotNull(); + assertThat(dynamodbBatchResponse.getBatchItemFailures().size()).isEqualTo(1); + assertThat(wasCalledAndFailed.get()).isTrue(); + StreamsEventResponse.BatchItemFailure batchItemFailure = dynamodbBatchResponse.getBatchItemFailures().get(0); + assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("4421584500000000017450439091"); + } + @ParameterizedTest @Event(value = "dynamo_event.json", type = DynamodbEvent.class) public void failingSuccessHandlerShouldntFailBatchButShouldFailMessage(DynamodbEvent event) { From 672ba5100f78241e02b3482f6fb8ffa5757bae78 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Thu, 27 Jul 2023 09:11:00 +0200 Subject: [PATCH 39/79] Tell sonar to ignore dupes in examples --- sonar-project.properties | 1 + 1 file changed, 1 insertion(+) create mode 100644 sonar-project.properties diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 000000000..94e934c33 --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1 @@ +sonar.cpd.exclusions=examples/**/* \ No newline at end of file From 70a08ba6ffa1fcb68a1bfab3eb9a578a0665203c Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Thu, 27 Jul 2023 10:04:13 +0200 Subject: [PATCH 40/79] Add docs --- docs/utilities/batch.md | 572 ++++++++++++------------------------ docs/utilities/sqs_batch.md | 460 +++++++++++++++++++++++++++++ mkdocs.yml | 1 + sonar-project.properties | 1 + 4 files changed, 654 insertions(+), 380 deletions(-) create mode 100644 docs/utilities/sqs_batch.md diff --git a/docs/utilities/batch.md b/docs/utilities/batch.md index 95704b8a0..ad309e8c7 100644 --- a/docs/utilities/batch.md +++ b/docs/utilities/batch.md @@ -1,460 +1,272 @@ --- -title: SQS Batch Processing +title: Batch Processing description: Utility --- -The SQS batch processing utility provides a way to handle partial failures when processing batches of messages from SQS. -The utility handles batch processing for both -[standard](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/standard-queues.html) and -[FIFO](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/FIFO-queues.html) SQS queues. +The batch processing utility provides a way to handle partial failures when processing batches of messages from SQS queues, +SQS FIFO queues, Kinesis Streams, or DynamoDB Streams. **Key Features** -* Prevent successfully processed messages from being returned to SQS -* A simple interface for individually processing messages from a batch + +* Reports batch item failures to reduce number of retries for a record upon errors +* Simple interface to process each batch record +* Integrates with Java Events library and the deserialization module +* Build your own batch processor by extending primitives **Background** -When using SQS as a Lambda event source mapping, Lambda functions can be triggered with a batch of messages from SQS. -If your function fails to process any message from the batch, the entire batch returns to your SQS queue, and your -Lambda function will be triggered with the same batch again. With this utility, messages within a batch will be handled individually - only messages that were not successfully processed -are returned to the queue. +When using SQS, Kinesis Data Streams, or DynamoDB Streams as a Lambda event source, your Lambda functions are +triggered with a batch of messages. +If your function fails to process any message from the batch, the entire batch returns to your queue or stream. +This same batch is then retried until either condition happens first: +**a)** your Lambda function returns a successful response , +**b)** record reaches maximum retry attempts, or +**c)** when records expire. + +With this utility, batch records are processed individually – only messages that failed to be processed +return to the queue or stream for a further retry. You simply build a `BatchProcessor` in your handler, +and return its response from the handler's `processMessage` implementation. Exceptions are handled +internally and an appropriate partial response for the message source is returned to Lambda for you. !!! warning - While this utility lowers the chance of processing messages more than once, it is not guaranteed. We recommend implementing processing logic in an idempotent manner wherever possible. - More details on how Lambda works with SQS can be found in the [AWS documentation](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html) +While this utility lowers the chance of processing messages more than once, but it is not guaranteed. +We recommend implementing processing logic in an idempotent manner wherever possible, for instance, +by taking advantage of [the idempotency module](idempotency.md). +More details on how Lambda works with SQS can be found in the [AWS documentation](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html) ## Install -Depending on your version of Java (either Java 1.8 or 11+), the configuration slightly changes. +We simply add `powertools-batch` to our build dependencies. Note - if you are using other Powertools +modules that require code-weaving, you will need to configure that also. Batch does not. -=== "Maven Java 11+" +=== "Maven" - ```xml hl_lines="3-7 16 18 24-27"" + ```xml ... software.amazon.lambda - powertools-sqs + powertools-batch {{ powertools.version }} ... - ... - - - - ... - - dev.aspectj - aspectj-maven-plugin - 1.13.1 - - 11 - 11 - 11 - - - software.amazon.lambda - powertools-sqs - - - - - - - compile - - - - - ... - - - ``` - -=== "Maven Java 1.8" - - ```xml hl_lines="3-7 16 18 24-27" - - ... - - software.amazon.lambda - powertools-sqs - {{ powertools.version }} - - ... - - ... - - - - ... - - org.codehaus.mojo - aspectj-maven-plugin - 1.14.0 - - 1.8 - 1.8 - 1.8 - - - software.amazon.lambda - powertools-sqs - - - - - - - compile - - - - - ... - - - ``` - -=== "Gradle Java 11+" - - ```groovy hl_lines="3 11" - plugins { - id 'java' - id 'io.freefair.aspectj.post-compile-weaving' version '8.1.0' - } - - repositories { - mavenCentral() - } - - dependencies { - aspect 'software.amazon.lambda:powertools-sqs:{{ powertools.version }}' - } - - sourceCompatibility = 11 // or higher - targetCompatibility = 11 // or higher - ``` -=== "Gradle Java 1.8" +=== "Gradle" - ```groovy hl_lines="3 11" - plugins { - id 'java' - id 'io.freefair.aspectj.post-compile-weaving' version '6.6.3' - } + ```groovy repositories { mavenCentral() } dependencies { - aspect 'software.amazon.lambda:powertools-sqs:{{ powertools.version }}' + aspect 'software.amazon.lambda:powertools-batch:{{ powertools.version }}' } - - sourceCompatibility = 1.8 - targetCompatibility = 1.8 ``` ## IAM Permissions -This utility requires additional permissions to work as expected. Lambda functions using this utility require the `sqs:DeleteMessageBatch` permission. - -If you are also using [nonRetryableExceptions](#move-non-retryable-messages-to-a-dead-letter-queue) attribute, utility will need additional permission of `sqs:GetQueueAttributes` on source SQS. -It also needs `sqs:SendMessage` and `sqs:SendMessageBatch` on configured dead letter queue. - -If source or dead letter queue is configured to use encryption at rest using [AWS Key Management Service (KMS)](https://aws.amazon.com/kms/), function will need additional permissions of -`kms:GenerateDataKey` and `kms:Decrypt` on the KMS key being used for encryption. Refer [docs](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-key-management.html#compatibility-with-aws-services) for more details. - -Refer [example project](https://github.com/aws-samples/aws-lambda-powertools-examples/blob/main/java/SqsBatchProcessing/template.yaml#L105) for policy details example. +TODO - I don't know that we actually do here? +## Getting Started +A complete [Serverless Application Model](https://aws.amazon.com/serverless/sam/) example can be found +[here](https://github.com/aws-powertools/powertools-lambda-java/tree/main/examples/powertools-examples-batch) covering +all of the batch sources. ## Processing messages from SQS -You can use either **[SqsBatch annotation](#sqsbatch-annotation)**, or **[SqsUtils Utility API](#sqsutils-utility-api)** as a fluent API. - -Both have nearly the same behaviour when it comes to processing messages from the batch: - -* **Entire batch has been successfully processed**, where your Lambda handler returned successfully, we will let SQS delete the batch to optimize your cost -* **Entire Batch has been partially processed successfully**, where exceptions were raised within your `SqsMessageHandler` interface implementation, we will: - - **1)** Delete successfully processed messages from the queue by directly calling `sqs:DeleteMessageBatch` - - **2)** If a message with a [message group ID](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/using-messagegroupid-property.html) fails, - the processing of the batch will be stopped and the remainder of the messages will be returned to SQS. - This behaviour [is required to handle SQS FIFO queues](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#services-sqs-batchfailurereporting). - - **3)** if non retryable exceptions occur, messages resulting in configured exceptions during processing will be immediately moved to the dead letter queue associated to the source SQS queue or deleted from the source SQS queue if `deleteNonRetryableMessageFromQueue` is set to `true`. - - **4)** Raise `SQSBatchProcessingException` to ensure failed messages return to your SQS queue - -The only difference is that **SqsUtils Utility API** will give you access to return from the processed messages if you need. Exception `SQSBatchProcessingException` thrown from the -utility will have access to both successful and failed messaged along with failure exceptions. - -## Functional Interface SqsMessageHandler +=== App.java +```java +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; +import com.amazonaws.services.lambda.runtime.events.SQSEvent; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import software.amazon.lambda.powertools.batch.BatchMessageHandlerBuilder; +import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; + +public class SqsBatchHandler implements RequestHandler { + private final static Logger LOGGER = LogManager.getLogger(SqsBatchHandler.class); + private final BatchMessageHandler handler; + + public SqsBatchHandler() { + handler = new BatchMessageHandlerBuilder() + .withSqsBatchHandler() + .buildWithMessageHandler(this::processMessage, Product.class); + } -Both [annotation](#sqsbatch-annotation) and [SqsUtils Utility API](#sqsutils-utility-api) requires an implementation of functional interface `SqsMessageHandler`. + @Override + public SQSBatchResponse handleRequest(SQSEvent sqsEvent, Context context) { + return handler.processBatch(sqsEvent, context); + } -This implementation is responsible for processing each individual message from the batch, and to raise an exception if unable to process any of the messages sent. -**Any non-exception/successful return from your record handler function** will instruct utility to queue up each individual message for deletion. + private void processMessage(Product p, Context c) { + LOGGER.info("Processing product " + p); + } -### SqsBatch annotation +} +``` -When using this annotation, you need provide a class implementation of `SqsMessageHandler` that will process individual messages from the batch - It should raise an exception if it is unable to process the record. +=== Product.java +```java +public class Product { + private long id; -All records in the batch will be passed to this handler for processing, even if exceptions are thrown - Here's the behaviour after completing the batch: + private String name; -* **Any successfully processed messages**, we will delete them from the queue via `sqs:DeleteMessageBatch`. -* **if, nonRetryableExceptions attribute is used**, messages resulting in configured exceptions during processing will be immediately moved to the dead letter queue associated to the source SQS queue or deleted from the source SQS queue if `deleteNonRetryableMessageFromQueue` is set to `true`. -* **Any unprocessed messages detected**, we will raise `SQSBatchProcessingException` to ensure failed messages return to your SQS queue. + private double price; -!!! warning - You will not have access to the **processed messages** within the Lambda Handler - all processing logic will and should be performed by the implemented `#!java SqsMessageHandler#process()` function. - -=== "AppSqsEvent.java" - - ```java hl_lines="7" - import software.amazon.lambda.powertools.sqs.SqsBatch; - import software.amazon.lambda.powertools.sqs.SqsMessageHandler; - import software.amazon.lambda.powertools.sqs.SqsUtils; - - public class AppSqsEvent implements RequestHandler { - @Override - @SqsBatch(SampleMessageHandler.class) - public String handleRequest(SQSEvent input, Context context) { - return "{\"statusCode\": 200}"; - } - - public class SampleMessageHandler implements SqsMessageHandler { - - @Override - public String process(SQSMessage message) { - // This will be called for each individual message from a batch - // It should raise an exception if the message was not processed successfully - String returnVal = doSomething(message.getBody()); - return returnVal; - } - } + public Product() { } - ``` -=== "AppSqsEventWithNonRetryableExceptions.java" - - ```java hl_lines="7 21" - import software.amazon.lambda.powertools.sqs.SqsBatch; - import software.amazon.lambda.powertools.sqs.SqsMessageHandler; - import software.amazon.lambda.powertools.sqs.SqsUtils; - - public class AppSqsEvent implements RequestHandler { - @Override - @SqsBatch(value = SampleMessageHandler.class, nonRetryableExceptions = {IllegalArgumentException.class}) - public String handleRequest(SQSEvent input, Context context) { - return "{\"statusCode\": 200}"; - } - - public class SampleMessageHandler implements SqsMessageHandler { - - @Override - public String process(SQSMessage message) { - // This will be called for each individual message from a batch - // It should raise an exception if the message was not processed successfully - String returnVal = doSomething(message.getBody()); - - if(/**Business validation failure**/) { - throw new IllegalArgumentException("Failed business validation. No point of retrying. Move me to DLQ." + message.getMessageId()); - } - - return returnVal; - } - } + public Product(long id, String name, double price) { + this.id = id; + this.name = name; + this.price = price; } - ``` - - -### SqsUtils Utility API - -If you require access to the result of processed messages, you can use this utility. The result from calling **`#!java SqsUtils#batchProcessor()`** on the context manager will be a list of all the return values -from your **`#!java SqsMessageHandler#process()`** function. -You can also use the utility in functional way by providing inline implementation of functional interface **`#!java SqsMessageHandler#process()`** + public long getId() { + return id; + } + public void setId(long id) { + this.id = id; + } -=== "Utility API" - - ```java hl_lines="4" - public class AppSqsEvent implements RequestHandler> { - @Override - public List handleRequest(SQSEvent input, Context context) { - List returnValues = SqsUtils.batchProcessor(input, SampleMessageHandler.class); - - return returnValues; - } - - public class SampleMessageHandler implements SqsMessageHandler { - - @Override - public String process(SQSMessage message) { - // This will be called for each individual message from a batch - // It should raise an exception if the message was not processed successfully - String returnVal = doSomething(message.getBody()); - return returnVal; - } - } + public String getName() { + return name; } - ``` -=== "Function implementation" - - ```java hl_lines="5 6 7 8 9 10" - public class AppSqsEvent implements RequestHandler> { - - @Override - public List handleRequest(SQSEvent input, Context context) { - List returnValues = SqsUtils.batchProcessor(input, (message) -> { - // This will be called for each individual message from a batch - // It should raise an exception if the message was not processed successfully - String returnVal = doSomething(message.getBody()); - return returnVal; - }); - - return returnValues; - } + public void setName(String name) { + this.name = name; } - ``` -## Passing custom SqsClient + public double getPrice() { + return price; + } -If you need to pass custom SqsClient such as region to the SDK, you can pass your own `SqsClient` to be used by utility either for -**[SqsBatch annotation](#sqsbatch-annotation)**, or **[SqsUtils Utility API](#sqsutils-utility-api)**. + public void setPrice(double price) { + this.price = price; + } +} +``` + +## Processing messages from Kinesis Streams +=== App.java +```java +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.amazonaws.services.lambda.runtime.events.KinesisEvent; +import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import software.amazon.lambda.powertools.batch.BatchMessageHandlerBuilder; +import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; + +public class KinesisBatchHandler implements RequestHandler { + + private final static Logger LOGGER = LogManager.getLogger(org.demo.batch.sqs.SqsBatchHandler.class); + private final BatchMessageHandler handler; + + public KinesisBatchHandler() { + handler = new BatchMessageHandlerBuilder() + .withKinesisBatchHandler() + .buildWithMessageHandler(this::processMessage, Product.class); + } -=== "App.java" + @Override + public StreamsEventResponse handleRequest(KinesisEvent kinesisEvent, Context context) { + return handler.processBatch(kinesisEvent, context); + } - ```java hl_lines="3 4" - public class AppSqsEvent implements RequestHandler> { - static { - SqsUtils.overrideSqsClient(SqsClient.builder() - .build()); - } - - @Override - public List handleRequest(SQSEvent input, Context context) { - List returnValues = SqsUtils.batchProcessor(input, SampleMessageHandler.class); - - return returnValues; - } - - public class SampleMessageHandler implements SqsMessageHandler { - - @Override - public String process(SQSMessage message) { - // This will be called for each individual message from a batch - // It should raise an exception if the message was not processed successfully - String returnVal = doSomething(message.getBody()); - return returnVal; - } - } + private void processMessage(Product p, Context c) { + LOGGER.info("Processing product " + p); } - ``` -## Suppressing exceptions +} +``` -If you want to disable the default behavior where `SQSBatchProcessingException` is raised if there are any exception, you can pass the `suppressException` boolean argument. +=== Product.java +```java +public class Product { + private long id; -=== "Within SqsBatch annotation" + private String name; - ```java hl_lines="2" - @Override - @SqsBatch(value = SampleMessageHandler.class, suppressException = true) - public String handleRequest(SQSEvent input, Context context) { - return "{\"statusCode\": 200}"; - } - ``` + private double price; -=== "Within SqsUtils Utility API" + public Product() { + } - ```java hl_lines="3" - @Override - public List handleRequest(SQSEvent input, Context context) { - List returnValues = SqsUtils.batchProcessor(input, true, SampleMessageHandler.class); - - return returnValues; - } - ``` + public Product(long id, String name, double price) { + this.id = id; + this.name = name; + this.price = price; + } -## Move non retryable messages to a dead letter queue + public long getId() { + return id; + } -If you want certain exceptions to be treated as permanent failures during batch processing, i.e. exceptions where the result of retrying will -always be a failure and want these can be immediately moved to the dead letter queue associated to the source SQS queue, you can use `SqsBatch#nonRetryableExceptions()` -to configure such exceptions. + public void setId(long id) { + this.id = id; + } -If you want such messages to be deleted instead, set `SqsBatch#deleteNonRetryableMessageFromQueue()` to `true`. By default, its value is `false`. + public String getName() { + return name; + } -Same capability is also provided by [SqsUtils Utility API](#sqsutils-utility-api). + public void setName(String name) { + this.name = name; + } -!!! info - Make sure the lambda function has required permissions needed by utility. Refer [this section](#iam-permissions). + public double getPrice() { + return price; + } -=== "SqsBatch annotation" + public void setPrice(double price) { + this.price = price; + } +} +``` + + +## Processing messages from DynamoDB Streams +=== App.java +```java +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; +import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import software.amazon.lambda.powertools.batch.BatchMessageHandlerBuilder; +import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; + +public class DynamoDBStreamBatchHandler implements RequestHandler { + + private final static Logger LOGGER = LogManager.getLogger(DynamoDBStreamBatchHandler.class); + private final BatchMessageHandler handler; + + public DynamoDBStreamBatchHandler() { + handler = new BatchMessageHandlerBuilder() + .withDynamoDbBatchHandler() + .buildWithRawMessageHandler(this::processMessage); + } - ```java hl_lines="7 21" - import software.amazon.lambda.powertools.sqs.SqsBatch; - import software.amazon.lambda.powertools.sqs.SqsMessageHandler; - import software.amazon.lambda.powertools.sqs.SqsUtils; - - public class AppSqsEvent implements RequestHandler { - @Override - @SqsBatch(value = SampleMessageHandler.class, nonRetryableExceptions = {IllegalArgumentException.class}) - public String handleRequest(SQSEvent input, Context context) { - return "{\"statusCode\": 200}"; - } - - public class SampleMessageHandler implements SqsMessageHandler { - - @Override - public String process(SQSMessage message) { - // This will be called for each individual message from a batch - // It should raise an exception if the message was not processed successfully - String returnVal = doSomething(message.getBody()); - - if(/**Business validation failure**/) { - throw new IllegalArgumentException("Failed business validation. No point of retrying. Move me to DLQ." + message.getMessageId()); - } - - return returnVal; - } - } + private void processMessage(DynamodbEvent.DynamodbStreamRecord dynamodbStreamRecord, Context context) { + LOGGER.info("Processing DynamoDB Stream Record" + dynamodbStreamRecord); } - ``` -=== "SqsBatch API" - - ```java hl_lines="9 23" - import software.amazon.lambda.powertools.sqs.SqsBatch; - import software.amazon.lambda.powertools.sqs.SqsMessageHandler; - import software.amazon.lambda.powertools.sqs.SqsUtils; - - public class AppSqsEvent implements RequestHandler { - @Override - public String handleRequest(SQSEvent input, Context context) { - - SqsUtils.batchProcessor(input, BatchProcessor.class, IllegalArgumentException.class); - - return "{\"statusCode\": 200}"; - } - - public class SampleMessageHandler implements SqsMessageHandler { - - @Override - public String process(SQSMessage message) { - // This will be called for each individual message from a batch - // It should raise an exception if the message was not processed successfully - String returnVal = doSomething(message.getBody()); - - if(/**Business validation failure**/) { - throw new IllegalArgumentException("Failed business validation. No point of retrying. Move me to DLQ." + message.getMessageId()); - } - - return returnVal; - } - } + @Override + public StreamsEventResponse handleRequest(DynamodbEvent ddbEvent, Context context) { + return handler.processBatch(ddbEvent, context); } - ``` + + +} +``` \ No newline at end of file diff --git a/docs/utilities/sqs_batch.md b/docs/utilities/sqs_batch.md new file mode 100644 index 000000000..5b35a84d3 --- /dev/null +++ b/docs/utilities/sqs_batch.md @@ -0,0 +1,460 @@ +--- +title: SQS Batch Processing (Deprecated) +description: Utility +--- + +The SQS batch processing utility provides a way to handle partial failures when processing batches of messages from SQS. +The utility handles batch processing for both +[standard](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/standard-queues.html) and +[FIFO](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/FIFO-queues.html) SQS queues. + +**Key Features** + +* Prevent successfully processed messages from being returned to SQS +* A simple interface for individually processing messages from a batch + +**Background** + +When using SQS as a Lambda event source mapping, Lambda functions can be triggered with a batch of messages from SQS. +If your function fails to process any message from the batch, the entire batch returns to your SQS queue, and your +Lambda function will be triggered with the same batch again. With this utility, messages within a batch will be handled individually - only messages that were not successfully processed +are returned to the queue. + +!!! warning + While this utility lowers the chance of processing messages more than once, it is not guaranteed. We recommend implementing processing logic in an idempotent manner wherever possible. + More details on how Lambda works with SQS can be found in the [AWS documentation](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html) + +## Install + +Depending on your version of Java (either Java 1.8 or 11+), the configuration slightly changes. + +=== "Maven Java 11+" + + ```xml hl_lines="3-7 16 18 24-27"" + + ... + + software.amazon.lambda + powertools-sqs + {{ powertools.version }} + + ... + + ... + + + + ... + + dev.aspectj + aspectj-maven-plugin + 1.13.1 + + 11 + 11 + 11 + + + software.amazon.lambda + powertools-sqs + + + + + + + compile + + + + + ... + + + ``` + +=== "Maven Java 1.8" + + ```xml hl_lines="3-7 16 18 24-27" + + ... + + software.amazon.lambda + powertools-sqs + {{ powertools.version }} + + ... + + ... + + + + ... + + org.codehaus.mojo + aspectj-maven-plugin + 1.14.0 + + 1.8 + 1.8 + 1.8 + + + software.amazon.lambda + powertools-sqs + + + + + + + compile + + + + + ... + + + ``` + +=== "Gradle Java 11+" + + ```groovy hl_lines="3 11" + plugins { + id 'java' + id 'io.freefair.aspectj.post-compile-weaving' version '8.1.0' + } + + repositories { + mavenCentral() + } + + dependencies { + aspect 'software.amazon.lambda:powertools-sqs:{{ powertools.version }}' + } + + sourceCompatibility = 11 // or higher + targetCompatibility = 11 // or higher + ``` + +=== "Gradle Java 1.8" + + ```groovy hl_lines="3 11" + plugins { + id 'java' + id 'io.freefair.aspectj.post-compile-weaving' version '6.6.3' + } + + repositories { + mavenCentral() + } + + dependencies { + aspect 'software.amazon.lambda:powertools-sqs:{{ powertools.version }}' + } + + sourceCompatibility = 1.8 + targetCompatibility = 1.8 + ``` + +## IAM Permissions + +This utility requires additional permissions to work as expected. Lambda functions using this utility require the `sqs:DeleteMessageBatch` permission. + +If you are also using [nonRetryableExceptions](#move-non-retryable-messages-to-a-dead-letter-queue) attribute, utility will need additional permission of `sqs:GetQueueAttributes` on source SQS. +It also needs `sqs:SendMessage` and `sqs:SendMessageBatch` on configured dead letter queue. + +If source or dead letter queue is configured to use encryption at rest using [AWS Key Management Service (KMS)](https://aws.amazon.com/kms/), function will need additional permissions of +`kms:GenerateDataKey` and `kms:Decrypt` on the KMS key being used for encryption. Refer [docs](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-key-management.html#compatibility-with-aws-services) for more details. + +Refer [example project](https://github.com/aws-samples/aws-lambda-powertools-examples/blob/main/java/SqsBatchProcessing/template.yaml#L105) for policy details example. + + +## Processing messages from SQS + +You can use either **[SqsBatch annotation](#sqsbatch-annotation)**, or **[SqsUtils Utility API](#sqsutils-utility-api)** as a fluent API. + +Both have nearly the same behaviour when it comes to processing messages from the batch: + +* **Entire batch has been successfully processed**, where your Lambda handler returned successfully, we will let SQS delete the batch to optimize your cost +* **Entire Batch has been partially processed successfully**, where exceptions were raised within your `SqsMessageHandler` interface implementation, we will: + - **1)** Delete successfully processed messages from the queue by directly calling `sqs:DeleteMessageBatch` + - **2)** If a message with a [message group ID](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/using-messagegroupid-property.html) fails, + the processing of the batch will be stopped and the remainder of the messages will be returned to SQS. + This behaviour [is required to handle SQS FIFO queues](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#services-sqs-batchfailurereporting). + - **3)** if non retryable exceptions occur, messages resulting in configured exceptions during processing will be immediately moved to the dead letter queue associated to the source SQS queue or deleted from the source SQS queue if `deleteNonRetryableMessageFromQueue` is set to `true`. + - **4)** Raise `SQSBatchProcessingException` to ensure failed messages return to your SQS queue + +The only difference is that **SqsUtils Utility API** will give you access to return from the processed messages if you need. Exception `SQSBatchProcessingException` thrown from the +utility will have access to both successful and failed messaged along with failure exceptions. + +## Functional Interface SqsMessageHandler + +Both [annotation](#sqsbatch-annotation) and [SqsUtils Utility API](#sqsutils-utility-api) requires an implementation of functional interface `SqsMessageHandler`. + +This implementation is responsible for processing each individual message from the batch, and to raise an exception if unable to process any of the messages sent. + +**Any non-exception/successful return from your record handler function** will instruct utility to queue up each individual message for deletion. + +### SqsBatch annotation + +When using this annotation, you need provide a class implementation of `SqsMessageHandler` that will process individual messages from the batch - It should raise an exception if it is unable to process the record. + +All records in the batch will be passed to this handler for processing, even if exceptions are thrown - Here's the behaviour after completing the batch: + +* **Any successfully processed messages**, we will delete them from the queue via `sqs:DeleteMessageBatch`. +* **if, nonRetryableExceptions attribute is used**, messages resulting in configured exceptions during processing will be immediately moved to the dead letter queue associated to the source SQS queue or deleted from the source SQS queue if `deleteNonRetryableMessageFromQueue` is set to `true`. +* **Any unprocessed messages detected**, we will raise `SQSBatchProcessingException` to ensure failed messages return to your SQS queue. + +!!! warning + You will not have access to the **processed messages** within the Lambda Handler - all processing logic will and should be performed by the implemented `#!java SqsMessageHandler#process()` function. + +=== "AppSqsEvent.java" + + ```java hl_lines="7" + import software.amazon.lambda.powertools.sqs.SqsBatch; + import software.amazon.lambda.powertools.sqs.SqsMessageHandler; + import software.amazon.lambda.powertools.sqs.SqsUtils; + + public class AppSqsEvent implements RequestHandler { + @Override + @SqsBatch(SampleMessageHandler.class) + public String handleRequest(SQSEvent input, Context context) { + return "{\"statusCode\": 200}"; + } + + public class SampleMessageHandler implements SqsMessageHandler { + + @Override + public String process(SQSMessage message) { + // This will be called for each individual message from a batch + // It should raise an exception if the message was not processed successfully + String returnVal = doSomething(message.getBody()); + return returnVal; + } + } + } + ``` + +=== "AppSqsEventWithNonRetryableExceptions.java" + + ```java hl_lines="7 21" + import software.amazon.lambda.powertools.sqs.SqsBatch; + import software.amazon.lambda.powertools.sqs.SqsMessageHandler; + import software.amazon.lambda.powertools.sqs.SqsUtils; + + public class AppSqsEvent implements RequestHandler { + @Override + @SqsBatch(value = SampleMessageHandler.class, nonRetryableExceptions = {IllegalArgumentException.class}) + public String handleRequest(SQSEvent input, Context context) { + return "{\"statusCode\": 200}"; + } + + public class SampleMessageHandler implements SqsMessageHandler { + + @Override + public String process(SQSMessage message) { + // This will be called for each individual message from a batch + // It should raise an exception if the message was not processed successfully + String returnVal = doSomething(message.getBody()); + + if(/**Business validation failure**/) { + throw new IllegalArgumentException("Failed business validation. No point of retrying. Move me to DLQ." + message.getMessageId()); + } + + return returnVal; + } + } + } + ``` + + +### SqsUtils Utility API + +If you require access to the result of processed messages, you can use this utility. The result from calling **`#!java SqsUtils#batchProcessor()`** on the context manager will be a list of all the return values +from your **`#!java SqsMessageHandler#process()`** function. + +You can also use the utility in functional way by providing inline implementation of functional interface **`#!java SqsMessageHandler#process()`** + + +=== "Utility API" + + ```java hl_lines="4" + public class AppSqsEvent implements RequestHandler> { + @Override + public List handleRequest(SQSEvent input, Context context) { + List returnValues = SqsUtils.batchProcessor(input, SampleMessageHandler.class); + + return returnValues; + } + + public class SampleMessageHandler implements SqsMessageHandler { + + @Override + public String process(SQSMessage message) { + // This will be called for each individual message from a batch + // It should raise an exception if the message was not processed successfully + String returnVal = doSomething(message.getBody()); + return returnVal; + } + } + } + ``` + +=== "Function implementation" + + ```java hl_lines="5 6 7 8 9 10" + public class AppSqsEvent implements RequestHandler> { + + @Override + public List handleRequest(SQSEvent input, Context context) { + List returnValues = SqsUtils.batchProcessor(input, (message) -> { + // This will be called for each individual message from a batch + // It should raise an exception if the message was not processed successfully + String returnVal = doSomething(message.getBody()); + return returnVal; + }); + + return returnValues; + } + } + ``` + +## Passing custom SqsClient + +If you need to pass custom SqsClient such as region to the SDK, you can pass your own `SqsClient` to be used by utility either for +**[SqsBatch annotation](#sqsbatch-annotation)**, or **[SqsUtils Utility API](#sqsutils-utility-api)**. + +=== "App.java" + + ```java hl_lines="3 4" + public class AppSqsEvent implements RequestHandler> { + static { + SqsUtils.overrideSqsClient(SqsClient.builder() + .build()); + } + + @Override + public List handleRequest(SQSEvent input, Context context) { + List returnValues = SqsUtils.batchProcessor(input, SampleMessageHandler.class); + + return returnValues; + } + + public class SampleMessageHandler implements SqsMessageHandler { + + @Override + public String process(SQSMessage message) { + // This will be called for each individual message from a batch + // It should raise an exception if the message was not processed successfully + String returnVal = doSomething(message.getBody()); + return returnVal; + } + } + } + ``` + +## Suppressing exceptions + +If you want to disable the default behavior where `SQSBatchProcessingException` is raised if there are any exception, you can pass the `suppressException` boolean argument. + +=== "Within SqsBatch annotation" + + ```java hl_lines="2" + @Override + @SqsBatch(value = SampleMessageHandler.class, suppressException = true) + public String handleRequest(SQSEvent input, Context context) { + return "{\"statusCode\": 200}"; + } + ``` + +=== "Within SqsUtils Utility API" + + ```java hl_lines="3" + @Override + public List handleRequest(SQSEvent input, Context context) { + List returnValues = SqsUtils.batchProcessor(input, true, SampleMessageHandler.class); + + return returnValues; + } + ``` + +## Move non retryable messages to a dead letter queue + +If you want certain exceptions to be treated as permanent failures during batch processing, i.e. exceptions where the result of retrying will +always be a failure and want these can be immediately moved to the dead letter queue associated to the source SQS queue, you can use `SqsBatch#nonRetryableExceptions()` +to configure such exceptions. + +If you want such messages to be deleted instead, set `SqsBatch#deleteNonRetryableMessageFromQueue()` to `true`. By default, its value is `false`. + +Same capability is also provided by [SqsUtils Utility API](#sqsutils-utility-api). + +!!! info + Make sure the lambda function has required permissions needed by utility. Refer [this section](#iam-permissions). + +=== "SqsBatch annotation" + + ```java hl_lines="7 21" + import software.amazon.lambda.powertools.sqs.SqsBatch; + import software.amazon.lambda.powertools.sqs.SqsMessageHandler; + import software.amazon.lambda.powertools.sqs.SqsUtils; + + public class AppSqsEvent implements RequestHandler { + @Override + @SqsBatch(value = SampleMessageHandler.class, nonRetryableExceptions = {IllegalArgumentException.class}) + public String handleRequest(SQSEvent input, Context context) { + return "{\"statusCode\": 200}"; + } + + public class SampleMessageHandler implements SqsMessageHandler { + + @Override + public String process(SQSMessage message) { + // This will be called for each individual message from a batch + // It should raise an exception if the message was not processed successfully + String returnVal = doSomething(message.getBody()); + + if(/**Business validation failure**/) { + throw new IllegalArgumentException("Failed business validation. No point of retrying. Move me to DLQ." + message.getMessageId()); + } + + return returnVal; + } + } + } + ``` + +=== "SqsBatch API" + + ```java hl_lines="9 23" + import software.amazon.lambda.powertools.sqs.SqsBatch; + import software.amazon.lambda.powertools.sqs.SqsMessageHandler; + import software.amazon.lambda.powertools.sqs.SqsUtils; + + public class AppSqsEvent implements RequestHandler { + @Override + public String handleRequest(SQSEvent input, Context context) { + + SqsUtils.batchProcessor(input, BatchProcessor.class, IllegalArgumentException.class); + + return "{\"statusCode\": 200}"; + } + + public class SampleMessageHandler implements SqsMessageHandler { + + @Override + public String process(SQSMessage message) { + // This will be called for each individual message from a batch + // It should raise an exception if the message was not processed successfully + String returnVal = doSomething(message.getBody()); + + if(/**Business validation failure**/) { + throw new IllegalArgumentException("Failed business validation. No point of retrying. Move me to DLQ." + message.getMessageId()); + } + + return returnVal; + } + } + } + ``` diff --git a/mkdocs.yml b/mkdocs.yml index da00b24d1..1c8b4a0f4 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -15,6 +15,7 @@ nav: - utilities/parameters.md - utilities/sqs_large_message_handling.md - utilities/batch.md + - utilities/sqs_batch.md - utilities/validation.md - utilities/custom_resources.md - utilities/serialization.md diff --git a/sonar-project.properties b/sonar-project.properties index 94e934c33..00f44a64e 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1 +1,2 @@ +# Ignore code duplicates in the examples sonar.cpd.exclusions=examples/**/* \ No newline at end of file From dae61310a5705524b7c7bb562ede4ed863e7f219 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Thu, 27 Jul 2023 10:25:49 +0200 Subject: [PATCH 41/79] Add warning --- docs/utilities/sqs_batch.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/utilities/sqs_batch.md b/docs/utilities/sqs_batch.md index 5b35a84d3..b5cf19024 100644 --- a/docs/utilities/sqs_batch.md +++ b/docs/utilities/sqs_batch.md @@ -3,6 +3,8 @@ title: SQS Batch Processing (Deprecated) description: Utility --- +!!! warn "The SQS batch module is now deprecated and will be removed in v2. Use the batch module." + The SQS batch processing utility provides a way to handle partial failures when processing batches of messages from SQS. The utility handles batch processing for both [standard](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/standard-queues.html) and From 6d842daabdc0f1dfcbc3c859c4ee69c03d9544f9 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Thu, 27 Jul 2023 10:31:52 +0200 Subject: [PATCH 42/79] More doco --- docs/utilities/sqs_batch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/utilities/sqs_batch.md b/docs/utilities/sqs_batch.md index b5cf19024..134c89ad3 100644 --- a/docs/utilities/sqs_batch.md +++ b/docs/utilities/sqs_batch.md @@ -3,7 +3,7 @@ title: SQS Batch Processing (Deprecated) description: Utility --- -!!! warn "The SQS batch module is now deprecated and will be removed in v2. Use the batch module." +!!! warn "The SQS batch module is now deprecated and will be removed in v2. Use the [batch module](batch.md)." The SQS batch processing utility provides a way to handle partial failures when processing batches of messages from SQS. The utility handles batch processing for both From 9de7d0af41c96fb806f11a85ca7c86c24c372a3f Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Thu, 27 Jul 2023 11:33:46 +0200 Subject: [PATCH 43/79] Format --- examples/powertools-examples-batch/README.md | 6 ++-- .../deploy/kinesis/template.yml | 14 ++++---- .../deploy/sqs/template.yml | 26 +++++++-------- examples/powertools-examples-batch/pom.xml | 4 +-- .../dynamo/DynamoDBStreamBatchHandler.java | 1 - .../org/demo/batch/dynamo/DynamoDBWriter.java | 14 ++++---- .../batch/kinesis/KinesisBatchSender.java | 16 +++------ .../java/org/demo/batch/model/DdbProduct.java | 12 ++++--- .../java/org/demo/batch/model/Product.java | 9 +++-- .../org/demo/batch/sqs/SqsBatchHandler.java | 4 +-- .../org/demo/batch/sqs/SqsBatchSender.java | 11 +++---- .../src/main/resources/log4j2.xml | 2 +- .../template-dynamodb.yaml | 2 +- powertools-batch/pom.xml | 4 +-- .../batch/BatchMessageHandlerBuilder.java | 4 ++- .../AbstractBatchMessageHandlerBuilder.java | 21 ++++++------ .../DynamoDbBatchMessageHandlerBuilder.java | 12 ++++--- .../KinesisBatchMessageHandlerBuilder.java | 25 +++++++------- .../SqsBatchMessageHandlerBuilder.java | 25 +++++++------- .../batch/handler/BatchMessageHandler.java | 3 +- .../handler/DynamoDbBatchMessageHandler.java | 18 +++++----- .../KinesisStreamsBatchMessageHandler.java | 22 ++++++------- .../batch/handler/SqsBatchMessageHandler.java | 33 +++++++++++-------- .../batch/DdbBatchProcessorTest.java | 7 ++-- .../batch/KinesisBatchProcessorTest.java | 25 ++++++++------ .../batch/SQSBatchProcessorTest.java | 8 ++--- .../lambda/powertools/batch/model/Basket.java | 23 ++++++++----- .../powertools/batch/model/Product.java | 8 +++-- .../src/test/resources/sqs_event.json | 3 -- .../src/test/resources/sqs_fifo_event.json | 3 -- 30 files changed, 191 insertions(+), 174 deletions(-) diff --git a/examples/powertools-examples-batch/README.md b/examples/powertools-examples-batch/README.md index ef832e035..d65fb584a 100644 --- a/examples/powertools-examples-batch/README.md +++ b/examples/powertools-examples-batch/README.md @@ -1,6 +1,7 @@ # Powertools for AWS Lambda (Java) - Batch Example -This project contains examples of Lambda function using the batch processing module of Powertools for AWS Lambda (Java). For more information on this module, please refer to the +This project contains examples of Lambda function using the batch processing module of Powertools for AWS Lambda (Java). +For more information on this module, please refer to the [documentation](https://docs.powertools.aws.dev/lambda-java/utilities/batch/). Three different examples and SAM deployments are included, covering each of the batch sources: @@ -17,6 +18,7 @@ started with SAM in [the examples directory](../README.md) This sample contains three different deployments, depending on which batch processor you'd like to use, you can change to the subdirectory containing the example SAM template, and deploy. For instance, for the SQS batch deployment: + ```bash cd deploy/sqs sam build @@ -25,7 +27,7 @@ sam deploy --guided ## Test the application -Each of the examples uses a Lambda scheduled every 5 minutes to push a batch, and a separate lambda to read it. To +Each of the examples uses a Lambda scheduled every 5 minutes to push a batch, and a separate lambda to read it. To see this in action, we can simply tail the logs of our stack: ```bash diff --git a/examples/powertools-examples-batch/deploy/kinesis/template.yml b/examples/powertools-examples-batch/deploy/kinesis/template.yml index dbe017558..dcece61b8 100644 --- a/examples/powertools-examples-batch/deploy/kinesis/template.yml +++ b/examples/powertools-examples-batch/deploy/kinesis/template.yml @@ -39,18 +39,18 @@ Resources: STREAM_NAME: !Ref DemoKinesisStream Policies: - Statement: - - Sid: WriteToKinesis - Effect: Allow - Action: - - kinesis:PutRecords - - kinesis:DescribeStream - Resource: !GetAtt DemoKinesisStream.Arn + - Sid: WriteToKinesis + Effect: Allow + Action: + - kinesis:PutRecords + - kinesis:DescribeStream + Resource: !GetAtt DemoKinesisStream.Arn Events: CWSchedule: Type: Schedule Properties: Schedule: 'rate(5 minutes)' - Name: !Join ["-", ["message-producer-schedule", !Select [0, !Split [-, !Select [2, !Split [/, !Ref AWS::StackId ]]]]]] + Name: !Join [ "-", [ "message-producer-schedule", !Select [ 0, !Split [ -, !Select [ 2, !Split [ /, !Ref AWS::StackId ] ] ] ] ] ] Description: Produce message to Kinesis via a Lambda function Enabled: true diff --git a/examples/powertools-examples-batch/deploy/sqs/template.yml b/examples/powertools-examples-batch/deploy/sqs/template.yml index d038362b0..764ba4863 100644 --- a/examples/powertools-examples-batch/deploy/sqs/template.yml +++ b/examples/powertools-examples-batch/deploy/sqs/template.yml @@ -72,24 +72,24 @@ Resources: QUEUE_URL: !Ref DemoSqsQueue Policies: - Statement: - - Sid: SQSSendMessageBatch - Effect: Allow - Action: - - sqs:SendMessageBatch - - sqs:SendMessage - Resource: !GetAtt DemoSqsQueue.Arn - - Sid: SQSKMSKey - Effect: Allow - Action: - - kms:GenerateDataKey - - kms:Decrypt - Resource: !GetAtt CustomerKey.Arn + - Sid: SQSSendMessageBatch + Effect: Allow + Action: + - sqs:SendMessageBatch + - sqs:SendMessage + Resource: !GetAtt DemoSqsQueue.Arn + - Sid: SQSKMSKey + Effect: Allow + Action: + - kms:GenerateDataKey + - kms:Decrypt + Resource: !GetAtt CustomerKey.Arn Events: CWSchedule: Type: Schedule Properties: Schedule: 'rate(5 minutes)' - Name: !Join ["-", ["message-producer-schedule", !Select [0, !Split [-, !Select [2, !Split [/, !Ref AWS::StackId ]]]]]] + Name: !Join [ "-", [ "message-producer-schedule", !Select [ 0, !Split [ -, !Select [ 2, !Split [ /, !Ref AWS::StackId ] ] ] ] ] ] Description: Produce message to SQS via a Lambda function Enabled: true diff --git a/examples/powertools-examples-batch/pom.xml b/examples/powertools-examples-batch/pom.xml index efb2b565d..d3c4bc49b 100644 --- a/examples/powertools-examples-batch/pom.xml +++ b/examples/powertools-examples-batch/pom.xml @@ -1,6 +1,6 @@ - 4.0.0 diff --git a/examples/powertools-examples-batch/src/main/java/org/demo/batch/dynamo/DynamoDBStreamBatchHandler.java b/examples/powertools-examples-batch/src/main/java/org/demo/batch/dynamo/DynamoDBStreamBatchHandler.java index c8cb81bc5..d8b6f0bbd 100644 --- a/examples/powertools-examples-batch/src/main/java/org/demo/batch/dynamo/DynamoDBStreamBatchHandler.java +++ b/examples/powertools-examples-batch/src/main/java/org/demo/batch/dynamo/DynamoDBStreamBatchHandler.java @@ -6,7 +6,6 @@ import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.demo.batch.model.Product; import software.amazon.lambda.powertools.batch.BatchMessageHandlerBuilder; import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; diff --git a/examples/powertools-examples-batch/src/main/java/org/demo/batch/dynamo/DynamoDBWriter.java b/examples/powertools-examples-batch/src/main/java/org/demo/batch/dynamo/DynamoDBWriter.java index 4f2b9f9da..7178ff885 100644 --- a/examples/powertools-examples-batch/src/main/java/org/demo/batch/dynamo/DynamoDBWriter.java +++ b/examples/powertools-examples-batch/src/main/java/org/demo/batch/dynamo/DynamoDBWriter.java @@ -3,6 +3,9 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.RequestHandler; import com.amazonaws.services.lambda.runtime.events.ScheduledEvent; +import java.security.SecureRandom; +import java.util.UUID; +import java.util.stream.IntStream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.demo.batch.model.DdbProduct; @@ -14,10 +17,6 @@ import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; -import java.security.SecureRandom; -import java.util.UUID; -import java.util.stream.IntStream; - public class DynamoDBWriter implements RequestHandler { private static final Logger LOGGER = LogManager.getLogger(DynamoDBWriter.class); @@ -54,9 +53,10 @@ public String handleRequest(ScheduledEvent scheduledEvent, Context context) { productBuilder.addPutItem(product); }); - BatchWriteItemEnhancedRequest batchWriteItemEnhancedRequest = BatchWriteItemEnhancedRequest.builder().writeBatches( - productBuilder.build()) - .build(); + BatchWriteItemEnhancedRequest batchWriteItemEnhancedRequest = + BatchWriteItemEnhancedRequest.builder().writeBatches( + productBuilder.build()) + .build(); BatchWriteResult batchWriteResult = enhancedClient.batchWriteItem(batchWriteItemEnhancedRequest); diff --git a/examples/powertools-examples-batch/src/main/java/org/demo/batch/kinesis/KinesisBatchSender.java b/examples/powertools-examples-batch/src/main/java/org/demo/batch/kinesis/KinesisBatchSender.java index 84b79436e..0bc7dc42c 100644 --- a/examples/powertools-examples-batch/src/main/java/org/demo/batch/kinesis/KinesisBatchSender.java +++ b/examples/powertools-examples-batch/src/main/java/org/demo/batch/kinesis/KinesisBatchSender.java @@ -1,30 +1,24 @@ package org.demo.batch.kinesis; +import static java.util.stream.Collectors.toList; + import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.RequestHandler; import com.amazonaws.services.lambda.runtime.events.ScheduledEvent; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import java.security.SecureRandom; +import java.util.List; +import java.util.stream.IntStream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.demo.batch.model.Product; -import software.amazon.awssdk.core.BytesWrapper; import software.amazon.awssdk.core.SdkBytes; import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; import software.amazon.awssdk.services.kinesis.KinesisClient; import software.amazon.awssdk.services.kinesis.model.PutRecordsRequest; import software.amazon.awssdk.services.kinesis.model.PutRecordsRequestEntry; import software.amazon.awssdk.services.kinesis.model.PutRecordsResponse; -import software.amazon.awssdk.services.sqs.SqsClient; -import software.amazon.awssdk.services.sqs.model.SendMessageBatchRequest; -import software.amazon.awssdk.services.sqs.model.SendMessageBatchRequestEntry; -import software.amazon.awssdk.services.sqs.model.SendMessageBatchResponse; - -import java.security.SecureRandom; -import java.util.List; -import java.util.stream.IntStream; - -import static java.util.stream.Collectors.toList; /** diff --git a/examples/powertools-examples-batch/src/main/java/org/demo/batch/model/DdbProduct.java b/examples/powertools-examples-batch/src/main/java/org/demo/batch/model/DdbProduct.java index 91e9943d2..9d69eac5d 100644 --- a/examples/powertools-examples-batch/src/main/java/org/demo/batch/model/DdbProduct.java +++ b/examples/powertools-examples-batch/src/main/java/org/demo/batch/model/DdbProduct.java @@ -11,13 +11,13 @@ * limitations under the License. * */ + package org.demo.batch.model; +import java.util.Objects; import software.amazon.awssdk.enhanced.dynamodb.mapper.annotations.DynamoDbBean; import software.amazon.awssdk.enhanced.dynamodb.mapper.annotations.DynamoDbPartitionKey; -import java.util.Objects; - @DynamoDbBean public class DdbProduct { private String id; @@ -62,8 +62,12 @@ public void setPrice(double price) { @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } DdbProduct that = (DdbProduct) o; return Double.compare(that.price, price) == 0 && Objects.equals(id, that.id) && Objects.equals(name, that.name); } diff --git a/examples/powertools-examples-batch/src/main/java/org/demo/batch/model/Product.java b/examples/powertools-examples-batch/src/main/java/org/demo/batch/model/Product.java index b513226b0..64da1804e 100644 --- a/examples/powertools-examples-batch/src/main/java/org/demo/batch/model/Product.java +++ b/examples/powertools-examples-batch/src/main/java/org/demo/batch/model/Product.java @@ -11,6 +11,7 @@ * limitations under the License. * */ + package org.demo.batch.model; import java.util.Objects; @@ -57,8 +58,12 @@ public void setPrice(double price) { @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } Product product = (Product) o; return id == product.id && Double.compare(product.price, price) == 0 && Objects.equals(name, product.name); } diff --git a/examples/powertools-examples-batch/src/main/java/org/demo/batch/sqs/SqsBatchHandler.java b/examples/powertools-examples-batch/src/main/java/org/demo/batch/sqs/SqsBatchHandler.java index 5d508b5bf..bb9d704d3 100644 --- a/examples/powertools-examples-batch/src/main/java/org/demo/batch/sqs/SqsBatchHandler.java +++ b/examples/powertools-examples-batch/src/main/java/org/demo/batch/sqs/SqsBatchHandler.java @@ -4,14 +4,14 @@ import com.amazonaws.services.lambda.runtime.RequestHandler; import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; import com.amazonaws.services.lambda.runtime.events.SQSEvent; -import org.demo.batch.model.Product; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.demo.batch.model.Product; import software.amazon.lambda.powertools.batch.BatchMessageHandlerBuilder; import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; public class SqsBatchHandler implements RequestHandler { - private final static Logger LOGGER = LogManager.getLogger(SqsBatchHandler.class); + private final static Logger LOGGER = LogManager.getLogger(SqsBatchHandler.class); private final BatchMessageHandler handler; public SqsBatchHandler() { diff --git a/examples/powertools-examples-batch/src/main/java/org/demo/batch/sqs/SqsBatchSender.java b/examples/powertools-examples-batch/src/main/java/org/demo/batch/sqs/SqsBatchSender.java index 1938414ae..af78bed5a 100644 --- a/examples/powertools-examples-batch/src/main/java/org/demo/batch/sqs/SqsBatchSender.java +++ b/examples/powertools-examples-batch/src/main/java/org/demo/batch/sqs/SqsBatchSender.java @@ -1,10 +1,15 @@ package org.demo.batch.sqs; +import static java.util.stream.Collectors.toList; + import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.RequestHandler; import com.amazonaws.services.lambda.runtime.events.ScheduledEvent; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import java.security.SecureRandom; +import java.util.List; +import java.util.stream.IntStream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.demo.batch.model.Product; @@ -14,12 +19,6 @@ import software.amazon.awssdk.services.sqs.model.SendMessageBatchRequestEntry; import software.amazon.awssdk.services.sqs.model.SendMessageBatchResponse; -import java.security.SecureRandom; -import java.util.List; -import java.util.stream.IntStream; - -import static java.util.stream.Collectors.toList; - /** * A Lambda handler used to send message batches to SQS. This is only here diff --git a/examples/powertools-examples-batch/src/main/resources/log4j2.xml b/examples/powertools-examples-batch/src/main/resources/log4j2.xml index e1fd14cea..ea3ecf474 100644 --- a/examples/powertools-examples-batch/src/main/resources/log4j2.xml +++ b/examples/powertools-examples-batch/src/main/resources/log4j2.xml @@ -2,7 +2,7 @@ - + diff --git a/examples/powertools-examples-batch/template-dynamodb.yaml b/examples/powertools-examples-batch/template-dynamodb.yaml index 7c32514bc..54ca26c2c 100644 --- a/examples/powertools-examples-batch/template-dynamodb.yaml +++ b/examples/powertools-examples-batch/template-dynamodb.yaml @@ -66,7 +66,7 @@ Resources: Type: Schedule Properties: Schedule: 'rate(5 minutes)' - Name: !Join ["-", ["ddb-writer-schedule", !Select [0, !Split [-, !Select [2, !Split [/, !Ref AWS::StackId ]]]]]] + Name: !Join [ "-", [ "ddb-writer-schedule", !Select [ 0, !Split [ -, !Select [ 2, !Split [ /, !Ref AWS::StackId ] ] ] ] ] ] Description: Produce message to SQS via a Lambda function Enabled: true diff --git a/powertools-batch/pom.xml b/powertools-batch/pom.xml index 4008cc36e..c788b206d 100644 --- a/powertools-batch/pom.xml +++ b/powertools-batch/pom.xml @@ -1,6 +1,6 @@ - 4.0.0 diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java index 97c7c235b..03a2b498a 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java @@ -1,6 +1,8 @@ package software.amazon.lambda.powertools.batch; -import software.amazon.lambda.powertools.batch.builder.*; +import software.amazon.lambda.powertools.batch.builder.DynamoDbBatchMessageHandlerBuilder; +import software.amazon.lambda.powertools.batch.builder.KinesisBatchMessageHandlerBuilder; +import software.amazon.lambda.powertools.batch.builder.SqsBatchMessageHandlerBuilder; /** * A builder-style interface we can use to build batch processing handlers for SQS, Kinesis Streams, diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/AbstractBatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/AbstractBatchMessageHandlerBuilder.java index c6019a626..69121f7e1 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/AbstractBatchMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/AbstractBatchMessageHandlerBuilder.java @@ -1,17 +1,15 @@ package software.amazon.lambda.powertools.batch.builder; import com.amazonaws.services.lambda.runtime.Context; -import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; - import java.util.function.BiConsumer; import java.util.function.Consumer; +import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; /** - * * An abstract class to capture common arguments used across all the message-binding-specific batch processing * builders. The builders provide a fluent interface to configure the batch processors. Any arguments specific * to a particular batch binding can be added to the child builder. - * + *

* We capture types for the various messages involved, so that we can provide an interface that makes * sense for the concrete child. * @@ -28,7 +26,7 @@ abstract class AbstractBatchMessageHandlerBuilder { * Provides an (Optional!) success handler. A success handler is invoked * once for each message after it has been processed by the user-provided * handler. - * + *

* If the success handler throws, the item in the batch will be * marked failed. * @@ -45,7 +43,7 @@ public C withSuccessHandler(Consumer handler) { * user-provided handler. This gives the user's code a useful hook to do * anything else that might have to be done in response to a failure - for * instance, updating a metric, or writing a detailed log. - * + *

* Please note that this method has nothing to do with the partial batch * failure mechanism. Regardless of whether a failure handler is * specified, partial batch failures and responses to the Lambda environment @@ -64,12 +62,12 @@ public C withFailureHandler(BiConsumer handler) { * takes a function that consumes a raw message and the Lambda context. This * is useful for handlers that need access to the entire message object, not * just the deserialized contents of the body. - * + *

* Note: If you don't need the Lambda context, use the variant of this function * that does not require it. * * @param handler Takes a raw message - the underlying AWS Events Library event - to process. - * For instance for SQS this would be an SQSMessage. + * For instance for SQS this would be an SQSMessage. * @return A BatchMessageHandler for processing the batch */ public abstract BatchMessageHandler buildWithRawMessageHandler(BiConsumer handler); @@ -82,7 +80,7 @@ public C withFailureHandler(BiConsumer handler) { * just the deserialized contents of the body. * * @param handler Takes a raw message - the underlying AWS Events Library event - to process. - * For instance for SQS this would be an SQSMessage. + * For instance for SQS this would be an SQSMessage. * @return A BatchMessageHandler for processing the batch */ public BatchMessageHandler buildWithRawMessageHandler(Consumer handler) { @@ -101,13 +99,14 @@ public BatchMessageHandler buildWithRawMessageHandler(Consumer handler) * @param handler Processes the deserialized body of the message * @return A BatchMessageHandler for processing the batch */ - public abstract BatchMessageHandler buildWithMessageHandler(BiConsumer handler, Class messageClass); + public abstract BatchMessageHandler buildWithMessageHandler(BiConsumer handler, + Class messageClass); /** * Builds a BatchMessageHandler that can be used to process batches, given * a user-defined handler to process each item in the batch. This variant * takes a function that consumes the deserialized body of the given message - * If deserialization fails, it will be treated as + * If deserialization fails, it will be treated as * failure of the processing of that item in the batch. * Note: If you don't need the Lambda context, use the variant of this function * that does not require it. diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/DynamoDbBatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/DynamoDbBatchMessageHandlerBuilder.java index b1d90fd99..065a90d46 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/DynamoDbBatchMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/DynamoDbBatchMessageHandlerBuilder.java @@ -3,23 +3,24 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; +import java.util.function.BiConsumer; import software.amazon.lambda.powertools.batch.exception.DeserializationNotSupportedException; import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; import software.amazon.lambda.powertools.batch.handler.DynamoDbBatchMessageHandler; -import java.util.function.BiConsumer; - /** * Builds a batch processor for processing DynamoDB Streams batch events **/ -public class DynamoDbBatchMessageHandlerBuilder extends AbstractBatchMessageHandlerBuilder { @Override - public BatchMessageHandler buildWithRawMessageHandler(BiConsumer rawMessageHandler) { + public BatchMessageHandler buildWithRawMessageHandler( + BiConsumer rawMessageHandler) { return new DynamoDbBatchMessageHandler( this.successHandler, this.failureHandler, @@ -27,7 +28,8 @@ public BatchMessageHandler buildWithRawMess } @Override - public BatchMessageHandler buildWithMessageHandler(BiConsumer handler, Class messageClass) { + public BatchMessageHandler buildWithMessageHandler( + BiConsumer handler, Class messageClass) { // The DDB provider streams DynamoDB changes, and therefore does not have a customizable payload throw new DeserializationNotSupportedException(); } diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/KinesisBatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/KinesisBatchMessageHandlerBuilder.java index 96e9705dc..c5b7daf48 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/KinesisBatchMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/KinesisBatchMessageHandlerBuilder.java @@ -3,31 +3,32 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.KinesisEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; +import java.util.function.BiConsumer; import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; import software.amazon.lambda.powertools.batch.handler.KinesisStreamsBatchMessageHandler; -import java.util.function.BiConsumer; - /** * Builds a batch processor for processing Kinesis Streams batch events */ -public class KinesisBatchMessageHandlerBuilder extends AbstractBatchMessageHandlerBuilder -{ + StreamsEventResponse> { @Override - public BatchMessageHandler buildWithRawMessageHandler(BiConsumer rawMessageHandler) { + public BatchMessageHandler buildWithRawMessageHandler( + BiConsumer rawMessageHandler) { return new KinesisStreamsBatchMessageHandler( - rawMessageHandler, - null, - null, - successHandler, - failureHandler); + rawMessageHandler, + null, + null, + successHandler, + failureHandler); } @Override - public BatchMessageHandler buildWithMessageHandler(BiConsumer messageHandler, Class messageClass) { + public BatchMessageHandler buildWithMessageHandler( + BiConsumer messageHandler, Class messageClass) { return new KinesisStreamsBatchMessageHandler<>( null, messageHandler, diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/SqsBatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/SqsBatchMessageHandlerBuilder.java index 51b5eef88..5c8c03787 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/SqsBatchMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/SqsBatchMessageHandlerBuilder.java @@ -3,11 +3,10 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; import com.amazonaws.services.lambda.runtime.events.SQSEvent; +import java.util.function.BiConsumer; import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; import software.amazon.lambda.powertools.batch.handler.SqsBatchMessageHandler; -import java.util.function.BiConsumer; - /** * Builds a batch processor for the SQS event source. */ @@ -18,18 +17,20 @@ public class SqsBatchMessageHandlerBuilder extends AbstractBatchMessageHandlerBu @Override - public BatchMessageHandler buildWithRawMessageHandler(BiConsumer rawMessageHandler) { - return new SqsBatchMessageHandler( - null, - null, - rawMessageHandler, - successHandler, - failureHandler - ); + public BatchMessageHandler buildWithRawMessageHandler( + BiConsumer rawMessageHandler) { + return new SqsBatchMessageHandler( + null, + null, + rawMessageHandler, + successHandler, + failureHandler + ); } @Override - public BatchMessageHandler buildWithMessageHandler(BiConsumer messageHandler, Class messageClass) { + public BatchMessageHandler buildWithMessageHandler( + BiConsumer messageHandler, Class messageClass) { return new SqsBatchMessageHandler<>( messageHandler, messageClass, @@ -46,6 +47,4 @@ protected SqsBatchMessageHandlerBuilder getThis() { } - - } diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/BatchMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/BatchMessageHandler.java index 14fe52508..49ef5976e 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/BatchMessageHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/BatchMessageHandler.java @@ -3,7 +3,6 @@ import com.amazonaws.services.lambda.runtime.Context; /** - * * The basic interface a batch message handler must meet. * * @param The type of the Lambda batch event @@ -16,7 +15,7 @@ public interface BatchMessageHandler { * response indicating the success and failure of individual * messages within the batch. * - * @param event The Lambda event containing the batch to process + * @param event The Lambda event containing the batch to process * @param context The lambda context * @return A partial batch response */ diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/DynamoDbBatchMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/DynamoDbBatchMessageHandler.java index 3d46a90fa..107161c3a 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/DynamoDbBatchMessageHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/DynamoDbBatchMessageHandler.java @@ -3,28 +3,28 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.ArrayList; import java.util.List; import java.util.function.BiConsumer; import java.util.function.Consumer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A batch message processor for DynamoDB Streams batches. * * @see DynamoDB Streams batch failure reporting - * */ -public class DynamoDbBatchMessageHandler implements BatchMessageHandler{ +public class DynamoDbBatchMessageHandler implements BatchMessageHandler { private final static Logger LOGGER = LoggerFactory.getLogger(DynamoDbBatchMessageHandler.class); private final Consumer successHandler; private final BiConsumer failureHandler; private final BiConsumer rawMessageHandler; - public DynamoDbBatchMessageHandler(Consumer successHandler, BiConsumer failureHandler, BiConsumer rawMessageHandler) { + public DynamoDbBatchMessageHandler(Consumer successHandler, + BiConsumer failureHandler, + BiConsumer rawMessageHandler) { this.successHandler = successHandler; this.failureHandler = failureHandler; this.rawMessageHandler = rawMessageHandler; @@ -44,7 +44,8 @@ public StreamsEventResponse processBatch(DynamodbEvent event, Context context) { } } catch (Throwable t) { String sequenceNumber = record.getDynamodb().getSequenceNumber(); - LOGGER.error("Error while processing record with id {}: {}, adding it to batch item failures", sequenceNumber, t.getMessage()); + LOGGER.error("Error while processing record with id {}: {}, adding it to batch item failures", + sequenceNumber, t.getMessage()); batchFailures.add(new StreamsEventResponse.BatchItemFailure(sequenceNumber)); // Report failure if we have a handler @@ -52,8 +53,7 @@ public StreamsEventResponse processBatch(DynamodbEvent event, Context context) { // A failing failure handler is no reason to fail the batch try { this.failureHandler.accept(record, t); - } - catch (Throwable t2) { + } catch (Throwable t2) { LOGGER.warn("failureHandler threw handling failure", t2); } } diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/KinesisStreamsBatchMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/KinesisStreamsBatchMessageHandler.java index 17fda7860..034911199 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/KinesisStreamsBatchMessageHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/KinesisStreamsBatchMessageHandler.java @@ -4,22 +4,22 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.KinesisEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import software.amazon.lambda.powertools.utilities.EventDeserializer; - import java.util.ArrayList; import java.util.List; import java.util.function.BiConsumer; import java.util.function.Consumer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.lambda.powertools.utilities.EventDeserializer; /** * A batch message processor for Kinesis Streams batch processing. - * + *

* Refer to Kinesis Batch failure reporting + * * @param The user-defined type of the Kinesis record payload */ -public class KinesisStreamsBatchMessageHandler implements BatchMessageHandler { +public class KinesisStreamsBatchMessageHandler implements BatchMessageHandler { private final static Logger LOGGER = LoggerFactory.getLogger(KinesisStreamsBatchMessageHandler.class); private final BiConsumer rawMessageHandler; @@ -29,9 +29,10 @@ public class KinesisStreamsBatchMessageHandler implements BatchMessageHandle private final BiConsumer failureHandler; public KinesisStreamsBatchMessageHandler(BiConsumer rawMessageHandler, - BiConsumer messageHandler, - Class messageClass, Consumer successHandler, - BiConsumer failureHandler) { + BiConsumer messageHandler, + Class messageClass, + Consumer successHandler, + BiConsumer failureHandler) { this.rawMessageHandler = rawMessageHandler; this.messageHandler = messageHandler; @@ -65,8 +66,7 @@ public StreamsEventResponse processBatch(KinesisEvent event, Context context) { // A failing failure handler is no reason to fail the batch try { this.failureHandler.accept(record, t); - } - catch (Throwable t2) { + } catch (Throwable t2) { LOGGER.warn("failureHandler threw handling failure", t2); } } diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java index e6f489b37..240eb2aae 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java @@ -3,21 +3,20 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; import com.amazonaws.services.lambda.runtime.events.SQSEvent; -import software.amazon.lambda.powertools.utilities.EventDeserializer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.ArrayList; import java.util.function.BiConsumer; import java.util.function.Consumer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.lambda.powertools.utilities.EventDeserializer; /** * A batch message processor for SQS batches. * - * @see SQS Batch failure reporting * @param The user-defined type of the message payload + * @see SQS Batch failure reporting */ -public class SqsBatchMessageHandler implements BatchMessageHandler { +public class SqsBatchMessageHandler implements BatchMessageHandler { private final static Logger LOGGER = LoggerFactory.getLogger(SqsBatchMessageHandler.class); // The attribute on an SQS-FIFO message used to record the message group ID @@ -30,7 +29,10 @@ public class SqsBatchMessageHandler implements BatchMessageHandler successHandler; private final BiConsumer failureHandler; - public SqsBatchMessageHandler(BiConsumer messageHandler, Class messageClass, BiConsumer rawMessageHandler, Consumer successHandler, BiConsumer failureHandler) { + public SqsBatchMessageHandler(BiConsumer messageHandler, Class messageClass, + BiConsumer rawMessageHandler, + Consumer successHandler, + BiConsumer failureHandler) { this.messageHandler = messageHandler; this.messageClass = messageClass; this.rawMessageHandler = rawMessageHandler; @@ -68,11 +70,15 @@ public SQSBatchResponse processBatch(SQSEvent event, Context context) { } } catch (Throwable t) { - LOGGER.error("Error while processing message with messageId {}: {}, adding it to batch item failures", message.getMessageId(), t.getMessage()); - response.getBatchItemFailures().add(SQSBatchResponse.BatchItemFailure.builder().withItemIdentifier(message.getMessageId()).build()); + LOGGER.error("Error while processing message with messageId {}: {}, adding it to batch item failures", + message.getMessageId(), t.getMessage()); + response.getBatchItemFailures() + .add(SQSBatchResponse.BatchItemFailure.builder().withItemIdentifier(message.getMessageId()) + .build()); if (messageGroupId != null) { failWholeBatch = true; - LOGGER.info("A message in a batch with messageGroupId {} and messageId {} failed; failing the rest of the batch too" + LOGGER.info( + "A message in a batch with messageGroupId {} and messageId {} failed; failing the rest of the batch too" , messageGroupId, message.getMessageId()); } @@ -81,8 +87,7 @@ public SQSBatchResponse processBatch(SQSEvent event, Context context) { // A failing failure handler is no reason to fail the batch try { this.failureHandler.accept(message, t); - } - catch (Throwable t2) { + } catch (Throwable t2) { LOGGER.warn("failureHandler threw handling failure", t2); } } @@ -94,7 +99,9 @@ public SQSBatchResponse processBatch(SQSEvent event, Context context) { // Add the remaining messages to the batch item failures event.getRecords() .subList(messageCursor, event.getRecords().size()) - .forEach(message -> response.getBatchItemFailures().add(SQSBatchResponse.BatchItemFailure.builder().withItemIdentifier(message.getMessageId()).build())); + .forEach(message -> response.getBatchItemFailures() + .add(SQSBatchResponse.BatchItemFailure.builder().withItemIdentifier(message.getMessageId()) + .build())); } return response; } diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/DdbBatchProcessorTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/DdbBatchProcessorTest.java index 8800eb25f..d428f06cb 100644 --- a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/DdbBatchProcessorTest.java +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/DdbBatchProcessorTest.java @@ -1,17 +1,16 @@ package software.amazon.lambda.powertools.batch; +import static org.assertj.core.api.Assertions.assertThat; + import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; import com.amazonaws.services.lambda.runtime.tests.annotations.Event; +import java.util.concurrent.atomic.AtomicBoolean; import org.junit.jupiter.params.ParameterizedTest; import org.mockito.Mock; import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; -import java.util.concurrent.atomic.AtomicBoolean; - -import static org.assertj.core.api.Assertions.assertThat; - public class DdbBatchProcessorTest { @Mock diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/KinesisBatchProcessorTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/KinesisBatchProcessorTest.java index 405091db8..f60af48f5 100644 --- a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/KinesisBatchProcessorTest.java +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/KinesisBatchProcessorTest.java @@ -1,18 +1,17 @@ package software.amazon.lambda.powertools.batch; +import static org.assertj.core.api.Assertions.assertThat; + import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.KinesisEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; import com.amazonaws.services.lambda.runtime.tests.annotations.Event; +import java.util.concurrent.atomic.AtomicBoolean; import org.junit.jupiter.params.ParameterizedTest; import org.mockito.Mock; import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; import software.amazon.lambda.powertools.batch.model.Product; -import java.util.concurrent.atomic.AtomicBoolean; - -import static org.assertj.core.api.Assertions.assertThat; - public class KinesisBatchProcessorTest { @Mock @@ -23,7 +22,8 @@ private void processMessageSucceeds(KinesisEvent.KinesisEventRecord record, Cont } private void processMessageFailsForFixedMessage(KinesisEvent.KinesisEventRecord record, Context context) { - if (record.getKinesis().getSequenceNumber().equals("49545115243490985018280067714973144582180062593244200961")) { + if (record.getKinesis().getSequenceNumber() + .equals("49545115243490985018280067714973144582180062593244200961")) { throw new RuntimeException("fake exception"); } } @@ -64,7 +64,8 @@ public void shouldAddMessageToBatchFailure_whenException_withMessage(KinesisEven // Assert assertThat(kinesisBatchResponse.getBatchItemFailures()).hasSize(1); StreamsEventResponse.BatchItemFailure batchItemFailure = kinesisBatchResponse.getBatchItemFailures().get(0); - assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("49545115243490985018280067714973144582180062593244200961"); + assertThat(batchItemFailure.getItemIdentifier()).isEqualTo( + "49545115243490985018280067714973144582180062593244200961"); } @ParameterizedTest @@ -81,7 +82,8 @@ public void shouldAddMessageToBatchFailure_whenException_withProduct(KinesisEven // Assert assertThat(kinesisBatchResponse.getBatchItemFailures()).hasSize(1); StreamsEventResponse.BatchItemFailure batchItemFailure = kinesisBatchResponse.getBatchItemFailures().get(0); - assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("49545115243490985018280067714973144582180062593244200961"); + assertThat(batchItemFailure.getItemIdentifier()).isEqualTo( + "49545115243490985018280067714973144582180062593244200961"); } @ParameterizedTest @@ -105,7 +107,8 @@ public void failingFailureHandlerShouldntFailBatch(KinesisEvent event) { assertThat(kinesisBatchResponse.getBatchItemFailures().size()).isEqualTo(1); assertThat(wasCalled.get()).isTrue(); StreamsEventResponse.BatchItemFailure batchItemFailure = kinesisBatchResponse.getBatchItemFailures().get(0); - assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("49545115243490985018280067714973144582180062593244200961"); + assertThat(batchItemFailure.getItemIdentifier()).isEqualTo( + "49545115243490985018280067714973144582180062593244200961"); } @ParameterizedTest @@ -116,7 +119,8 @@ public void failingSuccessHandlerShouldntFailBatchButShouldFailMessage(KinesisEv BatchMessageHandler handler = new BatchMessageHandlerBuilder() .withKinesisBatchHandler() .withSuccessHandler((e) -> { - if (e.getKinesis().getSequenceNumber().equals("49545115243490985018280067714973144582180062593244200961")) { + if (e.getKinesis().getSequenceNumber() + .equals("49545115243490985018280067714973144582180062593244200961")) { wasCalledAndFailed.set(true); throw new RuntimeException("Success handler throws"); } @@ -131,7 +135,8 @@ public void failingSuccessHandlerShouldntFailBatchButShouldFailMessage(KinesisEv assertThat(kinesisBatchResponse.getBatchItemFailures().size()).isEqualTo(1); assertThat(wasCalledAndFailed.get()).isTrue(); StreamsEventResponse.BatchItemFailure batchItemFailure = kinesisBatchResponse.getBatchItemFailures().get(0); - assertThat(batchItemFailure.getItemIdentifier()).isEqualTo("49545115243490985018280067714973144582180062593244200961"); + assertThat(batchItemFailure.getItemIdentifier()).isEqualTo( + "49545115243490985018280067714973144582180062593244200961"); } } diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java index 97e3596c3..4bd9e53ec 100644 --- a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java @@ -1,18 +1,17 @@ package software.amazon.lambda.powertools.batch; +import static org.assertj.core.api.Assertions.assertThat; + import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; import com.amazonaws.services.lambda.runtime.events.SQSEvent; import com.amazonaws.services.lambda.runtime.tests.annotations.Event; +import java.util.concurrent.atomic.AtomicBoolean; import org.junit.jupiter.params.ParameterizedTest; import org.mockito.Mock; import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; import software.amazon.lambda.powertools.batch.model.Product; -import java.util.concurrent.atomic.AtomicBoolean; - -import static org.assertj.core.api.Assertions.assertThat; - public class SQSBatchProcessorTest { @Mock private Context context; @@ -155,5 +154,4 @@ public void failingSuccessHandlerShouldntFailBatchButShouldFailMessage(SQSEvent } - } diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/model/Basket.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/model/Basket.java index fd1c9fcc8..6009e79d6 100644 --- a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/model/Basket.java +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/model/Basket.java @@ -11,6 +11,7 @@ * limitations under the License. * */ + package software.amazon.lambda.powertools.batch.model; import java.util.ArrayList; @@ -21,19 +22,19 @@ public class Basket { private List products = new ArrayList<>(); - public List getProducts() { - return products; + public Basket() { } - public void setProducts(List products) { - this.products = products; + public Basket(Product... p) { + products.addAll(Arrays.asList(p)); } - public Basket() { + public List getProducts() { + return products; } - public Basket( Product ...p){ - products.addAll(Arrays.asList(p)); + public void setProducts(List products) { + this.products = products; } public void add(Product product) { @@ -42,8 +43,12 @@ public void add(Product product) { @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } Basket basket = (Basket) o; return products.equals(basket.products); } diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/model/Product.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/model/Product.java index 20f9da9a4..2695578f9 100644 --- a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/model/Product.java +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/model/Product.java @@ -58,8 +58,12 @@ public void setPrice(double price) { @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } Product product = (Product) o; return id == product.id && Double.compare(product.price, price) == 0 && Objects.equals(name, product.name); } diff --git a/powertools-batch/src/test/resources/sqs_event.json b/powertools-batch/src/test/resources/sqs_event.json index 1366029d3..7fdad096f 100644 --- a/powertools-batch/src/test/resources/sqs_event.json +++ b/powertools-batch/src/test/resources/sqs_event.json @@ -11,7 +11,6 @@ "ApproximateFirstReceiveTimestamp": "1601975706499" }, "messageAttributes": { - }, "md5OfBody": "13e7f7851d2eaa5c01f208ebadbf1e72", "eventSource": "aws:sqs", @@ -29,7 +28,6 @@ "ApproximateFirstReceiveTimestamp": "1601975706499" }, "messageAttributes": { - }, "md5OfBody": "13e7f7851d2eaa5c01f208ebadbf1e72", "eventSource": "aws:sqs", @@ -47,7 +45,6 @@ "ApproximateFirstReceiveTimestamp": "1601975706499" }, "messageAttributes": { - }, "md5OfBody": "13e7f7851d2eaa5c01f208ebadbf1e72", "eventSource": "aws:sqs", diff --git a/powertools-batch/src/test/resources/sqs_fifo_event.json b/powertools-batch/src/test/resources/sqs_fifo_event.json index 8ac6a4b8f..e5abb1e5a 100644 --- a/powertools-batch/src/test/resources/sqs_fifo_event.json +++ b/powertools-batch/src/test/resources/sqs_fifo_event.json @@ -12,7 +12,6 @@ "ApproximateFirstReceiveTimestamp": "1601975706499" }, "messageAttributes": { - }, "md5OfBody": "13e7f7851d2eaa5c01f208ebadbf1e72", "eventSource": "aws:sqs", @@ -31,7 +30,6 @@ "ApproximateFirstReceiveTimestamp": "1601975706499" }, "messageAttributes": { - }, "md5OfBody": "13e7f7851d2eaa5c01f208ebadbf1e72", "eventSource": "aws:sqs", @@ -50,7 +48,6 @@ "ApproximateFirstReceiveTimestamp": "1601975706499" }, "messageAttributes": { - }, "md5OfBody": "13e7f7851d2eaa5c01f208ebadbf1e72", "eventSource": "aws:sqs", From 0e95238f847cd4056b643ad791fb2580e0315a34 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Thu, 27 Jul 2023 12:41:28 +0200 Subject: [PATCH 44/79] Docs good --- docs/utilities/batch.md | 383 +++++++++++++++++++----------------- docs/utilities/sqs_batch.md | 29 ++- 2 files changed, 227 insertions(+), 185 deletions(-) diff --git a/docs/utilities/batch.md b/docs/utilities/batch.md index ad309e8c7..de6a83146 100644 --- a/docs/utilities/batch.md +++ b/docs/utilities/batch.md @@ -30,10 +30,10 @@ and return its response from the handler's `processMessage` implementation. Exce internally and an appropriate partial response for the message source is returned to Lambda for you. !!! warning -While this utility lowers the chance of processing messages more than once, but it is not guaranteed. -We recommend implementing processing logic in an idempotent manner wherever possible, for instance, -by taking advantage of [the idempotency module](idempotency.md). -More details on how Lambda works with SQS can be found in the [AWS documentation](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html) + While this utility lowers the chance of processing messages more than once, but it is not guaranteed. + We recommend implementing processing logic in an idempotent manner wherever possible, for instance, + by taking advantage of [the idempotency module](idempotency.md). + More details on how Lambda works with SQS can be found in the [AWS documentation](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html) ## Install @@ -52,6 +52,7 @@ modules that require code-weaving, you will need to configure that also. Batch d ... + ``` === "Gradle" @@ -68,7 +69,14 @@ modules that require code-weaving, you will need to configure that also. Batch d ## IAM Permissions -TODO - I don't know that we actually do here? +The [Lambda execution role](https://docs.aws.amazon.com/lambda/latest/dg/lambda-intro-execution-role.html) of your function +requires appropriate permissions for the message source you are using. In each case you should create a policy that restricts +access to the queue, stream, or table, that your lambda is reading batches with. + +* **SQS** - `SQS:ReceiveMessage`, ``SQS::DeleteMessage``, ``SQS::GetQueueAttributes`` - [further details](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#events-sqs-permissions) +* **Kinesis Streams** - ``kinesis:DescribeStream``, ``kinesis:DescribeStreamSummary``, *kinesis:GetRecords*, ``kinesis:GetShardIterator``, + **kinesis:ListShards**, ``kinesis:ListStreams``, ``kinesis:SubscribeToShard`` - [further details](https://docs.aws.amazon.com/lambda/latest/dg/with-kinesis.html#events-kinesis-permissions) +* **DynamoDB Streams** - ``dynamodb:DescribeStream``, ``dynamodb:GetRecords``, ``dynamodb:GetShardIterator``, ``dynamodb:ListStreams`` - [further details](https://docs.aws.amazon.com/lambda/latest/dg/with-ddb.html#events-dynamodb-permissions) ## Getting Started A complete [Serverless Application Model](https://aws.amazon.com/serverless/sam/) example can be found @@ -77,196 +85,203 @@ all of the batch sources. ## Processing messages from SQS -=== App.java -```java -import com.amazonaws.services.lambda.runtime.Context; -import com.amazonaws.services.lambda.runtime.RequestHandler; -import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; -import com.amazonaws.services.lambda.runtime.events.SQSEvent; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import software.amazon.lambda.powertools.batch.BatchMessageHandlerBuilder; -import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; - -public class SqsBatchHandler implements RequestHandler { - private final static Logger LOGGER = LogManager.getLogger(SqsBatchHandler.class); - private final BatchMessageHandler handler; - - public SqsBatchHandler() { - handler = new BatchMessageHandlerBuilder() - .withSqsBatchHandler() - .buildWithMessageHandler(this::processMessage, Product.class); - } - - @Override - public SQSBatchResponse handleRequest(SQSEvent sqsEvent, Context context) { - return handler.processBatch(sqsEvent, context); - } - - - private void processMessage(Product p, Context c) { - LOGGER.info("Processing product " + p); - } - -} -``` - -=== Product.java -```java -public class Product { - private long id; - - private String name; - - private double price; - - public Product() { - } - - public Product(long id, String name, double price) { - this.id = id; - this.name = name; - this.price = price; - } - - public long getId() { - return id; - } - - public void setId(long id) { - this.id = id; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public double getPrice() { - return price; +=== "App.java" + + ```java + import com.amazonaws.services.lambda.runtime.Context; + import com.amazonaws.services.lambda.runtime.RequestHandler; + import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; + import com.amazonaws.services.lambda.runtime.events.SQSEvent; + import org.apache.logging.log4j.LogManager; + import org.apache.logging.log4j.Logger; + import software.amazon.lambda.powertools.batch.BatchMessageHandlerBuilder; + import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; + + public class SqsBatchHandler implements RequestHandler { + private final static Logger LOGGER = LogManager.getLogger(SqsBatchHandler.class); + private final BatchMessageHandler handler; + + public SqsBatchHandler() { + handler = new BatchMessageHandlerBuilder() + .withSqsBatchHandler() + .buildWithMessageHandler(this::processMessage, Product.class); + } + + @Override + public SQSBatchResponse handleRequest(SQSEvent sqsEvent, Context context) { + return handler.processBatch(sqsEvent, context); + } + + + private void processMessage(Product p, Context c) { + LOGGER.info("Processing product " + p); + } + } + ``` - public void setPrice(double price) { - this.price = price; +=== "Product.java" + + ```java + public class Product { + private long id; + + private String name; + + private double price; + + public Product() { + } + + public Product(long id, String name, double price) { + this.id = id; + this.name = name; + this.price = price; + } + + public long getId() { + return id; + } + + public void setId(long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public double getPrice() { + return price; + } + + public void setPrice(double price) { + this.price = price; + } } -} -``` + ``` ## Processing messages from Kinesis Streams -=== App.java -```java -import com.amazonaws.services.lambda.runtime.Context; -import com.amazonaws.services.lambda.runtime.RequestHandler; -import com.amazonaws.services.lambda.runtime.events.KinesisEvent; -import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import software.amazon.lambda.powertools.batch.BatchMessageHandlerBuilder; -import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; - -public class KinesisBatchHandler implements RequestHandler { - - private final static Logger LOGGER = LogManager.getLogger(org.demo.batch.sqs.SqsBatchHandler.class); - private final BatchMessageHandler handler; - - public KinesisBatchHandler() { - handler = new BatchMessageHandlerBuilder() - .withKinesisBatchHandler() - .buildWithMessageHandler(this::processMessage, Product.class); - } - @Override - public StreamsEventResponse handleRequest(KinesisEvent kinesisEvent, Context context) { - return handler.processBatch(kinesisEvent, context); - } - - private void processMessage(Product p, Context c) { - LOGGER.info("Processing product " + p); - } - -} -``` - -=== Product.java -```java -public class Product { - private long id; - - private String name; - - private double price; - - public Product() { - } - - public Product(long id, String name, double price) { - this.id = id; - this.name = name; - this.price = price; - } - - public long getId() { - return id; - } - - public void setId(long id) { - this.id = id; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public double getPrice() { - return price; +=== "App.java" + + ```java + import com.amazonaws.services.lambda.runtime.Context; + import com.amazonaws.services.lambda.runtime.RequestHandler; + import com.amazonaws.services.lambda.runtime.events.KinesisEvent; + import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; + import org.apache.logging.log4j.LogManager; + import org.apache.logging.log4j.Logger; + import software.amazon.lambda.powertools.batch.BatchMessageHandlerBuilder; + import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; + + public class KinesisBatchHandler implements RequestHandler { + + private final static Logger LOGGER = LogManager.getLogger(org.demo.batch.sqs.SqsBatchHandler.class); + private final BatchMessageHandler handler; + + public KinesisBatchHandler() { + handler = new BatchMessageHandlerBuilder() + .withKinesisBatchHandler() + .buildWithMessageHandler(this::processMessage, Product.class); + } + + @Override + public StreamsEventResponse handleRequest(KinesisEvent kinesisEvent, Context context) { + return handler.processBatch(kinesisEvent, context); + } + + private void processMessage(Product p, Context c) { + LOGGER.info("Processing product " + p); + } + } + ``` - public void setPrice(double price) { - this.price = price; +=== "Product.java" + + ```java + public class Product { + private long id; + + private String name; + + private double price; + + public Product() { + } + + public Product(long id, String name, double price) { + this.id = id; + this.name = name; + this.price = price; + } + + public long getId() { + return id; + } + + public void setId(long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public double getPrice() { + return price; + } + + public void setPrice(double price) { + this.price = price; + } } -} -``` + ``` ## Processing messages from DynamoDB Streams -=== App.java -```java -import com.amazonaws.services.lambda.runtime.Context; -import com.amazonaws.services.lambda.runtime.RequestHandler; -import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; -import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import software.amazon.lambda.powertools.batch.BatchMessageHandlerBuilder; -import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; - -public class DynamoDBStreamBatchHandler implements RequestHandler { - private final static Logger LOGGER = LogManager.getLogger(DynamoDBStreamBatchHandler.class); - private final BatchMessageHandler handler; - - public DynamoDBStreamBatchHandler() { - handler = new BatchMessageHandlerBuilder() - .withDynamoDbBatchHandler() - .buildWithRawMessageHandler(this::processMessage); - } - - private void processMessage(DynamodbEvent.DynamodbStreamRecord dynamodbStreamRecord, Context context) { - LOGGER.info("Processing DynamoDB Stream Record" + dynamodbStreamRecord); - } - - @Override - public StreamsEventResponse handleRequest(DynamodbEvent ddbEvent, Context context) { - return handler.processBatch(ddbEvent, context); +=== "App.java" + + ```java + import com.amazonaws.services.lambda.runtime.Context; + import com.amazonaws.services.lambda.runtime.RequestHandler; + import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; + import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; + import org.apache.logging.log4j.LogManager; + import org.apache.logging.log4j.Logger; + import software.amazon.lambda.powertools.batch.BatchMessageHandlerBuilder; + import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; + + public class DynamoDBStreamBatchHandler implements RequestHandler { + + private final static Logger LOGGER = LogManager.getLogger(DynamoDBStreamBatchHandler.class); + private final BatchMessageHandler handler; + + public DynamoDBStreamBatchHandler() { + handler = new BatchMessageHandlerBuilder() + .withDynamoDbBatchHandler() + .buildWithRawMessageHandler(this::processMessage); + } + + private void processMessage(DynamodbEvent.DynamodbStreamRecord dynamodbStreamRecord, Context context) { + LOGGER.info("Processing DynamoDB Stream Record" + dynamodbStreamRecord); + } + + @Override + public StreamsEventResponse handleRequest(DynamodbEvent ddbEvent, Context context) { + return handler.processBatch(ddbEvent, context); + } + + } - - -} -``` \ No newline at end of file + ``` \ No newline at end of file diff --git a/docs/utilities/sqs_batch.md b/docs/utilities/sqs_batch.md index 134c89ad3..658f7b085 100644 --- a/docs/utilities/sqs_batch.md +++ b/docs/utilities/sqs_batch.md @@ -3,7 +3,9 @@ title: SQS Batch Processing (Deprecated) description: Utility --- -!!! warn "The SQS batch module is now deprecated and will be removed in v2. Use the [batch module](batch.md)." +!!! warning + The SQS batch module is now deprecated and will be removed in v2 of the library. Use the [batch module](batch.md), + and check out **[migrating to the batch library](#migrating-to-the-batch-library)** for migration instructions. The SQS batch processing utility provides a way to handle partial failures when processing batches of messages from SQS. The utility handles batch processing for both @@ -460,3 +462,28 @@ Same capability is also provided by [SqsUtils Utility API](#sqsutils-utility-api } } ``` + +## Migrating to the Batch Library +The [batch processing library](batch.md) provides a way to process messages and gracefully handle partial failures for +SQS, Kinesis Streams, and DynamoDB Streams batch sources. In comparison the legacy SQS Batch library, it relies on +[Lambda partial batch responses](https://aws.amazon.com/about-aws/whats-new/2021/11/aws-lambda-partial-batch-response-sqs-event-source/), +which allows the library to provide a simpler, reliable interface for processing batches. + +In order to get started, check out the [processing messages from SQS](batch/#processing-messages-from-sqs) documentation. +In most cases, you will simply be able to retain your existing batch message handler function, and wrap it with the new +batch processing interface. Unlike this module, As the batch processor uses *partial batch responses* to communicate to +Lambda which messages have been processed and must be removed from the queue, the return of the handler's process function +must be returned to Lambda. + +The new library also no longer requires the `SQS:DeleteMessage` action on the Lambda function's role policy, as Lambda +itself now manages removal of messages from the queue. + +!!! info + Some tuneables from this library are no longer provided. + + * **Non-retryable Exceptions** - there is no mechanism to indicate in a partial batch response that a particular message + should not be retried and instead moved to DLQ - a message either succeeds, or fails and is retried. A message + will be moved to the DLQ once the normal retry process has expired. + * **Suppress Exception** - The new batch processor does not throw an exception on failure of a handler. Instead, + its result must be returned by your code from your message handler to Lambda, so that Lambda can manage + the completed messages and retry behaviour. \ No newline at end of file From ba17efdd5404bfa8ae9824201265cb597a3820b9 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Thu, 27 Jul 2023 13:03:23 +0200 Subject: [PATCH 45/79] Disabling formatting check for now as its breaking the build and I can't work out how to autoapply it from intellij properly --- pom.xml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2bceb557b..9508c7837 100644 --- a/pom.xml +++ b/pom.xml @@ -571,7 +571,10 @@ - check + + From e87e6dc2642753287b4875272da29ea7aea5a221 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Thu, 27 Jul 2023 13:35:12 +0200 Subject: [PATCH 46/79] Make checkstyle happy --- pom.xml | 9 +++------ .../batch/BatchMessageHandlerBuilder.java | 14 ++++++++++++++ .../AbstractBatchMessageHandlerBuilder.java | 14 ++++++++++++++ .../DynamoDbBatchMessageHandlerBuilder.java | 14 ++++++++++++++ .../builder/KinesisBatchMessageHandlerBuilder.java | 14 ++++++++++++++ .../builder/SqsBatchMessageHandlerBuilder.java | 14 ++++++++++++++ .../DeserializationNotSupportedException.java | 14 ++++++++++++++ .../batch/handler/BatchMessageHandler.java | 14 ++++++++++++++ .../batch/handler/DynamoDbBatchMessageHandler.java | 14 ++++++++++++++ .../handler/KinesisStreamsBatchMessageHandler.java | 14 ++++++++++++++ .../batch/handler/SqsBatchMessageHandler.java | 14 ++++++++++++++ .../powertools/batch/DdbBatchProcessorTest.java | 14 ++++++++++++++ .../batch/KinesisBatchProcessorTest.java | 14 ++++++++++++++ .../powertools/batch/SQSBatchProcessorTest.java | 14 ++++++++++++++ 14 files changed, 185 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index 9508c7837..727b09dcf 100644 --- a/pom.xml +++ b/pom.xml @@ -541,9 +541,9 @@ - jdk11 + notJava8 - 11 + [9,) @@ -571,10 +571,7 @@ - - + check diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java index 03a2b498a..4ed44453b 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/BatchMessageHandlerBuilder.java @@ -1,3 +1,17 @@ +/* + * Copyright 2023 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package software.amazon.lambda.powertools.batch; import software.amazon.lambda.powertools.batch.builder.DynamoDbBatchMessageHandlerBuilder; diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/AbstractBatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/AbstractBatchMessageHandlerBuilder.java index 69121f7e1..9b0647770 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/AbstractBatchMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/AbstractBatchMessageHandlerBuilder.java @@ -1,3 +1,17 @@ +/* + * Copyright 2023 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package software.amazon.lambda.powertools.batch.builder; import com.amazonaws.services.lambda.runtime.Context; diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/DynamoDbBatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/DynamoDbBatchMessageHandlerBuilder.java index 065a90d46..8513322b3 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/DynamoDbBatchMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/DynamoDbBatchMessageHandlerBuilder.java @@ -1,3 +1,17 @@ +/* + * Copyright 2023 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package software.amazon.lambda.powertools.batch.builder; import com.amazonaws.services.lambda.runtime.Context; diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/KinesisBatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/KinesisBatchMessageHandlerBuilder.java index c5b7daf48..30bfcab65 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/KinesisBatchMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/KinesisBatchMessageHandlerBuilder.java @@ -1,3 +1,17 @@ +/* + * Copyright 2023 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package software.amazon.lambda.powertools.batch.builder; import com.amazonaws.services.lambda.runtime.Context; diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/SqsBatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/SqsBatchMessageHandlerBuilder.java index 5c8c03787..ee2dc23f6 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/SqsBatchMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/SqsBatchMessageHandlerBuilder.java @@ -1,3 +1,17 @@ +/* + * Copyright 2023 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package software.amazon.lambda.powertools.batch.builder; import com.amazonaws.services.lambda.runtime.Context; diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/exception/DeserializationNotSupportedException.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/exception/DeserializationNotSupportedException.java index d95968e38..6f3206c99 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/exception/DeserializationNotSupportedException.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/exception/DeserializationNotSupportedException.java @@ -1,3 +1,17 @@ +/* + * Copyright 2023 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package software.amazon.lambda.powertools.batch.exception; /** diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/BatchMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/BatchMessageHandler.java index 49ef5976e..730211feb 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/BatchMessageHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/BatchMessageHandler.java @@ -1,3 +1,17 @@ +/* + * Copyright 2023 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package software.amazon.lambda.powertools.batch.handler; import com.amazonaws.services.lambda.runtime.Context; diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/DynamoDbBatchMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/DynamoDbBatchMessageHandler.java index 107161c3a..911da6944 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/DynamoDbBatchMessageHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/DynamoDbBatchMessageHandler.java @@ -1,3 +1,17 @@ +/* + * Copyright 2023 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package software.amazon.lambda.powertools.batch.handler; import com.amazonaws.services.lambda.runtime.Context; diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/KinesisStreamsBatchMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/KinesisStreamsBatchMessageHandler.java index 034911199..04bfe963c 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/KinesisStreamsBatchMessageHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/KinesisStreamsBatchMessageHandler.java @@ -1,3 +1,17 @@ +/* + * Copyright 2023 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package software.amazon.lambda.powertools.batch.handler; diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java index 240eb2aae..539a3840c 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java @@ -1,3 +1,17 @@ +/* + * Copyright 2023 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package software.amazon.lambda.powertools.batch.handler; import com.amazonaws.services.lambda.runtime.Context; diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/DdbBatchProcessorTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/DdbBatchProcessorTest.java index d428f06cb..9e2c211e2 100644 --- a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/DdbBatchProcessorTest.java +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/DdbBatchProcessorTest.java @@ -1,3 +1,17 @@ +/* + * Copyright 2023 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package software.amazon.lambda.powertools.batch; import static org.assertj.core.api.Assertions.assertThat; diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/KinesisBatchProcessorTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/KinesisBatchProcessorTest.java index f60af48f5..d78638e1d 100644 --- a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/KinesisBatchProcessorTest.java +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/KinesisBatchProcessorTest.java @@ -1,3 +1,17 @@ +/* + * Copyright 2023 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package software.amazon.lambda.powertools.batch; import static org.assertj.core.api.Assertions.assertThat; diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java index 4bd9e53ec..2f9429fa3 100644 --- a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java @@ -1,3 +1,17 @@ +/* + * Copyright 2023 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package software.amazon.lambda.powertools.batch; import static org.assertj.core.api.Assertions.assertThat; From 5c046a63b4bc15e782a72440abbbef5fd92829de Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Thu, 27 Jul 2023 17:04:04 +0200 Subject: [PATCH 47/79] Add docs from heitor --- docs/utilities/batch.md | 65 +++++++++++++++++++++++++++++++---------- 1 file changed, 50 insertions(+), 15 deletions(-) diff --git a/docs/utilities/batch.md b/docs/utilities/batch.md index de6a83146..4ecb836db 100644 --- a/docs/utilities/batch.md +++ b/docs/utilities/batch.md @@ -6,6 +6,26 @@ description: Utility The batch processing utility provides a way to handle partial failures when processing batches of messages from SQS queues, SQS FIFO queues, Kinesis Streams, or DynamoDB Streams. +```mermaid +stateDiagram-v2 + direction LR + BatchSource: Amazon SQS

Amazon Kinesis Data Streams

Amazon DynamoDB Streams

+ LambdaInit: Lambda invocation + BatchProcessor: Batch Processor + RecordHandler: Record Handler function + YourLogic: Your logic to process each batch item + LambdaResponse: Lambda response + BatchSource --> LambdaInit + LambdaInit --> BatchProcessor + BatchProcessor --> RecordHandler + state BatchProcessor { + [*] --> RecordHandler: Your function + RecordHandler --> YourLogic + } + RecordHandler --> BatchProcessor: Collect results + BatchProcessor --> LambdaResponse: Report items that failed processing +``` + **Key Features** @@ -18,11 +38,27 @@ SQS FIFO queues, Kinesis Streams, or DynamoDB Streams. When using SQS, Kinesis Data Streams, or DynamoDB Streams as a Lambda event source, your Lambda functions are triggered with a batch of messages. -If your function fails to process any message from the batch, the entire batch returns to your queue or stream. -This same batch is then retried until either condition happens first: -**a)** your Lambda function returns a successful response , +If your function fails to process any message from the batch, the entire batch returns to your queue or stream. +This same batch is then retried until either condition happens first: +**a)** your Lambda function returns a successful response, **b)** record reaches maximum retry attempts, or -**c)** when records expire. +**c)** records expire. + +```mermaid +journey + section Conditions + Successful response: 5: Success + Maximum retries: 3: Failure + Records expired: 1: Failure +``` + +This behavior changes when you enable Report Batch Item Failures feature in your Lambda function event source configuration: + + +* [**SQS queues**](#sqs-standard). Only messages reported as failure will return to the queue for a retry, while successful ones will be deleted. + * [**Kinesis data streams**](#kinesis-and-dynamodb-streams) and [**DynamoDB streams**](#kinesis-and-dynamodb-streams). + Single reported failure will use its sequence number as the stream checkpoint. + Multiple reported failures will use the lowest sequence number as checkpoint. With this utility, batch records are processed individually – only messages that failed to be processed return to the queue or stream for a further retry. You simply build a `BatchProcessor` in your handler, @@ -66,22 +102,21 @@ modules that require code-weaving, you will need to configure that also. Batch d aspect 'software.amazon.lambda:powertools-batch:{{ powertools.version }}' } ``` +## Getting Started -## IAM Permissions - -The [Lambda execution role](https://docs.aws.amazon.com/lambda/latest/dg/lambda-intro-execution-role.html) of your function -requires appropriate permissions for the message source you are using. In each case you should create a policy that restricts -access to the queue, stream, or table, that your lambda is reading batches with. +For this feature to work, you need to **(1)** configure your Lambda function event source to use `ReportBatchItemFailures`, +and **(2)** return [a specific response](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#services-sqs-batchfailurereporting){target="_blank" rel="nofollow"} +to report which records failed to be processed. -* **SQS** - `SQS:ReceiveMessage`, ``SQS::DeleteMessage``, ``SQS::GetQueueAttributes`` - [further details](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#events-sqs-permissions) -* **Kinesis Streams** - ``kinesis:DescribeStream``, ``kinesis:DescribeStreamSummary``, *kinesis:GetRecords*, ``kinesis:GetShardIterator``, - **kinesis:ListShards**, ``kinesis:ListStreams``, ``kinesis:SubscribeToShard`` - [further details](https://docs.aws.amazon.com/lambda/latest/dg/with-kinesis.html#events-kinesis-permissions) -* **DynamoDB Streams** - ``dynamodb:DescribeStream``, ``dynamodb:GetRecords``, ``dynamodb:GetShardIterator``, ``dynamodb:ListStreams`` - [further details](https://docs.aws.amazon.com/lambda/latest/dg/with-ddb.html#events-dynamodb-permissions) +You use your preferred deployment framework to set the correct configuration while this utility handles the correct response to be returned. -## Getting Started A complete [Serverless Application Model](https://aws.amazon.com/serverless/sam/) example can be found [here](https://github.com/aws-powertools/powertools-lambda-java/tree/main/examples/powertools-examples-batch) covering -all of the batch sources. +all of the batch sources. + + + +!!! note "You do not need any additional IAM permissions to use this utility, except for what each event source requires." ## Processing messages from SQS From 56af4d04eabe652fa87d0a011a3ca937b9cfe361 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Fri, 28 Jul 2023 08:41:57 +0200 Subject: [PATCH 48/79] More docs changes --- docs/utilities/batch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/utilities/batch.md b/docs/utilities/batch.md index 4ecb836db..712afdd4a 100644 --- a/docs/utilities/batch.md +++ b/docs/utilities/batch.md @@ -99,7 +99,7 @@ modules that require code-weaving, you will need to configure that also. Batch d } dependencies { - aspect 'software.amazon.lambda:powertools-batch:{{ powertools.version }}' + implementation 'software.amazon.lambda:powertools-batch:{{ powertools.version }}' } ``` ## Getting Started From 033a922cf7e4fa325e8420634f27da56a78165a5 Mon Sep 17 00:00:00 2001 From: Michele Ricciardi Date: Wed, 2 Aug 2023 13:19:17 +0200 Subject: [PATCH 49/79] move ddb template in the right folder --- .../{template-dynamodb.yaml => deploy/ddb-streams/template.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/powertools-examples-batch/{template-dynamodb.yaml => deploy/ddb-streams/template.yaml} (100%) diff --git a/examples/powertools-examples-batch/template-dynamodb.yaml b/examples/powertools-examples-batch/deploy/ddb-streams/template.yaml similarity index 100% rename from examples/powertools-examples-batch/template-dynamodb.yaml rename to examples/powertools-examples-batch/deploy/ddb-streams/template.yaml From 13baa6af3b52acecd0caf170c3507ec5ca251b4b Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Wed, 2 Aug 2023 13:37:21 +0200 Subject: [PATCH 50/79] Changes --- powertools-e2e-tests/handlers/batch/pom.xml | 68 +++++++++++++++ .../lambda/powertools/e2e/Function.java | 84 +++++++++++++++++++ .../lambda/powertools/e2e/model/Product.java | 56 +++++++++++++ .../batch/src/main/resources/log4j2.xml | 16 ++++ powertools-e2e-tests/handlers/pom.xml | 5 ++ .../amazon/lambda/powertools/BatchE2ET.java | 83 ++++++++++++++++++ .../powertools/testutils/Infrastructure.java | 29 +++++++ 7 files changed, 341 insertions(+) create mode 100644 powertools-e2e-tests/handlers/batch/pom.xml create mode 100644 powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java create mode 100644 powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/model/Product.java create mode 100644 powertools-e2e-tests/handlers/batch/src/main/resources/log4j2.xml create mode 100644 powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java diff --git a/powertools-e2e-tests/handlers/batch/pom.xml b/powertools-e2e-tests/handlers/batch/pom.xml new file mode 100644 index 000000000..347b79426 --- /dev/null +++ b/powertools-e2e-tests/handlers/batch/pom.xml @@ -0,0 +1,68 @@ + + 4.0.0 + + + software.amazon.lambda + e2e-test-handlers-parent + 1.0.0 + + + e2e-test-handler-batch + jar + A Lambda function using Powertools for AWS Lambda (Java) batch + + + + software.amazon.lambda + powertools-batch + + + software.amazon.lambda + powertools-logging + + + com.amazonaws + aws-lambda-java-events + + + org.apache.logging.log4j + log4j-slf4j2-impl + + + + + + + dev.aspectj + aspectj-maven-plugin + + ${maven.compiler.source} + ${maven.compiler.target} + ${maven.compiler.target} + + + software.amazon.lambda + powertools-large-messages + + + software.amazon.lambda + powertools-logging + + + + + + + compile + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + diff --git a/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java b/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java new file mode 100644 index 000000000..fde312230 --- /dev/null +++ b/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java @@ -0,0 +1,84 @@ +/* + * Copyright 2023 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package software.amazon.lambda.powertools.e2e; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; +import com.amazonaws.services.lambda.runtime.events.KinesisEvent; +import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; +import com.amazonaws.services.lambda.runtime.events.SQSEvent; +import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import software.amazon.lambda.powertools.batch.BatchMessageHandlerBuilder; +import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; +import software.amazon.lambda.powertools.e2e.model.Product; +import software.amazon.lambda.powertools.logging.Logging; + + +public class Function implements RequestHandler { + + private final static Logger LOGGER = LogManager.getLogger(Function.class); + + private final BatchMessageHandler sqsHandler; + private final BatchMessageHandler kinesisHandler; + private final BatchMessageHandler ddbHandler; + private final String ddbOutputTable; + + public Function() { + sqsHandler = new BatchMessageHandlerBuilder() + .withSqsBatchHandler() + .buildWithMessageHandler(this::processProductMessage, Product.class); + + kinesisHandler = new BatchMessageHandlerBuilder() + .withKinesisBatchHandler() + .buildWithMessageHandler(this::processProductMessage, Product.class); + + ddbHandler = new BatchMessageHandlerBuilder() + .withDynamoDbBatchHandler() + .buildWithRawMessageHandler(this::processDdbMessage); + + this.ddbOutputTable = System.getenv("BATCH_OUTPUT_TABLE"); + } + + @Logging(logEvent = true) + public Object handleRequest(Object input, Context context) { + // TODO - make this work by working out whether or not we can convert the input + // TODO to each of the different types. Doing it with the ENV thing will make it hard with the E2E framework. + String streamType = System.getenv("STREAM_TYPE"); + switch (streamType) { + case "sqs": + return sqsHandler.processBatch((SQSEvent) input, context); + case "kinesis": + return kinesisHandler.processBatch((KinesisEvent) input, context); + case "dynamo": + return ddbHandler.processBatch((DynamodbEvent) input, context); + } + throw new RuntimeException("Whoops! Expected to find sqs/kinesis/dynamo in env var STREAM_TYPE but found " + streamType); + } + + private void processProductMessage(Product p, Context c) { + LOGGER.info("Processing product " + p); + + // TODO - write product details to output table + } + + private void processDdbMessage(DynamodbEvent.DynamodbStreamRecord dynamodbStreamRecord, Context context) { + LOGGER.info("Processing DynamoDB Stream Record" + dynamodbStreamRecord); + + // TODO write DDB change details to batch output table + } +} \ No newline at end of file diff --git a/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/model/Product.java b/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/model/Product.java new file mode 100644 index 000000000..74bb5ff9f --- /dev/null +++ b/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/model/Product.java @@ -0,0 +1,56 @@ +/* + * Copyright 2023 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package software.amazon.lambda.powertools.e2e.model; + +public class Product { + private long id; + + private String name; + + private double price; + + public Product() { + } + + public Product(long id, String name, double price) { + this.id = id; + this.name = name; + this.price = price; + } + + public long getId() { + return id; + } + + public void setId(long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public double getPrice() { + return price; + } + + public void setPrice(double price) { + this.price = price; + } +} diff --git a/powertools-e2e-tests/handlers/batch/src/main/resources/log4j2.xml b/powertools-e2e-tests/handlers/batch/src/main/resources/log4j2.xml new file mode 100644 index 000000000..8925f70b9 --- /dev/null +++ b/powertools-e2e-tests/handlers/batch/src/main/resources/log4j2.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/powertools-e2e-tests/handlers/pom.xml b/powertools-e2e-tests/handlers/pom.xml index 4dd8cbb45..37154f659 100644 --- a/powertools-e2e-tests/handlers/pom.xml +++ b/powertools-e2e-tests/handlers/pom.xml @@ -72,6 +72,11 @@ powertools-large-messages ${lambda.powertools.version} + + software.amazon.lambda + powertools-batch + ${lambda.powertools.version} + com.amazonaws aws-lambda-java-core diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java new file mode 100644 index 000000000..94b3b0552 --- /dev/null +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java @@ -0,0 +1,83 @@ +/* + * Copyright 2023 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package software.amazon.lambda.powertools; + +import static software.amazon.lambda.powertools.testutils.Infrastructure.FUNCTION_NAME_OUTPUT; +import static software.amazon.lambda.powertools.testutils.lambda.LambdaInvoker.invokeFunction; + +import java.time.Year; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import software.amazon.lambda.powertools.testutils.Infrastructure; +import software.amazon.lambda.powertools.testutils.lambda.InvocationResult; + +public class BatchE2ET { + private static Infrastructure infrastructure; + private static String functionName; + + @BeforeAll + @Timeout(value = 5, unit = TimeUnit.MINUTES) + public static void setup() { + String random = UUID.randomUUID().toString().substring(0, 6); + String queueName = "batchqueue" + random; + String kinesisStreamName = "batchstream" + random; + + infrastructure = Infrastructure.builder() + .testName(BatchE2ET.class.getSimpleName()) + .pathToFunction("batch") + .idempotencyTable("idempo" + random) + .queue(queueName) + .kinesisStream(kinesisStreamName) + .build(); + Map outputs = infrastructure.deploy(); + functionName = outputs.get(FUNCTION_NAME_OUTPUT); + } + + @AfterAll + public static void tearDown() { + if (infrastructure != null) { + infrastructure.destroy(); + } + } + + @Test + public void test_ttlNotExpired_sameResult_ttlExpired_differentResult() throws InterruptedException { + // GIVEN + String event = "{\"message\":\"TTL 10sec\"}"; + + // WHEN + // First invocation + InvocationResult result1 = invokeFunction(functionName, event); + + // Second invocation (should get same result) + InvocationResult result2 = invokeFunction(functionName, event); + + Thread.sleep(12000); + + // Third invocation (should get different result) + InvocationResult result3 = invokeFunction(functionName, event); + + // THEN + Assertions.assertThat(result1.getResult()).contains(Year.now().toString()); + Assertions.assertThat(result2.getResult()).isEqualTo(result1.getResult()); + Assertions.assertThat(result3.getResult()).isNotEqualTo(result2.getResult()); + } +} diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java index 996f49bd4..417248c91 100644 --- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java @@ -16,6 +16,7 @@ import static java.util.Collections.singletonList; +import com.amazonaws.services.lambda.runtime.events.KinesisEvent; import com.fasterxml.jackson.databind.JsonNode; import java.io.File; import java.io.IOException; @@ -49,10 +50,14 @@ import software.amazon.awscdk.services.dynamodb.AttributeType; import software.amazon.awscdk.services.dynamodb.BillingMode; import software.amazon.awscdk.services.dynamodb.Table; +import software.amazon.awscdk.services.events.targets.KinesisStream; import software.amazon.awscdk.services.iam.PolicyStatement; +import software.amazon.awscdk.services.kinesis.Stream; +import software.amazon.awscdk.services.kinesis.StreamMode; import software.amazon.awscdk.services.lambda.Code; import software.amazon.awscdk.services.lambda.Function; import software.amazon.awscdk.services.lambda.Tracing; +import software.amazon.awscdk.services.lambda.eventsources.KinesisEventSource; import software.amazon.awscdk.services.lambda.eventsources.SqsEventSource; import software.amazon.awscdk.services.logs.LogGroup; import software.amazon.awscdk.services.logs.RetentionDays; @@ -110,6 +115,7 @@ public class Infrastructure { private final AppConfig appConfig; private final SdkHttpClient httpClient; private final String queue; + private final String kinesisStream; private final String largeMessagesBucket; private String functionName; @@ -126,6 +132,7 @@ private Infrastructure(Builder builder) { this.idempotencyTable = builder.idemPotencyTable; this.appConfig = builder.appConfig; this.queue = builder.queue; + this.kinesisStream = builder.kinesisStream; this.largeMessagesBucket = builder.largeMessagesBucket; this.app = new App(); @@ -287,6 +294,22 @@ private Stack createStackWithLambda() { .build(); createTableForAsyncTests = true; } + if (!StringUtils.isEmpty(kinesisStream)) { + Stream stream = Stream.Builder + .create(stack, "KinesisStream") + .streamMode(StreamMode.ON_DEMAND) + .streamName(kinesisStream) + .shardCount(1) + .build(); + + stream.grantRead(function); + KinesisEventSource kinesisEventSource = KinesisEventSource.Builder.create(stream).enabled(true).batchSize(3).build(); + function.addEventSource(kinesisEventSource); + CfnOutput.Builder + .create(stack, "KinesisStreamName") + .value(stream.getStreamName()) + .build(); + } if (!StringUtils.isEmpty(largeMessagesBucket)) { Bucket offloadBucket = Bucket.Builder @@ -451,6 +474,7 @@ public static class Builder { private Map environmentVariables = new HashMap<>(); private String idemPotencyTable; private String queue; + private String kinesisStream; private Builder() { getJavaRuntime(); @@ -526,6 +550,11 @@ public Builder queue(String queue) { return this; } + public Builder kinesisStream(String stream) { + this.kinesisStream = stream; + return this; + } + public Builder largeMessagesBucket(String largeMessagesBucket) { this.largeMessagesBucket = largeMessagesBucket; return this; From 00fe0b05e200431c10c37139a79e188284c4a85c Mon Sep 17 00:00:00 2001 From: Michele Ricciardi Date: Wed, 2 Aug 2023 15:02:12 +0200 Subject: [PATCH 51/79] add items updates and deletions to ddb example --- .../deploy/ddb-streams/template.yaml | 52 +++++++------- .../dynamo/DynamoDBStreamBatchHandler.java | 7 +- .../org/demo/batch/dynamo/DynamoDBWriter.java | 68 +++++++++++++++---- 3 files changed, 84 insertions(+), 43 deletions(-) diff --git a/examples/powertools-examples-batch/deploy/ddb-streams/template.yaml b/examples/powertools-examples-batch/deploy/ddb-streams/template.yaml index 54ca26c2c..91f8799c4 100644 --- a/examples/powertools-examples-batch/deploy/ddb-streams/template.yaml +++ b/examples/powertools-examples-batch/deploy/ddb-streams/template.yaml @@ -1,15 +1,12 @@ AWSTemplateFormatVersion: '2010-09-09' Transform: AWS::Serverless-2016-10-31 Description: > - powertools-examples-batch-ddb + DynamoDB Streams batch processing demo - Sample SAM Template for powertools-examples-batch-ddb - -# More info about Globals: https://github.com/awslabs/serverless-application-model/blob/master/docs/globals.rst Globals: Function: Timeout: 20 - Runtime: java17 + Runtime: java11 MemorySize: 512 Tracing: Active Architectures: @@ -36,37 +33,42 @@ Resources: StreamSpecification: StreamViewType: NEW_IMAGE - DemoDynamoDBStreamsConsumerFunction: - Type: AWS::Serverless::Function - Properties: - CodeUri: . - Handler: org.demo.batch.dynamo.DynamoDBStreamBatchHandler::handleRequest - Policies: AWSLambdaDynamoDBExecutionRole - Events: - Stream: - Type: DynamoDB - Properties: - Stream: !GetAtt DynamoDBTable.StreamArn - BatchSize: 100 - StartingPosition: TRIM_HORIZON DemoDynamoDBWriter: - Type: AWS::Serverless::Function # More info about Function Resource: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#awsserverlessfunction + Type: AWS::Serverless::Function Properties: - CodeUri: . + CodeUri: ../.. Handler: org.demo.batch.dynamo.DynamoDBWriter::handleRequest + Environment: + Variables: + POWERTOOLS_SERVICE_NAME: ddbstreams-demo + TABLE_NAME: !Ref DynamoDBTable Policies: - DynamoDBCrudPolicy: TableName: !Ref DynamoDBTable - Environment: # More info about Env Vars: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#environment-object - Variables: - TABLE_NAME: !Ref DynamoDBTable Events: CWSchedule: Type: Schedule Properties: - Schedule: 'rate(5 minutes)' + Schedule: 'rate(1 minute)' Name: !Join [ "-", [ "ddb-writer-schedule", !Select [ 0, !Split [ -, !Select [ 2, !Split [ /, !Ref AWS::StackId ] ] ] ] ] ] - Description: Produce message to SQS via a Lambda function + Description: Write records to DynamoDB via a Lambda function Enabled: true + DemoDynamoDBStreamsConsumerFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: ../.. + Handler: org.demo.batch.dynamo.DynamoDBStreamBatchHandler::handleRequest + Environment: + Variables: + POWERTOOLS_SERVICE_NAME: ddbstreams-batch-demo + Policies: AWSLambdaDynamoDBExecutionRole + Events: + Stream: + Type: DynamoDB + Properties: + Stream: !GetAtt DynamoDBTable.StreamArn + BatchSize: 100 + StartingPosition: TRIM_HORIZON + diff --git a/examples/powertools-examples-batch/src/main/java/org/demo/batch/dynamo/DynamoDBStreamBatchHandler.java b/examples/powertools-examples-batch/src/main/java/org/demo/batch/dynamo/DynamoDBStreamBatchHandler.java index d8b6f0bbd..988c49e86 100644 --- a/examples/powertools-examples-batch/src/main/java/org/demo/batch/dynamo/DynamoDBStreamBatchHandler.java +++ b/examples/powertools-examples-batch/src/main/java/org/demo/batch/dynamo/DynamoDBStreamBatchHandler.java @@ -20,14 +20,13 @@ public DynamoDBStreamBatchHandler() { .buildWithRawMessageHandler(this::processMessage); } - private void processMessage(DynamodbEvent.DynamodbStreamRecord dynamodbStreamRecord, Context context) { - LOGGER.info("Processing DynamoDB Stream Record" + dynamodbStreamRecord); - } - @Override public StreamsEventResponse handleRequest(DynamodbEvent ddbEvent, Context context) { return handler.processBatch(ddbEvent, context); } + private void processMessage(DynamodbEvent.DynamodbStreamRecord dynamodbStreamRecord, Context context) { + LOGGER.info("Processing DynamoDB Stream Record" + dynamodbStreamRecord); + } } diff --git a/examples/powertools-examples-batch/src/main/java/org/demo/batch/dynamo/DynamoDBWriter.java b/examples/powertools-examples-batch/src/main/java/org/demo/batch/dynamo/DynamoDBWriter.java index 7178ff885..953ba8f23 100644 --- a/examples/powertools-examples-batch/src/main/java/org/demo/batch/dynamo/DynamoDBWriter.java +++ b/examples/powertools-examples-batch/src/main/java/org/demo/batch/dynamo/DynamoDBWriter.java @@ -4,7 +4,9 @@ import com.amazonaws.services.lambda.runtime.RequestHandler; import com.amazonaws.services.lambda.runtime.events.ScheduledEvent; import java.security.SecureRandom; +import java.util.List; import java.util.UUID; +import java.util.stream.Collectors; import java.util.stream.IntStream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -42,27 +44,65 @@ public String handleRequest(ScheduledEvent scheduledEvent, Context context) { LOGGER.info("handleRequest"); - WriteBatch.Builder productBuilder = WriteBatch.builder(DdbProduct.class) + List products = createProducts(tableName); + List updatedProducts = updateProducts(tableName, products); + deleteProducts(tableName, updatedProducts); + + return "Success"; + } + + private void deleteProducts(String tableName, List updatedProducts) { + WriteBatch.Builder productDeleteBuilder = WriteBatch.builder(DdbProduct.class) .mappedTableResource(enhancedClient.table(tableName, TableSchema.fromBean(DdbProduct.class))); - IntStream.range(0, 5).forEach(i -> { - String id = UUID.randomUUID().toString(); + updatedProducts.forEach(productDeleteBuilder::addDeleteItem); - float price = random.nextFloat(); - DdbProduct product = new DdbProduct(id, "product-" + id, price); - productBuilder.addPutItem(product); - }); + BatchWriteResult batchDeleteResult = enhancedClient + .batchWriteItem(BatchWriteItemEnhancedRequest.builder().writeBatches( + productDeleteBuilder.build()) + .build()); + LOGGER.info("Deleted batch of objects from DynamoDB: {}", batchDeleteResult); + } - BatchWriteItemEnhancedRequest batchWriteItemEnhancedRequest = - BatchWriteItemEnhancedRequest.builder().writeBatches( - productBuilder.build()) - .build(); + private List updateProducts(String tableName, List products) { + WriteBatch.Builder productUpdateBuilder = WriteBatch.builder(DdbProduct.class) + .mappedTableResource(enhancedClient.table(tableName, TableSchema.fromBean(DdbProduct.class))); - BatchWriteResult batchWriteResult = enhancedClient.batchWriteItem(batchWriteItemEnhancedRequest); + List updatedProducts = products.stream().map(product -> { + // Update the price of the product and add it to the batch + LOGGER.info("Updating product: {}", product); + float price = random.nextFloat(); + DdbProduct updatedProduct = new DdbProduct(product.getId(), "updated-product-" + product.getId(), price); + productUpdateBuilder.addPutItem(updatedProduct); + return updatedProduct; + }).collect(Collectors.toList()); + + BatchWriteResult batchUpdateResult = enhancedClient + .batchWriteItem(BatchWriteItemEnhancedRequest.builder().writeBatches( + productUpdateBuilder.build()) + .build()); + LOGGER.info("Updated batch of objects to DynamoDB: {}", batchUpdateResult); + return updatedProducts; + } + public List createProducts(String tableName) { + WriteBatch.Builder productBuilder = WriteBatch.builder(DdbProduct.class) + .mappedTableResource(enhancedClient.table(tableName, TableSchema.fromBean(DdbProduct.class))); - LOGGER.info("Wrote batch of messages to DynamoDB: {}", batchWriteResult); + List ddbProductStream = IntStream.range(0, 5).mapToObj(i -> { + String id = UUID.randomUUID().toString(); + float price = random.nextFloat(); + // Create a new product and add it to the batch + final DdbProduct product = new DdbProduct(id, "product-" + id, price); + productBuilder.addPutItem(product); + return product; + }).collect(Collectors.toList()); - return "Success"; + BatchWriteResult batchWriteResult = enhancedClient + .batchWriteItem(BatchWriteItemEnhancedRequest.builder().writeBatches( + productBuilder.build()) + .build()); + LOGGER.info("Wrote batch of objects to DynamoDB: {}", batchWriteResult); + return ddbProductStream; } } From 1f65cebf9fc8c538b824ea6a52447bc803c01d34 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Wed, 2 Aug 2023 15:05:00 +0200 Subject: [PATCH 52/79] Will it blend? --- .../amazon/lambda/powertools/BatchE2ET.java | 147 ++++++++++++++++-- 1 file changed, 133 insertions(+), 14 deletions(-) diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java index 94b3b0552..1bf463de4 100644 --- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java @@ -17,21 +17,53 @@ import static software.amazon.lambda.powertools.testutils.Infrastructure.FUNCTION_NAME_OUTPUT; import static software.amazon.lambda.powertools.testutils.lambda.LambdaInvoker.invokeFunction; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.ObjectMapper; import java.time.Year; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; +import software.amazon.awssdk.http.SdkHttpClient; +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.DynamoDbClientBuilder; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; +import software.amazon.awssdk.services.dynamodb.model.DeleteItemRequest; +import software.amazon.awssdk.services.dynamodb.model.ScanRequest; +import software.amazon.awssdk.services.dynamodb.model.ScanResponse; +import software.amazon.awssdk.services.sqs.SqsClient; +import software.amazon.awssdk.services.sqs.model.SendMessageBatchRequest; +import software.amazon.awssdk.services.sqs.model.SendMessageBatchRequestEntry; +import software.amazon.awssdk.services.sqs.model.SendMessageRequest; import software.amazon.lambda.powertools.testutils.Infrastructure; import software.amazon.lambda.powertools.testutils.lambda.InvocationResult; +import software.amazon.lambda.powertools.utilities.JsonConfig; +import static org.assertj.core.api.Assertions.assertThat; public class BatchE2ET { private static Infrastructure infrastructure; private static String functionName; + private static final SdkHttpClient httpClient = UrlConnectionHttpClient.builder().build(); + private static final Region region = Region.of(System.getProperty("AWS_DEFAULT_REGION", "eu-west-1")); + private static String queueUrl; + private static ObjectMapper objectMapper; + private static String outputTable; + private static DynamoDbClient ddbClient; + private static SqsClient sqsClient; + private final List testProducts; @BeforeAll @Timeout(value = 5, unit = TimeUnit.MINUTES) @@ -39,6 +71,7 @@ public static void setup() { String random = UUID.randomUUID().toString().substring(0, 6); String queueName = "batchqueue" + random; String kinesisStreamName = "batchstream" + random; + objectMapper = JsonConfig.get().getObjectMapper(); infrastructure = Infrastructure.builder() .testName(BatchE2ET.class.getSimpleName()) @@ -47,8 +80,47 @@ public static void setup() { .queue(queueName) .kinesisStream(kinesisStreamName) .build(); + Map outputs = infrastructure.deploy(); functionName = outputs.get(FUNCTION_NAME_OUTPUT); + queueUrl = outputs.get("QueueURL"); + kinesisStreamName = outputs.get("KinesisStreamName"); + outputTable = outputs.get("TableNameForAsyncTests"); + + ddbClient = DynamoDbClient.builder() + .region(region) + .httpClient(httpClient) + .build(); + + // GIVEN + sqsClient = SqsClient.builder() + .httpClient(httpClient) + .region(region) + .build(); + + } + + @AfterEach + public void cleanUpTest() { + // Delete everything in the output table + ScanResponse items = ddbClient.scan(ScanRequest.builder() + .tableName(outputTable) + .build()); + + for (Map key :items.items()) { + ddbClient.deleteItem(DeleteItemRequest.builder() + .tableName(outputTable) + .key(key) + .build()); + } + } + + public BatchE2ET() { + testProducts = Arrays.asList( + new Product(1, "product1", 1.23), + new Product(2, "product2", 4.56), + new Product(3, "product3", 6.78) + ); } @AfterAll @@ -59,25 +131,72 @@ public static void tearDown() { } @Test - public void test_ttlNotExpired_sameResult_ttlExpired_differentResult() throws InterruptedException { - // GIVEN - String event = "{\"message\":\"TTL 10sec\"}"; + public void sqsBatchProcessingSucceeds() throws InterruptedException { + List entries = testProducts.stream() + .map(p -> { + try { + return SendMessageBatchRequestEntry.builder() + .id(p.getName()) + .messageBody(objectMapper.writeValueAsString(p)) + .build(); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toList()); // WHEN - // First invocation - InvocationResult result1 = invokeFunction(functionName, event); + sqsClient.sendMessageBatch(SendMessageBatchRequest.builder() + .entries(entries) + .queueUrl(queueUrl) + .build()); + Thread.sleep(30000); // wait for function to be executed + + // THEN + ScanResponse items = ddbClient.scan(ScanRequest.builder() + .tableName(outputTable) + .build()); + validateAllItemsHandled(items); + } - // Second invocation (should get same result) - InvocationResult result2 = invokeFunction(functionName, event); + private void validateAllItemsHandled(ScanResponse items) { + for (Product p : testProducts) { + boolean foundIt = false; + for (Map a : items.items()) { + if (a.get("id").equals(Long.toString(p.id))) { + foundIt = true; + } + } + assertThat(foundIt).isTrue(); + } + } - Thread.sleep(12000); + class Product { + private long id; - // Third invocation (should get different result) - InvocationResult result3 = invokeFunction(functionName, event); + private String name; - // THEN - Assertions.assertThat(result1.getResult()).contains(Year.now().toString()); - Assertions.assertThat(result2.getResult()).isEqualTo(result1.getResult()); - Assertions.assertThat(result3.getResult()).isNotEqualTo(result2.getResult()); + private double price; + + public Product() { + } + + public Product(long id, String name, double price) { + this.id = id; + this.name = name; + this.price = price; + } + + public long getId() { + return id; + } + + public String getName() { + return name; + } + + public double getPrice() { + return price; + } } } From bb74fe46a280d88a7f40c5ad36c675c1dddb7cd4 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Wed, 2 Aug 2023 15:13:17 +0200 Subject: [PATCH 53/79] More changes --- powertools-e2e-tests/handlers/batch/pom.xml | 4 ---- .../lambda/powertools/testutils/Infrastructure.java | 9 +++++++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/powertools-e2e-tests/handlers/batch/pom.xml b/powertools-e2e-tests/handlers/batch/pom.xml index 347b79426..1257e9bde 100644 --- a/powertools-e2e-tests/handlers/batch/pom.xml +++ b/powertools-e2e-tests/handlers/batch/pom.xml @@ -41,10 +41,6 @@ ${maven.compiler.target} ${maven.compiler.target} - - software.amazon.lambda - powertools-large-messages - software.amazon.lambda powertools-logging diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java index 417248c91..acba3818b 100644 --- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java @@ -56,6 +56,7 @@ import software.amazon.awscdk.services.kinesis.StreamMode; import software.amazon.awscdk.services.lambda.Code; import software.amazon.awscdk.services.lambda.Function; +import software.amazon.awscdk.services.lambda.StartingPosition; import software.amazon.awscdk.services.lambda.Tracing; import software.amazon.awscdk.services.lambda.eventsources.KinesisEventSource; import software.amazon.awscdk.services.lambda.eventsources.SqsEventSource; @@ -299,11 +300,15 @@ private Stack createStackWithLambda() { .create(stack, "KinesisStream") .streamMode(StreamMode.ON_DEMAND) .streamName(kinesisStream) - .shardCount(1) .build(); stream.grantRead(function); - KinesisEventSource kinesisEventSource = KinesisEventSource.Builder.create(stream).enabled(true).batchSize(3).build(); + KinesisEventSource kinesisEventSource = KinesisEventSource.Builder + .create(stream) + .enabled(true) + .batchSize(3) + .startingPosition(StartingPosition.LATEST) + .build(); function.addEventSource(kinesisEventSource); CfnOutput.Builder .create(stack, "KinesisStreamName") From 76e51a6e8c4a0fb7380da96c1e5bdc09ead6fda6 Mon Sep 17 00:00:00 2001 From: Michele Ricciardi Date: Wed, 2 Aug 2023 15:13:36 +0200 Subject: [PATCH 54/79] e2e test handler --- .../lambda/powertools/e2e/Function.java | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java b/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java index fde312230..bdff15df4 100644 --- a/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java +++ b/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java @@ -51,23 +51,21 @@ public Function() { .withDynamoDbBatchHandler() .buildWithRawMessageHandler(this::processDdbMessage); - this.ddbOutputTable = System.getenv("BATCH_OUTPUT_TABLE"); + this.ddbOutputTable = System.getenv("TABLE_FOR_ASYNC_TESTS"); } @Logging(logEvent = true) public Object handleRequest(Object input, Context context) { - // TODO - make this work by working out whether or not we can convert the input - // TODO to each of the different types. Doing it with the ENV thing will make it hard with the E2E framework. - String streamType = System.getenv("STREAM_TYPE"); - switch (streamType) { - case "sqs": - return sqsHandler.processBatch((SQSEvent) input, context); - case "kinesis": - return kinesisHandler.processBatch((KinesisEvent) input, context); - case "dynamo": - return ddbHandler.processBatch((DynamodbEvent) input, context); + + if(input instanceof KinesisEvent){ + return kinesisHandler.processBatch((KinesisEvent) input, context); + } else if (input instanceof SQSEvent) { + return sqsHandler.processBatch((SQSEvent) input, context); + } else if (input instanceof DynamodbEvent) { + return ddbHandler.processBatch((DynamodbEvent) input, context); + } else { + throw new RuntimeException("Whoops! Expected to find sqs/kinesis/dynamo in input but found " + input.getClass()); } - throw new RuntimeException("Whoops! Expected to find sqs/kinesis/dynamo in env var STREAM_TYPE but found " + streamType); } private void processProductMessage(Product p, Context c) { From 0e4b018083a10dd16fbe5fdd73864def9fffea57 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Wed, 2 Aug 2023 15:25:29 +0200 Subject: [PATCH 55/79] Try work for SQS only --- powertools-e2e-tests/handlers/batch/pom.xml | 4 ++ .../lambda/powertools/e2e/Function.java | 42 ++++++++++++------- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/powertools-e2e-tests/handlers/batch/pom.xml b/powertools-e2e-tests/handlers/batch/pom.xml index 1257e9bde..ff803aac6 100644 --- a/powertools-e2e-tests/handlers/batch/pom.xml +++ b/powertools-e2e-tests/handlers/batch/pom.xml @@ -29,6 +29,10 @@ org.apache.logging.log4j log4j-slf4j2-impl + + software.amazon.awssdk + dynamodb + diff --git a/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java b/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java index fde312230..1c8702671 100644 --- a/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java +++ b/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java @@ -21,15 +21,19 @@ import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; import com.amazonaws.services.lambda.runtime.events.SQSEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; +import java.util.HashMap; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; +import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; import software.amazon.lambda.powertools.batch.BatchMessageHandlerBuilder; import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; import software.amazon.lambda.powertools.e2e.model.Product; import software.amazon.lambda.powertools.logging.Logging; -public class Function implements RequestHandler { +public class Function implements RequestHandler { private final static Logger LOGGER = LogManager.getLogger(Function.class); @@ -37,6 +41,7 @@ public class Function implements RequestHandler { private final BatchMessageHandler kinesisHandler; private final BatchMessageHandler ddbHandler; private final String ddbOutputTable; + private DynamoDbClient ddbClient; public Function() { sqsHandler = new BatchMessageHandlerBuilder() @@ -55,25 +60,34 @@ public Function() { } @Logging(logEvent = true) - public Object handleRequest(Object input, Context context) { - // TODO - make this work by working out whether or not we can convert the input - // TODO to each of the different types. Doing it with the ENV thing will make it hard with the E2E framework. - String streamType = System.getenv("STREAM_TYPE"); - switch (streamType) { - case "sqs": - return sqsHandler.processBatch((SQSEvent) input, context); - case "kinesis": - return kinesisHandler.processBatch((KinesisEvent) input, context); - case "dynamo": - return ddbHandler.processBatch((DynamodbEvent) input, context); - } - throw new RuntimeException("Whoops! Expected to find sqs/kinesis/dynamo in env var STREAM_TYPE but found " + streamType); + public Object handleRequest(SQSEvent input, Context context) { + // TODO - this should work for all the different types, by working + // TODO out what we can deserialize to. Making it work _just for_ SQS for + // TODO now to test the E2E framework. + return sqsHandler.processBatch(input, context); } private void processProductMessage(Product p, Context c) { LOGGER.info("Processing product " + p); // TODO - write product details to output table + ddbClient = DynamoDbClient.builder() + .build(); + ddbClient.putItem(PutItemRequest.builder() + .tableName(ddbOutputTable) + .item(new HashMap() { + { + put("id", AttributeValue.builder() + .s(Long.toString(p.getId())) + .build()); + put("name", AttributeValue.builder() + .s(p.getName()) + .build()); + put("price", AttributeValue.builder() + .n(Double.toString(p.getPrice())) + .build()); + }}) + .build()); } private void processDdbMessage(DynamodbEvent.DynamodbStreamRecord dynamodbStreamRecord, Context context) { From c641d4b2c627463092afdd57b08138f58dbf0d43 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Wed, 2 Aug 2023 15:46:08 +0200 Subject: [PATCH 56/79] More greatness --- .../lambda/powertools/e2e/Function.java | 21 ++++++++----------- .../amazon/lambda/powertools/BatchE2ET.java | 3 ++- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java b/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java index 655275193..2017b7362 100644 --- a/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java +++ b/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java @@ -33,7 +33,7 @@ import software.amazon.lambda.powertools.logging.Logging; -public class Function implements RequestHandler { +public class Function implements RequestHandler { private final static Logger LOGGER = LogManager.getLogger(Function.class); @@ -60,17 +60,11 @@ public Function() { } @Logging(logEvent = true) - public Object handleRequest(Object input, Context context) { - - if(input instanceof KinesisEvent){ - return kinesisHandler.processBatch((KinesisEvent) input, context); - } else if (input instanceof SQSEvent) { - return sqsHandler.processBatch((SQSEvent) input, context); - } else if (input instanceof DynamodbEvent) { - return ddbHandler.processBatch((DynamodbEvent) input, context); - } else { - throw new RuntimeException("Whoops! Expected to find sqs/kinesis/dynamo in input but found " + input.getClass()); - } + public Object handleRequest(SQSEvent input, Context context) { + // TODO - this should work for all the different types, by working + // TODO out what we can deserialize to. Making it work _just for_ SQS for + // TODO now to test the E2E framework. + return sqsHandler.processBatch(input, context); } private void processProductMessage(Product p, Context c) { @@ -83,6 +77,9 @@ private void processProductMessage(Product p, Context c) { .tableName(ddbOutputTable) .item(new HashMap() { { + put("functionName", AttributeValue.builder() + .s(c.getFunctionName()) + .build()); put("id", AttributeValue.builder() .s(Long.toString(p.getId())) .build()); diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java index 1bf463de4..9331e92bf 100644 --- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java @@ -125,8 +125,9 @@ public BatchE2ET() { @AfterAll public static void tearDown() { + // TODO bring this back after testing if (infrastructure != null) { - infrastructure.destroy(); + // infrastructure.destroy(); } } From 0dbf1a38ae947feffe6f71254e0577b3b5dc2f23 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Wed, 2 Aug 2023 15:59:32 +0200 Subject: [PATCH 57/79] Almost good --- .../test/java/software/amazon/lambda/powertools/BatchE2ET.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java index 9331e92bf..a60af06e7 100644 --- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java @@ -164,7 +164,7 @@ private void validateAllItemsHandled(ScanResponse items) { for (Product p : testProducts) { boolean foundIt = false; for (Map a : items.items()) { - if (a.get("id").equals(Long.toString(p.id))) { + if (a.get("id").s().equals(Long.toString(p.id))) { foundIt = true; } } From 7834a6ee8fc1861571d684c85f1a2149907ddef4 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Wed, 2 Aug 2023 16:09:32 +0200 Subject: [PATCH 58/79] SQS works --- .../amazon/lambda/powertools/BatchE2ET.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java index a60af06e7..a56b8ce84 100644 --- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java @@ -23,6 +23,7 @@ import java.time.Year; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; @@ -107,7 +108,17 @@ public void cleanUpTest() { .tableName(outputTable) .build()); - for (Map key :items.items()) { + for (Map item :items.items()) { + HashMap key = new HashMap() { + { + put("functionName", AttributeValue.builder() + .s(item.get("functionName").s()) + .build()); + put("id", AttributeValue.builder() + .s(item.get("id").s()) + .build()); + }}; + ddbClient.deleteItem(DeleteItemRequest.builder() .tableName(outputTable) .key(key) From 062870f9a846899ac2f6c5a63311c3981ef2e9fa Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Wed, 2 Aug 2023 16:26:16 +0200 Subject: [PATCH 59/79] Also kinesis e2e --- powertools-e2e-tests/pom.xml | 7 ++- .../amazon/lambda/powertools/BatchE2ET.java | 43 ++++++++++++++++++- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/powertools-e2e-tests/pom.xml b/powertools-e2e-tests/pom.xml index 500b7f30a..cc51fac0e 100644 --- a/powertools-e2e-tests/pom.xml +++ b/powertools-e2e-tests/pom.xml @@ -57,7 +57,12 @@ ${aws.sdk.version} test - + + software.amazon.awssdk + kinesis + ${aws.sdk.version} + test + software.amazon.awssdk cloudwatch diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java index a56b8ce84..1ccc0acbf 100644 --- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java @@ -36,6 +36,7 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; +import software.amazon.awssdk.core.SdkBytes; import software.amazon.awssdk.http.SdkHttpClient; import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; import software.amazon.awssdk.regions.Region; @@ -45,6 +46,9 @@ import software.amazon.awssdk.services.dynamodb.model.DeleteItemRequest; import software.amazon.awssdk.services.dynamodb.model.ScanRequest; import software.amazon.awssdk.services.dynamodb.model.ScanResponse; +import software.amazon.awssdk.services.kinesis.KinesisClient; +import software.amazon.awssdk.services.kinesis.model.PutRecordsRequest; +import software.amazon.awssdk.services.kinesis.model.PutRecordsRequestEntry; import software.amazon.awssdk.services.sqs.SqsClient; import software.amazon.awssdk.services.sqs.model.SendMessageBatchRequest; import software.amazon.awssdk.services.sqs.model.SendMessageBatchRequestEntry; @@ -60,10 +64,13 @@ public class BatchE2ET { private static final SdkHttpClient httpClient = UrlConnectionHttpClient.builder().build(); private static final Region region = Region.of(System.getProperty("AWS_DEFAULT_REGION", "eu-west-1")); private static String queueUrl; + private static String kinesisStreamName; + private static ObjectMapper objectMapper; private static String outputTable; private static DynamoDbClient ddbClient; private static SqsClient sqsClient; + private static KinesisClient kinesisClient; private final List testProducts; @BeforeAll @@ -71,7 +78,7 @@ public class BatchE2ET { public static void setup() { String random = UUID.randomUUID().toString().substring(0, 6); String queueName = "batchqueue" + random; - String kinesisStreamName = "batchstream" + random; + kinesisStreamName = "batchstream" + random; objectMapper = JsonConfig.get().getObjectMapper(); infrastructure = Infrastructure.builder() @@ -98,7 +105,10 @@ public static void setup() { .httpClient(httpClient) .region(region) .build(); - + kinesisClient = KinesisClient.builder() + .httpClient(httpClient) + .region(region) + .build(); } @AfterEach @@ -171,6 +181,35 @@ public void sqsBatchProcessingSucceeds() throws InterruptedException { validateAllItemsHandled(items); } + @Test + public void kinesisBatchProcessingSucceeds() throws InterruptedException { + List entries = testProducts.stream() + .map(p -> { + try { + return PutRecordsRequestEntry.builder() + .partitionKey("1") + .data(SdkBytes.fromUtf8String(objectMapper.writeValueAsString(p))) + .build(); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toList()); + + // WHEN + kinesisClient.putRecords(PutRecordsRequest.builder() + .streamName(kinesisStreamName) + .records(entries) + .build()); + Thread.sleep(30000); // wait for function to be executed + + // THEN + ScanResponse items = ddbClient.scan(ScanRequest.builder() + .tableName(outputTable) + .build()); + validateAllItemsHandled(items); + } + private void validateAllItemsHandled(ScanResponse items) { for (Product p : testProducts) { boolean foundIt = false; From 5548dc01d36c9f232381fd5e3fcf7075613971da Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Wed, 2 Aug 2023 16:54:30 +0200 Subject: [PATCH 60/79] Lets try doing it with streams --- .../lambda/powertools/e2e/Function.java | 76 ++++++++++++++++--- 1 file changed, 67 insertions(+), 9 deletions(-) diff --git a/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java b/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java index 2017b7362..5f9025b83 100644 --- a/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java +++ b/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java @@ -16,12 +16,26 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.amazonaws.services.lambda.runtime.RequestStreamHandler; import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; import com.amazonaws.services.lambda.runtime.events.KinesisEvent; import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; import com.amazonaws.services.lambda.runtime.events.SQSEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; import java.util.HashMap; +import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; @@ -31,9 +45,10 @@ import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; import software.amazon.lambda.powertools.e2e.model.Product; import software.amazon.lambda.powertools.logging.Logging; +import software.amazon.lambda.powertools.utilities.JsonConfig; -public class Function implements RequestHandler { +public class Function implements RequestStreamHandler { private final static Logger LOGGER = LogManager.getLogger(Function.class); @@ -59,13 +74,6 @@ public Function() { this.ddbOutputTable = System.getenv("TABLE_FOR_ASYNC_TESTS"); } - @Logging(logEvent = true) - public Object handleRequest(SQSEvent input, Context context) { - // TODO - this should work for all the different types, by working - // TODO out what we can deserialize to. Making it work _just for_ SQS for - // TODO now to test the E2E framework. - return sqsHandler.processBatch(input, context); - } private void processProductMessage(Product p, Context c) { LOGGER.info("Processing product " + p); @@ -98,4 +106,54 @@ private void processDdbMessage(DynamodbEvent.DynamodbStreamRecord dynamodbStream // TODO write DDB change details to batch output table } -} \ No newline at end of file + + @Override + @Logging(logEvent = true) + public void handleRequest(InputStream inputStream, OutputStream outputStream, Context context) throws IOException { + String input = new BufferedReader( + new InputStreamReader(inputStream, StandardCharsets.UTF_8)) + .lines() + .collect(Collectors.joining("\n")); + + String ret = createResult(input, context); + BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(outputStream)); + writer.write(ret); + writer.flush(); + } + + private String createResult(String input, Context context) { + + // TODO - this should work for all the different types, by working + // TODO out what we can deserialize to. Making it work _just for_ SQS for + // TODO now to test the E2E framework. + ObjectMapper obj = JsonConfig.get().getObjectMapper(); + + try { + SQSEvent event = obj.readValue(input, SQSEvent.class); + SQSBatchResponse result = sqsHandler.processBatch(event, context); + return obj.writeValueAsString(result); + } catch (Exception e) { + LOGGER.warn("Wasn't SQS"); + } + + try { + KinesisEvent event = obj.readValue(input, KinesisEvent.class); + StreamsEventResponse result = kinesisHandler.processBatch(event, context); + return obj.writeValueAsString(result); + } catch (Exception e) { + LOGGER.warn("Wasn't Kinesis"); + } + + try { + DynamodbEvent event = obj.readValue(input, DynamodbEvent.class); + StreamsEventResponse result = ddbHandler.processBatch(event, context); + return obj.writeValueAsString(result); + } catch (Exception e) { + LOGGER.warn("Wasn't DynamoDB"); + } + + throw new RuntimeException("Couldn't deserialize an event we understood. Giving up!"); + + } + +} From 3cf034aa0408b473f3e25883aa05ecea7fe82a1a Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Wed, 2 Aug 2023 17:02:50 +0200 Subject: [PATCH 61/79] Try make it work with streams --- .../software/amazon/lambda/powertools/e2e/Function.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java b/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java index 5f9025b83..cfdcab092 100644 --- a/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java +++ b/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java @@ -128,12 +128,14 @@ private String createResult(String input, Context context) { // TODO now to test the E2E framework. ObjectMapper obj = JsonConfig.get().getObjectMapper(); + LOGGER.info(input); + try { SQSEvent event = obj.readValue(input, SQSEvent.class); SQSBatchResponse result = sqsHandler.processBatch(event, context); return obj.writeValueAsString(result); } catch (Exception e) { - LOGGER.warn("Wasn't SQS"); + LOGGER.warn("Wasn't SQS", e); } try { @@ -141,7 +143,7 @@ private String createResult(String input, Context context) { StreamsEventResponse result = kinesisHandler.processBatch(event, context); return obj.writeValueAsString(result); } catch (Exception e) { - LOGGER.warn("Wasn't Kinesis"); + LOGGER.warn("Wasn't Kinesis", e); } try { From 55cbbf42755836a8c220efffacecf3c93cbaadab Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Thu, 3 Aug 2023 08:47:02 +0200 Subject: [PATCH 62/79] Streams? --- powertools-e2e-tests/handlers/batch/pom.xml | 4 ++ .../lambda/powertools/e2e/Function.java | 51 ++++++++----------- powertools-e2e-tests/handlers/pom.xml | 6 +++ 3 files changed, 32 insertions(+), 29 deletions(-) diff --git a/powertools-e2e-tests/handlers/batch/pom.xml b/powertools-e2e-tests/handlers/batch/pom.xml index ff803aac6..995121e2a 100644 --- a/powertools-e2e-tests/handlers/batch/pom.xml +++ b/powertools-e2e-tests/handlers/batch/pom.xml @@ -25,6 +25,10 @@ com.amazonaws aws-lambda-java-events + + com.amazonaws + aws-lambda-java-serialization + org.apache.logging.log4j log4j-slf4j2-impl diff --git a/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java b/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java index cfdcab092..33e994d51 100644 --- a/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java +++ b/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java @@ -14,18 +14,18 @@ package software.amazon.lambda.powertools.e2e; +import com.amazonaws.lambda.thirdparty.com.fasterxml.jackson.databind.ObjectMapper; import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.RequestHandler; -import com.amazonaws.services.lambda.runtime.RequestStreamHandler; import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; import com.amazonaws.services.lambda.runtime.events.KinesisEvent; import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; import com.amazonaws.services.lambda.runtime.events.SQSEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; +import com.amazonaws.services.lambda.runtime.serialization.factories.JacksonFactory; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.IOException; @@ -38,6 +38,7 @@ import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.util.IOUtils; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; @@ -48,7 +49,7 @@ import software.amazon.lambda.powertools.utilities.JsonConfig; -public class Function implements RequestStreamHandler { +public class Function implements RequestHandler { private final static Logger LOGGER = LogManager.getLogger(Function.class); @@ -74,7 +75,6 @@ public Function() { this.ddbOutputTable = System.getenv("TABLE_FOR_ASYNC_TESTS"); } - private void processProductMessage(Product p, Context c) { LOGGER.info("Processing product " + p); @@ -107,49 +107,32 @@ private void processDdbMessage(DynamodbEvent.DynamodbStreamRecord dynamodbStream // TODO write DDB change details to batch output table } - @Override - @Logging(logEvent = true) - public void handleRequest(InputStream inputStream, OutputStream outputStream, Context context) throws IOException { - String input = new BufferedReader( - new InputStreamReader(inputStream, StandardCharsets.UTF_8)) - .lines() - .collect(Collectors.joining("\n")); - - String ret = createResult(input, context); - BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(outputStream)); - writer.write(ret); - writer.flush(); - } - - private String createResult(String input, Context context) { + private Object createResult(String input, Context context) { // TODO - this should work for all the different types, by working // TODO out what we can deserialize to. Making it work _just for_ SQS for // TODO now to test the E2E framework. - ObjectMapper obj = JsonConfig.get().getObjectMapper(); + ObjectMapper mapper = JacksonFactory.getInstance().getMapper(); LOGGER.info(input); try { - SQSEvent event = obj.readValue(input, SQSEvent.class); - SQSBatchResponse result = sqsHandler.processBatch(event, context); - return obj.writeValueAsString(result); + SQSEvent event = mapper.readValue(input, SQSEvent.class); + return sqsHandler.processBatch(event, context); } catch (Exception e) { LOGGER.warn("Wasn't SQS", e); } try { - KinesisEvent event = obj.readValue(input, KinesisEvent.class); - StreamsEventResponse result = kinesisHandler.processBatch(event, context); - return obj.writeValueAsString(result); + KinesisEvent event = mapper.readValue(input, KinesisEvent.class); + return kinesisHandler.processBatch(event, context); } catch (Exception e) { LOGGER.warn("Wasn't Kinesis", e); } try { - DynamodbEvent event = obj.readValue(input, DynamodbEvent.class); - StreamsEventResponse result = ddbHandler.processBatch(event, context); - return obj.writeValueAsString(result); + DynamodbEvent event = mapper.readValue(input, DynamodbEvent.class); + return ddbHandler.processBatch(event, context); } catch (Exception e) { LOGGER.warn("Wasn't DynamoDB"); } @@ -158,4 +141,14 @@ private String createResult(String input, Context context) { } + @Override + public Object handleRequest(InputStream inputStream, Context context) { + + String input = new BufferedReader( + new InputStreamReader(inputStream, StandardCharsets.UTF_8)) + .lines() + .collect(Collectors.joining("\n")); + + return createResult(input, context); + } } diff --git a/powertools-e2e-tests/handlers/pom.xml b/powertools-e2e-tests/handlers/pom.xml index 37154f659..6e82c7aec 100644 --- a/powertools-e2e-tests/handlers/pom.xml +++ b/powertools-e2e-tests/handlers/pom.xml @@ -16,6 +16,7 @@ 1.8 1.2.2 + 1.1.2 3.11.2 3.5.0 1.13.1 @@ -87,6 +88,11 @@ aws-lambda-java-events ${lambda.java.events} + + com.amazonaws + aws-lambda-java-serialization + ${lambda.java.serialization} + org.apache.logging.log4j log4j-slf4j2-impl From 6e44fe6bfbab7e3bc548b83efacf0b9d7014bbb7 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Thu, 3 Aug 2023 09:12:55 +0200 Subject: [PATCH 63/79] Make SQS test work --- .../amazon/lambda/powertools/e2e/Function.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java b/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java index 33e994d51..c78fdf60d 100644 --- a/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java +++ b/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java @@ -22,6 +22,8 @@ import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; import com.amazonaws.services.lambda.runtime.events.SQSEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; +import com.amazonaws.services.lambda.runtime.serialization.PojoSerializer; +import com.amazonaws.services.lambda.runtime.serialization.events.LambdaEventSerializers; import com.amazonaws.services.lambda.runtime.serialization.factories.JacksonFactory; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonMappingException; @@ -109,29 +111,27 @@ private void processDdbMessage(DynamodbEvent.DynamodbStreamRecord dynamodbStream private Object createResult(String input, Context context) { - // TODO - this should work for all the different types, by working - // TODO out what we can deserialize to. Making it work _just for_ SQS for - // TODO now to test the E2E framework. - ObjectMapper mapper = JacksonFactory.getInstance().getMapper(); - LOGGER.info(input); try { - SQSEvent event = mapper.readValue(input, SQSEvent.class); + PojoSerializer serializer = LambdaEventSerializers.serializerFor(SQSEvent.class, this.getClass().getClassLoader()); + SQSEvent event = serializer.fromJson(input); return sqsHandler.processBatch(event, context); } catch (Exception e) { LOGGER.warn("Wasn't SQS", e); } try { - KinesisEvent event = mapper.readValue(input, KinesisEvent.class); + PojoSerializer serializer = LambdaEventSerializers.serializerFor(KinesisEvent.class, this.getClass().getClassLoader()); + KinesisEvent event = serializer.fromJson(input); return kinesisHandler.processBatch(event, context); } catch (Exception e) { LOGGER.warn("Wasn't Kinesis", e); } try { - DynamodbEvent event = mapper.readValue(input, DynamodbEvent.class); + PojoSerializer serializer = LambdaEventSerializers.serializerFor(DynamodbEvent.class, this.getClass().getClassLoader()); + DynamodbEvent event = serializer.fromJson(input); return ddbHandler.processBatch(event, context); } catch (Exception e) { LOGGER.warn("Wasn't DynamoDB"); From 0c9f8b8570817b83ff817240e5270d4336b55832 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Thu, 3 Aug 2023 18:38:03 +0200 Subject: [PATCH 64/79] SQS and Kinesis work --- .../lambda/powertools/e2e/Function.java | 38 ++++++++----------- .../src/test/java/TestSerialization.java | 29 ++++++++++++++ .../amazon/lambda/powertools/BatchE2ET.java | 3 +- .../powertools/testutils/Infrastructure.java | 3 +- 4 files changed, 49 insertions(+), 24 deletions(-) create mode 100644 powertools-e2e-tests/handlers/batch/src/test/java/TestSerialization.java diff --git a/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java b/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java index c78fdf60d..35c67a844 100644 --- a/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java +++ b/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java @@ -109,36 +109,30 @@ private void processDdbMessage(DynamodbEvent.DynamodbStreamRecord dynamodbStream // TODO write DDB change details to batch output table } - private Object createResult(String input, Context context) { + public Object createResult(String input, Context context) { LOGGER.info(input); - try { - PojoSerializer serializer = LambdaEventSerializers.serializerFor(SQSEvent.class, this.getClass().getClassLoader()); - SQSEvent event = serializer.fromJson(input); + PojoSerializer serializer = LambdaEventSerializers.serializerFor(SQSEvent.class, this.getClass().getClassLoader()); + SQSEvent event = serializer.fromJson(input); + if (event.getRecords().get(0).getEventSource().equals("aws:sqs")) { + LOGGER.info("Running for SQS"); + LOGGER.info(event); return sqsHandler.processBatch(event, context); - } catch (Exception e) { - LOGGER.warn("Wasn't SQS", e); } - try { - PojoSerializer serializer = LambdaEventSerializers.serializerFor(KinesisEvent.class, this.getClass().getClassLoader()); - KinesisEvent event = serializer.fromJson(input); - return kinesisHandler.processBatch(event, context); - } catch (Exception e) { - LOGGER.warn("Wasn't Kinesis", e); + PojoSerializer kinesisSerializer = LambdaEventSerializers.serializerFor(KinesisEvent.class, this.getClass().getClassLoader()); + KinesisEvent kinesisEvent = kinesisSerializer.fromJson(input); + if (kinesisEvent.getRecords().get(0).getEventSource().equals("aws:kinesis")) { + LOGGER.info("Running for Kinesis"); + return kinesisHandler.processBatch(kinesisEvent, context); } - try { - PojoSerializer serializer = LambdaEventSerializers.serializerFor(DynamodbEvent.class, this.getClass().getClassLoader()); - DynamodbEvent event = serializer.fromJson(input); - return ddbHandler.processBatch(event, context); - } catch (Exception e) { - LOGGER.warn("Wasn't DynamoDB"); - } - - throw new RuntimeException("Couldn't deserialize an event we understood. Giving up!"); - + // Well, let's try dynamo + PojoSerializer ddbSerializer = LambdaEventSerializers.serializerFor(DynamodbEvent.class, this.getClass().getClassLoader()); + LOGGER.info("Running for DynamoDB"); + DynamodbEvent ddbEvent = ddbSerializer.fromJson(input); + return ddbHandler.processBatch(ddbEvent, context); } @Override diff --git a/powertools-e2e-tests/handlers/batch/src/test/java/TestSerialization.java b/powertools-e2e-tests/handlers/batch/src/test/java/TestSerialization.java new file mode 100644 index 000000000..c9ca6e472 --- /dev/null +++ b/powertools-e2e-tests/handlers/batch/src/test/java/TestSerialization.java @@ -0,0 +1,29 @@ +import com.amazonaws.lambda.thirdparty.com.fasterxml.jackson.databind.ObjectMapper; +import com.amazonaws.services.lambda.runtime.events.KinesisEvent; +import com.amazonaws.services.lambda.runtime.events.SQSEvent; +import com.amazonaws.services.lambda.runtime.serialization.PojoSerializer; +import com.amazonaws.services.lambda.runtime.serialization.events.LambdaEventSerializers; +import com.amazonaws.services.lambda.runtime.serialization.factories.JacksonFactory; +import software.amazon.lambda.powertools.e2e.Function; + +public class TestSerialization { + + static String sqsMsg = "{\"Records\":[{\"messageId\":\"0c957063-07a7-4495-ab2a-d4678ce8c641\",\"receiptHandle\":\"AQEBJUG48GuQg8eTMDe8roYS4HKyxnramDvOYG+HqQrZkH/fz1ZQRoXzMvYiVsod/eR5wYviLr2RL+eX1kcqJfLhugRErFonuuF7mNj6RnOcBUHRxYrlCc5yP84WRGkRXCFhZ0esYNOgL4SABkVkJxOm72wmkz2PD5stzxpBq+SzrLEIroyx3TJ+2Tmp1m5lDHAZnrj9b3LcRBBF2JJ1FPibQ1WcXnmzWZxtefAbUppWxPM7M1yeTgJNuPMd9/r8u3RDl5AXCfao/FTMPAWrvi3Ih7FJUWD/jwCXC4sa9X/xOc2pyQBqsrkQWTANDh20XHMYYp5wcvj+ceom+BZ1NfmotLLtutoP7DMR2d5M3oAx5+me251jtBulq4LGHRgzdtSJcxJlFGXunWTZbG78u61BxQ==\",\"body\":\"{\\\"id\\\":1,\\\"name\\\":\\\"product1\\\",\\\"price\\\":1.23}\",\"attributes\":{\"ApproximateReceiveCount\":\"1\",\"SentTimestamp\":\"1691046046134\",\"SenderId\":\"AROAT2IWOUCAKCSJQ5M5X:gerrings-Isengard\",\"ApproximateFirstReceiveTimestamp\":\"1691046046146\"},\"messageAttributes\":{},\"md5OfBody\":\"6070c6b8b6961c80279de16f37838cc6\",\"eventSource\":\"aws:sqs\",\"eventSourceARN\":\"arn:aws:sqs:eu-west-1:262576971904:batchqueuefdedf4\",\"awsRegion\":\"eu-west-1\"}]}"; + static String kinesisMsg = "{\"Records\":[{\"kinesis\":{\"kinesisSchemaVersion\":\"1.0\",\"partitionKey\":\"1\",\"sequenceNumber\":\"49643250745023848610933107459883370427365077215172624434\",\"data\":\"eyJpZCI6MSwibmFtZSI6InByb2R1Y3QxIiwicHJpY2UiOjEuMjN9\",\"approximateArrivalTimestamp\":1.69106871717E9},\"eventSource\":\"aws:kinesis\",\"eventVersion\":\"1.0\",\"eventID\":\"shardId-000000000003:49643250745023848610933107459883370427365077215172624434\",\"eventName\":\"aws:kinesis:record\",\"invokeIdentityArn\":\"arn:aws:iam::262576971904:role/BatchE2ET-2b1e1d07d19a-BatchE2ET2b1e1d07d19afuncti-W4MDBKGVWKQA\",\"awsRegion\":\"eu-west-1\",\"eventSourceARN\":\"arn:aws:kinesis:eu-west-1:262576971904:stream/batchstream261626\"},{\"kinesis\":{\"kinesisSchemaVersion\":\"1.0\",\"partitionKey\":\"1\",\"sequenceNumber\":\"49643250745023848610933107459884579353184691844347330610\",\"data\":\"eyJpZCI6MiwibmFtZSI6InByb2R1Y3QyIiwicHJpY2UiOjQuNTZ9\",\"approximateArrivalTimestamp\":1.691068717172E9},\"eventSource\":\"aws:kinesis\",\"eventVersion\":\"1.0\",\"eventID\":\"shardId-000000000003:49643250745023848610933107459884579353184691844347330610\",\"eventName\":\"aws:kinesis:record\",\"invokeIdentityArn\":\"arn:aws:iam::262576971904:role/BatchE2ET-2b1e1d07d19a-BatchE2ET2b1e1d07d19afuncti-W4MDBKGVWKQA\",\"awsRegion\":\"eu-west-1\",\"eventSourceARN\":\"arn:aws:kinesis:eu-west-1:262576971904:stream/batchstream261626\"},{\"kinesis\":{\"kinesisSchemaVersion\":\"1.0\",\"partitionKey\":\"1\",\"sequenceNumber\":\"49643250745023848610933107459885788279004306473522036786\",\"data\":\"eyJpZCI6MywibmFtZSI6InByb2R1Y3QzIiwicHJpY2UiOjYuNzh9\",\"approximateArrivalTimestamp\":1.691068717172E9},\"eventSource\":\"aws:kinesis\",\"eventVersion\":\"1.0\",\"eventID\":\"shardId-000000000003:49643250745023848610933107459885788279004306473522036786\",\"eventName\":\"aws:kinesis:record\",\"invokeIdentityArn\":\"arn:aws:iam::262576971904:role/BatchE2ET-2b1e1d07d19a-BatchE2ET2b1e1d07d19afuncti-W4MDBKGVWKQA\",\"awsRegion\":\"eu-west-1\",\"eventSourceARN\":\"arn:aws:kinesis:eu-west-1:262576971904:stream/batchstream261626\"}]}"; + public static void main(String [] args) throws Exception { + + Function f = new Function(); + f.createResult(kinesisMsg, null); + + ObjectMapper mapper = JacksonFactory.getInstance().getMapper(); + + PojoSerializer serializer = LambdaEventSerializers.serializerFor(SQSEvent.class, TestSerialization.class.getClassLoader()); + PojoSerializer kinesisSerializer = LambdaEventSerializers.serializerFor(KinesisEvent.class, TestSerialization.class.getClassLoader()); + + SQSEvent sqsAsSqs = serializer.fromJson(sqsMsg); + SQSEvent kinesisAsSqs = serializer.fromJson(kinesisMsg); + + + System.out.println(kinesisAsSqs); + } +} diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java index 1ccc0acbf..6ccb58aef 100644 --- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java @@ -49,6 +49,7 @@ import software.amazon.awssdk.services.kinesis.KinesisClient; import software.amazon.awssdk.services.kinesis.model.PutRecordsRequest; import software.amazon.awssdk.services.kinesis.model.PutRecordsRequestEntry; +import software.amazon.awssdk.services.kinesis.model.PutRecordsResponse; import software.amazon.awssdk.services.sqs.SqsClient; import software.amazon.awssdk.services.sqs.model.SendMessageBatchRequest; import software.amazon.awssdk.services.sqs.model.SendMessageBatchRequestEntry; @@ -197,7 +198,7 @@ public void kinesisBatchProcessingSucceeds() throws InterruptedException { .collect(Collectors.toList()); // WHEN - kinesisClient.putRecords(PutRecordsRequest.builder() + PutRecordsResponse result = kinesisClient.putRecords(PutRecordsRequest.builder() .streamName(kinesisStreamName) .records(entries) .build()); diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java index acba3818b..1ffc4981b 100644 --- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java @@ -307,7 +307,8 @@ private Stack createStackWithLambda() { .create(stream) .enabled(true) .batchSize(3) - .startingPosition(StartingPosition.LATEST) + .startingPosition(StartingPosition.TRIM_HORIZON) + .maxBatchingWindow(Duration.seconds(1)) .build(); function.addEventSource(kinesisEventSource); CfnOutput.Builder From 006b9e0a7cc529cb3d097f66ebcd5132d41dd195 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Thu, 3 Aug 2023 19:41:46 +0200 Subject: [PATCH 65/79] DynamoDB E2E works --- .../handler/DynamoDbBatchMessageHandler.java | 1 + .../KinesisStreamsBatchMessageHandler.java | 5 ++ .../batch/handler/SqsBatchMessageHandler.java | 2 + .../lambda/powertools/e2e/Function.java | 51 +++++++++++++------ .../src/test/java/TestSerialization.java | 29 ----------- .../amazon/lambda/powertools/BatchE2ET.java | 36 +++++++++++-- .../powertools/testutils/Infrastructure.java | 33 +++++++++--- 7 files changed, 102 insertions(+), 55 deletions(-) delete mode 100644 powertools-e2e-tests/handlers/batch/src/test/java/TestSerialization.java diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/DynamoDbBatchMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/DynamoDbBatchMessageHandler.java index 911da6944..aa6eba839 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/DynamoDbBatchMessageHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/DynamoDbBatchMessageHandler.java @@ -60,6 +60,7 @@ public StreamsEventResponse processBatch(DynamodbEvent event, Context context) { String sequenceNumber = record.getDynamodb().getSequenceNumber(); LOGGER.error("Error while processing record with id {}: {}, adding it to batch item failures", sequenceNumber, t.getMessage()); + LOGGER.error("Error was", t); batchFailures.add(new StreamsEventResponse.BatchItemFailure(sequenceNumber)); // Report failure if we have a handler diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/KinesisStreamsBatchMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/KinesisStreamsBatchMessageHandler.java index 04bfe963c..fe1aaf354 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/KinesisStreamsBatchMessageHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/KinesisStreamsBatchMessageHandler.java @@ -73,6 +73,11 @@ public StreamsEventResponse processBatch(KinesisEvent event, Context context) { this.successHandler.accept(record); } } catch (Throwable t) { + String sequenceNumber = record.getEventID(); + LOGGER.error("Error while processing record with eventID {}: {}, adding it to batch item failures", + sequenceNumber, t.getMessage()); + LOGGER.error("Error was", t); + batchFailures.add(new StreamsEventResponse.BatchItemFailure(record.getKinesis().getSequenceNumber())); // Report failure if we have a handler diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java index 539a3840c..b3c416a69 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java @@ -86,6 +86,8 @@ public SQSBatchResponse processBatch(SQSEvent event, Context context) { } catch (Throwable t) { LOGGER.error("Error while processing message with messageId {}: {}, adding it to batch item failures", message.getMessageId(), t.getMessage()); + LOGGER.error("Error was", t); + response.getBatchItemFailures() .add(SQSBatchResponse.BatchItemFailure.builder().withItemIdentifier(message.getMessageId()) .build()); diff --git a/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java b/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java index 35c67a844..fe347ba16 100644 --- a/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java +++ b/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java @@ -37,6 +37,7 @@ import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; import java.util.HashMap; +import java.util.Map; import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -50,6 +51,8 @@ import software.amazon.lambda.powertools.logging.Logging; import software.amazon.lambda.powertools.utilities.JsonConfig; +import javax.management.Attribute; + public class Function implements RequestHandler { @@ -83,30 +86,46 @@ private void processProductMessage(Product p, Context c) { // TODO - write product details to output table ddbClient = DynamoDbClient.builder() .build(); + Map results = new HashMap<>(); + results.put("functionName", AttributeValue.builder() + .s(c.getFunctionName()) + .build()); + results.put("id", AttributeValue.builder() + .s(Long.toString(p.getId())) + .build()); + results.put("name", AttributeValue.builder() + .s(p.getName()) + .build()); + results.put("price", AttributeValue.builder() + .n(Double.toString(p.getPrice())) + .build()); ddbClient.putItem(PutItemRequest.builder() .tableName(ddbOutputTable) - .item(new HashMap() { - { - put("functionName", AttributeValue.builder() - .s(c.getFunctionName()) - .build()); - put("id", AttributeValue.builder() - .s(Long.toString(p.getId())) - .build()); - put("name", AttributeValue.builder() - .s(p.getName()) - .build()); - put("price", AttributeValue.builder() - .n(Double.toString(p.getPrice())) - .build()); - }}) + .item(results) .build()); } private void processDdbMessage(DynamodbEvent.DynamodbStreamRecord dynamodbStreamRecord, Context context) { LOGGER.info("Processing DynamoDB Stream Record" + dynamodbStreamRecord); - // TODO write DDB change details to batch output table + ddbClient = DynamoDbClient.builder() + .build(); + + String id = dynamodbStreamRecord.getDynamodb().getKeys().get("id").getS(); + LOGGER.info("Incoming ID is " + id); + + Map results = new HashMap<>(); + results.put("functionName", AttributeValue.builder() + .s(context.getFunctionName()) + .build()); + results.put("id", AttributeValue.builder() + .s(id) + .build()); + + ddbClient.putItem(PutItemRequest.builder() + .tableName(ddbOutputTable) + .item(results) + .build()); } public Object createResult(String input, Context context) { diff --git a/powertools-e2e-tests/handlers/batch/src/test/java/TestSerialization.java b/powertools-e2e-tests/handlers/batch/src/test/java/TestSerialization.java deleted file mode 100644 index c9ca6e472..000000000 --- a/powertools-e2e-tests/handlers/batch/src/test/java/TestSerialization.java +++ /dev/null @@ -1,29 +0,0 @@ -import com.amazonaws.lambda.thirdparty.com.fasterxml.jackson.databind.ObjectMapper; -import com.amazonaws.services.lambda.runtime.events.KinesisEvent; -import com.amazonaws.services.lambda.runtime.events.SQSEvent; -import com.amazonaws.services.lambda.runtime.serialization.PojoSerializer; -import com.amazonaws.services.lambda.runtime.serialization.events.LambdaEventSerializers; -import com.amazonaws.services.lambda.runtime.serialization.factories.JacksonFactory; -import software.amazon.lambda.powertools.e2e.Function; - -public class TestSerialization { - - static String sqsMsg = "{\"Records\":[{\"messageId\":\"0c957063-07a7-4495-ab2a-d4678ce8c641\",\"receiptHandle\":\"AQEBJUG48GuQg8eTMDe8roYS4HKyxnramDvOYG+HqQrZkH/fz1ZQRoXzMvYiVsod/eR5wYviLr2RL+eX1kcqJfLhugRErFonuuF7mNj6RnOcBUHRxYrlCc5yP84WRGkRXCFhZ0esYNOgL4SABkVkJxOm72wmkz2PD5stzxpBq+SzrLEIroyx3TJ+2Tmp1m5lDHAZnrj9b3LcRBBF2JJ1FPibQ1WcXnmzWZxtefAbUppWxPM7M1yeTgJNuPMd9/r8u3RDl5AXCfao/FTMPAWrvi3Ih7FJUWD/jwCXC4sa9X/xOc2pyQBqsrkQWTANDh20XHMYYp5wcvj+ceom+BZ1NfmotLLtutoP7DMR2d5M3oAx5+me251jtBulq4LGHRgzdtSJcxJlFGXunWTZbG78u61BxQ==\",\"body\":\"{\\\"id\\\":1,\\\"name\\\":\\\"product1\\\",\\\"price\\\":1.23}\",\"attributes\":{\"ApproximateReceiveCount\":\"1\",\"SentTimestamp\":\"1691046046134\",\"SenderId\":\"AROAT2IWOUCAKCSJQ5M5X:gerrings-Isengard\",\"ApproximateFirstReceiveTimestamp\":\"1691046046146\"},\"messageAttributes\":{},\"md5OfBody\":\"6070c6b8b6961c80279de16f37838cc6\",\"eventSource\":\"aws:sqs\",\"eventSourceARN\":\"arn:aws:sqs:eu-west-1:262576971904:batchqueuefdedf4\",\"awsRegion\":\"eu-west-1\"}]}"; - static String kinesisMsg = "{\"Records\":[{\"kinesis\":{\"kinesisSchemaVersion\":\"1.0\",\"partitionKey\":\"1\",\"sequenceNumber\":\"49643250745023848610933107459883370427365077215172624434\",\"data\":\"eyJpZCI6MSwibmFtZSI6InByb2R1Y3QxIiwicHJpY2UiOjEuMjN9\",\"approximateArrivalTimestamp\":1.69106871717E9},\"eventSource\":\"aws:kinesis\",\"eventVersion\":\"1.0\",\"eventID\":\"shardId-000000000003:49643250745023848610933107459883370427365077215172624434\",\"eventName\":\"aws:kinesis:record\",\"invokeIdentityArn\":\"arn:aws:iam::262576971904:role/BatchE2ET-2b1e1d07d19a-BatchE2ET2b1e1d07d19afuncti-W4MDBKGVWKQA\",\"awsRegion\":\"eu-west-1\",\"eventSourceARN\":\"arn:aws:kinesis:eu-west-1:262576971904:stream/batchstream261626\"},{\"kinesis\":{\"kinesisSchemaVersion\":\"1.0\",\"partitionKey\":\"1\",\"sequenceNumber\":\"49643250745023848610933107459884579353184691844347330610\",\"data\":\"eyJpZCI6MiwibmFtZSI6InByb2R1Y3QyIiwicHJpY2UiOjQuNTZ9\",\"approximateArrivalTimestamp\":1.691068717172E9},\"eventSource\":\"aws:kinesis\",\"eventVersion\":\"1.0\",\"eventID\":\"shardId-000000000003:49643250745023848610933107459884579353184691844347330610\",\"eventName\":\"aws:kinesis:record\",\"invokeIdentityArn\":\"arn:aws:iam::262576971904:role/BatchE2ET-2b1e1d07d19a-BatchE2ET2b1e1d07d19afuncti-W4MDBKGVWKQA\",\"awsRegion\":\"eu-west-1\",\"eventSourceARN\":\"arn:aws:kinesis:eu-west-1:262576971904:stream/batchstream261626\"},{\"kinesis\":{\"kinesisSchemaVersion\":\"1.0\",\"partitionKey\":\"1\",\"sequenceNumber\":\"49643250745023848610933107459885788279004306473522036786\",\"data\":\"eyJpZCI6MywibmFtZSI6InByb2R1Y3QzIiwicHJpY2UiOjYuNzh9\",\"approximateArrivalTimestamp\":1.691068717172E9},\"eventSource\":\"aws:kinesis\",\"eventVersion\":\"1.0\",\"eventID\":\"shardId-000000000003:49643250745023848610933107459885788279004306473522036786\",\"eventName\":\"aws:kinesis:record\",\"invokeIdentityArn\":\"arn:aws:iam::262576971904:role/BatchE2ET-2b1e1d07d19a-BatchE2ET2b1e1d07d19afuncti-W4MDBKGVWKQA\",\"awsRegion\":\"eu-west-1\",\"eventSourceARN\":\"arn:aws:kinesis:eu-west-1:262576971904:stream/batchstream261626\"}]}"; - public static void main(String [] args) throws Exception { - - Function f = new Function(); - f.createResult(kinesisMsg, null); - - ObjectMapper mapper = JacksonFactory.getInstance().getMapper(); - - PojoSerializer serializer = LambdaEventSerializers.serializerFor(SQSEvent.class, TestSerialization.class.getClassLoader()); - PojoSerializer kinesisSerializer = LambdaEventSerializers.serializerFor(KinesisEvent.class, TestSerialization.class.getClassLoader()); - - SQSEvent sqsAsSqs = serializer.fromJson(sqsMsg); - SQSEvent kinesisAsSqs = serializer.fromJson(kinesisMsg); - - - System.out.println(kinesisAsSqs); - } -} diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java index 6ccb58aef..1a00a31f7 100644 --- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java @@ -42,10 +42,7 @@ import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.dynamodb.DynamoDbClientBuilder; -import software.amazon.awssdk.services.dynamodb.model.AttributeValue; -import software.amazon.awssdk.services.dynamodb.model.DeleteItemRequest; -import software.amazon.awssdk.services.dynamodb.model.ScanRequest; -import software.amazon.awssdk.services.dynamodb.model.ScanResponse; +import software.amazon.awssdk.services.dynamodb.model.*; import software.amazon.awssdk.services.kinesis.KinesisClient; import software.amazon.awssdk.services.kinesis.model.PutRecordsRequest; import software.amazon.awssdk.services.kinesis.model.PutRecordsRequestEntry; @@ -72,6 +69,7 @@ public class BatchE2ET { private static DynamoDbClient ddbClient; private static SqsClient sqsClient; private static KinesisClient kinesisClient; + private static String ddbStreamsTestTable; private final List testProducts; @BeforeAll @@ -80,6 +78,8 @@ public static void setup() { String random = UUID.randomUUID().toString().substring(0, 6); String queueName = "batchqueue" + random; kinesisStreamName = "batchstream" + random; + ddbStreamsTestTable = "ddbstreams" + random; + objectMapper = JsonConfig.get().getObjectMapper(); infrastructure = Infrastructure.builder() @@ -87,6 +87,7 @@ public static void setup() { .pathToFunction("batch") .idempotencyTable("idempo" + random) .queue(queueName) + .ddbStreamsTableName(ddbStreamsTestTable) .kinesisStream(kinesisStreamName) .build(); @@ -95,6 +96,7 @@ public static void setup() { queueUrl = outputs.get("QueueURL"); kinesisStreamName = outputs.get("KinesisStreamName"); outputTable = outputs.get("TableNameForAsyncTests"); + ddbStreamsTestTable = outputs.get("DdbStreamsTestTable"); ddbClient = DynamoDbClient.builder() .region(region) @@ -211,6 +213,32 @@ public void kinesisBatchProcessingSucceeds() throws InterruptedException { validateAllItemsHandled(items); } + @Test + public void ddbStreamsBatchProcessingSucceeds() throws InterruptedException { + // GIVEN + String theId = "my-test-id"; + + // WHEN + ddbClient.putItem(PutItemRequest.builder() + .tableName(ddbStreamsTestTable) + .item(new HashMap() { + { + put("id", AttributeValue.builder() + .s(theId) + .build()); + }}) + .build()); + Thread.sleep(90000); // wait for function to be executed + + // THEN + ScanResponse items = ddbClient.scan(ScanRequest.builder() + .tableName(outputTable) + .build()); + + assertThat(items.count()).isEqualTo(1); + assertThat(items.items().get(0).get("id").s()).isEqualTo(theId); + } + private void validateAllItemsHandled(ScanResponse items) { for (Product p : testProducts) { boolean foundIt = false; diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java index 1ffc4981b..473a540dc 100644 --- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java @@ -16,7 +16,6 @@ import static java.util.Collections.singletonList; -import com.amazonaws.services.lambda.runtime.events.KinesisEvent; import com.fasterxml.jackson.databind.JsonNode; import java.io.File; import java.io.IOException; @@ -46,11 +45,7 @@ import software.amazon.awscdk.services.appconfig.CfnDeploymentStrategy; import software.amazon.awscdk.services.appconfig.CfnEnvironment; import software.amazon.awscdk.services.appconfig.CfnHostedConfigurationVersion; -import software.amazon.awscdk.services.dynamodb.Attribute; -import software.amazon.awscdk.services.dynamodb.AttributeType; -import software.amazon.awscdk.services.dynamodb.BillingMode; -import software.amazon.awscdk.services.dynamodb.Table; -import software.amazon.awscdk.services.events.targets.KinesisStream; +import software.amazon.awscdk.services.dynamodb.*; import software.amazon.awscdk.services.iam.PolicyStatement; import software.amazon.awscdk.services.kinesis.Stream; import software.amazon.awscdk.services.kinesis.StreamMode; @@ -58,6 +53,7 @@ import software.amazon.awscdk.services.lambda.Function; import software.amazon.awscdk.services.lambda.StartingPosition; import software.amazon.awscdk.services.lambda.Tracing; +import software.amazon.awscdk.services.lambda.eventsources.DynamoEventSource; import software.amazon.awscdk.services.lambda.eventsources.KinesisEventSource; import software.amazon.awscdk.services.lambda.eventsources.SqsEventSource; import software.amazon.awscdk.services.logs.LogGroup; @@ -118,6 +114,7 @@ public class Infrastructure { private final String queue; private final String kinesisStream; private final String largeMessagesBucket; + private String ddbStreamsTableName; private String functionName; private Object cfnTemplate; @@ -135,6 +132,7 @@ private Infrastructure(Builder builder) { this.queue = builder.queue; this.kinesisStream = builder.kinesisStream; this.largeMessagesBucket = builder.largeMessagesBucket; + this.ddbStreamsTableName = builder.ddbStreamsTableName; this.app = new App(); this.stack = createStackWithLambda(); @@ -317,6 +315,23 @@ private Stack createStackWithLambda() { .build(); } + if (!StringUtils.isEmpty(ddbStreamsTableName)) { + Table ddbStreamsTable = Table.Builder.create(stack, "DDBStreamsTable") + .tableName(ddbStreamsTableName) + .stream(StreamViewType.KEYS_ONLY) + .removalPolicy(RemovalPolicy.DESTROY) + .partitionKey(Attribute.builder().name("id").type(AttributeType.STRING).build()) + .build(); + + DynamoEventSource ddbEventSource = DynamoEventSource.Builder.create(ddbStreamsTable) + .batchSize(1) + .startingPosition(StartingPosition.TRIM_HORIZON) + .maxBatchingWindow(Duration.seconds(1)) + .build(); + function.addEventSource(ddbEventSource); + CfnOutput.Builder.create(stack, "DdbStreamsTestTable").value(ddbStreamsTable.getTableName()).build(); + } + if (!StringUtils.isEmpty(largeMessagesBucket)) { Bucket offloadBucket = Bucket.Builder .create(stack, "LargeMessagesOffloadBucket") @@ -481,6 +496,7 @@ public static class Builder { private String idemPotencyTable; private String queue; private String kinesisStream; + private String ddbStreamsTableName; private Builder() { getJavaRuntime(); @@ -561,6 +577,11 @@ public Builder kinesisStream(String stream) { return this; } + public Builder ddbStreamsTableName(String tableName) { + this.ddbStreamsTableName = tableName; + return this; + } + public Builder largeMessagesBucket(String largeMessagesBucket) { this.largeMessagesBucket = largeMessagesBucket; return this; From c3ae363ba949b3fdfe3255b2615dbaa0f4787660 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Thu, 3 Aug 2023 19:42:54 +0200 Subject: [PATCH 66/79] Formatting --- .../AbstractBatchMessageHandlerBuilder.java | 3 +- .../DynamoDbBatchMessageHandlerBuilder.java | 3 +- .../KinesisBatchMessageHandlerBuilder.java | 3 +- .../SqsBatchMessageHandlerBuilder.java | 3 +- .../handler/DynamoDbBatchMessageHandler.java | 5 +- .../KinesisStreamsBatchMessageHandler.java | 7 +- .../batch/handler/SqsBatchMessageHandler.java | 7 +- .../batch/DdbBatchProcessorTest.java | 7 +- .../batch/KinesisBatchProcessorTest.java | 7 +- .../batch/SQSBatchProcessorTest.java | 7 +- .../lambda/powertools/e2e/Function.java | 10 +- .../amazon/lambda/powertools/BatchE2ET.java | 98 ++++++++----------- 12 files changed, 79 insertions(+), 81 deletions(-) diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/AbstractBatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/AbstractBatchMessageHandlerBuilder.java index 9b0647770..e28429503 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/AbstractBatchMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/AbstractBatchMessageHandlerBuilder.java @@ -15,9 +15,10 @@ package software.amazon.lambda.powertools.batch.builder; import com.amazonaws.services.lambda.runtime.Context; +import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; + import java.util.function.BiConsumer; import java.util.function.Consumer; -import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; /** * An abstract class to capture common arguments used across all the message-binding-specific batch processing diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/DynamoDbBatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/DynamoDbBatchMessageHandlerBuilder.java index 8513322b3..6e42f8d7b 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/DynamoDbBatchMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/DynamoDbBatchMessageHandlerBuilder.java @@ -17,11 +17,12 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; -import java.util.function.BiConsumer; import software.amazon.lambda.powertools.batch.exception.DeserializationNotSupportedException; import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; import software.amazon.lambda.powertools.batch.handler.DynamoDbBatchMessageHandler; +import java.util.function.BiConsumer; + /** * Builds a batch processor for processing DynamoDB Streams batch events **/ diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/KinesisBatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/KinesisBatchMessageHandlerBuilder.java index 30bfcab65..52426bdc7 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/KinesisBatchMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/KinesisBatchMessageHandlerBuilder.java @@ -17,10 +17,11 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.KinesisEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; -import java.util.function.BiConsumer; import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; import software.amazon.lambda.powertools.batch.handler.KinesisStreamsBatchMessageHandler; +import java.util.function.BiConsumer; + /** * Builds a batch processor for processing Kinesis Streams batch events */ diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/SqsBatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/SqsBatchMessageHandlerBuilder.java index ee2dc23f6..bf5d18a0b 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/SqsBatchMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/SqsBatchMessageHandlerBuilder.java @@ -17,10 +17,11 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; import com.amazonaws.services.lambda.runtime.events.SQSEvent; -import java.util.function.BiConsumer; import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; import software.amazon.lambda.powertools.batch.handler.SqsBatchMessageHandler; +import java.util.function.BiConsumer; + /** * Builds a batch processor for the SQS event source. */ diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/DynamoDbBatchMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/DynamoDbBatchMessageHandler.java index aa6eba839..9f4491329 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/DynamoDbBatchMessageHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/DynamoDbBatchMessageHandler.java @@ -17,12 +17,13 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.util.ArrayList; import java.util.List; import java.util.function.BiConsumer; import java.util.function.Consumer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * A batch message processor for DynamoDB Streams batches. diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/KinesisStreamsBatchMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/KinesisStreamsBatchMessageHandler.java index fe1aaf354..4d215bf76 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/KinesisStreamsBatchMessageHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/KinesisStreamsBatchMessageHandler.java @@ -18,13 +18,14 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.KinesisEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.lambda.powertools.utilities.EventDeserializer; + import java.util.ArrayList; import java.util.List; import java.util.function.BiConsumer; import java.util.function.Consumer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import software.amazon.lambda.powertools.utilities.EventDeserializer; /** * A batch message processor for Kinesis Streams batch processing. diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java index b3c416a69..d1c75e966 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java @@ -17,13 +17,14 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; import com.amazonaws.services.lambda.runtime.events.SQSEvent; -import java.util.ArrayList; -import java.util.function.BiConsumer; -import java.util.function.Consumer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.lambda.powertools.utilities.EventDeserializer; +import java.util.ArrayList; +import java.util.function.BiConsumer; +import java.util.function.Consumer; + /** * A batch message processor for SQS batches. * diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/DdbBatchProcessorTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/DdbBatchProcessorTest.java index 9e2c211e2..0983838c7 100644 --- a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/DdbBatchProcessorTest.java +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/DdbBatchProcessorTest.java @@ -14,17 +14,18 @@ package software.amazon.lambda.powertools.batch; -import static org.assertj.core.api.Assertions.assertThat; - import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; import com.amazonaws.services.lambda.runtime.tests.annotations.Event; -import java.util.concurrent.atomic.AtomicBoolean; import org.junit.jupiter.params.ParameterizedTest; import org.mockito.Mock; import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.assertj.core.api.Assertions.assertThat; + public class DdbBatchProcessorTest { @Mock diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/KinesisBatchProcessorTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/KinesisBatchProcessorTest.java index d78638e1d..9d9c355d2 100644 --- a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/KinesisBatchProcessorTest.java +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/KinesisBatchProcessorTest.java @@ -14,18 +14,19 @@ package software.amazon.lambda.powertools.batch; -import static org.assertj.core.api.Assertions.assertThat; - import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.KinesisEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; import com.amazonaws.services.lambda.runtime.tests.annotations.Event; -import java.util.concurrent.atomic.AtomicBoolean; import org.junit.jupiter.params.ParameterizedTest; import org.mockito.Mock; import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; import software.amazon.lambda.powertools.batch.model.Product; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.assertj.core.api.Assertions.assertThat; + public class KinesisBatchProcessorTest { @Mock diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java index 2f9429fa3..4e3623924 100644 --- a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java @@ -14,18 +14,19 @@ package software.amazon.lambda.powertools.batch; -import static org.assertj.core.api.Assertions.assertThat; - import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; import com.amazonaws.services.lambda.runtime.events.SQSEvent; import com.amazonaws.services.lambda.runtime.tests.annotations.Event; -import java.util.concurrent.atomic.AtomicBoolean; import org.junit.jupiter.params.ParameterizedTest; import org.mockito.Mock; import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; import software.amazon.lambda.powertools.batch.model.Product; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.assertj.core.api.Assertions.assertThat; + public class SQSBatchProcessorTest { @Mock private Context context; diff --git a/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java b/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java index fe347ba16..1b234fefb 100644 --- a/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java +++ b/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java @@ -28,6 +28,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.JsonNode; + import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.IOException; @@ -39,6 +40,7 @@ import java.util.HashMap; import java.util.Map; import java.util.stream.Collectors; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.core.util.IOUtils; @@ -91,8 +93,8 @@ private void processProductMessage(Product p, Context c) { .s(c.getFunctionName()) .build()); results.put("id", AttributeValue.builder() - .s(Long.toString(p.getId())) - .build()); + .s(Long.toString(p.getId())) + .build()); results.put("name", AttributeValue.builder() .s(p.getName()) .build()); @@ -100,8 +102,8 @@ private void processProductMessage(Product p, Context c) { .n(Double.toString(p.getPrice())) .build()); ddbClient.putItem(PutItemRequest.builder() - .tableName(ddbOutputTable) - .item(results) + .tableName(ddbOutputTable) + .item(results) .build()); } diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java index 1a00a31f7..fea007fc1 100644 --- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java @@ -14,34 +14,14 @@ package software.amazon.lambda.powertools; -import static software.amazon.lambda.powertools.testutils.Infrastructure.FUNCTION_NAME_OUTPUT; -import static software.amazon.lambda.powertools.testutils.lambda.LambdaInvoker.invokeFunction; - import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; -import java.time.Year; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.*; import software.amazon.awssdk.core.SdkBytes; import software.amazon.awssdk.http.SdkHttpClient; import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; -import software.amazon.awssdk.services.dynamodb.DynamoDbClientBuilder; import software.amazon.awssdk.services.dynamodb.model.*; import software.amazon.awssdk.services.kinesis.KinesisClient; import software.amazon.awssdk.services.kinesis.model.PutRecordsRequest; @@ -50,17 +30,21 @@ import software.amazon.awssdk.services.sqs.SqsClient; import software.amazon.awssdk.services.sqs.model.SendMessageBatchRequest; import software.amazon.awssdk.services.sqs.model.SendMessageBatchRequestEntry; -import software.amazon.awssdk.services.sqs.model.SendMessageRequest; import software.amazon.lambda.powertools.testutils.Infrastructure; -import software.amazon.lambda.powertools.testutils.lambda.InvocationResult; import software.amazon.lambda.powertools.utilities.JsonConfig; + +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + import static org.assertj.core.api.Assertions.assertThat; +import static software.amazon.lambda.powertools.testutils.Infrastructure.FUNCTION_NAME_OUTPUT; public class BatchE2ET { - private static Infrastructure infrastructure; - private static String functionName; private static final SdkHttpClient httpClient = UrlConnectionHttpClient.builder().build(); private static final Region region = Region.of(System.getProperty("AWS_DEFAULT_REGION", "eu-west-1")); + private static Infrastructure infrastructure; + private static String functionName; private static String queueUrl; private static String kinesisStreamName; @@ -72,6 +56,14 @@ public class BatchE2ET { private static String ddbStreamsTestTable; private final List testProducts; + public BatchE2ET() { + testProducts = Arrays.asList( + new Product(1, "product1", 1.23), + new Product(2, "product2", 4.56), + new Product(3, "product3", 6.78) + ); + } + @BeforeAll @Timeout(value = 5, unit = TimeUnit.MINUTES) public static void setup() { @@ -97,7 +89,7 @@ public static void setup() { kinesisStreamName = outputs.get("KinesisStreamName"); outputTable = outputs.get("TableNameForAsyncTests"); ddbStreamsTestTable = outputs.get("DdbStreamsTestTable"); - + ddbClient = DynamoDbClient.builder() .region(region) .httpClient(httpClient) @@ -114,6 +106,14 @@ public static void setup() { .build(); } + @AfterAll + public static void tearDown() { + // TODO bring this back after testing + if (infrastructure != null) { + // infrastructure.destroy(); + } + } + @AfterEach public void cleanUpTest() { // Delete everything in the output table @@ -121,7 +121,7 @@ public void cleanUpTest() { .tableName(outputTable) .build()); - for (Map item :items.items()) { + for (Map item : items.items()) { HashMap key = new HashMap() { { put("functionName", AttributeValue.builder() @@ -130,28 +130,13 @@ public void cleanUpTest() { put("id", AttributeValue.builder() .s(item.get("id").s()) .build()); - }}; + } + }; ddbClient.deleteItem(DeleteItemRequest.builder() - .tableName(outputTable) - .key(key) - .build()); - } - } - - public BatchE2ET() { - testProducts = Arrays.asList( - new Product(1, "product1", 1.23), - new Product(2, "product2", 4.56), - new Product(3, "product3", 6.78) - ); - } - - @AfterAll - public static void tearDown() { - // TODO bring this back after testing - if (infrastructure != null) { - // infrastructure.destroy(); + .tableName(outputTable) + .key(key) + .build()); } } @@ -219,15 +204,16 @@ public void ddbStreamsBatchProcessingSucceeds() throws InterruptedException { String theId = "my-test-id"; // WHEN - ddbClient.putItem(PutItemRequest.builder() - .tableName(ddbStreamsTestTable) - .item(new HashMap() { - { - put("id", AttributeValue.builder() - .s(theId) - .build()); - }}) - .build()); + ddbClient.putItem(PutItemRequest.builder() + .tableName(ddbStreamsTestTable) + .item(new HashMap() { + { + put("id", AttributeValue.builder() + .s(theId) + .build()); + } + }) + .build()); Thread.sleep(90000); // wait for function to be executed // THEN From 570d051b86bac35f3cd36c2fbc0d27c89b36320a Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Thu, 3 Aug 2023 19:57:32 +0200 Subject: [PATCH 67/79] Try exclude e2e-tests from dupe checking --- sonar-project.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sonar-project.properties b/sonar-project.properties index 00f44a64e..1bd93ed9e 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,2 +1,2 @@ # Ignore code duplicates in the examples -sonar.cpd.exclusions=examples/**/* \ No newline at end of file +sonar.cpd.exclusions=examples/**/*,powertools-e2e-tests/**/* \ No newline at end of file From 6303a75ea83575e7b761d662f7bbc09609b15577 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Thu, 3 Aug 2023 20:03:02 +0200 Subject: [PATCH 68/79] Rename sonar file --- sonar-project.properties => .sonarcloud.properties | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename sonar-project.properties => .sonarcloud.properties (100%) diff --git a/sonar-project.properties b/.sonarcloud.properties similarity index 100% rename from sonar-project.properties rename to .sonarcloud.properties From f91c2c12a02a30c66e135ec217a72e41808b0bf7 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Thu, 3 Aug 2023 20:10:39 +0200 Subject: [PATCH 69/79] Formatting --- .../lambda/powertools/e2e/Function.java | 9 ++++-- .../amazon/lambda/powertools/BatchE2ET.java | 29 +++++++++++++------ 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java b/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java index 1b234fefb..64f5a02c2 100644 --- a/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java +++ b/powertools-e2e-tests/handlers/batch/src/main/java/software/amazon/lambda/powertools/e2e/Function.java @@ -134,7 +134,8 @@ public Object createResult(String input, Context context) { LOGGER.info(input); - PojoSerializer serializer = LambdaEventSerializers.serializerFor(SQSEvent.class, this.getClass().getClassLoader()); + PojoSerializer serializer = + LambdaEventSerializers.serializerFor(SQSEvent.class, this.getClass().getClassLoader()); SQSEvent event = serializer.fromJson(input); if (event.getRecords().get(0).getEventSource().equals("aws:sqs")) { LOGGER.info("Running for SQS"); @@ -142,7 +143,8 @@ public Object createResult(String input, Context context) { return sqsHandler.processBatch(event, context); } - PojoSerializer kinesisSerializer = LambdaEventSerializers.serializerFor(KinesisEvent.class, this.getClass().getClassLoader()); + PojoSerializer kinesisSerializer = + LambdaEventSerializers.serializerFor(KinesisEvent.class, this.getClass().getClassLoader()); KinesisEvent kinesisEvent = kinesisSerializer.fromJson(input); if (kinesisEvent.getRecords().get(0).getEventSource().equals("aws:kinesis")) { LOGGER.info("Running for Kinesis"); @@ -150,7 +152,8 @@ public Object createResult(String input, Context context) { } // Well, let's try dynamo - PojoSerializer ddbSerializer = LambdaEventSerializers.serializerFor(DynamodbEvent.class, this.getClass().getClassLoader()); + PojoSerializer ddbSerializer = + LambdaEventSerializers.serializerFor(DynamodbEvent.class, this.getClass().getClassLoader()); LOGGER.info("Running for DynamoDB"); DynamodbEvent ddbEvent = ddbSerializer.fromJson(input); return ddbHandler.processBatch(ddbEvent, context); diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java index fea007fc1..2997fb0ab 100644 --- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java @@ -14,15 +14,33 @@ package software.amazon.lambda.powertools; +import static org.assertj.core.api.Assertions.assertThat; +import static software.amazon.lambda.powertools.testutils.Infrastructure.FUNCTION_NAME_OUTPUT; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.jupiter.api.*; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import software.amazon.awssdk.core.SdkBytes; import software.amazon.awssdk.http.SdkHttpClient; import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; -import software.amazon.awssdk.services.dynamodb.model.*; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; +import software.amazon.awssdk.services.dynamodb.model.DeleteItemRequest; +import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; +import software.amazon.awssdk.services.dynamodb.model.ScanRequest; +import software.amazon.awssdk.services.dynamodb.model.ScanResponse; import software.amazon.awssdk.services.kinesis.KinesisClient; import software.amazon.awssdk.services.kinesis.model.PutRecordsRequest; import software.amazon.awssdk.services.kinesis.model.PutRecordsRequestEntry; @@ -33,13 +51,6 @@ import software.amazon.lambda.powertools.testutils.Infrastructure; import software.amazon.lambda.powertools.utilities.JsonConfig; -import java.util.*; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -import static org.assertj.core.api.Assertions.assertThat; -import static software.amazon.lambda.powertools.testutils.Infrastructure.FUNCTION_NAME_OUTPUT; - public class BatchE2ET { private static final SdkHttpClient httpClient = UrlConnectionHttpClient.builder().build(); private static final Region region = Region.of(System.getProperty("AWS_DEFAULT_REGION", "eu-west-1")); From db0e7f1aa923a5f26a92ad3c8d6cf054d37c5bfa Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Fri, 4 Aug 2023 08:20:31 +0200 Subject: [PATCH 70/79] Update docs/utilities/batch.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jérôme Van Der Linden <117538+jeromevdl@users.noreply.github.com> --- docs/utilities/batch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/utilities/batch.md b/docs/utilities/batch.md index 712afdd4a..188cbafc7 100644 --- a/docs/utilities/batch.md +++ b/docs/utilities/batch.md @@ -55,7 +55,7 @@ journey This behavior changes when you enable Report Batch Item Failures feature in your Lambda function event source configuration: -* [**SQS queues**](#sqs-standard). Only messages reported as failure will return to the queue for a retry, while successful ones will be deleted. + * [**SQS queues**](#sqs-standard). Only messages reported as failure will return to the queue for a retry, while successful ones will be deleted. * [**Kinesis data streams**](#kinesis-and-dynamodb-streams) and [**DynamoDB streams**](#kinesis-and-dynamodb-streams). Single reported failure will use its sequence number as the stream checkpoint. Multiple reported failures will use the lowest sequence number as checkpoint. From fa9aa11e879365b6fe1f017074cea9fc6a6c84ca Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Fri, 4 Aug 2023 08:20:51 +0200 Subject: [PATCH 71/79] Update docs/utilities/batch.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jérôme Van Der Linden <117538+jeromevdl@users.noreply.github.com> --- docs/utilities/batch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/utilities/batch.md b/docs/utilities/batch.md index 188cbafc7..19cc53009 100644 --- a/docs/utilities/batch.md +++ b/docs/utilities/batch.md @@ -58,7 +58,7 @@ This behavior changes when you enable Report Batch Item Failures feature in your * [**SQS queues**](#sqs-standard). Only messages reported as failure will return to the queue for a retry, while successful ones will be deleted. * [**Kinesis data streams**](#kinesis-and-dynamodb-streams) and [**DynamoDB streams**](#kinesis-and-dynamodb-streams). Single reported failure will use its sequence number as the stream checkpoint. - Multiple reported failures will use the lowest sequence number as checkpoint. + Multiple reported failures will use the lowest sequence number as checkpoint. With this utility, batch records are processed individually – only messages that failed to be processed return to the queue or stream for a further retry. You simply build a `BatchProcessor` in your handler, From 710a94c3a1c434847ad7626f27d9314d01ce9220 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Fri, 4 Aug 2023 08:52:20 +0200 Subject: [PATCH 72/79] Address review comments --- docs/utilities/batch.md | 214 +++++++++++++++--- .../amazon/lambda/powertools/BatchE2ET.java | 4 +- .../powertools/testutils/Infrastructure.java | 10 +- 3 files changed, 193 insertions(+), 35 deletions(-) diff --git a/docs/utilities/batch.md b/docs/utilities/batch.md index 712afdd4a..5b70c95e9 100644 --- a/docs/utilities/batch.md +++ b/docs/utilities/batch.md @@ -28,7 +28,6 @@ stateDiagram-v2 **Key Features** - * Reports batch item failures to reduce number of retries for a record upon errors * Simple interface to process each batch record * Integrates with Java Events library and the deserialization module @@ -56,9 +55,9 @@ This behavior changes when you enable Report Batch Item Failures feature in your * [**SQS queues**](#sqs-standard). Only messages reported as failure will return to the queue for a retry, while successful ones will be deleted. - * [**Kinesis data streams**](#kinesis-and-dynamodb-streams) and [**DynamoDB streams**](#kinesis-and-dynamodb-streams). - Single reported failure will use its sequence number as the stream checkpoint. - Multiple reported failures will use the lowest sequence number as checkpoint. +* [**Kinesis data streams**](#kinesis-and-dynamodb-streams) and [**DynamoDB streams**](#kinesis-and-dynamodb-streams). +Single reported failure will use its sequence number as the stream checkpoint. +Multiple reported failures will use the lowest sequence number as checkpoint. With this utility, batch records are processed individually – only messages that failed to be processed return to the queue or stream for a further retry. You simply build a `BatchProcessor` in your handler, @@ -74,7 +73,7 @@ internally and an appropriate partial response for the message source is returne ## Install We simply add `powertools-batch` to our build dependencies. Note - if you are using other Powertools -modules that require code-weaving, you will need to configure that also. Batch does not. +modules that require code-weaving, such as `powertools-core`, you will need to configure that also. === "Maven" @@ -105,15 +104,22 @@ modules that require code-weaving, you will need to configure that also. Batch d ## Getting Started For this feature to work, you need to **(1)** configure your Lambda function event source to use `ReportBatchItemFailures`, -and **(2)** return [a specific response](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#services-sqs-batchfailurereporting){target="_blank" rel="nofollow"} -to report which records failed to be processed. +and **(2)** return a specific response to report which records failed to be processed. -You use your preferred deployment framework to set the correct configuration while this utility handles the correct response to be returned. +You can use your preferred deployment framework to set the correct configuration while this utility, +while the `powertools-batch` module handles generating the response, which simply needs to be returned as the result of +your Lambda handler. A complete [Serverless Application Model](https://aws.amazon.com/serverless/sam/) example can be found [here](https://github.com/aws-powertools/powertools-lambda-java/tree/main/examples/powertools-examples-batch) covering all of the batch sources. +For more information on configuring `ReportBatchItemFailures`, +see the details for [SQS](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#services-sqs-batchfailurereporting), +[Kinesis](https://docs.aws.amazon.com/lambda/latest/dg/with-kinesis.html#services-kinesis-batchfailurereporting),and +[DynamoDB Streams](https://docs.aws.amazon.com/lambda/latest/dg/with-ddb.html#services-ddb-batchfailurereporting). + + !!! note "You do not need any additional IAM permissions to use this utility, except for what each event source requires." @@ -122,18 +128,16 @@ all of the batch sources. === "App.java" - ```java + ```java hl_lines="10 13-15 20 25" import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.RequestHandler; import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; import com.amazonaws.services.lambda.runtime.events.SQSEvent; - import org.apache.logging.log4j.LogManager; - import org.apache.logging.log4j.Logger; import software.amazon.lambda.powertools.batch.BatchMessageHandlerBuilder; import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; public class SqsBatchHandler implements RequestHandler { - private final static Logger LOGGER = LogManager.getLogger(SqsBatchHandler.class); + private final BatchMessageHandler handler; public SqsBatchHandler() { @@ -149,7 +153,7 @@ all of the batch sources. private void processMessage(Product p, Context c) { - LOGGER.info("Processing product " + p); + // Process the product } } @@ -200,23 +204,62 @@ all of the batch sources. } ``` +=== "Example Event" + + ```json + { + "Records": [ + { + "messageId": "d9144555-9a4f-4ec3-99a0-34ce359b4b54", + "receiptHandle": "13e7f7851d2eaa5c01f208ebadbf1e72==", + "body": "{\n \"id\": 1234,\n \"name\": \"product\",\n \"price\": 42\n}", + "attributes": { + "ApproximateReceiveCount": "1", + "SentTimestamp": "1601975706495", + "SenderId": "AROAIFU437PVZ5L2J53F5", + "ApproximateFirstReceiveTimestamp": "1601975706499" + }, + "messageAttributes": { + }, + "md5OfBody": "13e7f7851d2eaa5c01f208ebadbf1e72", + "eventSource": "aws:sqs", + "eventSourceARN": "arn:aws:sqs:eu-central-1:123456789012:TestLambda", + "awsRegion": "eu-central-1" + }, + { + "messageId": "e9144555-9a4f-4ec3-99a0-34ce359b4b54", + "receiptHandle": "13e7f7851d2eaa5c01f208ebadbf1e72==", + "body": "{\n \"id\": 12345,\n \"name\": \"product5\",\n \"price\": 45\n}", + "attributes": { + "ApproximateReceiveCount": "1", + "SentTimestamp": "1601975706495", + "SenderId": "AROAIFU437PVZ5L2J53F5", + "ApproximateFirstReceiveTimestamp": "1601975706499" + }, + "messageAttributes": { + }, + "md5OfBody": "13e7f7851d2eaa5c01f208ebadbf1e72", + "eventSource": "aws:sqs", + "eventSourceARN": "arn:aws:sqs:eu-central-1:123456789012:TestLambda", + "awsRegion": "eu-central-1" + }] + } + ``` + ## Processing messages from Kinesis Streams === "App.java" - ```java + ```java hl_lines="10 13-15 20 24" import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.RequestHandler; import com.amazonaws.services.lambda.runtime.events.KinesisEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; - import org.apache.logging.log4j.LogManager; - import org.apache.logging.log4j.Logger; import software.amazon.lambda.powertools.batch.BatchMessageHandlerBuilder; import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; public class KinesisBatchHandler implements RequestHandler { - private final static Logger LOGGER = LogManager.getLogger(org.demo.batch.sqs.SqsBatchHandler.class); private final BatchMessageHandler handler; public KinesisBatchHandler() { @@ -231,7 +274,7 @@ all of the batch sources. } private void processMessage(Product p, Context c) { - LOGGER.info("Processing product " + p); + // process the product } } @@ -282,24 +325,62 @@ all of the batch sources. } ``` - +=== "Example Event" + + ```json + { + "Records": [ + { + "kinesis": { + "partitionKey": "partitionKey-03", + "kinesisSchemaVersion": "1.0", + "data": "eyJpZCI6MTIzNCwgIm5hbWUiOiJwcm9kdWN0IiwgInByaWNlIjo0Mn0=", + "sequenceNumber": "49545115243490985018280067714973144582180062593244200961", + "approximateArrivalTimestamp": 1428537600, + "encryptionType": "NONE" + }, + "eventSource": "aws:kinesis", + "eventID": "shardId-000000000000:49545115243490985018280067714973144582180062593244200961", + "invokeIdentityArn": "arn:aws:iam::EXAMPLE", + "eventVersion": "1.0", + "eventName": "aws:kinesis:record", + "eventSourceARN": "arn:aws:kinesis:EXAMPLE", + "awsRegion": "eu-central-1" + }, + { + "kinesis": { + "partitionKey": "partitionKey-03", + "kinesisSchemaVersion": "1.0", + "data": "eyJpZCI6MTIzNDUsICJuYW1lIjoicHJvZHVjdDUiLCAicHJpY2UiOjQ1fQ==", + "sequenceNumber": "49545115243490985018280067714973144582180062593244200962", + "approximateArrivalTimestamp": 1428537600, + "encryptionType": "NONE" + }, + "eventSource": "aws:kinesis", + "eventID": "shardId-000000000000:49545115243490985018280067714973144582180062593244200961", + "invokeIdentityArn": "arn:aws:iam::EXAMPLE", + "eventVersion": "1.0", + "eventName": "aws:kinesis:record", + "eventSourceARN": "arn:aws:kinesis:EXAMPLE", + "awsRegion": "eu-central-1" + } + ] + } + ``` ## Processing messages from DynamoDB Streams === "App.java" - ```java + ```java hl_lines="10 13-15 20 24" import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.RequestHandler; import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; - import org.apache.logging.log4j.LogManager; - import org.apache.logging.log4j.Logger; import software.amazon.lambda.powertools.batch.BatchMessageHandlerBuilder; import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; public class DynamoDBStreamBatchHandler implements RequestHandler { - private final static Logger LOGGER = LogManager.getLogger(DynamoDBStreamBatchHandler.class); private final BatchMessageHandler handler; public DynamoDBStreamBatchHandler() { @@ -307,16 +388,89 @@ all of the batch sources. .withDynamoDbBatchHandler() .buildWithRawMessageHandler(this::processMessage); } - - private void processMessage(DynamodbEvent.DynamodbStreamRecord dynamodbStreamRecord, Context context) { - LOGGER.info("Processing DynamoDB Stream Record" + dynamodbStreamRecord); - } - + @Override public StreamsEventResponse handleRequest(DynamodbEvent ddbEvent, Context context) { return handler.processBatch(ddbEvent, context); } - - + + private void processMessage(DynamodbEvent.DynamodbStreamRecord dynamodbStreamRecord, Context context) { + // Process the change record + } } + ``` + +=== "Example Event" + + ```json + { + "Records": [ + { + "eventID": "c4ca4238a0b923820dcc509a6f75849b", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "eu-central-1", + "dynamodb": { + "Keys": { + "Id": { + "N": "101" + } + }, + "NewImage": { + "Message": { + "S": "New item!" + }, + "Id": { + "N": "101" + } + }, + "ApproximateCreationDateTime": 1428537600, + "SequenceNumber": "4421584500000000017450439091", + "SizeBytes": 26, + "StreamViewType": "NEW_AND_OLD_IMAGES" + }, + "eventSourceARN": "arn:aws:dynamodb:eu-central-1:123456789012:table/ExampleTableWithStream/stream/2015-06-27T00:48:05.899", + "userIdentity": { + "principalId": "dynamodb.amazonaws.com", + "type": "Service" + } + }, + { + "eventID": "c81e728d9d4c2f636f067f89cc14862c", + "eventName": "MODIFY", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "eu-central-1", + "dynamodb": { + "Keys": { + "Id": { + "N": "101" + } + }, + "NewImage": { + "Message": { + "S": "This item has changed" + }, + "Id": { + "N": "101" + } + }, + "OldImage": { + "Message": { + "S": "New item!" + }, + "Id": { + "N": "101" + } + }, + "ApproximateCreationDateTime": 1428537600, + "SequenceNumber": "4421584500000000017450439092", + "SizeBytes": 59, + "StreamViewType": "NEW_AND_OLD_IMAGES" + }, + "eventSourceARN": "arn:aws:dynamodb:eu-central-1:123456789012:table/ExampleTableWithStream/stream/2015-06-27T00:48:05.899" + } + ] + } ``` \ No newline at end of file diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java index 2997fb0ab..c5f74594d 100644 --- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/BatchE2ET.java @@ -88,7 +88,6 @@ public static void setup() { infrastructure = Infrastructure.builder() .testName(BatchE2ET.class.getSimpleName()) .pathToFunction("batch") - .idempotencyTable("idempo" + random) .queue(queueName) .ddbStreamsTableName(ddbStreamsTestTable) .kinesisStream(kinesisStreamName) @@ -119,9 +118,8 @@ public static void setup() { @AfterAll public static void tearDown() { - // TODO bring this back after testing if (infrastructure != null) { - // infrastructure.destroy(); + infrastructure.destroy(); } } diff --git a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java index 473a540dc..2a1af093c 100644 --- a/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java +++ b/powertools-e2e-tests/src/test/java/software/amazon/lambda/powertools/testutils/Infrastructure.java @@ -115,7 +115,6 @@ public class Infrastructure { private final String kinesisStream; private final String largeMessagesBucket; private String ddbStreamsTableName; - private String functionName; private Object cfnTemplate; private String cfnAssetDirectory; @@ -285,7 +284,12 @@ private Stack createStackWithLambda() { .maxReceiveCount(1) // do not retry in case of error .build(); sqsQueue.grantConsumeMessages(function); - SqsEventSource sqsEventSource = SqsEventSource.Builder.create(sqsQueue).enabled(true).batchSize(1).build(); + SqsEventSource sqsEventSource = SqsEventSource.Builder + .create(sqsQueue) + .enabled(true) + .reportBatchItemFailures(true) + .batchSize(1) + .build(); function.addEventSource(sqsEventSource); CfnOutput.Builder .create(stack, "QueueURL") @@ -305,6 +309,7 @@ private Stack createStackWithLambda() { .create(stream) .enabled(true) .batchSize(3) + .reportBatchItemFailures(true) .startingPosition(StartingPosition.TRIM_HORIZON) .maxBatchingWindow(Duration.seconds(1)) .build(); @@ -327,6 +332,7 @@ private Stack createStackWithLambda() { .batchSize(1) .startingPosition(StartingPosition.TRIM_HORIZON) .maxBatchingWindow(Duration.seconds(1)) + .reportBatchItemFailures(true) .build(); function.addEventSource(ddbEventSource); CfnOutput.Builder.create(stack, "DdbStreamsTestTable").value(ddbStreamsTable.getTableName()).build(); From 294313fc1b428e13976adcf27528f6331b69abbf Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Fri, 4 Aug 2023 08:55:31 +0200 Subject: [PATCH 73/79] Missed one --- docs/utilities/batch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/utilities/batch.md b/docs/utilities/batch.md index e5a86a78d..f2f3d5061 100644 --- a/docs/utilities/batch.md +++ b/docs/utilities/batch.md @@ -65,7 +65,7 @@ and return its response from the handler's `processMessage` implementation. Exce internally and an appropriate partial response for the message source is returned to Lambda for you. !!! warning - While this utility lowers the chance of processing messages more than once, but it is not guaranteed. + While this utility lowers the chance of processing messages more than once, it is still not guaranteed. We recommend implementing processing logic in an idempotent manner wherever possible, for instance, by taking advantage of [the idempotency module](idempotency.md). More details on how Lambda works with SQS can be found in the [AWS documentation](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html) From c63fb28bf0525d4fe7b2cce2df04139363433244 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Fri, 4 Aug 2023 09:05:11 +0200 Subject: [PATCH 74/79] Formatting --- .../batch/builder/AbstractBatchMessageHandlerBuilder.java | 3 +-- .../batch/builder/DynamoDbBatchMessageHandlerBuilder.java | 3 +-- .../batch/builder/KinesisBatchMessageHandlerBuilder.java | 3 +-- .../batch/builder/SqsBatchMessageHandlerBuilder.java | 3 +-- .../batch/handler/DynamoDbBatchMessageHandler.java | 5 ++--- .../batch/handler/KinesisStreamsBatchMessageHandler.java | 7 +++---- .../powertools/batch/handler/SqsBatchMessageHandler.java | 7 +++---- .../lambda/powertools/batch/DdbBatchProcessorTest.java | 7 +++---- .../lambda/powertools/batch/KinesisBatchProcessorTest.java | 7 +++---- .../lambda/powertools/batch/SQSBatchProcessorTest.java | 7 +++---- 10 files changed, 21 insertions(+), 31 deletions(-) diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/AbstractBatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/AbstractBatchMessageHandlerBuilder.java index e28429503..9b0647770 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/AbstractBatchMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/AbstractBatchMessageHandlerBuilder.java @@ -15,10 +15,9 @@ package software.amazon.lambda.powertools.batch.builder; import com.amazonaws.services.lambda.runtime.Context; -import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; - import java.util.function.BiConsumer; import java.util.function.Consumer; +import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; /** * An abstract class to capture common arguments used across all the message-binding-specific batch processing diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/DynamoDbBatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/DynamoDbBatchMessageHandlerBuilder.java index 6e42f8d7b..8513322b3 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/DynamoDbBatchMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/DynamoDbBatchMessageHandlerBuilder.java @@ -17,12 +17,11 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; +import java.util.function.BiConsumer; import software.amazon.lambda.powertools.batch.exception.DeserializationNotSupportedException; import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; import software.amazon.lambda.powertools.batch.handler.DynamoDbBatchMessageHandler; -import java.util.function.BiConsumer; - /** * Builds a batch processor for processing DynamoDB Streams batch events **/ diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/KinesisBatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/KinesisBatchMessageHandlerBuilder.java index 52426bdc7..30bfcab65 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/KinesisBatchMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/KinesisBatchMessageHandlerBuilder.java @@ -17,11 +17,10 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.KinesisEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; +import java.util.function.BiConsumer; import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; import software.amazon.lambda.powertools.batch.handler.KinesisStreamsBatchMessageHandler; -import java.util.function.BiConsumer; - /** * Builds a batch processor for processing Kinesis Streams batch events */ diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/SqsBatchMessageHandlerBuilder.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/SqsBatchMessageHandlerBuilder.java index bf5d18a0b..ee2dc23f6 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/SqsBatchMessageHandlerBuilder.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/builder/SqsBatchMessageHandlerBuilder.java @@ -17,11 +17,10 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; import com.amazonaws.services.lambda.runtime.events.SQSEvent; +import java.util.function.BiConsumer; import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; import software.amazon.lambda.powertools.batch.handler.SqsBatchMessageHandler; -import java.util.function.BiConsumer; - /** * Builds a batch processor for the SQS event source. */ diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/DynamoDbBatchMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/DynamoDbBatchMessageHandler.java index 9f4491329..aa6eba839 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/DynamoDbBatchMessageHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/DynamoDbBatchMessageHandler.java @@ -17,13 +17,12 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.ArrayList; import java.util.List; import java.util.function.BiConsumer; import java.util.function.Consumer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A batch message processor for DynamoDB Streams batches. diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/KinesisStreamsBatchMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/KinesisStreamsBatchMessageHandler.java index 4d215bf76..fe1aaf354 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/KinesisStreamsBatchMessageHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/KinesisStreamsBatchMessageHandler.java @@ -18,14 +18,13 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.KinesisEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import software.amazon.lambda.powertools.utilities.EventDeserializer; - import java.util.ArrayList; import java.util.List; import java.util.function.BiConsumer; import java.util.function.Consumer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.lambda.powertools.utilities.EventDeserializer; /** * A batch message processor for Kinesis Streams batch processing. diff --git a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java index d1c75e966..b3c416a69 100644 --- a/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java +++ b/powertools-batch/src/main/java/software/amazon/lambda/powertools/batch/handler/SqsBatchMessageHandler.java @@ -17,13 +17,12 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; import com.amazonaws.services.lambda.runtime.events.SQSEvent; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import software.amazon.lambda.powertools.utilities.EventDeserializer; - import java.util.ArrayList; import java.util.function.BiConsumer; import java.util.function.Consumer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.lambda.powertools.utilities.EventDeserializer; /** * A batch message processor for SQS batches. diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/DdbBatchProcessorTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/DdbBatchProcessorTest.java index 0983838c7..9e2c211e2 100644 --- a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/DdbBatchProcessorTest.java +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/DdbBatchProcessorTest.java @@ -14,18 +14,17 @@ package software.amazon.lambda.powertools.batch; +import static org.assertj.core.api.Assertions.assertThat; + import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.DynamodbEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; import com.amazonaws.services.lambda.runtime.tests.annotations.Event; +import java.util.concurrent.atomic.AtomicBoolean; import org.junit.jupiter.params.ParameterizedTest; import org.mockito.Mock; import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; -import java.util.concurrent.atomic.AtomicBoolean; - -import static org.assertj.core.api.Assertions.assertThat; - public class DdbBatchProcessorTest { @Mock diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/KinesisBatchProcessorTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/KinesisBatchProcessorTest.java index 9d9c355d2..d78638e1d 100644 --- a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/KinesisBatchProcessorTest.java +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/KinesisBatchProcessorTest.java @@ -14,19 +14,18 @@ package software.amazon.lambda.powertools.batch; +import static org.assertj.core.api.Assertions.assertThat; + import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.KinesisEvent; import com.amazonaws.services.lambda.runtime.events.StreamsEventResponse; import com.amazonaws.services.lambda.runtime.tests.annotations.Event; +import java.util.concurrent.atomic.AtomicBoolean; import org.junit.jupiter.params.ParameterizedTest; import org.mockito.Mock; import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; import software.amazon.lambda.powertools.batch.model.Product; -import java.util.concurrent.atomic.AtomicBoolean; - -import static org.assertj.core.api.Assertions.assertThat; - public class KinesisBatchProcessorTest { @Mock diff --git a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java index 4e3623924..2f9429fa3 100644 --- a/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java +++ b/powertools-batch/src/test/java/software/amazon/lambda/powertools/batch/SQSBatchProcessorTest.java @@ -14,19 +14,18 @@ package software.amazon.lambda.powertools.batch; +import static org.assertj.core.api.Assertions.assertThat; + import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.events.SQSBatchResponse; import com.amazonaws.services.lambda.runtime.events.SQSEvent; import com.amazonaws.services.lambda.runtime.tests.annotations.Event; +import java.util.concurrent.atomic.AtomicBoolean; import org.junit.jupiter.params.ParameterizedTest; import org.mockito.Mock; import software.amazon.lambda.powertools.batch.handler.BatchMessageHandler; import software.amazon.lambda.powertools.batch.model.Product; -import java.util.concurrent.atomic.AtomicBoolean; - -import static org.assertj.core.api.Assertions.assertThat; - public class SQSBatchProcessorTest { @Mock private Context context; From e27a31fce53fbfc410b121cf53c89c78604ae3bb Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Fri, 4 Aug 2023 10:48:14 +0200 Subject: [PATCH 75/79] Cleanup doc linking --- docs/utilities/sqs_large_message_handling.md | 8 +++++--- mkdocs.yml | 4 +++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/utilities/sqs_large_message_handling.md b/docs/utilities/sqs_large_message_handling.md index 6308f1c79..0924d01cf 100644 --- a/docs/utilities/sqs_large_message_handling.md +++ b/docs/utilities/sqs_large_message_handling.md @@ -1,11 +1,13 @@ --- -title: SQS Large Message Handling +title: SQS Large Message Handling (Deprecated) description: Utility --- !!! warning -This module is now deprecated and will be removed in version 2. -See [Large Message Handling](large_messages.md) for the new module (`powertools-large-messages`) documentation. + This module is now deprecated and will be removed in version 2. + See [Large Message Handling](large_messages.md) and + [the migration guide](http://localhost:8000/lambda-java/utilities/large_messages/#migration-from-the-sqs-large-message-utility) + for the new module (`powertools-large-messages`) documentation The large message handling utility handles SQS messages which have had their payloads offloaded to S3 due to them being larger than the SQS maximum. diff --git a/mkdocs.yml b/mkdocs.yml index a59855923..d54ece508 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -15,10 +15,12 @@ nav: - utilities/parameters.md - utilities/large_messages.md - utilities/batch.md - - utilities/sqs_batch.md - utilities/validation.md - utilities/custom_resources.md - utilities/serialization.md + - Deprecated: + - utilities/sqs_large_message_handling.md + - utilities/sqs_batch.md - Processes: - processes/maintainers.md From 0929fb1f764bac929a002a4d2e2e541ff71f281a Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Fri, 4 Aug 2023 11:58:28 +0200 Subject: [PATCH 76/79] More doco --- docs/utilities/batch.md | 115 +++++++++++++++++++++++++++++++++++----- 1 file changed, 103 insertions(+), 12 deletions(-) diff --git a/docs/utilities/batch.md b/docs/utilities/batch.md index f2f3d5061..6ac5f3911 100644 --- a/docs/utilities/batch.md +++ b/docs/utilities/batch.md @@ -124,9 +124,9 @@ see the details for [SQS](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs. !!! note "You do not need any additional IAM permissions to use this utility, except for what each event source requires." -## Processing messages from SQS +### Processing messages from SQS -=== "App.java" +=== "SQSBatchHandler" ```java hl_lines="10 13-15 20 25" import com.amazonaws.services.lambda.runtime.Context; @@ -159,7 +159,7 @@ see the details for [SQS](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs. } ``` -=== "Product.java" +=== "SQS Product" ```java public class Product { @@ -204,7 +204,7 @@ see the details for [SQS](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs. } ``` -=== "Example Event" +=== "SQS Example Event" ```json { @@ -246,9 +246,9 @@ see the details for [SQS](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs. } ``` -## Processing messages from Kinesis Streams +### Processing messages from Kinesis Streams -=== "App.java" +=== "KinesisBatchHandler" ```java hl_lines="10 13-15 20 24" import com.amazonaws.services.lambda.runtime.Context; @@ -280,7 +280,7 @@ see the details for [SQS](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs. } ``` -=== "Product.java" +=== "Kinesis Product" ```java public class Product { @@ -325,7 +325,7 @@ see the details for [SQS](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs. } ``` -=== "Example Event" +=== "Kinesis Example Event" ```json { @@ -367,9 +367,9 @@ see the details for [SQS](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs. ] } ``` -## Processing messages from DynamoDB Streams +### Processing messages from DynamoDB Streams -=== "App.java" +=== "DynamoDBStreamBatchHandler" ```java hl_lines="10 13-15 20 24" import com.amazonaws.services.lambda.runtime.Context; @@ -400,7 +400,7 @@ see the details for [SQS](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs. } ``` -=== "Example Event" +=== "DynamoDB Example Event" ```json { @@ -473,4 +473,95 @@ see the details for [SQS](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs. } ] } - ``` \ No newline at end of file + ``` + + +## Handling Messages + +### Raw message and deserialized message handlers +You must provide either a raw message handler, or a deserialized message handler. The raw message handler receives +the envelope record type relevant for the particular event source - for instance, the SQS event source provides +[SQSMessage](https://javadoc.io/doc/com.amazonaws/aws-lambda-java-events/2.2.2/com/amazonaws/services/lambda/runtime/events/SQSEvent.html) +instances. The deserialized message handler extracts the body from this envelope, and deserializes it to a user-defined +type. + +In general, the deserialized message handler should be used unless you need access to information on the envelope. + +=== "Raw Message Handler" + + ```java + public void setup() { + BatchMessageHandler handler = new BatchMessageHandlerBuilder() + .withSqsBatchHandler() + .buildWithRawMessageHandler(this::processRawMessage); + } + + private void processRawMessage(SQSEvent.SQSMessage sqsMessage) { + // Do something with the raw message + } + + ``` + +=== "Deserialized Message Handler" + + ```java + public void setup() { + BatchMessageHandler handler = new BatchMessageHandlerBuilder() + .withSqsBatchHandler() + .buildWitMessageHandler(this::processRawMessage, Product.class); + } + + private void processMessage(Product product) { + // Do something with the raw message + } + + ``` + +### Success and failure handlers + +You can register a success or failure handler which will be invoked as each message is processed by the batch +module. + +Handlers can be provided when building the batch processor and are available for all event sources. +For instance for DynamoDB: + +```java +BatchMessageHandler handler = new BatchMessageHandlerBuilder() + .withDynamoDbBatchHandler() + .withSuccessHandler((m) -> { + // Success handler receives the raw message + LOGGER.info("Message with sequenceNumber {} was successfully processed", + m.getDynamodb().getSequenceNumber()); + }) + .withFailureHandler((m, e) -> { + // Failure handler receives the raw message and the exception thrown + LOGGER.info("Message with sequenceNumber {} failed to be processed: {}" + , e.getDynamodb().getSequenceNumber(), e); + }) + .buildWithMessageHander(this::processMessage); +``` + +!!! info + If the success handler throws an exception, the item it is processing will be marked as failed by the + batch processor. + If the failure handler throws, the batch processing will continue; the item it is processing has + already been marked as failed. + + +### Lambda Context + +Both raw and deserialized message handlers can choose to take the Lambda context as an argument if they +need it, or not: + +```java + public class ClassWithHandlers { + + private void processMessage(Product product) { + // Do something with the raw message + } + + private void processMessageWithContext(Product product, Context context) { + // Do something with the raw message and the lambda Context + } + } +``` From 3d8aca8b25e3e73e3f9a6cba32750dbb8e8cdad2 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Fri, 4 Aug 2023 13:06:16 +0200 Subject: [PATCH 77/79] Update docs/utilities/batch.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jérôme Van Der Linden <117538+jeromevdl@users.noreply.github.com> --- docs/utilities/batch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/utilities/batch.md b/docs/utilities/batch.md index 6ac5f3911..e13059fc3 100644 --- a/docs/utilities/batch.md +++ b/docs/utilities/batch.md @@ -512,7 +512,7 @@ In general, the deserialized message handler should be used unless you need acce } private void processMessage(Product product) { - // Do something with the raw message + // Do something with the deserialized message } ``` From 1ff4658695dee10cc41a16a63d425aa01e046d15 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Fri, 4 Aug 2023 13:13:15 +0200 Subject: [PATCH 78/79] Update batch.md Address review comments --- docs/utilities/batch.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/utilities/batch.md b/docs/utilities/batch.md index e13059fc3..6b38b438c 100644 --- a/docs/utilities/batch.md +++ b/docs/utilities/batch.md @@ -483,7 +483,8 @@ You must provide either a raw message handler, or a deserialized message handler the envelope record type relevant for the particular event source - for instance, the SQS event source provides [SQSMessage](https://javadoc.io/doc/com.amazonaws/aws-lambda-java-events/2.2.2/com/amazonaws/services/lambda/runtime/events/SQSEvent.html) instances. The deserialized message handler extracts the body from this envelope, and deserializes it to a user-defined -type. +type. Note that deserialized message handlers are not relevant for the DynamoDB provider, as the format of the inner +message is fixed by DynamoDB. In general, the deserialized message handler should be used unless you need access to information on the envelope. @@ -520,7 +521,10 @@ In general, the deserialized message handler should be used unless you need acce ### Success and failure handlers You can register a success or failure handler which will be invoked as each message is processed by the batch -module. +module. This may be useful for reporting - for instance, writing metrics or logging failures. + +These handlers are optional. Batch failures are handled by the module regardless of whether or not you +provide a custom failure handler. Handlers can be provided when building the batch processor and are available for all event sources. For instance for DynamoDB: @@ -534,7 +538,7 @@ BatchMessageHandler handler = new BatchMess m.getDynamodb().getSequenceNumber()); }) .withFailureHandler((m, e) -> { - // Failure handler receives the raw message and the exception thrown + // Failure handler receives the raw message and the exception thrown. LOGGER.info("Message with sequenceNumber {} failed to be processed: {}" , e.getDynamodb().getSequenceNumber(), e); }) From 83fdf127e28413c5d56a2c58c87c68389cdb84f5 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Fri, 4 Aug 2023 13:53:46 +0200 Subject: [PATCH 79/79] Skip aspectj run --- powertools-batch/pom.xml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/powertools-batch/pom.xml b/powertools-batch/pom.xml index c788b206d..9e25dabd8 100644 --- a/powertools-batch/pom.xml +++ b/powertools-batch/pom.xml @@ -9,6 +9,18 @@ 1.17.0-SNAPSHOT + + + + dev.aspectj + aspectj-maven-plugin + + true + + + + + powertools-batch @@ -24,11 +36,6 @@ powertools-serialization ${project.version} - - org.aspectj - aspectjrt - compile -