Merge branch 'main' into 1776-json-replacer-fn

Author: dreamorosi

.github/workflows/codeql.yml

@@ -35,7 +35,7 @@ jobs:
           # repo_token: ${{ secrets.SCORECARD_TOKEN }}  # read-only fine-grained token to read branch protection settings
 
       - name: "Upload results"
-        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+        uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4
         with:
           name: SARIF file
           path: results.sarif

.github/workflows/ossf_scorecard.yml

@@ -45,7 +45,7 @@ jobs:
       - name: Zip output
         run: zip -r cdk.out.zip layers/cdk.out
       - name: Archive CDK artifacts
-        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+        uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4
         with:
           name: cdk-layer-artifact
           path: cdk.out.zip
@@ -97,7 +97,7 @@ jobs:
         with:
           ref: ${{ github.sha }}
       - name: Download CDK layer artifacts
-        uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
         with:
           path: cdk-layer-stack
           pattern: cdk-layer-stack-* # merge all Layer artifacts created per region earlier (reusable_deploy_layer_stack.yml; step "Save Layer ARN artifact")

.github/workflows/publish_layer.yml

@@ -53,7 +53,7 @@ jobs:
           script: |
             const script = require('.github/scripts/save_pr_details.js')
             await script({github, context, core})
-      - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+      - uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4
         with:
           name: pr
           path: pr.txt

.github/workflows/record_pr.yml

@@ -78,7 +78,7 @@ jobs:
       - name: Setup dependencies
         uses: ./.github/actions/cached-node-modules
       - name: Download artifact
-        uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
         with:
           name: ${{ inputs.artifact-name }}
       - name: Unzip artifact
@@ -93,7 +93,7 @@ jobs:
           cat cdk-layer-stack/${{ matrix.region }}-layer-version.txt
       - name: Save Layer ARN artifact
         if: ${{ inputs.stage == 'PROD' }}
-        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+        uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4
         with:
           name: cdk-layer-stack-${{ matrix.region }}
           path: ./cdk-layer-stack/* # NOTE: upload-artifact does not inherit working-directory setting.

.github/workflows/reusable_deploy_layer_stack.yml

@@ -96,7 +96,7 @@ jobs:
         run: |
           cp -r api site/
       - name: Create Artifact (Site)
-        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+        uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4
         with:
           name: site
           path: site

.github/workflows/reusable_publish_docs.yml

@@ -261,7 +261,7 @@ All records in the batch will be passed to this handler for processing, even if
 
 * **All records successfully processed**. We will return an empty list of item failures `{'batchItemFailures': []}`
 * **Partial success with some exceptions**. We will return a list of all item IDs/sequence numbers that failed processing
-* **All records failed to be processed**. We will raise `BatchProcessingError` exception with a list of all exceptions raised when processing
+* **All records failed to be processed**. We will throw a `FullBatchFailureError` error with a list of all the errors thrown while processing unless `throwOnFullBatchFailure` is disabled.
 
 The following sequence diagrams explain how each Batch processor behaves under different scenarios.
 
@@ -450,6 +450,18 @@ We can automatically inject the [Lambda context](https://docs.aws.amazon.com/lam
 --8<-- "examples/snippets/batch/accessLambdaContext.ts"
 ```
 
+### Working with full batch failures
+
+By default, the `BatchProcessor` will throw a `FullBatchFailureError` if all records in the batch fail to process, we do this to reflect the failure in your operational metrics.
+
+When working with functions that handle batches with a small number of records, or when you use errors as a flow control mechanism, this behavior might not be desirable as your function might generate an unnaturally high number of errors. When this happens, the [Lambda service will scale down the concurrency of your function](https://docs.aws.amazon.com/lambda/latest/dg/services-sqs-errorhandling.html#services-sqs-backoff-strategy){target="_blank"}, potentially impacting performance.
+
+For these scenarios, you can set the `throwOnFullBatchFailure` option to `false` when calling.
+
+```typescript hl_lines="17"
+--8<-- "examples/snippets/batch/noThrowOnFullBatchFailure.ts"
+```
+
 ### Extending BatchProcessor
 
 You might want to bring custom logic to the existing `BatchProcessor` to slightly override how we handle successes and failures.

docs/utilities/batch.md

@@ -0,0 +1,18 @@
+import {
+  BatchProcessor,
+  EventType,
+  processPartialResponse,
+} from '@aws-lambda-powertools/batch';
+import type { SQSHandler, SQSRecord } from 'aws-lambda';
+
+const processor = new BatchProcessor(EventType.SQS);
+
+const recordHandler = async (_record: SQSRecord): Promise<void> => {
+  // Process the record
+};
+
+export const handler: SQSHandler = async (event, context) =>
+  processPartialResponse(event, recordHandler, processor, {
+    context,
+    throwOnFullBatchFailure: false,
+  });

examples/snippets/batch/noThrowOnFullBatchFailure.ts

@@ -63,9 +63,8 @@ abstract class BasePartialBatchProcessor extends BasePartialProcessor {
   /**
    * Clean up logic to be run after processing a batch
    *
-   * If the entire batch failed, and the utility is not configured otherwise,
-   * this method will throw a `FullBatchFailureError` with the list of errors
-   * that occurred during processing.
+   * If the entire batch failed this method will throw a {@link FullBatchFailureError | `FullBatchFailureError`} with the list of
+   * errors that occurred during processing, unless the `throwOnFullBatchFailure` option is set to `false`.
    *
    * Otherwise, it will build the partial failure response based on the event type.
    */
@@ -74,7 +73,10 @@ abstract class BasePartialBatchProcessor extends BasePartialProcessor {
       return;
     }
 
-    if (this.entireBatchFailed()) {
+    if (
+      this.options?.throwOnFullBatchFailure !== false &&
+      this.entireBatchFailed()
+    ) {
       throw new FullBatchFailureError(this.errors);
     }
 

packages/batch/src/BasePartialBatchProcessor.ts

@@ -42,6 +42,31 @@ import type {
  *   });
  * ```
  *
+ * By default, if the entire batch fails, the function will throw an error.
+ * If you want to prevent this behavior, you can set the `throwOnFullBatchFailure` to `false`
+ *
+ * @example
+ * ```typescript
+ * import {
+ *   BatchProcessor,
+ *   EventType,
+ *   processPartialResponse,
+ * } from '@aws-lambda-powertools/batch';
+ * import type { KinesisStreamHandler, KinesisStreamRecord } from 'aws-lambda';
+ *
+ * const processor = new BatchProcessor(EventType.KinesisDataStreams);
+ *
+ * const recordHandler = async (record: KinesisStreamRecord): Promise<void> => {
+ *   const payload = JSON.parse(record.kinesis.data);
+ * };
+ *
+ * export const handler: KinesisStreamHandler = async (event, context) =>
+ *   processPartialResponse(event, recordHandler, processor, {
+ *     context,
+ *     throwOnFullBatchFailure: false
+ *   });
+ * ```
+ *
  * @param event The event object containing the batch of records
  * @param recordHandler Async function to process each record from the batch
  * @param processor Batch processor instance to handle the batch processing

packages/batch/src/processPartialResponse.ts

@@ -68,6 +68,30 @@ import type {
  *   });
  * ```
  *
+ * By default, if the entire batch fails, the function will throw an error.
+ * If you want to prevent this behavior, you can set the `throwOnFullBatchFailure` to `false`
+ *
+ * @example
+ * ```typescript
+ * import {
+ *   SqsFifoPartialProcessor,
+ *   processPartialResponseSync,
+ * } from '@aws-lambda-powertools/batch';
+ * import type { SQSRecord, SQSHandler } from 'aws-lambda';
+ *
+ * const processor = new SqsFifoPartialProcessor();
+ *
+ * const recordHandler = async (record: SQSRecord): Promise<void> => {
+ *   const payload = JSON.parse(record.body);
+ * };
+ *
+ * export const handler: SQSHandler = async (event, context) =>
+ *   processPartialResponseSync(event, recordHandler, processor, {
+ *     context,
+ *     throwOnFullBatchFailure: false
+ *   });
+ * ```
+ *
  * @param event The event object containing the batch of records
  * @param recordHandler Sync function to process each record from the batch
  * @param processor Batch processor instance to handle the batch processing

packages/batch/src/processPartialResponseSync.ts

@@ -13,6 +13,7 @@ import { BasePartialBatchProcessor } from './BasePartialBatchProcessor.js';
  * @template T The type of the batch processor, defaults to BasePartialBatchProcessor
  * @property context The context object provided by the AWS Lambda runtime
  * @property skipGroupOnError The option to group on error during processing
+ * @property throwOnFullBatchFailure The option to throw an error if the entire batch fails
  */
 type BatchProcessingOptions<T = BasePartialBatchProcessor> = {
   /**
@@ -25,6 +26,10 @@ type BatchProcessingOptions<T = BasePartialBatchProcessor> = {
    * If true skip the group on error during processing.
    */
   skipGroupOnError?: T extends SqsFifoPartialProcessor ? boolean : never;
+  /**
+   *  Set this to false to prevent throwing an error if the entire batch fails.
+   */
+  throwOnFullBatchFailure?: boolean;
 };
 
 /**