Merge branch 'develop' into feat/parameter-env-variable

Author: leandrodamascena

CHANGELOG.md

@@ -4,18 +4,39 @@
 <a name="unreleased"></a>
 # Unreleased
 
+## Bug Fixes
+
+* **batch:** handle early validation errors for pydantic models (poison pill) [#2091](https://github.com/awslabs/aws-lambda-powertools-python/issues/2091) ([#2099](https://github.com/awslabs/aws-lambda-powertools-python/issues/2099))
+
 ## Documentation
 
+* **batch:** use newly supported Json model ([#2100](https://github.com/awslabs/aws-lambda-powertools-python/issues/2100))
+* **homepage:** remove banner for end-of-support v1 ([#2098](https://github.com/awslabs/aws-lambda-powertools-python/issues/2098))
+* **idempotency:** fixes to testing your code section ([#2073](https://github.com/awslabs/aws-lambda-powertools-python/issues/2073))
 * **idempotency:** new sequence diagrams, fix idempotency record vs DynamoDB TTL confusion ([#2074](https://github.com/awslabs/aws-lambda-powertools-python/issues/2074))
 * **parser:** fix highlighted line ([#2064](https://github.com/awslabs/aws-lambda-powertools-python/issues/2064))
 
+## Features
+
+* **batch:** reduce boilerplate with process_partial_response ([#2090](https://github.com/awslabs/aws-lambda-powertools-python/issues/2090))
+* **idempotency:** allow custom sdk clients in DynamoDBPersistenceLayer ([#2087](https://github.com/awslabs/aws-lambda-powertools-python/issues/2087))
+
 ## Maintenance
 
+* **deps:** bump aws-xray-sdk from 2.11.0 to 2.12.0 ([#2080](https://github.com/awslabs/aws-lambda-powertools-python/issues/2080))
 * **deps:** bump peaceiris/actions-gh-pages from 3.9.2 to 3.9.3 ([#2069](https://github.com/awslabs/aws-lambda-powertools-python/issues/2069))
+* **deps-dev:** bump aws-cdk-lib from 2.72.1 to 2.73.0 ([#2097](https://github.com/awslabs/aws-lambda-powertools-python/issues/2097))
+* **deps-dev:** bump aws-cdk from 2.72.1 to 2.73.0 ([#2093](https://github.com/awslabs/aws-lambda-powertools-python/issues/2093))
+* **deps-dev:** bump mypy-boto3-cloudformation from 1.26.60 to 1.26.108 ([#2095](https://github.com/awslabs/aws-lambda-powertools-python/issues/2095))
+* **deps-dev:** bump types-python-dateutil from 2.8.19.11 to 2.8.19.12 ([#2085](https://github.com/awslabs/aws-lambda-powertools-python/issues/2085))
+* **deps-dev:** bump cfn-lint from 0.76.1 to 0.76.2 ([#2084](https://github.com/awslabs/aws-lambda-powertools-python/issues/2084))
+* **deps-dev:** bump aws-cdk from 2.72.0 to 2.72.1 ([#2081](https://github.com/awslabs/aws-lambda-powertools-python/issues/2081))
+* **deps-dev:** bump coverage from 7.2.2 to 7.2.3 ([#2092](https://github.com/awslabs/aws-lambda-powertools-python/issues/2092))
 * **deps-dev:** bump mkdocs-material from 9.1.4 to 9.1.5 ([#2077](https://github.com/awslabs/aws-lambda-powertools-python/issues/2077))
 * **deps-dev:** bump aws-cdk-lib from 2.72.0 to 2.72.1 ([#2076](https://github.com/awslabs/aws-lambda-powertools-python/issues/2076))
 * **deps-dev:** bump mypy-boto3-s3 from 1.26.99 to 1.26.104 ([#2075](https://github.com/awslabs/aws-lambda-powertools-python/issues/2075))
 * **deps-dev:** bump aws-cdk from 2.71.0 to 2.72.0 ([#2071](https://github.com/awslabs/aws-lambda-powertools-python/issues/2071))
+* **deps-dev:** bump filelock from 3.10.7 to 3.11.0 ([#2094](https://github.com/awslabs/aws-lambda-powertools-python/issues/2094))
 * **deps-dev:** bump aws-cdk-lib from 2.71.0 to 2.72.0 ([#2070](https://github.com/awslabs/aws-lambda-powertools-python/issues/2070))
 * **deps-dev:** bump black from 23.1.0 to 23.3.0 ([#2066](https://github.com/awslabs/aws-lambda-powertools-python/issues/2066))
 * **deps-dev:** bump aws-cdk from 2.70.0 to 2.71.0 ([#2067](https://github.com/awslabs/aws-lambda-powertools-python/issues/2067))

aws_lambda_powertools/utilities/batch/__init__.py

@@ -12,8 +12,12 @@
     EventType,
     FailureResponse,
     SuccessResponse,
+)
+from aws_lambda_powertools.utilities.batch.decorators import (
     async_batch_processor,
+    async_process_partial_response,
     batch_processor,
+    process_partial_response,
 )
 from aws_lambda_powertools.utilities.batch.exceptions import ExceptionInfo
 from aws_lambda_powertools.utilities.batch.sqs_fifo_partial_processor import (
@@ -22,6 +26,10 @@
 from aws_lambda_powertools.utilities.batch.types import BatchTypeModels
 
 __all__ = (
+    "async_batch_processor",
+    "async_process_partial_response",
+    "batch_processor",
+    "process_partial_response",
     "BatchProcessor",
     "AsyncBatchProcessor",
     "BasePartialProcessor",
@@ -32,6 +40,4 @@
     "FailureResponse",
     "SuccessResponse",
     "SqsFifoPartialProcessor",
-    "batch_processor",
-    "async_batch_processor",
 )

aws_lambda_powertools/utilities/batch/base.py

@@ -11,19 +11,8 @@
 import sys
 from abc import ABC, abstractmethod
 from enum import Enum
-from typing import (
-    Any,
-    Awaitable,
-    Callable,
-    Dict,
-    List,
-    Optional,
-    Tuple,
-    Union,
-    overload,
-)
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union, overload
 
-from aws_lambda_powertools.middleware_factory import lambda_handler_decorator
 from aws_lambda_powertools.shared import constants
 from aws_lambda_powertools.utilities.batch.exceptions import (
     BatchProcessingError,
@@ -37,6 +26,7 @@
     KinesisStreamRecord,
 )
 from aws_lambda_powertools.utilities.data_classes.sqs_event import SQSRecord
+from aws_lambda_powertools.utilities.parser import ValidationError
 from aws_lambda_powertools.utilities.typing import LambdaContext
 
 logger = logging.getLogger(__name__)
@@ -316,21 +306,36 @@ def _get_messages_to_report(self) -> List[Dict[str, str]]:
     def _collect_sqs_failures(self):
         failures = []
         for msg in self.fail_messages:
-            msg_id = msg.messageId if self.model else msg.message_id
+            # If a message failed due to model validation (e.g., poison pill)
+            # we convert to an event source data class...but self.model is still true
+            # therefore, we do an additional check on whether the failed message is still a model
+            # see https://github.com/awslabs/aws-lambda-powertools-python/issues/2091
+            if self.model and getattr(msg, "parse_obj", None):
+                msg_id = msg.messageId
+            else:
+                msg_id = msg.message_id
             failures.append({"itemIdentifier": msg_id})
         return failures
 
     def _collect_kinesis_failures(self):
         failures = []
         for msg in self.fail_messages:
-            msg_id = msg.kinesis.sequenceNumber if self.model else msg.kinesis.sequence_number
+            # # see https://github.com/awslabs/aws-lambda-powertools-python/issues/2091
+            if self.model and getattr(msg, "parse_obj", None):
+                msg_id = msg.kinesis.sequenceNumber
+            else:
+                msg_id = msg.kinesis.sequence_number
             failures.append({"itemIdentifier": msg_id})
         return failures
 
     def _collect_dynamodb_failures(self):
         failures = []
         for msg in self.fail_messages:
-            msg_id = msg.dynamodb.SequenceNumber if self.model else msg.dynamodb.sequence_number
+            # see https://github.com/awslabs/aws-lambda-powertools-python/issues/2091
+            if self.model and getattr(msg, "parse_obj", None):
+                msg_id = msg.dynamodb.SequenceNumber
+            else:
+                msg_id = msg.dynamodb.sequence_number
             failures.append({"itemIdentifier": msg_id})
         return failures
 
@@ -347,6 +352,17 @@ def _to_batch_type(self, record: dict, event_type: EventType, model: Optional["B
             return model.parse_obj(record)
         return self._DATA_CLASS_MAPPING[event_type](record)
 
+    def _register_model_validation_error_record(self, record: dict):
+        """Convert and register failure due to poison pills where model failed validation early"""
+        # Parser will fail validation if record is a poison pill (malformed input)
+        # this means we can't collect the message id if we try transforming again
+        # so we convert into to the equivalent batch type model (e.g., SQS, Kinesis, DynamoDB Stream)
+        # and downstream we can correctly collect the correct message id identifier and make the failed record available
+        # see https://github.com/awslabs/aws-lambda-powertools-python/issues/2091
+        logger.debug("Record cannot be converted to customer's model; converting without model")
+        failed_record: "EventSourceDataClassTypes" = self._to_batch_type(record=record, event_type=self.event_type)
+        return self.failure_handler(record=failed_record, exception=sys.exc_info())
+
 
 class BatchProcessor(BasePartialBatchProcessor):  # Keep old name for compatibility
     """Process native partial responses from SQS, Kinesis Data Streams, and DynamoDB.
@@ -471,63 +487,21 @@ def _process_record(self, record: dict) -> Union[SuccessResponse, FailureRespons
         record: dict
             A batch record to be processed.
         """
-        data = self._to_batch_type(record=record, event_type=self.event_type, model=self.model)
+        data: Optional["BatchTypeModels"] = None
         try:
+            data = self._to_batch_type(record=record, event_type=self.event_type, model=self.model)
             if self._handler_accepts_lambda_context:
                 result = self.handler(record=data, lambda_context=self.lambda_context)
             else:
                 result = self.handler(record=data)
 
             return self.success_handler(record=record, result=result)
+        except ValidationError:
+            return self._register_model_validation_error_record(record)
         except Exception:
             return self.failure_handler(record=data, exception=sys.exc_info())
 
 
-@lambda_handler_decorator
-def batch_processor(
-    handler: Callable, event: Dict, context: LambdaContext, record_handler: Callable, processor: BatchProcessor
-):
-    """
-    Middleware to handle batch event processing
-
-    Parameters
-    ----------
-    handler: Callable
-        Lambda's handler
-    event: Dict
-        Lambda's Event
-    context: LambdaContext
-        Lambda's Context
-    record_handler: Callable
-        Callable or corutine to process each record from the batch
-    processor: BatchProcessor
-        Batch Processor to handle partial failure cases
-
-    Examples
-    --------
-    **Processes Lambda's event with a BasePartialProcessor**
-
-        >>> from aws_lambda_powertools.utilities.batch import batch_processor, BatchProcessor
-        >>>
-        >>> def record_handler(record):
-        >>>     return record["body"]
-        >>>
-        >>> @batch_processor(record_handler=record_handler, processor=BatchProcessor())
-        >>> def handler(event, context):
-        >>>     return {"StatusCode": 200}
-
-    Limitations
-    -----------
-    * Async batch processors. Use `async_batch_processor` instead.
-    """
-    records = event["Records"]
-
-    with processor(records, record_handler, lambda_context=context):
-        processor.process()
-
-    return handler(event, context)
-
-
 class AsyncBatchProcessor(BasePartialBatchProcessor):
     """Process native partial responses from SQS, Kinesis Data Streams, and DynamoDB asynchronously.
 
@@ -651,62 +625,16 @@ async def _async_process_record(self, record: dict) -> Union[SuccessResponse, Fa
         record: dict
             A batch record to be processed.
         """
-        data = self._to_batch_type(record=record, event_type=self.event_type, model=self.model)
+        data: Optional["BatchTypeModels"] = None
         try:
+            data = self._to_batch_type(record=record, event_type=self.event_type, model=self.model)
             if self._handler_accepts_lambda_context:
                 result = await self.handler(record=data, lambda_context=self.lambda_context)
             else:
                 result = await self.handler(record=data)
 
             return self.success_handler(record=record, result=result)
+        except ValidationError:
+            return self._register_model_validation_error_record(record)
         except Exception:
             return self.failure_handler(record=data, exception=sys.exc_info())
-
-
-@lambda_handler_decorator
-def async_batch_processor(
-    handler: Callable,
-    event: Dict,
-    context: LambdaContext,
-    record_handler: Callable[..., Awaitable[Any]],
-    processor: AsyncBatchProcessor,
-):
-    """
-    Middleware to handle batch event processing
-    Parameters
-    ----------
-    handler: Callable
-        Lambda's handler
-    event: Dict
-        Lambda's Event
-    context: LambdaContext
-        Lambda's Context
-    record_handler: Callable[..., Awaitable[Any]]
-        Callable to process each record from the batch
-    processor: AsyncBatchProcessor
-        Batch Processor to handle partial failure cases
-    Examples
-    --------
-    **Processes Lambda's event with a BasePartialProcessor**
-        >>> from aws_lambda_powertools.utilities.batch import async_batch_processor, AsyncBatchProcessor
-        >>>
-        >>> async def async_record_handler(record):
-        >>>     payload: str = record.body
-        >>>     return payload
-        >>>
-        >>> processor = AsyncBatchProcessor(event_type=EventType.SQS)
-        >>>
-        >>> @async_batch_processor(record_handler=async_record_handler, processor=processor)
-        >>> async def lambda_handler(event, context: LambdaContext):
-        >>>     return processor.response()
-
-    Limitations
-    -----------
-    * Sync batch processors. Use `batch_processor` instead.
-    """
-    records = event["Records"]
-
-    with processor(records, record_handler, lambda_context=context):
-        processor.async_process()
-
-    return handler(event, context)