chore(idempotency): remove param expires_in_progress

Author: rubenfonseca

aws_lambda_powertools/utilities/idempotency/base.py

@@ -190,12 +190,8 @@ def _handle_for_status(self, data_record: DataRecord) -> Optional[Dict[Any, Any]
             raise IdempotencyInconsistentStateError("save_inprogress and get_record return inconsistent results.")
 
         if data_record.status == STATUS_CONSTANTS["INPROGRESS"]:
-            # This code path will only be triggered if the expires_in_progress option is enabled, and the item
-            # became expored between the save_inprogress call an dhere
-            if (
-                self.config.expires_in_progress
-                and data_record.in_progress_expiry_timestamp is not None
-                and data_record.in_progress_expiry_timestamp < int(datetime.datetime.now().timestamp())
+            if data_record.in_progress_expiry_timestamp is not None and data_record.in_progress_expiry_timestamp < int(
+                datetime.datetime.now().timestamp()
             ):
                 raise IdempotencyInconsistentStateError(
                     "item should have been expired in-progress because it already time-outed."

aws_lambda_powertools/utilities/idempotency/config.py

@@ -9,7 +9,6 @@ def __init__(
         jmespath_options: Optional[Dict] = None,
         raise_on_no_idempotency_key: bool = False,
         expires_after_seconds: int = 60 * 60,  # 1 hour default
-        expires_in_progress: bool = False,
         use_local_cache: bool = False,
         local_cache_max_items: int = 256,
         hash_function: str = "md5",
@@ -27,8 +26,6 @@ def __init__(
             Raise exception if no idempotency key was found in the request, by default False
         expires_after_seconds: int
             The number of seconds to wait before a record is expired
-        expires_in_progress: bool, optional
-            Whether to expire units of work that timed-out during invocation, by default False
         use_local_cache: bool, optional
             Whether to locally cache idempotency results, by default False
         local_cache_max_items: int, optional
@@ -41,7 +38,6 @@ def __init__(
         self.jmespath_options = jmespath_options
         self.raise_on_no_idempotency_key = raise_on_no_idempotency_key
         self.expires_after_seconds = expires_after_seconds
-        self.expires_in_progress = expires_in_progress
         self.use_local_cache = use_local_cache
         self.local_cache_max_items = local_cache_max_items
         self.hash_function = hash_function

aws_lambda_powertools/utilities/idempotency/persistence/base.py

@@ -125,7 +125,6 @@ def __init__(self):
         self.validation_key_jmespath = None
         self.raise_on_no_idempotency_key = False
         self.expires_after_seconds: int = 60 * 60  # 1 hour default
-        self.expires_in_progress: bool = False
         self.use_local_cache = False
         self.hash_function = None
 
@@ -158,7 +157,6 @@ def configure(self, config: IdempotencyConfig, function_name: Optional[str] = No
             self.payload_validation_enabled = True
         self.raise_on_no_idempotency_key = config.raise_on_no_idempotency_key
         self.expires_after_seconds = config.expires_after_seconds
-        self.expires_in_progress = config.expires_in_progress
         self.use_local_cache = config.use_local_cache
         if self.use_local_cache:
             self._cache = LRUDict(max_items=config.local_cache_max_items)
@@ -353,19 +351,18 @@ def save_inprogress(self, data: Dict[str, Any], remaining_time_in_millis: Option
             payload_hash=self._get_hashed_payload(data=data),
         )
 
-        if self.expires_in_progress:
-            if remaining_time_in_millis:
-                now = datetime.datetime.now()
-                period = datetime.timedelta(milliseconds=remaining_time_in_millis)
+        if remaining_time_in_millis:
+            now = datetime.datetime.now()
+            period = datetime.timedelta(milliseconds=remaining_time_in_millis)
 
-                # It's very important to use math.ceil here. Otherwise, we might return an integer that will be smaller
-                # than the current time in milliseconds, due to rounding. This will create a scenario where the record
-                # looks already expired in the store, but the invocation is still running.
-                timestamp = ceil((now + period).timestamp())
+            # It's very important to use math.ceil here. Otherwise, we might return an integer that will be smaller
+            # than the current time in milliseconds, due to rounding. This will create a scenario where the record
+            # looks already expired in the store, but the invocation is still running.
+            timestamp = ceil((now + period).timestamp())
 
-                data_record.in_progress_expiry_timestamp = timestamp
-            else:
-                warnings.warn("Expires in progress is enabled but we couldn't determine the remaining time left")
+            data_record.in_progress_expiry_timestamp = timestamp
+        else:
+            warnings.warn("Expires in progress is enabled but we couldn't determine the remaining time left")
 
         logger.debug(f"Saving in progress record for idempotency key: {data_record.idempotency_key}")
 

aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py

@@ -168,64 +168,48 @@ def _put_record(self, data_record: DataRecord) -> None:
         try:
             logger.debug(f"Putting record for idempotency key: {data_record.idempotency_key}")
 
-            condition_expression = "attribute_not_exists(#id) OR #now < :now"
-            expression_attribute_names = {
-                "#id": self.key_attr,
-                "#now": self.expiry_attr,
-                "#status": self.status_attr,
-            }
-            expression_attribute_values: Dict[str, Any] = {":now": int(now.timestamp())}
-
-            # When we want to expire_in_progress invocations, we check if the in_progress timestamp exists
-            # and we are past that timestamp. We also make sure the status is INPROGRESS because we don't
-            # want to repeat COMPLETE invocations.
-            #
-            # We put this in an if block because customers might want to disable the feature,
-            # reverting to the old behavior that relies just on the idempotency key.
-            if self.expires_in_progress:
-                condition_expression += (
-                    " OR ("
-                    "attribute_exists(#in_progress_expiry) AND "
-                    "#in_progress_expiry < :now AND #status = :inprogress"
-                    ")"
-                )
-                expression_attribute_names["#in_progress_expiry"] = self.in_progress_expiry_attr
-                expression_attribute_values[":inprogress"] = STATUS_CONSTANTS["INPROGRESS"]
-
             self.table.put_item(
                 Item=item,
-                ConditionExpression=condition_expression,
-                ExpressionAttributeNames=expression_attribute_names,
-                ExpressionAttributeValues=expression_attribute_values,
+                ConditionExpression=(
+                    "attribute_not_exists(#id) OR #now < :now OR "
+                    "(attribute_exists(#in_progress_expiry) AND #in_progress_expiry < :now AND #status = :inprogress)"
+                ),
+                ExpressionAttributeNames={
+                    "#id": self.key_attr,
+                    "#now": self.expiry_attr,
+                    "#in_progress_expiry": self.in_progress_expiry_attr,
+                    "#status": self.status_attr,
+                },
+                ExpressionAttributeValues={":now": int(now.timestamp()), ":inprogress": STATUS_CONSTANTS["INPROGRESS"]},
             )
         except self.table.meta.client.exceptions.ConditionalCheckFailedException:
             logger.debug(f"Failed to put record for already existing idempotency key: {data_record.idempotency_key}")
             raise IdempotencyItemAlreadyExistsError
 
     def _update_record(self, data_record: DataRecord):
         logger.debug(f"Updating record for idempotency key: {data_record.idempotency_key}")
-        update_expression = "SET #response_data = :response_data, #expiry = :expiry, #status = :status"
+        update_expression = (
+            "SET #response_data = :response_data, #expiry = :expiry, "
+            "#status = :status, #in_progress_expiry = :in_progress_expiry"
+        )
         expression_attr_values = {
             ":expiry": data_record.expiry_timestamp,
             ":response_data": data_record.response_data,
             ":status": data_record.status,
+            ":in_progress_expiry": data_record.in_progress_expiry_timestamp,
         }
         expression_attr_names = {
             "#response_data": self.data_attr,
             "#expiry": self.expiry_attr,
             "#status": self.status_attr,
+            "#in_progress_expiry": self.in_progress_expiry_attr,
         }
 
         if self.payload_validation_enabled:
             update_expression += ", #validation_key = :validation_key"
             expression_attr_values[":validation_key"] = data_record.payload_hash
             expression_attr_names["#validation_key"] = self.validation_key_attr
 
-        if self.expires_in_progress:
-            update_expression += ", #in_progress_expiry = :in_progress_expiry"
-            expression_attr_values[":in_progress_expiry"] = data_record.in_progress_expiry_timestamp
-            expression_attr_names["#in_progress_expiry"] = self.in_progress_expiry_attr
-
         kwargs = {
             "Key": self._get_key(data_record.idempotency_key),
             "UpdateExpression": update_expression,

docs/utilities/idempotency.md

@@ -370,12 +370,12 @@ sequenceDiagram
         Lambda->>Persistence Layer: Update record with Lambda handler results
         deactivate Persistence Layer
         Persistence Layer-->>Persistence Layer: Update record with result
-        Lambda--xClient: Response not received by client
+        Lambda-->>Client: Response sent to client
     else retried request:
         Client->>Lambda: Invoke (event)
         Lambda->>Persistence Layer: Get or set (id=event.search(payload))
         Persistence Layer-->>Lambda: Already exists in persistence layer. Return result
-        Lambda-->>Client: Response  sent to client
+        Lambda-->>Client: Response sent to client
     end
 ```
 <i>Idempotent sequence</i>
@@ -386,6 +386,59 @@ The client was successful in receiving the result after the retry. Since the Lam
 ???+ note
     Bear in mind that the entire Lambda handler is treated as a single idempotent operation. If your Lambda handler can cause multiple side effects, consider splitting it into separate functions.
 
+#### Lambda timeouts
+
+In cases where the [Lambda invocation
+expires](https://aws.amazon.com/premiumsupport/knowledge-center/lambda-verify-invocation-timeouts/),
+Powertools doesn't have the chance to set the idempotency record to `EXPIRED`.
+This means that the record would normally have been locked until `expire_seconds` have
+passsed.
+
+However, when Powertools has access to the Lambda invocation context, we are able to calculate the remaining
+available time for the invocation, and save it on the idempotency record. This way, if a second invocation happens
+after this timestamp, and the record is still marked `INPROGRESS`, we execute the inovcation again as if it was
+already expired. This means that if an invocation expired during execution, it will be quickly executed again on the
+next retry.
+
+???+ info "Info: Calculating the remaining available time"
+    For now this only works with the `idempotent` decorator. At the moment we
+    don't have access to the Lambda context when using the
+    `idempotent_function` so enabling this option is a no-op in that scenario.
+
+<center>
+```mermaid
+sequenceDiagram
+    participant Client
+    participant Lambda
+    participant Persistence Layer
+    alt initial request:
+        Client->>Lambda: Invoke (event)
+        Lambda->>Persistence Layer: Get or set (id=event.search(payload))
+        activate Persistence Layer
+        Note right of Persistence Layer: Locked during this time. Prevents multiple<br/>Lambda invocations with the same<br/>payload running concurrently.
+        Lambda--xLambda: Run Lambda handler (event).<br/>Time out
+        Lambda-->>Client: Return error response
+        deactivate Persistence Layer
+    else retried before the Lambda timeout:
+        Client->>Lambda: Invoke (event)
+        Lambda->>Persistence Layer: Get or set (id=event.search(payload))
+        Persistence Layer-->>Lambda: Already exists in persistence layer. Still in-progress
+        Lambda--xClient: Return Invocation Already In Progress error
+    else retried after the Lambda timeout:
+        Client->>Lambda: Invoke (event)
+        Lambda->>Persistence Layer: Get or set (id=event.search(payload))
+        activate Persistence Layer
+        Note right of Persistence Layer: Locked during this time. Prevents multiple<br/>Lambda invocations with the same<br/>payload running concurrently.
+        Lambda-->>Lambda: Run Lambda handler (event)
+        Lambda->>Persistence Layer: Update record with Lambda handler results
+        deactivate Persistence Layer
+        Persistence Layer-->>Persistence Layer: Update record with result
+        Lambda-->>Client: Response sent to client
+    end
+```
+<i>Idempotent sequence for Lambda timeouts</i>
+</center>
+
 ### Handling exceptions
 
 If you are using the `idempotent` decorator on your Lambda handler, any unhandled exceptions that are raised during the code execution will cause **the record in the persistence layer to be deleted**.
@@ -483,7 +536,6 @@ Parameter | Default | Description
 **payload_validation_jmespath** | `""` | JMESPath expression to validate whether certain parameters have changed in the event while the event payload
 **raise_on_no_idempotency_key** | `False` | Raise exception if no idempotency key was found in the request
 **expires_after_seconds** | 3600 | The number of seconds to wait before a record is expired
-**expires_in_progress** | `False`| Enables expiry of invocations that time out during execution
 **use_local_cache** | `False` | Whether to locally cache idempotency results
 **local_cache_max_items** | 256 | Max number of items to store in local cache
 **hash_function** | `md5` | Function to use for calculating hashes, as provided by [hashlib](https://docs.python.org/3/library/hashlib.html) in the standard library.
@@ -897,58 +949,6 @@ class DynamoDBPersistenceLayer(BasePersistenceLayer):
 
     For example, the `_put_record` method needs to raise an exception if a non-expired record already exists in the data store with a matching key.
 
-### Expiring in-progress invocations
-
-The default behavior when a Lambda invocation times out is for the event to stay locked until `expire_seconds` have passed. Powertools
-has no way of knowing if it's safe to retry the operation in this scenario, so we assume the safest approach: to not
-retry the operation.
-
-However, certain types of invocation have less strict requirements, and can benefit from faster expiry of invocations. Ideally, we
-can make an in-progress invocation expire as soon as the [Lambda invocation expires](https://aws.amazon.com/premiumsupport/knowledge-center/lambda-verify-invocation-timeouts/).
-
-When using this option, powertools will calculate the remaining available time for the invocation, and save it on the idempotency record.
-This way, if a second invocation happens after this timestamp, and the record is stil marked `INPROGRESS`, we execute the invocation again
-as if it was already expired. This means that if an invocation expired during execution, it will be quickly executed again on the next try.
-
-This setting introduces no change on the regular behavior where if an invocation succeeds, the results are cached for `expire_seconds` seconds.
-
-???+ warning "Warning"
-    Consider whenever you really want this behavior. Powertools can't make any garantee on which state your application was
-    when it time outed. Ensure that your business logic can be retried at any stage.
-
-???+ info "Info: Calculating the remaining available time"
-    For now this only works with the `idempotent` decorator. At the moment we don't have access to the Lambda context when using
-    the `idempotent_function` so enabling this option is a no-op in that scenario.
-
-To activate this behaviour, enable the `expires_in_progress` option on the configuration:
-
-=== "app.py"
-
-    ```python hl_lines="8"
-    from aws_lambda_powertools.utilities.idempotency import (
-        DynamoDBPersistenceLayer, IdempotencyConfig, idempotent
-    )
-
-    persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable")
-
-    config = IdempotencyConfig(
-        expires_in_progress=True,
-    )
-
-    @idempotent(persistence_store=persistence_layer, config=config)
-    def handler(event, context):
-        payment = create_subscription_payment(
-            user=event['user'],
-            product=event['product_id']
-        )
-        ...
-        return {
-            "payment_id": payment.id,
-            "message": "success",
-            "statusCode": 200,
-        }
-    ```
-
 ## Compatibility with other utilities
 
 ### Validation utility