From 3d5062c3892fd0fc135e729d7f47116b8f404513 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sun, 29 Jan 2023 17:29:55 +0000 Subject: [PATCH 01/13] (redis): initial commit --- .../utilities/idempotency/__init__.py | 12 +- .../utilities/idempotency/exceptions.py | 6 + .../idempotency/persistence/redis.py | 171 ++++++++++++++++++ poetry.lock | 43 ++++- pyproject.toml | 2 + 5 files changed, 229 insertions(+), 5 deletions(-) create mode 100644 aws_lambda_powertools/utilities/idempotency/persistence/redis.py diff --git a/aws_lambda_powertools/utilities/idempotency/__init__.py b/aws_lambda_powertools/utilities/idempotency/__init__.py index 148b291ea6d..30447acb28c 100644 --- a/aws_lambda_powertools/utilities/idempotency/__init__.py +++ b/aws_lambda_powertools/utilities/idempotency/__init__.py @@ -8,7 +8,17 @@ from aws_lambda_powertools.utilities.idempotency.persistence.dynamodb import ( DynamoDBPersistenceLayer, ) +from aws_lambda_powertools.utilities.idempotency.persistence.redis import ( + RedisCachePersistenceLayer, +) from .idempotency import IdempotencyConfig, idempotent, idempotent_function -__all__ = ("DynamoDBPersistenceLayer", "BasePersistenceLayer", "idempotent", "idempotent_function", "IdempotencyConfig") +__all__ = ( + "DynamoDBPersistenceLayer", + "BasePersistenceLayer", + "idempotent", + "idempotent_function", + "IdempotencyConfig", + "RedisCachePersistenceLayer", +) diff --git a/aws_lambda_powertools/utilities/idempotency/exceptions.py b/aws_lambda_powertools/utilities/idempotency/exceptions.py index 69ab420850a..f27d5044da0 100644 --- a/aws_lambda_powertools/utilities/idempotency/exceptions.py +++ b/aws_lambda_powertools/utilities/idempotency/exceptions.py @@ -71,3 +71,9 @@ class IdempotencyKeyError(BaseError): """ Payload does not contain an idempotent key """ + + +class IdempotencyRedisConnectionError(BaseError): + """ + Payload does not contain an idempotent key + """ diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/redis.py b/aws_lambda_powertools/utilities/idempotency/persistence/redis.py new file mode 100644 index 00000000000..407fdacba44 --- /dev/null +++ b/aws_lambda_powertools/utilities/idempotency/persistence/redis.py @@ -0,0 +1,171 @@ +import logging +import os +from typing import Any, Dict, Optional + +import redis + +from aws_lambda_powertools.shared import constants +from aws_lambda_powertools.utilities.idempotency import BasePersistenceLayer +from aws_lambda_powertools.utilities.idempotency.exceptions import ( + IdempotencyItemNotFoundError, + IdempotencyPersistenceLayerError, + IdempotencyRedisConnectionError, +) +from aws_lambda_powertools.utilities.idempotency.persistence.base import DataRecord + +logger = logging.getLogger(__name__) + + +class RedisCachePersistenceLayer(BasePersistenceLayer): + def __init__( + self, + host: str, + port: int = "6379", + username: Optional[str] = None, + password: Optional[str] = None, + db_index: int = "0", + static_pk_value: Optional[str] = None, + expiry_attr: str = "expiration", + in_progress_expiry_attr: str = "in_progress_expiration", + status_attr: str = "status", + data_attr: str = "data", + validation_key_attr: str = "validation", + ): + """ + Initialize the Redis client + Parameters + ---------- + host: str + Name of the host to connect to Redis instance/cluster + port: int + Number of the port to connect to Redis instance/cluster + username: str + Name of the username to connect to Redis instance/cluster in case of using ACL + See: https://redis.io/docs/management/security/acl/ + password: str + Password to connect to Redis instance/cluster + db_index: int + Index of Redis database + See: https://redis.io/commands/select/ + static_pk_value: str, optional + Redis attribute value for cache key, by default "idempotency#". + expiry_attr: str, optional + Redis hash attribute name for expiry timestamp, by default "expiration" + in_progress_expiry_attr: str, optional + Redis hash attribute name for in-progress expiry timestamp, by default "in_progress_expiration" + status_attr: str, optional + Redis hash attribute name for status, by default "status" + data_attr: str, optional + Redis hash attribute name for response data, by default "data" + """ + + self._connection = None + self.host = host + self.port = port + self.username = username + self.password = password + self.db_index = db_index + + if static_pk_value is None: + static_pk_value = f"idempotency#{os.getenv(constants.LAMBDA_FUNCTION_NAME_ENV, '')}" + + self.static_pk_value = static_pk_value + self.in_progress_expiry_attr = in_progress_expiry_attr + self.expiry_attr = expiry_attr + self.status_attr = status_attr + self.data_attr = data_attr + self.validation_key_attr = validation_key_attr + super(RedisCachePersistenceLayer, self).__init__() + + @property + def connection(self): + """ + Caching property to store redis connection + """ + if self._connection: + return self._connection + + logger.info(f"Trying to connect to Redis Host/Cluster: {self.host}") + + try: + self._connection = redis.Redis( + host=self.host, port=self.port, username=self.username, password=self.password, db=self.db_index + ) + except redis.exceptions.ConnectionError as exc: + logger.debug(f"Cannot connect in Redis Host: {self.host}") + raise IdempotencyRedisConnectionError("Could not to connect to Redis", exc) from exc + return self._connection + + @connection.setter + def connection(self, connection): + """ + Allow redis connection variable to be set directly, primarily for use in tests + """ + self._connection = connection + + def _get_key(self, idempotency_key: str) -> dict: + # Need to review this after adding GETKEY logic + if self.sort_key_attr: + return {self.key_attr: self.static_pk_value, self.sort_key_attr: idempotency_key} + return {self.key_attr: idempotency_key} + + def _item_to_data_record(self, item: Dict[str, Any]) -> DataRecord: + # Need to review this after adding GETKEY logic + return DataRecord( + idempotency_key=item[self.key_attr], + status=item[self.status_attr], + expiry_timestamp=item[self.expiry_attr], + in_progress_expiry_timestamp=item.get(self.in_progress_expiry_attr), + response_data=item.get(self.data_attr), + payload_hash=item.get(self.validation_key_attr), + ) + + def _get_record(self, idempotency_key) -> DataRecord: + # See: https://redis.io/commands/hgetall/ + response = self.connection.hgetall(idempotency_key) + + try: + item = response + except KeyError: + raise IdempotencyItemNotFoundError + return self._item_to_data_record(item) + + def _put_record(self, data_record: DataRecord) -> None: + + # Redis works with hset to support hashing keys with multiple attributes + # See: https://redis.io/commands/hset/ + item = { + "name": data_record.idempotency_key, + "mapping": { + self.in_progress_expiry_attr: data_record.in_progress_expiry_timestamp, + self.status_attr: data_record.status, + }, + } + + try: + logger.debug(f"Putting record on Redis for idempotency key: {data_record.idempotency_key}") + self.connection.hset(**item) + # hset type must set expiration after adding the record + # Need to review this to get ttl in seconds + self.connection.expire(name=data_record.idempotency_key, time=60) + except Exception as exc: + logger.debug(f"Failed to add record idempotency key: {data_record.idempotency_key}") + raise IdempotencyPersistenceLayerError( + f"Failed to add record idempotency key: {data_record.idempotency_key}", exc + ) from exc + + def _update_record(self, data_record: DataRecord) -> None: + item = { + "name": data_record.idempotency_key, + "mapping": { + self.data_attr: data_record.response_data, + self.status_attr: data_record.status, + }, + } + logger.debug(f"Updating record for idempotency key: {data_record.idempotency_key}") + self.connection.hset(**item) + + def _delete_record(self, data_record: DataRecord) -> None: + logger.debug(f"Deleting record for idempotency key: {data_record.idempotency_key}") + # See: https://redis.io/commands/del/ + self.connection.delete(data_record.idempotency_key) diff --git a/poetry.lock b/poetry.lock index 83c36e27073..76a72f1a8a1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,5 +1,20 @@ # This file is automatically @generated by Poetry and should not be changed by hand. +[[package]] +name = "async-timeout" +version = "4.0.2" +description = "Timeout context manager for asyncio programs" +category = "main" +optional = true +python-versions = ">=3.6" +files = [ + {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, + {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, +] + +[package.dependencies] +typing-extensions = {version = ">=3.6.5", markers = "python_version < \"3.8\""} + [[package]] name = "attrs" version = "22.1.0" @@ -870,7 +885,7 @@ files = [ name = "importlib-metadata" version = "6.0.0" description = "Read metadata from Python packages" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1062,7 +1077,6 @@ category = "dev" optional = false python-versions = "*" files = [ - {file = "junit-xml-1.9.tar.gz", hash = "sha256:de16a051990d4e25a3982b2dd9e89d671067548718866416faec14d9de56db9f"}, {file = "junit_xml-1.9-py2.py3-none-any.whl", hash = "sha256:ec5ca1a55aefdd76d28fcc0b135251d156c7106fa979686a4b48d62b761b4732"}, ] @@ -2139,6 +2153,27 @@ colorama = {version = ">=0.4.1", markers = "python_version > \"3.4\""} future = "*" mando = ">=0.6,<0.7" +[[package]] +name = "redis" +version = "4.4.2" +description = "Python client for Redis database and key-value store" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ + {file = "redis-4.4.2-py3-none-any.whl", hash = "sha256:e6206448e2f8a432871d07d432c13ed6c2abcf6b74edb436c99752b1371be387"}, + {file = "redis-4.4.2.tar.gz", hash = "sha256:a010f6cb7378065040a02839c3f75c7e0fb37a87116fb4a95be82a95552776c7"}, +] + +[package.dependencies] +async-timeout = ">=4.0.2" +importlib-metadata = {version = ">=1.0", markers = "python_version < \"3.8\""} +typing-extensions = {version = "*", markers = "python_version < \"3.8\""} + +[package.extras] +hiredis = ["hiredis (>=1.0.0)"] +ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==20.0.1)", "requests (>=2.26.0)"] + [[package]] name = "regex" version = "2022.10.31" @@ -2644,7 +2679,7 @@ requests = ">=2.0,<3.0" name = "zipp" version = "3.11.0" description = "Backport of pathlib-compatible object wrapper for zip files" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2666,4 +2701,4 @@ validation = ["fastjsonschema"] [metadata] lock-version = "2.0" python-versions = "^3.7.4" -content-hash = "6593de2a17ba398072a78ad12e6d3ec19bb3f0ec70ee572502e5e8b3c1866fb9" +content-hash = "471c33ef48bd747ea58e5e131a924e16ca843fa8dce74bfd1ed21104df52a9e9" diff --git a/pyproject.toml b/pyproject.toml index 71b9ad96319..7e3451cd825 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ fastjsonschema = { version = "^2.14.5", optional = true } pydantic = { version = "^1.8.2", optional = true } boto3 = { version = "^1.20.32", optional = true } typing-extensions = "^4.4.0" +redis = {version = "^4.4.2", optional = true} [tool.poetry.dev-dependencies] coverage = {extras = ["toml"], version = "^7.1"} @@ -88,6 +89,7 @@ tracer = ["aws-xray-sdk"] all = ["pydantic", "aws-xray-sdk", "fastjsonschema"] # allow customers to run code locally without emulators (SAM CLI, etc.) aws-sdk = ["boto3"] +redis = ["redis"] [tool.poetry.group.dev.dependencies] cfn-lint = "0.67.0" From aed821a6f1583e272d34732c02832df2dd78cf46 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 6 Feb 2023 21:02:17 +0000 Subject: [PATCH 02/13] feat(redis/idempotency): creating redis connections --- .../utilities/database/__init__.py | 3 + .../utilities/database/exceptions.py | 4 ++ .../utilities/database/redis.py | 56 +++++++++++++++ .../idempotency/persistence/redis.py | 68 +++---------------- 4 files changed, 72 insertions(+), 59 deletions(-) create mode 100644 aws_lambda_powertools/utilities/database/__init__.py create mode 100644 aws_lambda_powertools/utilities/database/exceptions.py create mode 100644 aws_lambda_powertools/utilities/database/redis.py diff --git a/aws_lambda_powertools/utilities/database/__init__.py b/aws_lambda_powertools/utilities/database/__init__.py new file mode 100644 index 00000000000..3939531172d --- /dev/null +++ b/aws_lambda_powertools/utilities/database/__init__.py @@ -0,0 +1,3 @@ +from aws_lambda_powertools.utilities.database.redis import RedisStandalone + +__all__ = RedisStandalone diff --git a/aws_lambda_powertools/utilities/database/exceptions.py b/aws_lambda_powertools/utilities/database/exceptions.py new file mode 100644 index 00000000000..5107cc0f485 --- /dev/null +++ b/aws_lambda_powertools/utilities/database/exceptions.py @@ -0,0 +1,4 @@ +class RedisConnectionError(Exception): + """ + Payload does not contain an idempotent key + """ diff --git a/aws_lambda_powertools/utilities/database/redis.py b/aws_lambda_powertools/utilities/database/redis.py new file mode 100644 index 00000000000..33edb6f02cb --- /dev/null +++ b/aws_lambda_powertools/utilities/database/redis.py @@ -0,0 +1,56 @@ +import logging +from typing import Optional + +import redis + +from aws_lambda_powertools.utilities.database.exceptions import RedisConnectionError + +logger = logging.getLogger(__name__) + + +class RedisStandalone: + def __init__( + self, host: str, port: int, username: Optional[str] = None, password: Optional[str] = None, db_index: int = "0" + ) -> None: + """ + Initialize the Redis standalone client + Parameters + ---------- + host: str + Name of the host to connect to Redis instance/cluster + port: int + Number of the port to connect to Redis instance/cluster + username: str + Name of the username to connect to Redis instance/cluster in case of using ACL + See: https://redis.io/docs/management/security/acl/ + password: str + Passwod to connect to Redis instance/cluster + db_index: int + Index of Redis database + See: https://redis.io/commands/select/ + """ + + self.host = host + self.port = port + self.username = username + self.password = password + self.db_index = db_index + + def _init_connection(self): + """ + Connection is cached, so returning this + """ + if self._connection: + return self._connection + + logger.info(f"Trying to connect to Redis Host/Cluster: {self.host}") + + try: + self._connection = redis.Redis( + host=self.host, port=self.port, username=self.username, password=self.password, db=self.db_index + ) + except redis.exceptions.ConnectionError as exc: + logger.debug(f"Cannot connect in Redis Host: {self.host}") + raise RedisConnectionError("Could not to connect to Redis Standalone", exc) from exc + + return self._connection diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/redis.py b/aws_lambda_powertools/utilities/idempotency/persistence/redis.py index 407fdacba44..9ba41c1c600 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/redis.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/redis.py @@ -2,14 +2,11 @@ import os from typing import Any, Dict, Optional -import redis - from aws_lambda_powertools.shared import constants from aws_lambda_powertools.utilities.idempotency import BasePersistenceLayer from aws_lambda_powertools.utilities.idempotency.exceptions import ( IdempotencyItemNotFoundError, IdempotencyPersistenceLayerError, - IdempotencyRedisConnectionError, ) from aws_lambda_powertools.utilities.idempotency.persistence.base import DataRecord @@ -19,11 +16,7 @@ class RedisCachePersistenceLayer(BasePersistenceLayer): def __init__( self, - host: str, - port: int = "6379", - username: Optional[str] = None, - password: Optional[str] = None, - db_index: int = "0", + connection, static_pk_value: Optional[str] = None, expiry_attr: str = "expiration", in_progress_expiry_attr: str = "in_progress_expiration", @@ -32,21 +25,9 @@ def __init__( validation_key_attr: str = "validation", ): """ - Initialize the Redis client + Initialize the Redis Persistence Layer Parameters ---------- - host: str - Name of the host to connect to Redis instance/cluster - port: int - Number of the port to connect to Redis instance/cluster - username: str - Name of the username to connect to Redis instance/cluster in case of using ACL - See: https://redis.io/docs/management/security/acl/ - password: str - Password to connect to Redis instance/cluster - db_index: int - Index of Redis database - See: https://redis.io/commands/select/ static_pk_value: str, optional Redis attribute value for cache key, by default "idempotency#". expiry_attr: str, optional @@ -59,12 +40,8 @@ def __init__( Redis hash attribute name for response data, by default "data" """ - self._connection = None - self.host = host - self.port = port - self.username = username - self.password = password - self.db_index = db_index + # Initialize connection with Redis + self._connection = connection._init_connection() if static_pk_value is None: static_pk_value = f"idempotency#{os.getenv(constants.LAMBDA_FUNCTION_NAME_ENV, '')}" @@ -77,32 +54,6 @@ def __init__( self.validation_key_attr = validation_key_attr super(RedisCachePersistenceLayer, self).__init__() - @property - def connection(self): - """ - Caching property to store redis connection - """ - if self._connection: - return self._connection - - logger.info(f"Trying to connect to Redis Host/Cluster: {self.host}") - - try: - self._connection = redis.Redis( - host=self.host, port=self.port, username=self.username, password=self.password, db=self.db_index - ) - except redis.exceptions.ConnectionError as exc: - logger.debug(f"Cannot connect in Redis Host: {self.host}") - raise IdempotencyRedisConnectionError("Could not to connect to Redis", exc) from exc - return self._connection - - @connection.setter - def connection(self, connection): - """ - Allow redis connection variable to be set directly, primarily for use in tests - """ - self._connection = connection - def _get_key(self, idempotency_key: str) -> dict: # Need to review this after adding GETKEY logic if self.sort_key_attr: @@ -122,7 +73,7 @@ def _item_to_data_record(self, item: Dict[str, Any]) -> DataRecord: def _get_record(self, idempotency_key) -> DataRecord: # See: https://redis.io/commands/hgetall/ - response = self.connection.hgetall(idempotency_key) + response = self._connection.hgetall(idempotency_key) try: item = response @@ -131,7 +82,6 @@ def _get_record(self, idempotency_key) -> DataRecord: return self._item_to_data_record(item) def _put_record(self, data_record: DataRecord) -> None: - # Redis works with hset to support hashing keys with multiple attributes # See: https://redis.io/commands/hset/ item = { @@ -144,10 +94,10 @@ def _put_record(self, data_record: DataRecord) -> None: try: logger.debug(f"Putting record on Redis for idempotency key: {data_record.idempotency_key}") - self.connection.hset(**item) + self._connection.hset(**item) # hset type must set expiration after adding the record # Need to review this to get ttl in seconds - self.connection.expire(name=data_record.idempotency_key, time=60) + self._connection.expire(name=data_record.idempotency_key, time=60) except Exception as exc: logger.debug(f"Failed to add record idempotency key: {data_record.idempotency_key}") raise IdempotencyPersistenceLayerError( @@ -163,9 +113,9 @@ def _update_record(self, data_record: DataRecord) -> None: }, } logger.debug(f"Updating record for idempotency key: {data_record.idempotency_key}") - self.connection.hset(**item) + self._connection.hset(**item) def _delete_record(self, data_record: DataRecord) -> None: logger.debug(f"Deleting record for idempotency key: {data_record.idempotency_key}") # See: https://redis.io/commands/del/ - self.connection.delete(data_record.idempotency_key) + self._connection.delete(data_record.idempotency_key) From ffcf16d41fe024cdf62a9a770b187a28b68dbb94 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 6 Feb 2023 21:57:56 +0000 Subject: [PATCH 03/13] feat(redis/idempotency): creating redis connections --- .../utilities/database/__init__.py | 4 +- .../utilities/database/redis.py | 91 +++++++++++++++++-- 2 files changed, 87 insertions(+), 8 deletions(-) diff --git a/aws_lambda_powertools/utilities/database/__init__.py b/aws_lambda_powertools/utilities/database/__init__.py index 3939531172d..214a04d3329 100644 --- a/aws_lambda_powertools/utilities/database/__init__.py +++ b/aws_lambda_powertools/utilities/database/__init__.py @@ -1,3 +1,3 @@ -from aws_lambda_powertools.utilities.database.redis import RedisStandalone +from aws_lambda_powertools.utilities.database.redis import RedisCluster, RedisStandalone -__all__ = RedisStandalone +__all__ = (RedisStandalone, RedisCluster) diff --git a/aws_lambda_powertools/utilities/database/redis.py b/aws_lambda_powertools/utilities/database/redis.py index 33edb6f02cb..e78ec92a0d9 100644 --- a/aws_lambda_powertools/utilities/database/redis.py +++ b/aws_lambda_powertools/utilities/database/redis.py @@ -10,7 +10,13 @@ class RedisStandalone: def __init__( - self, host: str, port: int, username: Optional[str] = None, password: Optional[str] = None, db_index: int = "0" + self, + host: Optional[str] = None, + port: Optional[int] = None, + username: Optional[str] = None, + password: Optional[str] = None, + db_index: Optional[int] = None, + url: Optional[str] = None, ) -> None: """ Initialize the Redis standalone client @@ -24,19 +30,24 @@ def __init__( Name of the username to connect to Redis instance/cluster in case of using ACL See: https://redis.io/docs/management/security/acl/ password: str - Passwod to connect to Redis instance/cluster + Password to connect to Redis instance/cluster db_index: int Index of Redis database See: https://redis.io/commands/select/ + url: str + Redis client object configured from the given URL + See: https://redis.readthedocs.io/en/latest/connections.html#redis.Redis.from_url """ + self.url = url self.host = host self.port = port self.username = username self.password = password self.db_index = db_index + self._connection = None - def _init_connection(self): + def get_redis_connection(self): """ Connection is cached, so returning this """ @@ -46,11 +57,79 @@ def _init_connection(self): logger.info(f"Trying to connect to Redis Host/Cluster: {self.host}") try: - self._connection = redis.Redis( - host=self.host, port=self.port, username=self.username, password=self.password, db=self.db_index - ) + if self.url: + logger.debug(f"Using URL format to connect to Redis: {self.host}") + self._connection = redis.Redis.from_url(url=self.url) + else: + logger.debug(f"Using other parameters to connect to Redis: {self.host}") + self._connection = redis.Redis( + host=self.host, port=self.port, username=self.username, password=self.password, db=self.db_index + ) except redis.exceptions.ConnectionError as exc: logger.debug(f"Cannot connect in Redis Host: {self.host}") raise RedisConnectionError("Could not to connect to Redis Standalone", exc) from exc return self._connection + + +class RedisCluster: + def __init__( + self, + host: Optional[str] = None, + port: Optional[int] = None, + read_from_replicas: Optional[bool] = False, + url: Optional[str] = None, + ) -> None: + """ + Initialize the Redis standalone client + Parameters + ---------- + host: str + Name of the host to connect to Redis instance/cluster + port: int + Number of the port to connect to Redis instance/cluster + username: str + Name of the username to connect to Redis instance/cluster in case of using ACL + See: https://redis.io/docs/management/security/acl/ + password: str + Passwod to connect to Redis instance/cluster + db_index: int + Index of Redis database + See: https://redis.io/commands/select/ + url: str + Redis client object configured from the given URL + See: https://redis.readthedocs.io/en/latest/connections.html#redis.Redis.from_url + """ + + self.url = url + self.host = host + self.port = port + self.read_from_replicas = read_from_replicas + self._connection = None + + def get_redis_connection(self): + """ + Connection is cached, so returning this + """ + if self._connection: + return self._connection + + logger.info(f"Trying to connect to Redis Cluster: {self.host}") + + try: + if self.url: + logger.debug(f"Using URL format to connect to Redis Cluster: {self.host}") + self._connection = redis.Redis.from_url(url=self.url) + else: + logger.debug(f"Using other parameters to connect to Redis Cluster: {self.host}") + self._connection = redis.cluster.RedisCluster( + host=self.host, + port=self.port, + server_type=None, + read_from_replicas=self.read_from_replicas, + ) + except redis.exceptions.ConnectionError as exc: + logger.debug(f"Cannot connect in Redis Cluster: {self.host}") + raise RedisConnectionError("Could not to connect to Redis Cluster", exc) from exc + + return self._connection From b5c579189c57737f8a04cde2447aebfff159c701 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 6 Feb 2023 22:17:48 +0000 Subject: [PATCH 04/13] feat(redis/idempotency): fixing import --- aws_lambda_powertools/utilities/database/exceptions.py | 2 +- aws_lambda_powertools/utilities/idempotency/exceptions.py | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/aws_lambda_powertools/utilities/database/exceptions.py b/aws_lambda_powertools/utilities/database/exceptions.py index 5107cc0f485..b4426c2b142 100644 --- a/aws_lambda_powertools/utilities/database/exceptions.py +++ b/aws_lambda_powertools/utilities/database/exceptions.py @@ -1,4 +1,4 @@ class RedisConnectionError(Exception): """ - Payload does not contain an idempotent key + Redis connection error """ diff --git a/aws_lambda_powertools/utilities/idempotency/exceptions.py b/aws_lambda_powertools/utilities/idempotency/exceptions.py index f27d5044da0..69ab420850a 100644 --- a/aws_lambda_powertools/utilities/idempotency/exceptions.py +++ b/aws_lambda_powertools/utilities/idempotency/exceptions.py @@ -71,9 +71,3 @@ class IdempotencyKeyError(BaseError): """ Payload does not contain an idempotent key """ - - -class IdempotencyRedisConnectionError(BaseError): - """ - Payload does not contain an idempotent key - """ From 6902e7334399d9656f183df8335fb6fc9562eca6 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Tue, 7 Feb 2023 16:13:47 +0000 Subject: [PATCH 05/13] feat(redis/idempotency): adding base class --- .../utilities/connections/__init__.py | 6 ++++++ .../utilities/connections/base_sync.py | 7 +++++++ .../{database => connections}/exceptions.py | 0 .../utilities/{database => connections}/redis.py | 13 +++++++------ .../utilities/database/__init__.py | 3 --- .../utilities/idempotency/persistence/redis.py | 2 +- 6 files changed, 21 insertions(+), 10 deletions(-) create mode 100644 aws_lambda_powertools/utilities/connections/__init__.py create mode 100644 aws_lambda_powertools/utilities/connections/base_sync.py rename aws_lambda_powertools/utilities/{database => connections}/exceptions.py (100%) rename aws_lambda_powertools/utilities/{database => connections}/redis.py (93%) delete mode 100644 aws_lambda_powertools/utilities/database/__init__.py diff --git a/aws_lambda_powertools/utilities/connections/__init__.py b/aws_lambda_powertools/utilities/connections/__init__.py new file mode 100644 index 00000000000..b9c93b96b68 --- /dev/null +++ b/aws_lambda_powertools/utilities/connections/__init__.py @@ -0,0 +1,6 @@ +from aws_lambda_powertools.utilities.connections.redis import ( + RedisCluster, + RedisStandalone, +) + +__all__ = (RedisStandalone, RedisCluster) diff --git a/aws_lambda_powertools/utilities/connections/base_sync.py b/aws_lambda_powertools/utilities/connections/base_sync.py new file mode 100644 index 00000000000..f67149c3277 --- /dev/null +++ b/aws_lambda_powertools/utilities/connections/base_sync.py @@ -0,0 +1,7 @@ +from abc import ABC, abstractmethod + + +class BaseConnectionSync(ABC): + @abstractmethod + def init_connection(self): + raise NotImplementedError() # pragma: no cover diff --git a/aws_lambda_powertools/utilities/database/exceptions.py b/aws_lambda_powertools/utilities/connections/exceptions.py similarity index 100% rename from aws_lambda_powertools/utilities/database/exceptions.py rename to aws_lambda_powertools/utilities/connections/exceptions.py diff --git a/aws_lambda_powertools/utilities/database/redis.py b/aws_lambda_powertools/utilities/connections/redis.py similarity index 93% rename from aws_lambda_powertools/utilities/database/redis.py rename to aws_lambda_powertools/utilities/connections/redis.py index e78ec92a0d9..6dfd1e839b9 100644 --- a/aws_lambda_powertools/utilities/database/redis.py +++ b/aws_lambda_powertools/utilities/connections/redis.py @@ -3,12 +3,13 @@ import redis -from aws_lambda_powertools.utilities.database.exceptions import RedisConnectionError +from .base_sync import BaseConnectionSync +from .exceptions import RedisConnectionError logger = logging.getLogger(__name__) -class RedisStandalone: +class RedisStandalone(BaseConnectionSync): def __init__( self, host: Optional[str] = None, @@ -30,7 +31,7 @@ def __init__( Name of the username to connect to Redis instance/cluster in case of using ACL See: https://redis.io/docs/management/security/acl/ password: str - Password to connect to Redis instance/cluster + Passwod to connect to Redis instance/cluster db_index: int Index of Redis database See: https://redis.io/commands/select/ @@ -47,7 +48,7 @@ def __init__( self.db_index = db_index self._connection = None - def get_redis_connection(self): + def init_connection(self): """ Connection is cached, so returning this """ @@ -72,7 +73,7 @@ def get_redis_connection(self): return self._connection -class RedisCluster: +class RedisCluster(BaseConnectionSync): def __init__( self, host: Optional[str] = None, @@ -107,7 +108,7 @@ def __init__( self.read_from_replicas = read_from_replicas self._connection = None - def get_redis_connection(self): + def init_connection(self): """ Connection is cached, so returning this """ diff --git a/aws_lambda_powertools/utilities/database/__init__.py b/aws_lambda_powertools/utilities/database/__init__.py deleted file mode 100644 index 214a04d3329..00000000000 --- a/aws_lambda_powertools/utilities/database/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from aws_lambda_powertools.utilities.database.redis import RedisCluster, RedisStandalone - -__all__ = (RedisStandalone, RedisCluster) diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/redis.py b/aws_lambda_powertools/utilities/idempotency/persistence/redis.py index 9ba41c1c600..ee88e5854fb 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/redis.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/redis.py @@ -41,7 +41,7 @@ def __init__( """ # Initialize connection with Redis - self._connection = connection._init_connection() + self._connection = connection.init_connection() if static_pk_value is None: static_pk_value = f"idempotency#{os.getenv(constants.LAMBDA_FUNCTION_NAME_ENV, '')}" From 504d2589d32e30fbb956bc373899928508137a0b Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Tue, 7 Feb 2023 22:51:30 +0000 Subject: [PATCH 06/13] feat(redis/idempotency): adding logic to get record --- .../utilities/connections/redis.py | 7 +++- .../utilities/idempotency/persistence/base.py | 2 +- .../idempotency/persistence/redis.py | 37 +++++++++++++++---- 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/aws_lambda_powertools/utilities/connections/redis.py b/aws_lambda_powertools/utilities/connections/redis.py index 6dfd1e839b9..aa29d8ba39a 100644 --- a/aws_lambda_powertools/utilities/connections/redis.py +++ b/aws_lambda_powertools/utilities/connections/redis.py @@ -64,7 +64,12 @@ def init_connection(self): else: logger.debug(f"Using other parameters to connect to Redis: {self.host}") self._connection = redis.Redis( - host=self.host, port=self.port, username=self.username, password=self.password, db=self.db_index + host=self.host, + port=self.port, + username=self.username, + password=self.password, + db=self.db_index, + decode_responses=True, ) except redis.exceptions.ConnectionError as exc: logger.debug(f"Cannot connect in Redis Host: {self.host}") diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/base.py b/aws_lambda_powertools/utilities/idempotency/persistence/base.py index a87980d7fe0..aaf550e4785 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/base.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/base.py @@ -37,7 +37,7 @@ class DataRecord: def __init__( self, - idempotency_key, + idempotency_key: Optional[str] = "", status: str = "", expiry_timestamp: Optional[int] = None, in_progress_expiry_timestamp: Optional[int] = None, diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/redis.py b/aws_lambda_powertools/utilities/idempotency/persistence/redis.py index ee88e5854fb..6976a65642a 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/redis.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/redis.py @@ -5,8 +5,8 @@ from aws_lambda_powertools.shared import constants from aws_lambda_powertools.utilities.idempotency import BasePersistenceLayer from aws_lambda_powertools.utilities.idempotency.exceptions import ( + IdempotencyItemAlreadyExistsError, IdempotencyItemNotFoundError, - IdempotencyPersistenceLayerError, ) from aws_lambda_powertools.utilities.idempotency.persistence.base import DataRecord @@ -63,7 +63,6 @@ def _get_key(self, idempotency_key: str) -> dict: def _item_to_data_record(self, item: Dict[str, Any]) -> DataRecord: # Need to review this after adding GETKEY logic return DataRecord( - idempotency_key=item[self.key_attr], status=item[self.status_attr], expiry_timestamp=item[self.expiry_attr], in_progress_expiry_timestamp=item.get(self.in_progress_expiry_attr), @@ -89,20 +88,44 @@ def _put_record(self, data_record: DataRecord) -> None: "mapping": { self.in_progress_expiry_attr: data_record.in_progress_expiry_timestamp, self.status_attr: data_record.status, + self.expiry_attr: data_record.expiry_timestamp, }, } + if data_record.in_progress_expiry_timestamp is not None: + item["mapping"][self.in_progress_expiry_attr] = data_record.in_progress_expiry_timestamp + + if self.payload_validation_enabled: + item["mapping"][self.validation_key_attr] = data_record.payload_hash + try: + # | LOCKED | RETRY if status = "INPROGRESS" | RETRY + # |----------------|-------------------------------------------------------|-------------> .... (time) + # | Lambda Idempotency Record + # | Timeout Timeout + # | (in_progress_expiry) (expiry) + + # Conditions to successfully save a record: + + # The idempotency key does not exist: + # - first time that this invocation key is used + # - previous invocation with the same key was deleted due to TTL + idempotency_key_not_exist = self._connection.exists(data_record.idempotency_key) + + # key exists + if idempotency_key_not_exist == 1: + raise + + # missing logic to compare expiration + logger.debug(f"Putting record on Redis for idempotency key: {data_record.idempotency_key}") self._connection.hset(**item) # hset type must set expiration after adding the record # Need to review this to get ttl in seconds self._connection.expire(name=data_record.idempotency_key, time=60) - except Exception as exc: - logger.debug(f"Failed to add record idempotency key: {data_record.idempotency_key}") - raise IdempotencyPersistenceLayerError( - f"Failed to add record idempotency key: {data_record.idempotency_key}", exc - ) from exc + except Exception: + logger.debug(f"Failed to put record for already existing idempotency key: {data_record.idempotency_key}") + raise IdempotencyItemAlreadyExistsError def _update_record(self, data_record: DataRecord) -> None: item = { From 0d1e3e991be70a7b37f9d2e690333112f6cd7aa5 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Tue, 7 Feb 2023 23:01:52 +0000 Subject: [PATCH 07/13] feat(redis/idempotency): adding expiry timeout --- .../utilities/idempotency/persistence/base.py | 10 +++++++--- .../utilities/idempotency/persistence/redis.py | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/base.py b/aws_lambda_powertools/utilities/idempotency/persistence/base.py index aaf550e4785..c14232ca9bf 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/base.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/base.py @@ -116,6 +116,7 @@ class BasePersistenceLayer(ABC): def __init__(self): """Initialize the defaults""" self.function_name = "" + self.backend = "" self.configured = False self.event_key_jmespath: Optional[str] = None self.event_key_compiled_jmespath = None @@ -262,9 +263,12 @@ def _get_expiry_timestamp(self) -> int: unix timestamp of expiry date for idempotency record """ - now = datetime.datetime.now() - period = datetime.timedelta(seconds=self.expires_after_seconds) - return int((now + period).timestamp()) + if self.backend == "redis": + return self.expires_after_seconds + else: + now = datetime.datetime.now() + period = datetime.timedelta(seconds=self.expires_after_seconds) + return int((now + period).timestamp()) def _save_to_cache(self, data_record: DataRecord): """ diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/redis.py b/aws_lambda_powertools/utilities/idempotency/persistence/redis.py index 6976a65642a..4603c98e944 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/redis.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/redis.py @@ -122,7 +122,7 @@ def _put_record(self, data_record: DataRecord) -> None: self._connection.hset(**item) # hset type must set expiration after adding the record # Need to review this to get ttl in seconds - self._connection.expire(name=data_record.idempotency_key, time=60) + self._connection.expire(name=data_record.idempotency_key, time=self.expires_after_seconds) except Exception: logger.debug(f"Failed to put record for already existing idempotency key: {data_record.idempotency_key}") raise IdempotencyItemAlreadyExistsError From 3126885a4ced64e052e8296803f7bd2ca4cae029 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Fri, 17 Feb 2023 00:04:13 +0000 Subject: [PATCH 08/13] feat(redis) - refactoring connection and fixing mypy errors --- .../utilities/connections/__init__.py | 2 +- .../utilities/connections/redis.py | 107 ++++++----- .../utilities/idempotency/persistence/base.py | 2 +- .../idempotency/persistence/dynamodb.py | 2 +- .../idempotency/persistence/redis.py | 14 +- poetry.lock | 168 +++++++++++++++++- pyproject.toml | 1 + 7 files changed, 225 insertions(+), 71 deletions(-) diff --git a/aws_lambda_powertools/utilities/connections/__init__.py b/aws_lambda_powertools/utilities/connections/__init__.py index b9c93b96b68..660c559447f 100644 --- a/aws_lambda_powertools/utilities/connections/__init__.py +++ b/aws_lambda_powertools/utilities/connections/__init__.py @@ -3,4 +3,4 @@ RedisStandalone, ) -__all__ = (RedisStandalone, RedisCluster) +__all__ = ["RedisStandalone", "RedisCluster"] diff --git a/aws_lambda_powertools/utilities/connections/redis.py b/aws_lambda_powertools/utilities/connections/redis.py index aa29d8ba39a..10a314db5d8 100644 --- a/aws_lambda_powertools/utilities/connections/redis.py +++ b/aws_lambda_powertools/utilities/connections/redis.py @@ -1,5 +1,5 @@ import logging -from typing import Optional +from typing import Optional, Type, Union import redis @@ -9,36 +9,19 @@ logger = logging.getLogger(__name__) -class RedisStandalone(BaseConnectionSync): +class RedisConnection(BaseConnectionSync): def __init__( self, + client: Type[Union[redis.Redis, redis.RedisCluster]], host: Optional[str] = None, port: Optional[int] = None, username: Optional[str] = None, password: Optional[str] = None, db_index: Optional[int] = None, url: Optional[str] = None, + **extra_options, ) -> None: - """ - Initialize the Redis standalone client - Parameters - ---------- - host: str - Name of the host to connect to Redis instance/cluster - port: int - Number of the port to connect to Redis instance/cluster - username: str - Name of the username to connect to Redis instance/cluster in case of using ACL - See: https://redis.io/docs/management/security/acl/ - password: str - Passwod to connect to Redis instance/cluster - db_index: int - Index of Redis database - See: https://redis.io/commands/select/ - url: str - Redis client object configured from the given URL - See: https://redis.readthedocs.io/en/latest/connections.html#redis.Redis.from_url - """ + self.extra_options: dict = {} self.url = url self.host = host @@ -46,7 +29,9 @@ def __init__( self.username = username self.password = password self.db_index = db_index + self.extra_options.update(**extra_options) self._connection = None + self._client = client def init_connection(self): """ @@ -55,36 +40,40 @@ def init_connection(self): if self._connection: return self._connection - logger.info(f"Trying to connect to Redis Host/Cluster: {self.host}") + logger.info(f"Trying to connect to Redis: {self.host}") try: if self.url: logger.debug(f"Using URL format to connect to Redis: {self.host}") - self._connection = redis.Redis.from_url(url=self.url) + self._connection = self._client.from_url(url=self.url) else: logger.debug(f"Using other parameters to connect to Redis: {self.host}") - self._connection = redis.Redis( + self._connection = self._client( host=self.host, port=self.port, username=self.username, password=self.password, db=self.db_index, decode_responses=True, + **self.extra_options, ) except redis.exceptions.ConnectionError as exc: - logger.debug(f"Cannot connect in Redis Host: {self.host}") - raise RedisConnectionError("Could not to connect to Redis Standalone", exc) from exc + logger.debug(f"Cannot connect in Redis: {self.host}") + raise RedisConnectionError("Could not to connect to Redis", exc) from exc return self._connection -class RedisCluster(BaseConnectionSync): +class RedisStandalone(RedisConnection): def __init__( self, host: Optional[str] = None, port: Optional[int] = None, - read_from_replicas: Optional[bool] = False, + username: Optional[str] = None, + password: Optional[str] = None, + db_index: Optional[int] = None, url: Optional[str] = None, + **extra_options, ) -> None: """ Initialize the Redis standalone client @@ -106,36 +95,40 @@ def __init__( Redis client object configured from the given URL See: https://redis.readthedocs.io/en/latest/connections.html#redis.Redis.from_url """ + print(extra_options) + super().__init__(redis.Redis, host, port, username, password, db_index, url, **extra_options) - self.url = url - self.host = host - self.port = port - self.read_from_replicas = read_from_replicas - self._connection = None - def init_connection(self): +class RedisCluster(RedisConnection): + def __init__( + self, + host: Optional[str] = None, + port: Optional[int] = None, + username: Optional[str] = None, + password: Optional[str] = None, + db_index: Optional[int] = None, + url: Optional[str] = None, + **extra_options, + ) -> None: """ - Connection is cached, so returning this + Initialize the Redis standalone client + Parameters + ---------- + host: str + Name of the host to connect to Redis instance/cluster + port: int + Number of the port to connect to Redis instance/cluster + username: str + Name of the username to connect to Redis instance/cluster in case of using ACL + See: https://redis.io/docs/management/security/acl/ + password: str + Passwod to connect to Redis instance/cluster + db_index: int + Index of Redis database + See: https://redis.io/commands/select/ + url: str + Redis client object configured from the given URL + See: https://redis.readthedocs.io/en/latest/connections.html#redis.Redis.from_url """ - if self._connection: - return self._connection - logger.info(f"Trying to connect to Redis Cluster: {self.host}") - - try: - if self.url: - logger.debug(f"Using URL format to connect to Redis Cluster: {self.host}") - self._connection = redis.Redis.from_url(url=self.url) - else: - logger.debug(f"Using other parameters to connect to Redis Cluster: {self.host}") - self._connection = redis.cluster.RedisCluster( - host=self.host, - port=self.port, - server_type=None, - read_from_replicas=self.read_from_replicas, - ) - except redis.exceptions.ConnectionError as exc: - logger.debug(f"Cannot connect in Redis Cluster: {self.host}") - raise RedisConnectionError("Could not to connect to Redis Cluster", exc) from exc - - return self._connection + super().__init__(redis.cluster.RedisCluster, host, port, username, password, db_index, url, **extra_options) diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/base.py b/aws_lambda_powertools/utilities/idempotency/persistence/base.py index c14232ca9bf..a19dd073d7a 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/base.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/base.py @@ -37,7 +37,7 @@ class DataRecord: def __init__( self, - idempotency_key: Optional[str] = "", + idempotency_key: str = "", status: str = "", expiry_timestamp: Optional[int] = None, in_progress_expiry_timestamp: Optional[int] = None, diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py index b05d8216b50..fe47d12845c 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py @@ -26,7 +26,7 @@ class DynamoDBPersistenceLayer(BasePersistenceLayer): def __init__( self, table_name: str, - key_attr: str = "id", + key_attr: Optional[str] = "id", static_pk_value: Optional[str] = None, sort_key_attr: Optional[str] = None, expiry_attr: str = "expiration", diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/redis.py b/aws_lambda_powertools/utilities/idempotency/persistence/redis.py index 4603c98e944..b291a5e2ef1 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/redis.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/redis.py @@ -19,10 +19,10 @@ def __init__( connection, static_pk_value: Optional[str] = None, expiry_attr: str = "expiration", - in_progress_expiry_attr: str = "in_progress_expiration", + in_progress_expiry_attr="in_progress_expiration", status_attr: str = "status", data_attr: str = "data", - validation_key_attr: str = "validation", + validation_key_attr="validation", ): """ Initialize the Redis Persistence Layer @@ -54,12 +54,6 @@ def __init__( self.validation_key_attr = validation_key_attr super(RedisCachePersistenceLayer, self).__init__() - def _get_key(self, idempotency_key: str) -> dict: - # Need to review this after adding GETKEY logic - if self.sort_key_attr: - return {self.key_attr: self.static_pk_value, self.sort_key_attr: idempotency_key} - return {self.key_attr: idempotency_key} - def _item_to_data_record(self, item: Dict[str, Any]) -> DataRecord: # Need to review this after adding GETKEY logic return DataRecord( @@ -93,10 +87,10 @@ def _put_record(self, data_record: DataRecord) -> None: } if data_record.in_progress_expiry_timestamp is not None: - item["mapping"][self.in_progress_expiry_attr] = data_record.in_progress_expiry_timestamp + item.update({"mapping": {self.in_progress_expiry_attr: data_record.in_progress_expiry_timestamp}}) if self.payload_validation_enabled: - item["mapping"][self.validation_key_attr] = data_record.payload_hash + item.update({"mapping": {self.validation_key_attr: data_record.payload_hash}}) try: # | LOCKED | RETRY if status = "INPROGRESS" | RETRY diff --git a/poetry.lock b/poetry.lock index d9d1ee6fb0f..040311acba0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -360,6 +360,83 @@ files = [ {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, ] +[[package]] +name = "cffi" +version = "1.15.1" +description = "Foreign Function Interface for Python calling C code." +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914"}, + {file = "cffi-1.15.1-cp27-cp27m-win32.whl", hash = "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3"}, + {file = "cffi-1.15.1-cp27-cp27m-win_amd64.whl", hash = "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e"}, + {file = "cffi-1.15.1-cp310-cp310-win32.whl", hash = "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2"}, + {file = "cffi-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8"}, + {file = "cffi-1.15.1-cp311-cp311-win32.whl", hash = "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d"}, + {file = "cffi-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104"}, + {file = "cffi-1.15.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e"}, + {file = "cffi-1.15.1-cp36-cp36m-win32.whl", hash = "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf"}, + {file = "cffi-1.15.1-cp36-cp36m-win_amd64.whl", hash = "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497"}, + {file = "cffi-1.15.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426"}, + {file = "cffi-1.15.1-cp37-cp37m-win32.whl", hash = "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9"}, + {file = "cffi-1.15.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045"}, + {file = "cffi-1.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192"}, + {file = "cffi-1.15.1-cp38-cp38-win32.whl", hash = "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314"}, + {file = "cffi-1.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3"}, + {file = "cffi-1.15.1-cp39-cp39-win32.whl", hash = "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee"}, + {file = "cffi-1.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c"}, + {file = "cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9"}, +] + +[package.dependencies] +pycparser = "*" + [[package]] name = "cfn-lint" version = "0.67.0" @@ -604,6 +681,52 @@ tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.1 [package.extras] toml = ["tomli"] +[[package]] +name = "cryptography" +version = "39.0.1" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ + {file = "cryptography-39.0.1-cp36-abi3-macosx_10_12_universal2.whl", hash = "sha256:6687ef6d0a6497e2b58e7c5b852b53f62142cfa7cd1555795758934da363a965"}, + {file = "cryptography-39.0.1-cp36-abi3-macosx_10_12_x86_64.whl", hash = "sha256:706843b48f9a3f9b9911979761c91541e3d90db1ca905fd63fee540a217698bc"}, + {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:5d2d8b87a490bfcd407ed9d49093793d0f75198a35e6eb1a923ce1ee86c62b41"}, + {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83e17b26de248c33f3acffb922748151d71827d6021d98c70e6c1a25ddd78505"}, + {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e124352fd3db36a9d4a21c1aa27fd5d051e621845cb87fb851c08f4f75ce8be6"}, + {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:5aa67414fcdfa22cf052e640cb5ddc461924a045cacf325cd164e65312d99502"}, + {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:35f7c7d015d474f4011e859e93e789c87d21f6f4880ebdc29896a60403328f1f"}, + {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f24077a3b5298a5a06a8e0536e3ea9ec60e4c7ac486755e5fb6e6ea9b3500106"}, + {file = "cryptography-39.0.1-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:f0c64d1bd842ca2633e74a1a28033d139368ad959872533b1bab8c80e8240a0c"}, + {file = "cryptography-39.0.1-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:0f8da300b5c8af9f98111ffd512910bc792b4c77392a9523624680f7956a99d4"}, + {file = "cryptography-39.0.1-cp36-abi3-win32.whl", hash = "sha256:fe913f20024eb2cb2f323e42a64bdf2911bb9738a15dba7d3cce48151034e3a8"}, + {file = "cryptography-39.0.1-cp36-abi3-win_amd64.whl", hash = "sha256:ced4e447ae29ca194449a3f1ce132ded8fcab06971ef5f618605aacaa612beac"}, + {file = "cryptography-39.0.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:807ce09d4434881ca3a7594733669bd834f5b2c6d5c7e36f8c00f691887042ad"}, + {file = "cryptography-39.0.1-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c5caeb8188c24888c90b5108a441c106f7faa4c4c075a2bcae438c6e8ca73cef"}, + {file = "cryptography-39.0.1-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4789d1e3e257965e960232345002262ede4d094d1a19f4d3b52e48d4d8f3b885"}, + {file = "cryptography-39.0.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:96f1157a7c08b5b189b16b47bc9db2332269d6680a196341bf30046330d15388"}, + {file = "cryptography-39.0.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e422abdec8b5fa8462aa016786680720d78bdce7a30c652b7fadf83a4ba35336"}, + {file = "cryptography-39.0.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:b0afd054cd42f3d213bf82c629efb1ee5f22eba35bf0eec88ea9ea7304f511a2"}, + {file = "cryptography-39.0.1-pp39-pypy39_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:6f8ba7f0328b79f08bdacc3e4e66fb4d7aab0c3584e0bd41328dce5262e26b2e"}, + {file = "cryptography-39.0.1-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:ef8b72fa70b348724ff1218267e7f7375b8de4e8194d1636ee60510aae104cd0"}, + {file = "cryptography-39.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:aec5a6c9864be7df2240c382740fcf3b96928c46604eaa7f3091f58b878c0bb6"}, + {file = "cryptography-39.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:fdd188c8a6ef8769f148f88f859884507b954cc64db6b52f66ef199bb9ad660a"}, + {file = "cryptography-39.0.1.tar.gz", hash = "sha256:d1f6198ee6d9148405e49887803907fe8962a23e6c6f83ea7d98f1c0de375695"}, +] + +[package.dependencies] +cffi = ">=1.12" + +[package.extras] +docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] +docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] +pep8test = ["black", "check-manifest", "mypy", "ruff", "types-pytz", "types-requests"] +sdist = ["setuptools-rust (>=0.11.4)"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-shard (>=0.1.2)", "pytest-subtests", "pytest-xdist", "pytz"] +test-randomorder = ["pytest-randomly"] +tox = ["tox"] + [[package]] name = "decorator" version = "5.1.1" @@ -1912,6 +2035,18 @@ files = [ {file = "pycodestyle-2.9.1.tar.gz", hash = "sha256:2c9607871d58c76354b697b42f5d57e1ada7d261c261efac224b664affdc5785"}, ] +[[package]] +name = "pycparser" +version = "2.21" +description = "C parser in Python" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, + {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, +] + [[package]] name = "pydantic" version = "1.10.4" @@ -2639,6 +2774,21 @@ files = [ doc = ["sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] test = ["mypy", "pytest", "typing-extensions"] +[[package]] +name = "types-pyopenssl" +version = "23.0.0.3" +description = "Typing stubs for pyOpenSSL" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "types-pyOpenSSL-23.0.0.3.tar.gz", hash = "sha256:6ca54d593f8b946f9570f9ed7457c41da3b518feff5e344851941a6209bea62b"}, + {file = "types_pyOpenSSL-23.0.0.3-py3-none-any.whl", hash = "sha256:847ab17a16475a882dc29898648a6a35ad0d3e11a5bba5aa8ab2f3435a8647cb"}, +] + +[package.dependencies] +cryptography = ">=35.0.0" + [[package]] name = "types-python-dateutil" version = "2.8.19.6" @@ -2651,6 +2801,22 @@ files = [ {file = "types_python_dateutil-2.8.19.6-py3-none-any.whl", hash = "sha256:cfb7d31021c6bce6f3362c69af6e3abb48fe3e08854f02487e844ff910deec2a"}, ] +[[package]] +name = "types-redis" +version = "4.5.1.1" +description = "Typing stubs for redis" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "types-redis-4.5.1.1.tar.gz", hash = "sha256:c072e4824855f46d0a968509c3e0fa4789fc13b62d472064527bad3d1815aeed"}, + {file = "types_redis-4.5.1.1-py3-none-any.whl", hash = "sha256:081dfeec730df6e3f32ccbdafe3198873b7c02516c22d79cc2a40efdd69a3963"}, +] + +[package.dependencies] +cryptography = ">=35.0.0" +types-pyOpenSSL = "*" + [[package]] name = "types-requests" version = "2.28.11.12" @@ -2881,4 +3047,4 @@ validation = ["fastjsonschema"] [metadata] lock-version = "2.0" python-versions = "^3.7.4" -content-hash = "7b24ff99d31f27beb822b0917cfb5cdf1be6a3b49aec62b5a35ef0ca21ec49e4" +content-hash = "8018f0b880e13d656975c8a26b840469485e97cf539c2624ae9e5a27cb6f0959" diff --git a/pyproject.toml b/pyproject.toml index 70ae7356f80..50cad7b5ad5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -105,6 +105,7 @@ redis = ["redis"] cfn-lint = "0.67.0" mypy = "^0.982" types-python-dateutil = "^2.8.19.6" +types-redis = "^4.5.1.1" [tool.coverage.run] source = ["aws_lambda_powertools"] From 81c141eb0b337c3da5decc81c065a05507cbda2e Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Fri, 17 Feb 2023 00:06:36 +0000 Subject: [PATCH 09/13] feat(redis) - removing wrong print --- aws_lambda_powertools/utilities/connections/redis.py | 1 - 1 file changed, 1 deletion(-) diff --git a/aws_lambda_powertools/utilities/connections/redis.py b/aws_lambda_powertools/utilities/connections/redis.py index 10a314db5d8..d394cba4777 100644 --- a/aws_lambda_powertools/utilities/connections/redis.py +++ b/aws_lambda_powertools/utilities/connections/redis.py @@ -95,7 +95,6 @@ def __init__( Redis client object configured from the given URL See: https://redis.readthedocs.io/en/latest/connections.html#redis.Redis.from_url """ - print(extra_options) super().__init__(redis.Redis, host, port, username, password, db_index, url, **extra_options) From 5d6b0d0a24f54cfde90f9d456e058192fe325dd1 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Fri, 17 Feb 2023 23:32:38 +0000 Subject: [PATCH 10/13] feat(redis) - removing fields and adding additional logic to validate the idempotency key --- .../idempotency/persistence/dynamodb.py | 2 + .../idempotency/persistence/redis.py | 61 ++++++++++--------- 2 files changed, 34 insertions(+), 29 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py index fe47d12845c..26f4e4c841d 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py @@ -59,6 +59,8 @@ def __init__( DynamoDB attribute name for status, by default "status" data_attr: str, optional DynamoDB attribute name for response data, by default "data" + validation_key_attr: str, optional + DynamoDB attribute name for hashed representation of the parts of the event used for validation boto_config: botocore.config.Config, optional Botocore configuration to pass during client initialization boto3_session : boto3.session.Session, optional diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/redis.py b/aws_lambda_powertools/utilities/idempotency/persistence/redis.py index b291a5e2ef1..56913a70d20 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/redis.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/redis.py @@ -1,14 +1,18 @@ +import datetime import logging -import os -from typing import Any, Dict, Optional +from typing import Any, Dict, Union + +import redis -from aws_lambda_powertools.shared import constants from aws_lambda_powertools.utilities.idempotency import BasePersistenceLayer from aws_lambda_powertools.utilities.idempotency.exceptions import ( IdempotencyItemAlreadyExistsError, IdempotencyItemNotFoundError, ) -from aws_lambda_powertools.utilities.idempotency.persistence.base import DataRecord +from aws_lambda_powertools.utilities.idempotency.persistence.base import ( + STATUS_CONSTANTS, + DataRecord, +) logger = logging.getLogger(__name__) @@ -17,48 +21,37 @@ class RedisCachePersistenceLayer(BasePersistenceLayer): def __init__( self, connection, - static_pk_value: Optional[str] = None, - expiry_attr: str = "expiration", - in_progress_expiry_attr="in_progress_expiration", + in_progress_expiry_attr: str = "in_progress_expiration", status_attr: str = "status", data_attr: str = "data", - validation_key_attr="validation", + validation_key_attr: str = "validation", ): """ Initialize the Redis Persistence Layer Parameters ---------- - static_pk_value: str, optional - Redis attribute value for cache key, by default "idempotency#". - expiry_attr: str, optional - Redis hash attribute name for expiry timestamp, by default "expiration" in_progress_expiry_attr: str, optional Redis hash attribute name for in-progress expiry timestamp, by default "in_progress_expiration" status_attr: str, optional Redis hash attribute name for status, by default "status" data_attr: str, optional Redis hash attribute name for response data, by default "data" + validation_key_attr: str, optional + Redis hash attribute name for hashed representation of the parts of the event used for validation """ # Initialize connection with Redis - self._connection = connection.init_connection() - - if static_pk_value is None: - static_pk_value = f"idempotency#{os.getenv(constants.LAMBDA_FUNCTION_NAME_ENV, '')}" + self._connection: Union[redis.Redis, redis.RedisCluster] = connection.init_connection() - self.static_pk_value = static_pk_value self.in_progress_expiry_attr = in_progress_expiry_attr - self.expiry_attr = expiry_attr self.status_attr = status_attr self.data_attr = data_attr self.validation_key_attr = validation_key_attr super(RedisCachePersistenceLayer, self).__init__() def _item_to_data_record(self, item: Dict[str, Any]) -> DataRecord: - # Need to review this after adding GETKEY logic return DataRecord( status=item[self.status_attr], - expiry_timestamp=item[self.expiry_attr], in_progress_expiry_timestamp=item.get(self.in_progress_expiry_attr), response_data=item.get(self.data_attr), payload_hash=item.get(self.validation_key_attr), @@ -75,23 +68,24 @@ def _get_record(self, idempotency_key) -> DataRecord: return self._item_to_data_record(item) def _put_record(self, data_record: DataRecord) -> None: + item: Dict[str, Any] = {} + # Redis works with hset to support hashing keys with multiple attributes # See: https://redis.io/commands/hset/ item = { "name": data_record.idempotency_key, "mapping": { - self.in_progress_expiry_attr: data_record.in_progress_expiry_timestamp, self.status_attr: data_record.status, - self.expiry_attr: data_record.expiry_timestamp, }, } if data_record.in_progress_expiry_timestamp is not None: - item.update({"mapping": {self.in_progress_expiry_attr: data_record.in_progress_expiry_timestamp}}) + item["mapping"][self.in_progress_expiry_attr] = data_record.in_progress_expiry_timestamp if self.payload_validation_enabled: - item.update({"mapping": {self.validation_key_attr: data_record.payload_hash}}) + item["mapping"][self.validation_key_attr] = data_record.payload_hash + now = datetime.datetime.now() try: # | LOCKED | RETRY if status = "INPROGRESS" | RETRY # |----------------|-------------------------------------------------------|-------------> .... (time) @@ -104,13 +98,20 @@ def _put_record(self, data_record: DataRecord) -> None: # The idempotency key does not exist: # - first time that this invocation key is used # - previous invocation with the same key was deleted due to TTL - idempotency_key_not_exist = self._connection.exists(data_record.idempotency_key) + idempotency_record = self._connection.hgetall(data_record.idempotency_key) + if len(idempotency_record) > 0: + # record already exists. - # key exists - if idempotency_key_not_exist == 1: - raise + # status is completed, so raise exception because it exists and still valid + if idempotency_record[self.status_attr] == STATUS_CONSTANTS["COMPLETED"]: + raise - # missing logic to compare expiration + # checking if in_progress_expiry_attr exists + # if in_progress_expiry_attr exist, must be lower than now + if self.in_progress_expiry_attr in idempotency_record and int( + idempotency_record[self.in_progress_expiry_attr] + ) > int(now.timestamp() * 1000): + raise logger.debug(f"Putting record on Redis for idempotency key: {data_record.idempotency_key}") self._connection.hset(**item) @@ -122,6 +123,8 @@ def _put_record(self, data_record: DataRecord) -> None: raise IdempotencyItemAlreadyExistsError def _update_record(self, data_record: DataRecord) -> None: + item: Dict[str, Any] = {} + item = { "name": data_record.idempotency_key, "mapping": { From 038926cd3418278e10f23c9785bda92c86d01d6e Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Fri, 17 Feb 2023 23:43:20 +0000 Subject: [PATCH 11/13] feat(redis) - adding redis as dev dependency --- poetry.lock | 6 +++--- pyproject.toml | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 00226bc6eec..871a2472a9b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -27,7 +27,7 @@ name = "async-timeout" version = "4.0.2" description = "Timeout context manager for asyncio programs" category = "main" -optional = true +optional = false python-versions = ">=3.6" files = [ {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, @@ -2536,7 +2536,7 @@ name = "redis" version = "4.5.1" description = "Python client for Redis database and key-value store" category = "main" -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "redis-4.5.1-py3-none-any.whl", hash = "sha256:5deb072d26e67d2be1712603bfb7947ec3431fb0eec9c578994052e33035af6d"}, @@ -3144,4 +3144,4 @@ validation = ["fastjsonschema"] [metadata] lock-version = "2.0" python-versions = "^3.7.4" -content-hash = "ddcef7738045cc2c082cb5a58200d7a2305e82376415a382d56d03227ae83ba1" +content-hash = "bdbccdd716558b962a801c46c062aeb7892153a1f821c3cfee836294a102cb0f" diff --git a/pyproject.toml b/pyproject.toml index e82119375e8..54f06b6a849 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -107,6 +107,7 @@ mypy = "^0.982" types-python-dateutil = "^2.8.19.6" types-redis = "^4.5.1.1" httpx = "^0.23.3" +redis = "^4.5.1" [tool.coverage.run] source = ["aws_lambda_powertools"] From ca9b16db14f945f6f2696258e035f5247817d597 Mon Sep 17 00:00:00 2001 From: Vandita Patidar Date: Thu, 23 Mar 2023 14:11:04 -0700 Subject: [PATCH 12/13] Update idempotency.md Signed-off-by: Vandita Patidar --- docs/utilities/idempotency.md | 254 ++++++++++++++++++++++++++++++++++ 1 file changed, 254 insertions(+) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 49a028168b3..081ad3a509e 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -28,6 +28,7 @@ times with the same parameters**. This makes idempotent operations safe to retry ### IAM Permissions +#### DynamoDB Your Lambda function IAM Role must have `dynamodb:GetItem`, `dynamodb:PutItem`, `dynamodb:UpdateItem` and `dynamodb:DeleteItem` IAM permissions before using this feature. ???+ note @@ -35,10 +36,14 @@ Your Lambda function IAM Role must have `dynamodb:GetItem`, `dynamodb:PutItem`, ### Required resources +_**DynamoDB**_ Before getting started, you need to create a persistent storage layer where the idempotency utility can store its state - your lambda functions will need read and write access to it. As of now, Amazon DynamoDB is the only supported persistent storage layer, so you'll need to create a table first. +_**Redis**_ +Before getting started you need to setup your [EC2 Instance](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EC2_GetStarted.html) and [ElastiCache for Redis cluster](https://docs.aws.amazon.com/AmazonElastiCache/latest/red-ug/GettingStarted.html). + **Default table configuration** If you're not [changing the default configuration for the DynamoDB persistence layer](#dynamodbpersistencelayer), this is the expected default configuration: @@ -90,6 +95,8 @@ Resources: ### Idempotent decorator +_**DynamoDB**_ + You can quickly start by initializing the `DynamoDBPersistenceLayer` class and using it with the `idempotent` decorator on your lambda handler. === "app.py" @@ -123,6 +130,31 @@ You can quickly start by initializing the `DynamoDBPersistenceLayer` class and u "product_id": "123456789" } ``` + +_**Redis**_ + +You can initialize `RedisCachePersistenceLayer` class and use it with `idempotent` decorator on your lambda handler. + +=== "app.py" + +``` + from aws_lambda_powertools.utilities.connections import RedisStandalone, RedisCluster + from aws_lambda_powertools.utilities.idempotency import ( + idempotent, + RedisCachePersistenceLayer, + IdempotencyConfig + ) + # For connection using Redis Standalone architecture + redis_connection = RedisStandalone(host="192.168.68.112", port=6379, password="pass", db_index=0) + + persistence_layer = RedisCachePersistenceLayer(connection=redis_connection) + config = IdempotencyConfig( + expires_after_seconds=1*60, # 1 minutes + ) + @idempotent(config=config, persistence_store=persistence_layer) + def lambda_handler(event, context): + return {"message":"Hello"} +``` ### Idempotent_function decorator @@ -565,6 +597,57 @@ When using DynamoDB as a persistence layer, you can alter the attribute names by | **sort_key_attr** | | | Sort key of the table (if table is configured with a sort key). | | **static_pk_value** | | `idempotency#{LAMBDA_FUNCTION_NAME}` | Static value to use as the partition key. Only used when **sort_key_attr** is set. | +#### RedisCachePersistenceLayer + +This persistence layer is built-in and you can use ElastiCache to store and see the keys. + +``` + from aws_lambda_powertools.utilities.idempotency import RedisCachePersistenceLayer + persistence_layer = RedisCachePersistenceLayer( + static_pk_value: Optional[str] = None, + expiry_attr: str = "expiration", + in_progress_expiry_attr: str = "in_progress_expiration", + status_attr: str = "status", + data_attr: str = "data", + validation_key_attr: str = "validation", + ) +``` + +When using ElastiCache for Redis as a persistence layer, you can alter the attribute names by passing these parameters when initializing the persistence layer: + +| Parameter | Required | Default | Description | +| --------------------------- | ------------------ | ------------------------------------ | -------------------------------------------------------------------------------------------------------- | +| **static_pk_value** | | `idempotency#{LAMBDA_FUNCTION_NAME}` | Static value to use as the partition key. Only used when **sort_key_attr** is set. | +| **expiry_attr** | | `expiration` | Unix timestamp of when record expires | +| **in_progress_expiry_attr** | | `in_progress_expiration` | Unix timestamp of when record expires while in progress (in case of the invocation times out) | +| **status_attr** | | `status` | Stores status of the lambda execution during and after invocation | +| **data_attr** | | `data` | Stores results of successfully executed Lambda handlers | +| **validation_key_attr** | | `validation` | Hashed representation of the parts of the event used for validation | + +#### RedisStandalone/RedisCluster: + +``` +from aws_lambda_powertools.utilities.connections import RedisStandalone,RedisCluster + +redis_connection = RedisStandalone( + host="192.168.68.112", + port=6379, + username = "abc" + password="pass", + db_index=0, + url = None +) +``` + +| Parameter | Required | Default | Description | +| --------------------------- | ------------------ | ------------------------------------ | -------------------------------------------------------------------------------------------------------- | +| **host** | | `localhost` | Name of the host to connect to Redis instance/cluster | +| **port** | | 6379 | Number of the port to connect to Redis instance/cluster | +| **username** | | `None` | Name of the username to connect to Redis instance/cluster in case of using ACL | +| **password** | | `None` | Passwod to connect to Redis instance/cluster | +| **db_index** | | 0. | Index of Redis database | +| **url** | | `None` | Redis client object configured from the given URL. | + ## Advanced ### Customizing the default behavior @@ -626,6 +709,8 @@ In most cases, it is not desirable to store the idempotency records forever. Rat You can change this window with the **`expires_after_seconds`** parameter: +_**DynamoDB**_ + ```python hl_lines="8 11" title="Adjusting cache TTL" from aws_lambda_powertools.utilities.idempotency import ( IdempotencyConfig, DynamoDBPersistenceLayer, idempotent @@ -642,6 +727,24 @@ def handler(event, context): ... ``` +_**Redis**_ + +``` +from aws_lambda_powertools.utilities.connections import RedisStandalone, RedisCluster +from aws_lambda_powertools.utilities.idempotency import ( + idempotent, + RedisCachePersistenceLayer, + IdempotencyConfig +) +# For connection using Redis Standalone architecture +redis_connection = RedisStandalone(host="192.168.68.112", port=6379, password="pass", db_index=0) + +persistence_layer = RedisCachePersistenceLayer(connection=redis_connection) +config = IdempotencyConfig( + expires_after_seconds=5*60, # 5 minutes +) +``` + This will mark any records older than 5 minutes as expired, and the lambda handler will be executed as normal if it is invoked with a matching payload. ???+ note "Note: DynamoDB time-to-live field" @@ -856,6 +959,8 @@ This utility provides an abstract base class (ABC), so that you can implement yo You can inherit from the `BasePersistenceLayer` class and implement the abstract methods `_get_record`, `_put_record`, `_update_record` and `_delete_record`. +_**DynamoDB**_ + ```python hl_lines="8-13 57 65 74 96 124" title="Excerpt DynamoDB Persistence Layer implementation for reference" import datetime import logging @@ -985,6 +1090,155 @@ class DynamoDBPersistenceLayer(BasePersistenceLayer): self.table.delete_item(Key={self.key_attr: data_record.idempotency_key},) ``` +_**Redis**_ + +``` +import logging +import os +from typing import Any, Dict, Optional + +from aws_lambda_powertools.shared import constants +from aws_lambda_powertools.utilities.idempotency import BasePersistenceLayer +from aws_lambda_powertools.utilities.idempotency.exceptions import ( + IdempotencyItemAlreadyExistsError, + IdempotencyItemNotFoundError, +) +from aws_lambda_powertools.utilities.idempotency.persistence.base import DataRecord + +logger = logging.getLogger(__name__) + + +class RedisCachePersistenceLayer(BasePersistenceLayer): + def __init__( + self, + connection, + static_pk_value: Optional[str] = None, + expiry_attr: str = "expiration", + in_progress_expiry_attr: str = "in_progress_expiration", + status_attr: str = "status", + data_attr: str = "data", + validation_key_attr: str = "validation", + ): + """ + Initialize the Redis Persistence Layer + Parameters + ---------- + static_pk_value: str, optional + Redis attribute value for cache key, by default "idempotency#". + expiry_attr: str, optional + Redis hash attribute name for expiry timestamp, by default "expiration" + in_progress_expiry_attr: str, optional + Redis hash attribute name for in-progress expiry timestamp, by default "in_progress_expiration" + status_attr: str, optional + Redis hash attribute name for status, by default "status" + data_attr: str, optional + Redis hash attribute name for response data, by default "data" + """ + + # Initialize connection with Redis + self._connection = connection.init_connection() + + if static_pk_value is None: + static_pk_value = f"idempotency#{os.getenv(constants.LAMBDA_FUNCTION_NAME_ENV, '')}" + + self.static_pk_value = static_pk_value + self.in_progress_expiry_attr = in_progress_expiry_attr + self.expiry_attr = expiry_attr + self.status_attr = status_attr + self.data_attr = data_attr + self.validation_key_attr = validation_key_attr + super(RedisCachePersistenceLayer, self).__init__() + + def _get_key(self, idempotency_key: str) -> dict: + # Need to review this after adding GETKEY logic + if self.sort_key_attr: + return {self.key_attr: self.static_pk_value, self.sort_key_attr: idempotency_key} + return {self.key_attr: idempotency_key} + + def _item_to_data_record(self, item: Dict[str, Any]) -> DataRecord: + # Need to review this after adding GETKEY logic + return DataRecord( + status=item[self.status_attr], + expiry_timestamp=item[self.expiry_attr], + in_progress_expiry_timestamp=item.get(self.in_progress_expiry_attr), + response_data=item.get(self.data_attr), + payload_hash=item.get(self.validation_key_attr), + ) + + def _get_record(self, idempotency_key) -> DataRecord: + # See: https://redis.io/commands/hgetall/ + response = self._connection.hgetall(idempotency_key) + + try: + item = response + except KeyError: + raise IdempotencyItemNotFoundError + return self._item_to_data_record(item) + + def _put_record(self, data_record: DataRecord) -> None: + # Redis works with hset to support hashing keys with multiple attributes + # See: https://redis.io/commands/hset/ + item = { + "name": data_record.idempotency_key, + "mapping": { + self.in_progress_expiry_attr: data_record.in_progress_expiry_timestamp, + self.status_attr: data_record.status, + self.expiry_attr: data_record.expiry_timestamp, + }, + } + + if data_record.in_progress_expiry_timestamp is not None: + item["mapping"][self.in_progress_expiry_attr] = data_record.in_progress_expiry_timestamp + + if self.payload_validation_enabled: + item["mapping"][self.validation_key_attr] = data_record.payload_hash + + try: + # | LOCKED | RETRY if status = "INPROGRESS" | RETRY + # |----------------|-------------------------------------------------------|-------------> .... (time) + # | Lambda Idempotency Record + # | Timeout Timeout + # | (in_progress_expiry) (expiry) + + # Conditions to successfully save a record: + + # The idempotency key does not exist: + # - first time that this invocation key is used + # - previous invocation with the same key was deleted due to TTL + idempotency_key_not_exist = self._connection.exists(data_record.idempotency_key) + + # key exists + if idempotency_key_not_exist == 1: + raise + + # missing logic to compare expiration + + logger.debug(f"Putting record on Redis for idempotency key: {data_record.idempotency_key}") + self._connection.hset(**item) + # hset type must set expiration after adding the record + # Need to review this to get ttl in seconds + self._connection.expire(name=data_record.idempotency_key, time=self.expires_after_seconds) + except Exception: + logger.debug(f"Failed to put record for already existing idempotency key: {data_record.idempotency_key}") + raise IdempotencyItemAlreadyExistsError + + def _update_record(self, data_record: DataRecord) -> None: + item = { + "name": data_record.idempotency_key, + "mapping": { + self.data_attr: data_record.response_data, + self.status_attr: data_record.status, + }, + } + logger.debug(f"Updating record for idempotency key: {data_record.idempotency_key}") + self._connection.hset(**item) + + def _delete_record(self, data_record: DataRecord) -> None: + logger.debug(f"Deleting record for idempotency key: {data_record.idempotency_key}") + # See: https://redis.io/commands/del/ + self._connection.delete(data_record.idempotency_key) +``` + ???+ danger Pay attention to the documentation for each - you may need to perform additional checks inside these methods to ensure the idempotency guarantees remain intact. From b8bcfe4b90503c13c7bbdce1a486a3d24ffcc806 Mon Sep 17 00:00:00 2001 From: Vandita Patidar Date: Sun, 26 Mar 2023 21:35:44 -0700 Subject: [PATCH 13/13] Update idempotency.md Signed-off-by: Vandita Patidar --- docs/utilities/idempotency.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 081ad3a509e..8b0c1800c94 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -37,11 +37,13 @@ Your Lambda function IAM Role must have `dynamodb:GetItem`, `dynamodb:PutItem`, ### Required resources _**DynamoDB**_ + Before getting started, you need to create a persistent storage layer where the idempotency utility can store its state - your lambda functions will need read and write access to it. As of now, Amazon DynamoDB is the only supported persistent storage layer, so you'll need to create a table first. _**Redis**_ + Before getting started you need to setup your [EC2 Instance](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EC2_GetStarted.html) and [ElastiCache for Redis cluster](https://docs.aws.amazon.com/AmazonElastiCache/latest/red-ug/GettingStarted.html). **Default table configuration**