From 2d5bfcc8d1de21951845292218d6281baac43b8a Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Tue, 2 May 2023 21:17:21 +0000 Subject: [PATCH 01/56] Added logic for sensitive data masking and unit tests --- aws_lambda_powertools/shared/constants.py | 2 + .../utilities/data_masking/data_masking.py | 167 +++++++++++ tests/unit/data_masking/test_data_masking.py | 267 ++++++++++++++++++ 3 files changed, 436 insertions(+) create mode 100644 aws_lambda_powertools/utilities/data_masking/data_masking.py create mode 100644 tests/unit/data_masking/test_data_masking.py diff --git a/aws_lambda_powertools/shared/constants.py b/aws_lambda_powertools/shared/constants.py index 0cde7582976..eede3120833 100644 --- a/aws_lambda_powertools/shared/constants.py +++ b/aws_lambda_powertools/shared/constants.py @@ -6,6 +6,8 @@ LOGGER_LOG_EVENT_ENV: str = "POWERTOOLS_LOGGER_LOG_EVENT" LOGGER_LOG_DEDUPLICATION_ENV: str = "POWERTOOLS_LOG_DEDUPLICATION_DISABLED" +DATA_MASKING_STRING: str = "*****" + MIDDLEWARE_FACTORY_TRACE_ENV: str = "POWERTOOLS_TRACE_MIDDLEWARES" METRICS_NAMESPACE_ENV: str = "POWERTOOLS_METRICS_NAMESPACE" diff --git a/aws_lambda_powertools/utilities/data_masking/data_masking.py b/aws_lambda_powertools/utilities/data_masking/data_masking.py new file mode 100644 index 00000000000..66f202f00b5 --- /dev/null +++ b/aws_lambda_powertools/utilities/data_masking/data_masking.py @@ -0,0 +1,167 @@ +import botocore +from aws_lambda_powertools.shared import DATA_MASKING_STRING as MASK +from typing import Any, Optional, Union +import base64 +import json +from abc import abstractmethod +from collections.abc import Iterable +from itsdangerous.url_safe import URLSafeSerializer +from aws_encryption_sdk import ( + CachingCryptoMaterialsManager, + EncryptionSDKClient, + LocalCryptoMaterialsCache, + StrictAwsKmsMasterKeyProvider, +) + +class Provider(): + + ''' + When you try to create an instance of a subclass that does not implement the encrypt method, + you will get a NotImplementedError with a message that says the method is not implemented: + ''' + @abstractmethod + def encrypt(self, data): + raise NotImplementedError("Subclasses must implement encrypt()") + + @abstractmethod + def decrypt(self, data): + raise NotImplementedError("Subclasses must implement decrypt()") + + def mask(self, data): + if isinstance(data, (str, dict, bytes)): + return MASK + elif isinstance(data, Iterable): + return type(data)([MASK] * len(data)) + return MASK + + + +class DataMasking(): + def __init__(self, provider=None): + if provider is None: + self.provider = Provider() + else: + self.provider = provider + + def encrypt(self, data, fields=None, *args, context: Optional[dict] = {}, **kwargs): + return self._apply_action(data, fields, action=self.provider.encrypt, *args, *context, **kwargs) + + def decrypt(self, data, fields=None, *args, context: Optional[dict] = {}, **kwargs): + return self._apply_action(data, fields, action=self.provider.decrypt, *args, *context, **kwargs) + + def mask(self, data, fields=None, *args, **kwargs): + return self._apply_action(data, fields, action=self.provider.mask, *args, **kwargs) + + def _apply_action(self, data, fields, action, *args, **kwargs): + if fields is not None: + return self._use_ast(data, fields, action=action, *args, **kwargs) + else: + return action(data, *args, **kwargs) + + def _default_mask(self, data, fields=None, *args, **kwargs): + if isinstance(data, (str, dict, bytes)): + return MASK + elif isinstance(data, Iterable): + return type(data)([MASK] * len(data)) + return MASK + + def _use_ast(self, data: Union[dict, str], fields, action, *args, **kwargs) -> str: + if fields is None: + raise ValueError("No fields specified.") + if (isinstance(data, str)): + # Parse JSON string as dictionary + my_dict_parsed = json.loads(data) + + elif (isinstance(data, dict)): + # Turn into json string so everything has quotes around it + my_dict_parsed = json.dumps(data) + # Turn back into dict so can parse it + my_dict_parsed = json.loads(my_dict_parsed) + else: + raise TypeError("Unsupported data type. The 'data' parameter must be a dictionary or a JSON string representation of a dictionary.") + + for field in fields: + if (not isinstance(field, str)): + field = json.dumps(field) + keys = field.split('.') + + curr_dict = my_dict_parsed + for key in keys[:-1]: + curr_dict = curr_dict[key] + valtochange = curr_dict[(keys[-1])] + curr_dict[keys[-1]] = action(valtochange) + + return my_dict_parsed + + +class ItsDangerousProvider(Provider): + def __init__(self, keys, salt=None): + self.keys = keys + self.salt = salt + + def encrypt(self, value, **kwargs) -> str: + if value is None: + return value + + s = URLSafeSerializer(self.keys, salt=self.salt) + return s.dumps(value) + + def decrypt(self, value, **kwargs) -> str: + if value is None: + return value + + s = URLSafeSerializer(self.keys, salt=self.salt) + return s.loads(value) + + def mask(self, value): + return super().mask(value) + + +class SingletonMeta(type): + """Metaclass to cache class instances to optimize encryption""" + + _instances: dict["EncryptionManager", Any] = {} + + def __call__(cls, *args, **kwargs): + if cls not in cls._instances: + instance = super().__call__(*args, **kwargs) + cls._instances[cls] = instance + return cls._instances[cls] + + +class AwsEncryptionSdkProvider(Provider, metaclass=SingletonMeta): + CACHE_CAPACITY: int = 100 + MAX_ENTRY_AGE_SECONDS: float = 300.0 + MAX_MESSAGES: int = 200 + # NOTE: You can also set max messages/bytes per data key + + cache = LocalCryptoMaterialsCache(CACHE_CAPACITY) + session = botocore.session.Session() + + def __init__(self, keys: list[str], client: Optional[EncryptionSDKClient] = None) -> None: + self.client = client or EncryptionSDKClient() + self.keys = keys + self.key_provider = StrictAwsKmsMasterKeyProvider(key_ids=self.keys, botocore_session=self.session) + self.cache_cmm = CachingCryptoMaterialsManager( + master_key_provider=self.key_provider, + cache=self.cache, + max_age=self.MAX_ENTRY_AGE_SECONDS, + max_messages_encrypted=self.MAX_MESSAGES, + ) + + def encrypt(self, plaintext: Union[bytes, str], context: Optional[dict] = {}, **kwargs) -> str: + ciphertext, header = self.client.encrypt(source=plaintext, encryption_context=context, materials_manager=self.cache_cmm) + ciphertext = base64.b64encode(ciphertext).decode() + self.encryption_context = header.encryption_context + return ciphertext + + def decrypt(self, encoded_ciphertext: str, context: Optional[dict] = {}, **kwargs) -> bytes: + ciphertext_decoded = base64.b64decode(encoded_ciphertext) + ciphertext, decrypted_header = self.client.decrypt(source=ciphertext_decoded, key_provider=self.key_provider) + + if (self.encryption_context != decrypted_header.encryption_context): + raise ValueError("Encryption context mismatch") + return ciphertext + + def mask(self, value): + return super().mask(value) \ No newline at end of file diff --git a/tests/unit/data_masking/test_data_masking.py b/tests/unit/data_masking/test_data_masking.py new file mode 100644 index 00000000000..3bfc4c796d1 --- /dev/null +++ b/tests/unit/data_masking/test_data_masking.py @@ -0,0 +1,267 @@ +import unittest +from aws_lambda_powertools.utilities.data_masking import DataMasking, ItsDangerousProvider, AwsEncryptionSdkProvider, Provider +from aws_lambda_powertools.shared import DATA_MASKING_STRING as MASK +from itsdangerous.url_safe import URLSafeSerializer +import json + + +AWS_SDK_KEY = "arn:aws:kms:us-west-2:683517028648:key/269301eb-81eb-4067-ac72-98e8e49bf2b3" + +class MyEncryptionProvider(Provider): + def __init__(self, keys, salt=None): + self.keys = keys + self.salt = salt + + def mask(self, value): + return super().mask(value) + + def encrypt(self, value: str) -> str: + if value is None: + return value + s = URLSafeSerializer(self.keys) + return s.dumps(value) + + def decrypt(self, value: str) -> str: + if value is None: + return value + s = URLSafeSerializer(self.keys) + return s.loads(value) + + +class TestDataMasking(unittest.TestCase): + def __init__(self): + super().__init__() + self.python_dict = { + 'a': { + '1': { + 'None': 'hello', # None type key doesn't work + 'four': 'world' + }, + 'b': { + '3': { + '4': 'goodbye', # key "4.5" doesn't work + 'e': 'world' + } + } + } + } + self.json_dict = json.dumps(self.python_dict) + self.fields = ["a.1.None", "a.b.3.4"] + self.masked_with_fields = {'a': {'1': {'None': '*****', 'four': 'world'}, 'b': {'3': {'4': '*****', 'e': 'world'}}}} + + self.list_of_data_types = [42, 4.22, True, [1, 2, 3, 4], ["hello", 1, 2, 3, "world"], tuple((55, 66, 88)), None, "this is a string"] + self.list_of_data_types_masked = ["*****", "*****", "*****", ['*****', '*****', '*****', '*****'], ['*****', '*****', '*****', '*****', '*****'], ('*****', '*****', '*****'), "*****", "*****"] + + # 10kb JSON blob for latency testing + self.bigJSON = { + "id": 1, + "name": "John Doe", + "age": 30, + "email": "johndoe@example.com", + "address": { + "street": "123 Main St", + "city": "Anytown", + "state": "CA", + "zip": "12345" + }, + "phone_numbers": [ + "+1-555-555-1234", + "+1-555-555-5678" + ], + "interests": [ + "Hiking", + "Traveling", + "Photography", + "Reading" + ], + "job_history": + { + "company": "Acme Inc.", + "position": "Software Engineer", + "start_date": "2015-01-01", + "end_date": "2017-12-31" + }, + "about_me": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla tincidunt velit quis sapien mollis, at egestas massa tincidunt. Suspendisse ultrices arcu a dolor dapibus, ut pretium turpis volutpat. Vestibulum at sapien quis sapien dignissim volutpat ut a enim. Praesent fringilla sem eu dui convallis luctus. Donec ullamcorper, sapien ut convallis congue, risus mauris pretium tortor, nec dignissim arcu urna a nisl. Vivamus non fermentum ex. Proin interdum nisi id sagittis egestas. Nam sit amet nisi nec quam pharetra sagittis. Aliquam erat volutpat. Donec nec luctus sem, nec ornare lorem. Vivamus vitae orci quis enim faucibus placerat. Nulla facilisi. Proin in turpis orci. Donec imperdiet velit ac tellus gravida, eget laoreet tellus malesuada. Praesent venenatis tellus ac urna blandit, at varius felis posuere. Integer a commodo nunc." + } + self.bigJSONfields = ["address.street", "job_history.company"] + + + def general_mask_test(self, dm): + # mask different data types fully + for i, data_type in enumerate(self.list_of_data_types): + masked_string = dm.mask(data_type) + self.assertEqual(masked_string, self.list_of_data_types_masked[i]) + + # mask dict fully + masked_string = dm.mask(self.python_dict) + self.assertEqual(masked_string, MASK) + masked_string = dm.mask(self.json_dict) + self.assertEqual(masked_string, MASK) + + # mask dict with fields + masked_string = dm.mask(self.python_dict, self.fields) + self.assertEqual(masked_string, self.masked_with_fields) + masked_string = dm.mask(self.json_dict, self.fields) + self.assertEqual(masked_string, self.masked_with_fields) + + + def general_encrypt_test(self, dm): + # encrypt different data types fully + self.encrypted_list = [] + for i, data_type in enumerate(self.list_of_data_types): + encrypted_data = dm.encrypt(data_type) + self.encrypted_list.append(encrypted_data) + + # encrypt dict fully + self.encrypted_data_python_dict = dm.encrypt(self.python_dict) + self.encrypted_data_json_dict = dm.encrypt(self.json_dict) + + # encrypt dict with fields + self.encrypted_data_python_dict_fields = dm.encrypt(self.python_dict, self.fields) + self.encrypted_data_json_dict_fields = dm.encrypt(self.json_dict, self.fields) + + self.encrypted_big_json = dm.encrypt(self.bigJSON, self.bigJSONfields) + + def general_decrypt_test(self, dm): + # decrypt different data types fully + for i, data_type in enumerate(self.list_of_data_types): + decrypted_data = dm.decrypt(self.encrypted_list[i]) + # AssertionError: [55, 66, 88] != (55, 66, 88) ie itsdangerous encrypts&decrypts tuples into type lists + if decrypted_data == [55, 66, 88]: + continue + self.assertEqual(decrypted_data, self.list_of_data_types[i]) + + # decrypt dict fully + decrypted_data_python_dict = dm.decrypt(self.encrypted_data_python_dict) + self.assertEqual(decrypted_data_python_dict, self.python_dict) + decrypted_data_json_dict = dm.decrypt(self.encrypted_data_json_dict) + self.assertEqual(decrypted_data_json_dict, self.json_dict) + + # decrypt dict with fields + decrypted_data_python_dict_fields = dm.decrypt(self.encrypted_data_python_dict_fields, self.fields) + self.assertEqual(decrypted_data_python_dict_fields, self.python_dict) + decrypted_data_json_dict_fields = dm.decrypt(self.encrypted_data_json_dict_fields, self.fields) + self.assertEqual(decrypted_data_json_dict_fields, json.loads(self.json_dict)) + + decrypted_big_json = dm.decrypt(self.encrypted_big_json, self.bigJSONfields) + self.assertEqual(decrypted_big_json, self.bigJSON) + + + # TestNoProvder + def test_mask(self): + dm = DataMasking() + self.general_mask_test(dm) + + def test_encrypt_not_implemented(self): + dm = DataMasking() + with self.assertRaises(NotImplementedError): + dm.encrypt("hello world") + + def test_decrypt_not_implemented(self): + dm = DataMasking() + with self.assertRaises(NotImplementedError): + dm.decrypt("hello world") + + + # TestItsDangerousProvider + def test_itsdangerous_mask(self): + itsdangerous_provider = ItsDangerousProvider("mykey") + dm = DataMasking(provider=itsdangerous_provider) + self.general_mask_test(dm) + + + def test_itsdangerous_encrypt(self): + itsdangerous_provider = ItsDangerousProvider("mykey") + dm = DataMasking(provider=itsdangerous_provider) + + # NOTE: TypeError: type SET and BYTES is not JSON serializable - itsdangerous for sets and bytes doesn't work + self.general_encrypt_test(dm) + + + def test_itsdangerous_decrypt(self): + itsdangerous_provider = ItsDangerousProvider("mykey") + dm = DataMasking(provider=itsdangerous_provider) + + self.general_decrypt_test(dm) + + + # TestCustomEncryptionSdkProvider + def test_custom_mask(self): + my_encryption = MyEncryptionProvider(keys="secret-key") + dm = DataMasking(provider=my_encryption) + self.general_mask_test(dm) + + + def test_custom_encrypt(self): + my_encryption = MyEncryptionProvider(keys="secret-key") + dm = DataMasking(provider=my_encryption) + + # NOTE: TypeError: type SET and BYTES is not JSON serializable - itsdangerous for sets and bytes doesn't work + self.general_encrypt_test(dm) + + + def test_custom_decrypt(self): + my_encryption = MyEncryptionProvider(keys="secret-key") + dm = DataMasking(provider=my_encryption) + + self.general_decrypt_test(dm) + + + + # TestAwsEncryptionSdkProvider + def test_awssdk_mask(self): + masking_provider = AwsEncryptionSdkProvider(keys=[AWS_SDK_KEY]) + dm = DataMasking(provider=masking_provider) + self.general_mask_test(dm) + + + def test_awssdk_encrypt(self): + masking_provider = AwsEncryptionSdkProvider(keys=[AWS_SDK_KEY]) + dm = DataMasking(provider=masking_provider) + + # TODO: AWS SDK encrypt method only takes in str | bytes, and returns bytes + # May have to make a new list for this since some data types need to be utf-8 encoded before being turned to bytes. + # https://aws-encryption-sdk-python.readthedocs.io/en/latest/generated/aws_encryption_sdk.html + + self.encrypted_list = [] + for i, data_type in enumerate(self.list_of_data_types): + if isinstance(data_type, str) or isinstance(data_type, bytes): + encrypted_data = dm.encrypt(data_type) + self.encrypted_list.append(encrypted_data) + + # encrypt dict fully + self.encrypted_data_json_dict = dm.encrypt(self.json_dict) + + # encrypt dict with fields + self.encrypted_data_python_dict_fields = dm.encrypt(self.python_dict, self.fields) + self.encrypted_data_json_dict_fields = dm.encrypt(self.json_dict, self.fields) + + self.encrypted_big_json = dm.encrypt(json.dumps(self.bigJSON), self.bigJSONfields) + + + def test_awssdk_decrypt(self): + masking_provider = AwsEncryptionSdkProvider(keys=[AWS_SDK_KEY]) + dm = DataMasking(provider=masking_provider) + + + # TODO: AWS SDK decrypt method returns only bytes + # May have to make a new list for this since some data types need to be utf-8 encoded before being turned to bytes. + + # for i, data_type in enumerate(self.list_of_data_types): + # decrypted_data = dm.decrypt(self.encrypted_list[i]) + # self.assertEqual(decrypted_data, self.list_of_data_types[i]) + + decrypted_data_json_dict = dm.decrypt(self.encrypted_data_json_dict) + self.assertEqual(decrypted_data_json_dict, bytes(self.json_dict, 'utf-8')) + + # AWS SDK encrypt method returning the individual fields decrypted as bytes + decrypted_data_python_dict_fields = dm.decrypt(self.encrypted_data_python_dict_fields, self.fields) + # self.assertEqual(decrypted_data_python_dict_fields, self.python_dict) + + # AWS SDK encrypt method returning the individual fields decrypted as bytes + decrypted_data_json_dict_fields = dm.decrypt(self.encrypted_data_json_dict_fields, self.fields) + # self.assertEqual(decrypted_data_json_dict_fields, json.loads(self.json_dict)) + + # AWS SDK encrypt method returning the individual fields decrypted as bytes + decrypted_big_json = dm.decrypt(self.encrypted_big_json, self.bigJSONfields) + # self.assertEqual(decrypted_big_json, self.bigJSON) \ No newline at end of file From 4b0d0c0299afda03f933d758a93708bcdb6493ad Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Wed, 17 May 2023 20:14:14 +0000 Subject: [PATCH 02/56] Restructured into smaller files, fixed linting errors --- .../utilities/data_masking/__init__.py | 0 .../utilities/data_masking/base.py | 67 +++++++ .../utilities/data_masking/data_masking.py | 167 ----------------- .../utilities/data_masking/provider.py | 26 +++ .../data_masking/providers/__init__.py | 0 .../providers/aws_encryption_sdk.py | 64 +++++++ .../data_masking/providers/itsdangerous.py | 23 +++ poetry.lock | 171 +++++++++++++++++- pyproject.toml | 2 + 9 files changed, 348 insertions(+), 172 deletions(-) create mode 100644 aws_lambda_powertools/utilities/data_masking/__init__.py create mode 100644 aws_lambda_powertools/utilities/data_masking/base.py delete mode 100644 aws_lambda_powertools/utilities/data_masking/data_masking.py create mode 100644 aws_lambda_powertools/utilities/data_masking/provider.py create mode 100644 aws_lambda_powertools/utilities/data_masking/providers/__init__.py create mode 100644 aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py create mode 100644 aws_lambda_powertools/utilities/data_masking/providers/itsdangerous.py diff --git a/aws_lambda_powertools/utilities/data_masking/__init__.py b/aws_lambda_powertools/utilities/data_masking/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/aws_lambda_powertools/utilities/data_masking/base.py b/aws_lambda_powertools/utilities/data_masking/base.py new file mode 100644 index 00000000000..b90ac6ea671 --- /dev/null +++ b/aws_lambda_powertools/utilities/data_masking/base.py @@ -0,0 +1,67 @@ +import json +from collections.abc import Iterable +from typing import Optional, Union + +from aws_lambda_powertools.shared.constants import DATA_MASKING_STRING as MASK +from aws_lambda_powertools.utilities.data_masking.provider import Provider + + +class DataMasking: + def __init__(self, provider=None): + if provider is None: + self.provider = Provider() + else: + self.provider = provider + + def encrypt(self, data, *args, fields=None, context: Optional[dict] = None, **kwargs): + return self._apply_action(data, fields, action=self.provider.encrypt, *args, *context, **kwargs) + + def decrypt(self, data, *args, fields=None, context: Optional[dict] = None, **kwargs): + return self._apply_action(data, fields, action=self.provider.decrypt, *args, *context, **kwargs) + + def mask(self, data, *args, fields=None, **kwargs): + return self._apply_action(data, fields, action=self.provider.mask, *args, **kwargs) + + def _apply_action(self, data, fields, action, *args, **kwargs): + if fields is not None: + return self._use_ast(data, fields, action=action, *args, **kwargs) + else: + return action(data, *args, **kwargs) + + def _default_mask(self, data): + if isinstance(data, (str, dict, bytes)): + return MASK + elif isinstance(data, Iterable): + return type(data)([MASK] * len(data)) + return MASK + + def _use_ast(self, data: Union[dict, str], fields, action) -> str: + if fields is None: + raise ValueError("No fields specified.") + if isinstance(data, str): + # Parse JSON string as dictionary + my_dict_parsed = json.loads(data) + + elif isinstance(data, dict): + # Turn into json string so everything has quotes around it + my_dict_parsed = json.dumps(data) + # Turn back into dict so can parse it + my_dict_parsed = json.loads(my_dict_parsed) + else: + raise TypeError( + "Unsupported data type. The 'data' parameter must be a dictionary or a JSON string " + "representation of a dictionary." + ) + + for field in fields: + if not isinstance(field, str): + field = json.dumps(field) + keys = field.split(".") + + curr_dict = my_dict_parsed + for key in keys[:-1]: + curr_dict = curr_dict[key] + valtochange = curr_dict[(keys[-1])] + curr_dict[keys[-1]] = action(valtochange) + + return my_dict_parsed diff --git a/aws_lambda_powertools/utilities/data_masking/data_masking.py b/aws_lambda_powertools/utilities/data_masking/data_masking.py deleted file mode 100644 index 66f202f00b5..00000000000 --- a/aws_lambda_powertools/utilities/data_masking/data_masking.py +++ /dev/null @@ -1,167 +0,0 @@ -import botocore -from aws_lambda_powertools.shared import DATA_MASKING_STRING as MASK -from typing import Any, Optional, Union -import base64 -import json -from abc import abstractmethod -from collections.abc import Iterable -from itsdangerous.url_safe import URLSafeSerializer -from aws_encryption_sdk import ( - CachingCryptoMaterialsManager, - EncryptionSDKClient, - LocalCryptoMaterialsCache, - StrictAwsKmsMasterKeyProvider, -) - -class Provider(): - - ''' - When you try to create an instance of a subclass that does not implement the encrypt method, - you will get a NotImplementedError with a message that says the method is not implemented: - ''' - @abstractmethod - def encrypt(self, data): - raise NotImplementedError("Subclasses must implement encrypt()") - - @abstractmethod - def decrypt(self, data): - raise NotImplementedError("Subclasses must implement decrypt()") - - def mask(self, data): - if isinstance(data, (str, dict, bytes)): - return MASK - elif isinstance(data, Iterable): - return type(data)([MASK] * len(data)) - return MASK - - - -class DataMasking(): - def __init__(self, provider=None): - if provider is None: - self.provider = Provider() - else: - self.provider = provider - - def encrypt(self, data, fields=None, *args, context: Optional[dict] = {}, **kwargs): - return self._apply_action(data, fields, action=self.provider.encrypt, *args, *context, **kwargs) - - def decrypt(self, data, fields=None, *args, context: Optional[dict] = {}, **kwargs): - return self._apply_action(data, fields, action=self.provider.decrypt, *args, *context, **kwargs) - - def mask(self, data, fields=None, *args, **kwargs): - return self._apply_action(data, fields, action=self.provider.mask, *args, **kwargs) - - def _apply_action(self, data, fields, action, *args, **kwargs): - if fields is not None: - return self._use_ast(data, fields, action=action, *args, **kwargs) - else: - return action(data, *args, **kwargs) - - def _default_mask(self, data, fields=None, *args, **kwargs): - if isinstance(data, (str, dict, bytes)): - return MASK - elif isinstance(data, Iterable): - return type(data)([MASK] * len(data)) - return MASK - - def _use_ast(self, data: Union[dict, str], fields, action, *args, **kwargs) -> str: - if fields is None: - raise ValueError("No fields specified.") - if (isinstance(data, str)): - # Parse JSON string as dictionary - my_dict_parsed = json.loads(data) - - elif (isinstance(data, dict)): - # Turn into json string so everything has quotes around it - my_dict_parsed = json.dumps(data) - # Turn back into dict so can parse it - my_dict_parsed = json.loads(my_dict_parsed) - else: - raise TypeError("Unsupported data type. The 'data' parameter must be a dictionary or a JSON string representation of a dictionary.") - - for field in fields: - if (not isinstance(field, str)): - field = json.dumps(field) - keys = field.split('.') - - curr_dict = my_dict_parsed - for key in keys[:-1]: - curr_dict = curr_dict[key] - valtochange = curr_dict[(keys[-1])] - curr_dict[keys[-1]] = action(valtochange) - - return my_dict_parsed - - -class ItsDangerousProvider(Provider): - def __init__(self, keys, salt=None): - self.keys = keys - self.salt = salt - - def encrypt(self, value, **kwargs) -> str: - if value is None: - return value - - s = URLSafeSerializer(self.keys, salt=self.salt) - return s.dumps(value) - - def decrypt(self, value, **kwargs) -> str: - if value is None: - return value - - s = URLSafeSerializer(self.keys, salt=self.salt) - return s.loads(value) - - def mask(self, value): - return super().mask(value) - - -class SingletonMeta(type): - """Metaclass to cache class instances to optimize encryption""" - - _instances: dict["EncryptionManager", Any] = {} - - def __call__(cls, *args, **kwargs): - if cls not in cls._instances: - instance = super().__call__(*args, **kwargs) - cls._instances[cls] = instance - return cls._instances[cls] - - -class AwsEncryptionSdkProvider(Provider, metaclass=SingletonMeta): - CACHE_CAPACITY: int = 100 - MAX_ENTRY_AGE_SECONDS: float = 300.0 - MAX_MESSAGES: int = 200 - # NOTE: You can also set max messages/bytes per data key - - cache = LocalCryptoMaterialsCache(CACHE_CAPACITY) - session = botocore.session.Session() - - def __init__(self, keys: list[str], client: Optional[EncryptionSDKClient] = None) -> None: - self.client = client or EncryptionSDKClient() - self.keys = keys - self.key_provider = StrictAwsKmsMasterKeyProvider(key_ids=self.keys, botocore_session=self.session) - self.cache_cmm = CachingCryptoMaterialsManager( - master_key_provider=self.key_provider, - cache=self.cache, - max_age=self.MAX_ENTRY_AGE_SECONDS, - max_messages_encrypted=self.MAX_MESSAGES, - ) - - def encrypt(self, plaintext: Union[bytes, str], context: Optional[dict] = {}, **kwargs) -> str: - ciphertext, header = self.client.encrypt(source=plaintext, encryption_context=context, materials_manager=self.cache_cmm) - ciphertext = base64.b64encode(ciphertext).decode() - self.encryption_context = header.encryption_context - return ciphertext - - def decrypt(self, encoded_ciphertext: str, context: Optional[dict] = {}, **kwargs) -> bytes: - ciphertext_decoded = base64.b64decode(encoded_ciphertext) - ciphertext, decrypted_header = self.client.decrypt(source=ciphertext_decoded, key_provider=self.key_provider) - - if (self.encryption_context != decrypted_header.encryption_context): - raise ValueError("Encryption context mismatch") - return ciphertext - - def mask(self, value): - return super().mask(value) \ No newline at end of file diff --git a/aws_lambda_powertools/utilities/data_masking/provider.py b/aws_lambda_powertools/utilities/data_masking/provider.py new file mode 100644 index 00000000000..8e53c9686ff --- /dev/null +++ b/aws_lambda_powertools/utilities/data_masking/provider.py @@ -0,0 +1,26 @@ +from abc import abstractmethod +from collections.abc import Iterable + +from aws_lambda_powertools.shared.constants import DATA_MASKING_STRING as MASK + + +class Provider: + """ + When you try to create an instance of a subclass that does not implement the encrypt method, + you will get a NotImplementedError with a message that says the method is not implemented: + """ + + @abstractmethod + def encrypt(self, data): + raise NotImplementedError("Subclasses must implement encrypt()") + + @abstractmethod + def decrypt(self, data): + raise NotImplementedError("Subclasses must implement decrypt()") + + def mask(self, data): + if isinstance(data, (str, dict, bytes)): + return MASK + elif isinstance(data, Iterable): + return type(data)([MASK] * len(data)) + return MASK diff --git a/aws_lambda_powertools/utilities/data_masking/providers/__init__.py b/aws_lambda_powertools/utilities/data_masking/providers/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py new file mode 100644 index 00000000000..3a74637eb54 --- /dev/null +++ b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py @@ -0,0 +1,64 @@ +import base64 +from typing import Any, Optional, Union + +import botocore +from aws_encryption_sdk import ( + CachingCryptoMaterialsManager, + EncryptionSDKClient, + LocalCryptoMaterialsCache, + StrictAwsKmsMasterKeyProvider, +) + +from aws_lambda_powertools.utilities.data_masking.provider import Provider + + +class SingletonMeta(type): + """Metaclass to cache class instances to optimize encryption""" + + _instances: dict["EncryptionManager", Any] = {} + + def __call__(cls, *args, **kwargs): + if cls not in cls._instances: + instance = super().__call__(*args, **kwargs) + cls._instances[cls] = instance + return cls._instances[cls] + + +class AwsEncryptionSdkProvider(Provider, metaclass=SingletonMeta): + CACHE_CAPACITY: int = 100 + MAX_ENTRY_AGE_SECONDS: float = 300.0 + MAX_MESSAGES: int = 200 + # NOTE: You can also set max messages/bytes per data key + + cache = LocalCryptoMaterialsCache(CACHE_CAPACITY) + session = botocore.session.Session() + + def __init__(self, keys: list[str], client: Optional[EncryptionSDKClient()] = None) -> None: + self.client = client or EncryptionSDKClient() + self.keys = keys + self.key_provider = StrictAwsKmsMasterKeyProvider(key_ids=self.keys, botocore_session=self.session) + self.cache_cmm = CachingCryptoMaterialsManager( + master_key_provider=self.key_provider, + cache=self.cache, + max_age=self.MAX_ENTRY_AGE_SECONDS, + max_messages_encrypted=self.MAX_MESSAGES, + ) + self.encryption_context = None + + def encrypt(self, data: Union[bytes, str], context: Optional[dict] = None, **kwargs) -> str: + ciphertext, header = self.client.encrypt( + source=data, encryption_context=context, materials_manager=self.cache_cmm, **kwargs + ) + ciphertext = base64.b64encode(ciphertext).decode() + self.encryption_context = header.encryption_context + return ciphertext + + def decrypt(self, data: str, **kwargs) -> bytes: + ciphertext_decoded = base64.b64decode(data) + ciphertext, decrypted_header = self.client.decrypt( + source=ciphertext_decoded, key_provider=self.key_provider, **kwargs + ) + + if self.encryption_context != decrypted_header.encryption_context: + raise ValueError("Encryption context mismatch") + return ciphertext diff --git a/aws_lambda_powertools/utilities/data_masking/providers/itsdangerous.py b/aws_lambda_powertools/utilities/data_masking/providers/itsdangerous.py new file mode 100644 index 00000000000..2d7da619ce2 --- /dev/null +++ b/aws_lambda_powertools/utilities/data_masking/providers/itsdangerous.py @@ -0,0 +1,23 @@ +from itsdangerous.url_safe import URLSafeSerializer + +from aws_lambda_powertools.utilities.data_masking.provider import Provider + + +class ItsDangerousProvider(Provider): + def __init__(self, keys, salt=None): + self.keys = keys + self.salt = salt + + def encrypt(self, data, **kwargs) -> str: + if data is None: + return data + + serialized = URLSafeSerializer(self.keys, salt=self.salt, **kwargs) + return serialized.dumps(data) + + def decrypt(self, data, **kwargs) -> str: + if data is None: + return data + + serialized = URLSafeSerializer(self.keys, salt=self.salt, **kwargs) + return serialized.loads(data) diff --git a/poetry.lock b/poetry.lock index cba1912511a..eeeb3843500 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. +# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. [[package]] name = "anyio" @@ -26,7 +26,7 @@ trio = ["trio (>=0.16,<0.22)"] name = "attrs" version = "22.2.0" description = "Classes Without Boilerplate" -category = "dev" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -172,6 +172,24 @@ jsii = ">=1.78.1,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" +[[package]] +name = "aws-encryption-sdk" +version = "3.1.1" +description = "AWS Encryption SDK implementation for Python" +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "aws-encryption-sdk-3.1.1.tar.gz", hash = "sha256:8d5fbf018fc68d6b1cacbe4dd037fd805296c7736a9fe457eb684d053f7f9563"}, + {file = "aws_encryption_sdk-3.1.1-py2.py3-none-any.whl", hash = "sha256:a3cbbf04e0b9038b9180af8b03da896af19083e00ca011dcfcb403421458ad02"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +boto3 = ">=1.10.0" +cryptography = ">=2.5.0" +wrapt = ">=1.10.11" + [[package]] name = "aws-requests-auth" version = "0.4.3" @@ -368,6 +386,83 @@ files = [ {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, ] +[[package]] +name = "cffi" +version = "1.15.1" +description = "Foreign Function Interface for Python calling C code." +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914"}, + {file = "cffi-1.15.1-cp27-cp27m-win32.whl", hash = "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3"}, + {file = "cffi-1.15.1-cp27-cp27m-win_amd64.whl", hash = "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e"}, + {file = "cffi-1.15.1-cp310-cp310-win32.whl", hash = "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2"}, + {file = "cffi-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8"}, + {file = "cffi-1.15.1-cp311-cp311-win32.whl", hash = "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d"}, + {file = "cffi-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104"}, + {file = "cffi-1.15.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e"}, + {file = "cffi-1.15.1-cp36-cp36m-win32.whl", hash = "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf"}, + {file = "cffi-1.15.1-cp36-cp36m-win_amd64.whl", hash = "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497"}, + {file = "cffi-1.15.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426"}, + {file = "cffi-1.15.1-cp37-cp37m-win32.whl", hash = "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9"}, + {file = "cffi-1.15.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045"}, + {file = "cffi-1.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192"}, + {file = "cffi-1.15.1-cp38-cp38-win32.whl", hash = "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314"}, + {file = "cffi-1.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3"}, + {file = "cffi-1.15.1-cp39-cp39-win32.whl", hash = "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee"}, + {file = "cffi-1.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c"}, + {file = "cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9"}, +] + +[package.dependencies] +pycparser = "*" + [[package]] name = "cfn-lint" version = "0.77.4" @@ -601,6 +696,48 @@ tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.1 [package.extras] toml = ["tomli"] +[[package]] +name = "cryptography" +version = "40.0.2" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +category = "main" +optional = false +python-versions = ">=3.6" +files = [ + {file = "cryptography-40.0.2-cp36-abi3-macosx_10_12_universal2.whl", hash = "sha256:8f79b5ff5ad9d3218afb1e7e20ea74da5f76943ee5edb7f76e56ec5161ec782b"}, + {file = "cryptography-40.0.2-cp36-abi3-macosx_10_12_x86_64.whl", hash = "sha256:05dc219433b14046c476f6f09d7636b92a1c3e5808b9a6536adf4932b3b2c440"}, + {file = "cryptography-40.0.2-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4df2af28d7bedc84fe45bd49bc35d710aede676e2a4cb7fc6d103a2adc8afe4d"}, + {file = "cryptography-40.0.2-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0dcca15d3a19a66e63662dc8d30f8036b07be851a8680eda92d079868f106288"}, + {file = "cryptography-40.0.2-cp36-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a04386fb7bc85fab9cd51b6308633a3c271e3d0d3eae917eebab2fac6219b6d2"}, + {file = "cryptography-40.0.2-cp36-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:adc0d980fd2760c9e5de537c28935cc32b9353baaf28e0814df417619c6c8c3b"}, + {file = "cryptography-40.0.2-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:d5a1bd0e9e2031465761dfa920c16b0065ad77321d8a8c1f5ee331021fda65e9"}, + {file = "cryptography-40.0.2-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:a95f4802d49faa6a674242e25bfeea6fc2acd915b5e5e29ac90a32b1139cae1c"}, + {file = "cryptography-40.0.2-cp36-abi3-win32.whl", hash = "sha256:aecbb1592b0188e030cb01f82d12556cf72e218280f621deed7d806afd2113f9"}, + {file = "cryptography-40.0.2-cp36-abi3-win_amd64.whl", hash = "sha256:b12794f01d4cacfbd3177b9042198f3af1c856eedd0a98f10f141385c809a14b"}, + {file = "cryptography-40.0.2-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:142bae539ef28a1c76794cca7f49729e7c54423f615cfd9b0b1fa90ebe53244b"}, + {file = "cryptography-40.0.2-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:956ba8701b4ffe91ba59665ed170a2ebbdc6fc0e40de5f6059195d9f2b33ca0e"}, + {file = "cryptography-40.0.2-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4f01c9863da784558165f5d4d916093737a75203a5c5286fde60e503e4276c7a"}, + {file = "cryptography-40.0.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:3daf9b114213f8ba460b829a02896789751626a2a4e7a43a28ee77c04b5e4958"}, + {file = "cryptography-40.0.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:48f388d0d153350f378c7f7b41497a54ff1513c816bcbbcafe5b829e59b9ce5b"}, + {file = "cryptography-40.0.2-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c0764e72b36a3dc065c155e5b22f93df465da9c39af65516fe04ed3c68c92636"}, + {file = "cryptography-40.0.2-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:cbaba590180cba88cb99a5f76f90808a624f18b169b90a4abb40c1fd8c19420e"}, + {file = "cryptography-40.0.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7a38250f433cd41df7fcb763caa3ee9362777fdb4dc642b9a349721d2bf47404"}, + {file = "cryptography-40.0.2.tar.gz", hash = "sha256:c33c0d32b8594fa647d2e01dbccc303478e16fdd7cf98652d5b3ed11aa5e5c99"}, +] + +[package.dependencies] +cffi = ">=1.12" + +[package.extras] +docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] +docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] +pep8test = ["black", "check-manifest", "mypy", "ruff"] +sdist = ["setuptools-rust (>=0.11.4)"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["iso8601", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-shard (>=0.1.2)", "pytest-subtests", "pytest-xdist"] +test-randomorder = ["pytest-randomly"] +tox = ["tox"] + [[package]] name = "decorator" version = "5.1.1" @@ -1159,6 +1296,18 @@ pipfile-deprecated-finder = ["pip-shims (>=0.5.2)", "pipreqs", "requirementslib" plugins = ["setuptools"] requirements-deprecated-finder = ["pip-api", "pipreqs"] +[[package]] +name = "itsdangerous" +version = "2.1.2" +description = "Safely pass data to untrusted environments and back." +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "itsdangerous-2.1.2-py3-none-any.whl", hash = "sha256:2c2349112351b88699d8d4b6b075022c0808887cb7ad10069318a8b0bc88db44"}, + {file = "itsdangerous-2.1.2.tar.gz", hash = "sha256:5dbbc68b317e5e42f327f9021763545dc3fc3bfe22e6deb96aaf1fc38874156a"}, +] + [[package]] name = "jinja2" version = "3.1.2" @@ -2031,6 +2180,18 @@ files = [ {file = "pycodestyle-2.9.1.tar.gz", hash = "sha256:2c9607871d58c76354b697b42f5d57e1ada7d261c261efac224b664affdc5785"}, ] +[[package]] +name = "pycparser" +version = "2.21" +description = "C parser in Python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, + {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, +] + [[package]] name = "pydantic" version = "1.10.7" @@ -2912,7 +3073,7 @@ name = "wrapt" version = "1.15.0" description = "Module for decorators, wrappers and monkey patching." category = "main" -optional = true +optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" files = [ {file = "wrapt-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ca1cccf838cd28d5a0883b342474c630ac48cac5df0ee6eacc9c7290f76b11c1"}, @@ -3026,7 +3187,7 @@ docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] [extras] -all = ["pydantic", "aws-xray-sdk", "fastjsonschema"] +all = ["aws-xray-sdk", "fastjsonschema", "pydantic"] aws-sdk = ["boto3"] parser = ["pydantic"] tracer = ["aws-xray-sdk"] @@ -3035,4 +3196,4 @@ validation = ["fastjsonschema"] [metadata] lock-version = "2.0" python-versions = "^3.7.4" -content-hash = "1b65a257f53a8c00d851795c7f6a984ab07676494ebe26777db71525eb03c436" +content-hash = "60ac9b8d6ba1c142121745ca9431e2190a0462a0e86586ae2d091891b4931ef2" diff --git a/pyproject.toml b/pyproject.toml index 4540fc83ec7..3b7b6a935a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,8 @@ fastjsonschema = { version = "^2.14.5", optional = true } pydantic = { version = "^1.8.2", optional = true } boto3 = { version = "^1.20.32", optional = true } typing-extensions = "^4.4.0" +itsdangerous = "^2.1.2" +aws-encryption-sdk = "^3.1.1" [tool.poetry.dev-dependencies] coverage = {extras = ["toml"], version = "^7.2"} From b34a1ca481434513c2b147381bac88f2537364c7 Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Wed, 17 May 2023 20:18:26 +0000 Subject: [PATCH 03/56] Fix linting errors --- .../utilities/data_masking/providers/aws_encryption_sdk.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py index 3a74637eb54..3efc176040d 100644 --- a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py +++ b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py @@ -2,6 +2,7 @@ from typing import Any, Optional, Union import botocore +from aws_lambda_powertools.utilities.data_masking.provider import Provider from aws_encryption_sdk import ( CachingCryptoMaterialsManager, EncryptionSDKClient, @@ -9,9 +10,6 @@ StrictAwsKmsMasterKeyProvider, ) -from aws_lambda_powertools.utilities.data_masking.provider import Provider - - class SingletonMeta(type): """Metaclass to cache class instances to optimize encryption""" From c6ec14911d386afc589577f256b70c8dd01335fa Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Thu, 18 May 2023 21:39:20 +0000 Subject: [PATCH 04/56] Lint tests --- .../providers/aws_encryption_sdk.py | 8 +- .../data_masking/providers/itsdangerous.py | 4 +- tests/unit/data_masking/test_data_masking.py | 320 ++++++++++-------- 3 files changed, 182 insertions(+), 150 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py index 3efc176040d..e9e965e363f 100644 --- a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py +++ b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py @@ -2,7 +2,6 @@ from typing import Any, Optional, Union import botocore -from aws_lambda_powertools.utilities.data_masking.provider import Provider from aws_encryption_sdk import ( CachingCryptoMaterialsManager, EncryptionSDKClient, @@ -10,10 +9,13 @@ StrictAwsKmsMasterKeyProvider, ) +from aws_lambda_powertools.utilities.data_masking.provider import Provider + + class SingletonMeta(type): """Metaclass to cache class instances to optimize encryption""" - _instances: dict["EncryptionManager", Any] = {} + _instances: dict["AwsEncryptionSdkProvider", Any] = {} def __call__(cls, *args, **kwargs): if cls not in cls._instances: @@ -31,7 +33,7 @@ class AwsEncryptionSdkProvider(Provider, metaclass=SingletonMeta): cache = LocalCryptoMaterialsCache(CACHE_CAPACITY) session = botocore.session.Session() - def __init__(self, keys: list[str], client: Optional[EncryptionSDKClient()] = None) -> None: + def __init__(self, keys: list[str], client: Optional[EncryptionSDKClient] = None) -> None: self.client = client or EncryptionSDKClient() self.keys = keys self.key_provider = StrictAwsKmsMasterKeyProvider(key_ids=self.keys, botocore_session=self.session) diff --git a/aws_lambda_powertools/utilities/data_masking/providers/itsdangerous.py b/aws_lambda_powertools/utilities/data_masking/providers/itsdangerous.py index 2d7da619ce2..afc2c36618a 100644 --- a/aws_lambda_powertools/utilities/data_masking/providers/itsdangerous.py +++ b/aws_lambda_powertools/utilities/data_masking/providers/itsdangerous.py @@ -8,14 +8,14 @@ def __init__(self, keys, salt=None): self.keys = keys self.salt = salt - def encrypt(self, data, **kwargs) -> str: + def encrypt(self, data, **kwargs): if data is None: return data serialized = URLSafeSerializer(self.keys, salt=self.salt, **kwargs) return serialized.dumps(data) - def decrypt(self, data, **kwargs) -> str: + def decrypt(self, data, **kwargs): if data is None: return data diff --git a/tests/unit/data_masking/test_data_masking.py b/tests/unit/data_masking/test_data_masking.py index 3bfc4c796d1..7ff0c096a86 100644 --- a/tests/unit/data_masking/test_data_masking.py +++ b/tests/unit/data_masking/test_data_masking.py @@ -1,267 +1,297 @@ +import json import unittest -from aws_lambda_powertools.utilities.data_masking import DataMasking, ItsDangerousProvider, AwsEncryptionSdkProvider, Provider -from aws_lambda_powertools.shared import DATA_MASKING_STRING as MASK + from itsdangerous.url_safe import URLSafeSerializer -import json +from aws_lambda_powertools.shared.constants import DATA_MASKING_STRING as MASK +from aws_lambda_powertools.utilities.data_masking.base import DataMasking +from aws_lambda_powertools.utilities.data_masking.provider import Provider +from aws_lambda_powertools.utilities.data_masking.providers.aws_encryption_sdk import ( + AwsEncryptionSdkProvider, +) +from aws_lambda_powertools.utilities.data_masking.providers.itsdangerous import ( + ItsDangerousProvider, +) AWS_SDK_KEY = "arn:aws:kms:us-west-2:683517028648:key/269301eb-81eb-4067-ac72-98e8e49bf2b3" + class MyEncryptionProvider(Provider): + """Custom encryption provider class""" + def __init__(self, keys, salt=None): self.keys = keys self.salt = salt - def mask(self, value): - return super().mask(value) - - def encrypt(self, value: str) -> str: - if value is None: - return value - s = URLSafeSerializer(self.keys) - return s.dumps(value) - - def decrypt(self, value: str) -> str: - if value is None: - return value - s = URLSafeSerializer(self.keys) - return s.loads(value) - + def encrypt(self, data: str) -> str: + if data is None: + return data + serialize = URLSafeSerializer(self.keys) + return serialize.dumps(data) + + def decrypt(self, data: str) -> str: + if data is None: + return data + serialize = URLSafeSerializer(self.keys) + return serialize.loads(data) + class TestDataMasking(unittest.TestCase): + """Tests for sensitive data masking utility""" + def __init__(self): super().__init__() self.python_dict = { - 'a': { - '1': { - 'None': 'hello', # None type key doesn't work - 'four': 'world' - }, - 'b': { - '3': { - '4': 'goodbye', # key "4.5" doesn't work - 'e': 'world' - } - } + "a": { + "1": {"None": "hello", "four": "world"}, # None type key doesn't work + "b": {"3": {"4": "goodbye", "e": "world"}}, # key "4.5" doesn't work } } self.json_dict = json.dumps(self.python_dict) self.fields = ["a.1.None", "a.b.3.4"] - self.masked_with_fields = {'a': {'1': {'None': '*****', 'four': 'world'}, 'b': {'3': {'4': '*****', 'e': 'world'}}}} + self.masked_with_fields = { + "a": {"1": {"None": "*****", "four": "world"}, "b": {"3": {"4": "*****", "e": "world"}}} + } - self.list_of_data_types = [42, 4.22, True, [1, 2, 3, 4], ["hello", 1, 2, 3, "world"], tuple((55, 66, 88)), None, "this is a string"] - self.list_of_data_types_masked = ["*****", "*****", "*****", ['*****', '*****', '*****', '*****'], ['*****', '*****', '*****', '*****', '*****'], ('*****', '*****', '*****'), "*****", "*****"] + self.list_of_data_types = [ + 42, + 4.22, + True, + [1, 2, 3, 4], + ["hello", 1, 2, 3, "world"], + (55, 66, 88), + None, + "this is a string", + ] + self.list_of_data_types_masked = [ + "*****", + "*****", + "*****", + ["*****", "*****", "*****", "*****"], + ["*****", "*****", "*****", "*****", "*****"], + ("*****", "*****", "*****"), + "*****", + "*****", + ] # 10kb JSON blob for latency testing - self.bigJSON = { + self.json_blob = { "id": 1, "name": "John Doe", "age": 30, "email": "johndoe@example.com", - "address": { - "street": "123 Main St", - "city": "Anytown", - "state": "CA", - "zip": "12345" + "address": {"street": "123 Main St", "city": "Anytown", "state": "CA", "zip": "12345"}, + "phone_numbers": ["+1-555-555-1234", "+1-555-555-5678"], + "interests": ["Hiking", "Traveling", "Photography", "Reading"], + "job_history": { + "company": "Acme Inc.", + "position": "Software Engineer", + "start_date": "2015-01-01", + "end_date": "2017-12-31", }, - "phone_numbers": [ - "+1-555-555-1234", - "+1-555-555-5678" - ], - "interests": [ - "Hiking", - "Traveling", - "Photography", - "Reading" - ], - "job_history": - { - "company": "Acme Inc.", - "position": "Software Engineer", - "start_date": "2015-01-01", - "end_date": "2017-12-31" - }, - "about_me": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla tincidunt velit quis sapien mollis, at egestas massa tincidunt. Suspendisse ultrices arcu a dolor dapibus, ut pretium turpis volutpat. Vestibulum at sapien quis sapien dignissim volutpat ut a enim. Praesent fringilla sem eu dui convallis luctus. Donec ullamcorper, sapien ut convallis congue, risus mauris pretium tortor, nec dignissim arcu urna a nisl. Vivamus non fermentum ex. Proin interdum nisi id sagittis egestas. Nam sit amet nisi nec quam pharetra sagittis. Aliquam erat volutpat. Donec nec luctus sem, nec ornare lorem. Vivamus vitae orci quis enim faucibus placerat. Nulla facilisi. Proin in turpis orci. Donec imperdiet velit ac tellus gravida, eget laoreet tellus malesuada. Praesent venenatis tellus ac urna blandit, at varius felis posuere. Integer a commodo nunc." + "about_me": """ + Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla tincidunt velit quis + sapien mollis, at egestas massa tincidunt. Suspendisse ultrices arcu a dolor dapibus, + ut pretium turpis volutpat. Vestibulum at sapien quis sapien dignissim volutpat ut a enim. + Praesent fringilla sem eu dui convallis luctus. Donec ullamcorper, sapien ut convallis congue, + risus mauris pretium tortor, nec dignissim arcu urna a nisl. Vivamus non fermentum ex. Proin + interdum nisi id sagittis egestas. Nam sit amet nisi nec quam pharetra sagittis. Aliquam erat + volutpat. Donec nec luctus sem, nec ornare lorem. Vivamus vitae orci quis enim faucibus placerat. + Nulla facilisi. Proin in turpis orci. Donec imperdiet velit ac tellus gravida, eget laoreet tellus + malesuada. Praesent venenatis tellus ac urna blandit, at varius felis posuere. Integer a commodo nunc. + """, } - self.bigJSONfields = ["address.street", "job_history.company"] - - - def general_mask_test(self, dm): + self.json_blob_fields = ["address.street", "job_history.company"] + self.encrypted_list = [] + self.encrypted_data_python_dict = None + self.encrypted_data_json_dict = None + self.encrypted_data_python_dict_fields = None + self.encrypted_data_json_dict_fields = None + self.encrypted_json_blob = None + + def general_mask_test(self, data_masker): + """Method to mask several different data types fully, + and specific values in nested dicts""" # mask different data types fully for i, data_type in enumerate(self.list_of_data_types): - masked_string = dm.mask(data_type) + masked_string = data_masker.mask(data_type) self.assertEqual(masked_string, self.list_of_data_types_masked[i]) - + # mask dict fully - masked_string = dm.mask(self.python_dict) + masked_string = data_masker.mask(self.python_dict) self.assertEqual(masked_string, MASK) - masked_string = dm.mask(self.json_dict) + masked_string = data_masker.mask(self.json_dict) self.assertEqual(masked_string, MASK) # mask dict with fields - masked_string = dm.mask(self.python_dict, self.fields) + masked_string = data_masker.mask(self.python_dict, self.fields) self.assertEqual(masked_string, self.masked_with_fields) - masked_string = dm.mask(self.json_dict, self.fields) + masked_string = data_masker.mask(self.json_dict, self.fields) self.assertEqual(masked_string, self.masked_with_fields) - - def general_encrypt_test(self, dm): + def general_encrypt_test(self, data_masker): + """Method to encrypt several different data types fully, + and specific values in nested dicts""" # encrypt different data types fully self.encrypted_list = [] - for i, data_type in enumerate(self.list_of_data_types): - encrypted_data = dm.encrypt(data_type) + for data_type in self.list_of_data_types: + encrypted_data = data_masker.encrypt(data_type) self.encrypted_list.append(encrypted_data) # encrypt dict fully - self.encrypted_data_python_dict = dm.encrypt(self.python_dict) - self.encrypted_data_json_dict = dm.encrypt(self.json_dict) + self.encrypted_data_python_dict = data_masker.encrypt(self.python_dict) + self.encrypted_data_json_dict = data_masker.encrypt(self.json_dict) # encrypt dict with fields - self.encrypted_data_python_dict_fields = dm.encrypt(self.python_dict, self.fields) - self.encrypted_data_json_dict_fields = dm.encrypt(self.json_dict, self.fields) + self.encrypted_data_python_dict_fields = data_masker.encrypt(self.python_dict, self.fields) + self.encrypted_data_json_dict_fields = data_masker.encrypt(self.json_dict, self.fields) - self.encrypted_big_json = dm.encrypt(self.bigJSON, self.bigJSONfields) + self.encrypted_json_blob = data_masker.encrypt(self.json_blob, self.json_blob_fields) - def general_decrypt_test(self, dm): + def general_decrypt_test(self, data_masker): + """Method to decrypt several different data types fully, + and specific values in nested dicts""" # decrypt different data types fully - for i, data_type in enumerate(self.list_of_data_types): - decrypted_data = dm.decrypt(self.encrypted_list[i]) - # AssertionError: [55, 66, 88] != (55, 66, 88) ie itsdangerous encrypts&decrypts tuples into type lists + for i, encrypted_data in enumerate(self.encrypted_list): + decrypted_data = data_masker.decrypt(encrypted_data) + # ie itsdangerous encrypts & decrypts tuples into type lists if decrypted_data == [55, 66, 88]: continue self.assertEqual(decrypted_data, self.list_of_data_types[i]) # decrypt dict fully - decrypted_data_python_dict = dm.decrypt(self.encrypted_data_python_dict) + decrypted_data_python_dict = data_masker.decrypt(self.encrypted_data_python_dict) self.assertEqual(decrypted_data_python_dict, self.python_dict) - decrypted_data_json_dict = dm.decrypt(self.encrypted_data_json_dict) + decrypted_data_json_dict = data_masker.decrypt(self.encrypted_data_json_dict) self.assertEqual(decrypted_data_json_dict, self.json_dict) # decrypt dict with fields - decrypted_data_python_dict_fields = dm.decrypt(self.encrypted_data_python_dict_fields, self.fields) + decrypted_data_python_dict_fields = data_masker.decrypt(self.encrypted_data_python_dict_fields, self.fields) self.assertEqual(decrypted_data_python_dict_fields, self.python_dict) - decrypted_data_json_dict_fields = dm.decrypt(self.encrypted_data_json_dict_fields, self.fields) + decrypted_data_json_dict_fields = data_masker.decrypt(self.encrypted_data_json_dict_fields, self.fields) self.assertEqual(decrypted_data_json_dict_fields, json.loads(self.json_dict)) - decrypted_big_json = dm.decrypt(self.encrypted_big_json, self.bigJSONfields) - self.assertEqual(decrypted_big_json, self.bigJSON) + decrypted_json_blob = data_masker.decrypt(self.encrypted_json_blob, self.json_blob_fields) + self.assertEqual(decrypted_json_blob, self.json_blob) - - # TestNoProvder def test_mask(self): - dm = DataMasking() - self.general_mask_test(dm) + """Test masking with no Provider""" + data_masker = DataMasking() + self.general_mask_test(data_masker) def test_encrypt_not_implemented(self): - dm = DataMasking() + """Test encrypting with no Provider""" + data_masker = DataMasking() with self.assertRaises(NotImplementedError): - dm.encrypt("hello world") + data_masker.encrypt("hello world") def test_decrypt_not_implemented(self): - dm = DataMasking() + """Test decrypting with no Provider""" + data_masker = DataMasking() with self.assertRaises(NotImplementedError): - dm.decrypt("hello world") - + data_masker.decrypt("hello world") - # TestItsDangerousProvider def test_itsdangerous_mask(self): + """Test masking with ItsDangerous provider""" itsdangerous_provider = ItsDangerousProvider("mykey") - dm = DataMasking(provider=itsdangerous_provider) - self.general_mask_test(dm) - + data_masker = DataMasking(provider=itsdangerous_provider) + self.general_mask_test(data_masker) def test_itsdangerous_encrypt(self): + """Test encrypting with ItsDangerous provider""" itsdangerous_provider = ItsDangerousProvider("mykey") - dm = DataMasking(provider=itsdangerous_provider) - - # NOTE: TypeError: type SET and BYTES is not JSON serializable - itsdangerous for sets and bytes doesn't work - self.general_encrypt_test(dm) + data_masker = DataMasking(provider=itsdangerous_provider) + # TypeError: type SET and BYTES is not JSON serializable, ie + # itsdangerous for sets and bytes doesn't work + self.general_encrypt_test(data_masker) def test_itsdangerous_decrypt(self): + """Test decrypting with ItsDangerous provider""" itsdangerous_provider = ItsDangerousProvider("mykey") - dm = DataMasking(provider=itsdangerous_provider) - - self.general_decrypt_test(dm) + data_masker = DataMasking(provider=itsdangerous_provider) + self.general_decrypt_test(data_masker) # TestCustomEncryptionSdkProvider def test_custom_mask(self): + """Test masking with a custom encryption provider""" my_encryption = MyEncryptionProvider(keys="secret-key") - dm = DataMasking(provider=my_encryption) - self.general_mask_test(dm) - + data_masker = DataMasking(provider=my_encryption) + self.general_mask_test(data_masker) def test_custom_encrypt(self): + """Test encrypting with a custom encryption provider""" my_encryption = MyEncryptionProvider(keys="secret-key") - dm = DataMasking(provider=my_encryption) - - # NOTE: TypeError: type SET and BYTES is not JSON serializable - itsdangerous for sets and bytes doesn't work - self.general_encrypt_test(dm) + data_masker = DataMasking(provider=my_encryption) + # TypeError: type SET and BYTES is not JSON serializable ie. + # itsdangerous for sets and bytes doesn't work + self.general_encrypt_test(data_masker) def test_custom_decrypt(self): + """Test decrypting with a custom encryption provider""" my_encryption = MyEncryptionProvider(keys="secret-key") - dm = DataMasking(provider=my_encryption) - - self.general_decrypt_test(dm) - + data_masker = DataMasking(provider=my_encryption) + self.general_decrypt_test(data_masker) # TestAwsEncryptionSdkProvider def test_awssdk_mask(self): + """Test masking with AwsEncryptionSdk provider""" masking_provider = AwsEncryptionSdkProvider(keys=[AWS_SDK_KEY]) - dm = DataMasking(provider=masking_provider) - self.general_mask_test(dm) - + data_masker = DataMasking(provider=masking_provider) + self.general_mask_test(data_masker) def test_awssdk_encrypt(self): + """Test encrypting with AwsEncryptionSdk provider""" masking_provider = AwsEncryptionSdkProvider(keys=[AWS_SDK_KEY]) - dm = DataMasking(provider=masking_provider) + data_masker = DataMasking(provider=masking_provider) - # TODO: AWS SDK encrypt method only takes in str | bytes, and returns bytes - # May have to make a new list for this since some data types need to be utf-8 encoded before being turned to bytes. + # AWS SDK encrypt method only takes in str | bytes, and returns bytes + # May have to make a new list for this as some data types need to be utf-8 encoded + # before being turned to bytes. # https://aws-encryption-sdk-python.readthedocs.io/en/latest/generated/aws_encryption_sdk.html self.encrypted_list = [] - for i, data_type in enumerate(self.list_of_data_types): + for data_type in self.list_of_data_types: if isinstance(data_type, str) or isinstance(data_type, bytes): - encrypted_data = dm.encrypt(data_type) + encrypted_data = data_masker.encrypt(data_type) self.encrypted_list.append(encrypted_data) - + # encrypt dict fully - self.encrypted_data_json_dict = dm.encrypt(self.json_dict) + self.encrypted_data_json_dict = data_masker.encrypt(self.json_dict) # encrypt dict with fields - self.encrypted_data_python_dict_fields = dm.encrypt(self.python_dict, self.fields) - self.encrypted_data_json_dict_fields = dm.encrypt(self.json_dict, self.fields) - - self.encrypted_big_json = dm.encrypt(json.dumps(self.bigJSON), self.bigJSONfields) + self.encrypted_data_python_dict_fields = data_masker.encrypt(self.python_dict, self.fields) + self.encrypted_data_json_dict_fields = data_masker.encrypt(self.json_dict, self.fields) + self.encrypted_json_blob = data_masker.encrypt(json.dumps(self.json_blob), self.json_blob_fields) def test_awssdk_decrypt(self): + """Test decrypting with AwsEncryptionSdk provider""" masking_provider = AwsEncryptionSdkProvider(keys=[AWS_SDK_KEY]) - dm = DataMasking(provider=masking_provider) + data_masker = DataMasking(provider=masking_provider) - - # TODO: AWS SDK decrypt method returns only bytes - # May have to make a new list for this since some data types need to be utf-8 encoded before being turned to bytes. + # AWS SDK decrypt method returns only bytes + # May have to make a new list for this as some data types need to be utf-8 + # encoded before being turned to bytes - # for i, data_type in enumerate(self.list_of_data_types): - # decrypted_data = dm.decrypt(self.encrypted_list[i]) - # self.assertEqual(decrypted_data, self.list_of_data_types[i]) + for i, encrypted_data in enumerate(self.encrypted_list): + decrypted_data = data_masker.decrypt(encrypted_data) + self.assertEqual(decrypted_data, self.list_of_data_types[i]) - decrypted_data_json_dict = dm.decrypt(self.encrypted_data_json_dict) - self.assertEqual(decrypted_data_json_dict, bytes(self.json_dict, 'utf-8')) + decrypted_data_json_dict = data_masker.decrypt(self.encrypted_data_json_dict) + self.assertEqual(decrypted_data_json_dict, bytes(self.json_dict, "utf-8")) # AWS SDK encrypt method returning the individual fields decrypted as bytes - decrypted_data_python_dict_fields = dm.decrypt(self.encrypted_data_python_dict_fields, self.fields) - # self.assertEqual(decrypted_data_python_dict_fields, self.python_dict) + decrypted_data_python_dict_fields = data_masker.decrypt(self.encrypted_data_python_dict_fields, self.fields) + self.assertEqual(decrypted_data_python_dict_fields, self.python_dict) # AWS SDK encrypt method returning the individual fields decrypted as bytes - decrypted_data_json_dict_fields = dm.decrypt(self.encrypted_data_json_dict_fields, self.fields) - # self.assertEqual(decrypted_data_json_dict_fields, json.loads(self.json_dict)) + decrypted_data_json_dict_fields = data_masker.decrypt(self.encrypted_data_json_dict_fields, self.fields) + self.assertEqual(decrypted_data_json_dict_fields, json.loads(self.json_dict)) # AWS SDK encrypt method returning the individual fields decrypted as bytes - decrypted_big_json = dm.decrypt(self.encrypted_big_json, self.bigJSONfields) - # self.assertEqual(decrypted_big_json, self.bigJSON) \ No newline at end of file + decrypted_json_blob = data_masker.decrypt(self.encrypted_json_blob, self.json_blob_fields) + self.assertEqual(decrypted_json_blob, self.json_blob) From 21759b52a9962013f38e50ed13eb409a54b81024 Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Fri, 19 May 2023 22:32:07 +0000 Subject: [PATCH 05/56] Fix mypy errors --- .../utilities/data_masking/base.py | 14 ++++++++------ mypy.ini | 6 ++++++ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_masking/base.py b/aws_lambda_powertools/utilities/data_masking/base.py index b90ac6ea671..735003df03a 100644 --- a/aws_lambda_powertools/utilities/data_masking/base.py +++ b/aws_lambda_powertools/utilities/data_masking/base.py @@ -1,6 +1,6 @@ import json from collections.abc import Iterable -from typing import Optional, Union +from typing import Union from aws_lambda_powertools.shared.constants import DATA_MASKING_STRING as MASK from aws_lambda_powertools.utilities.data_masking.provider import Provider @@ -13,18 +13,20 @@ def __init__(self, provider=None): else: self.provider = provider - def encrypt(self, data, *args, fields=None, context: Optional[dict] = None, **kwargs): - return self._apply_action(data, fields, action=self.provider.encrypt, *args, *context, **kwargs) + # def encrypt(self, data, *args, fields=None, context: Optional[dict] = None, **kwargs): + # think was after *args (*context) bc how else to get this param through when itsdangeorus doesn't have it? + def encrypt(self, data, *args, fields=None, **kwargs): + return self._apply_action(data, fields, action=self.provider.encrypt, *args, **kwargs) - def decrypt(self, data, *args, fields=None, context: Optional[dict] = None, **kwargs): - return self._apply_action(data, fields, action=self.provider.decrypt, *args, *context, **kwargs) + def decrypt(self, data, *args, fields=None, **kwargs): + return self._apply_action(data, fields, action=self.provider.decrypt, *args, **kwargs) def mask(self, data, *args, fields=None, **kwargs): return self._apply_action(data, fields, action=self.provider.mask, *args, **kwargs) def _apply_action(self, data, fields, action, *args, **kwargs): if fields is not None: - return self._use_ast(data, fields, action=action, *args, **kwargs) + return self._use_ast(data, fields, action, *args, **kwargs) else: return action(data, *args, **kwargs) diff --git a/mypy.ini b/mypy.ini index 4af89217fdc..41293ce2912 100644 --- a/mypy.ini +++ b/mypy.ini @@ -12,6 +12,12 @@ disable_error_code = annotation-unchecked [mypy-jmespath] ignore_missing_imports=True +[mypy-aws_encryption_sdk] +ignore_missing_imports=True + +[mypy-sentry_sdk] +ignore_missing_imports=True + [mypy-jmespath.exceptions] ignore_missing_imports=True From 6a2e98af47cc615a58645e514d1c79119d75fa44 Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Mon, 22 May 2023 20:33:55 +0000 Subject: [PATCH 06/56] Fixing tests --- tests/unit/data_masking/test_data_masking.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/data_masking/test_data_masking.py b/tests/unit/data_masking/test_data_masking.py index 7ff0c096a86..46f8578a97a 100644 --- a/tests/unit/data_masking/test_data_masking.py +++ b/tests/unit/data_masking/test_data_masking.py @@ -255,7 +255,7 @@ def test_awssdk_encrypt(self): self.encrypted_list = [] for data_type in self.list_of_data_types: - if isinstance(data_type, str) or isinstance(data_type, bytes): + if isinstance(data_type, (str, bytes)): encrypted_data = data_masker.encrypt(data_type) self.encrypted_list.append(encrypted_data) From d39d9563df596d5fae969f2fa2989159167257ba Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Tue, 23 May 2023 21:00:38 +0000 Subject: [PATCH 07/56] mypy fixes --- .../providers/aws_encryption_sdk.py | 6 +-- tests/unit/data_masking/test_data_masking.py | 37 ++++++++++--------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py index e9e965e363f..07f393ec1fb 100644 --- a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py +++ b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py @@ -1,5 +1,5 @@ import base64 -from typing import Any, Optional, Union +from typing import Any, Dict, List, Optional, Union import botocore from aws_encryption_sdk import ( @@ -15,7 +15,7 @@ class SingletonMeta(type): """Metaclass to cache class instances to optimize encryption""" - _instances: dict["AwsEncryptionSdkProvider", Any] = {} + _instances: Dict["AwsEncryptionSdkProvider", Any] = {} def __call__(cls, *args, **kwargs): if cls not in cls._instances: @@ -33,7 +33,7 @@ class AwsEncryptionSdkProvider(Provider, metaclass=SingletonMeta): cache = LocalCryptoMaterialsCache(CACHE_CAPACITY) session = botocore.session.Session() - def __init__(self, keys: list[str], client: Optional[EncryptionSDKClient] = None) -> None: + def __init__(self, keys: List[str], client: Optional[EncryptionSDKClient] = None) -> None: self.client = client or EncryptionSDKClient() self.keys = keys self.key_provider = StrictAwsKmsMasterKeyProvider(key_ids=self.keys, botocore_session=self.session) diff --git a/tests/unit/data_masking/test_data_masking.py b/tests/unit/data_masking/test_data_masking.py index 46f8578a97a..ff42e389b76 100644 --- a/tests/unit/data_masking/test_data_masking.py +++ b/tests/unit/data_masking/test_data_masking.py @@ -1,6 +1,7 @@ import json import unittest +import pytest from itsdangerous.url_safe import URLSafeSerializer from aws_lambda_powertools.shared.constants import DATA_MASKING_STRING as MASK @@ -115,19 +116,19 @@ def general_mask_test(self, data_masker): # mask different data types fully for i, data_type in enumerate(self.list_of_data_types): masked_string = data_masker.mask(data_type) - self.assertEqual(masked_string, self.list_of_data_types_masked[i]) + assert masked_string == self.list_of_data_types_masked[i] # mask dict fully masked_string = data_masker.mask(self.python_dict) - self.assertEqual(masked_string, MASK) + assert masked_string == MASK masked_string = data_masker.mask(self.json_dict) - self.assertEqual(masked_string, MASK) + assert masked_string == MASK # mask dict with fields masked_string = data_masker.mask(self.python_dict, self.fields) - self.assertEqual(masked_string, self.masked_with_fields) + assert masked_string == self.masked_with_fields masked_string = data_masker.mask(self.json_dict, self.fields) - self.assertEqual(masked_string, self.masked_with_fields) + assert masked_string == self.masked_with_fields def general_encrypt_test(self, data_masker): """Method to encrypt several different data types fully, @@ -157,22 +158,22 @@ def general_decrypt_test(self, data_masker): # ie itsdangerous encrypts & decrypts tuples into type lists if decrypted_data == [55, 66, 88]: continue - self.assertEqual(decrypted_data, self.list_of_data_types[i]) + assert decrypted_data == self.list_of_data_types[i] # decrypt dict fully decrypted_data_python_dict = data_masker.decrypt(self.encrypted_data_python_dict) - self.assertEqual(decrypted_data_python_dict, self.python_dict) + assert decrypted_data_python_dict == self.python_dict decrypted_data_json_dict = data_masker.decrypt(self.encrypted_data_json_dict) - self.assertEqual(decrypted_data_json_dict, self.json_dict) + assert decrypted_data_json_dict == self.json_dict # decrypt dict with fields decrypted_data_python_dict_fields = data_masker.decrypt(self.encrypted_data_python_dict_fields, self.fields) - self.assertEqual(decrypted_data_python_dict_fields, self.python_dict) + assert decrypted_data_python_dict_fields == self.python_dict decrypted_data_json_dict_fields = data_masker.decrypt(self.encrypted_data_json_dict_fields, self.fields) - self.assertEqual(decrypted_data_json_dict_fields, json.loads(self.json_dict)) + assert decrypted_data_json_dict_fields == json.loads(self.json_dict) decrypted_json_blob = data_masker.decrypt(self.encrypted_json_blob, self.json_blob_fields) - self.assertEqual(decrypted_json_blob, self.json_blob) + assert decrypted_json_blob == self.json_blob def test_mask(self): """Test masking with no Provider""" @@ -182,13 +183,13 @@ def test_mask(self): def test_encrypt_not_implemented(self): """Test encrypting with no Provider""" data_masker = DataMasking() - with self.assertRaises(NotImplementedError): + with pytest.raises(NotImplementedError): data_masker.encrypt("hello world") def test_decrypt_not_implemented(self): """Test decrypting with no Provider""" data_masker = DataMasking() - with self.assertRaises(NotImplementedError): + with pytest.raises(NotImplementedError): data_masker.decrypt("hello world") def test_itsdangerous_mask(self): @@ -279,19 +280,19 @@ def test_awssdk_decrypt(self): for i, encrypted_data in enumerate(self.encrypted_list): decrypted_data = data_masker.decrypt(encrypted_data) - self.assertEqual(decrypted_data, self.list_of_data_types[i]) + assert decrypted_data == self.list_of_data_types[i] decrypted_data_json_dict = data_masker.decrypt(self.encrypted_data_json_dict) - self.assertEqual(decrypted_data_json_dict, bytes(self.json_dict, "utf-8")) + assert decrypted_data_json_dict == bytes(self.json_dict, "utf-8") # AWS SDK encrypt method returning the individual fields decrypted as bytes decrypted_data_python_dict_fields = data_masker.decrypt(self.encrypted_data_python_dict_fields, self.fields) - self.assertEqual(decrypted_data_python_dict_fields, self.python_dict) + assert decrypted_data_python_dict_fields == self.python_dict # AWS SDK encrypt method returning the individual fields decrypted as bytes decrypted_data_json_dict_fields = data_masker.decrypt(self.encrypted_data_json_dict_fields, self.fields) - self.assertEqual(decrypted_data_json_dict_fields, json.loads(self.json_dict)) + assert decrypted_data_json_dict_fields == json.loads(self.json_dict) # AWS SDK encrypt method returning the individual fields decrypted as bytes decrypted_json_blob = data_masker.decrypt(self.encrypted_json_blob, self.json_blob_fields) - self.assertEqual(decrypted_json_blob, self.json_blob) + assert decrypted_json_blob == self.json_blob From 97c5b858cc58c106f441948de8d8c9ebb9ad0593 Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Wed, 24 May 2023 22:01:15 +0000 Subject: [PATCH 08/56] Fixed passing in context for aws encryption sdk provider --- .../utilities/data_masking/base.py | 15 ++------------- .../data_masking/providers/aws_encryption_sdk.py | 9 +++++---- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_masking/base.py b/aws_lambda_powertools/utilities/data_masking/base.py index 735003df03a..b15bb23ccb0 100644 --- a/aws_lambda_powertools/utilities/data_masking/base.py +++ b/aws_lambda_powertools/utilities/data_masking/base.py @@ -1,8 +1,6 @@ import json -from collections.abc import Iterable from typing import Union -from aws_lambda_powertools.shared.constants import DATA_MASKING_STRING as MASK from aws_lambda_powertools.utilities.data_masking.provider import Provider @@ -13,8 +11,6 @@ def __init__(self, provider=None): else: self.provider = provider - # def encrypt(self, data, *args, fields=None, context: Optional[dict] = None, **kwargs): - # think was after *args (*context) bc how else to get this param through when itsdangeorus doesn't have it? def encrypt(self, data, *args, fields=None, **kwargs): return self._apply_action(data, fields, action=self.provider.encrypt, *args, **kwargs) @@ -30,14 +26,7 @@ def _apply_action(self, data, fields, action, *args, **kwargs): else: return action(data, *args, **kwargs) - def _default_mask(self, data): - if isinstance(data, (str, dict, bytes)): - return MASK - elif isinstance(data, Iterable): - return type(data)([MASK] * len(data)) - return MASK - - def _use_ast(self, data: Union[dict, str], fields, action) -> str: + def _use_ast(self, data: Union[dict, str], fields, action, *args, **kwargs) -> str: if fields is None: raise ValueError("No fields specified.") if isinstance(data, str): @@ -64,6 +53,6 @@ def _use_ast(self, data: Union[dict, str], fields, action) -> str: for key in keys[:-1]: curr_dict = curr_dict[key] valtochange = curr_dict[(keys[-1])] - curr_dict[keys[-1]] = action(valtochange) + curr_dict[keys[-1]] = action(valtochange, *args, **kwargs) return my_dict_parsed diff --git a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py index 07f393ec1fb..a10c5f081e1 100644 --- a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py +++ b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py @@ -45,18 +45,19 @@ def __init__(self, keys: List[str], client: Optional[EncryptionSDKClient] = None ) self.encryption_context = None - def encrypt(self, data: Union[bytes, str], context: Optional[dict] = None, **kwargs) -> str: + def encrypt(self, data: Union[bytes, str], *args, **kwargs) -> str: + context = kwargs["context"] ciphertext, header = self.client.encrypt( - source=data, encryption_context=context, materials_manager=self.cache_cmm, **kwargs + source=data, encryption_context=context, materials_manager=self.cache_cmm, *args, **kwargs ) ciphertext = base64.b64encode(ciphertext).decode() self.encryption_context = header.encryption_context return ciphertext - def decrypt(self, data: str, **kwargs) -> bytes: + def decrypt(self, data: str, *args, **kwargs) -> bytes: ciphertext_decoded = base64.b64decode(data) ciphertext, decrypted_header = self.client.decrypt( - source=ciphertext_decoded, key_provider=self.key_provider, **kwargs + source=ciphertext_decoded, key_provider=self.key_provider, *args, **kwargs ) if self.encryption_context != decrypted_header.encryption_context: From f722e703877adee46cb09bd7baf1f674223f431c Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Fri, 26 May 2023 22:06:37 +0000 Subject: [PATCH 09/56] Use d pytest library for unit testing --- .../providers/aws_encryption_sdk.py | 8 +- tests/unit/data_masking/test_data_masking.py | 378 ++++++------------ 2 files changed, 125 insertions(+), 261 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py index a10c5f081e1..3b59697c431 100644 --- a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py +++ b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py @@ -48,7 +48,13 @@ def __init__(self, keys: List[str], client: Optional[EncryptionSDKClient] = None def encrypt(self, data: Union[bytes, str], *args, **kwargs) -> str: context = kwargs["context"] ciphertext, header = self.client.encrypt( - source=data, encryption_context=context, materials_manager=self.cache_cmm, *args, **kwargs + # Turn all data into string? bc we’re turning everything into a dict + # in order to get the key values even if they pass in a json str of a dict + source=str(data), + encryption_context=context, + materials_manager=self.cache_cmm, + *args, + **kwargs ) ciphertext = base64.b64encode(ciphertext).decode() self.encryption_context = header.encryption_context diff --git a/tests/unit/data_masking/test_data_masking.py b/tests/unit/data_masking/test_data_masking.py index ff42e389b76..83ab19027e0 100644 --- a/tests/unit/data_masking/test_data_masking.py +++ b/tests/unit/data_masking/test_data_masking.py @@ -1,5 +1,4 @@ import json -import unittest import pytest from itsdangerous.url_safe import URLSafeSerializer @@ -37,262 +36,121 @@ def decrypt(self, data: str) -> str: return serialize.loads(data) -class TestDataMasking(unittest.TestCase): - """Tests for sensitive data masking utility""" - - def __init__(self): - super().__init__() - self.python_dict = { - "a": { - "1": {"None": "hello", "four": "world"}, # None type key doesn't work - "b": {"3": {"4": "goodbye", "e": "world"}}, # key "4.5" doesn't work - } - } - self.json_dict = json.dumps(self.python_dict) - self.fields = ["a.1.None", "a.b.3.4"] - self.masked_with_fields = { - "a": {"1": {"None": "*****", "four": "world"}, "b": {"3": {"4": "*****", "e": "world"}}} - } - - self.list_of_data_types = [ - 42, - 4.22, - True, - [1, 2, 3, 4], - ["hello", 1, 2, 3, "world"], - (55, 66, 88), - None, - "this is a string", - ] - self.list_of_data_types_masked = [ - "*****", - "*****", - "*****", - ["*****", "*****", "*****", "*****"], - ["*****", "*****", "*****", "*****", "*****"], - ("*****", "*****", "*****"), - "*****", - "*****", - ] - - # 10kb JSON blob for latency testing - self.json_blob = { - "id": 1, - "name": "John Doe", - "age": 30, - "email": "johndoe@example.com", - "address": {"street": "123 Main St", "city": "Anytown", "state": "CA", "zip": "12345"}, - "phone_numbers": ["+1-555-555-1234", "+1-555-555-5678"], - "interests": ["Hiking", "Traveling", "Photography", "Reading"], - "job_history": { - "company": "Acme Inc.", - "position": "Software Engineer", - "start_date": "2015-01-01", - "end_date": "2017-12-31", - }, - "about_me": """ - Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla tincidunt velit quis - sapien mollis, at egestas massa tincidunt. Suspendisse ultrices arcu a dolor dapibus, - ut pretium turpis volutpat. Vestibulum at sapien quis sapien dignissim volutpat ut a enim. - Praesent fringilla sem eu dui convallis luctus. Donec ullamcorper, sapien ut convallis congue, - risus mauris pretium tortor, nec dignissim arcu urna a nisl. Vivamus non fermentum ex. Proin - interdum nisi id sagittis egestas. Nam sit amet nisi nec quam pharetra sagittis. Aliquam erat - volutpat. Donec nec luctus sem, nec ornare lorem. Vivamus vitae orci quis enim faucibus placerat. - Nulla facilisi. Proin in turpis orci. Donec imperdiet velit ac tellus gravida, eget laoreet tellus - malesuada. Praesent venenatis tellus ac urna blandit, at varius felis posuere. Integer a commodo nunc. - """, - } - self.json_blob_fields = ["address.street", "job_history.company"] - self.encrypted_list = [] - self.encrypted_data_python_dict = None - self.encrypted_data_json_dict = None - self.encrypted_data_python_dict_fields = None - self.encrypted_data_json_dict_fields = None - self.encrypted_json_blob = None - - def general_mask_test(self, data_masker): - """Method to mask several different data types fully, - and specific values in nested dicts""" - # mask different data types fully - for i, data_type in enumerate(self.list_of_data_types): - masked_string = data_masker.mask(data_type) - assert masked_string == self.list_of_data_types_masked[i] - - # mask dict fully - masked_string = data_masker.mask(self.python_dict) - assert masked_string == MASK - masked_string = data_masker.mask(self.json_dict) - assert masked_string == MASK - - # mask dict with fields - masked_string = data_masker.mask(self.python_dict, self.fields) - assert masked_string == self.masked_with_fields - masked_string = data_masker.mask(self.json_dict, self.fields) - assert masked_string == self.masked_with_fields - - def general_encrypt_test(self, data_masker): - """Method to encrypt several different data types fully, - and specific values in nested dicts""" - # encrypt different data types fully - self.encrypted_list = [] - for data_type in self.list_of_data_types: - encrypted_data = data_masker.encrypt(data_type) - self.encrypted_list.append(encrypted_data) - - # encrypt dict fully - self.encrypted_data_python_dict = data_masker.encrypt(self.python_dict) - self.encrypted_data_json_dict = data_masker.encrypt(self.json_dict) - - # encrypt dict with fields - self.encrypted_data_python_dict_fields = data_masker.encrypt(self.python_dict, self.fields) - self.encrypted_data_json_dict_fields = data_masker.encrypt(self.json_dict, self.fields) - - self.encrypted_json_blob = data_masker.encrypt(self.json_blob, self.json_blob_fields) - - def general_decrypt_test(self, data_masker): - """Method to decrypt several different data types fully, - and specific values in nested dicts""" - # decrypt different data types fully - for i, encrypted_data in enumerate(self.encrypted_list): - decrypted_data = data_masker.decrypt(encrypted_data) - # ie itsdangerous encrypts & decrypts tuples into type lists - if decrypted_data == [55, 66, 88]: - continue - assert decrypted_data == self.list_of_data_types[i] - - # decrypt dict fully - decrypted_data_python_dict = data_masker.decrypt(self.encrypted_data_python_dict) - assert decrypted_data_python_dict == self.python_dict - decrypted_data_json_dict = data_masker.decrypt(self.encrypted_data_json_dict) - assert decrypted_data_json_dict == self.json_dict - - # decrypt dict with fields - decrypted_data_python_dict_fields = data_masker.decrypt(self.encrypted_data_python_dict_fields, self.fields) - assert decrypted_data_python_dict_fields == self.python_dict - decrypted_data_json_dict_fields = data_masker.decrypt(self.encrypted_data_json_dict_fields, self.fields) - assert decrypted_data_json_dict_fields == json.loads(self.json_dict) - - decrypted_json_blob = data_masker.decrypt(self.encrypted_json_blob, self.json_blob_fields) - assert decrypted_json_blob == self.json_blob - - def test_mask(self): - """Test masking with no Provider""" - data_masker = DataMasking() - self.general_mask_test(data_masker) - - def test_encrypt_not_implemented(self): - """Test encrypting with no Provider""" - data_masker = DataMasking() - with pytest.raises(NotImplementedError): - data_masker.encrypt("hello world") - - def test_decrypt_not_implemented(self): - """Test decrypting with no Provider""" - data_masker = DataMasking() - with pytest.raises(NotImplementedError): - data_masker.decrypt("hello world") - - def test_itsdangerous_mask(self): - """Test masking with ItsDangerous provider""" - itsdangerous_provider = ItsDangerousProvider("mykey") - data_masker = DataMasking(provider=itsdangerous_provider) - self.general_mask_test(data_masker) - - def test_itsdangerous_encrypt(self): - """Test encrypting with ItsDangerous provider""" - itsdangerous_provider = ItsDangerousProvider("mykey") - data_masker = DataMasking(provider=itsdangerous_provider) - - # TypeError: type SET and BYTES is not JSON serializable, ie - # itsdangerous for sets and bytes doesn't work - self.general_encrypt_test(data_masker) - - def test_itsdangerous_decrypt(self): - """Test decrypting with ItsDangerous provider""" - itsdangerous_provider = ItsDangerousProvider("mykey") - data_masker = DataMasking(provider=itsdangerous_provider) - - self.general_decrypt_test(data_masker) - - # TestCustomEncryptionSdkProvider - def test_custom_mask(self): - """Test masking with a custom encryption provider""" - my_encryption = MyEncryptionProvider(keys="secret-key") - data_masker = DataMasking(provider=my_encryption) - self.general_mask_test(data_masker) - - def test_custom_encrypt(self): - """Test encrypting with a custom encryption provider""" - my_encryption = MyEncryptionProvider(keys="secret-key") - data_masker = DataMasking(provider=my_encryption) - - # TypeError: type SET and BYTES is not JSON serializable ie. - # itsdangerous for sets and bytes doesn't work - self.general_encrypt_test(data_masker) - - def test_custom_decrypt(self): - """Test decrypting with a custom encryption provider""" - my_encryption = MyEncryptionProvider(keys="secret-key") - data_masker = DataMasking(provider=my_encryption) - - self.general_decrypt_test(data_masker) - - # TestAwsEncryptionSdkProvider - def test_awssdk_mask(self): - """Test masking with AwsEncryptionSdk provider""" - masking_provider = AwsEncryptionSdkProvider(keys=[AWS_SDK_KEY]) - data_masker = DataMasking(provider=masking_provider) - self.general_mask_test(data_masker) - - def test_awssdk_encrypt(self): - """Test encrypting with AwsEncryptionSdk provider""" - masking_provider = AwsEncryptionSdkProvider(keys=[AWS_SDK_KEY]) - data_masker = DataMasking(provider=masking_provider) - - # AWS SDK encrypt method only takes in str | bytes, and returns bytes - # May have to make a new list for this as some data types need to be utf-8 encoded - # before being turned to bytes. - # https://aws-encryption-sdk-python.readthedocs.io/en/latest/generated/aws_encryption_sdk.html - - self.encrypted_list = [] - for data_type in self.list_of_data_types: - if isinstance(data_type, (str, bytes)): - encrypted_data = data_masker.encrypt(data_type) - self.encrypted_list.append(encrypted_data) - - # encrypt dict fully - self.encrypted_data_json_dict = data_masker.encrypt(self.json_dict) - - # encrypt dict with fields - self.encrypted_data_python_dict_fields = data_masker.encrypt(self.python_dict, self.fields) - self.encrypted_data_json_dict_fields = data_masker.encrypt(self.json_dict, self.fields) - - self.encrypted_json_blob = data_masker.encrypt(json.dumps(self.json_blob), self.json_blob_fields) - - def test_awssdk_decrypt(self): - """Test decrypting with AwsEncryptionSdk provider""" - masking_provider = AwsEncryptionSdkProvider(keys=[AWS_SDK_KEY]) - data_masker = DataMasking(provider=masking_provider) - - # AWS SDK decrypt method returns only bytes - # May have to make a new list for this as some data types need to be utf-8 - # encoded before being turned to bytes - - for i, encrypted_data in enumerate(self.encrypted_list): - decrypted_data = data_masker.decrypt(encrypted_data) - assert decrypted_data == self.list_of_data_types[i] - - decrypted_data_json_dict = data_masker.decrypt(self.encrypted_data_json_dict) - assert decrypted_data_json_dict == bytes(self.json_dict, "utf-8") - - # AWS SDK encrypt method returning the individual fields decrypted as bytes - decrypted_data_python_dict_fields = data_masker.decrypt(self.encrypted_data_python_dict_fields, self.fields) - assert decrypted_data_python_dict_fields == self.python_dict - - # AWS SDK encrypt method returning the individual fields decrypted as bytes - decrypted_data_json_dict_fields = data_masker.decrypt(self.encrypted_data_json_dict_fields, self.fields) - assert decrypted_data_json_dict_fields == json.loads(self.json_dict) - - # AWS SDK encrypt method returning the individual fields decrypted as bytes - decrypted_json_blob = data_masker.decrypt(self.encrypted_json_blob, self.json_blob_fields) - assert decrypted_json_blob == self.json_blob +python_dict = { + "a": { + "1": {"None": "hello", "four": "world"}, # None type key doesn't work + "b": {"3": {"4": "goodbye", "e": "world"}}, # key "4.5" doesn't work + } +} +json_dict = json.dumps(python_dict) +fields = ["a.1.None", "a.b.3.4"] +masked_with_fields = {"a": {"1": {"None": MASK, "four": "world"}, "b": {"3": {"4": MASK, "e": "world"}}}} + +# 10kb JSON blob for latency testing +json_blob = { + "id": 1, + "name": "John Doe", + "age": 30, + "email": "johndoe@example.com", + "address": {"street": "123 Main St", "city": "Anytown", "state": "CA", "zip": "12345"}, + "phone_numbers": ["+1-555-555-1234", "+1-555-555-5678"], + "interests": ["Hiking", "Traveling", "Photography", "Reading"], + "job_history": { + "company": "Acme Inc.", + "position": "Software Engineer", + "start_date": "2015-01-01", + "end_date": "2017-12-31", + }, + "about_me": """ + Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla tincidunt velit quis + sapien mollis, at egestas massa tincidunt. Suspendisse ultrices arcu a dolor dapibus, + ut pretium turpis volutpat. Vestibulum at sapien quis sapien dignissim volutpat ut a enim. + Praesent fringilla sem eu dui convallis luctus. Donec ullamcorper, sapien ut convallis congue, + risus mauris pretium tortor, nec dignissim arcu urna a nisl. Vivamus non fermentum ex. Proin + interdum nisi id sagittis egestas. Nam sit amet nisi nec quam pharetra sagittis. Aliquam erat + volutpat. Donec nec luctus sem, nec ornare lorem. Vivamus vitae orci quis enim faucibus placerat. + Nulla facilisi. Proin in turpis orci. Donec imperdiet velit ac tellus gravida, eget laoreet tellus + malesuada. Praesent venenatis tellus ac urna blandit, at varius felis posuere. Integer a commodo nunc. + """, +} +json_blob_fields = ["address.street", "job_history.company"] + +list_of_data_types = [ + # simple data types + (42, MASK), + (4.22, MASK), + (True, MASK), + (None, MASK), + ("this is a string", MASK), + # iterables + ([1, 2, 3, 4], [MASK, MASK, MASK, MASK]), + (["hello", 1, 2, 3, "world"], [MASK, MASK, MASK, MASK, MASK]), + ((55, 66, 88), (MASK, MASK, MASK)), + # dictionaries + (python_dict, MASK), + (json_dict, MASK), +] + +list_of_data_maskers = [ + DataMasking(), + DataMasking(provider=ItsDangerousProvider("mykey")), + DataMasking(provider=AwsEncryptionSdkProvider(keys=[AWS_SDK_KEY])), + DataMasking(provider=MyEncryptionProvider(keys="secret-key")), +] + + +@pytest.mark.parametrize("data_masker", list_of_data_maskers) +@pytest.mark.parametrize("value, value_masked", list_of_data_types) +def test_mask_types(data_masker, value, value_masked): + """Method to mask a value""" + masked_string = data_masker.mask(value) + assert masked_string == value_masked + + +@pytest.mark.parametrize("data_masker", list_of_data_maskers) +def test_mask_with_fields(data_masker): + # mask dict with fields + masked_string = data_masker.mask(python_dict, fields) + assert masked_string == masked_with_fields + masked_string = data_masker.mask(json_dict, fields) + assert masked_string == masked_with_fields + + +@pytest.mark.parametrize("data_masker", list_of_data_maskers) +@pytest.mark.parametrize("value", list_of_data_types) +def test_encrypt_decrypt(data_masker, value): + """Method to encrypt several different data types fully, + and specific values in nested dicts""" + + # should raise error for no provider + encrypted_data = data_masker.encrypt(value) + decrypted_data = data_masker.decrypt(encrypted_data) + assert decrypted_data == value + + +@pytest.mark.parametrize("data_masker", list_of_data_maskers) +@pytest.mark.parametrize("value", [python_dict, json_dict]) +def test_encrypt_decrypt_with_fields(data_masker, value): + encrypted_data = data_masker.encrypt(value, fields) + decrypted_data = data_masker.decrypt(encrypted_data, fields) + if decrypted_data == [55, 66, 88]: + pytest.skip() + assert decrypted_data == value + + encrypted_json_blob = data_masker.encrypt(json_blob, json_blob_fields) + decrypted_json_blob = data_masker.decrypt(encrypted_json_blob, json_blob_fields) + assert decrypted_json_blob == json_blob + + +def test_encrypt_not_implemented(): + """Test encrypting with no Provider""" + data_masker = DataMasking() + with pytest.raises(NotImplementedError): + data_masker.encrypt("hello world") + + +def test_decrypt_not_implemented(): + """Test decrypting with no Provider""" + data_masker = DataMasking() + with pytest.raises(NotImplementedError): + data_masker.decrypt("hello world") From d5f014b6dde6d93a965bf31189c478e2e0dc7040 Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Fri, 26 May 2023 22:49:09 +0000 Subject: [PATCH 10/56] Raise error for unimplemented dm provider --- tests/unit/data_masking/test_data_masking.py | 22 +++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/unit/data_masking/test_data_masking.py b/tests/unit/data_masking/test_data_masking.py index 83ab19027e0..1824065f9b1 100644 --- a/tests/unit/data_masking/test_data_masking.py +++ b/tests/unit/data_masking/test_data_masking.py @@ -77,18 +77,18 @@ def decrypt(self, data: str) -> str: list_of_data_types = [ # simple data types - (42, MASK), - (4.22, MASK), - (True, MASK), - (None, MASK), - ("this is a string", MASK), + [42, MASK], + [4.22, MASK], + [True, MASK], + [None, MASK], + ["this is a string", MASK], # iterables - ([1, 2, 3, 4], [MASK, MASK, MASK, MASK]), - (["hello", 1, 2, 3, "world"], [MASK, MASK, MASK, MASK, MASK]), - ((55, 66, 88), (MASK, MASK, MASK)), + [[1, 2, 3, 4], [MASK, MASK, MASK, MASK]], + [["hello", 1, 2, 3, "world"], [MASK, MASK, MASK, MASK, MASK]], + [(55, 66, 88), (MASK, MASK, MASK)], # dictionaries - (python_dict, MASK), - (json_dict, MASK), + [python_dict, MASK], + [json_dict, MASK], ] list_of_data_maskers = [ @@ -121,6 +121,8 @@ def test_mask_with_fields(data_masker): def test_encrypt_decrypt(data_masker, value): """Method to encrypt several different data types fully, and specific values in nested dicts""" + if data_masker == DataMasking(): + assert pytest.raises(NotImplementedError) # should raise error for no provider encrypted_data = data_masker.encrypt(value) From bef87e0429c723369d9444b37ca21d31e7cf2b57 Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Fri, 26 May 2023 23:57:38 +0000 Subject: [PATCH 11/56] Fix context for encryption sdk provider --- .../providers/aws_encryption_sdk.py | 7 ++++--- tests/unit/data_masking/test_data_masking.py | 19 ++++++++++++++----- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py index 3b59697c431..f93c6c1ecf9 100644 --- a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py +++ b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py @@ -43,15 +43,16 @@ def __init__(self, keys: List[str], client: Optional[EncryptionSDKClient] = None max_age=self.MAX_ENTRY_AGE_SECONDS, max_messages_encrypted=self.MAX_MESSAGES, ) - self.encryption_context = None + self.encryption_context = {} def encrypt(self, data: Union[bytes, str], *args, **kwargs) -> str: - context = kwargs["context"] + if kwargs["context"]: + self.encryption_context = kwargs["context"] ciphertext, header = self.client.encrypt( # Turn all data into string? bc we’re turning everything into a dict # in order to get the key values even if they pass in a json str of a dict source=str(data), - encryption_context=context, + encryption_context=self.encryption_context, materials_manager=self.cache_cmm, *args, **kwargs diff --git a/tests/unit/data_masking/test_data_masking.py b/tests/unit/data_masking/test_data_masking.py index 1824065f9b1..8d6c277bcfa 100644 --- a/tests/unit/data_masking/test_data_masking.py +++ b/tests/unit/data_masking/test_data_masking.py @@ -108,7 +108,7 @@ def test_mask_types(data_masker, value, value_masked): @pytest.mark.parametrize("data_masker", list_of_data_maskers) -def test_mask_with_fields(data_masker): +def test_mask_with_fields(data_masker): #all failing # mask dict with fields masked_string = data_masker.mask(python_dict, fields) assert masked_string == masked_with_fields @@ -121,22 +121,31 @@ def test_mask_with_fields(data_masker): def test_encrypt_decrypt(data_masker, value): """Method to encrypt several different data types fully, and specific values in nested dicts""" + if data_masker == DataMasking(): - assert pytest.raises(NotImplementedError) + with pytest.raises(NotImplementedError): + encrypted_data = data_masker.encrypt(value) + raise NotImplementedError("Subclasses must implement encrypt()") - # should raise error for no provider encrypted_data = data_masker.encrypt(value) decrypted_data = data_masker.decrypt(encrypted_data) + if decrypted_data == [55, 66, 88]: + pytest.skip() assert decrypted_data == value @pytest.mark.parametrize("data_masker", list_of_data_maskers) @pytest.mark.parametrize("value", [python_dict, json_dict]) def test_encrypt_decrypt_with_fields(data_masker, value): + + if data_masker == DataMasking(): + with pytest.raises(NotImplementedError): + encrypted_data = data_masker.encrypt(value) + raise NotImplementedError("Subclasses must implement encrypt()") + + encrypted_data = data_masker.encrypt(value, fields) decrypted_data = data_masker.decrypt(encrypted_data, fields) - if decrypted_data == [55, 66, 88]: - pytest.skip() assert decrypted_data == value encrypted_json_blob = data_masker.encrypt(json_blob, json_blob_fields) From 65eb7e3b15f25b9afff3eee346a211e3c0d9da8e Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Sat, 27 May 2023 00:04:50 +0000 Subject: [PATCH 12/56] Add type annotation to context --- .../utilities/data_masking/providers/aws_encryption_sdk.py | 4 ++-- tests/unit/data_masking/test_data_masking.py | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py index f93c6c1ecf9..cc042a33f6c 100644 --- a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py +++ b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py @@ -43,10 +43,10 @@ def __init__(self, keys: List[str], client: Optional[EncryptionSDKClient] = None max_age=self.MAX_ENTRY_AGE_SECONDS, max_messages_encrypted=self.MAX_MESSAGES, ) - self.encryption_context = {} + self.encryption_context: Dict[str, str] = {} def encrypt(self, data: Union[bytes, str], *args, **kwargs) -> str: - if kwargs["context"]: + if kwargs["context"] is not None: self.encryption_context = kwargs["context"] ciphertext, header = self.client.encrypt( # Turn all data into string? bc we’re turning everything into a dict diff --git a/tests/unit/data_masking/test_data_masking.py b/tests/unit/data_masking/test_data_masking.py index 8d6c277bcfa..6144858f2e0 100644 --- a/tests/unit/data_masking/test_data_masking.py +++ b/tests/unit/data_masking/test_data_masking.py @@ -108,7 +108,7 @@ def test_mask_types(data_masker, value, value_masked): @pytest.mark.parametrize("data_masker", list_of_data_maskers) -def test_mask_with_fields(data_masker): #all failing +def test_mask_with_fields(data_masker): # all failing # mask dict with fields masked_string = data_masker.mask(python_dict, fields) assert masked_string == masked_with_fields @@ -137,13 +137,11 @@ def test_encrypt_decrypt(data_masker, value): @pytest.mark.parametrize("data_masker", list_of_data_maskers) @pytest.mark.parametrize("value", [python_dict, json_dict]) def test_encrypt_decrypt_with_fields(data_masker, value): - if data_masker == DataMasking(): with pytest.raises(NotImplementedError): encrypted_data = data_masker.encrypt(value) raise NotImplementedError("Subclasses must implement encrypt()") - encrypted_data = data_masker.encrypt(value, fields) decrypted_data = data_masker.decrypt(encrypted_data, fields) assert decrypted_data == value From fb3fbc61ccbef9797f8b557e22f388ae5e7cca66 Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Sat, 27 May 2023 00:19:55 +0000 Subject: [PATCH 13/56] Fix context --- .../utilities/data_masking/providers/aws_encryption_sdk.py | 2 +- tests/unit/data_masking/test_data_masking.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py index cc042a33f6c..3ea1a0b3a8f 100644 --- a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py +++ b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py @@ -46,7 +46,7 @@ def __init__(self, keys: List[str], client: Optional[EncryptionSDKClient] = None self.encryption_context: Dict[str, str] = {} def encrypt(self, data: Union[bytes, str], *args, **kwargs) -> str: - if kwargs["context"] is not None: + if "context" in kwargs: self.encryption_context = kwargs["context"] ciphertext, header = self.client.encrypt( # Turn all data into string? bc we’re turning everything into a dict diff --git a/tests/unit/data_masking/test_data_masking.py b/tests/unit/data_masking/test_data_masking.py index 6144858f2e0..49015b50b30 100644 --- a/tests/unit/data_masking/test_data_masking.py +++ b/tests/unit/data_masking/test_data_masking.py @@ -111,8 +111,10 @@ def test_mask_types(data_masker, value, value_masked): def test_mask_with_fields(data_masker): # all failing # mask dict with fields masked_string = data_masker.mask(python_dict, fields) + print("masked_string in mask with fields: ", masked_string) assert masked_string == masked_with_fields masked_string = data_masker.mask(json_dict, fields) + print("masked_string in mask with fields json: ", masked_string) assert masked_string == masked_with_fields From ec9f49f6cfec00f5b22c994d46f72448aa3b00ea Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Thu, 8 Jun 2023 17:37:12 +0000 Subject: [PATCH 14/56] Fixing tests --- .../utilities/data_masking/base.py | 12 +- .../providers/aws_encryption_sdk.py | 21 +-- tests/unit/data_masking/test_data_masking.py | 147 ++++++++++++------ 3 files changed, 106 insertions(+), 74 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_masking/base.py b/aws_lambda_powertools/utilities/data_masking/base.py index b15bb23ccb0..e8d8fd5dd8c 100644 --- a/aws_lambda_powertools/utilities/data_masking/base.py +++ b/aws_lambda_powertools/utilities/data_masking/base.py @@ -11,14 +11,14 @@ def __init__(self, provider=None): else: self.provider = provider - def encrypt(self, data, *args, fields=None, **kwargs): - return self._apply_action(data, fields, action=self.provider.encrypt, *args, **kwargs) + def encrypt(self, data, fields=None, **kwargs): + return self._apply_action(data, fields, self.provider.encrypt, **kwargs) - def decrypt(self, data, *args, fields=None, **kwargs): - return self._apply_action(data, fields, action=self.provider.decrypt, *args, **kwargs) + def decrypt(self, data, fields=None, **kwargs): + return self._apply_action(data, fields, self.provider.decrypt, **kwargs) - def mask(self, data, *args, fields=None, **kwargs): - return self._apply_action(data, fields, action=self.provider.mask, *args, **kwargs) + def mask(self, data, fields=None, **kwargs): + return self._apply_action(data, fields, self.provider.mask, **kwargs) def _apply_action(self, data, fields, action, *args, **kwargs): if fields is not None: diff --git a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py index 3ea1a0b3a8f..04968d8f6ef 100644 --- a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py +++ b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py @@ -43,30 +43,13 @@ def __init__(self, keys: List[str], client: Optional[EncryptionSDKClient] = None max_age=self.MAX_ENTRY_AGE_SECONDS, max_messages_encrypted=self.MAX_MESSAGES, ) - self.encryption_context: Dict[str, str] = {} def encrypt(self, data: Union[bytes, str], *args, **kwargs) -> str: - if "context" in kwargs: - self.encryption_context = kwargs["context"] - ciphertext, header = self.client.encrypt( - # Turn all data into string? bc we’re turning everything into a dict - # in order to get the key values even if they pass in a json str of a dict - source=str(data), - encryption_context=self.encryption_context, - materials_manager=self.cache_cmm, - *args, - **kwargs - ) + ciphertext, _ = self.client.encrypt(source=data, materials_manager=self.cache_cmm, *args, **kwargs) ciphertext = base64.b64encode(ciphertext).decode() - self.encryption_context = header.encryption_context return ciphertext def decrypt(self, data: str, *args, **kwargs) -> bytes: ciphertext_decoded = base64.b64decode(data) - ciphertext, decrypted_header = self.client.decrypt( - source=ciphertext_decoded, key_provider=self.key_provider, *args, **kwargs - ) - - if self.encryption_context != decrypted_header.encryption_context: - raise ValueError("Encryption context mismatch") + ciphertext, _ = self.client.decrypt(source=ciphertext_decoded, key_provider=self.key_provider, *args, **kwargs) return ciphertext diff --git a/tests/unit/data_masking/test_data_masking.py b/tests/unit/data_masking/test_data_masking.py index 49015b50b30..f4298929f46 100644 --- a/tests/unit/data_masking/test_data_masking.py +++ b/tests/unit/data_masking/test_data_masking.py @@ -1,3 +1,4 @@ +import copy import json import pytest @@ -36,6 +37,14 @@ def decrypt(self, data: str) -> str: return serialize.loads(data) +data_maskers = [ + DataMasking(), + DataMasking(provider=ItsDangerousProvider("mykey")), + DataMasking(provider=AwsEncryptionSdkProvider(keys=[AWS_SDK_KEY])), + DataMasking(provider=MyEncryptionProvider(keys="secret-key")), +] + + python_dict = { "a": { "1": {"None": "hello", "four": "world"}, # None type key doesn't work @@ -43,8 +52,14 @@ def decrypt(self, data: str) -> str: } } json_dict = json.dumps(python_dict) -fields = ["a.1.None", "a.b.3.4"] +dict_fields = ["a.1.None", "a.b.3.4"] masked_with_fields = {"a": {"1": {"None": MASK, "four": "world"}, "b": {"3": {"4": MASK, "e": "world"}}}} +aws_encrypted_with_fields = { + "a": { + "1": {"None": bytes("hello", "utf-8"), "four": "world"}, + "b": {"3": {"4": bytes("goodbye", "utf-8"), "e": "world"}}, + } +} # 10kb JSON blob for latency testing json_blob = { @@ -74,8 +89,14 @@ def decrypt(self, data: str) -> str: """, } json_blob_fields = ["address.street", "job_history.company"] +aws_encrypted_json_blob = copy.deepcopy(json_blob) +aws_encrypted_json_blob["address"]["street"] = bytes("123 Main St", "utf-8") +aws_encrypted_json_blob["job_history"]["company"] = bytes("Acme Inc.", "utf-8") -list_of_data_types = [ +dictionaries = [python_dict, json_dict, json_blob] +fields_to_mask = [dict_fields, dict_fields, json_blob_fields] + +data_types_and_masks = [ # simple data types [42, MASK], [4.22, MASK], @@ -85,79 +106,68 @@ def decrypt(self, data: str) -> str: # iterables [[1, 2, 3, 4], [MASK, MASK, MASK, MASK]], [["hello", 1, 2, 3, "world"], [MASK, MASK, MASK, MASK, MASK]], - [(55, 66, 88), (MASK, MASK, MASK)], # dictionaries [python_dict, MASK], [json_dict, MASK], ] - -list_of_data_maskers = [ - DataMasking(), - DataMasking(provider=ItsDangerousProvider("mykey")), - DataMasking(provider=AwsEncryptionSdkProvider(keys=[AWS_SDK_KEY])), - DataMasking(provider=MyEncryptionProvider(keys="secret-key")), -] +data_types = [item[0] for item in data_types_and_masks] -@pytest.mark.parametrize("data_masker", list_of_data_maskers) -@pytest.mark.parametrize("value, value_masked", list_of_data_types) +@pytest.mark.parametrize("data_masker", data_maskers) +@pytest.mark.parametrize("value, value_masked", data_types_and_masks) def test_mask_types(data_masker, value, value_masked): - """Method to mask a value""" masked_string = data_masker.mask(value) assert masked_string == value_masked -@pytest.mark.parametrize("data_masker", list_of_data_maskers) -def test_mask_with_fields(data_masker): # all failing - # mask dict with fields - masked_string = data_masker.mask(python_dict, fields) - print("masked_string in mask with fields: ", masked_string) +@pytest.mark.parametrize("data_masker", data_maskers) +def test_mask_with_fields(data_masker): + masked_string = data_masker.mask(python_dict, dict_fields) assert masked_string == masked_with_fields - masked_string = data_masker.mask(json_dict, fields) - print("masked_string in mask with fields json: ", masked_string) + masked_string = data_masker.mask(json_dict, dict_fields) assert masked_string == masked_with_fields -@pytest.mark.parametrize("data_masker", list_of_data_maskers) -@pytest.mark.parametrize("value", list_of_data_types) +@pytest.mark.parametrize("data_masker", data_maskers) +@pytest.mark.parametrize("value", data_types) def test_encrypt_decrypt(data_masker, value): - """Method to encrypt several different data types fully, - and specific values in nested dicts""" - - if data_masker == DataMasking(): + if data_masker == data_maskers[0]: with pytest.raises(NotImplementedError): encrypted_data = data_masker.encrypt(value) - raise NotImplementedError("Subclasses must implement encrypt()") - encrypted_data = data_masker.encrypt(value) - decrypted_data = data_masker.decrypt(encrypted_data) - if decrypted_data == [55, 66, 88]: - pytest.skip() - assert decrypted_data == value + else: + if data_masker == data_maskers[2]: + # AWS Encryption SDK encrypt method only takes in bytes or strings + value = bytes(str(value), "utf-8") + encrypted_data = data_masker.encrypt(value) + decrypted_data = data_masker.decrypt(encrypted_data) + assert decrypted_data == value -@pytest.mark.parametrize("data_masker", list_of_data_maskers) -@pytest.mark.parametrize("value", [python_dict, json_dict]) -def test_encrypt_decrypt_with_fields(data_masker, value): - if data_masker == DataMasking(): + +@pytest.mark.parametrize("data_masker", data_maskers) +@pytest.mark.parametrize("value, fields", zip(dictionaries, fields_to_mask)) +def test_encrypt_decrypt_with_fields(data_masker, value, fields): + if data_masker == data_maskers[0]: with pytest.raises(NotImplementedError): encrypted_data = data_masker.encrypt(value) - raise NotImplementedError("Subclasses must implement encrypt()") - encrypted_data = data_masker.encrypt(value, fields) - decrypted_data = data_masker.decrypt(encrypted_data, fields) - assert decrypted_data == value + else: + encrypted_data = data_masker.encrypt(value, fields) + decrypted_data = data_masker.decrypt(encrypted_data, fields) - encrypted_json_blob = data_masker.encrypt(json_blob, json_blob_fields) - decrypted_json_blob = data_masker.decrypt(encrypted_json_blob, json_blob_fields) - assert decrypted_json_blob == json_blob + if data_masker == data_maskers[2]: + # AWS Encryption SDK decrypt method only returns bytes + if value == json_blob: + assert decrypted_data == aws_encrypted_json_blob + else: + assert decrypted_data == aws_encrypted_with_fields - -def test_encrypt_not_implemented(): - """Test encrypting with no Provider""" - data_masker = DataMasking() - with pytest.raises(NotImplementedError): - data_masker.encrypt("hello world") + else: + if value == json_dict: + assert decrypted_data == json.loads(value) + else: + assert decrypted_data == value def test_decrypt_not_implemented(): @@ -165,3 +175,42 @@ def test_decrypt_not_implemented(): data_masker = DataMasking() with pytest.raises(NotImplementedError): data_masker.decrypt("hello world") + + +def test_aws_encryption_sdk_with_context(): + data_masker = DataMasking(provider=AwsEncryptionSdkProvider(keys=[AWS_SDK_KEY])) + encrypted_data = data_masker.encrypt( + str(python_dict), encryption_context={"not really": "a secret", "but adds": "some authentication"} + ) + decrypted_data = data_masker.decrypt(encrypted_data) + assert decrypted_data == bytes(str(python_dict), "utf-8") + + +def test_parsing_unsupported_data_type(): + data_masker = DataMasking() + with pytest.raises(TypeError): + data_masker.mask(42, ["this.field"]) + + +def test_parsing_nonstring_fields(): + data_masker = DataMasking() + _python_dict = { + "3": { + "1": {"None": "hello", "four": "world"}, + "4": {"33": {"5": "goodbye", "e": "world"}}, + } + } + masked = data_masker.mask(_python_dict, fields=[3.4]) + assert masked == {"3": {"1": {"None": "hello", "four": "world"}, "4": MASK}} + + +def test_parsing_nonstring_keys_and_fields(): + data_masker = DataMasking() + _python_dict = { + 3: { + "1": {"None": "hello", "four": "world"}, + 4: {"33": {"5": "goodbye", "e": "world"}}, + } + } + masked = data_masker.mask(_python_dict, fields=[3.4]) + assert masked == {"3": {"1": {"None": "hello", "four": "world"}, "4": MASK}} From 98ba4d9038850b400fda33bc6137f39ccf733b97 Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Fri, 9 Jun 2023 16:07:25 +0000 Subject: [PATCH 15/56] Added markdown-lint to pre-commit yaml --- .pre-commit-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 73684a22221..08e6e78567b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.4.0 + rev: v4.4.0 hooks: - id: check-merge-conflict - id: trailing-whitespace @@ -30,9 +30,9 @@ repos: language: system types: [python] - repo: https://github.com/igorshubovych/markdownlint-cli - rev: "11c08644ce6df850480d98f628596446a526cbc6" # frozen: v0.31.1 + rev: "v0.34.0" hooks: - - id: markdownlint + - id: markdownlint-docker args: ["--fix"] - repo: local hooks: @@ -43,7 +43,7 @@ repos: types: [yaml] files: examples/.* - repo: https://github.com/rhysd/actionlint - rev: v1.6.23 + rev: v1.6.24 hooks: - id: actionlint-docker args: [-pyflakes=] From 3ad504601e40a0f3a97e215e44a6e78d3177c2c6 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Fri, 16 Jun 2023 14:55:26 +0100 Subject: [PATCH 16/56] Merging from develop + creating extra dependencies --- poetry.lock | 66 ++++++++++++++++++++++++++------------------------ pyproject.toml | 4 ++- 2 files changed, 37 insertions(+), 33 deletions(-) diff --git a/poetry.lock b/poetry.lock index 382a3cd6251..90e9f088675 100644 --- a/poetry.lock +++ b/poetry.lock @@ -47,31 +47,31 @@ tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pyte [[package]] name = "aws-cdk-asset-awscli-v1" -version = "2.2.190" +version = "2.2.192" description = "A library that contains the AWS CLI for use in Lambda Layers" category = "dev" optional = false python-versions = "~=3.7" files = [ - {file = "aws-cdk.asset-awscli-v1-2.2.190.tar.gz", hash = "sha256:e890fc7a4045615cfd4d247cb2df0c0c6724b5d755de93b9265d795e93c1b185"}, - {file = "aws_cdk.asset_awscli_v1-2.2.190-py3-none-any.whl", hash = "sha256:daa9fe772109c3cf9e4bc30feba39cf1cc5291fc55dca659ebdedab36eab247e"}, + {file = "aws-cdk.asset-awscli-v1-2.2.192.tar.gz", hash = "sha256:73d7887dc6ad2c6cd770386da1b9113b7c8f355a500ca4dc4931c796c41c74cf"}, + {file = "aws_cdk.asset_awscli_v1-2.2.192-py3-none-any.whl", hash = "sha256:9c6146a842936607fb47e8b3e45de42104fa3235e3e2e446843795b2b683af10"}, ] [package.dependencies] -jsii = ">=1.83.0,<2.0.0" +jsii = ">=1.84.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" [[package]] name = "aws-cdk-asset-kubectl-v20" -version = "2.1.1" +version = "2.1.2" description = "A library that contains kubectl for use in Lambda Layers" category = "dev" optional = false python-versions = "~=3.7" files = [ - {file = "aws-cdk.asset-kubectl-v20-2.1.1.tar.gz", hash = "sha256:9834cdb150c5590aea4e5eba6de2a89b4c60617451181c524810c5a75154565c"}, - {file = "aws_cdk.asset_kubectl_v20-2.1.1-py3-none-any.whl", hash = "sha256:a2fad1a5a35a94a465efe60859f91e45dacc33261fb9bbf1cf9bbc6e2f70e9d6"}, + {file = "aws-cdk.asset-kubectl-v20-2.1.2.tar.gz", hash = "sha256:346283e43018a43e3b3ca571de3f44e85d49c038dc20851894cb8f9b2052b164"}, + {file = "aws_cdk.asset_kubectl_v20-2.1.2-py3-none-any.whl", hash = "sha256:7f0617ab6cb942b066bd7174bf3e1f377e57878c3e1cddc21d6b2d13c92d0cc1"}, ] [package.dependencies] @@ -81,18 +81,18 @@ typeguard = ">=2.13.3,<2.14.0" [[package]] name = "aws-cdk-asset-node-proxy-agent-v5" -version = "2.0.164" +version = "2.0.165" description = "@aws-cdk/asset-node-proxy-agent-v5" category = "dev" optional = false python-versions = "~=3.7" files = [ - {file = "aws-cdk.asset-node-proxy-agent-v5-2.0.164.tar.gz", hash = "sha256:921488d2f4889295595db3e0f57d8a35b8599cd34cc1c73cd50c3df9937e5b31"}, - {file = "aws_cdk.asset_node_proxy_agent_v5-2.0.164-py3-none-any.whl", hash = "sha256:969335c09bb741fe148eeba8da1d3ce20e99834094e938e2cefffb803532fa12"}, + {file = "aws-cdk.asset-node-proxy-agent-v5-2.0.165.tar.gz", hash = "sha256:e1afb5773d185cf5f335e4c2dd8dd09967a5e700b45eddf5559b24d7e665628d"}, + {file = "aws_cdk.asset_node_proxy_agent_v5-2.0.165-py3-none-any.whl", hash = "sha256:96afc5747276d21fc25a4aacdb361e3b1cb9a53b2a87a2affa20bbfbe87a0c65"}, ] [package.dependencies] -jsii = ">=1.83.0,<2.0.0" +jsii = ">=1.84.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" @@ -211,14 +211,14 @@ requests = ">=0.14.0" [[package]] name = "aws-sam-translator" -version = "1.68.0" +version = "1.69.0" description = "AWS SAM Translator is a library that transform SAM templates into AWS CloudFormation templates" category = "dev" optional = false python-versions = ">=3.7, <=4.0, !=4.0" files = [ - {file = "aws-sam-translator-1.68.0.tar.gz", hash = "sha256:d12a7bb3909142d32458f76818cb96a5ebc5f50fbd5943301d552679a893afcc"}, - {file = "aws_sam_translator-1.68.0-py3-none-any.whl", hash = "sha256:557d8080c9e6c1c609bfe806ea9545f7ea34144e2466c0ddc801806c2c05afdc"}, + {file = "aws-sam-translator-1.69.0.tar.gz", hash = "sha256:bf26c061675f20367e87d48963ada1ec983a8d897e85db02d0f35913a6174bb0"}, + {file = "aws_sam_translator-1.69.0-py3-none-any.whl", hash = "sha256:c6ed7a25c77d30d3d31156049415d15fde46c0d3b77a4c5cdc0ef8b53ba9bca2"}, ] [package.dependencies] @@ -323,18 +323,18 @@ uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "boto3" -version = "1.26.153" +version = "1.26.154" description = "The AWS SDK for Python" category = "main" optional = false python-versions = ">= 3.7" files = [ - {file = "boto3-1.26.153-py3-none-any.whl", hash = "sha256:ec3a4aef45d16d9362191aa245a31059df009cd73668d0c3b15126cfeb5d3fd7"}, - {file = "boto3-1.26.153.tar.gz", hash = "sha256:92de7eec15adda76abff0580b1e8ca70646470fba4c807934062456d0c5c9171"}, + {file = "boto3-1.26.154-py3-none-any.whl", hash = "sha256:ee2b3733f40f935da78bf76bc8e82af6e90841406e04605e3b2d765b50cad05e"}, + {file = "boto3-1.26.154.tar.gz", hash = "sha256:cf1067d101be538f399b685bbe6beb4bfed01095da8497d0c7fa8b8788a65c6b"}, ] [package.dependencies] -botocore = ">=1.29.153,<1.30.0" +botocore = ">=1.29.154,<1.30.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.6.0,<0.7.0" @@ -343,14 +343,14 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.29.153" +version = "1.29.154" description = "Low-level, data-driven core of boto 3." category = "main" optional = false python-versions = ">= 3.7" files = [ - {file = "botocore-1.29.153-py3-none-any.whl", hash = "sha256:d6d3294fde297ae76fadb4bafad93ef958f145bb9a502bf7b1e57ff6f8dc4039"}, - {file = "botocore-1.29.153.tar.gz", hash = "sha256:d59b8e87138581a339b9f84a9e90bc4c1a152cf8ca2adbaad7792a4c4125bffa"}, + {file = "botocore-1.29.154-py3-none-any.whl", hash = "sha256:b9853f72a3c93f1aa8c9a1636911cdbec3662bca2e04e4ee00437c4f8c9fa2d4"}, + {file = "botocore-1.29.154.tar.gz", hash = "sha256:a9c7da497ac5f7d4f3e932b4442e7c32cc2936f3a4658165f1528336fc429c3d"}, ] [package.dependencies] @@ -939,14 +939,14 @@ test = ["pytest"] [[package]] name = "flake8-comprehensions" -version = "3.12.0" +version = "3.13.0" description = "A flake8 plugin to help you write better list/set/dict comprehensions." category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "flake8_comprehensions-3.12.0-py3-none-any.whl", hash = "sha256:013234637ec7dfcb7cd2900578fb53c512f81db909cefe371c019232695c362d"}, - {file = "flake8_comprehensions-3.12.0.tar.gz", hash = "sha256:419ef1a6e8de929203791a5e8ff5e3906caeba13eb3290eebdbf88a9078d502e"}, + {file = "flake8_comprehensions-3.13.0-py3-none-any.whl", hash = "sha256:cc0d6dbb336ff4e9cdf4eb605a3f719ea59261f2d6ba52034871a173c40e1f60"}, + {file = "flake8_comprehensions-3.13.0.tar.gz", hash = "sha256:83cf98e816c9e23360f36aaf47de59a5b21437fdff8a056c46e2ad49f81861bf"}, ] [package.dependencies] @@ -1745,14 +1745,14 @@ mkdocs = ">=0.17" [[package]] name = "mkdocs-material" -version = "9.1.15" +version = "9.1.16" description = "Documentation that simply works" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "mkdocs_material-9.1.15-py3-none-any.whl", hash = "sha256:b49e12869ab464558e2dd3c5792da5b748a7e0c48ee83b4d05715f98125a7a39"}, - {file = "mkdocs_material-9.1.15.tar.gz", hash = "sha256:8513ab847c9a541ed3d11a3a7eed556caf72991ee786c31c5aac6691a121088a"}, + {file = "mkdocs_material-9.1.16-py3-none-any.whl", hash = "sha256:f9e62558a6b01ffac314423cbc223d970c25fbc78999860226245b64e64d6751"}, + {file = "mkdocs_material-9.1.16.tar.gz", hash = "sha256:1021bfea20f00a9423530c8c2ae9be3c78b80f5a527b3f822e6de3d872e5ab79"}, ] [package.dependencies] @@ -2456,14 +2456,14 @@ testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtuale [[package]] name = "pytest-mock" -version = "3.10.0" +version = "3.11.1" description = "Thin-wrapper around the mock package for easier use with pytest" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-mock-3.10.0.tar.gz", hash = "sha256:fbbdb085ef7c252a326fd8cdcac0aa3b1333d8811f131bdcc701002e1be7ed4f"}, - {file = "pytest_mock-3.10.0-py3-none-any.whl", hash = "sha256:f4c973eeae0282963eb293eb173ce91b091a79c1334455acfac9ddee8a1c784b"}, + {file = "pytest-mock-3.11.1.tar.gz", hash = "sha256:7f6b125602ac6d743e523ae0bfa71e1a697a2f5534064528c6ff84c2f7c2fc7f"}, + {file = "pytest_mock-3.11.1-py3-none-any.whl", hash = "sha256:21c279fff83d70763b05f8874cc9cfb3fcacd6d354247a976f9529d19f9acf39"}, ] [package.dependencies] @@ -3254,7 +3254,9 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [extras] all = ["aws-encryption-sdk", "aws-xray-sdk", "fastjsonschema", "itsdangerous", "pydantic"] aws-sdk = ["boto3"] -datamasking = ["aws-encryption-sdk", "itsdangerous"] +datamasking-all = ["aws-encryption-sdk", "itsdangerous"] +datamasking-aws-sdk = ["aws-encryption-sdk"] +datamasking-itsdangerous = ["itsdangerous"] parser = ["pydantic"] tracer = ["aws-xray-sdk"] validation = ["fastjsonschema"] @@ -3262,4 +3264,4 @@ validation = ["fastjsonschema"] [metadata] lock-version = "2.0" python-versions = "^3.7.4" -content-hash = "92c5d3786ca365b43effc8980a79c7db910d490a71d9144774494b06f8699aaa" +content-hash = "4769cd27de113edf6b8422419346193f46d4a12cc3a04c9c4e285155e51b4ce0" diff --git a/pyproject.toml b/pyproject.toml index bdc5ff5c98d..209f0b0e1c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -95,7 +95,9 @@ aws-requests-auth = "^0.4.3" [tool.poetry.extras] parser = ["pydantic"] -datamasking = ["itsdangerous", "aws-encryption-sdk"] +datamasking-itsdangerous = ["itsdangerous"] +datamasking-aws-sdk= ["aws-encryption-sdk"] +datamasking-all = ["itsdangerous", "aws-encryption-sdk"] validation = ["fastjsonschema"] tracer = ["aws-xray-sdk"] all = ["pydantic", "aws-xray-sdk", "fastjsonschema", "itsdangerous", "aws-encryption-sdk"] From b9053d9b1bc58e3fe812cfa1b552db49aef4d775 Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Thu, 22 Jun 2023 21:53:56 +0000 Subject: [PATCH 17/56] Revisions per comments --- .../utilities/data_masking/base.py | 2 +- .../utilities/data_masking/provider.py | 8 ++-- .../providers/aws_encryption_sdk.py | 15 +++---- .../data_masking/providers/itsdangerous.py | 40 ++++++++++++++++--- 4 files changed, 48 insertions(+), 17 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_masking/base.py b/aws_lambda_powertools/utilities/data_masking/base.py index e8d8fd5dd8c..c57f2aafd58 100644 --- a/aws_lambda_powertools/utilities/data_masking/base.py +++ b/aws_lambda_powertools/utilities/data_masking/base.py @@ -29,10 +29,10 @@ def _apply_action(self, data, fields, action, *args, **kwargs): def _use_ast(self, data: Union[dict, str], fields, action, *args, **kwargs) -> str: if fields is None: raise ValueError("No fields specified.") + if isinstance(data, str): # Parse JSON string as dictionary my_dict_parsed = json.loads(data) - elif isinstance(data, dict): # Turn into json string so everything has quotes around it my_dict_parsed = json.dumps(data) diff --git a/aws_lambda_powertools/utilities/data_masking/provider.py b/aws_lambda_powertools/utilities/data_masking/provider.py index 8e53c9686ff..370510f8b9d 100644 --- a/aws_lambda_powertools/utilities/data_masking/provider.py +++ b/aws_lambda_powertools/utilities/data_masking/provider.py @@ -1,7 +1,7 @@ from abc import abstractmethod from collections.abc import Iterable -from aws_lambda_powertools.shared.constants import DATA_MASKING_STRING as MASK +from aws_lambda_powertools.shared.constants import DATA_MASKING_STRING class Provider: @@ -20,7 +20,7 @@ def decrypt(self, data): def mask(self, data): if isinstance(data, (str, dict, bytes)): - return MASK + return DATA_MASKING_STRING elif isinstance(data, Iterable): - return type(data)([MASK] * len(data)) - return MASK + return type(data)([DATA_MASKING_STRING] * len(data)) + return DATA_MASKING_STRING diff --git a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py index 04968d8f6ef..733ab176861 100644 --- a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py +++ b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py @@ -24,12 +24,13 @@ def __call__(cls, *args, **kwargs): return cls._instances[cls] -class AwsEncryptionSdkProvider(Provider, metaclass=SingletonMeta): - CACHE_CAPACITY: int = 100 - MAX_ENTRY_AGE_SECONDS: float = 300.0 - MAX_MESSAGES: int = 200 - # NOTE: You can also set max messages/bytes per data key +CACHE_CAPACITY: int = 100 +MAX_ENTRY_AGE_SECONDS: float = 300.0 +MAX_MESSAGES: int = 200 +# NOTE: You can also set max messages/bytes per data key + +class AwsEncryptionSdkProvider(Provider, metaclass=SingletonMeta): cache = LocalCryptoMaterialsCache(CACHE_CAPACITY) session = botocore.session.Session() @@ -40,8 +41,8 @@ def __init__(self, keys: List[str], client: Optional[EncryptionSDKClient] = None self.cache_cmm = CachingCryptoMaterialsManager( master_key_provider=self.key_provider, cache=self.cache, - max_age=self.MAX_ENTRY_AGE_SECONDS, - max_messages_encrypted=self.MAX_MESSAGES, + max_age=MAX_ENTRY_AGE_SECONDS, + max_messages_encrypted=MAX_MESSAGES, ) def encrypt(self, data: Union[bytes, str], *args, **kwargs) -> str: diff --git a/aws_lambda_powertools/utilities/data_masking/providers/itsdangerous.py b/aws_lambda_powertools/utilities/data_masking/providers/itsdangerous.py index afc2c36618a..c53299fe306 100644 --- a/aws_lambda_powertools/utilities/data_masking/providers/itsdangerous.py +++ b/aws_lambda_powertools/utilities/data_masking/providers/itsdangerous.py @@ -4,20 +4,50 @@ class ItsDangerousProvider(Provider): - def __init__(self, keys, salt=None): + def __init__( + self, + keys, + salt=None, + serializer=None, + serializer_kwargs=None, + signer=None, + signer_kwargs=None, + fallback_signers=None, + ): self.keys = keys self.salt = salt + self.serializer = serializer + self.serializer_kwargs = serializer_kwargs + self.signer = signer + self.signer_kwargs = signer_kwargs + self.fallback_signers = fallback_signers - def encrypt(self, data, **kwargs): + def encrypt(self, data): if data is None: return data - serialized = URLSafeSerializer(self.keys, salt=self.salt, **kwargs) + serialized = URLSafeSerializer( + self.keys, + salt=self.salt, + serializer=None, + serializer_kwargs=None, + signer=None, + signer_kwargs=None, + fallback_signers=None, + ) return serialized.dumps(data) - def decrypt(self, data, **kwargs): + def decrypt(self, data): if data is None: return data - serialized = URLSafeSerializer(self.keys, salt=self.salt, **kwargs) + serialized = URLSafeSerializer( + self.keys, + salt=self.salt, + serializer=None, + serializer_kwargs=None, + signer=None, + signer_kwargs=None, + fallback_signers=None, + ) return serialized.loads(data) From 0193ee6542fc4460a643052c628d703fc666420b Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Thu, 29 Jun 2023 20:38:22 +0000 Subject: [PATCH 18/56] Added performance benchmarking tests --- .../utilities/data_masking/base.py | 4 +- tests/performance/test_data_masking.py | 73 +++++++++++++++++++ tests/unit/data_masking/test_data_masking.py | 35 +++++---- 3 files changed, 95 insertions(+), 17 deletions(-) create mode 100644 tests/performance/test_data_masking.py diff --git a/aws_lambda_powertools/utilities/data_masking/base.py b/aws_lambda_powertools/utilities/data_masking/base.py index c57f2aafd58..82ec0e5ab62 100644 --- a/aws_lambda_powertools/utilities/data_masking/base.py +++ b/aws_lambda_powertools/utilities/data_masking/base.py @@ -22,11 +22,11 @@ def mask(self, data, fields=None, **kwargs): def _apply_action(self, data, fields, action, *args, **kwargs): if fields is not None: - return self._use_ast(data, fields, action, *args, **kwargs) + return self._apply_action_to_fields(data, fields, action, *args, **kwargs) else: return action(data, *args, **kwargs) - def _use_ast(self, data: Union[dict, str], fields, action, *args, **kwargs) -> str: + def _apply_action_to_fields(self, data: Union[dict, str], fields, action, *args, **kwargs) -> str: if fields is None: raise ValueError("No fields specified.") diff --git a/tests/performance/test_data_masking.py b/tests/performance/test_data_masking.py new file mode 100644 index 00000000000..33ebe42733f --- /dev/null +++ b/tests/performance/test_data_masking.py @@ -0,0 +1,73 @@ +import importlib +from types import ModuleType + +import pytest + +from aws_lambda_powertools.utilities.data_masking.base import DataMasking +from aws_lambda_powertools.utilities.data_masking.providers.itsdangerous import ( + ItsDangerousProvider, +) + +DATA_MASKING_PACKAGE = "aws_lambda_powertools.utilities.data_masking" +DATA_MASKING_INIT_SLA: float = 0.002 +DATA_MASKING_NESTED_ENCRYPT_SLA: float = 0.001 + +json_blob = { + "id": 1, + "name": "John Doe", + "age": 30, + "email": "johndoe@example.com", + "address": {"street": "123 Main St", "city": "Anytown", "state": "CA", "zip": "12345"}, + "phone_numbers": ["+1-555-555-1234", "+1-555-555-5678"], + "interests": ["Hiking", "Traveling", "Photography", "Reading"], + "job_history": { + "company": { + "company_name": "Acme Inc.", + "company_address": "5678 Interview Dr.", + }, + "position": "Software Engineer", + "start_date": "2015-01-01", + "end_date": "2017-12-31", + }, + "about_me": """ + Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla tincidunt velit quis + sapien mollis, at egestas massa tincidunt. Suspendisse ultrices arcu a dolor dapibus, + ut pretium turpis volutpat. Vestibulum at sapien quis sapien dignissim volutpat ut a enim. + Praesent fringilla sem eu dui convallis luctus. Donec ullamcorper, sapien ut convallis congue, + risus mauris pretium tortor, nec dignissim arcu urna a nisl. Vivamus non fermentum ex. Proin + interdum nisi id sagittis egestas. Nam sit amet nisi nec quam pharetra sagittis. Aliquam erat + volutpat. Donec nec luctus sem, nec ornare lorem. Vivamus vitae orci quis enim faucibus placerat. + Nulla facilisi. Proin in turpis orci. Donec imperdiet velit ac tellus gravida, eget laoreet tellus + malesuada. Praesent venenatis tellus ac urna blandit, at varius felis posuere. Integer a commodo nunc. + """, +} +json_blob_fields = ["address.street", "job_history.company.company_name"] + + +def import_data_masking_utility() -> ModuleType: + """Dynamically imports and return DataMasking module""" + return importlib.import_module(DATA_MASKING_PACKAGE) + + +@pytest.mark.perf +@pytest.mark.benchmark(group="core", disable_gc=True, warmup=False) +def test_data_masking_init(benchmark): + benchmark.pedantic(import_data_masking_utility) + stat = benchmark.stats.stats.max + if stat > DATA_MASKING_INIT_SLA: + pytest.fail(f"High level imports should be below {DATA_MASKING_INIT_SLA}s: {stat}") + + +def encrypt_json_blob(): + data_masker = DataMasking(provider=ItsDangerousProvider("mykey")) + encrypted = data_masker.encrypt(json_blob, json_blob_fields) + data_masker.decrypt(encrypted, json_blob_fields) + + +@pytest.mark.perf +@pytest.mark.benchmark(group="core", disable_gc=True, warmup=False) +def test_data_masking_encrypt_with_json_blob(benchmark): + benchmark.pedantic(encrypt_json_blob) + stat = benchmark.stats.stats.max + if stat > DATA_MASKING_NESTED_ENCRYPT_SLA: + pytest.fail(f"High level imports should be below {DATA_MASKING_NESTED_ENCRYPT_SLA}s: {stat}") diff --git a/tests/unit/data_masking/test_data_masking.py b/tests/unit/data_masking/test_data_masking.py index f4298929f46..c8dc6b8ed57 100644 --- a/tests/unit/data_masking/test_data_masking.py +++ b/tests/unit/data_masking/test_data_masking.py @@ -4,7 +4,7 @@ import pytest from itsdangerous.url_safe import URLSafeSerializer -from aws_lambda_powertools.shared.constants import DATA_MASKING_STRING as MASK +from aws_lambda_powertools.shared.constants import DATA_MASKING_STRING from aws_lambda_powertools.utilities.data_masking.base import DataMasking from aws_lambda_powertools.utilities.data_masking.provider import Provider from aws_lambda_powertools.utilities.data_masking.providers.aws_encryption_sdk import ( @@ -53,7 +53,9 @@ def decrypt(self, data: str) -> str: } json_dict = json.dumps(python_dict) dict_fields = ["a.1.None", "a.b.3.4"] -masked_with_fields = {"a": {"1": {"None": MASK, "four": "world"}, "b": {"3": {"4": MASK, "e": "world"}}}} +masked_with_fields = { + "a": {"1": {"None": DATA_MASKING_STRING, "four": "world"}, "b": {"3": {"4": DATA_MASKING_STRING, "e": "world"}}} +} aws_encrypted_with_fields = { "a": { "1": {"None": bytes("hello", "utf-8"), "four": "world"}, @@ -83,7 +85,7 @@ def decrypt(self, data: str) -> str: Praesent fringilla sem eu dui convallis luctus. Donec ullamcorper, sapien ut convallis congue, risus mauris pretium tortor, nec dignissim arcu urna a nisl. Vivamus non fermentum ex. Proin interdum nisi id sagittis egestas. Nam sit amet nisi nec quam pharetra sagittis. Aliquam erat - volutpat. Donec nec luctus sem, nec ornare lorem. Vivamus vitae orci quis enim faucibus placerat. + volutpat. Donec nec luctus sem, nec ornare lorem. Vivamus vitae orci quis enim faucibus placerat. Nulla facilisi. Proin in turpis orci. Donec imperdiet velit ac tellus gravida, eget laoreet tellus malesuada. Praesent venenatis tellus ac urna blandit, at varius felis posuere. Integer a commodo nunc. """, @@ -98,17 +100,20 @@ def decrypt(self, data: str) -> str: data_types_and_masks = [ # simple data types - [42, MASK], - [4.22, MASK], - [True, MASK], - [None, MASK], - ["this is a string", MASK], + [42, DATA_MASKING_STRING], + [4.22, DATA_MASKING_STRING], + [True, DATA_MASKING_STRING], + [None, DATA_MASKING_STRING], + ["this is a string", DATA_MASKING_STRING], # iterables - [[1, 2, 3, 4], [MASK, MASK, MASK, MASK]], - [["hello", 1, 2, 3, "world"], [MASK, MASK, MASK, MASK, MASK]], + [[1, 2, 3, 4], [DATA_MASKING_STRING, DATA_MASKING_STRING, DATA_MASKING_STRING, DATA_MASKING_STRING]], + [ + ["hello", 1, 2, 3, "world"], + [DATA_MASKING_STRING, DATA_MASKING_STRING, DATA_MASKING_STRING, DATA_MASKING_STRING, DATA_MASKING_STRING], + ], # dictionaries - [python_dict, MASK], - [json_dict, MASK], + [python_dict, DATA_MASKING_STRING], + [json_dict, DATA_MASKING_STRING], ] data_types = [item[0] for item in data_types_and_masks] @@ -180,7 +185,7 @@ def test_decrypt_not_implemented(): def test_aws_encryption_sdk_with_context(): data_masker = DataMasking(provider=AwsEncryptionSdkProvider(keys=[AWS_SDK_KEY])) encrypted_data = data_masker.encrypt( - str(python_dict), encryption_context={"not really": "a secret", "but adds": "some authentication"} + str(python_dict), encryption_context={"not really": "a secret", "but adds": "some auth"} ) decrypted_data = data_masker.decrypt(encrypted_data) assert decrypted_data == bytes(str(python_dict), "utf-8") @@ -201,7 +206,7 @@ def test_parsing_nonstring_fields(): } } masked = data_masker.mask(_python_dict, fields=[3.4]) - assert masked == {"3": {"1": {"None": "hello", "four": "world"}, "4": MASK}} + assert masked == {"3": {"1": {"None": "hello", "four": "world"}, "4": DATA_MASKING_STRING}} def test_parsing_nonstring_keys_and_fields(): @@ -213,4 +218,4 @@ def test_parsing_nonstring_keys_and_fields(): } } masked = data_masker.mask(_python_dict, fields=[3.4]) - assert masked == {"3": {"1": {"None": "hello", "four": "world"}, "4": MASK}} + assert masked == {"3": {"1": {"None": "hello", "four": "world"}, "4": DATA_MASKING_STRING}} From 22f0b4678d2c0ad00ffa9ee8abda9ce3c9b21c07 Mon Sep 17 00:00:00 2001 From: seshubaws <116689586+seshubaws@users.noreply.github.com> Date: Tue, 11 Jul 2023 11:33:23 -0700 Subject: [PATCH 19/56] Update aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py Co-authored-by: Roger Zhang Signed-off-by: seshubaws <116689586+seshubaws@users.noreply.github.com> --- .../utilities/data_masking/providers/aws_encryption_sdk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py index 733ab176861..53daa912111 100644 --- a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py +++ b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py @@ -33,7 +33,7 @@ def __call__(cls, *args, **kwargs): class AwsEncryptionSdkProvider(Provider, metaclass=SingletonMeta): cache = LocalCryptoMaterialsCache(CACHE_CAPACITY) session = botocore.session.Session() - +register_feature_to_botocore_session(session, "data-masking") def __init__(self, keys: List[str], client: Optional[EncryptionSDKClient] = None) -> None: self.client = client or EncryptionSDKClient() self.keys = keys From ece4643fb5f38be952a016294235e7b474de05ac Mon Sep 17 00:00:00 2001 From: seshubaws <116689586+seshubaws@users.noreply.github.com> Date: Tue, 11 Jul 2023 11:33:34 -0700 Subject: [PATCH 20/56] Update aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py Co-authored-by: Roger Zhang Signed-off-by: seshubaws <116689586+seshubaws@users.noreply.github.com> --- .../utilities/data_masking/providers/aws_encryption_sdk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py index 53daa912111..d69f4144f22 100644 --- a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py +++ b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py @@ -10,7 +10,7 @@ ) from aws_lambda_powertools.utilities.data_masking.provider import Provider - +from aws_lambda_powertools.shared.user_agent import register_feature_to_botocore_session class SingletonMeta(type): """Metaclass to cache class instances to optimize encryption""" From 82990394fbe94580d9557b0fd53271c0d9467bdb Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Wed, 2 Aug 2023 23:08:37 +0000 Subject: [PATCH 21/56] Removed args and ItsDangerous and commented on tests --- .../utilities/data_masking/base.py | 49 +++++-- .../providers/aws_encryption_sdk.py | 14 +- .../data_masking/providers/itsdangerous.py | 53 ------- tests/unit/data_masking/test_data_masking.py | 130 +++++++++++------- 4 files changed, 124 insertions(+), 122 deletions(-) delete mode 100644 aws_lambda_powertools/utilities/data_masking/providers/itsdangerous.py diff --git a/aws_lambda_powertools/utilities/data_masking/base.py b/aws_lambda_powertools/utilities/data_masking/base.py index 82ec0e5ab62..e67898fb58c 100644 --- a/aws_lambda_powertools/utilities/data_masking/base.py +++ b/aws_lambda_powertools/utilities/data_masking/base.py @@ -11,22 +11,49 @@ def __init__(self, provider=None): else: self.provider = provider - def encrypt(self, data, fields=None, **kwargs): - return self._apply_action(data, fields, self.provider.encrypt, **kwargs) + def encrypt(self, data, fields=None, **provider_options): + return self._apply_action(data, fields, self.provider.encrypt, **provider_options) - def decrypt(self, data, fields=None, **kwargs): - return self._apply_action(data, fields, self.provider.decrypt, **kwargs) + def decrypt(self, data, fields=None, **provider_options): + return self._apply_action(data, fields, self.provider.decrypt, **provider_options) - def mask(self, data, fields=None, **kwargs): - return self._apply_action(data, fields, self.provider.mask, **kwargs) + def mask(self, data, fields=None, **provider_options): + return self._apply_action(data, fields, self.provider.mask, **provider_options) - def _apply_action(self, data, fields, action, *args, **kwargs): + def _apply_action(self, data, fields, action, **provider_options): if fields is not None: - return self._apply_action_to_fields(data, fields, action, *args, **kwargs) + return self._apply_action_to_fields(data, fields, action, **provider_options) else: - return action(data, *args, **kwargs) + return action(data, **provider_options) + + def _apply_action_to_fields(self, data: Union[dict, str], fields, action, **provider_options) -> str: + """ + Apply the specified action to the specified fields in the input data. + + This method is takes the input data, which can be either a dictionary or a JSON string representation + of a dictionary, and applies a mask, an encryption, or a decryption to the specified fields. + + Parameters: + data (Union[dict, str]): The input data to process. It can be either a dictionary or a JSON string + representation of a dictionary. + fields (list): A list of fields to apply the action to. Each field can be specified as a string or + a list of strings representing nested keys in the dictionary. + action (callable): The action to apply to the fields. It should be a callable that takes the current + value of the field as the first argument and any additional arguments that might be required + for the action. It performs an operation on the current value using the provided arguments and + returns the modified value. + **provider_options: Additional keyword arguments to pass to the 'action' function. + + Returns: + str: A JSON string representation of the modified dictionary after applying the action to the + specified fields. + + Raises: + ValueError: If 'fields' parameter is None. + TypeError: If the 'data' parameter is not a dictionary or a JSON string representation of a dictionary. + KeyError: If specified 'fields' do not exist in input data + """ - def _apply_action_to_fields(self, data: Union[dict, str], fields, action, *args, **kwargs) -> str: if fields is None: raise ValueError("No fields specified.") @@ -53,6 +80,6 @@ def _apply_action_to_fields(self, data: Union[dict, str], fields, action, *args, for key in keys[:-1]: curr_dict = curr_dict[key] valtochange = curr_dict[(keys[-1])] - curr_dict[keys[-1]] = action(valtochange, *args, **kwargs) + curr_dict[keys[-1]] = action(valtochange, **provider_options) return my_dict_parsed diff --git a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py index 733ab176861..504b634ab96 100644 --- a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py +++ b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py @@ -17,9 +17,9 @@ class SingletonMeta(type): _instances: Dict["AwsEncryptionSdkProvider", Any] = {} - def __call__(cls, *args, **kwargs): + def __call__(cls, *args, **provider_options): if cls not in cls._instances: - instance = super().__call__(*args, **kwargs) + instance = super().__call__(*args, **provider_options) cls._instances[cls] = instance return cls._instances[cls] @@ -45,12 +45,14 @@ def __init__(self, keys: List[str], client: Optional[EncryptionSDKClient] = None max_messages_encrypted=MAX_MESSAGES, ) - def encrypt(self, data: Union[bytes, str], *args, **kwargs) -> str: - ciphertext, _ = self.client.encrypt(source=data, materials_manager=self.cache_cmm, *args, **kwargs) + def encrypt(self, data: Union[bytes, str], **provider_options) -> str: + ciphertext, _ = self.client.encrypt(source=data, materials_manager=self.cache_cmm, **provider_options) ciphertext = base64.b64encode(ciphertext).decode() return ciphertext - def decrypt(self, data: str, *args, **kwargs) -> bytes: + def decrypt(self, data: str, **provider_options) -> bytes: ciphertext_decoded = base64.b64decode(data) - ciphertext, _ = self.client.decrypt(source=ciphertext_decoded, key_provider=self.key_provider, *args, **kwargs) + ciphertext, _ = self.client.decrypt( + source=ciphertext_decoded, key_provider=self.key_provider, **provider_options + ) return ciphertext diff --git a/aws_lambda_powertools/utilities/data_masking/providers/itsdangerous.py b/aws_lambda_powertools/utilities/data_masking/providers/itsdangerous.py deleted file mode 100644 index c53299fe306..00000000000 --- a/aws_lambda_powertools/utilities/data_masking/providers/itsdangerous.py +++ /dev/null @@ -1,53 +0,0 @@ -from itsdangerous.url_safe import URLSafeSerializer - -from aws_lambda_powertools.utilities.data_masking.provider import Provider - - -class ItsDangerousProvider(Provider): - def __init__( - self, - keys, - salt=None, - serializer=None, - serializer_kwargs=None, - signer=None, - signer_kwargs=None, - fallback_signers=None, - ): - self.keys = keys - self.salt = salt - self.serializer = serializer - self.serializer_kwargs = serializer_kwargs - self.signer = signer - self.signer_kwargs = signer_kwargs - self.fallback_signers = fallback_signers - - def encrypt(self, data): - if data is None: - return data - - serialized = URLSafeSerializer( - self.keys, - salt=self.salt, - serializer=None, - serializer_kwargs=None, - signer=None, - signer_kwargs=None, - fallback_signers=None, - ) - return serialized.dumps(data) - - def decrypt(self, data): - if data is None: - return data - - serialized = URLSafeSerializer( - self.keys, - salt=self.salt, - serializer=None, - serializer_kwargs=None, - signer=None, - signer_kwargs=None, - fallback_signers=None, - ) - return serialized.loads(data) diff --git a/tests/unit/data_masking/test_data_masking.py b/tests/unit/data_masking/test_data_masking.py index c8dc6b8ed57..4917c26440d 100644 --- a/tests/unit/data_masking/test_data_masking.py +++ b/tests/unit/data_masking/test_data_masking.py @@ -7,14 +7,6 @@ from aws_lambda_powertools.shared.constants import DATA_MASKING_STRING from aws_lambda_powertools.utilities.data_masking.base import DataMasking from aws_lambda_powertools.utilities.data_masking.provider import Provider -from aws_lambda_powertools.utilities.data_masking.providers.aws_encryption_sdk import ( - AwsEncryptionSdkProvider, -) -from aws_lambda_powertools.utilities.data_masking.providers.itsdangerous import ( - ItsDangerousProvider, -) - -AWS_SDK_KEY = "arn:aws:kms:us-west-2:683517028648:key/269301eb-81eb-4067-ac72-98e8e49bf2b3" class MyEncryptionProvider(Provider): @@ -39,8 +31,6 @@ def decrypt(self, data: str) -> str: data_maskers = [ DataMasking(), - DataMasking(provider=ItsDangerousProvider("mykey")), - DataMasking(provider=AwsEncryptionSdkProvider(keys=[AWS_SDK_KEY])), DataMasking(provider=MyEncryptionProvider(keys="secret-key")), ] @@ -121,83 +111,109 @@ def decrypt(self, data: str) -> str: @pytest.mark.parametrize("data_masker", data_maskers) @pytest.mark.parametrize("value, value_masked", data_types_and_masks) def test_mask_types(data_masker, value, value_masked): + # GIVEN any data type + + # WHEN mask is called with no fields argument masked_string = data_masker.mask(value) + + # THEN the result is the full input data masked assert masked_string == value_masked @pytest.mark.parametrize("data_masker", data_maskers) def test_mask_with_fields(data_masker): + # GIVEN the data type is a dictionary, or a json representation of a dictionary + + # WHEN mask is called with a list of fields specified masked_string = data_masker.mask(python_dict, dict_fields) + masked_json_string = data_masker.mask(json_dict, dict_fields) + + # THEN the result is only the specified fields are masked assert masked_string == masked_with_fields - masked_string = data_masker.mask(json_dict, dict_fields) - assert masked_string == masked_with_fields + assert masked_json_string == masked_with_fields -@pytest.mark.parametrize("data_masker", data_maskers) @pytest.mark.parametrize("value", data_types) -def test_encrypt_decrypt(data_masker, value): - if data_masker == data_maskers[0]: - with pytest.raises(NotImplementedError): - encrypted_data = data_masker.encrypt(value) +def test_encrypt_decrypt(value): + # GIVEN an instantiation of DataMasking with a Provider + data_masker = DataMasking(provider=MyEncryptionProvider(keys="secret-key")) - else: - if data_masker == data_maskers[2]: - # AWS Encryption SDK encrypt method only takes in bytes or strings - value = bytes(str(value), "utf-8") + # WHEN encrypting and then decrypting the encrypted data + encrypted_data = data_masker.encrypt(value) + decrypted_data = data_masker.decrypt(encrypted_data) - encrypted_data = data_masker.encrypt(value) - decrypted_data = data_masker.decrypt(encrypted_data) - assert decrypted_data == value + # THEN the result is the original input data + assert decrypted_data == value -@pytest.mark.parametrize("data_masker", data_maskers) @pytest.mark.parametrize("value, fields", zip(dictionaries, fields_to_mask)) -def test_encrypt_decrypt_with_fields(data_masker, value, fields): - if data_masker == data_maskers[0]: - with pytest.raises(NotImplementedError): - encrypted_data = data_masker.encrypt(value) +def test_encrypt_decrypt_with_fields(value, fields): + # GIVEN an instantiation of DataMasking with a Provider + data_masker = DataMasking(provider=MyEncryptionProvider(keys="secret-key")) + + # WHEN encrypting and then decrypting the encrypted data with a list of fields + encrypted_data = data_masker.encrypt(value, fields) + decrypted_data = data_masker.decrypt(encrypted_data, fields) + # THEN the result is the original input data + if value == json_dict: + assert decrypted_data == json.loads(value) else: - encrypted_data = data_masker.encrypt(value, fields) - decrypted_data = data_masker.decrypt(encrypted_data, fields) + assert decrypted_data == value + + +def test_encrypt_not_implemented(): + # GIVEN DataMasking is not initialized with a Provider + data_masker = DataMasking() - if data_masker == data_maskers[2]: - # AWS Encryption SDK decrypt method only returns bytes - if value == json_blob: - assert decrypted_data == aws_encrypted_json_blob - else: - assert decrypted_data == aws_encrypted_with_fields + # WHEN attempting to call the encrypt method on the data - else: - if value == json_dict: - assert decrypted_data == json.loads(value) - else: - assert decrypted_data == value + # THEN the result is a NotImplementedError + with pytest.raises(NotImplementedError): + data_masker.encrypt("hello world") def test_decrypt_not_implemented(): - """Test decrypting with no Provider""" + # GIVEN DataMasking is not initialized with a Provider data_masker = DataMasking() - with pytest.raises(NotImplementedError): - data_masker.decrypt("hello world") + # WHEN attempting to call the decrypt method on the data -def test_aws_encryption_sdk_with_context(): - data_masker = DataMasking(provider=AwsEncryptionSdkProvider(keys=[AWS_SDK_KEY])) - encrypted_data = data_masker.encrypt( - str(python_dict), encryption_context={"not really": "a secret", "but adds": "some auth"} - ) - decrypted_data = data_masker.decrypt(encrypted_data) - assert decrypted_data == bytes(str(python_dict), "utf-8") + # THEN the result is a NotImplementedError + with pytest.raises(NotImplementedError): + data_masker.decrypt("hello world") def test_parsing_unsupported_data_type(): + # GIVEN an initialization of the DataMasking class data_masker = DataMasking() + + # WHEN attempting to pass in a list of fields with input data that is not a dict + + # THEN the result is a TypeError with pytest.raises(TypeError): data_masker.mask(42, ["this.field"]) +def test_parsing_nonexistent_fields(): + # GIVEN an initialization of the DataMasking class + data_masker = DataMasking() + _python_dict = { + "3": { + "1": {"None": "hello", "four": "world"}, + "4": {"33": {"5": "goodbye", "e": "world"}}, + } + } + + # WHEN attempting to pass in fields that do not exist in the input data + + # THEN the result is a KeyError + with pytest.raises(KeyError): + data_masker.mask(_python_dict, ["3.1.True"]) + + def test_parsing_nonstring_fields(): + # GIVEN an initialization of the DataMasking class data_masker = DataMasking() _python_dict = { "3": { @@ -205,17 +221,27 @@ def test_parsing_nonstring_fields(): "4": {"33": {"5": "goodbye", "e": "world"}}, } } + + # WHEN attempting to pass in a list of fields that are not strings masked = data_masker.mask(_python_dict, fields=[3.4]) + + # THEN the result is the value of the nested field should be masked as normal assert masked == {"3": {"1": {"None": "hello", "four": "world"}, "4": DATA_MASKING_STRING}} def test_parsing_nonstring_keys_and_fields(): + # GIVEN an initialization of the DataMasking class data_masker = DataMasking() + + # WHEN the input data is a dictionary with integer keys _python_dict = { 3: { "1": {"None": "hello", "four": "world"}, 4: {"33": {"5": "goodbye", "e": "world"}}, } } + masked = data_masker.mask(_python_dict, fields=[3.4]) + + # THEN the result is the value of the nested field should be masked as normal assert masked == {"3": {"1": {"None": "hello", "four": "world"}, "4": DATA_MASKING_STRING}} From 27eca179b10a8928eed6e13f7255bc53c2fdd9ca Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Wed, 9 Aug 2023 23:00:39 +0000 Subject: [PATCH 22/56] Added functional tests and put input data in separate file --- .../providers/aws_encryption_sdk.py | 4 +- tests/functional/data_masking/__init__.py | 0 .../data_masking/test_aws_encryption_sdk.py | 79 ++++++++++++ tests/unit/data_masking/setup.py | 117 ++++++++++++++++++ tests/unit/data_masking/test_data_masking.py | 105 +--------------- 5 files changed, 202 insertions(+), 103 deletions(-) create mode 100644 tests/functional/data_masking/__init__.py create mode 100644 tests/functional/data_masking/test_aws_encryption_sdk.py create mode 100644 tests/unit/data_masking/setup.py diff --git a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py index e92eedeb2fa..c57c44de755 100644 --- a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py +++ b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py @@ -12,6 +12,7 @@ from aws_lambda_powertools.utilities.data_masking.provider import Provider from aws_lambda_powertools.shared.user_agent import register_feature_to_botocore_session + class SingletonMeta(type): """Metaclass to cache class instances to optimize encryption""" @@ -33,7 +34,8 @@ def __call__(cls, *args, **provider_options): class AwsEncryptionSdkProvider(Provider, metaclass=SingletonMeta): cache = LocalCryptoMaterialsCache(CACHE_CAPACITY) session = botocore.session.Session() -register_feature_to_botocore_session(session, "data-masking") + register_feature_to_botocore_session(session, "data-masking") + def __init__(self, keys: List[str], client: Optional[EncryptionSDKClient] = None) -> None: self.client = client or EncryptionSDKClient() self.keys = keys diff --git a/tests/functional/data_masking/__init__.py b/tests/functional/data_masking/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/functional/data_masking/test_aws_encryption_sdk.py b/tests/functional/data_masking/test_aws_encryption_sdk.py new file mode 100644 index 00000000000..82a00744217 --- /dev/null +++ b/tests/functional/data_masking/test_aws_encryption_sdk.py @@ -0,0 +1,79 @@ +from unittest.mock import patch + +import pytest + +from aws_lambda_powertools.utilities.data_masking.base import DataMasking +from aws_lambda_powertools.utilities.data_masking.providers.aws_encryption_sdk import AwsEncryptionSdkProvider +from tests.unit.data_masking.setup import * + + +AWS_SDK_KEY = "arn:aws:kms:us-west-2:683517028648:key/269301eb-81eb-4067-ac72-98e8e49bf2b3" + + +@pytest.fixture +def data_masker(): + return DataMasking(provider=AwsEncryptionSdkProvider(keys=[AWS_SDK_KEY])) + + +@pytest.mark.parametrize("value, value_masked", data_types_and_masks) +def test_mask_types(value, value_masked, data_masker): + # GIVEN any data type + + # WHEN the AWS encryption provider's mask method is called with no fields argument + masked_string = data_masker.mask(value) + + # THEN the result is the full input data masked + assert masked_string == value_masked + + +def test_mask_with_fields(data_masker): + # GIVEN the data type is a dictionary, or a json representation of a dictionary + + # WHEN the AWS encryption provider's mask is called with a list of fields specified + masked_string = data_masker.mask(python_dict, dict_fields) + masked_json_string = data_masker.mask(json_dict, dict_fields) + + # THEN the result is only the specified fields are masked + assert masked_string == masked_with_fields + assert masked_json_string == masked_with_fields + + +@pytest.mark.parametrize("value", data_types) +def test_encrypt_decrypt(value, data_masker): + # GIVEN an instantiation of DataMasking with the AWS encryption provider + + # AWS Encryption SDK encrypt method only takes in bytes or strings + value = bytes(str(value), "utf-8") + + # WHEN encrypting and then decrypting the encrypted data + encrypted_data = data_masker.encrypt(value) + decrypted_data = data_masker.decrypt(encrypted_data) + + # THEN the result is the original input data + assert decrypted_data == value + + +@pytest.mark.parametrize("value, fields", zip(dictionaries, fields_to_mask)) +def test_encrypt_decrypt_with_fields(value, fields, data_masker): + # GIVEN an instantiation of DataMasking with the AWS encryption provider + + # WHEN encrypting and then decrypting the encrypted data with a list of fields + encrypted_data = data_masker.encrypt(value, fields) + decrypted_data = data_masker.decrypt(encrypted_data, fields) + + # THEN the result is the original input data + # AWS Encryption SDK decrypt method only returns bytes + if value == json_blob: + assert decrypted_data == aws_encrypted_json_blob + else: + assert decrypted_data == aws_encrypted_with_fields + + +@patch("aws_encryption_sdk.EncryptionSDKClient") +def test_mock(get_encryption_sdk_client_mock): + get_encryption_sdk_client_mock.return_value = "mock_value" + + d_m = DataMasking(provider=AwsEncryptionSdkProvider(keys=["mock_value"])) + encrypted_data = d_m.encrypt(b"secret_data") + decrypted_data = d_m.decrypt(encrypted_data) + assert decrypted_data == b"secret_data" diff --git a/tests/unit/data_masking/setup.py b/tests/unit/data_masking/setup.py new file mode 100644 index 00000000000..f2fd6bc969f --- /dev/null +++ b/tests/unit/data_masking/setup.py @@ -0,0 +1,117 @@ +import json +import copy +from itsdangerous.url_safe import URLSafeSerializer +from aws_lambda_powertools.shared.constants import DATA_MASKING_STRING +from aws_lambda_powertools.utilities.data_masking.base import DataMasking +from aws_lambda_powertools.utilities.data_masking.provider import Provider + + +class MyEncryptionProvider(Provider): + """Custom encryption provider class""" + + def __init__(self, keys, salt=None): + self.keys = keys + self.salt = salt + + def encrypt(self, data: str) -> str: + if data is None: + return data + serialize = URLSafeSerializer(self.keys) + return serialize.dumps(data) + + def decrypt(self, data: str) -> str: + if data is None: + return data + serialize = URLSafeSerializer(self.keys) + return serialize.loads(data) + + +data_maskers = [ + DataMasking(), + DataMasking(provider=MyEncryptionProvider(keys="secret-key")), +] + + +python_dict = { + "a": { + "1": {"None": "hello", "four": "world"}, # None type key doesn't work + "b": {"3": {"4": "goodbye", "e": "world"}}, # key "4.5" doesn't work + } +} + + +json_dict = json.dumps(python_dict) + + +dict_fields = ["a.1.None", "a.b.3.4"] + + +masked_with_fields = { + "a": {"1": {"None": DATA_MASKING_STRING, "four": "world"}, "b": {"3": {"4": DATA_MASKING_STRING, "e": "world"}}} +} + +aws_encrypted_with_fields = { + "a": { + "1": {"None": bytes("hello", "utf-8"), "four": "world"}, + "b": {"3": {"4": bytes("goodbye", "utf-8"), "e": "world"}}, + } +} + +# 10kb JSON blob for latency testing +json_blob = { + "id": 1, + "name": "John Doe", + "age": 30, + "email": "johndoe@example.com", + "address": {"street": "123 Main St", "city": "Anytown", "state": "CA", "zip": "12345"}, + "phone_numbers": ["+1-555-555-1234", "+1-555-555-5678"], + "interests": ["Hiking", "Traveling", "Photography", "Reading"], + "job_history": { + "company": "Acme Inc.", + "position": "Software Engineer", + "start_date": "2015-01-01", + "end_date": "2017-12-31", + }, + "about_me": """ + Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla tincidunt velit quis + sapien mollis, at egestas massa tincidunt. Suspendisse ultrices arcu a dolor dapibus, + ut pretium turpis volutpat. Vestibulum at sapien quis sapien dignissim volutpat ut a enim. + Praesent fringilla sem eu dui convallis luctus. Donec ullamcorper, sapien ut convallis congue, + risus mauris pretium tortor, nec dignissim arcu urna a nisl. Vivamus non fermentum ex. Proin + interdum nisi id sagittis egestas. Nam sit amet nisi nec quam pharetra sagittis. Aliquam erat + volutpat. Donec nec luctus sem, nec ornare lorem. Vivamus vitae orci quis enim faucibus placerat. + Nulla facilisi. Proin in turpis orci. Donec imperdiet velit ac tellus gravida, eget laoreet tellus + malesuada. Praesent venenatis tellus ac urna blandit, at varius felis posuere. Integer a commodo nunc. + """, +} + + +json_blob_fields = ["address.street", "job_history.company"] +aws_encrypted_json_blob = copy.deepcopy(json_blob) +aws_encrypted_json_blob["address"]["street"] = bytes("123 Main St", "utf-8") +aws_encrypted_json_blob["job_history"]["company"] = bytes("Acme Inc.", "utf-8") + +dictionaries = [python_dict, json_dict, json_blob] +fields_to_mask = [dict_fields, dict_fields, json_blob_fields] + + +data_types_and_masks = [ + # simple data types + [42, DATA_MASKING_STRING], + [4.22, DATA_MASKING_STRING], + [True, DATA_MASKING_STRING], + [None, DATA_MASKING_STRING], + ["this is a string", DATA_MASKING_STRING], + # iterables + [[1, 2, 3, 4], [DATA_MASKING_STRING, DATA_MASKING_STRING, DATA_MASKING_STRING, DATA_MASKING_STRING]], + [ + ["hello", 1, 2, 3, "world"], + [DATA_MASKING_STRING, DATA_MASKING_STRING, DATA_MASKING_STRING, DATA_MASKING_STRING, DATA_MASKING_STRING], + ], + # dictionaries + [python_dict, DATA_MASKING_STRING], + [json_dict, DATA_MASKING_STRING], +] + + +data_types = [item[0] for item in data_types_and_masks] diff --git a/tests/unit/data_masking/test_data_masking.py b/tests/unit/data_masking/test_data_masking.py index 4917c26440d..8e013e2e925 100644 --- a/tests/unit/data_masking/test_data_masking.py +++ b/tests/unit/data_masking/test_data_masking.py @@ -1,111 +1,12 @@ -import copy import json import pytest -from itsdangerous.url_safe import URLSafeSerializer - from aws_lambda_powertools.shared.constants import DATA_MASKING_STRING from aws_lambda_powertools.utilities.data_masking.base import DataMasking -from aws_lambda_powertools.utilities.data_masking.provider import Provider - - -class MyEncryptionProvider(Provider): - """Custom encryption provider class""" - - def __init__(self, keys, salt=None): - self.keys = keys - self.salt = salt - - def encrypt(self, data: str) -> str: - if data is None: - return data - serialize = URLSafeSerializer(self.keys) - return serialize.dumps(data) - - def decrypt(self, data: str) -> str: - if data is None: - return data - serialize = URLSafeSerializer(self.keys) - return serialize.loads(data) +from tests.unit.data_masking.setup import * - -data_maskers = [ - DataMasking(), - DataMasking(provider=MyEncryptionProvider(keys="secret-key")), -] - - -python_dict = { - "a": { - "1": {"None": "hello", "four": "world"}, # None type key doesn't work - "b": {"3": {"4": "goodbye", "e": "world"}}, # key "4.5" doesn't work - } -} -json_dict = json.dumps(python_dict) -dict_fields = ["a.1.None", "a.b.3.4"] -masked_with_fields = { - "a": {"1": {"None": DATA_MASKING_STRING, "four": "world"}, "b": {"3": {"4": DATA_MASKING_STRING, "e": "world"}}} -} -aws_encrypted_with_fields = { - "a": { - "1": {"None": bytes("hello", "utf-8"), "four": "world"}, - "b": {"3": {"4": bytes("goodbye", "utf-8"), "e": "world"}}, - } -} - -# 10kb JSON blob for latency testing -json_blob = { - "id": 1, - "name": "John Doe", - "age": 30, - "email": "johndoe@example.com", - "address": {"street": "123 Main St", "city": "Anytown", "state": "CA", "zip": "12345"}, - "phone_numbers": ["+1-555-555-1234", "+1-555-555-5678"], - "interests": ["Hiking", "Traveling", "Photography", "Reading"], - "job_history": { - "company": "Acme Inc.", - "position": "Software Engineer", - "start_date": "2015-01-01", - "end_date": "2017-12-31", - }, - "about_me": """ - Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla tincidunt velit quis - sapien mollis, at egestas massa tincidunt. Suspendisse ultrices arcu a dolor dapibus, - ut pretium turpis volutpat. Vestibulum at sapien quis sapien dignissim volutpat ut a enim. - Praesent fringilla sem eu dui convallis luctus. Donec ullamcorper, sapien ut convallis congue, - risus mauris pretium tortor, nec dignissim arcu urna a nisl. Vivamus non fermentum ex. Proin - interdum nisi id sagittis egestas. Nam sit amet nisi nec quam pharetra sagittis. Aliquam erat - volutpat. Donec nec luctus sem, nec ornare lorem. Vivamus vitae orci quis enim faucibus placerat. - Nulla facilisi. Proin in turpis orci. Donec imperdiet velit ac tellus gravida, eget laoreet tellus - malesuada. Praesent venenatis tellus ac urna blandit, at varius felis posuere. Integer a commodo nunc. - """, -} -json_blob_fields = ["address.street", "job_history.company"] -aws_encrypted_json_blob = copy.deepcopy(json_blob) -aws_encrypted_json_blob["address"]["street"] = bytes("123 Main St", "utf-8") -aws_encrypted_json_blob["job_history"]["company"] = bytes("Acme Inc.", "utf-8") - -dictionaries = [python_dict, json_dict, json_blob] -fields_to_mask = [dict_fields, dict_fields, json_blob_fields] - -data_types_and_masks = [ - # simple data types - [42, DATA_MASKING_STRING], - [4.22, DATA_MASKING_STRING], - [True, DATA_MASKING_STRING], - [None, DATA_MASKING_STRING], - ["this is a string", DATA_MASKING_STRING], - # iterables - [[1, 2, 3, 4], [DATA_MASKING_STRING, DATA_MASKING_STRING, DATA_MASKING_STRING, DATA_MASKING_STRING]], - [ - ["hello", 1, 2, 3, "world"], - [DATA_MASKING_STRING, DATA_MASKING_STRING, DATA_MASKING_STRING, DATA_MASKING_STRING, DATA_MASKING_STRING], - ], - # dictionaries - [python_dict, DATA_MASKING_STRING], - [json_dict, DATA_MASKING_STRING], -] -data_types = [item[0] for item in data_types_and_masks] +# should be conftest? no other conftest in unit tests +# didn't work when i made them all pytest.fixtures @pytest.mark.parametrize("data_masker", data_maskers) From fe37c507007f995c81e79404acfd509b4a447b05 Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Thu, 10 Aug 2023 11:53:07 -0700 Subject: [PATCH 23/56] Applied patch to update lock to latest range deps --- poetry.lock | 451 ++++++++++++++++++++-------------------------------- 1 file changed, 173 insertions(+), 278 deletions(-) diff --git a/poetry.lock b/poetry.lock index 850ab13ad31..4ca647d5721 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,10 +1,9 @@ -# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. [[package]] name = "anyio" version = "3.7.1" description = "High level compatibility layer for multiple asynchronous event loop implementations" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -27,7 +26,6 @@ trio = ["trio (<0.22)"] name = "attrs" version = "23.1.0" description = "Classes Without Boilerplate" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -65,7 +63,6 @@ typeguard = ">=2.13.3,<2.14.0" name = "aws-cdk-asset-kubectl-v20" version = "2.1.2" description = "A library that contains kubectl for use in Lambda Layers" -category = "dev" optional = false python-versions = "~=3.7" files = [ @@ -82,7 +79,6 @@ typeguard = ">=2.13.3,<2.14.0" name = "aws-cdk-asset-node-proxy-agent-v5" version = "2.0.166" description = "@aws-cdk/asset-node-proxy-agent-v5" -category = "dev" optional = false python-versions = "~=3.7" files = [ @@ -97,73 +93,69 @@ typeguard = ">=2.13.3,<2.14.0" [[package]] name = "aws-cdk-aws-apigatewayv2-alpha" -version = "2.89.0a0" +version = "2.90.0a0" description = "The CDK Construct Library for AWS::APIGatewayv2" -category = "dev" optional = false python-versions = "~=3.7" files = [ - {file = "aws-cdk.aws-apigatewayv2-alpha-2.89.0a0.tar.gz", hash = "sha256:8300431d4ef9d869066ad5dba955a6b9eca4825eb4ffcdb03d9ce34f82509d6a"}, - {file = "aws_cdk.aws_apigatewayv2_alpha-2.89.0a0-py3-none-any.whl", hash = "sha256:64a84542822bd085b03ac40e39f15c3fee1aaf649a0df34ecf0f288f7bc84c78"}, + {file = "aws-cdk.aws-apigatewayv2-alpha-2.90.0a0.tar.gz", hash = "sha256:b81e8321a2a76594fd6d79725eb6136dad844aef9122fe666b7bc5f11bb18de7"}, + {file = "aws_cdk.aws_apigatewayv2_alpha-2.90.0a0-py3-none-any.whl", hash = "sha256:154f146d5a88c602aa477869aea0109c0691437e9a0f6a9d61d98b9b52c83b51"}, ] [package.dependencies] -aws-cdk-lib = "2.89.0" +aws-cdk-lib = "2.90.0" constructs = ">=10.0.0,<11.0.0" -jsii = ">=1.85.0,<2.0.0" +jsii = ">=1.86.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" [[package]] name = "aws-cdk-aws-apigatewayv2-authorizers-alpha" -version = "2.89.0a0" +version = "2.90.0a0" description = "Authorizers for AWS APIGateway V2" -category = "dev" optional = false python-versions = "~=3.7" files = [ - {file = "aws-cdk.aws-apigatewayv2-authorizers-alpha-2.89.0a0.tar.gz", hash = "sha256:efa23f021efdca83f037569d41d7e96023c3750417fc976023688397f7f57715"}, - {file = "aws_cdk.aws_apigatewayv2_authorizers_alpha-2.89.0a0-py3-none-any.whl", hash = "sha256:7b56ea2889e8a340bfd4feb67f0798827bf58090d368763a59cd0223fe2dd916"}, + {file = "aws-cdk.aws-apigatewayv2-authorizers-alpha-2.90.0a0.tar.gz", hash = "sha256:7aeda815ca63f14362c2d08b06c8e85a4694fc07b8c7ba2a1c20b9346792c74c"}, + {file = "aws_cdk.aws_apigatewayv2_authorizers_alpha-2.90.0a0-py3-none-any.whl", hash = "sha256:ebaeaeddbdcb16b4130ce948f3d8db42254a68372b497700643bd654331207b6"}, ] [package.dependencies] -"aws-cdk.aws-apigatewayv2-alpha" = "2.89.0.a0" -aws-cdk-lib = "2.89.0" +"aws-cdk.aws-apigatewayv2-alpha" = "2.90.0.a0" +aws-cdk-lib = "2.90.0" constructs = ">=10.0.0,<11.0.0" -jsii = ">=1.85.0,<2.0.0" +jsii = ">=1.86.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" [[package]] name = "aws-cdk-aws-apigatewayv2-integrations-alpha" -version = "2.89.0a0" +version = "2.90.0a0" description = "Integrations for AWS APIGateway V2" -category = "dev" optional = false python-versions = "~=3.7" files = [ - {file = "aws-cdk.aws-apigatewayv2-integrations-alpha-2.89.0a0.tar.gz", hash = "sha256:81469d688a611d9ab10d528923692eba685cbb04a5d3401c02a4530b001a6a77"}, - {file = "aws_cdk.aws_apigatewayv2_integrations_alpha-2.89.0a0-py3-none-any.whl", hash = "sha256:3367cf5fa8e4bb1939fcd60e919af00ecc6d97a1d046938af25b9c5bef26b4c1"}, + {file = "aws-cdk.aws-apigatewayv2-integrations-alpha-2.90.0a0.tar.gz", hash = "sha256:558f12d2e951ae424f828ea8e2bfecc5c4271b4eb0c583feadd7420fdaca7cc7"}, + {file = "aws_cdk.aws_apigatewayv2_integrations_alpha-2.90.0a0-py3-none-any.whl", hash = "sha256:b26ee8a02b06f99110383dc4ad9a4542415cc5c50c35aaccc90c63d80aeeb1fd"}, ] [package.dependencies] -"aws-cdk.aws-apigatewayv2-alpha" = "2.89.0.a0" -aws-cdk-lib = "2.89.0" +"aws-cdk.aws-apigatewayv2-alpha" = "2.90.0.a0" +aws-cdk-lib = "2.90.0" constructs = ">=10.0.0,<11.0.0" -jsii = ">=1.85.0,<2.0.0" +jsii = ">=1.86.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" [[package]] name = "aws-cdk-lib" -version = "2.89.0" +version = "2.90.0" description = "Version 2 of the AWS Cloud Development Kit library" -category = "dev" optional = false python-versions = "~=3.7" files = [ - {file = "aws-cdk-lib-2.89.0.tar.gz", hash = "sha256:8fbd1d4ee0ffeb67bcc845bef5a10575dbc678ad07f74cdb3cb4243afc433db7"}, - {file = "aws_cdk_lib-2.89.0-py3-none-any.whl", hash = "sha256:92eeebd77fe17b36029fae20f46eb601710485ea7c808c3d33fe1c71fee125bd"}, + {file = "aws-cdk-lib-2.90.0.tar.gz", hash = "sha256:d99e304f96f1b04c41922cfa2fc98c1cdd7c88e45c6ebb980ecfc367cdc77e87"}, + {file = "aws_cdk_lib-2.90.0-py3-none-any.whl", hash = "sha256:ef481a40c3ece38aeaf15706ecbfeea19860b8a7b789ea7b28056a9f456c65d1"}, ] [package.dependencies] @@ -171,7 +163,7 @@ files = [ "aws-cdk.asset-kubectl-v20" = ">=2.1.2,<3.0.0" "aws-cdk.asset-node-proxy-agent-v5" = ">=2.0.166,<3.0.0" constructs = ">=10.0.0,<11.0.0" -jsii = ">=1.85.0,<2.0.0" +jsii = ">=1.86.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" @@ -179,7 +171,6 @@ typeguard = ">=2.13.3,<2.14.0" name = "aws-encryption-sdk" version = "3.1.1" description = "AWS Encryption SDK implementation for Python" -category = "main" optional = true python-versions = "*" files = [ @@ -197,7 +188,6 @@ wrapt = ">=1.10.11" name = "aws-requests-auth" version = "0.4.3" description = "AWS signature version 4 signing process for the python requests module" -category = "dev" optional = false python-versions = "*" files = [ @@ -210,30 +200,28 @@ requests = ">=0.14.0" [[package]] name = "aws-sam-translator" -version = "1.72.0" +version = "1.73.0" description = "AWS SAM Translator is a library that transform SAM templates into AWS CloudFormation templates" -category = "dev" optional = false python-versions = ">=3.7, <=4.0, !=4.0" files = [ - {file = "aws-sam-translator-1.72.0.tar.gz", hash = "sha256:e688aac30943bfe0352147b792d8bbe7c1b5ed648747cd7ef6280875b249e2d8"}, - {file = "aws_sam_translator-1.72.0-py3-none-any.whl", hash = "sha256:69fe3914d61ae6690034c3fc1055743e5415d83c59c35ec5ec9ceb26cc65c8a1"}, + {file = "aws-sam-translator-1.73.0.tar.gz", hash = "sha256:bfa7cad3a78f002edeec5e39fd61b616cf84f34f61010c5dc2f7a76845fe7a02"}, + {file = "aws_sam_translator-1.73.0-py3-none-any.whl", hash = "sha256:c0132b065d743773fcd2573ed1ae60e0129fa46043fad76430261b098a811924"}, ] [package.dependencies] -boto3 = ">=1.19.5,<2.0.0" +boto3 = ">=1.19.5,<2.dev0" jsonschema = ">=3.2,<5" pydantic = ">=1.8,<2.0" typing-extensions = ">=4.4,<5" [package.extras] -dev = ["black (==23.1.0)", "boto3 (>=1.23,<2)", "boto3-stubs[appconfig,serverlessrepo] (>=1.19.5,<2.0.0)", "coverage (>=5.3,<8)", "dateparser (>=1.1,<2.0)", "importlib-metadata", "mypy (>=1.1.0,<1.2.0)", "parameterized (>=0.7,<1.0)", "pytest (>=6.2,<8)", "pytest-cov (>=2.10,<5)", "pytest-env (>=0.6,<1)", "pytest-rerunfailures (>=9.1,<12)", "pytest-xdist (>=2.5,<4)", "pyyaml (>=6.0,<7.0)", "requests (>=2.28,<3.0)", "ruamel.yaml (==0.17.21)", "ruff (==0.0.263)", "tenacity (>=8.0,<9.0)", "types-PyYAML (>=6.0,<7.0)", "types-jsonschema (>=3.2,<4.0)"] +dev = ["black (==23.1.0)", "boto3 (>=1.23,<2)", "boto3-stubs[appconfig,serverlessrepo] (>=1.19.5,<2.dev0)", "coverage (>=5.3,<8)", "dateparser (>=1.1,<2.0)", "importlib-metadata", "mypy (>=1.1.0,<1.2.0)", "parameterized (>=0.7,<1.0)", "pytest (>=6.2,<8)", "pytest-cov (>=2.10,<5)", "pytest-env (>=0.6,<1)", "pytest-rerunfailures (>=9.1,<12)", "pytest-xdist (>=2.5,<4)", "pyyaml (>=6.0,<7.0)", "requests (>=2.28,<3.0)", "ruamel.yaml (==0.17.21)", "ruff (==0.0.263)", "tenacity (>=8.0,<9.0)", "types-PyYAML (>=6.0,<7.0)", "types-jsonschema (>=3.2,<4.0)"] [[package]] name = "aws-xray-sdk" version = "2.12.0" description = "The AWS X-Ray SDK for Python (the SDK) enables Python developers to record and emit information from within their applications to the AWS X-Ray service." -category = "main" optional = true python-versions = "*" files = [ @@ -249,7 +237,6 @@ wrapt = "*" name = "bandit" version = "1.7.5" description = "Security oriented static analyser for python code." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -273,7 +260,6 @@ yaml = ["PyYAML"] name = "black" version = "23.3.0" description = "The uncompromising code formatter." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -322,18 +308,17 @@ uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "boto3" -version = "1.28.16" +version = "1.28.23" description = "The AWS SDK for Python" -category = "main" optional = false python-versions = ">= 3.7" files = [ - {file = "boto3-1.28.16-py3-none-any.whl", hash = "sha256:d8e31f69fb919025a5961f8fbeb51fe92e2f753beb37fc1853138667a231cdaa"}, - {file = "boto3-1.28.16.tar.gz", hash = "sha256:aea48aedf3e8676e598e3202e732295064a4fcad5f2d2d2a699368b8c3ab492c"}, + {file = "boto3-1.28.23-py3-none-any.whl", hash = "sha256:807d4a4698ba9a76d5901a1663ff1943d13efbc388908f38b60f209c3511f1d6"}, + {file = "boto3-1.28.23.tar.gz", hash = "sha256:839deb868d1278dd5a3f87208cfc4a8e259c95ca3cbe607cc322d435f02f63b0"}, ] [package.dependencies] -botocore = ">=1.31.16,<1.32.0" +botocore = ">=1.31.23,<1.32.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.6.0,<0.7.0" @@ -342,14 +327,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.31.16" +version = "1.31.23" description = "Low-level, data-driven core of boto 3." -category = "main" optional = false python-versions = ">= 3.7" files = [ - {file = "botocore-1.31.16-py3-none-any.whl", hash = "sha256:92b240e2cb7b3afae5361651d2f48ee582f45d2dab53aef76eef7eec1d3ce582"}, - {file = "botocore-1.31.16.tar.gz", hash = "sha256:563e15979e763b93d78de58d0fc065f8615be12f41bab42f5ad9f412b6a224b3"}, + {file = "botocore-1.31.23-py3-none-any.whl", hash = "sha256:d0a95f74eb6bd99e8f52f16af0a430ba6cd1526744f40ffdd3fcccceeaf961c2"}, + {file = "botocore-1.31.23.tar.gz", hash = "sha256:f3258feaebce48f138eb2675168c4d33cc3d99e9f45af13cb8de47bdc2b9c573"}, ] [package.dependencies] @@ -364,7 +348,6 @@ crt = ["awscrt (==0.16.26)"] name = "cattrs" version = "23.1.2" description = "Composable complex class support for attrs and dataclasses." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -390,7 +373,6 @@ ujson = ["ujson (>=5.4.0,<6.0.0)"] name = "certifi" version = "2023.7.22" description = "Python package for providing Mozilla's CA Bundle." -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -402,7 +384,6 @@ files = [ name = "cffi" version = "1.15.1" description = "Foreign Function Interface for Python calling C code." -category = "main" optional = true python-versions = "*" files = [ @@ -479,7 +460,6 @@ pycparser = "*" name = "cfn-lint" version = "0.79.6" description = "Checks CloudFormation templates for practices and behaviour that could potentially be improved" -category = "dev" optional = false python-versions = ">=3.7, <=4.0, !=4.0" files = [ @@ -503,7 +483,6 @@ sympy = ">=1.0.0" name = "charset-normalizer" version = "3.2.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -category = "dev" optional = false python-versions = ">=3.7.0" files = [ @@ -588,7 +567,6 @@ files = [ name = "checksumdir" version = "1.2.0" description = "Compute a single hash of the file contents of a directory." -category = "dev" optional = false python-versions = ">=3.6,<4.0" files = [ @@ -600,7 +578,6 @@ files = [ name = "click" version = "8.1.6" description = "Composable command line interface toolkit" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -616,7 +593,6 @@ importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -628,7 +604,6 @@ files = [ name = "constructs" version = "10.2.69" description = "A programming model for software-defined state" -category = "dev" optional = false python-versions = "~=3.7" files = [ @@ -645,7 +620,6 @@ typeguard = ">=2.13.3,<2.14.0" name = "coverage" version = "7.2.7" description = "Code coverage measurement for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -719,31 +693,34 @@ toml = ["tomli"] [[package]] name = "cryptography" -version = "41.0.1" +version = "41.0.3" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." -category = "main" optional = true python-versions = ">=3.7" files = [ - {file = "cryptography-41.0.1-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:f73bff05db2a3e5974a6fd248af2566134d8981fd7ab012e5dd4ddb1d9a70699"}, - {file = "cryptography-41.0.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:1a5472d40c8f8e91ff7a3d8ac6dfa363d8e3138b961529c996f3e2df0c7a411a"}, - {file = "cryptography-41.0.1-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7fa01527046ca5facdf973eef2535a27fec4cb651e4daec4d043ef63f6ecd4ca"}, - {file = "cryptography-41.0.1-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b46e37db3cc267b4dea1f56da7346c9727e1209aa98487179ee8ebed09d21e43"}, - {file = "cryptography-41.0.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d198820aba55660b4d74f7b5fd1f17db3aa5eb3e6893b0a41b75e84e4f9e0e4b"}, - {file = "cryptography-41.0.1-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:948224d76c4b6457349d47c0c98657557f429b4e93057cf5a2f71d603e2fc3a3"}, - {file = "cryptography-41.0.1-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:059e348f9a3c1950937e1b5d7ba1f8e968508ab181e75fc32b879452f08356db"}, - {file = "cryptography-41.0.1-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:b4ceb5324b998ce2003bc17d519080b4ec8d5b7b70794cbd2836101406a9be31"}, - {file = "cryptography-41.0.1-cp37-abi3-win32.whl", hash = "sha256:8f4ab7021127a9b4323537300a2acfb450124b2def3756f64dc3a3d2160ee4b5"}, - {file = "cryptography-41.0.1-cp37-abi3-win_amd64.whl", hash = "sha256:1fee5aacc7367487b4e22484d3c7e547992ed726d14864ee33c0176ae43b0d7c"}, - {file = "cryptography-41.0.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:9a6c7a3c87d595608a39980ebaa04d5a37f94024c9f24eb7d10262b92f739ddb"}, - {file = "cryptography-41.0.1-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:5d092fdfedaec4cbbffbf98cddc915ba145313a6fdaab83c6e67f4e6c218e6f3"}, - {file = "cryptography-41.0.1-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:1a8e6c2de6fbbcc5e14fd27fb24414507cb3333198ea9ab1258d916f00bc3039"}, - {file = "cryptography-41.0.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:cb33ccf15e89f7ed89b235cff9d49e2e62c6c981a6061c9c8bb47ed7951190bc"}, - {file = "cryptography-41.0.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5f0ff6e18d13a3de56f609dd1fd11470918f770c6bd5d00d632076c727d35485"}, - {file = "cryptography-41.0.1-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:7bfc55a5eae8b86a287747053140ba221afc65eb06207bedf6e019b8934b477c"}, - {file = "cryptography-41.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:eb8163f5e549a22888c18b0d53d6bb62a20510060a22fd5a995ec8a05268df8a"}, - {file = "cryptography-41.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:8dde71c4169ec5ccc1087bb7521d54251c016f126f922ab2dfe6649170a3b8c5"}, - {file = "cryptography-41.0.1.tar.gz", hash = "sha256:d34579085401d3f49762d2f7d6634d6b6c2ae1242202e860f4d26b046e3a1006"}, + {file = "cryptography-41.0.3-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:652627a055cb52a84f8c448185922241dd5217443ca194d5739b44612c5e6507"}, + {file = "cryptography-41.0.3-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:8f09daa483aedea50d249ef98ed500569841d6498aa9c9f4b0531b9964658922"}, + {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4fd871184321100fb400d759ad0cddddf284c4b696568204d281c902fc7b0d81"}, + {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84537453d57f55a50a5b6835622ee405816999a7113267739a1b4581f83535bd"}, + {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3fb248989b6363906827284cd20cca63bb1a757e0a2864d4c1682a985e3dca47"}, + {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:42cb413e01a5d36da9929baa9d70ca90d90b969269e5a12d39c1e0d475010116"}, + {file = "cryptography-41.0.3-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:aeb57c421b34af8f9fe830e1955bf493a86a7996cc1338fe41b30047d16e962c"}, + {file = "cryptography-41.0.3-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:6af1c6387c531cd364b72c28daa29232162010d952ceb7e5ca8e2827526aceae"}, + {file = "cryptography-41.0.3-cp37-abi3-win32.whl", hash = "sha256:0d09fb5356f975974dbcb595ad2d178305e5050656affb7890a1583f5e02a306"}, + {file = "cryptography-41.0.3-cp37-abi3-win_amd64.whl", hash = "sha256:a983e441a00a9d57a4d7c91b3116a37ae602907a7618b882c8013b5762e80574"}, + {file = "cryptography-41.0.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5259cb659aa43005eb55a0e4ff2c825ca111a0da1814202c64d28a985d33b087"}, + {file = "cryptography-41.0.3-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:67e120e9a577c64fe1f611e53b30b3e69744e5910ff3b6e97e935aeb96005858"}, + {file = "cryptography-41.0.3-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:7efe8041897fe7a50863e51b77789b657a133c75c3b094e51b5e4b5cec7bf906"}, + {file = "cryptography-41.0.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ce785cf81a7bdade534297ef9e490ddff800d956625020ab2ec2780a556c313e"}, + {file = "cryptography-41.0.3-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:57a51b89f954f216a81c9d057bf1a24e2f36e764a1ca9a501a6964eb4a6800dd"}, + {file = "cryptography-41.0.3-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:4c2f0d35703d61002a2bbdcf15548ebb701cfdd83cdc12471d2bae80878a4207"}, + {file = "cryptography-41.0.3-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:23c2d778cf829f7d0ae180600b17e9fceea3c2ef8b31a99e3c694cbbf3a24b84"}, + {file = "cryptography-41.0.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:95dd7f261bb76948b52a5330ba5202b91a26fbac13ad0e9fc8a3ac04752058c7"}, + {file = "cryptography-41.0.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:41d7aa7cdfded09b3d73a47f429c298e80796c8e825ddfadc84c8a7f12df212d"}, + {file = "cryptography-41.0.3-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d0d651aa754ef58d75cec6edfbd21259d93810b73f6ec246436a21b7841908de"}, + {file = "cryptography-41.0.3-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:ab8de0d091acbf778f74286f4989cf3d1528336af1b59f3e5d2ebca8b5fe49e1"}, + {file = "cryptography-41.0.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a74fbcdb2a0d46fe00504f571a2a540532f4c188e6ccf26f1f178480117b33c4"}, + {file = "cryptography-41.0.3.tar.gz", hash = "sha256:6d192741113ef5e30d89dcb5b956ef4e1578f304708701b8b73d38e3e1461f34"}, ] [package.dependencies] @@ -763,7 +740,6 @@ test-randomorder = ["pytest-randomly"] name = "decorator" version = "5.1.1" description = "Decorators for Humans" -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -775,7 +751,6 @@ files = [ name = "exceptiongroup" version = "1.1.2" description = "Backport of PEP 654 (exception groups)" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -790,7 +765,6 @@ test = ["pytest (>=6)"] name = "execnet" version = "2.0.2" description = "execnet: rapid multi-Python deployment" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -805,7 +779,6 @@ testing = ["hatch", "pre-commit", "pytest", "tox"] name = "fastjsonschema" version = "2.18.0" description = "Fastest Python implementation of JSON schema" -category = "main" optional = true python-versions = "*" files = [ @@ -820,7 +793,6 @@ devel = ["colorama", "json-spec", "jsonschema", "pylint", "pytest", "pytest-benc name = "filelock" version = "3.12.2" description = "A platform independent file lock." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -836,7 +808,6 @@ testing = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "diff-cover (>=7.5)", "p name = "future" version = "0.18.3" description = "Clean single-source support for Python 3 and 2" -category = "dev" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -847,7 +818,6 @@ files = [ name = "ghp-import" version = "2.1.0" description = "Copy your docs directly to the gh-pages branch." -category = "dev" optional = false python-versions = "*" files = [ @@ -865,7 +835,6 @@ dev = ["flake8", "markdown", "twine", "wheel"] name = "gitdb" version = "4.0.10" description = "Git Object Database" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -880,7 +849,6 @@ smmap = ">=3.0.1,<6" name = "gitpython" version = "3.1.32" description = "GitPython is a Python library used to interact with Git repositories" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -896,7 +864,6 @@ typing-extensions = {version = ">=3.7.4.3", markers = "python_version < \"3.8\"" name = "h11" version = "0.14.0" description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -911,7 +878,6 @@ typing-extensions = {version = "*", markers = "python_version < \"3.8\""} name = "httpcore" version = "0.17.3" description = "A minimal low-level HTTP client." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -923,17 +889,16 @@ files = [ anyio = ">=3.0,<5.0" certifi = "*" h11 = ">=0.13,<0.15" -sniffio = ">=1.0.0,<2.0.0" +sniffio = "==1.*" [package.extras] http2 = ["h2 (>=3,<5)"] -socks = ["socksio (>=1.0.0,<2.0.0)"] +socks = ["socksio (==1.*)"] [[package]] name = "httpx" version = "0.24.1" description = "The next generation HTTP client." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -949,15 +914,14 @@ sniffio = "*" [package.extras] brotli = ["brotli", "brotlicffi"] -cli = ["click (>=8.0.0,<9.0.0)", "pygments (>=2.0.0,<3.0.0)", "rich (>=10,<14)"] +cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] http2 = ["h2 (>=3,<5)"] -socks = ["socksio (>=1.0.0,<2.0.0)"] +socks = ["socksio (==1.*)"] [[package]] name = "hvac" version = "1.1.1" description = "HashiCorp Vault API client" -category = "dev" optional = false python-versions = ">=3.6.2,<4.0.0" files = [ @@ -973,7 +937,6 @@ requests = ">=2.27.1,<3.0.0" name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -985,7 +948,6 @@ files = [ name = "ijson" version = "3.2.3" description = "Iterative JSON parser with standard Python iterator interfaces" -category = "dev" optional = false python-versions = "*" files = [ @@ -1073,7 +1035,6 @@ files = [ name = "importlib-metadata" version = "6.7.0" description = "Read metadata from Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1094,7 +1055,6 @@ testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs name = "importlib-resources" version = "5.12.0" description = "Read resources from Python packages" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1113,7 +1073,6 @@ testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-chec name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1125,7 +1084,6 @@ files = [ name = "isort" version = "5.11.5" description = "A Python utility / library to sort Python imports." -category = "dev" optional = false python-versions = ">=3.7.0" files = [ @@ -1143,7 +1101,6 @@ requirements-deprecated-finder = ["pip-api", "pipreqs"] name = "itsdangerous" version = "2.1.2" description = "Safely pass data to untrusted environments and back." -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1155,7 +1112,6 @@ files = [ name = "jinja2" version = "3.1.2" description = "A very fast and expressive template engine." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1173,7 +1129,6 @@ i18n = ["Babel (>=2.7)"] name = "jmespath" version = "1.0.1" description = "JSON Matching Expressions" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1185,7 +1140,6 @@ files = [ name = "jschema-to-python" version = "1.2.3" description = "Generate source code for Python classes from a JSON schema." -category = "dev" optional = false python-versions = ">= 2.7" files = [ @@ -1200,14 +1154,13 @@ pbr = "*" [[package]] name = "jsii" -version = "1.85.0" +version = "1.86.1" description = "Python client for jsii runtime" -category = "dev" optional = false python-versions = "~=3.7" files = [ - {file = "jsii-1.85.0-py3-none-any.whl", hash = "sha256:379feb1a1a3c4e449307564f42a7cddef05e43760cbfbbfe8434f6448cd668a0"}, - {file = "jsii-1.85.0.tar.gz", hash = "sha256:b77194cf053c06c6bdffc887a4d1d2a41113c6f4780a7d78d70a780a70998008"}, + {file = "jsii-1.86.1-py3-none-any.whl", hash = "sha256:32eb46ed4c9a35bc92b892ef049ed1996f13be38ffef964d607e8fe930471b3e"}, + {file = "jsii-1.86.1.tar.gz", hash = "sha256:44f9a820eea92c9508693f72d3129b5a080421c949c32303f4f7b2cc98a81f59"}, ] [package.dependencies] @@ -1223,7 +1176,6 @@ typing-extensions = ">=3.7,<5.0" name = "jsonpatch" version = "1.33" description = "Apply JSON-Patches (RFC 6902)" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" files = [ @@ -1238,7 +1190,6 @@ jsonpointer = ">=1.9" name = "jsonpickle" version = "3.0.1" description = "Python library for serializing any arbitrary object graph into JSON" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1258,7 +1209,6 @@ testing-libs = ["simplejson", "ujson"] name = "jsonpointer" version = "2.4" description = "Identify specific nodes in a JSON document (RFC 6901)" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" files = [ @@ -1270,7 +1220,6 @@ files = [ name = "jsonschema" version = "4.17.3" description = "An implementation of JSON Schema validation for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1294,7 +1243,6 @@ format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339- name = "junit-xml" version = "1.9" description = "Creates JUnit XML test result documents that can be read by tools such as Jenkins" -category = "dev" optional = false python-versions = "*" files = [ @@ -1309,7 +1257,6 @@ six = "*" name = "mako" version = "1.2.4" description = "A super-fast templating language that borrows the best ideas from the existing templating languages." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1330,7 +1277,6 @@ testing = ["pytest"] name = "mando" version = "0.6.4" description = "Create Python CLI apps with little to no effort at all!" -category = "dev" optional = false python-versions = "*" files = [ @@ -1366,7 +1312,6 @@ testing = ["coverage", "pyyaml"] name = "markdown-it-py" version = "2.2.0" description = "Python port of markdown-it. Markdown parsing, done right!" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1392,7 +1337,6 @@ testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] name = "markupsafe" version = "2.1.3" description = "Safely add untrusted strings to HTML/XML markup." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1452,7 +1396,6 @@ files = [ name = "mdurl" version = "0.1.2" description = "Markdown URL utilities" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1464,7 +1407,6 @@ files = [ name = "mergedeep" version = "1.3.4" description = "A deep merge function for 🐍." -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1476,7 +1418,6 @@ files = [ name = "mike" version = "1.1.2" description = "Manage multiple versions of your MkDocs-powered documentation" -category = "dev" optional = false python-versions = "*" files = [ @@ -1496,14 +1437,13 @@ test = ["coverage", "flake8 (>=3.0)", "shtab"] [[package]] name = "mkdocs" -version = "1.5.1" +version = "1.5.2" description = "Project documentation with Markdown." -category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "mkdocs-1.5.1-py3-none-any.whl", hash = "sha256:67e889f8d8ba1fe5decdfc59f5f8f21d6a8925a129339e93dede303bdea03a98"}, - {file = "mkdocs-1.5.1.tar.gz", hash = "sha256:f2f323c62fffdf1b71b84849e39aef56d6852b3f0a5571552bca32cefc650209"}, + {file = "mkdocs-1.5.2-py3-none-any.whl", hash = "sha256:60a62538519c2e96fe8426654a67ee177350451616118a41596ae7c876bb7eac"}, + {file = "mkdocs-1.5.2.tar.gz", hash = "sha256:70d0da09c26cff288852471be03c23f0f521fc15cf16ac89c7a3bfb9ae8d24f9"}, ] [package.dependencies] @@ -1531,7 +1471,6 @@ min-versions = ["babel (==2.9.0)", "click (==7.0)", "colorama (==0.4)", "ghp-imp name = "mkdocs-git-revision-date-plugin" version = "0.3.2" description = "MkDocs plugin for setting revision date from git per markdown file." -category = "dev" optional = false python-versions = ">=3.4" files = [ @@ -1547,7 +1486,6 @@ mkdocs = ">=0.17" name = "mkdocs-material" version = "9.1.21" description = "Documentation that simply works" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1570,7 +1508,6 @@ requests = ">=2.26" name = "mkdocs-material-extensions" version = "1.1.1" description = "Extension pack for Python Markdown and MkDocs Material." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1582,7 +1519,6 @@ files = [ name = "mpmath" version = "1.3.0" description = "Python library for arbitrary-precision floating-point arithmetic" -category = "dev" optional = false python-versions = "*" files = [ @@ -1600,7 +1536,6 @@ tests = ["pytest (>=4.6)"] name = "mypy" version = "1.4.1" description = "Optional static typing for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1802,7 +1737,6 @@ typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.9\""} name = "mypy-extensions" version = "1.0.0" description = "Type system extensions for programs checked with the mypy type checker." -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -1814,7 +1748,6 @@ files = [ name = "networkx" version = "2.6.3" description = "Python package for creating and manipulating graphs and networks" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1833,7 +1766,6 @@ test = ["codecov (>=2.1)", "pytest (>=6.2)", "pytest-cov (>=2.12)"] name = "packaging" version = "23.1" description = "Core utilities for Python packages" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1845,7 +1777,6 @@ files = [ name = "pathspec" version = "0.11.2" description = "Utility library for gitignore style pattern matching of file paths." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1857,7 +1788,6 @@ files = [ name = "pbr" version = "5.11.1" description = "Python Build Reasonableness" -category = "dev" optional = false python-versions = ">=2.6" files = [ @@ -1869,7 +1799,6 @@ files = [ name = "pdoc3" version = "0.10.0" description = "Auto-generate API documentation for Python projects." -category = "dev" optional = false python-versions = ">= 3.6" files = [ @@ -1885,7 +1814,6 @@ markdown = ">=3.0" name = "pkgutil-resolve-name" version = "1.3.10" description = "Resolve a name to an object." -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1897,7 +1825,6 @@ files = [ name = "platformdirs" version = "3.10.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1916,7 +1843,6 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-co name = "pluggy" version = "1.2.0" description = "plugin and hook calling mechanisms for python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1935,7 +1861,6 @@ testing = ["pytest", "pytest-benchmark"] name = "publication" version = "0.0.3" description = "Publication helps you maintain public-api-friendly modules by preventing unintentional access to private implementation details via introspection." -category = "dev" optional = false python-versions = "*" files = [ @@ -1947,7 +1872,6 @@ files = [ name = "py-cpuinfo" version = "9.0.0" description = "Get CPU info with pure Python" -category = "dev" optional = false python-versions = "*" files = [ @@ -1955,11 +1879,21 @@ files = [ {file = "py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5"}, ] +[[package]] +name = "pycparser" +version = "2.21" +description = "C parser in Python" +optional = true +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, + {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, +] + [[package]] name = "pydantic" version = "1.10.12" description = "Data validation and settings management using python type hints" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2010,14 +1944,13 @@ email = ["email-validator (>=1.0.3)"] [[package]] name = "pygments" -version = "2.15.1" +version = "2.16.1" description = "Pygments is a syntax highlighting package written in Python." -category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "Pygments-2.15.1-py3-none-any.whl", hash = "sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1"}, - {file = "Pygments-2.15.1.tar.gz", hash = "sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c"}, + {file = "Pygments-2.16.1-py3-none-any.whl", hash = "sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692"}, + {file = "Pygments-2.16.1.tar.gz", hash = "sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29"}, ] [package.extras] @@ -2027,7 +1960,6 @@ plugins = ["importlib-metadata"] name = "pyhcl" version = "0.4.4" description = "HCL configuration parser for python" -category = "dev" optional = false python-versions = "*" files = [ @@ -2038,7 +1970,6 @@ files = [ name = "pymdown-extensions" version = "10.1" description = "Extension pack for Python Markdown." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2054,7 +1985,6 @@ pyyaml = "*" name = "pyrsistent" version = "0.19.3" description = "Persistent/Functional/Immutable data structures" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2091,7 +2021,6 @@ files = [ name = "pytest" version = "7.4.0" description = "pytest: simple powerful testing with Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2115,7 +2044,6 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no name = "pytest-asyncio" version = "0.21.1" description = "Pytest support for asyncio" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2135,7 +2063,6 @@ testing = ["coverage (>=6.2)", "flaky (>=3.5.0)", "hypothesis (>=5.7.1)", "mypy name = "pytest-benchmark" version = "4.0.0" description = "A ``pytest`` fixture for benchmarking code. It will group the tests into rounds that are calibrated to the chosen timer." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2156,7 +2083,6 @@ histogram = ["pygal", "pygaljs"] name = "pytest-cov" version = "4.1.0" description = "Pytest plugin for measuring coverage." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2175,7 +2101,6 @@ testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtuale name = "pytest-mock" version = "3.11.1" description = "Thin-wrapper around the mock package for easier use with pytest" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2193,7 +2118,6 @@ dev = ["pre-commit", "pytest-asyncio", "tox"] name = "pytest-xdist" version = "3.3.1" description = "pytest xdist plugin for distributed testing, most importantly across multiple CPUs" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2214,7 +2138,6 @@ testing = ["filelock"] name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -2229,7 +2152,6 @@ six = ">=1.5" name = "pyyaml" version = "6.0.1" description = "YAML parser and emitter for Python" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -2279,7 +2201,6 @@ files = [ name = "pyyaml-env-tag" version = "0.1" description = "A custom YAML tag for referencing environment variables in YAML files. " -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -2294,7 +2215,6 @@ pyyaml = "*" name = "radon" version = "5.1.0" description = "Code Metrics in Python" -category = "dev" optional = false python-versions = "*" files = [ @@ -2309,107 +2229,105 @@ mando = ">=0.6,<0.7" [[package]] name = "regex" -version = "2023.6.3" +version = "2023.8.8" description = "Alternative regular expression module, to replace re." -category = "dev" optional = false python-versions = ">=3.6" files = [ - {file = "regex-2023.6.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:824bf3ac11001849aec3fa1d69abcb67aac3e150a933963fb12bda5151fe1bfd"}, - {file = "regex-2023.6.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:05ed27acdf4465c95826962528f9e8d41dbf9b1aa8531a387dee6ed215a3e9ef"}, - {file = "regex-2023.6.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b49c764f88a79160fa64f9a7b425620e87c9f46095ef9c9920542ab2495c8bc"}, - {file = "regex-2023.6.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8e3f1316c2293e5469f8f09dc2d76efb6c3982d3da91ba95061a7e69489a14ef"}, - {file = "regex-2023.6.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:43e1dd9d12df9004246bacb79a0e5886b3b6071b32e41f83b0acbf293f820ee8"}, - {file = "regex-2023.6.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4959e8bcbfda5146477d21c3a8ad81b185cd252f3d0d6e4724a5ef11c012fb06"}, - {file = "regex-2023.6.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:af4dd387354dc83a3bff67127a124c21116feb0d2ef536805c454721c5d7993d"}, - {file = "regex-2023.6.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2239d95d8e243658b8dbb36b12bd10c33ad6e6933a54d36ff053713f129aa536"}, - {file = "regex-2023.6.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:890e5a11c97cf0d0c550eb661b937a1e45431ffa79803b942a057c4fb12a2da2"}, - {file = "regex-2023.6.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a8105e9af3b029f243ab11ad47c19b566482c150c754e4c717900a798806b222"}, - {file = "regex-2023.6.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:25be746a8ec7bc7b082783216de8e9473803706723b3f6bef34b3d0ed03d57e2"}, - {file = "regex-2023.6.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:3676f1dd082be28b1266c93f618ee07741b704ab7b68501a173ce7d8d0d0ca18"}, - {file = "regex-2023.6.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:10cb847aeb1728412c666ab2e2000ba6f174f25b2bdc7292e7dd71b16db07568"}, - {file = "regex-2023.6.3-cp310-cp310-win32.whl", hash = "sha256:dbbbfce33cd98f97f6bffb17801b0576e653f4fdb1d399b2ea89638bc8d08ae1"}, - {file = "regex-2023.6.3-cp310-cp310-win_amd64.whl", hash = "sha256:c5f8037000eb21e4823aa485149f2299eb589f8d1fe4b448036d230c3f4e68e0"}, - {file = "regex-2023.6.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c123f662be8ec5ab4ea72ea300359023a5d1df095b7ead76fedcd8babbedf969"}, - {file = "regex-2023.6.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9edcbad1f8a407e450fbac88d89e04e0b99a08473f666a3f3de0fd292badb6aa"}, - {file = "regex-2023.6.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dcba6dae7de533c876255317c11f3abe4907ba7d9aa15d13e3d9710d4315ec0e"}, - {file = "regex-2023.6.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:29cdd471ebf9e0f2fb3cac165efedc3c58db841d83a518b082077e612d3ee5df"}, - {file = "regex-2023.6.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:12b74fbbf6cbbf9dbce20eb9b5879469e97aeeaa874145517563cca4029db65c"}, - {file = "regex-2023.6.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c29ca1bd61b16b67be247be87390ef1d1ef702800f91fbd1991f5c4421ebae8"}, - {file = "regex-2023.6.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d77f09bc4b55d4bf7cc5eba785d87001d6757b7c9eec237fe2af57aba1a071d9"}, - {file = "regex-2023.6.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ea353ecb6ab5f7e7d2f4372b1e779796ebd7b37352d290096978fea83c4dba0c"}, - {file = "regex-2023.6.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:10590510780b7541969287512d1b43f19f965c2ece6c9b1c00fc367b29d8dce7"}, - {file = "regex-2023.6.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e2fbd6236aae3b7f9d514312cdb58e6494ee1c76a9948adde6eba33eb1c4264f"}, - {file = "regex-2023.6.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:6b2675068c8b56f6bfd5a2bda55b8accbb96c02fd563704732fd1c95e2083461"}, - {file = "regex-2023.6.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:74419d2b50ecb98360cfaa2974da8689cb3b45b9deff0dcf489c0d333bcc1477"}, - {file = "regex-2023.6.3-cp311-cp311-win32.whl", hash = "sha256:fb5ec16523dc573a4b277663a2b5a364e2099902d3944c9419a40ebd56a118f9"}, - {file = "regex-2023.6.3-cp311-cp311-win_amd64.whl", hash = "sha256:09e4a1a6acc39294a36b7338819b10baceb227f7f7dbbea0506d419b5a1dd8af"}, - {file = "regex-2023.6.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:0654bca0cdf28a5956c83839162692725159f4cda8d63e0911a2c0dc76166525"}, - {file = "regex-2023.6.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:463b6a3ceb5ca952e66550a4532cef94c9a0c80dc156c4cc343041951aec1697"}, - {file = "regex-2023.6.3-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:87b2a5bb5e78ee0ad1de71c664d6eb536dc3947a46a69182a90f4410f5e3f7dd"}, - {file = "regex-2023.6.3-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6343c6928282c1f6a9db41f5fd551662310e8774c0e5ebccb767002fcf663ca9"}, - {file = "regex-2023.6.3-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b6192d5af2ccd2a38877bfef086d35e6659566a335b1492786ff254c168b1693"}, - {file = "regex-2023.6.3-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:74390d18c75054947e4194019077e243c06fbb62e541d8817a0fa822ea310c14"}, - {file = "regex-2023.6.3-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:742e19a90d9bb2f4a6cf2862b8b06dea5e09b96c9f2df1779e53432d7275331f"}, - {file = "regex-2023.6.3-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:8abbc5d54ea0ee80e37fef009e3cec5dafd722ed3c829126253d3e22f3846f1e"}, - {file = "regex-2023.6.3-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:c2b867c17a7a7ae44c43ebbeb1b5ff406b3e8d5b3e14662683e5e66e6cc868d3"}, - {file = "regex-2023.6.3-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:d831c2f8ff278179705ca59f7e8524069c1a989e716a1874d6d1aab6119d91d1"}, - {file = "regex-2023.6.3-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:ee2d1a9a253b1729bb2de27d41f696ae893507c7db224436abe83ee25356f5c1"}, - {file = "regex-2023.6.3-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:61474f0b41fe1a80e8dfa70f70ea1e047387b7cd01c85ec88fa44f5d7561d787"}, - {file = "regex-2023.6.3-cp36-cp36m-win32.whl", hash = "sha256:0b71e63226e393b534105fcbdd8740410dc6b0854c2bfa39bbda6b0d40e59a54"}, - {file = "regex-2023.6.3-cp36-cp36m-win_amd64.whl", hash = "sha256:bbb02fd4462f37060122e5acacec78e49c0fbb303c30dd49c7f493cf21fc5b27"}, - {file = "regex-2023.6.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b862c2b9d5ae38a68b92e215b93f98d4c5e9454fa36aae4450f61dd33ff48487"}, - {file = "regex-2023.6.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:976d7a304b59ede34ca2921305b57356694f9e6879db323fd90a80f865d355a3"}, - {file = "regex-2023.6.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:83320a09188e0e6c39088355d423aa9d056ad57a0b6c6381b300ec1a04ec3d16"}, - {file = "regex-2023.6.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9427a399501818a7564f8c90eced1e9e20709ece36be701f394ada99890ea4b3"}, - {file = "regex-2023.6.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7178bbc1b2ec40eaca599d13c092079bf529679bf0371c602edaa555e10b41c3"}, - {file = "regex-2023.6.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:837328d14cde912af625d5f303ec29f7e28cdab588674897baafaf505341f2fc"}, - {file = "regex-2023.6.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2d44dc13229905ae96dd2ae2dd7cebf824ee92bc52e8cf03dcead37d926da019"}, - {file = "regex-2023.6.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d54af539295392611e7efbe94e827311eb8b29668e2b3f4cadcfe6f46df9c777"}, - {file = "regex-2023.6.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:7117d10690c38a622e54c432dfbbd3cbd92f09401d622902c32f6d377e2300ee"}, - {file = "regex-2023.6.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bb60b503ec8a6e4e3e03a681072fa3a5adcbfa5479fa2d898ae2b4a8e24c4591"}, - {file = "regex-2023.6.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:65ba8603753cec91c71de423a943ba506363b0e5c3fdb913ef8f9caa14b2c7e0"}, - {file = "regex-2023.6.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:271f0bdba3c70b58e6f500b205d10a36fb4b58bd06ac61381b68de66442efddb"}, - {file = "regex-2023.6.3-cp37-cp37m-win32.whl", hash = "sha256:9beb322958aaca059f34975b0df135181f2e5d7a13b84d3e0e45434749cb20f7"}, - {file = "regex-2023.6.3-cp37-cp37m-win_amd64.whl", hash = "sha256:fea75c3710d4f31389eed3c02f62d0b66a9da282521075061ce875eb5300cf23"}, - {file = "regex-2023.6.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8f56fcb7ff7bf7404becdfc60b1e81a6d0561807051fd2f1860b0d0348156a07"}, - {file = "regex-2023.6.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d2da3abc88711bce7557412310dfa50327d5769a31d1c894b58eb256459dc289"}, - {file = "regex-2023.6.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a99b50300df5add73d307cf66abea093304a07eb017bce94f01e795090dea87c"}, - {file = "regex-2023.6.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5708089ed5b40a7b2dc561e0c8baa9535b77771b64a8330b684823cfd5116036"}, - {file = "regex-2023.6.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:687ea9d78a4b1cf82f8479cab23678aff723108df3edeac098e5b2498879f4a7"}, - {file = "regex-2023.6.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4d3850beab9f527f06ccc94b446c864059c57651b3f911fddb8d9d3ec1d1b25d"}, - {file = "regex-2023.6.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e8915cc96abeb8983cea1df3c939e3c6e1ac778340c17732eb63bb96247b91d2"}, - {file = "regex-2023.6.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:841d6e0e5663d4c7b4c8099c9997be748677d46cbf43f9f471150e560791f7ff"}, - {file = "regex-2023.6.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:9edce5281f965cf135e19840f4d93d55b3835122aa76ccacfd389e880ba4cf82"}, - {file = "regex-2023.6.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:b956231ebdc45f5b7a2e1f90f66a12be9610ce775fe1b1d50414aac1e9206c06"}, - {file = "regex-2023.6.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:36efeba71c6539d23c4643be88295ce8c82c88bbd7c65e8a24081d2ca123da3f"}, - {file = "regex-2023.6.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:cf67ca618b4fd34aee78740bea954d7c69fdda419eb208c2c0c7060bb822d747"}, - {file = "regex-2023.6.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b4598b1897837067a57b08147a68ac026c1e73b31ef6e36deeeb1fa60b2933c9"}, - {file = "regex-2023.6.3-cp38-cp38-win32.whl", hash = "sha256:f415f802fbcafed5dcc694c13b1292f07fe0befdb94aa8a52905bd115ff41e88"}, - {file = "regex-2023.6.3-cp38-cp38-win_amd64.whl", hash = "sha256:d4f03bb71d482f979bda92e1427f3ec9b220e62a7dd337af0aa6b47bf4498f72"}, - {file = "regex-2023.6.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ccf91346b7bd20c790310c4147eee6ed495a54ddb6737162a36ce9dbef3e4751"}, - {file = "regex-2023.6.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b28f5024a3a041009eb4c333863d7894d191215b39576535c6734cd88b0fcb68"}, - {file = "regex-2023.6.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e0bb18053dfcfed432cc3ac632b5e5e5c5b7e55fb3f8090e867bfd9b054dbcbf"}, - {file = "regex-2023.6.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9a5bfb3004f2144a084a16ce19ca56b8ac46e6fd0651f54269fc9e230edb5e4a"}, - {file = "regex-2023.6.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c6b48d0fa50d8f4df3daf451be7f9689c2bde1a52b1225c5926e3f54b6a9ed1"}, - {file = "regex-2023.6.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:051da80e6eeb6e239e394ae60704d2b566aa6a7aed6f2890a7967307267a5dc6"}, - {file = "regex-2023.6.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a4c3b7fa4cdaa69268748665a1a6ff70c014d39bb69c50fda64b396c9116cf77"}, - {file = "regex-2023.6.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:457b6cce21bee41ac292d6753d5e94dcbc5c9e3e3a834da285b0bde7aa4a11e9"}, - {file = "regex-2023.6.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:aad51907d74fc183033ad796dd4c2e080d1adcc4fd3c0fd4fd499f30c03011cd"}, - {file = "regex-2023.6.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:0385e73da22363778ef2324950e08b689abdf0b108a7d8decb403ad7f5191938"}, - {file = "regex-2023.6.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:c6a57b742133830eec44d9b2290daf5cbe0a2f1d6acee1b3c7b1c7b2f3606df7"}, - {file = "regex-2023.6.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:3e5219bf9e75993d73ab3d25985c857c77e614525fac9ae02b1bebd92f7cecac"}, - {file = "regex-2023.6.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e5087a3c59eef624a4591ef9eaa6e9a8d8a94c779dade95d27c0bc24650261cd"}, - {file = "regex-2023.6.3-cp39-cp39-win32.whl", hash = "sha256:20326216cc2afe69b6e98528160b225d72f85ab080cbdf0b11528cbbaba2248f"}, - {file = "regex-2023.6.3-cp39-cp39-win_amd64.whl", hash = "sha256:bdff5eab10e59cf26bc479f565e25ed71a7d041d1ded04ccf9aee1d9f208487a"}, - {file = "regex-2023.6.3.tar.gz", hash = "sha256:72d1a25bf36d2050ceb35b517afe13864865268dfb45910e2e17a84be6cbfeb0"}, + {file = "regex-2023.8.8-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:88900f521c645f784260a8d346e12a1590f79e96403971241e64c3a265c8ecdb"}, + {file = "regex-2023.8.8-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3611576aff55918af2697410ff0293d6071b7e00f4b09e005d614686ac4cd57c"}, + {file = "regex-2023.8.8-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b8a0ccc8f2698f120e9e5742f4b38dc944c38744d4bdfc427616f3a163dd9de5"}, + {file = "regex-2023.8.8-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c662a4cbdd6280ee56f841f14620787215a171c4e2d1744c9528bed8f5816c96"}, + {file = "regex-2023.8.8-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cf0633e4a1b667bfe0bb10b5e53fe0d5f34a6243ea2530eb342491f1adf4f739"}, + {file = "regex-2023.8.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:551ad543fa19e94943c5b2cebc54c73353ffff08228ee5f3376bd27b3d5b9800"}, + {file = "regex-2023.8.8-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54de2619f5ea58474f2ac211ceea6b615af2d7e4306220d4f3fe690c91988a61"}, + {file = "regex-2023.8.8-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5ec4b3f0aebbbe2fc0134ee30a791af522a92ad9f164858805a77442d7d18570"}, + {file = "regex-2023.8.8-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:3ae646c35cb9f820491760ac62c25b6d6b496757fda2d51be429e0e7b67ae0ab"}, + {file = "regex-2023.8.8-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ca339088839582d01654e6f83a637a4b8194d0960477b9769d2ff2cfa0fa36d2"}, + {file = "regex-2023.8.8-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:d9b6627408021452dcd0d2cdf8da0534e19d93d070bfa8b6b4176f99711e7f90"}, + {file = "regex-2023.8.8-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:bd3366aceedf274f765a3a4bc95d6cd97b130d1dda524d8f25225d14123c01db"}, + {file = "regex-2023.8.8-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7aed90a72fc3654fba9bc4b7f851571dcc368120432ad68b226bd593f3f6c0b7"}, + {file = "regex-2023.8.8-cp310-cp310-win32.whl", hash = "sha256:80b80b889cb767cc47f31d2b2f3dec2db8126fbcd0cff31b3925b4dc6609dcdb"}, + {file = "regex-2023.8.8-cp310-cp310-win_amd64.whl", hash = "sha256:b82edc98d107cbc7357da7a5a695901b47d6eb0420e587256ba3ad24b80b7d0b"}, + {file = "regex-2023.8.8-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1e7d84d64c84ad97bf06f3c8cb5e48941f135ace28f450d86af6b6512f1c9a71"}, + {file = "regex-2023.8.8-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ce0f9fbe7d295f9922c0424a3637b88c6c472b75eafeaff6f910494a1fa719ef"}, + {file = "regex-2023.8.8-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:06c57e14ac723b04458df5956cfb7e2d9caa6e9d353c0b4c7d5d54fcb1325c46"}, + {file = "regex-2023.8.8-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e7a9aaa5a1267125eef22cef3b63484c3241aaec6f48949b366d26c7250e0357"}, + {file = "regex-2023.8.8-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b7408511fca48a82a119d78a77c2f5eb1b22fe88b0d2450ed0756d194fe7a9a"}, + {file = "regex-2023.8.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14dc6f2d88192a67d708341f3085df6a4f5a0c7b03dec08d763ca2cd86e9f559"}, + {file = "regex-2023.8.8-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:48c640b99213643d141550326f34f0502fedb1798adb3c9eb79650b1ecb2f177"}, + {file = "regex-2023.8.8-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0085da0f6c6393428bf0d9c08d8b1874d805bb55e17cb1dfa5ddb7cfb11140bf"}, + {file = "regex-2023.8.8-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:964b16dcc10c79a4a2be9f1273fcc2684a9eedb3906439720598029a797b46e6"}, + {file = "regex-2023.8.8-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7ce606c14bb195b0e5108544b540e2c5faed6843367e4ab3deb5c6aa5e681208"}, + {file = "regex-2023.8.8-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:40f029d73b10fac448c73d6eb33d57b34607f40116e9f6e9f0d32e9229b147d7"}, + {file = "regex-2023.8.8-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3b8e6ea6be6d64104d8e9afc34c151926f8182f84e7ac290a93925c0db004bfd"}, + {file = "regex-2023.8.8-cp311-cp311-win32.whl", hash = "sha256:942f8b1f3b223638b02df7df79140646c03938d488fbfb771824f3d05fc083a8"}, + {file = "regex-2023.8.8-cp311-cp311-win_amd64.whl", hash = "sha256:51d8ea2a3a1a8fe4f67de21b8b93757005213e8ac3917567872f2865185fa7fb"}, + {file = "regex-2023.8.8-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e951d1a8e9963ea51efd7f150450803e3b95db5939f994ad3d5edac2b6f6e2b4"}, + {file = "regex-2023.8.8-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:704f63b774218207b8ccc6c47fcef5340741e5d839d11d606f70af93ee78e4d4"}, + {file = "regex-2023.8.8-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:22283c769a7b01c8ac355d5be0715bf6929b6267619505e289f792b01304d898"}, + {file = "regex-2023.8.8-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:91129ff1bb0619bc1f4ad19485718cc623a2dc433dff95baadbf89405c7f6b57"}, + {file = "regex-2023.8.8-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de35342190deb7b866ad6ba5cbcccb2d22c0487ee0cbb251efef0843d705f0d4"}, + {file = "regex-2023.8.8-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b993b6f524d1e274a5062488a43e3f9f8764ee9745ccd8e8193df743dbe5ee61"}, + {file = "regex-2023.8.8-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3026cbcf11d79095a32d9a13bbc572a458727bd5b1ca332df4a79faecd45281c"}, + {file = "regex-2023.8.8-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:293352710172239bf579c90a9864d0df57340b6fd21272345222fb6371bf82b3"}, + {file = "regex-2023.8.8-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:d909b5a3fff619dc7e48b6b1bedc2f30ec43033ba7af32f936c10839e81b9217"}, + {file = "regex-2023.8.8-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:3d370ff652323c5307d9c8e4c62efd1956fb08051b0e9210212bc51168b4ff56"}, + {file = "regex-2023.8.8-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:b076da1ed19dc37788f6a934c60adf97bd02c7eea461b73730513921a85d4235"}, + {file = "regex-2023.8.8-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:e9941a4ada58f6218694f382e43fdd256e97615db9da135e77359da257a7168b"}, + {file = "regex-2023.8.8-cp36-cp36m-win32.whl", hash = "sha256:a8c65c17aed7e15a0c824cdc63a6b104dfc530f6fa8cb6ac51c437af52b481c7"}, + {file = "regex-2023.8.8-cp36-cp36m-win_amd64.whl", hash = "sha256:aadf28046e77a72f30dcc1ab185639e8de7f4104b8cb5c6dfa5d8ed860e57236"}, + {file = "regex-2023.8.8-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:423adfa872b4908843ac3e7a30f957f5d5282944b81ca0a3b8a7ccbbfaa06103"}, + {file = "regex-2023.8.8-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ae594c66f4a7e1ea67232a0846649a7c94c188d6c071ac0210c3e86a5f92109"}, + {file = "regex-2023.8.8-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e51c80c168074faa793685656c38eb7a06cbad7774c8cbc3ea05552d615393d8"}, + {file = "regex-2023.8.8-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:09b7f4c66aa9d1522b06e31a54f15581c37286237208df1345108fcf4e050c18"}, + {file = "regex-2023.8.8-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e73e5243af12d9cd6a9d6a45a43570dbe2e5b1cdfc862f5ae2b031e44dd95a8"}, + {file = "regex-2023.8.8-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:941460db8fe3bd613db52f05259c9336f5a47ccae7d7def44cc277184030a116"}, + {file = "regex-2023.8.8-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f0ccf3e01afeb412a1a9993049cb160d0352dba635bbca7762b2dc722aa5742a"}, + {file = "regex-2023.8.8-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:2e9216e0d2cdce7dbc9be48cb3eacb962740a09b011a116fd7af8c832ab116ca"}, + {file = "regex-2023.8.8-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:5cd9cd7170459b9223c5e592ac036e0704bee765706445c353d96f2890e816c8"}, + {file = "regex-2023.8.8-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:4873ef92e03a4309b3ccd8281454801b291b689f6ad45ef8c3658b6fa761d7ac"}, + {file = "regex-2023.8.8-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:239c3c2a339d3b3ddd51c2daef10874410917cd2b998f043c13e2084cb191684"}, + {file = "regex-2023.8.8-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:1005c60ed7037be0d9dea1f9c53cc42f836188227366370867222bda4c3c6bd7"}, + {file = "regex-2023.8.8-cp37-cp37m-win32.whl", hash = "sha256:e6bd1e9b95bc5614a7a9c9c44fde9539cba1c823b43a9f7bc11266446dd568e3"}, + {file = "regex-2023.8.8-cp37-cp37m-win_amd64.whl", hash = "sha256:9a96edd79661e93327cfeac4edec72a4046e14550a1d22aa0dd2e3ca52aec921"}, + {file = "regex-2023.8.8-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f2181c20ef18747d5f4a7ea513e09ea03bdd50884a11ce46066bb90fe4213675"}, + {file = "regex-2023.8.8-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a2ad5add903eb7cdde2b7c64aaca405f3957ab34f16594d2b78d53b8b1a6a7d6"}, + {file = "regex-2023.8.8-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9233ac249b354c54146e392e8a451e465dd2d967fc773690811d3a8c240ac601"}, + {file = "regex-2023.8.8-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:920974009fb37b20d32afcdf0227a2e707eb83fe418713f7a8b7de038b870d0b"}, + {file = "regex-2023.8.8-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2b6c5dfe0929b6c23dde9624483380b170b6e34ed79054ad131b20203a1a63"}, + {file = "regex-2023.8.8-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:96979d753b1dc3b2169003e1854dc67bfc86edf93c01e84757927f810b8c3c93"}, + {file = "regex-2023.8.8-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2ae54a338191e1356253e7883d9d19f8679b6143703086245fb14d1f20196be9"}, + {file = "regex-2023.8.8-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2162ae2eb8b079622176a81b65d486ba50b888271302190870b8cc488587d280"}, + {file = "regex-2023.8.8-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c884d1a59e69e03b93cf0dfee8794c63d7de0ee8f7ffb76e5f75be8131b6400a"}, + {file = "regex-2023.8.8-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:cf9273e96f3ee2ac89ffcb17627a78f78e7516b08f94dc435844ae72576a276e"}, + {file = "regex-2023.8.8-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:83215147121e15d5f3a45d99abeed9cf1fe16869d5c233b08c56cdf75f43a504"}, + {file = "regex-2023.8.8-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:3f7454aa427b8ab9101f3787eb178057c5250478e39b99540cfc2b889c7d0586"}, + {file = "regex-2023.8.8-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f0640913d2c1044d97e30d7c41728195fc37e54d190c5385eacb52115127b882"}, + {file = "regex-2023.8.8-cp38-cp38-win32.whl", hash = "sha256:0c59122ceccb905a941fb23b087b8eafc5290bf983ebcb14d2301febcbe199c7"}, + {file = "regex-2023.8.8-cp38-cp38-win_amd64.whl", hash = "sha256:c12f6f67495ea05c3d542d119d270007090bad5b843f642d418eb601ec0fa7be"}, + {file = "regex-2023.8.8-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:82cd0a69cd28f6cc3789cc6adeb1027f79526b1ab50b1f6062bbc3a0ccb2dbc3"}, + {file = "regex-2023.8.8-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:bb34d1605f96a245fc39790a117ac1bac8de84ab7691637b26ab2c5efb8f228c"}, + {file = "regex-2023.8.8-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:987b9ac04d0b38ef4f89fbc035e84a7efad9cdd5f1e29024f9289182c8d99e09"}, + {file = "regex-2023.8.8-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9dd6082f4e2aec9b6a0927202c85bc1b09dcab113f97265127c1dc20e2e32495"}, + {file = "regex-2023.8.8-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7eb95fe8222932c10d4436e7a6f7c99991e3fdd9f36c949eff16a69246dee2dc"}, + {file = "regex-2023.8.8-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7098c524ba9f20717a56a8d551d2ed491ea89cbf37e540759ed3b776a4f8d6eb"}, + {file = "regex-2023.8.8-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b694430b3f00eb02c594ff5a16db30e054c1b9589a043fe9174584c6efa8033"}, + {file = "regex-2023.8.8-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:b2aeab3895d778155054abea5238d0eb9a72e9242bd4b43f42fd911ef9a13470"}, + {file = "regex-2023.8.8-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:988631b9d78b546e284478c2ec15c8a85960e262e247b35ca5eaf7ee22f6050a"}, + {file = "regex-2023.8.8-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:67ecd894e56a0c6108ec5ab1d8fa8418ec0cff45844a855966b875d1039a2e34"}, + {file = "regex-2023.8.8-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:14898830f0a0eb67cae2bbbc787c1a7d6e34ecc06fbd39d3af5fe29a4468e2c9"}, + {file = "regex-2023.8.8-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:f2200e00b62568cfd920127782c61bc1c546062a879cdc741cfcc6976668dfcf"}, + {file = "regex-2023.8.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9691a549c19c22d26a4f3b948071e93517bdf86e41b81d8c6ac8a964bb71e5a6"}, + {file = "regex-2023.8.8-cp39-cp39-win32.whl", hash = "sha256:6ab2ed84bf0137927846b37e882745a827458689eb969028af8032b1b3dac78e"}, + {file = "regex-2023.8.8-cp39-cp39-win_amd64.whl", hash = "sha256:5543c055d8ec7801901e1193a51570643d6a6ab8751b1f7dd9af71af467538bb"}, + {file = "regex-2023.8.8.tar.gz", hash = "sha256:fcbdc5f2b0f1cd0f6a56cdb46fe41d2cce1e644e3b68832f3eeebc5fb0f7712e"}, ] [[package]] name = "requests" version = "2.31.0" description = "Python HTTP for Humans." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2431,7 +2349,6 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "retry2" version = "0.9.5" description = "Easy to use retry decorator." -category = "dev" optional = false python-versions = ">=2.6" files = [ @@ -2445,7 +2362,6 @@ decorator = ">=3.4.2" name = "rich" version = "13.5.2" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" -category = "dev" optional = false python-versions = ">=3.7.0" files = [ @@ -2491,7 +2407,6 @@ files = [ name = "s3transfer" version = "0.6.1" description = "An Amazon S3 Transfer Manager" -category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -2509,7 +2424,6 @@ crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] name = "sarif-om" version = "1.0.4" description = "Classes implementing the SARIF 2.1.0 object model." -category = "dev" optional = false python-versions = ">= 2.7" files = [ @@ -2525,7 +2439,6 @@ pbr = "*" name = "sentry-sdk" version = "1.29.2" description = "Python client for Sentry (https://sentry.io)" -category = "dev" optional = false python-versions = "*" files = [ @@ -2568,7 +2481,6 @@ tornado = ["tornado (>=5)"] name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -2580,7 +2492,6 @@ files = [ name = "smmap" version = "5.0.0" description = "A pure Python implementation of a sliding window memory map manager" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -2592,7 +2503,6 @@ files = [ name = "sniffio" version = "1.3.0" description = "Sniff out which async library your code is running under" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2604,7 +2514,6 @@ files = [ name = "stevedore" version = "3.5.2" description = "Manage dynamic plugins for Python applications" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -2620,7 +2529,6 @@ pbr = ">=2.0.0,<2.1.0 || >2.1.0" name = "sympy" version = "1.10.1" description = "Computer algebra system (CAS) in Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2635,7 +2543,6 @@ mpmath = ">=0.19" name = "tomli" version = "2.0.1" description = "A lil' TOML parser" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2647,7 +2554,6 @@ files = [ name = "typed-ast" version = "1.5.5" description = "a fork of Python 2 and 3 ast modules with type comment support" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -2698,7 +2604,6 @@ files = [ name = "typeguard" version = "2.13.3" description = "Run-time type checker for Python" -category = "dev" optional = false python-versions = ">=3.5.3" files = [ @@ -2714,7 +2619,6 @@ test = ["mypy", "pytest", "typing-extensions"] name = "types-python-dateutil" version = "2.8.19.14" description = "Typing stubs for python-dateutil" -category = "dev" optional = false python-versions = "*" files = [ @@ -2726,7 +2630,6 @@ files = [ name = "types-requests" version = "2.31.0.2" description = "Typing stubs for requests" -category = "dev" optional = false python-versions = "*" files = [ @@ -2741,7 +2644,6 @@ types-urllib3 = "*" name = "types-urllib3" version = "1.26.25.14" description = "Typing stubs for urllib3" -category = "dev" optional = false python-versions = "*" files = [ @@ -2753,7 +2655,6 @@ files = [ name = "typing-extensions" version = "4.7.1" description = "Backported and Experimental Type Hints for Python 3.7+" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2765,7 +2666,6 @@ files = [ name = "urllib3" version = "1.26.16" description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ @@ -2782,7 +2682,6 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] name = "verspec" version = "0.1.0" description = "Flexible version handling" -category = "dev" optional = false python-versions = "*" files = [ @@ -2797,7 +2696,6 @@ test = ["coverage", "flake8 (>=3.7)", "mypy", "pretend", "pytest"] name = "watchdog" version = "3.0.0" description = "Filesystem events monitoring" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2837,7 +2735,6 @@ watchmedo = ["PyYAML (>=3.10)"] name = "wrapt" version = "1.15.0" description = "Module for decorators, wrappers and monkey patching." -category = "main" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" files = [ @@ -2922,7 +2819,6 @@ files = [ name = "xenon" version = "0.9.0" description = "Monitor code metrics for Python on your CI server" -category = "dev" optional = false python-versions = "*" files = [ @@ -2939,7 +2835,6 @@ requests = ">=2.0,<3.0" name = "zipp" version = "3.15.0" description = "Backport of pathlib-compatible object wrapper for zip files" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2952,7 +2847,7 @@ docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] [extras] -all = ["aws-encryption-sdk", "aws-xray-sdk", "fastjsonschema", "itsdangerous", "pydantic"] +all = ["aws-encryption-sdk", "aws-xray-sdk", "fastjsonschema", "pydantic"] aws-sdk = ["boto3"] datamasking-all = ["aws-encryption-sdk", "itsdangerous"] datamasking-aws-sdk = ["aws-encryption-sdk"] @@ -2964,4 +2859,4 @@ validation = ["fastjsonschema"] [metadata] lock-version = "2.0" python-versions = "^3.7.4" -content-hash = "47c1e7d74f594a0f31f4cef267382bb214b9ee2bbb139a7cc692659a1e589cdd" +content-hash = "287b2aff9a85ab1983ec41515388f72b9a048b53c1af42e957b29c968c69056d" From 2eab50b596faa6ae3eeff11e10c1956d0a76245c Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Fri, 11 Aug 2023 02:53:39 +0000 Subject: [PATCH 24/56] Made unit tests more legible, removed parameterization --- tests/unit/data_masking/test_data_masking.py | 334 ++++++++++++++++--- 1 file changed, 295 insertions(+), 39 deletions(-) diff --git a/tests/unit/data_masking/test_data_masking.py b/tests/unit/data_masking/test_data_masking.py index 8e013e2e925..3ed1b203d20 100644 --- a/tests/unit/data_masking/test_data_masking.py +++ b/tests/unit/data_masking/test_data_masking.py @@ -1,66 +1,323 @@ import json - import pytest +from itsdangerous.url_safe import URLSafeSerializer from aws_lambda_powertools.shared.constants import DATA_MASKING_STRING from aws_lambda_powertools.utilities.data_masking.base import DataMasking -from tests.unit.data_masking.setup import * +from aws_lambda_powertools.utilities.data_masking.provider import Provider + + +class MyEncryptionProvider(Provider): + """Custom encryption provider class""" + + def __init__(self, keys, salt=None): + self.keys = keys + self.salt = salt + + def encrypt(self, data: str) -> str: + if data is None: + return data + serialize = URLSafeSerializer(self.keys) + return serialize.dumps(data) + + def decrypt(self, data: str) -> str: + if data is None: + return data + serialize = URLSafeSerializer(self.keys) + return serialize.loads(data) + + +# @pytest.fixture +# def data_masker(): +# return DataMasking() + + +# @pytest.fixture +# def custom_data_masker(): +# return DataMasking(provider=MyEncryptionProvider(keys="secret-key")) + + +def test_mask_int(): + data_masker = DataMasking() + + # GIVEN an int data type + + # WHEN mask is called with no fields argument + masked_string = data_masker.mask(42) + + # THEN the result is the data masked + assert masked_string == DATA_MASKING_STRING + + +def test_mask_float(): + data_masker = DataMasking() + + # GIVEN a float data type + + # WHEN mask is called with no fields argument + masked_string = data_masker.mask(4.2) + + # THEN the result is the data masked + assert masked_string == DATA_MASKING_STRING + + +def test_mask_bool(): + data_masker = DataMasking() + + # GIVEN a bool data type + + # WHEN mask is called with no fields argument + masked_string = data_masker.mask(True) + + # THEN the result is the data masked + assert masked_string == DATA_MASKING_STRING + + +def test_mask_none(): + data_masker = DataMasking() -# should be conftest? no other conftest in unit tests -# didn't work when i made them all pytest.fixtures + # GIVEN a None data type + + # WHEN mask is called with no fields argument + masked_string = data_masker.mask(None) + + # THEN the result is the data masked + assert masked_string == DATA_MASKING_STRING + + +def test_mask_str(): + data_masker = DataMasking() + + # GIVEN a str data type + + # WHEN mask is called with no fields argument + masked_string = data_masker.mask("this is a string") + + # THEN the result is the data masked + assert masked_string == DATA_MASKING_STRING + + +def test_mask_list(): + data_masker = DataMasking() + + # GIVEN a list data type + + # WHEN mask is called with no fields argument + masked_string = data_masker.mask([1, 2, "string", 3]) + + # THEN the result is the data masked, while maintaining type list + assert masked_string == [DATA_MASKING_STRING, DATA_MASKING_STRING, DATA_MASKING_STRING, DATA_MASKING_STRING] -@pytest.mark.parametrize("data_masker", data_maskers) -@pytest.mark.parametrize("value, value_masked", data_types_and_masks) -def test_mask_types(data_masker, value, value_masked): - # GIVEN any data type +def test_mask_dict(): + data_masker = DataMasking() + + # GIVEN a dict data type + data = { + "a": { + "1": {"None": "hello", "four": "world"}, + "b": {"3": {"4": "goodbye", "e": "world"}}, + } + } # WHEN mask is called with no fields argument - masked_string = data_masker.mask(value) + masked_string = data_masker.mask(data) - # THEN the result is the full input data masked - assert masked_string == value_masked + # THEN the result is the data masked + assert masked_string == DATA_MASKING_STRING -@pytest.mark.parametrize("data_masker", data_maskers) -def test_mask_with_fields(data_masker): - # GIVEN the data type is a dictionary, or a json representation of a dictionary +def test_mask_dict_with_fields(): + data_masker = DataMasking() + + # GIVEN the data type is a dictionary + data = { + "a": { + "1": {"None": "hello", "four": "world"}, + "b": {"3": {"4": "goodbye", "e": "world"}}, + } + } # WHEN mask is called with a list of fields specified - masked_string = data_masker.mask(python_dict, dict_fields) - masked_json_string = data_masker.mask(json_dict, dict_fields) + masked_string = data_masker.mask(data, fields=["a.1.None", "a.b.3.4"]) # THEN the result is only the specified fields are masked - assert masked_string == masked_with_fields - assert masked_json_string == masked_with_fields + assert masked_string == { + "a": {"1": {"None": DATA_MASKING_STRING, "four": "world"}, "b": {"3": {"4": DATA_MASKING_STRING, "e": "world"}}} + } + + +def test_mask_json_dict_with_fields(): + data_masker = DataMasking() + + # GIVEN the data type is a json representation of a dictionary + data = json.dumps( + { + "a": { + "1": {"None": "hello", "four": "world"}, + "b": {"3": {"4": "goodbye", "e": "world"}}, + } + } + ) + # WHEN mask is called with a list of fields specified + masked_json_string = data_masker.mask(data, fields=["a.1.None", "a.b.3.4"]) -@pytest.mark.parametrize("value", data_types) -def test_encrypt_decrypt(value): + # THEN the result is only the specified fields are masked + assert masked_json_string == { + "a": {"1": {"None": DATA_MASKING_STRING, "four": "world"}, "b": {"3": {"4": DATA_MASKING_STRING, "e": "world"}}} + } + + +def test_encrypt_decrypt_int(): + # GIVEN an instantiation of DataMasking with a Provider + data_masker = DataMasking(provider=MyEncryptionProvider(keys="secret-key")) + + # WHEN encrypting and then decrypting an int + encrypted_data = data_masker.encrypt(42) + decrypted_data = data_masker.decrypt(encrypted_data) + + # THEN the result is the original input data + assert decrypted_data == 42 + + +def test_encrypt_decrypt_float(): + # GIVEN an instantiation of DataMasking with a Provider + data_masker = DataMasking(provider=MyEncryptionProvider(keys="secret-key")) + + # WHEN encrypting and then decrypting a float + encrypted_data = data_masker.encrypt(4.2) + decrypted_data = data_masker.decrypt(encrypted_data) + + # THEN the result is the original input data + assert decrypted_data == 4.2 + + +def test_encrypt_decrypt_bool(): # GIVEN an instantiation of DataMasking with a Provider data_masker = DataMasking(provider=MyEncryptionProvider(keys="secret-key")) - # WHEN encrypting and then decrypting the encrypted data - encrypted_data = data_masker.encrypt(value) + # WHEN encrypting and then decrypting a bool + encrypted_data = data_masker.encrypt(True) decrypted_data = data_masker.decrypt(encrypted_data) # THEN the result is the original input data - assert decrypted_data == value + assert decrypted_data == True -@pytest.mark.parametrize("value, fields", zip(dictionaries, fields_to_mask)) -def test_encrypt_decrypt_with_fields(value, fields): +def test_encrypt_decrypt_none(): # GIVEN an instantiation of DataMasking with a Provider data_masker = DataMasking(provider=MyEncryptionProvider(keys="secret-key")) - # WHEN encrypting and then decrypting the encrypted data with a list of fields - encrypted_data = data_masker.encrypt(value, fields) - decrypted_data = data_masker.decrypt(encrypted_data, fields) + # WHEN encrypting and then decrypting a None type + encrypted_data = data_masker.encrypt(None) + decrypted_data = data_masker.decrypt(encrypted_data) # THEN the result is the original input data - if value == json_dict: - assert decrypted_data == json.loads(value) - else: - assert decrypted_data == value + assert decrypted_data == None + + +def test_encrypt_decrypt_str(): + # GIVEN an instantiation of DataMasking with a Provider + data_masker = DataMasking(provider=MyEncryptionProvider(keys="secret-key")) + + # WHEN encrypting and then decrypting a string + encrypted_data = data_masker.encrypt("this is a string") + decrypted_data = data_masker.decrypt(encrypted_data) + + # THEN the result is the original input data + assert decrypted_data == "this is a string" + + +def test_encrypt_decrypt_list(): + # GIVEN an instantiation of DataMasking with a Provider + data_masker = DataMasking(provider=MyEncryptionProvider(keys="secret-key")) + + # WHEN encrypting and then decrypting a list + encrypted_data = data_masker.encrypt([1, 2, "string", 4]) + decrypted_data = data_masker.decrypt(encrypted_data) + + # THEN the result is the original input data + assert decrypted_data == [1, 2, "string", 4] + + +def test_dict_encryption_with_fields(): + # GIVEN an instantiation of DataMasking with a Provider + data_masker = DataMasking(provider=MyEncryptionProvider(keys="secret-key")) + + data = { + "a": { + "1": {"None": "hello", "four": "world"}, + "b": {"3": {"4": "goodbye", "e": "world"}}, + } + } + + # WHEN encrypting and decrypting the data with a list of fields + encrypted_data = data_masker.encrypt(data, fields=["a.1.None", "a.b.3.4"]) + decrypted_data = data_masker.decrypt(encrypted_data, fields=["a.1.None", "a.b.3.4"]) + + # THEN the result is the original input data + assert decrypted_data == data + + +def test_json_encryption_with_fields(): + # GIVEN an instantiation of DataMasking with a Provider + data_masker = DataMasking(provider=MyEncryptionProvider(keys="secret-key")) + + data = json.dumps( + { + "a": { + "1": {"None": "hello", "four": "world"}, + "b": {"3": {"4": "goodbye", "e": "world"}}, + } + } + ) + + # WHEN encrypting and decrypting a json representation of a dictionary with a list of fields + encrypted_data = data_masker.encrypt(data, fields=["a.1.None", "a.b.3.4"]) + decrypted_data = data_masker.decrypt(encrypted_data, fields=["a.1.None", "a.b.3.4"]) + + # THEN the result is the original input data + assert decrypted_data == json.loads(data) + + +def test_big_data_encryption_with_fields(): + # GIVEN an instantiation of DataMasking with a Provider + data_masker = DataMasking(provider=MyEncryptionProvider(keys="secret-key")) + + # 10kb JSON blob for latency testing + data = { + "id": 1, + "name": "John Doe", + "age": 30, + "email": "johndoe@example.com", + "address": {"street": "123 Main St", "city": "Anytown", "state": "CA", "zip": "12345"}, + "phone_numbers": ["+1-555-555-1234", "+1-555-555-5678"], + "interests": ["Hiking", "Traveling", "Photography", "Reading"], + "job_history": { + "company": "Acme Inc.", + "position": "Software Engineer", + "start_date": "2015-01-01", + "end_date": "2017-12-31", + }, + "about_me": """ + Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla tincidunt velit quis + sapien mollis, at egestas massa tincidunt. Suspendisse ultrices arcu a dolor dapibus, + ut pretium turpis volutpat. Vestibulum at sapien quis sapien dignissim volutpat ut a enim. + Praesent fringilla sem eu dui convallis luctus. Donec ullamcorper, sapien ut convallis congue, + risus mauris pretium tortor, nec dignissim arcu urna a nisl. Vivamus non fermentum ex. Proin + interdum nisi id sagittis egestas. Nam sit amet nisi nec quam pharetra sagittis. Aliquam erat + volutpat. Donec nec luctus sem, nec ornare lorem. Vivamus vitae orci quis enim faucibus placerat. + Nulla facilisi. Proin in turpis orci. Donec imperdiet velit ac tellus gravida, eget laoreet tellus + malesuada. Praesent venenatis tellus ac urna blandit, at varius felis posuere. Integer a commodo nunc. + """, + } + + # WHEN encrypting and decrypting the data with a list of fields + encrypted_data = data_masker.encrypt(data, fields=["address.street", "job_history.company"]) + decrypted_data = data_masker.decrypt(encrypted_data, fields=["address.street", "job_history.company"]) + + # THEN the result is the original input data + assert decrypted_data == data def test_encrypt_not_implemented(): @@ -99,7 +356,7 @@ def test_parsing_unsupported_data_type(): def test_parsing_nonexistent_fields(): # GIVEN an initialization of the DataMasking class data_masker = DataMasking() - _python_dict = { + data = { "3": { "1": {"None": "hello", "four": "world"}, "4": {"33": {"5": "goodbye", "e": "world"}}, @@ -110,13 +367,13 @@ def test_parsing_nonexistent_fields(): # THEN the result is a KeyError with pytest.raises(KeyError): - data_masker.mask(_python_dict, ["3.1.True"]) + data_masker.mask(data, ["3.1.True"]) def test_parsing_nonstring_fields(): # GIVEN an initialization of the DataMasking class data_masker = DataMasking() - _python_dict = { + data = { "3": { "1": {"None": "hello", "four": "world"}, "4": {"33": {"5": "goodbye", "e": "world"}}, @@ -124,7 +381,7 @@ def test_parsing_nonstring_fields(): } # WHEN attempting to pass in a list of fields that are not strings - masked = data_masker.mask(_python_dict, fields=[3.4]) + masked = data_masker.mask(data, fields=[3.4]) # THEN the result is the value of the nested field should be masked as normal assert masked == {"3": {"1": {"None": "hello", "four": "world"}, "4": DATA_MASKING_STRING}} @@ -135,14 +392,13 @@ def test_parsing_nonstring_keys_and_fields(): data_masker = DataMasking() # WHEN the input data is a dictionary with integer keys - _python_dict = { + data = { 3: { "1": {"None": "hello", "four": "world"}, 4: {"33": {"5": "goodbye", "e": "world"}}, } } - - masked = data_masker.mask(_python_dict, fields=[3.4]) + masked = data_masker.mask(data, fields=[3.4]) # THEN the result is the value of the nested field should be masked as normal assert masked == {"3": {"1": {"None": "hello", "four": "world"}, "4": DATA_MASKING_STRING}} From 57a5a3ae31ce6f0141cc8edc695bb31eb6a474d8 Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Mon, 21 Aug 2023 18:30:57 +0000 Subject: [PATCH 25/56] Adding E2E tests (wip) --- .../providers/aws_encryption_sdk.py | 2 +- tests/e2e/data_masking/__init__.py | 0 tests/e2e/data_masking/conftest.py | 18 ++++ .../data_masking/handlers/basic_handler.py | 11 +++ tests/e2e/data_masking/infrastructure.py | 14 +++ tests/e2e/data_masking/test_data_masking.py | 99 +++++++++++++++++++ 6 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 tests/e2e/data_masking/__init__.py create mode 100644 tests/e2e/data_masking/conftest.py create mode 100644 tests/e2e/data_masking/handlers/basic_handler.py create mode 100644 tests/e2e/data_masking/infrastructure.py create mode 100644 tests/e2e/data_masking/test_data_masking.py diff --git a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py index c57c44de755..44b68dc1745 100644 --- a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py +++ b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py @@ -31,7 +31,7 @@ def __call__(cls, *args, **provider_options): # NOTE: You can also set max messages/bytes per data key -class AwsEncryptionSdkProvider(Provider, metaclass=SingletonMeta): +class AwsEncryptionSdkProvider(Provider): cache = LocalCryptoMaterialsCache(CACHE_CAPACITY) session = botocore.session.Session() register_feature_to_botocore_session(session, "data-masking") diff --git a/tests/e2e/data_masking/__init__.py b/tests/e2e/data_masking/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/data_masking/conftest.py b/tests/e2e/data_masking/conftest.py new file mode 100644 index 00000000000..0430c274b80 --- /dev/null +++ b/tests/e2e/data_masking/conftest.py @@ -0,0 +1,18 @@ +import pytest +from tests.e2e.data_masking.infrastructure import DataMaskingStack + + +@pytest.fixture(autouse=True, scope="package") +def infrastructure(): + """Setup and teardown logic for E2E test infrastructure + + Yields + ------ + Dict[str, str] + CloudFormation Outputs from deployed infrastructure + """ + stack = DataMaskingStack() + try: + yield stack.deploy() + finally: + stack.delete() diff --git a/tests/e2e/data_masking/handlers/basic_handler.py b/tests/e2e/data_masking/handlers/basic_handler.py new file mode 100644 index 00000000000..0f0dd46b4aa --- /dev/null +++ b/tests/e2e/data_masking/handlers/basic_handler.py @@ -0,0 +1,11 @@ +from aws_lambda_powertools import Logger + +logger = Logger() + + +@logger.inject_lambda_context +def lambda_handler(event, context): + message, append_keys = event.get("message", ""), event.get("append_keys", {}) + logger.append_keys(**append_keys) + logger.info(message) + return "success" diff --git a/tests/e2e/data_masking/infrastructure.py b/tests/e2e/data_masking/infrastructure.py new file mode 100644 index 00000000000..e1de36fd730 --- /dev/null +++ b/tests/e2e/data_masking/infrastructure.py @@ -0,0 +1,14 @@ +import aws_cdk.aws_kms as kms +from aws_cdk import CfnOutput +from tests.e2e.utils.infrastructure import BaseInfrastructure + + +class DataMaskingStack(BaseInfrastructure): + def create_resources(self): + self.create_lambda_functions() + + key1 = kms.Key(self.stack, "MyKMSKey1", description="My KMS Key1") + CfnOutput(self.stack, "KMSKey1Arn", value=key1.key_arn, description="ARN of the created KMS Key1") + + key2 = kms.Key(self.stack, "MyKMSKey2", description="My KMS Key2") + CfnOutput(self.stack, "KMSKey2Arn", value=key2.key_arn, description="ARN of the created KMS Key2") diff --git a/tests/e2e/data_masking/test_data_masking.py b/tests/e2e/data_masking/test_data_masking.py new file mode 100644 index 00000000000..75dfea221d6 --- /dev/null +++ b/tests/e2e/data_masking/test_data_masking.py @@ -0,0 +1,99 @@ +import json +from uuid import uuid4 +from aws_encryption_sdk.exceptions import DecryptKeyError +import pytest +from tests.e2e.utils import data_fetcher +from aws_lambda_powertools.utilities.data_masking.base import DataMasking +from aws_lambda_powertools.utilities.data_masking.providers.aws_encryption_sdk import AwsEncryptionSdkProvider +from aws_lambda_powertools.shared.constants import DATA_MASKING_STRING + + +@pytest.fixture +def basic_handler_fn(infrastructure: dict) -> str: + return infrastructure.get("BasicHandler", "") + + +@pytest.fixture +def basic_handler_fn_arn(infrastructure: dict) -> str: + return infrastructure.get("BasicHandlerArn", "") + + +@pytest.fixture +def kms_key1_arn(infrastructure: dict) -> str: + return infrastructure.get("KMSKey1Arn", "") + + +@pytest.fixture +def kms_key2_arn(infrastructure: dict) -> str: + return infrastructure.get("KMSKey2Arn", "") + + +@pytest.fixture +def data_masker(kms_key1_arn) -> DataMasking: + return DataMasking(provider=AwsEncryptionSdkProvider(keys=[kms_key1_arn])) + + +@pytest.mark.xdist_group(name="data_masking") +def test_encryption(data_masker): + # GIVEN an instantiation of DataMasking with the AWS encryption provider + + # AWS Encryption SDK encrypt method only takes in bytes or strings + value = bytes(str([1, 2, "string", 4.5]), "utf-8") + + # WHEN encrypting and then decrypting the encrypted data + encrypted_data = data_masker.encrypt(value) + decrypted_data = data_masker.decrypt(encrypted_data) + + # THEN the result is the original input data + assert decrypted_data == value + + +# TODO: Waiting on EncryptionSDK team to answer tt.amazon.com/V1005246120 +@pytest.mark.xdist_group(name="data_masking") +def test_encryption_context(data_masker): + # GIVEN an instantiation of DataMasking with the AWS encryption provider + + value = bytes(str([1, 2, "string", 4.5]), "utf-8") + + # WHEN encrypting and then decrypting the encrypted data + encrypted_data = data_masker.encrypt(value, encryption_context={"this": "is_secure"}) + decrypted_data = data_masker.decrypt(encrypted_data) + + # THEN the result is the original input data + assert decrypted_data == value + + +# TODO: metaclass? +@pytest.mark.xdist_group(name="data_masking") +def test_encryption_key_fail(kms_key2_arn, data_masker): + # GIVEN an instantiation of DataMasking with the AWS encryption provider with a certain key + + # WHEN encrypting and then decrypting the encrypted data + value = bytes(str([1, 2, "string", 4.5]), "utf-8") + encrypted_data = data_masker.encrypt(value) + + data_masker_key2 = DataMasking(provider=AwsEncryptionSdkProvider(keys=[kms_key2_arn])) + + with pytest.raises(DecryptKeyError): + data_masker_key2.decrypt(encrypted_data) + + +@pytest.mark.xdist_group(name="data_masking") +def test_masked_in_logs(basic_handler_fn, basic_handler_fn_arn): + # GIVEN an instantiation of DataMasking with the AWS encryption provider + data_masker = DataMasking(provider=AwsEncryptionSdkProvider(keys=[kms_key1_arn])) + + # WHEN masking a value and logging it + masked_data = data_masker.mask([1, 2, "string", 4.5]) + message = masked_data + custom_key = "order_id" + additional_keys = {custom_key: f"{uuid4()}"} + payload = json.dumps({"message": message, "append_keys": additional_keys}) + + _, execution_time = data_fetcher.get_lambda_response(lambda_arn=basic_handler_fn_arn, payload=payload) + data_fetcher.get_lambda_response(lambda_arn=basic_handler_fn_arn, payload=payload) + + # THEN the logs should show only the obfuscated data + logs = data_fetcher.get_logs(function_name=basic_handler_fn, start_time=execution_time, minimum_log_entries=2) + + assert logs.have_keys("message") is True From 8aabc7f6f79f2eb65dcbfa19c1badc6185fa23d4 Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Mon, 21 Aug 2023 19:08:42 +0000 Subject: [PATCH 26/56] Added data_masking constants, made into BaseProvider and added types --- aws_lambda_powertools/shared/constants.py | 2 -- aws_lambda_powertools/utilities/data_masking/base.py | 11 ++++++----- .../utilities/data_masking/constants.py | 1 + .../utilities/data_masking/provider.py | 5 ++--- .../data_masking/providers/aws_encryption_sdk.py | 8 ++++---- tests/e2e/data_masking/test_data_masking.py | 1 - tests/unit/data_masking/setup.py | 6 +++--- tests/unit/data_masking/test_data_masking.py | 6 +++--- 8 files changed, 19 insertions(+), 21 deletions(-) create mode 100644 aws_lambda_powertools/utilities/data_masking/constants.py diff --git a/aws_lambda_powertools/shared/constants.py b/aws_lambda_powertools/shared/constants.py index eede3120833..0cde7582976 100644 --- a/aws_lambda_powertools/shared/constants.py +++ b/aws_lambda_powertools/shared/constants.py @@ -6,8 +6,6 @@ LOGGER_LOG_EVENT_ENV: str = "POWERTOOLS_LOGGER_LOG_EVENT" LOGGER_LOG_DEDUPLICATION_ENV: str = "POWERTOOLS_LOG_DEDUPLICATION_DISABLED" -DATA_MASKING_STRING: str = "*****" - MIDDLEWARE_FACTORY_TRACE_ENV: str = "POWERTOOLS_TRACE_MIDDLEWARES" METRICS_NAMESPACE_ENV: str = "POWERTOOLS_METRICS_NAMESPACE" diff --git a/aws_lambda_powertools/utilities/data_masking/base.py b/aws_lambda_powertools/utilities/data_masking/base.py index e67898fb58c..e152841de72 100644 --- a/aws_lambda_powertools/utilities/data_masking/base.py +++ b/aws_lambda_powertools/utilities/data_masking/base.py @@ -1,15 +1,16 @@ import json from typing import Union -from aws_lambda_powertools.utilities.data_masking.provider import Provider +from aws_lambda_powertools.utilities.data_masking.provider import BaseProvider class DataMasking: def __init__(self, provider=None): - if provider is None: - self.provider = Provider() - else: - self.provider = provider + # if provider is None: + # self.provider = Provider() + # else: + # self.provider = provider + self.provider = provider or BaseProvider() def encrypt(self, data, fields=None, **provider_options): return self._apply_action(data, fields, self.provider.encrypt, **provider_options) diff --git a/aws_lambda_powertools/utilities/data_masking/constants.py b/aws_lambda_powertools/utilities/data_masking/constants.py new file mode 100644 index 00000000000..cd493378629 --- /dev/null +++ b/aws_lambda_powertools/utilities/data_masking/constants.py @@ -0,0 +1 @@ +DATA_MASKING_STRING: str = "*****" diff --git a/aws_lambda_powertools/utilities/data_masking/provider.py b/aws_lambda_powertools/utilities/data_masking/provider.py index 370510f8b9d..84fff5e815c 100644 --- a/aws_lambda_powertools/utilities/data_masking/provider.py +++ b/aws_lambda_powertools/utilities/data_masking/provider.py @@ -1,10 +1,9 @@ from abc import abstractmethod from collections.abc import Iterable -from aws_lambda_powertools.shared.constants import DATA_MASKING_STRING +from aws_lambda_powertools.utilities.data_masking.constants import DATA_MASKING_STRING - -class Provider: +class BaseProvider(): """ When you try to create an instance of a subclass that does not implement the encrypt method, you will get a NotImplementedError with a message that says the method is not implemented: diff --git a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py index 44b68dc1745..eca7062fbb2 100644 --- a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py +++ b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py @@ -9,7 +9,7 @@ StrictAwsKmsMasterKeyProvider, ) -from aws_lambda_powertools.utilities.data_masking.provider import Provider +from aws_lambda_powertools.utilities.data_masking.provider import BaseProvider from aws_lambda_powertools.shared.user_agent import register_feature_to_botocore_session @@ -31,7 +31,7 @@ def __call__(cls, *args, **provider_options): # NOTE: You can also set max messages/bytes per data key -class AwsEncryptionSdkProvider(Provider): +class AwsEncryptionSdkProvider(BaseProvider): cache = LocalCryptoMaterialsCache(CACHE_CAPACITY) session = botocore.session.Session() register_feature_to_botocore_session(session, "data-masking") @@ -47,12 +47,12 @@ def __init__(self, keys: List[str], client: Optional[EncryptionSDKClient] = None max_messages_encrypted=MAX_MESSAGES, ) - def encrypt(self, data: Union[bytes, str], **provider_options) -> str: + def encrypt(self, data: Union[bytes, str], **provider_options) -> Union[bytes, str]: ciphertext, _ = self.client.encrypt(source=data, materials_manager=self.cache_cmm, **provider_options) ciphertext = base64.b64encode(ciphertext).decode() return ciphertext - def decrypt(self, data: str, **provider_options) -> bytes: + def decrypt(self, data: str, **provider_options) -> Union[bytes, str]: ciphertext_decoded = base64.b64decode(data) ciphertext, _ = self.client.decrypt( source=ciphertext_decoded, key_provider=self.key_provider, **provider_options diff --git a/tests/e2e/data_masking/test_data_masking.py b/tests/e2e/data_masking/test_data_masking.py index 75dfea221d6..6cb61ce6cf7 100644 --- a/tests/e2e/data_masking/test_data_masking.py +++ b/tests/e2e/data_masking/test_data_masking.py @@ -5,7 +5,6 @@ from tests.e2e.utils import data_fetcher from aws_lambda_powertools.utilities.data_masking.base import DataMasking from aws_lambda_powertools.utilities.data_masking.providers.aws_encryption_sdk import AwsEncryptionSdkProvider -from aws_lambda_powertools.shared.constants import DATA_MASKING_STRING @pytest.fixture diff --git a/tests/unit/data_masking/setup.py b/tests/unit/data_masking/setup.py index f2fd6bc969f..5655d4cc84d 100644 --- a/tests/unit/data_masking/setup.py +++ b/tests/unit/data_masking/setup.py @@ -1,12 +1,12 @@ import json import copy from itsdangerous.url_safe import URLSafeSerializer -from aws_lambda_powertools.shared.constants import DATA_MASKING_STRING +from aws_lambda_powertools.utilities.data_masking.constants import DATA_MASKING_STRING from aws_lambda_powertools.utilities.data_masking.base import DataMasking -from aws_lambda_powertools.utilities.data_masking.provider import Provider +from aws_lambda_powertools.utilities.data_masking.provider import BaseProvider -class MyEncryptionProvider(Provider): +class MyEncryptionProvider(BaseProvider): """Custom encryption provider class""" def __init__(self, keys, salt=None): diff --git a/tests/unit/data_masking/test_data_masking.py b/tests/unit/data_masking/test_data_masking.py index 3ed1b203d20..ed471705629 100644 --- a/tests/unit/data_masking/test_data_masking.py +++ b/tests/unit/data_masking/test_data_masking.py @@ -1,12 +1,12 @@ import json import pytest from itsdangerous.url_safe import URLSafeSerializer -from aws_lambda_powertools.shared.constants import DATA_MASKING_STRING +from aws_lambda_powertools.utilities.data_masking.constants import DATA_MASKING_STRING from aws_lambda_powertools.utilities.data_masking.base import DataMasking -from aws_lambda_powertools.utilities.data_masking.provider import Provider +from aws_lambda_powertools.utilities.data_masking.provider import BaseProvider -class MyEncryptionProvider(Provider): +class MyEncryptionProvider(BaseProvider): """Custom encryption provider class""" def __init__(self, keys, salt=None): From bbeaa4e4d17085250468c38359e6d9303e59c26a Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Tue, 22 Aug 2023 00:44:06 +0000 Subject: [PATCH 27/56] Add check for encryption_context in Encryption SDK --- .../utilities/data_masking/base.py | 8 ++--- .../utilities/data_masking/provider.py | 10 +++--- .../providers/aws_encryption_sdk.py | 16 ++++++--- tests/e2e/data_masking/test_data_masking.py | 34 +++++++++++++++++-- .../data_masking/test_aws_encryption_sdk.py | 10 ------ 5 files changed, 51 insertions(+), 27 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_masking/base.py b/aws_lambda_powertools/utilities/data_masking/base.py index e152841de72..ae4deaea07c 100644 --- a/aws_lambda_powertools/utilities/data_masking/base.py +++ b/aws_lambda_powertools/utilities/data_masking/base.py @@ -6,10 +6,6 @@ class DataMasking: def __init__(self, provider=None): - # if provider is None: - # self.provider = Provider() - # else: - # self.provider = provider self.provider = provider or BaseProvider() def encrypt(self, data, fields=None, **provider_options): @@ -18,7 +14,7 @@ def encrypt(self, data, fields=None, **provider_options): def decrypt(self, data, fields=None, **provider_options): return self._apply_action(data, fields, self.provider.decrypt, **provider_options) - def mask(self, data, fields=None, **provider_options): + def mask(self, data, fields=None, **provider_options) -> str: return self._apply_action(data, fields, self.provider.mask, **provider_options) def _apply_action(self, data, fields, action, **provider_options): @@ -27,7 +23,7 @@ def _apply_action(self, data, fields, action, **provider_options): else: return action(data, **provider_options) - def _apply_action_to_fields(self, data: Union[dict, str], fields, action, **provider_options) -> str: + def _apply_action_to_fields(self, data: Union[dict, str], fields, action, **provider_options) -> Union[dict, str]: """ Apply the specified action to the specified fields in the input data. diff --git a/aws_lambda_powertools/utilities/data_masking/provider.py b/aws_lambda_powertools/utilities/data_masking/provider.py index 84fff5e815c..4f0b3a51968 100644 --- a/aws_lambda_powertools/utilities/data_masking/provider.py +++ b/aws_lambda_powertools/utilities/data_masking/provider.py @@ -1,23 +1,25 @@ from abc import abstractmethod from collections.abc import Iterable +from typing import Union from aws_lambda_powertools.utilities.data_masking.constants import DATA_MASKING_STRING -class BaseProvider(): + +class BaseProvider: """ When you try to create an instance of a subclass that does not implement the encrypt method, you will get a NotImplementedError with a message that says the method is not implemented: """ @abstractmethod - def encrypt(self, data): + def encrypt(self, data) -> Union[bytes, str]: raise NotImplementedError("Subclasses must implement encrypt()") @abstractmethod - def decrypt(self, data): + def decrypt(self, data) -> Union[bytes, str]: raise NotImplementedError("Subclasses must implement decrypt()") - def mask(self, data): + def mask(self, data) -> str: if isinstance(data, (str, dict, bytes)): return DATA_MASKING_STRING elif isinstance(data, Iterable): diff --git a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py index eca7062fbb2..85727616006 100644 --- a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py +++ b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py @@ -36,7 +36,7 @@ class AwsEncryptionSdkProvider(BaseProvider): session = botocore.session.Session() register_feature_to_botocore_session(session, "data-masking") - def __init__(self, keys: List[str], client: Optional[EncryptionSDKClient] = None) -> None: + def __init__(self, keys: List[str], client: Optional[EncryptionSDKClient] = None): self.client = client or EncryptionSDKClient() self.keys = keys self.key_provider = StrictAwsKmsMasterKeyProvider(key_ids=self.keys, botocore_session=self.session) @@ -47,14 +47,22 @@ def __init__(self, keys: List[str], client: Optional[EncryptionSDKClient] = None max_messages_encrypted=MAX_MESSAGES, ) - def encrypt(self, data: Union[bytes, str], **provider_options) -> Union[bytes, str]: + def encrypt(self, data: Union[bytes, str], **provider_options) -> str: ciphertext, _ = self.client.encrypt(source=data, materials_manager=self.cache_cmm, **provider_options) ciphertext = base64.b64encode(ciphertext).decode() return ciphertext - def decrypt(self, data: str, **provider_options) -> Union[bytes, str]: + def decrypt(self, data: str, **provider_options) -> bytes: ciphertext_decoded = base64.b64decode(data) - ciphertext, _ = self.client.decrypt( + + expected_context = provider_options.pop("encryption_context", {}) + + ciphertext, decryptor_header = self.client.decrypt( source=ciphertext_decoded, key_provider=self.key_provider, **provider_options ) + + for key, value in expected_context.items(): + if decryptor_header.encryption_context.get(key) != value: + raise ValueError(f"Encryption Context does not match expected value for key: {key}") + return ciphertext diff --git a/tests/e2e/data_masking/test_data_masking.py b/tests/e2e/data_masking/test_data_masking.py index 6cb61ce6cf7..06bed84e2f9 100644 --- a/tests/e2e/data_masking/test_data_masking.py +++ b/tests/e2e/data_masking/test_data_masking.py @@ -47,21 +47,49 @@ def test_encryption(data_masker): assert decrypted_data == value -# TODO: Waiting on EncryptionSDK team to answer tt.amazon.com/V1005246120 @pytest.mark.xdist_group(name="data_masking") def test_encryption_context(data_masker): # GIVEN an instantiation of DataMasking with the AWS encryption provider value = bytes(str([1, 2, "string", 4.5]), "utf-8") - # WHEN encrypting and then decrypting the encrypted data + # WHEN encrypting and then decrypting the encrypted data with an encryption_context encrypted_data = data_masker.encrypt(value, encryption_context={"this": "is_secure"}) - decrypted_data = data_masker.decrypt(encrypted_data) + decrypted_data = data_masker.decrypt(encrypted_data, encryption_context={"this": "is_secure"}) # THEN the result is the original input data assert decrypted_data == value +@pytest.mark.xdist_group(name="data_masking") +def test_encryption_context_fail(data_masker): + # GIVEN an instantiation of DataMasking with the AWS encryption provider + + value = bytes(str([1, 2, "string", 4.5]), "utf-8") + + # WHEN encrypting with a encryption_context + encrypted_data = data_masker.encrypt(value, encryption_context={"this": "is_secure"}) + + # THEN decrypting with a different encryption_context should raise a ValueError + with pytest.raises(ValueError): + data_masker.decrypt(encrypted_data, encryption_context={"not": "same_context"}) + + +# TODO: this should fail +@pytest.mark.xdist_group(name="data_masking") +def test_encryption_no_context_fail(data_masker): + # GIVEN an instantiation of DataMasking with the AWS encryption provider + + value = bytes(str([1, 2, "string", 4.5]), "utf-8") + + # WHEN encrypting with an encryption_context + encrypted_data = data_masker.encrypt(value, encryption_context={"this": "is_secure"}) + + # THEN decrypting with no encryption_context should raise a ValueError + with pytest.raises(ValueError): + data_masker.decrypt(encrypted_data) + + # TODO: metaclass? @pytest.mark.xdist_group(name="data_masking") def test_encryption_key_fail(kms_key2_arn, data_masker): diff --git a/tests/functional/data_masking/test_aws_encryption_sdk.py b/tests/functional/data_masking/test_aws_encryption_sdk.py index 82a00744217..e3b0aef833e 100644 --- a/tests/functional/data_masking/test_aws_encryption_sdk.py +++ b/tests/functional/data_masking/test_aws_encryption_sdk.py @@ -67,13 +67,3 @@ def test_encrypt_decrypt_with_fields(value, fields, data_masker): assert decrypted_data == aws_encrypted_json_blob else: assert decrypted_data == aws_encrypted_with_fields - - -@patch("aws_encryption_sdk.EncryptionSDKClient") -def test_mock(get_encryption_sdk_client_mock): - get_encryption_sdk_client_mock.return_value = "mock_value" - - d_m = DataMasking(provider=AwsEncryptionSdkProvider(keys=["mock_value"])) - encrypted_data = d_m.encrypt(b"secret_data") - decrypted_data = d_m.decrypt(encrypted_data) - assert decrypted_data == b"secret_data" From 5b794f7bdfbcef2b9c022fbe72fd6d88d36afffe Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Tue, 22 Aug 2023 16:26:37 +0000 Subject: [PATCH 28/56] Fixing enc_context e2e tests --- tests/e2e/data_masking/test_data_masking.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/e2e/data_masking/test_data_masking.py b/tests/e2e/data_masking/test_data_masking.py index 06bed84e2f9..ed37d9053a1 100644 --- a/tests/e2e/data_masking/test_data_masking.py +++ b/tests/e2e/data_masking/test_data_masking.py @@ -62,7 +62,7 @@ def test_encryption_context(data_masker): @pytest.mark.xdist_group(name="data_masking") -def test_encryption_context_fail(data_masker): +def test_encryption_diff_context_fail(data_masker): # GIVEN an instantiation of DataMasking with the AWS encryption provider value = bytes(str([1, 2, "string", 4.5]), "utf-8") @@ -75,19 +75,18 @@ def test_encryption_context_fail(data_masker): data_masker.decrypt(encrypted_data, encryption_context={"not": "same_context"}) -# TODO: this should fail @pytest.mark.xdist_group(name="data_masking") def test_encryption_no_context_fail(data_masker): # GIVEN an instantiation of DataMasking with the AWS encryption provider value = bytes(str([1, 2, "string", 4.5]), "utf-8") - # WHEN encrypting with an encryption_context - encrypted_data = data_masker.encrypt(value, encryption_context={"this": "is_secure"}) + # WHEN encrypting with no encryption_context + encrypted_data = data_masker.encrypt(value) - # THEN decrypting with no encryption_context should raise a ValueError + # THEN decrypting with an encryption_context should raise a ValueError with pytest.raises(ValueError): - data_masker.decrypt(encrypted_data) + data_masker.decrypt(encrypted_data, encryption_context={"this": "is_secure"}) # TODO: metaclass? From 2955c9cd3ac2132a2335a175422e7c703c621cbc Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Tue, 22 Aug 2023 19:58:49 +0000 Subject: [PATCH 29/56] Added test to encrypt&decrypt from logs in e2e tests --- tests/e2e/data_masking/test_data_masking.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/e2e/data_masking/test_data_masking.py b/tests/e2e/data_masking/test_data_masking.py index ed37d9053a1..d801f2326fb 100644 --- a/tests/e2e/data_masking/test_data_masking.py +++ b/tests/e2e/data_masking/test_data_masking.py @@ -91,13 +91,14 @@ def test_encryption_no_context_fail(data_masker): # TODO: metaclass? @pytest.mark.xdist_group(name="data_masking") -def test_encryption_key_fail(kms_key2_arn, data_masker): +def test_encryption_key_fail(data_masker, kms_key2_arn): # GIVEN an instantiation of DataMasking with the AWS encryption provider with a certain key # WHEN encrypting and then decrypting the encrypted data value = bytes(str([1, 2, "string", 4.5]), "utf-8") encrypted_data = data_masker.encrypt(value) + # THEN when decrypting with a different key it should fail data_masker_key2 = DataMasking(provider=AwsEncryptionSdkProvider(keys=[kms_key2_arn])) with pytest.raises(DecryptKeyError): @@ -105,13 +106,13 @@ def test_encryption_key_fail(kms_key2_arn, data_masker): @pytest.mark.xdist_group(name="data_masking") -def test_masked_in_logs(basic_handler_fn, basic_handler_fn_arn): +def test_encrypted_in_logs(data_masker, basic_handler_fn, basic_handler_fn_arn): # GIVEN an instantiation of DataMasking with the AWS encryption provider - data_masker = DataMasking(provider=AwsEncryptionSdkProvider(keys=[kms_key1_arn])) - # WHEN masking a value and logging it - masked_data = data_masker.mask([1, 2, "string", 4.5]) - message = masked_data + # WHEN encrypting a value and logging it + value = bytes(str([1, 2, "string", 4.5]), "utf-8") + encrypted_data = data_masker.encrypt(value) + message = encrypted_data custom_key = "order_id" additional_keys = {custom_key: f"{uuid4()}"} payload = json.dumps({"message": message, "append_keys": additional_keys}) @@ -119,7 +120,10 @@ def test_masked_in_logs(basic_handler_fn, basic_handler_fn_arn): _, execution_time = data_fetcher.get_lambda_response(lambda_arn=basic_handler_fn_arn, payload=payload) data_fetcher.get_lambda_response(lambda_arn=basic_handler_fn_arn, payload=payload) - # THEN the logs should show only the obfuscated data logs = data_fetcher.get_logs(function_name=basic_handler_fn, start_time=execution_time, minimum_log_entries=2) - assert logs.have_keys("message") is True + # THEN decrypting it from the logs should show the original value + for log in logs.get_log(key=custom_key): + encrypted_data = log.message + decrypted_data = data_masker.decrypt(encrypted_data) + assert decrypted_data == value From b15b8665fcefa1884ddb078888dfde8337438758 Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Sun, 3 Sep 2023 21:19:19 +0000 Subject: [PATCH 30/56] Added custom exception for enc_context mismatch, used pytest fixtures in unit tests --- .../providers/aws_encryption_sdk.py | 60 +++++++- tests/e2e/data_masking/test_data_masking.py | 24 ++-- tests/unit/data_masking/test_data_masking.py | 131 +++++++----------- 3 files changed, 118 insertions(+), 97 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py index 85727616006..e05bb58e2c0 100644 --- a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py +++ b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py @@ -8,11 +8,16 @@ LocalCryptoMaterialsCache, StrictAwsKmsMasterKeyProvider, ) - from aws_lambda_powertools.utilities.data_masking.provider import BaseProvider from aws_lambda_powertools.shared.user_agent import register_feature_to_botocore_session +class ContextMismatchError(Exception): + def __init__(self, key): + super().__init__(f"Encryption Context does not match expected value for key: {key}") + self.key = key + + class SingletonMeta(type): """Metaclass to cache class instances to optimize encryption""" @@ -32,27 +37,70 @@ def __call__(cls, *args, **provider_options): class AwsEncryptionSdkProvider(BaseProvider): - cache = LocalCryptoMaterialsCache(CACHE_CAPACITY) + """ + The AwsEncryptionSdkProvider is to be used as a Provider for the Datamasking class. + Example: + >>> data_masker = DataMasking(provider=AwsEncryptionSdkProvider(keys=[keyARN1, keyARN2,...,])) + >>> encrypted_data = data_masker.encrypt("a string") + "encrptedBase64String" + >>> decrypted_data = data_masker.decrypt(encrypted_data) + "a string" + """ + session = botocore.session.Session() register_feature_to_botocore_session(session, "data-masking") - def __init__(self, keys: List[str], client: Optional[EncryptionSDKClient] = None): + def __init__( + self, + keys: List[str], + client: Optional[EncryptionSDKClient] = None, + local_cache_capacity: Optional[int] = CACHE_CAPACITY, + max_cache_age_seconds: Optional[float] = MAX_ENTRY_AGE_SECONDS, + max_messages: Optional[int] = MAX_MESSAGES, + ): self.client = client or EncryptionSDKClient() self.keys = keys + self.cache = LocalCryptoMaterialsCache(local_cache_capacity) self.key_provider = StrictAwsKmsMasterKeyProvider(key_ids=self.keys, botocore_session=self.session) self.cache_cmm = CachingCryptoMaterialsManager( master_key_provider=self.key_provider, cache=self.cache, - max_age=MAX_ENTRY_AGE_SECONDS, - max_messages_encrypted=MAX_MESSAGES, + max_age=max_cache_age_seconds, + max_messages_encrypted=max_messages, ) def encrypt(self, data: Union[bytes, str], **provider_options) -> str: + """ + Encrypt data using the AwsEncryptionSdkProvider. + + Parameters: + - data (Union[bytes, str]): + The data to be encrypted. + - provider_options: + Additional options for the aws_encryption_sdk.EncryptionSDKClient + + Returns: + - ciphertext (str): + The encrypted data, as a base64-encoded string. + """ ciphertext, _ = self.client.encrypt(source=data, materials_manager=self.cache_cmm, **provider_options) ciphertext = base64.b64encode(ciphertext).decode() return ciphertext def decrypt(self, data: str, **provider_options) -> bytes: + """ + Decrypt data using AwsEncryptionSdkProvider. + + Parameters: + - data (Union[bytes, str]): + The encrypted data, as a base64-encoded string. + - provider_options: + Additional options for the aws_encryption_sdk.EncryptionSDKClient + + Returns: + - ciphertext (bytes): + The decrypted data in bytes + """ ciphertext_decoded = base64.b64decode(data) expected_context = provider_options.pop("encryption_context", {}) @@ -63,6 +111,6 @@ def decrypt(self, data: str, **provider_options) -> bytes: for key, value in expected_context.items(): if decryptor_header.encryption_context.get(key) != value: - raise ValueError(f"Encryption Context does not match expected value for key: {key}") + raise ContextMismatchError(key) return ciphertext diff --git a/tests/e2e/data_masking/test_data_masking.py b/tests/e2e/data_masking/test_data_masking.py index d801f2326fb..67a25fcd806 100644 --- a/tests/e2e/data_masking/test_data_masking.py +++ b/tests/e2e/data_masking/test_data_masking.py @@ -4,7 +4,10 @@ import pytest from tests.e2e.utils import data_fetcher from aws_lambda_powertools.utilities.data_masking.base import DataMasking -from aws_lambda_powertools.utilities.data_masking.providers.aws_encryption_sdk import AwsEncryptionSdkProvider +from aws_lambda_powertools.utilities.data_masking.providers.aws_encryption_sdk import ( + AwsEncryptionSdkProvider, + ContextMismatchError, +) @pytest.fixture @@ -52,17 +55,18 @@ def test_encryption_context(data_masker): # GIVEN an instantiation of DataMasking with the AWS encryption provider value = bytes(str([1, 2, "string", 4.5]), "utf-8") + context = {"this": "is_secure"} # WHEN encrypting and then decrypting the encrypted data with an encryption_context - encrypted_data = data_masker.encrypt(value, encryption_context={"this": "is_secure"}) - decrypted_data = data_masker.decrypt(encrypted_data, encryption_context={"this": "is_secure"}) + encrypted_data = data_masker.encrypt(value, encryption_context=context) + decrypted_data = data_masker.decrypt(encrypted_data, encryption_context=context) # THEN the result is the original input data assert decrypted_data == value @pytest.mark.xdist_group(name="data_masking") -def test_encryption_diff_context_fail(data_masker): +def test_encryption_context_mismatch(data_masker): # GIVEN an instantiation of DataMasking with the AWS encryption provider value = bytes(str([1, 2, "string", 4.5]), "utf-8") @@ -70,8 +74,8 @@ def test_encryption_diff_context_fail(data_masker): # WHEN encrypting with a encryption_context encrypted_data = data_masker.encrypt(value, encryption_context={"this": "is_secure"}) - # THEN decrypting with a different encryption_context should raise a ValueError - with pytest.raises(ValueError): + # THEN decrypting with a different encryption_context should raise a ContextMismatchError + with pytest.raises(ContextMismatchError): data_masker.decrypt(encrypted_data, encryption_context={"not": "same_context"}) @@ -84,14 +88,14 @@ def test_encryption_no_context_fail(data_masker): # WHEN encrypting with no encryption_context encrypted_data = data_masker.encrypt(value) - # THEN decrypting with an encryption_context should raise a ValueError - with pytest.raises(ValueError): + # THEN decrypting with an encryption_context should raise a ContextMismatchError + with pytest.raises(ContextMismatchError): data_masker.decrypt(encrypted_data, encryption_context={"this": "is_secure"}) # TODO: metaclass? @pytest.mark.xdist_group(name="data_masking") -def test_encryption_key_fail(data_masker, kms_key2_arn): +def test_encryption_decryption_key_mismatch(data_masker, kms_key2_arn): # GIVEN an instantiation of DataMasking with the AWS encryption provider with a certain key # WHEN encrypting and then decrypting the encrypted data @@ -106,7 +110,7 @@ def test_encryption_key_fail(data_masker, kms_key2_arn): @pytest.mark.xdist_group(name="data_masking") -def test_encrypted_in_logs(data_masker, basic_handler_fn, basic_handler_fn_arn): +def test_encryption_in_logs(data_masker, basic_handler_fn, basic_handler_fn_arn): # GIVEN an instantiation of DataMasking with the AWS encryption provider # WHEN encrypting a value and logging it diff --git a/tests/unit/data_masking/test_data_masking.py b/tests/unit/data_masking/test_data_masking.py index ed471705629..e9e33dccee3 100644 --- a/tests/unit/data_masking/test_data_masking.py +++ b/tests/unit/data_masking/test_data_masking.py @@ -26,19 +26,17 @@ def decrypt(self, data: str) -> str: return serialize.loads(data) -# @pytest.fixture -# def data_masker(): -# return DataMasking() +@pytest.fixture +def data_masker() -> DataMasking: + return DataMasking() -# @pytest.fixture -# def custom_data_masker(): -# return DataMasking(provider=MyEncryptionProvider(keys="secret-key")) +@pytest.fixture +def custom_data_masker() -> DataMasking: + return DataMasking(provider=MyEncryptionProvider(keys="secret-key")) -def test_mask_int(): - data_masker = DataMasking() - +def test_mask_int(data_masker): # GIVEN an int data type # WHEN mask is called with no fields argument @@ -48,9 +46,7 @@ def test_mask_int(): assert masked_string == DATA_MASKING_STRING -def test_mask_float(): - data_masker = DataMasking() - +def test_mask_float(data_masker): # GIVEN a float data type # WHEN mask is called with no fields argument @@ -60,9 +56,7 @@ def test_mask_float(): assert masked_string == DATA_MASKING_STRING -def test_mask_bool(): - data_masker = DataMasking() - +def test_mask_bool(data_masker): # GIVEN a bool data type # WHEN mask is called with no fields argument @@ -72,9 +66,7 @@ def test_mask_bool(): assert masked_string == DATA_MASKING_STRING -def test_mask_none(): - data_masker = DataMasking() - +def test_mask_none(data_masker): # GIVEN a None data type # WHEN mask is called with no fields argument @@ -84,9 +76,7 @@ def test_mask_none(): assert masked_string == DATA_MASKING_STRING -def test_mask_str(): - data_masker = DataMasking() - +def test_mask_str(data_masker): # GIVEN a str data type # WHEN mask is called with no fields argument @@ -96,9 +86,7 @@ def test_mask_str(): assert masked_string == DATA_MASKING_STRING -def test_mask_list(): - data_masker = DataMasking() - +def test_mask_list(data_masker): # GIVEN a list data type # WHEN mask is called with no fields argument @@ -108,9 +96,7 @@ def test_mask_list(): assert masked_string == [DATA_MASKING_STRING, DATA_MASKING_STRING, DATA_MASKING_STRING, DATA_MASKING_STRING] -def test_mask_dict(): - data_masker = DataMasking() - +def test_mask_dict(data_masker): # GIVEN a dict data type data = { "a": { @@ -126,9 +112,7 @@ def test_mask_dict(): assert masked_string == DATA_MASKING_STRING -def test_mask_dict_with_fields(): - data_masker = DataMasking() - +def test_mask_dict_with_fields(data_masker): # GIVEN the data type is a dictionary data = { "a": { @@ -146,9 +130,7 @@ def test_mask_dict_with_fields(): } -def test_mask_json_dict_with_fields(): - data_masker = DataMasking() - +def test_mask_json_dict_with_fields(data_masker): # GIVEN the data type is a json representation of a dictionary data = json.dumps( { @@ -168,81 +150,74 @@ def test_mask_json_dict_with_fields(): } -def test_encrypt_decrypt_int(): +def test_encrypt_decrypt_int(custom_data_masker): # GIVEN an instantiation of DataMasking with a Provider - data_masker = DataMasking(provider=MyEncryptionProvider(keys="secret-key")) # WHEN encrypting and then decrypting an int - encrypted_data = data_masker.encrypt(42) - decrypted_data = data_masker.decrypt(encrypted_data) + encrypted_data = custom_data_masker.encrypt(42) + decrypted_data = custom_data_masker.decrypt(encrypted_data) # THEN the result is the original input data assert decrypted_data == 42 -def test_encrypt_decrypt_float(): +def test_encrypt_decrypt_float(custom_data_masker): # GIVEN an instantiation of DataMasking with a Provider - data_masker = DataMasking(provider=MyEncryptionProvider(keys="secret-key")) # WHEN encrypting and then decrypting a float - encrypted_data = data_masker.encrypt(4.2) - decrypted_data = data_masker.decrypt(encrypted_data) + encrypted_data = custom_data_masker.encrypt(4.2) + decrypted_data = custom_data_masker.decrypt(encrypted_data) # THEN the result is the original input data assert decrypted_data == 4.2 -def test_encrypt_decrypt_bool(): +def test_encrypt_decrypt_bool(custom_data_masker): # GIVEN an instantiation of DataMasking with a Provider - data_masker = DataMasking(provider=MyEncryptionProvider(keys="secret-key")) # WHEN encrypting and then decrypting a bool - encrypted_data = data_masker.encrypt(True) - decrypted_data = data_masker.decrypt(encrypted_data) + encrypted_data = custom_data_masker.encrypt(True) + decrypted_data = custom_data_masker.decrypt(encrypted_data) # THEN the result is the original input data assert decrypted_data == True -def test_encrypt_decrypt_none(): +def test_encrypt_decrypt_none(custom_data_masker): # GIVEN an instantiation of DataMasking with a Provider - data_masker = DataMasking(provider=MyEncryptionProvider(keys="secret-key")) # WHEN encrypting and then decrypting a None type - encrypted_data = data_masker.encrypt(None) - decrypted_data = data_masker.decrypt(encrypted_data) + encrypted_data = custom_data_masker.encrypt(None) + decrypted_data = custom_data_masker.decrypt(encrypted_data) # THEN the result is the original input data assert decrypted_data == None -def test_encrypt_decrypt_str(): +def test_encrypt_decrypt_str(custom_data_masker): # GIVEN an instantiation of DataMasking with a Provider - data_masker = DataMasking(provider=MyEncryptionProvider(keys="secret-key")) # WHEN encrypting and then decrypting a string - encrypted_data = data_masker.encrypt("this is a string") - decrypted_data = data_masker.decrypt(encrypted_data) + encrypted_data = custom_data_masker.encrypt("this is a string") + decrypted_data = custom_data_masker.decrypt(encrypted_data) # THEN the result is the original input data assert decrypted_data == "this is a string" -def test_encrypt_decrypt_list(): +def test_encrypt_decrypt_list(custom_data_masker): # GIVEN an instantiation of DataMasking with a Provider - data_masker = DataMasking(provider=MyEncryptionProvider(keys="secret-key")) # WHEN encrypting and then decrypting a list - encrypted_data = data_masker.encrypt([1, 2, "string", 4]) - decrypted_data = data_masker.decrypt(encrypted_data) + encrypted_data = custom_data_masker.encrypt([1, 2, "string", 4]) + decrypted_data = custom_data_masker.decrypt(encrypted_data) # THEN the result is the original input data assert decrypted_data == [1, 2, "string", 4] -def test_dict_encryption_with_fields(): +def test_dict_encryption_with_fields(custom_data_masker): # GIVEN an instantiation of DataMasking with a Provider - data_masker = DataMasking(provider=MyEncryptionProvider(keys="secret-key")) data = { "a": { @@ -252,16 +227,15 @@ def test_dict_encryption_with_fields(): } # WHEN encrypting and decrypting the data with a list of fields - encrypted_data = data_masker.encrypt(data, fields=["a.1.None", "a.b.3.4"]) - decrypted_data = data_masker.decrypt(encrypted_data, fields=["a.1.None", "a.b.3.4"]) + encrypted_data = custom_data_masker.encrypt(data, fields=["a.1.None", "a.b.3.4"]) + decrypted_data = custom_data_masker.decrypt(encrypted_data, fields=["a.1.None", "a.b.3.4"]) # THEN the result is the original input data assert decrypted_data == data -def test_json_encryption_with_fields(): +def test_json_encryption_with_fields(custom_data_masker): # GIVEN an instantiation of DataMasking with a Provider - data_masker = DataMasking(provider=MyEncryptionProvider(keys="secret-key")) data = json.dumps( { @@ -273,16 +247,15 @@ def test_json_encryption_with_fields(): ) # WHEN encrypting and decrypting a json representation of a dictionary with a list of fields - encrypted_data = data_masker.encrypt(data, fields=["a.1.None", "a.b.3.4"]) - decrypted_data = data_masker.decrypt(encrypted_data, fields=["a.1.None", "a.b.3.4"]) + encrypted_data = custom_data_masker.encrypt(data, fields=["a.1.None", "a.b.3.4"]) + decrypted_data = custom_data_masker.decrypt(encrypted_data, fields=["a.1.None", "a.b.3.4"]) # THEN the result is the original input data assert decrypted_data == json.loads(data) -def test_big_data_encryption_with_fields(): +def test_big_data_encryption_with_fields(custom_data_masker): # GIVEN an instantiation of DataMasking with a Provider - data_masker = DataMasking(provider=MyEncryptionProvider(keys="secret-key")) # 10kb JSON blob for latency testing data = { @@ -313,16 +286,15 @@ def test_big_data_encryption_with_fields(): } # WHEN encrypting and decrypting the data with a list of fields - encrypted_data = data_masker.encrypt(data, fields=["address.street", "job_history.company"]) - decrypted_data = data_masker.decrypt(encrypted_data, fields=["address.street", "job_history.company"]) + encrypted_data = custom_data_masker.encrypt(data, fields=["address.street", "job_history.company"]) + decrypted_data = custom_data_masker.decrypt(encrypted_data, fields=["address.street", "job_history.company"]) # THEN the result is the original input data assert decrypted_data == data -def test_encrypt_not_implemented(): +def test_encrypt_not_implemented(data_masker): # GIVEN DataMasking is not initialized with a Provider - data_masker = DataMasking() # WHEN attempting to call the encrypt method on the data @@ -331,9 +303,8 @@ def test_encrypt_not_implemented(): data_masker.encrypt("hello world") -def test_decrypt_not_implemented(): +def test_decrypt_not_implemented(data_masker): # GIVEN DataMasking is not initialized with a Provider - data_masker = DataMasking() # WHEN attempting to call the decrypt method on the data @@ -342,9 +313,8 @@ def test_decrypt_not_implemented(): data_masker.decrypt("hello world") -def test_parsing_unsupported_data_type(): +def test_parsing_unsupported_data_type(data_masker): # GIVEN an initialization of the DataMasking class - data_masker = DataMasking() # WHEN attempting to pass in a list of fields with input data that is not a dict @@ -353,9 +323,9 @@ def test_parsing_unsupported_data_type(): data_masker.mask(42, ["this.field"]) -def test_parsing_nonexistent_fields(): +def test_parsing_nonexistent_fields(data_masker): # GIVEN an initialization of the DataMasking class - data_masker = DataMasking() + data = { "3": { "1": {"None": "hello", "four": "world"}, @@ -370,9 +340,9 @@ def test_parsing_nonexistent_fields(): data_masker.mask(data, ["3.1.True"]) -def test_parsing_nonstring_fields(): +def test_parsing_nonstring_fields(data_masker): # GIVEN an initialization of the DataMasking class - data_masker = DataMasking() + data = { "3": { "1": {"None": "hello", "four": "world"}, @@ -387,9 +357,8 @@ def test_parsing_nonstring_fields(): assert masked == {"3": {"1": {"None": "hello", "four": "world"}, "4": DATA_MASKING_STRING}} -def test_parsing_nonstring_keys_and_fields(): +def test_parsing_nonstring_keys_and_fields(data_masker): # GIVEN an initialization of the DataMasking class - data_masker = DataMasking() # WHEN the input data is a dictionary with integer keys data = { From ee3dddcbfe98c959cc5b6143b7e0111422c2e497 Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Mon, 4 Sep 2023 01:56:09 +0000 Subject: [PATCH 31/56] Added some docstrings and typing --- .../utilities/data_masking/base.py | 61 ++++++++++++++----- .../utilities/data_masking/provider.py | 8 +-- .../providers/aws_encryption_sdk.py | 30 +++++---- 3 files changed, 66 insertions(+), 33 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_masking/base.py b/aws_lambda_powertools/utilities/data_masking/base.py index ae4deaea07c..0aab71050ba 100644 --- a/aws_lambda_powertools/utilities/data_masking/base.py +++ b/aws_lambda_powertools/utilities/data_masking/base.py @@ -1,11 +1,11 @@ import json -from typing import Union +from typing import Union, Optional from aws_lambda_powertools.utilities.data_masking.provider import BaseProvider class DataMasking: - def __init__(self, provider=None): + def __init__(self, provider: Optional[BaseProvider] = None): self.provider = provider or BaseProvider() def encrypt(self, data, fields=None, **provider_options): @@ -18,6 +18,26 @@ def mask(self, data, fields=None, **provider_options) -> str: return self._apply_action(data, fields, self.provider.mask, **provider_options) def _apply_action(self, data, fields, action, **provider_options): + """ + Helper method to determine whether to apply a given action to the entire input data + or to specific fields if the 'fields' argument is specified. + + Parameters + ---------- + data : any + The input data to process. + fields : Optional[List[any]] = None + A list of fields to apply the action to. If 'None', the action is applied to the entire 'data'. + action : Callable + The action to apply to the data. It should be a callable that performs an operation on the data + and returns the modified value. + + Returns + ------- + any + The modified data after applying the action. + """ + if fields is not None: return self._apply_action_to_fields(data, fields, action, **provider_options) else: @@ -25,30 +45,39 @@ def _apply_action(self, data, fields, action, **provider_options): def _apply_action_to_fields(self, data: Union[dict, str], fields, action, **provider_options) -> Union[dict, str]: """ - Apply the specified action to the specified fields in the input data. - - This method is takes the input data, which can be either a dictionary or a JSON string representation + This method takes the input data, which can be either a dictionary or a JSON string representation of a dictionary, and applies a mask, an encryption, or a decryption to the specified fields. - Parameters: - data (Union[dict, str]): The input data to process. It can be either a dictionary or a JSON string + Parameters + ---------- + data : Union[dict, str]) + The input data to process. It can be either a dictionary or a JSON string representation of a dictionary. - fields (list): A list of fields to apply the action to. Each field can be specified as a string or + fields : List + A list of fields to apply the action to. Each field can be specified as a string or a list of strings representing nested keys in the dictionary. - action (callable): The action to apply to the fields. It should be a callable that takes the current + action : Callable + The action to apply to the fields. It should be a callable that takes the current value of the field as the first argument and any additional arguments that might be required for the action. It performs an operation on the current value using the provided arguments and returns the modified value. - **provider_options: Additional keyword arguments to pass to the 'action' function. + **provider_options: + Additional keyword arguments to pass to the 'action' function. - Returns: - str: A JSON string representation of the modified dictionary after applying the action to the + Returns + ------- + str + A JSON string representation of the modified dictionary after applying the action to the specified fields. - Raises: - ValueError: If 'fields' parameter is None. - TypeError: If the 'data' parameter is not a dictionary or a JSON string representation of a dictionary. - KeyError: If specified 'fields' do not exist in input data + Raises + ------- + ValueError + If 'fields' parameter is None. + TypeError + If the 'data' parameter is not a dictionary or a JSON string representation of a dictionary. + KeyError + If specified 'fields' do not exist in input data """ if fields is None: diff --git a/aws_lambda_powertools/utilities/data_masking/provider.py b/aws_lambda_powertools/utilities/data_masking/provider.py index 4f0b3a51968..2542a258b81 100644 --- a/aws_lambda_powertools/utilities/data_masking/provider.py +++ b/aws_lambda_powertools/utilities/data_masking/provider.py @@ -1,22 +1,20 @@ -from abc import abstractmethod +from abc import ABC from collections.abc import Iterable from typing import Union from aws_lambda_powertools.utilities.data_masking.constants import DATA_MASKING_STRING -class BaseProvider: +class BaseProvider(ABC): """ When you try to create an instance of a subclass that does not implement the encrypt method, you will get a NotImplementedError with a message that says the method is not implemented: """ - @abstractmethod def encrypt(self, data) -> Union[bytes, str]: raise NotImplementedError("Subclasses must implement encrypt()") - @abstractmethod - def decrypt(self, data) -> Union[bytes, str]: + def decrypt(self, data) -> any: raise NotImplementedError("Subclasses must implement decrypt()") def mask(self, data) -> str: diff --git a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py index e05bb58e2c0..1b83b37cc2a 100644 --- a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py +++ b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py @@ -39,7 +39,9 @@ def __call__(cls, *args, **provider_options): class AwsEncryptionSdkProvider(BaseProvider): """ The AwsEncryptionSdkProvider is to be used as a Provider for the Datamasking class. - Example: + + Example + ------- >>> data_masker = DataMasking(provider=AwsEncryptionSdkProvider(keys=[keyARN1, keyARN2,...,])) >>> encrypted_data = data_masker.encrypt("a string") "encrptedBase64String" @@ -73,14 +75,16 @@ def encrypt(self, data: Union[bytes, str], **provider_options) -> str: """ Encrypt data using the AwsEncryptionSdkProvider. - Parameters: - - data (Union[bytes, str]): + Parameters + ------- + data : Union[bytes, str] The data to be encrypted. - - provider_options: + provider_options Additional options for the aws_encryption_sdk.EncryptionSDKClient - Returns: - - ciphertext (str): + Returns + ------- + ciphertext : str The encrypted data, as a base64-encoded string. """ ciphertext, _ = self.client.encrypt(source=data, materials_manager=self.cache_cmm, **provider_options) @@ -91,14 +95,16 @@ def decrypt(self, data: str, **provider_options) -> bytes: """ Decrypt data using AwsEncryptionSdkProvider. - Parameters: - - data (Union[bytes, str]): - The encrypted data, as a base64-encoded string. - - provider_options: + Parameters + ------- + data : Union[bytes, str] + The encrypted data, as a base64-encoded string + provider_options Additional options for the aws_encryption_sdk.EncryptionSDKClient - Returns: - - ciphertext (bytes): + Returns + ------- + ciphertext : bytes The decrypted data in bytes """ ciphertext_decoded = base64.b64decode(data) From a79f3df115b18286103ae29690dd7114cf203280 Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Tue, 5 Sep 2023 23:30:26 +0000 Subject: [PATCH 32/56] Added test for using DataMasking in a lambda handler, wip due to incorrect dependencies --- .../utilities/data_masking/base.py | 2 +- .../data_masking/providers/aws_encryption_sdk.py | 2 +- tests/e2e/data_masking/handlers/basic_handler.py | 13 ++++++++++++- tests/e2e/data_masking/infrastructure.py | 4 ++-- tests/e2e/data_masking/test_data_masking.py | 15 +++++++++++++++ 5 files changed, 31 insertions(+), 5 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_masking/base.py b/aws_lambda_powertools/utilities/data_masking/base.py index 0aab71050ba..92d487fe6cc 100644 --- a/aws_lambda_powertools/utilities/data_masking/base.py +++ b/aws_lambda_powertools/utilities/data_masking/base.py @@ -61,7 +61,7 @@ def _apply_action_to_fields(self, data: Union[dict, str], fields, action, **prov value of the field as the first argument and any additional arguments that might be required for the action. It performs an operation on the current value using the provided arguments and returns the modified value. - **provider_options: + **provider_options: Additional keyword arguments to pass to the 'action' function. Returns diff --git a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py index 1b83b37cc2a..d8a1f3aa1df 100644 --- a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py +++ b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py @@ -39,7 +39,7 @@ def __call__(cls, *args, **provider_options): class AwsEncryptionSdkProvider(BaseProvider): """ The AwsEncryptionSdkProvider is to be used as a Provider for the Datamasking class. - + Example ------- >>> data_masker = DataMasking(provider=AwsEncryptionSdkProvider(keys=[keyARN1, keyARN2,...,])) diff --git a/tests/e2e/data_masking/handlers/basic_handler.py b/tests/e2e/data_masking/handlers/basic_handler.py index 0f0dd46b4aa..34c1415ef59 100644 --- a/tests/e2e/data_masking/handlers/basic_handler.py +++ b/tests/e2e/data_masking/handlers/basic_handler.py @@ -1,3 +1,5 @@ +from aws_lambda_powertools.utilities.data_masking.base import DataMasking +from aws_lambda_powertools.utilities.data_masking.providers.aws_encryption_sdk import AwsEncryptionSdkProvider from aws_lambda_powertools import Logger logger = Logger() @@ -5,7 +7,16 @@ @logger.inject_lambda_context def lambda_handler(event, context): + # Generating logs for test_encryption_in_logs test message, append_keys = event.get("message", ""), event.get("append_keys", {}) logger.append_keys(**append_keys) logger.info(message) - return "success" + + kms_key = event.get("kms_key") + data_masker = DataMasking(provider=AwsEncryptionSdkProvider(keys=[kms_key])) + value = bytes(str([1, 2, "string", 4.5]), "utf-8") + encrypted_data = data_masker.encrypt(value) + response = {} + response["encrypted_data"] = encrypted_data + + return response diff --git a/tests/e2e/data_masking/infrastructure.py b/tests/e2e/data_masking/infrastructure.py index e1de36fd730..9e6560a8eff 100644 --- a/tests/e2e/data_masking/infrastructure.py +++ b/tests/e2e/data_masking/infrastructure.py @@ -1,11 +1,11 @@ import aws_cdk.aws_kms as kms -from aws_cdk import CfnOutput +from aws_cdk import CfnOutput, Duration from tests.e2e.utils.infrastructure import BaseInfrastructure class DataMaskingStack(BaseInfrastructure): def create_resources(self): - self.create_lambda_functions() + self.create_lambda_functions(function_props={"timeout": Duration.seconds(10)}) key1 = kms.Key(self.stack, "MyKMSKey1", description="My KMS Key1") CfnOutput(self.stack, "KMSKey1Arn", value=key1.key_arn, description="ARN of the created KMS Key1") diff --git a/tests/e2e/data_masking/test_data_masking.py b/tests/e2e/data_masking/test_data_masking.py index 67a25fcd806..f142ad95bc6 100644 --- a/tests/e2e/data_masking/test_data_masking.py +++ b/tests/e2e/data_masking/test_data_masking.py @@ -131,3 +131,18 @@ def test_encryption_in_logs(data_masker, basic_handler_fn, basic_handler_fn_arn) encrypted_data = log.message decrypted_data = data_masker.decrypt(encrypted_data) assert decrypted_data == value + +# NOTE: This test is failing currently, need to find a fix for building correct dependencies +@pytest.mark.xdist_group(name="data_masking") +def test_encryption_in_handler(basic_handler_fn_arn, kms_key1_arn): + payload = {"kms_key": kms_key1_arn, "append_keys": {"order_id": f"{uuid4()}"}} + + # WHEN a lambda handler for encryption is invoked + handler_result, _ = data_fetcher.get_lambda_response(lambda_arn=basic_handler_fn_arn, payload=json.dumps(payload)) + + response = json.loads(handler_result["Payload"].read()) + encrypted_data = response["encrypted_data"] + decrypted_data = data_masker.decrypt(encrypted_data) + + # THEN decrypting the encrypted data from the response should result in the original value + assert decrypted_data == bytes(str([1, 2, "string", 4.5]), "utf-8") From 7883a4838ef007094789a73fb6a4e39cf57eb8a0 Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Fri, 8 Sep 2023 00:08:04 +0000 Subject: [PATCH 33/56] Revised singleton class to allow for one instance per different configuration --- .../utilities/data_masking/base.py | 4 +- .../providers/aws_encryption_sdk.py | 38 +++++++++++-------- tests/e2e/data_masking/conftest.py | 1 + .../data_masking/handlers/basic_handler.py | 2 +- tests/e2e/data_masking/infrastructure.py | 1 + tests/e2e/data_masking/test_data_masking.py | 23 +++++++++-- .../data_masking/test_aws_encryption_sdk.py | 3 -- tests/unit/data_masking/test_data_masking.py | 38 +++++++++++-------- 8 files changed, 70 insertions(+), 40 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_masking/base.py b/aws_lambda_powertools/utilities/data_masking/base.py index 92d487fe6cc..8cb7eb4c9d9 100644 --- a/aws_lambda_powertools/utilities/data_masking/base.py +++ b/aws_lambda_powertools/utilities/data_masking/base.py @@ -1,5 +1,5 @@ import json -from typing import Union, Optional +from typing import Optional, Union from aws_lambda_powertools.utilities.data_masking.provider import BaseProvider @@ -94,7 +94,7 @@ def _apply_action_to_fields(self, data: Union[dict, str], fields, action, **prov else: raise TypeError( "Unsupported data type. The 'data' parameter must be a dictionary or a JSON string " - "representation of a dictionary." + "representation of a dictionary.", ) for field in fields: diff --git a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py index d8a1f3aa1df..435ffcbcf9f 100644 --- a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py +++ b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py @@ -8,8 +8,9 @@ LocalCryptoMaterialsCache, StrictAwsKmsMasterKeyProvider, ) -from aws_lambda_powertools.utilities.data_masking.provider import BaseProvider + from aws_lambda_powertools.shared.user_agent import register_feature_to_botocore_session +from aws_lambda_powertools.utilities.data_masking.provider import BaseProvider class ContextMismatchError(Exception): @@ -18,25 +19,28 @@ def __init__(self, key): self.key = key -class SingletonMeta(type): - """Metaclass to cache class instances to optimize encryption""" +class Singleton: + _instances: Dict[Any, "AwsEncryptionSdkProvider"] = {} - _instances: Dict["AwsEncryptionSdkProvider", Any] = {} + def __new__(cls, *args, **kwargs): + # Generate a unique key based on the configuration + # Create a tuple by iterating through the values in kwargs, sorting them, + # and then adding them to the tuple. + config_key = tuple(v for value in kwargs.values() for v in sorted(value)) - def __call__(cls, *args, **provider_options): - if cls not in cls._instances: - instance = super().__call__(*args, **provider_options) - cls._instances[cls] = instance - return cls._instances[cls] + if config_key not in cls._instances: + cls._instances[config_key] = super(Singleton, cls).__new__(cls, *args) + print("in if class instances:", cls._instances) + return cls._instances[config_key] CACHE_CAPACITY: int = 100 -MAX_ENTRY_AGE_SECONDS: float = 300.0 -MAX_MESSAGES: int = 200 +MAX_CACHE_AGE_SECONDS: float = 300.0 +MAX_MESSAGES_ENCRYPTED: int = 200 # NOTE: You can also set max messages/bytes per data key -class AwsEncryptionSdkProvider(BaseProvider): +class AwsEncryptionSdkProvider(BaseProvider, Singleton): """ The AwsEncryptionSdkProvider is to be used as a Provider for the Datamasking class. @@ -57,8 +61,8 @@ def __init__( keys: List[str], client: Optional[EncryptionSDKClient] = None, local_cache_capacity: Optional[int] = CACHE_CAPACITY, - max_cache_age_seconds: Optional[float] = MAX_ENTRY_AGE_SECONDS, - max_messages: Optional[int] = MAX_MESSAGES, + max_cache_age_seconds: Optional[float] = MAX_CACHE_AGE_SECONDS, + max_messages_encrypted: Optional[int] = MAX_MESSAGES_ENCRYPTED, ): self.client = client or EncryptionSDKClient() self.keys = keys @@ -68,7 +72,7 @@ def __init__( master_key_provider=self.key_provider, cache=self.cache, max_age=max_cache_age_seconds, - max_messages_encrypted=max_messages, + max_messages_encrypted=max_messages_encrypted, ) def encrypt(self, data: Union[bytes, str], **provider_options) -> str: @@ -112,7 +116,9 @@ def decrypt(self, data: str, **provider_options) -> bytes: expected_context = provider_options.pop("encryption_context", {}) ciphertext, decryptor_header = self.client.decrypt( - source=ciphertext_decoded, key_provider=self.key_provider, **provider_options + source=ciphertext_decoded, + key_provider=self.key_provider, + **provider_options, ) for key, value in expected_context.items(): diff --git a/tests/e2e/data_masking/conftest.py b/tests/e2e/data_masking/conftest.py index 0430c274b80..f1892d7c0c9 100644 --- a/tests/e2e/data_masking/conftest.py +++ b/tests/e2e/data_masking/conftest.py @@ -1,4 +1,5 @@ import pytest + from tests.e2e.data_masking.infrastructure import DataMaskingStack diff --git a/tests/e2e/data_masking/handlers/basic_handler.py b/tests/e2e/data_masking/handlers/basic_handler.py index 34c1415ef59..93da716784c 100644 --- a/tests/e2e/data_masking/handlers/basic_handler.py +++ b/tests/e2e/data_masking/handlers/basic_handler.py @@ -1,6 +1,6 @@ +from aws_lambda_powertools import Logger from aws_lambda_powertools.utilities.data_masking.base import DataMasking from aws_lambda_powertools.utilities.data_masking.providers.aws_encryption_sdk import AwsEncryptionSdkProvider -from aws_lambda_powertools import Logger logger = Logger() diff --git a/tests/e2e/data_masking/infrastructure.py b/tests/e2e/data_masking/infrastructure.py index 9e6560a8eff..f6590622b49 100644 --- a/tests/e2e/data_masking/infrastructure.py +++ b/tests/e2e/data_masking/infrastructure.py @@ -1,5 +1,6 @@ import aws_cdk.aws_kms as kms from aws_cdk import CfnOutput, Duration + from tests.e2e.utils.infrastructure import BaseInfrastructure diff --git a/tests/e2e/data_masking/test_data_masking.py b/tests/e2e/data_masking/test_data_masking.py index f142ad95bc6..f8bd3b7e371 100644 --- a/tests/e2e/data_masking/test_data_masking.py +++ b/tests/e2e/data_masking/test_data_masking.py @@ -1,13 +1,15 @@ import json from uuid import uuid4 -from aws_encryption_sdk.exceptions import DecryptKeyError + import pytest -from tests.e2e.utils import data_fetcher +from aws_encryption_sdk.exceptions import DecryptKeyError + from aws_lambda_powertools.utilities.data_masking.base import DataMasking from aws_lambda_powertools.utilities.data_masking.providers.aws_encryption_sdk import ( AwsEncryptionSdkProvider, ContextMismatchError, ) +from tests.e2e.utils import data_fetcher @pytest.fixture @@ -93,7 +95,6 @@ def test_encryption_no_context_fail(data_masker): data_masker.decrypt(encrypted_data, encryption_context={"this": "is_secure"}) -# TODO: metaclass? @pytest.mark.xdist_group(name="data_masking") def test_encryption_decryption_key_mismatch(data_masker, kms_key2_arn): # GIVEN an instantiation of DataMasking with the AWS encryption provider with a certain key @@ -109,6 +110,21 @@ def test_encryption_decryption_key_mismatch(data_masker, kms_key2_arn): data_masker_key2.decrypt(encrypted_data) +def test_encryption_provider_singleton(data_masker, kms_key1_arn, kms_key2_arn): + data_masker_2 = DataMasking(provider=AwsEncryptionSdkProvider(keys=[kms_key1_arn])) + assert data_masker.provider is data_masker_2.provider + + # WHEN encrypting and then decrypting the encrypted data + encrypted_data = data_masker.encrypt("string") + decrypted_data = data_masker_2.decrypt(encrypted_data) + + # THEN the result is the original input data + assert decrypted_data == bytes("string", "utf-8") + + data_masker_3 = DataMasking(provider=AwsEncryptionSdkProvider(keys=[kms_key2_arn])) + assert data_masker_2.provider is not data_masker_3.provider + + @pytest.mark.xdist_group(name="data_masking") def test_encryption_in_logs(data_masker, basic_handler_fn, basic_handler_fn_arn): # GIVEN an instantiation of DataMasking with the AWS encryption provider @@ -132,6 +148,7 @@ def test_encryption_in_logs(data_masker, basic_handler_fn, basic_handler_fn_arn) decrypted_data = data_masker.decrypt(encrypted_data) assert decrypted_data == value + # NOTE: This test is failing currently, need to find a fix for building correct dependencies @pytest.mark.xdist_group(name="data_masking") def test_encryption_in_handler(basic_handler_fn_arn, kms_key1_arn): diff --git a/tests/functional/data_masking/test_aws_encryption_sdk.py b/tests/functional/data_masking/test_aws_encryption_sdk.py index e3b0aef833e..d31c242e03b 100644 --- a/tests/functional/data_masking/test_aws_encryption_sdk.py +++ b/tests/functional/data_masking/test_aws_encryption_sdk.py @@ -1,12 +1,9 @@ -from unittest.mock import patch - import pytest from aws_lambda_powertools.utilities.data_masking.base import DataMasking from aws_lambda_powertools.utilities.data_masking.providers.aws_encryption_sdk import AwsEncryptionSdkProvider from tests.unit.data_masking.setup import * - AWS_SDK_KEY = "arn:aws:kms:us-west-2:683517028648:key/269301eb-81eb-4067-ac72-98e8e49bf2b3" diff --git a/tests/unit/data_masking/test_data_masking.py b/tests/unit/data_masking/test_data_masking.py index e9e33dccee3..b30f18904f8 100644 --- a/tests/unit/data_masking/test_data_masking.py +++ b/tests/unit/data_masking/test_data_masking.py @@ -1,8 +1,10 @@ import json + import pytest from itsdangerous.url_safe import URLSafeSerializer -from aws_lambda_powertools.utilities.data_masking.constants import DATA_MASKING_STRING + from aws_lambda_powertools.utilities.data_masking.base import DataMasking +from aws_lambda_powertools.utilities.data_masking.constants import DATA_MASKING_STRING from aws_lambda_powertools.utilities.data_masking.provider import BaseProvider @@ -102,7 +104,7 @@ def test_mask_dict(data_masker): "a": { "1": {"None": "hello", "four": "world"}, "b": {"3": {"4": "goodbye", "e": "world"}}, - } + }, } # WHEN mask is called with no fields argument @@ -118,7 +120,7 @@ def test_mask_dict_with_fields(data_masker): "a": { "1": {"None": "hello", "four": "world"}, "b": {"3": {"4": "goodbye", "e": "world"}}, - } + }, } # WHEN mask is called with a list of fields specified @@ -126,7 +128,10 @@ def test_mask_dict_with_fields(data_masker): # THEN the result is only the specified fields are masked assert masked_string == { - "a": {"1": {"None": DATA_MASKING_STRING, "four": "world"}, "b": {"3": {"4": DATA_MASKING_STRING, "e": "world"}}} + "a": { + "1": {"None": DATA_MASKING_STRING, "four": "world"}, + "b": {"3": {"4": DATA_MASKING_STRING, "e": "world"}}, + }, } @@ -137,8 +142,8 @@ def test_mask_json_dict_with_fields(data_masker): "a": { "1": {"None": "hello", "four": "world"}, "b": {"3": {"4": "goodbye", "e": "world"}}, - } - } + }, + }, ) # WHEN mask is called with a list of fields specified @@ -146,7 +151,10 @@ def test_mask_json_dict_with_fields(data_masker): # THEN the result is only the specified fields are masked assert masked_json_string == { - "a": {"1": {"None": DATA_MASKING_STRING, "four": "world"}, "b": {"3": {"4": DATA_MASKING_STRING, "e": "world"}}} + "a": { + "1": {"None": DATA_MASKING_STRING, "four": "world"}, + "b": {"3": {"4": DATA_MASKING_STRING, "e": "world"}}, + }, } @@ -180,7 +188,7 @@ def test_encrypt_decrypt_bool(custom_data_masker): decrypted_data = custom_data_masker.decrypt(encrypted_data) # THEN the result is the original input data - assert decrypted_data == True + assert decrypted_data is True def test_encrypt_decrypt_none(custom_data_masker): @@ -191,7 +199,7 @@ def test_encrypt_decrypt_none(custom_data_masker): decrypted_data = custom_data_masker.decrypt(encrypted_data) # THEN the result is the original input data - assert decrypted_data == None + assert decrypted_data is None def test_encrypt_decrypt_str(custom_data_masker): @@ -223,7 +231,7 @@ def test_dict_encryption_with_fields(custom_data_masker): "a": { "1": {"None": "hello", "four": "world"}, "b": {"3": {"4": "goodbye", "e": "world"}}, - } + }, } # WHEN encrypting and decrypting the data with a list of fields @@ -242,8 +250,8 @@ def test_json_encryption_with_fields(custom_data_masker): "a": { "1": {"None": "hello", "four": "world"}, "b": {"3": {"4": "goodbye", "e": "world"}}, - } - } + }, + }, ) # WHEN encrypting and decrypting a json representation of a dictionary with a list of fields @@ -330,7 +338,7 @@ def test_parsing_nonexistent_fields(data_masker): "3": { "1": {"None": "hello", "four": "world"}, "4": {"33": {"5": "goodbye", "e": "world"}}, - } + }, } # WHEN attempting to pass in fields that do not exist in the input data @@ -347,7 +355,7 @@ def test_parsing_nonstring_fields(data_masker): "3": { "1": {"None": "hello", "four": "world"}, "4": {"33": {"5": "goodbye", "e": "world"}}, - } + }, } # WHEN attempting to pass in a list of fields that are not strings @@ -365,7 +373,7 @@ def test_parsing_nonstring_keys_and_fields(data_masker): 3: { "1": {"None": "hello", "four": "world"}, 4: {"33": {"5": "goodbye", "e": "world"}}, - } + }, } masked = data_masker.mask(data, fields=[3.4]) From 7127c9cd2eaa43611106096aaf6157dc7f4cdc25 Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Fri, 8 Sep 2023 16:33:22 +0000 Subject: [PATCH 34/56] Removed itsdangerous dependencies --- poetry.lock | 223 +++++++++++-------- pyproject.toml | 6 +- tests/unit/data_masking/setup.py | 39 +--- tests/unit/data_masking/test_data_masking.py | 170 -------------- 4 files changed, 145 insertions(+), 293 deletions(-) diff --git a/poetry.lock b/poetry.lock index 866dc4c9667..6d0c63e808e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. [[package]] name = "anyio" @@ -93,69 +93,69 @@ typeguard = ">=2.13.3,<2.14.0" [[package]] name = "aws-cdk-aws-apigatewayv2-alpha" -version = "2.93.0a0" +version = "2.95.0a0" description = "The CDK Construct Library for AWS::APIGatewayv2" optional = false python-versions = "~=3.7" files = [ - {file = "aws-cdk.aws-apigatewayv2-alpha-2.93.0a0.tar.gz", hash = "sha256:67b5c1cb5a3405f321a25da185ef949460793d9b33313f13544106bed2ce2180"}, - {file = "aws_cdk.aws_apigatewayv2_alpha-2.93.0a0-py3-none-any.whl", hash = "sha256:962d52fdfbc922f104381943d2edb0d535f1d793fd73f4518fb25fb7d63041f4"}, + {file = "aws-cdk.aws-apigatewayv2-alpha-2.95.0a0.tar.gz", hash = "sha256:ae6eb5fe0a06910a6aa0ac2fcc708585dba9cb288991811c8dd65520f121169b"}, + {file = "aws_cdk.aws_apigatewayv2_alpha-2.95.0a0-py3-none-any.whl", hash = "sha256:fab88cb71c2a3bec81851981c004c915114521944364b80f8a2e63cbaa79cbe1"}, ] [package.dependencies] -aws-cdk-lib = "2.93.0" +aws-cdk-lib = "2.95.0" constructs = ">=10.0.0,<11.0.0" -jsii = ">=1.87.0,<2.0.0" +jsii = ">=1.88.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" [[package]] name = "aws-cdk-aws-apigatewayv2-authorizers-alpha" -version = "2.93.0a0" +version = "2.95.0a0" description = "Authorizers for AWS APIGateway V2" optional = false python-versions = "~=3.7" files = [ - {file = "aws-cdk.aws-apigatewayv2-authorizers-alpha-2.93.0a0.tar.gz", hash = "sha256:495969d05ca85942bc3da6fac7d0a6df5893265b644921d9e891441ee845fdfd"}, - {file = "aws_cdk.aws_apigatewayv2_authorizers_alpha-2.93.0a0-py3-none-any.whl", hash = "sha256:6b22e4d94afa481c94fcafdc62c2cf22ea08ea0d985e738569b39da4ba4ffbb0"}, + {file = "aws-cdk.aws-apigatewayv2-authorizers-alpha-2.95.0a0.tar.gz", hash = "sha256:416b73ed01611d6a44578395b9236c1cba1f0878e88c0f6a84d0f43f00a7d15c"}, + {file = "aws_cdk.aws_apigatewayv2_authorizers_alpha-2.95.0a0-py3-none-any.whl", hash = "sha256:1ec30cf64211d8ee39dd6c51604faa01a448fe9a5611bd315172373dd6f18e85"}, ] [package.dependencies] -"aws-cdk.aws-apigatewayv2-alpha" = "2.93.0.a0" -aws-cdk-lib = "2.93.0" +"aws-cdk.aws-apigatewayv2-alpha" = "2.95.0.a0" +aws-cdk-lib = "2.95.0" constructs = ">=10.0.0,<11.0.0" -jsii = ">=1.87.0,<2.0.0" +jsii = ">=1.88.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" [[package]] name = "aws-cdk-aws-apigatewayv2-integrations-alpha" -version = "2.93.0a0" +version = "2.95.0a0" description = "Integrations for AWS APIGateway V2" optional = false python-versions = "~=3.7" files = [ - {file = "aws-cdk.aws-apigatewayv2-integrations-alpha-2.93.0a0.tar.gz", hash = "sha256:4c581f67634fab19b11025751e3ee825f055ee9d1bc77d9cbc5009f261456e62"}, - {file = "aws_cdk.aws_apigatewayv2_integrations_alpha-2.93.0a0-py3-none-any.whl", hash = "sha256:48479656dca9e446ae625e5936ddd940863bd478eb86cdd62889c6b5fee9f751"}, + {file = "aws-cdk.aws-apigatewayv2-integrations-alpha-2.95.0a0.tar.gz", hash = "sha256:59f41a6e4002687d20f01bcbc4249887f8b29d3dc27255976a0348e5a43cf158"}, + {file = "aws_cdk.aws_apigatewayv2_integrations_alpha-2.95.0a0-py3-none-any.whl", hash = "sha256:cc2236b2d901e2907ac523739e0d59be7dbf8c187801d3ef52ae2cb305402119"}, ] [package.dependencies] -"aws-cdk.aws-apigatewayv2-alpha" = "2.93.0.a0" -aws-cdk-lib = "2.93.0" +"aws-cdk.aws-apigatewayv2-alpha" = "2.95.0.a0" +aws-cdk-lib = "2.95.0" constructs = ">=10.0.0,<11.0.0" -jsii = ">=1.87.0,<2.0.0" +jsii = ">=1.88.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" [[package]] name = "aws-cdk-lib" -version = "2.93.0" +version = "2.95.0" description = "Version 2 of the AWS Cloud Development Kit library" optional = false python-versions = "~=3.7" files = [ - {file = "aws-cdk-lib-2.93.0.tar.gz", hash = "sha256:54252c8df547d2bd83584278529f47506fa2c27adcbfa623f00322b685f24c18"}, - {file = "aws_cdk_lib-2.93.0-py3-none-any.whl", hash = "sha256:063e7c1f2588a254766229130347fb60e0bd7dd2a6d222d3ae2aa145a6059554"}, + {file = "aws-cdk-lib-2.95.0.tar.gz", hash = "sha256:db87e5dc5cc2d7ea709f8618aa90fc815d43e750dc2e807b0a10801522d7a40b"}, + {file = "aws_cdk_lib-2.95.0-py3-none-any.whl", hash = "sha256:10c276e889eb7d036cd34be9076cc5f92e0960763b14dc79f2600caf5af13c4b"}, ] [package.dependencies] @@ -163,7 +163,7 @@ files = [ "aws-cdk.asset-kubectl-v20" = ">=2.1.2,<3.0.0" "aws-cdk.asset-node-proxy-agent-v6" = ">=2.0.1,<3.0.0" constructs = ">=10.0.0,<11.0.0" -jsii = ">=1.87.0,<2.0.0" +jsii = ">=1.88.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" @@ -200,13 +200,13 @@ requests = ">=0.14.0" [[package]] name = "aws-sam-translator" -version = "1.73.0" +version = "1.74.0" description = "AWS SAM Translator is a library that transform SAM templates into AWS CloudFormation templates" optional = false python-versions = ">=3.7, <=4.0, !=4.0" files = [ - {file = "aws-sam-translator-1.73.0.tar.gz", hash = "sha256:bfa7cad3a78f002edeec5e39fd61b616cf84f34f61010c5dc2f7a76845fe7a02"}, - {file = "aws_sam_translator-1.73.0-py3-none-any.whl", hash = "sha256:c0132b065d743773fcd2573ed1ae60e0129fa46043fad76430261b098a811924"}, + {file = "aws-sam-translator-1.74.0.tar.gz", hash = "sha256:6975ddf0798f45952554d6ea2d5e95de8f321ecb7d3299bb127073d4ffb53c90"}, + {file = "aws_sam_translator-1.74.0-py3-none-any.whl", hash = "sha256:74b386ed3780786a8b08d122c77c462f1872db4049617b63c497953b51008bf0"}, ] [package.dependencies] @@ -322,17 +322,17 @@ uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "boto3" -version = "1.28.35" +version = "1.28.43" description = "The AWS SDK for Python" optional = false python-versions = ">= 3.7" files = [ - {file = "boto3-1.28.35-py3-none-any.whl", hash = "sha256:d77415f22bbc14f3d72eaed2fc9f96d161f3ba7686922ad26d6bbc9d4985f3df"}, - {file = "boto3-1.28.35.tar.gz", hash = "sha256:580b584e36967155abed7cc9b088b3bd784e8242ae4d8841f58cb50ab05520dc"}, + {file = "boto3-1.28.43-py3-none-any.whl", hash = "sha256:4cd3e96900fb50bddc9f48007176c80d15396d08c5248b25a41220f3570e014f"}, + {file = "boto3-1.28.43.tar.gz", hash = "sha256:c0211a3e830432851c73fa1e136b14dbb6d02b5c9a5e1272c557e63538620b88"}, ] [package.dependencies] -botocore = ">=1.31.35,<1.32.0" +botocore = ">=1.31.43,<1.32.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.6.0,<0.7.0" @@ -341,13 +341,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.31.35" +version = "1.31.43" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">= 3.7" files = [ - {file = "botocore-1.31.35-py3-none-any.whl", hash = "sha256:943e1465aad66db4933b06809134bd08c5b05e8eb18c19742ffec82f54769457"}, - {file = "botocore-1.31.35.tar.gz", hash = "sha256:7e4534325262f43293a9cc9937cb3f1711365244ffde8b925a6ee862bcf30a83"}, + {file = "botocore-1.31.43-py3-none-any.whl", hash = "sha256:d8b0c41c8c75d82f15fee57f7d54a852a99810faacbeb9d6f3f022558a2c330e"}, + {file = "botocore-1.31.43.tar.gz", hash = "sha256:b4a3a1fcf75011351e2b0d3eb991f51f8d44a375d3e065f907dac67db232fc97"}, ] [package.dependencies] @@ -371,13 +371,13 @@ files = [ [[package]] name = "bytecode" -version = "0.14.2" +version = "0.15.0" description = "Python module to generate and modify bytecode" optional = false python-versions = ">=3.8" files = [ - {file = "bytecode-0.14.2-py3-none-any.whl", hash = "sha256:e368a2b9bbd7c986133c951250db94fb32f774cfc49752a9db9073bcf9899762"}, - {file = "bytecode-0.14.2.tar.gz", hash = "sha256:386378d9025d68ddb144870ae74330a492717b11b8c9164c4034e88add808f0c"}, + {file = "bytecode-0.15.0-py3-none-any.whl", hash = "sha256:a66718dc1d246b4fec52b5850c15592344a56c8bdb28fd243c895ccf00f8371f"}, + {file = "bytecode-0.15.0.tar.gz", hash = "sha256:0908a8348cabf366b5c1865daabcdc0d650cb0cbdeb1750cc90564852f81945c"}, ] [package.dependencies] @@ -641,17 +641,17 @@ files = [ [[package]] name = "constructs" -version = "10.2.69" +version = "10.2.70" description = "A programming model for software-defined state" optional = false python-versions = "~=3.7" files = [ - {file = "constructs-10.2.69-py3-none-any.whl", hash = "sha256:27a60f5ce4faa4d43c91c73f24e1a245c0a1ef67ea1c8a3df9ca6af9adf618df"}, - {file = "constructs-10.2.69.tar.gz", hash = "sha256:520ddd665cc336df90be06bb1bd49f3a9a7400d886cad8aef7b0155593b4ffa4"}, + {file = "constructs-10.2.70-py3-none-any.whl", hash = "sha256:ade1b5224830e78724ed50ce91ec2e6ce437c9983713c2b8ca541272283c5d37"}, + {file = "constructs-10.2.70.tar.gz", hash = "sha256:f4ae2e0705baff188519e0233ad2129537c8eca40d68242873ca444a659549f8"}, ] [package.dependencies] -jsii = ">=1.84.0,<2.0.0" +jsii = ">=1.88.0,<2.0.0" publication = ">=0.0.3" typeguard = ">=2.13.3,<2.14.0" @@ -730,15 +730,60 @@ tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.1 [package.extras] toml = ["tomli"] +[[package]] +name = "cryptography" +version = "41.0.3" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = true +python-versions = ">=3.7" +files = [ + {file = "cryptography-41.0.3-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:652627a055cb52a84f8c448185922241dd5217443ca194d5739b44612c5e6507"}, + {file = "cryptography-41.0.3-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:8f09daa483aedea50d249ef98ed500569841d6498aa9c9f4b0531b9964658922"}, + {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4fd871184321100fb400d759ad0cddddf284c4b696568204d281c902fc7b0d81"}, + {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84537453d57f55a50a5b6835622ee405816999a7113267739a1b4581f83535bd"}, + {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3fb248989b6363906827284cd20cca63bb1a757e0a2864d4c1682a985e3dca47"}, + {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:42cb413e01a5d36da9929baa9d70ca90d90b969269e5a12d39c1e0d475010116"}, + {file = "cryptography-41.0.3-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:aeb57c421b34af8f9fe830e1955bf493a86a7996cc1338fe41b30047d16e962c"}, + {file = "cryptography-41.0.3-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:6af1c6387c531cd364b72c28daa29232162010d952ceb7e5ca8e2827526aceae"}, + {file = "cryptography-41.0.3-cp37-abi3-win32.whl", hash = "sha256:0d09fb5356f975974dbcb595ad2d178305e5050656affb7890a1583f5e02a306"}, + {file = "cryptography-41.0.3-cp37-abi3-win_amd64.whl", hash = "sha256:a983e441a00a9d57a4d7c91b3116a37ae602907a7618b882c8013b5762e80574"}, + {file = "cryptography-41.0.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5259cb659aa43005eb55a0e4ff2c825ca111a0da1814202c64d28a985d33b087"}, + {file = "cryptography-41.0.3-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:67e120e9a577c64fe1f611e53b30b3e69744e5910ff3b6e97e935aeb96005858"}, + {file = "cryptography-41.0.3-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:7efe8041897fe7a50863e51b77789b657a133c75c3b094e51b5e4b5cec7bf906"}, + {file = "cryptography-41.0.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ce785cf81a7bdade534297ef9e490ddff800d956625020ab2ec2780a556c313e"}, + {file = "cryptography-41.0.3-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:57a51b89f954f216a81c9d057bf1a24e2f36e764a1ca9a501a6964eb4a6800dd"}, + {file = "cryptography-41.0.3-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:4c2f0d35703d61002a2bbdcf15548ebb701cfdd83cdc12471d2bae80878a4207"}, + {file = "cryptography-41.0.3-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:23c2d778cf829f7d0ae180600b17e9fceea3c2ef8b31a99e3c694cbbf3a24b84"}, + {file = "cryptography-41.0.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:95dd7f261bb76948b52a5330ba5202b91a26fbac13ad0e9fc8a3ac04752058c7"}, + {file = "cryptography-41.0.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:41d7aa7cdfded09b3d73a47f429c298e80796c8e825ddfadc84c8a7f12df212d"}, + {file = "cryptography-41.0.3-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d0d651aa754ef58d75cec6edfbd21259d93810b73f6ec246436a21b7841908de"}, + {file = "cryptography-41.0.3-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:ab8de0d091acbf778f74286f4989cf3d1528336af1b59f3e5d2ebca8b5fe49e1"}, + {file = "cryptography-41.0.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a74fbcdb2a0d46fe00504f571a2a540532f4c188e6ccf26f1f178480117b33c4"}, + {file = "cryptography-41.0.3.tar.gz", hash = "sha256:6d192741113ef5e30d89dcb5b956ef4e1578f304708701b8b73d38e3e1461f34"}, +] + +[package.dependencies] +cffi = ">=1.12" + +[package.extras] +docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] +docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] +nox = ["nox"] +pep8test = ["black", "check-sdist", "mypy", "ruff"] +sdist = ["build"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test-randomorder = ["pytest-randomly"] + [[package]] name = "datadog" -version = "0.46.0" +version = "0.47.0" description = "The Datadog Python library" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ - {file = "datadog-0.46.0-py2.py3-none-any.whl", hash = "sha256:3d7bcda6177b43be4cdb52e16b4bdd4f9005716c0dd7cfea009e018c36bb7a3d"}, - {file = "datadog-0.46.0.tar.gz", hash = "sha256:e4fbc92a85e2b0919a226896ae45fc5e4b356c0c57f1c2659659dfbe0789c674"}, + {file = "datadog-0.47.0-py2.py3-none-any.whl", hash = "sha256:a45ec997ab554208837e8c44d81d0e1456539dc14da5743687250e028bc809b7"}, + {file = "datadog-0.47.0.tar.gz", hash = "sha256:47be3b2c3d709a7f5b709eb126ed4fe6cc7977d618fe5c158dd89c2a9f7d9916"}, ] [package.dependencies] @@ -746,13 +791,13 @@ requests = ">=2.6.0" [[package]] name = "datadog-lambda" -version = "4.78.0" +version = "4.79.0" description = "The Datadog AWS Lambda Library" optional = false python-versions = ">=3.7.0,<4" files = [ - {file = "datadog_lambda-4.78.0-py3-none-any.whl", hash = "sha256:660bae6057f3b2033b0c035e9d542af491e40f9ce57b97b4891c491262b9148c"}, - {file = "datadog_lambda-4.78.0.tar.gz", hash = "sha256:3e57faa8f80ddd43b595355b92045fde8f9ed87efe8619133e82cebb87cbe434"}, + {file = "datadog_lambda-4.79.0-py3-none-any.whl", hash = "sha256:b297ff278a453dd0900e5cbe1332b9381bd2624d9ad8f21c8b464ef3c2dc85cc"}, + {file = "datadog_lambda-4.79.0.tar.gz", hash = "sha256:b0cf2d727edf9d652b75ab3f6b9e9eff3e5549394fcff7aa02f2d7cd955435da"}, ] [package.dependencies] @@ -1010,13 +1055,13 @@ smmap = ">=3.0.1,<6" [[package]] name = "gitpython" -version = "3.1.32" +version = "3.1.35" description = "GitPython is a Python library used to interact with Git repositories" optional = false python-versions = ">=3.7" files = [ - {file = "GitPython-3.1.32-py3-none-any.whl", hash = "sha256:e3d59b1c2c6ebb9dfa7a184daf3b6dd4914237e7488a1730a6d8f6f5d0b4187f"}, - {file = "GitPython-3.1.32.tar.gz", hash = "sha256:8d9b8cb1e80b9735e8717c9362079d3ce4c6e5ddeebedd0361b228c3a67a62f6"}, + {file = "GitPython-3.1.35-py3-none-any.whl", hash = "sha256:c19b4292d7a1d3c0f653858db273ff8a6614100d1eb1528b014ec97286193c09"}, + {file = "GitPython-3.1.35.tar.gz", hash = "sha256:9cbefbd1789a5fe9bcf621bb34d3f441f3a90c8461d377f84eda73e721d9b06b"}, ] [package.dependencies] @@ -1083,13 +1128,13 @@ socks = ["socksio (==1.*)"] [[package]] name = "hvac" -version = "1.1.1" +version = "1.2.0" description = "HashiCorp Vault API client" optional = false python-versions = ">=3.6.2,<4.0.0" files = [ - {file = "hvac-1.1.1-py3-none-any.whl", hash = "sha256:466e883665b4082933106b292649f9fba3bc0709a1ec1729e9e35b29477164b3"}, - {file = "hvac-1.1.1.tar.gz", hash = "sha256:f9dbcc46b98b250c785eb1050aa11ee34a0c8b6616b75218cf1346a9817992f9"}, + {file = "hvac-1.2.0-py3-none-any.whl", hash = "sha256:95716e0a6c081214d5f6dc74548d6e388aca895cd7be152cfaf177f7520b3d6e"}, + {file = "hvac-1.2.0.tar.gz", hash = "sha256:6f5aa0d6b8138b585d4656d1fe01b5d87616310c80484b909cc84c2cb8f064fd"}, ] [package.dependencies] @@ -1260,17 +1305,6 @@ pipfile-deprecated-finder = ["pip-shims (>=0.5.2)", "pipreqs", "requirementslib" plugins = ["setuptools"] requirements-deprecated-finder = ["pip-api", "pipreqs"] -[[package]] -name = "itsdangerous" -version = "2.1.2" -description = "Safely pass data to untrusted environments and back." -optional = true -python-versions = ">=3.7" -files = [ - {file = "itsdangerous-2.1.2-py3-none-any.whl", hash = "sha256:2c2349112351b88699d8d4b6b075022c0808887cb7ad10069318a8b0bc88db44"}, - {file = "itsdangerous-2.1.2.tar.gz", hash = "sha256:5dbbc68b317e5e42f327f9021763545dc3fc3bfe22e6deb96aaf1fc38874156a"}, -] - [[package]] name = "jinja2" version = "3.1.2" @@ -1523,6 +1557,16 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -1929,13 +1973,13 @@ test = ["codecov (>=2.1)", "pytest (>=6.2)", "pytest-cov (>=2.12)"] [[package]] name = "opentelemetry-api" -version = "1.19.0" +version = "1.20.0" description = "OpenTelemetry Python API" optional = false python-versions = ">=3.7" files = [ - {file = "opentelemetry_api-1.19.0-py3-none-any.whl", hash = "sha256:dcd2a0ad34b691964947e1d50f9e8c415c32827a1d87f0459a72deb9afdf5597"}, - {file = "opentelemetry_api-1.19.0.tar.gz", hash = "sha256:db374fb5bea00f3c7aa290f5d94cea50b659e6ea9343384c5f6c2bb5d5e8db65"}, + {file = "opentelemetry_api-1.20.0-py3-none-any.whl", hash = "sha256:982b76036fec0fdaf490ae3dfd9f28c81442a33414f737abc687a32758cdcba5"}, + {file = "opentelemetry_api-1.20.0.tar.gz", hash = "sha256:06abe351db7572f8afdd0fb889ce53f3c992dbf6f6262507b385cc1963e06983"}, ] [package.dependencies] @@ -2049,24 +2093,24 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "protobuf" -version = "4.24.2" +version = "4.24.3" description = "" optional = false python-versions = ">=3.7" files = [ - {file = "protobuf-4.24.2-cp310-abi3-win32.whl", hash = "sha256:58e12d2c1aa428ece2281cef09bbaa6938b083bcda606db3da4e02e991a0d924"}, - {file = "protobuf-4.24.2-cp310-abi3-win_amd64.whl", hash = "sha256:77700b55ba41144fc64828e02afb41901b42497b8217b558e4a001f18a85f2e3"}, - {file = "protobuf-4.24.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:237b9a50bd3b7307d0d834c1b0eb1a6cd47d3f4c2da840802cd03ea288ae8880"}, - {file = "protobuf-4.24.2-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:25ae91d21e3ce8d874211110c2f7edd6384816fb44e06b2867afe35139e1fd1c"}, - {file = "protobuf-4.24.2-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:c00c3c7eb9ad3833806e21e86dca448f46035242a680f81c3fe068ff65e79c74"}, - {file = "protobuf-4.24.2-cp37-cp37m-win32.whl", hash = "sha256:4e69965e7e54de4db989289a9b971a099e626f6167a9351e9d112221fc691bc1"}, - {file = "protobuf-4.24.2-cp37-cp37m-win_amd64.whl", hash = "sha256:c5cdd486af081bf752225b26809d2d0a85e575b80a84cde5172a05bbb1990099"}, - {file = "protobuf-4.24.2-cp38-cp38-win32.whl", hash = "sha256:6bd26c1fa9038b26c5c044ee77e0ecb18463e957fefbaeb81a3feb419313a54e"}, - {file = "protobuf-4.24.2-cp38-cp38-win_amd64.whl", hash = "sha256:bb7aa97c252279da65584af0456f802bd4b2de429eb945bbc9b3d61a42a8cd16"}, - {file = "protobuf-4.24.2-cp39-cp39-win32.whl", hash = "sha256:2b23bd6e06445699b12f525f3e92a916f2dcf45ffba441026357dea7fa46f42b"}, - {file = "protobuf-4.24.2-cp39-cp39-win_amd64.whl", hash = "sha256:839952e759fc40b5d46be319a265cf94920174d88de31657d5622b5d8d6be5cd"}, - {file = "protobuf-4.24.2-py3-none-any.whl", hash = "sha256:3b7b170d3491ceed33f723bbf2d5a260f8a4e23843799a3906f16ef736ef251e"}, - {file = "protobuf-4.24.2.tar.gz", hash = "sha256:7fda70797ddec31ddfa3576cbdcc3ddbb6b3078b737a1a87ab9136af0570cd6e"}, + {file = "protobuf-4.24.3-cp310-abi3-win32.whl", hash = "sha256:20651f11b6adc70c0f29efbe8f4a94a74caf61b6200472a9aea6e19898f9fcf4"}, + {file = "protobuf-4.24.3-cp310-abi3-win_amd64.whl", hash = "sha256:3d42e9e4796a811478c783ef63dc85b5a104b44aaaca85d4864d5b886e4b05e3"}, + {file = "protobuf-4.24.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:6e514e8af0045be2b56e56ae1bb14f43ce7ffa0f68b1c793670ccbe2c4fc7d2b"}, + {file = "protobuf-4.24.3-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:ba53c2f04798a326774f0e53b9c759eaef4f6a568ea7072ec6629851c8435959"}, + {file = "protobuf-4.24.3-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:f6ccbcf027761a2978c1406070c3788f6de4a4b2cc20800cc03d52df716ad675"}, + {file = "protobuf-4.24.3-cp37-cp37m-win32.whl", hash = "sha256:1b182c7181a2891e8f7f3a1b5242e4ec54d1f42582485a896e4de81aa17540c2"}, + {file = "protobuf-4.24.3-cp37-cp37m-win_amd64.whl", hash = "sha256:b0271a701e6782880d65a308ba42bc43874dabd1a0a0f41f72d2dac3b57f8e76"}, + {file = "protobuf-4.24.3-cp38-cp38-win32.whl", hash = "sha256:e29d79c913f17a60cf17c626f1041e5288e9885c8579832580209de8b75f2a52"}, + {file = "protobuf-4.24.3-cp38-cp38-win_amd64.whl", hash = "sha256:067f750169bc644da2e1ef18c785e85071b7c296f14ac53e0900e605da588719"}, + {file = "protobuf-4.24.3-cp39-cp39-win32.whl", hash = "sha256:2da777d34b4f4f7613cdf85c70eb9a90b1fbef9d36ae4a0ccfe014b0b07906f1"}, + {file = "protobuf-4.24.3-cp39-cp39-win_amd64.whl", hash = "sha256:f631bb982c5478e0c1c70eab383af74a84be66945ebf5dd6b06fc90079668d0b"}, + {file = "protobuf-4.24.3-py3-none-any.whl", hash = "sha256:f6f8dc65625dadaad0c8545319c2e2f0424fede988368893ca3844261342c11a"}, + {file = "protobuf-4.24.3.tar.gz", hash = "sha256:12e9ad2ec079b833176d2921be2cb24281fa591f0b119b208b788adc48c2561d"}, ] [[package]] @@ -2170,12 +2214,13 @@ plugins = ["importlib-metadata"] [[package]] name = "pyhcl" -version = "0.4.4" +version = "0.4.5" description = "HCL configuration parser for python" optional = false python-versions = "*" files = [ - {file = "pyhcl-0.4.4.tar.gz", hash = "sha256:2d9b9dcdf1023d812bfed561ba72c99104c5b3f52e558d595130a44ce081b003"}, + {file = "pyhcl-0.4.5-py3-none-any.whl", hash = "sha256:30ee337d330d1f90c9f5ed8f49c468f66c8e6e43192bdc7c6ece1420beb3070c"}, + {file = "pyhcl-0.4.5.tar.gz", hash = "sha256:c47293a51ccdd25e18bb5c8c0ab0ffe355b37c87f8d6f9d3280dc41efd4740bc"}, ] [[package]] @@ -2234,13 +2279,13 @@ files = [ [[package]] name = "pytest" -version = "7.4.1" +version = "7.4.2" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-7.4.1-py3-none-any.whl", hash = "sha256:460c9a59b14e27c602eb5ece2e47bec99dc5fc5f6513cf924a7d03a578991b1f"}, - {file = "pytest-7.4.1.tar.gz", hash = "sha256:2f2301e797521b23e4d2585a0a3d7b5e50fdddaaf7e7d6773ea26ddb17c213ab"}, + {file = "pytest-7.4.2-py3-none-any.whl", hash = "sha256:1d881c6124e08ff0a1bb75ba3ec0bfd8b5354a01c194ddd5a0a870a48d99b002"}, + {file = "pytest-7.4.2.tar.gz", hash = "sha256:a766259cfab564a2ad52cb1aae1b881a75c3eb7e34ca3779697c23ed47c47069"}, ] [package.dependencies] @@ -2365,13 +2410,13 @@ six = ">=1.5" [[package]] name = "pytz" -version = "2023.3" +version = "2023.3.post1" description = "World timezone definitions, modern and historical" optional = false python-versions = "*" files = [ - {file = "pytz-2023.3-py2.py3-none-any.whl", hash = "sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb"}, - {file = "pytz-2023.3.tar.gz", hash = "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588"}, + {file = "pytz-2023.3.post1-py2.py3-none-any.whl", hash = "sha256:ce42d816b81b68506614c11e8937d3aa9e41007ceb50bfdcb0749b921bf646c7"}, + {file = "pytz-2023.3.post1.tar.gz", hash = "sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b"}, ] [[package]] @@ -3100,6 +3145,8 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more all = ["aws-encryption-sdk", "aws-xray-sdk", "fastjsonschema", "pydantic"] aws-sdk = ["boto3"] datadog = ["datadog-lambda"] +datamasking-all = ["aws-encryption-sdk"] +datamasking-aws-sdk = ["aws-encryption-sdk"] parser = ["pydantic"] tracer = ["aws-xray-sdk"] validation = ["fastjsonschema"] @@ -3107,4 +3154,4 @@ validation = ["fastjsonschema"] [metadata] lock-version = "2.0" python-versions = "^3.7.4" -content-hash = "ae359f90595dce4d0659a4725d7f8117dd152a7805212194986595d1e2b60bcd" +content-hash = "e4e70e86e2f3eeb3ce4ee1807edc3571f083557032e5913163d022bdf82280b2" diff --git a/pyproject.toml b/pyproject.toml index 65ee706db1f..7eefa02a299 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ pydantic = { version = "^1.8.2", optional = true } boto3 = { version = "^1.20.32", optional = true } typing-extensions = "^4.6.2" datadog-lambda = { version = "^4.77.0", optional = true } +aws-encryption-sdk = { version = "^3.1.1", optional = true } [tool.poetry.dev-dependencies] coverage = {extras = ["toml"], version = "^7.2"} @@ -81,12 +82,11 @@ datadog-lambda = "^4.77.0" [tool.poetry.extras] parser = ["pydantic"] -datamasking-itsdangerous = ["itsdangerous"] datamasking-aws-sdk= ["aws-encryption-sdk"] -datamasking-all = ["itsdangerous", "aws-encryption-sdk"] +datamasking-all = ["aws-encryption-sdk"] validation = ["fastjsonschema"] tracer = ["aws-xray-sdk"] -all = ["pydantic", "aws-xray-sdk", "fastjsonschema", "itsdangerous", "aws-encryption-sdk"] +all = ["pydantic", "aws-xray-sdk", "fastjsonschema", "aws-encryption-sdk"] # allow customers to run code locally without emulators (SAM CLI, etc.) aws-sdk = ["boto3"] datadog=["datadog-lambda"] diff --git a/tests/unit/data_masking/setup.py b/tests/unit/data_masking/setup.py index 5655d4cc84d..1e8ee182aff 100644 --- a/tests/unit/data_masking/setup.py +++ b/tests/unit/data_masking/setup.py @@ -1,42 +1,17 @@ -import json import copy -from itsdangerous.url_safe import URLSafeSerializer -from aws_lambda_powertools.utilities.data_masking.constants import DATA_MASKING_STRING -from aws_lambda_powertools.utilities.data_masking.base import DataMasking -from aws_lambda_powertools.utilities.data_masking.provider import BaseProvider - - -class MyEncryptionProvider(BaseProvider): - """Custom encryption provider class""" - - def __init__(self, keys, salt=None): - self.keys = keys - self.salt = salt - - def encrypt(self, data: str) -> str: - if data is None: - return data - serialize = URLSafeSerializer(self.keys) - return serialize.dumps(data) +import json - def decrypt(self, data: str) -> str: - if data is None: - return data - serialize = URLSafeSerializer(self.keys) - return serialize.loads(data) +from aws_lambda_powertools.utilities.data_masking.base import DataMasking +from aws_lambda_powertools.utilities.data_masking.constants import DATA_MASKING_STRING - -data_maskers = [ - DataMasking(), - DataMasking(provider=MyEncryptionProvider(keys="secret-key")), -] +data_maskers = [DataMasking()] python_dict = { "a": { "1": {"None": "hello", "four": "world"}, # None type key doesn't work "b": {"3": {"4": "goodbye", "e": "world"}}, # key "4.5" doesn't work - } + }, } @@ -47,14 +22,14 @@ def decrypt(self, data: str) -> str: masked_with_fields = { - "a": {"1": {"None": DATA_MASKING_STRING, "four": "world"}, "b": {"3": {"4": DATA_MASKING_STRING, "e": "world"}}} + "a": {"1": {"None": DATA_MASKING_STRING, "four": "world"}, "b": {"3": {"4": DATA_MASKING_STRING, "e": "world"}}}, } aws_encrypted_with_fields = { "a": { "1": {"None": bytes("hello", "utf-8"), "four": "world"}, "b": {"3": {"4": bytes("goodbye", "utf-8"), "e": "world"}}, - } + }, } # 10kb JSON blob for latency testing diff --git a/tests/unit/data_masking/test_data_masking.py b/tests/unit/data_masking/test_data_masking.py index b30f18904f8..b227f5eba6f 100644 --- a/tests/unit/data_masking/test_data_masking.py +++ b/tests/unit/data_masking/test_data_masking.py @@ -1,31 +1,9 @@ import json import pytest -from itsdangerous.url_safe import URLSafeSerializer from aws_lambda_powertools.utilities.data_masking.base import DataMasking from aws_lambda_powertools.utilities.data_masking.constants import DATA_MASKING_STRING -from aws_lambda_powertools.utilities.data_masking.provider import BaseProvider - - -class MyEncryptionProvider(BaseProvider): - """Custom encryption provider class""" - - def __init__(self, keys, salt=None): - self.keys = keys - self.salt = salt - - def encrypt(self, data: str) -> str: - if data is None: - return data - serialize = URLSafeSerializer(self.keys) - return serialize.dumps(data) - - def decrypt(self, data: str) -> str: - if data is None: - return data - serialize = URLSafeSerializer(self.keys) - return serialize.loads(data) @pytest.fixture @@ -33,11 +11,6 @@ def data_masker() -> DataMasking: return DataMasking() -@pytest.fixture -def custom_data_masker() -> DataMasking: - return DataMasking(provider=MyEncryptionProvider(keys="secret-key")) - - def test_mask_int(data_masker): # GIVEN an int data type @@ -158,149 +131,6 @@ def test_mask_json_dict_with_fields(data_masker): } -def test_encrypt_decrypt_int(custom_data_masker): - # GIVEN an instantiation of DataMasking with a Provider - - # WHEN encrypting and then decrypting an int - encrypted_data = custom_data_masker.encrypt(42) - decrypted_data = custom_data_masker.decrypt(encrypted_data) - - # THEN the result is the original input data - assert decrypted_data == 42 - - -def test_encrypt_decrypt_float(custom_data_masker): - # GIVEN an instantiation of DataMasking with a Provider - - # WHEN encrypting and then decrypting a float - encrypted_data = custom_data_masker.encrypt(4.2) - decrypted_data = custom_data_masker.decrypt(encrypted_data) - - # THEN the result is the original input data - assert decrypted_data == 4.2 - - -def test_encrypt_decrypt_bool(custom_data_masker): - # GIVEN an instantiation of DataMasking with a Provider - - # WHEN encrypting and then decrypting a bool - encrypted_data = custom_data_masker.encrypt(True) - decrypted_data = custom_data_masker.decrypt(encrypted_data) - - # THEN the result is the original input data - assert decrypted_data is True - - -def test_encrypt_decrypt_none(custom_data_masker): - # GIVEN an instantiation of DataMasking with a Provider - - # WHEN encrypting and then decrypting a None type - encrypted_data = custom_data_masker.encrypt(None) - decrypted_data = custom_data_masker.decrypt(encrypted_data) - - # THEN the result is the original input data - assert decrypted_data is None - - -def test_encrypt_decrypt_str(custom_data_masker): - # GIVEN an instantiation of DataMasking with a Provider - - # WHEN encrypting and then decrypting a string - encrypted_data = custom_data_masker.encrypt("this is a string") - decrypted_data = custom_data_masker.decrypt(encrypted_data) - - # THEN the result is the original input data - assert decrypted_data == "this is a string" - - -def test_encrypt_decrypt_list(custom_data_masker): - # GIVEN an instantiation of DataMasking with a Provider - - # WHEN encrypting and then decrypting a list - encrypted_data = custom_data_masker.encrypt([1, 2, "string", 4]) - decrypted_data = custom_data_masker.decrypt(encrypted_data) - - # THEN the result is the original input data - assert decrypted_data == [1, 2, "string", 4] - - -def test_dict_encryption_with_fields(custom_data_masker): - # GIVEN an instantiation of DataMasking with a Provider - - data = { - "a": { - "1": {"None": "hello", "four": "world"}, - "b": {"3": {"4": "goodbye", "e": "world"}}, - }, - } - - # WHEN encrypting and decrypting the data with a list of fields - encrypted_data = custom_data_masker.encrypt(data, fields=["a.1.None", "a.b.3.4"]) - decrypted_data = custom_data_masker.decrypt(encrypted_data, fields=["a.1.None", "a.b.3.4"]) - - # THEN the result is the original input data - assert decrypted_data == data - - -def test_json_encryption_with_fields(custom_data_masker): - # GIVEN an instantiation of DataMasking with a Provider - - data = json.dumps( - { - "a": { - "1": {"None": "hello", "four": "world"}, - "b": {"3": {"4": "goodbye", "e": "world"}}, - }, - }, - ) - - # WHEN encrypting and decrypting a json representation of a dictionary with a list of fields - encrypted_data = custom_data_masker.encrypt(data, fields=["a.1.None", "a.b.3.4"]) - decrypted_data = custom_data_masker.decrypt(encrypted_data, fields=["a.1.None", "a.b.3.4"]) - - # THEN the result is the original input data - assert decrypted_data == json.loads(data) - - -def test_big_data_encryption_with_fields(custom_data_masker): - # GIVEN an instantiation of DataMasking with a Provider - - # 10kb JSON blob for latency testing - data = { - "id": 1, - "name": "John Doe", - "age": 30, - "email": "johndoe@example.com", - "address": {"street": "123 Main St", "city": "Anytown", "state": "CA", "zip": "12345"}, - "phone_numbers": ["+1-555-555-1234", "+1-555-555-5678"], - "interests": ["Hiking", "Traveling", "Photography", "Reading"], - "job_history": { - "company": "Acme Inc.", - "position": "Software Engineer", - "start_date": "2015-01-01", - "end_date": "2017-12-31", - }, - "about_me": """ - Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla tincidunt velit quis - sapien mollis, at egestas massa tincidunt. Suspendisse ultrices arcu a dolor dapibus, - ut pretium turpis volutpat. Vestibulum at sapien quis sapien dignissim volutpat ut a enim. - Praesent fringilla sem eu dui convallis luctus. Donec ullamcorper, sapien ut convallis congue, - risus mauris pretium tortor, nec dignissim arcu urna a nisl. Vivamus non fermentum ex. Proin - interdum nisi id sagittis egestas. Nam sit amet nisi nec quam pharetra sagittis. Aliquam erat - volutpat. Donec nec luctus sem, nec ornare lorem. Vivamus vitae orci quis enim faucibus placerat. - Nulla facilisi. Proin in turpis orci. Donec imperdiet velit ac tellus gravida, eget laoreet tellus - malesuada. Praesent venenatis tellus ac urna blandit, at varius felis posuere. Integer a commodo nunc. - """, - } - - # WHEN encrypting and decrypting the data with a list of fields - encrypted_data = custom_data_masker.encrypt(data, fields=["address.street", "job_history.company"]) - decrypted_data = custom_data_masker.decrypt(encrypted_data, fields=["address.street", "job_history.company"]) - - # THEN the result is the original input data - assert decrypted_data == data - - def test_encrypt_not_implemented(data_masker): # GIVEN DataMasking is not initialized with a Provider From 01885a557f0ea0f48049bbacbd05d226113b1833 Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Fri, 8 Sep 2023 23:04:05 +0000 Subject: [PATCH 35/56] Added serializer for aws enc sdk --- .../providers/aws_encryption_sdk.py | 8 ++++++ .../data_masking/handlers/basic_handler.py | 2 +- tests/e2e/data_masking/test_data_masking.py | 26 ++++++++++--------- .../data_masking/test_aws_encryption_sdk.py | 14 +++++----- tests/unit/data_masking/setup.py | 4 --- 5 files changed, 31 insertions(+), 23 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py index 435ffcbcf9f..25c306e85d6 100644 --- a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py +++ b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py @@ -75,6 +75,12 @@ def __init__( max_messages_encrypted=max_messages_encrypted, ) + def _serialize(self, data: Any): + return bytes(str(data), "utf-8") + + def _deserialize(self, data: bytes): + return data.decode("utf-8") + def encrypt(self, data: Union[bytes, str], **provider_options) -> str: """ Encrypt data using the AwsEncryptionSdkProvider. @@ -91,6 +97,7 @@ def encrypt(self, data: Union[bytes, str], **provider_options) -> str: ciphertext : str The encrypted data, as a base64-encoded string. """ + data = self._serialize(data) ciphertext, _ = self.client.encrypt(source=data, materials_manager=self.cache_cmm, **provider_options) ciphertext = base64.b64encode(ciphertext).decode() return ciphertext @@ -125,4 +132,5 @@ def decrypt(self, data: str, **provider_options) -> bytes: if decryptor_header.encryption_context.get(key) != value: raise ContextMismatchError(key) + ciphertext = self._deserialize(ciphertext) return ciphertext diff --git a/tests/e2e/data_masking/handlers/basic_handler.py b/tests/e2e/data_masking/handlers/basic_handler.py index 93da716784c..8df533f377d 100644 --- a/tests/e2e/data_masking/handlers/basic_handler.py +++ b/tests/e2e/data_masking/handlers/basic_handler.py @@ -14,7 +14,7 @@ def lambda_handler(event, context): kms_key = event.get("kms_key") data_masker = DataMasking(provider=AwsEncryptionSdkProvider(keys=[kms_key])) - value = bytes(str([1, 2, "string", 4.5]), "utf-8") + value = [1, 2, "string", 4.5] encrypted_data = data_masker.encrypt(value) response = {} response["encrypted_data"] = encrypted_data diff --git a/tests/e2e/data_masking/test_data_masking.py b/tests/e2e/data_masking/test_data_masking.py index f8bd3b7e371..9b5451f7018 100644 --- a/tests/e2e/data_masking/test_data_masking.py +++ b/tests/e2e/data_masking/test_data_masking.py @@ -42,21 +42,21 @@ def test_encryption(data_masker): # GIVEN an instantiation of DataMasking with the AWS encryption provider # AWS Encryption SDK encrypt method only takes in bytes or strings - value = bytes(str([1, 2, "string", 4.5]), "utf-8") + value = [1, 2, "string", 4.5] # WHEN encrypting and then decrypting the encrypted data encrypted_data = data_masker.encrypt(value) decrypted_data = data_masker.decrypt(encrypted_data) # THEN the result is the original input data - assert decrypted_data == value + assert decrypted_data == str(value) @pytest.mark.xdist_group(name="data_masking") def test_encryption_context(data_masker): # GIVEN an instantiation of DataMasking with the AWS encryption provider - value = bytes(str([1, 2, "string", 4.5]), "utf-8") + value = [1, 2, "string", 4.5] context = {"this": "is_secure"} # WHEN encrypting and then decrypting the encrypted data with an encryption_context @@ -64,14 +64,14 @@ def test_encryption_context(data_masker): decrypted_data = data_masker.decrypt(encrypted_data, encryption_context=context) # THEN the result is the original input data - assert decrypted_data == value + assert decrypted_data == str(value) @pytest.mark.xdist_group(name="data_masking") def test_encryption_context_mismatch(data_masker): # GIVEN an instantiation of DataMasking with the AWS encryption provider - value = bytes(str([1, 2, "string", 4.5]), "utf-8") + value = [1, 2, "string", 4.5] # WHEN encrypting with a encryption_context encrypted_data = data_masker.encrypt(value, encryption_context={"this": "is_secure"}) @@ -85,7 +85,7 @@ def test_encryption_context_mismatch(data_masker): def test_encryption_no_context_fail(data_masker): # GIVEN an instantiation of DataMasking with the AWS encryption provider - value = bytes(str([1, 2, "string", 4.5]), "utf-8") + value = [1, 2, "string", 4.5] # WHEN encrypting with no encryption_context encrypted_data = data_masker.encrypt(value) @@ -100,7 +100,7 @@ def test_encryption_decryption_key_mismatch(data_masker, kms_key2_arn): # GIVEN an instantiation of DataMasking with the AWS encryption provider with a certain key # WHEN encrypting and then decrypting the encrypted data - value = bytes(str([1, 2, "string", 4.5]), "utf-8") + value = [1, 2, "string", 4.5] encrypted_data = data_masker.encrypt(value) # THEN when decrypting with a different key it should fail @@ -114,12 +114,14 @@ def test_encryption_provider_singleton(data_masker, kms_key1_arn, kms_key2_arn): data_masker_2 = DataMasking(provider=AwsEncryptionSdkProvider(keys=[kms_key1_arn])) assert data_masker.provider is data_masker_2.provider + value = [1, 2, "string", 4.5] + # WHEN encrypting and then decrypting the encrypted data - encrypted_data = data_masker.encrypt("string") + encrypted_data = data_masker.encrypt(value) decrypted_data = data_masker_2.decrypt(encrypted_data) # THEN the result is the original input data - assert decrypted_data == bytes("string", "utf-8") + assert decrypted_data == str(value) data_masker_3 = DataMasking(provider=AwsEncryptionSdkProvider(keys=[kms_key2_arn])) assert data_masker_2.provider is not data_masker_3.provider @@ -130,7 +132,7 @@ def test_encryption_in_logs(data_masker, basic_handler_fn, basic_handler_fn_arn) # GIVEN an instantiation of DataMasking with the AWS encryption provider # WHEN encrypting a value and logging it - value = bytes(str([1, 2, "string", 4.5]), "utf-8") + value = [1, 2, "string", 4.5] encrypted_data = data_masker.encrypt(value) message = encrypted_data custom_key = "order_id" @@ -146,7 +148,7 @@ def test_encryption_in_logs(data_masker, basic_handler_fn, basic_handler_fn_arn) for log in logs.get_log(key=custom_key): encrypted_data = log.message decrypted_data = data_masker.decrypt(encrypted_data) - assert decrypted_data == value + assert decrypted_data == str(value) # NOTE: This test is failing currently, need to find a fix for building correct dependencies @@ -162,4 +164,4 @@ def test_encryption_in_handler(basic_handler_fn_arn, kms_key1_arn): decrypted_data = data_masker.decrypt(encrypted_data) # THEN decrypting the encrypted data from the response should result in the original value - assert decrypted_data == bytes(str([1, 2, "string", 4.5]), "utf-8") + assert decrypted_data == str([1, 2, "string", 4.5]) diff --git a/tests/functional/data_masking/test_aws_encryption_sdk.py b/tests/functional/data_masking/test_aws_encryption_sdk.py index d31c242e03b..ed2a356f2f1 100644 --- a/tests/functional/data_masking/test_aws_encryption_sdk.py +++ b/tests/functional/data_masking/test_aws_encryption_sdk.py @@ -39,15 +39,12 @@ def test_mask_with_fields(data_masker): def test_encrypt_decrypt(value, data_masker): # GIVEN an instantiation of DataMasking with the AWS encryption provider - # AWS Encryption SDK encrypt method only takes in bytes or strings - value = bytes(str(value), "utf-8") - # WHEN encrypting and then decrypting the encrypted data encrypted_data = data_masker.encrypt(value) decrypted_data = data_masker.decrypt(encrypted_data) # THEN the result is the original input data - assert decrypted_data == value + assert decrypted_data == str(value) @pytest.mark.parametrize("value, fields", zip(dictionaries, fields_to_mask)) @@ -60,7 +57,12 @@ def test_encrypt_decrypt_with_fields(value, fields, data_masker): # THEN the result is the original input data # AWS Encryption SDK decrypt method only returns bytes + print("value:", value) if value == json_blob: - assert decrypted_data == aws_encrypted_json_blob + print("json blob!!!!") + assert decrypted_data == value else: - assert decrypted_data == aws_encrypted_with_fields + print("json_blob_fields!!!!") + assert decrypted_data == str(value) + print("decrypted_data:", decrypted_data) + print("aws_encrypted_with_fields:", aws_encrypted_with_fields) diff --git a/tests/unit/data_masking/setup.py b/tests/unit/data_masking/setup.py index 1e8ee182aff..be79712fd21 100644 --- a/tests/unit/data_masking/setup.py +++ b/tests/unit/data_masking/setup.py @@ -1,12 +1,8 @@ import copy import json -from aws_lambda_powertools.utilities.data_masking.base import DataMasking from aws_lambda_powertools.utilities.data_masking.constants import DATA_MASKING_STRING -data_maskers = [DataMasking()] - - python_dict = { "a": { "1": {"None": "hello", "four": "world"}, # None type key doesn't work From 5b83b660f7f94fccd458f68b82dcf78d42386438 Mon Sep 17 00:00:00 2001 From: Heitor Lessa Date: Sat, 9 Sep 2023 01:51:32 +0200 Subject: [PATCH 36/56] chore: fix merge conflict, remove itsdangerous leftovers (#2) * fix(parameters): make cache aware of single vs multiple calls Signed-off-by: heitorlessa * chore: cleanup, add test for single and nested Signed-off-by: heitorlessa * chore(deps): bump pypa/gh-action-pypi-publish from 1.8.8 to 1.8.9 (#2943) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps-dev): bump the boto-typing group with 1 update (#2944) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Leandro Damascena * chore(ci): changelog rebuild (#2945) Co-authored-by: Powertools for AWS Lambda (Python) bot * chore(deps-dev): bump aws-cdk from 2.90.0 to 2.91.0 (#2947) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump gitpython from 3.1.31 to 3.1.32 in /docs (#2948) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump pypa/gh-action-pypi-publish from 1.8.9 to 1.8.10 (#2946) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Leandro Damascena * chore(ci): changelog rebuild (#2952) Co-authored-by: Powertools for AWS Lambda (Python) bot * feat(event_handler): allow stripping route prefixes using regexes (#2521) Co-authored-by: Roy Assis Co-authored-by: Ruben Fonseca * feat(metrics): add Datadog observability provider (#2906) Co-authored-by: Leandro Damascena Co-authored-by: heitorlessa * chore(deps-dev): bump xenon from 0.9.0 to 0.9.1 (#2955) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump actions/setup-node from 3.7.0 to 3.8.0 (#2957) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps-dev): bump cfn-lint from 0.79.6 to 0.79.7 (#2956) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Leandro Damascena * chore(ci): changelog rebuild (#2958) Co-authored-by: Powertools for AWS Lambda (Python) bot * docs(metrics): update Datadog integration diagram (#2954) Co-authored-by: Leandro Damascena * docs(roadmap): add GovCloud and China region item (#2960) * fix(parameters): make cache aware of single vs multiple calls Signed-off-by: heitorlessa * chore: cleanup, add test for single and nested Signed-off-by: heitorlessa * chore(test): remove itsdangerous from perf test Signed-off-by: heitorlessa * chore(deps): remove itsdangerous dependencies * chore: disable sockets in encryption sdk tests Signed-off-by: heitorlessa * refactor(tests): use a test double * chore: address make pr errors Signed-off-by: heitorlessa --------- Signed-off-by: heitorlessa Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Leandro Damascena Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Powertools for AWS Lambda (Python) bot Co-authored-by: roy <34189329+royassis@users.noreply.github.com> Co-authored-by: Roy Assis Co-authored-by: Ruben Fonseca Co-authored-by: Roger Zhang Co-authored-by: aal80 Co-authored-by: Seshu Brahma --- .../utilities/data_masking/base.py | 7 +- poetry.lock | 14 ++++ pyproject.toml | 82 +++++++++++-------- tests/functional/data_masking/conftest.py | 23 ++++++ .../data_masking/test_aws_encryption_sdk.py | 36 +++++--- tests/performance/test_data_masking.py | 12 +-- 6 files changed, 120 insertions(+), 54 deletions(-) create mode 100644 tests/functional/data_masking/conftest.py diff --git a/aws_lambda_powertools/utilities/data_masking/base.py b/aws_lambda_powertools/utilities/data_masking/base.py index 8cb7eb4c9d9..356dee91c9f 100644 --- a/aws_lambda_powertools/utilities/data_masking/base.py +++ b/aws_lambda_powertools/utilities/data_masking/base.py @@ -98,9 +98,12 @@ def _apply_action_to_fields(self, data: Union[dict, str], fields, action, **prov ) for field in fields: + # prevent overriding loop variable + current_field = field if not isinstance(field, str): - field = json.dumps(field) - keys = field.split(".") + current_field = json.dumps(field) + + keys = current_field.split(".") curr_dict = my_dict_parsed for key in keys[:-1]: diff --git a/poetry.lock b/poetry.lock index 6d0c63e808e..c6a3e91bf19 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2374,6 +2374,20 @@ pytest = ">=5.0" [package.extras] dev = ["pre-commit", "pytest-asyncio", "tox"] +[[package]] +name = "pytest-socket" +version = "0.6.0" +description = "Pytest Plugin to disable socket calls during tests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "pytest_socket-0.6.0-py3-none-any.whl", hash = "sha256:cca72f134ff01e0023c402e78d31b32e68da3efdf3493bf7788f8eba86a6824c"}, + {file = "pytest_socket-0.6.0.tar.gz", hash = "sha256:363c1d67228315d4fc7912f1aabfd570de29d0e3db6217d61db5728adacd7138"}, +] + +[package.dependencies] +pytest = ">=3.6.3" + [[package]] name = "pytest-xdist" version = "3.3.1" diff --git a/pyproject.toml b/pyproject.toml index 7eefa02a299..f8af23604cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,21 +4,31 @@ version = "2.23.1" description = "Powertools for AWS Lambda (Python) is a developer toolkit to implement Serverless best practices and increase developer velocity." authors = ["Amazon Web Services"] include = ["aws_lambda_powertools/py.typed", "THIRD-PARTY-LICENSES"] -classifiers=[ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "License :: OSI Approved :: MIT No Attribution License (MIT-0)", - "Natural Language :: English", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT No Attribution License (MIT-0)", + "Natural Language :: English", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ] repository = "https://github.com/aws-powertools/powertools-lambda-python" documentation = "https://docs.powertools.aws.dev/lambda/python/" readme = "README.md" -keywords = ["aws_lambda_powertools", "aws", "tracing", "logging", "lambda", "powertools", "feature_flags", "idempotency", "middleware"] +keywords = [ + "aws_lambda_powertools", + "aws", + "tracing", + "logging", + "lambda", + "powertools", + "feature_flags", + "idempotency", + "middleware", +] # MIT-0 is not recognized as an existing license from poetry. # By using `MIT` as a license value, a `License :: OSI Approved :: MIT License` classifier is added to the classifiers list. license = "MIT" @@ -38,7 +48,7 @@ datadog-lambda = { version = "^4.77.0", optional = true } aws-encryption-sdk = { version = "^3.1.1", optional = true } [tool.poetry.dev-dependencies] -coverage = {extras = ["toml"], version = "^7.2"} +coverage = { extras = ["toml"], version = "^7.2" } pytest = "^7.4.1" black = "^23.3" boto3 = "^1.18" @@ -75,7 +85,7 @@ filelock = "^3.12.2" checksumdir = "^1.2.0" mypy-boto3-appconfigdata = "^1.28.36" ijson = "^3.2.2" -typed-ast = { version = "^1.5.5", python = "< 3.8"} +typed-ast = { version = "^1.5.5", python = "< 3.8" } hvac = "^1.1.1" aws-requests-auth = "^0.4.3" datadog-lambda = "^4.77.0" @@ -89,7 +99,7 @@ tracer = ["aws-xray-sdk"] all = ["pydantic", "aws-xray-sdk", "fastjsonschema", "aws-encryption-sdk"] # allow customers to run code locally without emulators (SAM CLI, etc.) aws-sdk = ["boto3"] -datadog=["datadog-lambda"] +datadog = ["datadog-lambda"] [tool.poetry.group.dev.dependencies] cfn-lint = "0.79.9" @@ -99,10 +109,16 @@ httpx = ">=0.23.3,<0.25.0" sentry-sdk = "^1.22.2" ruff = ">=0.0.272,<0.0.288" retry2 = "^0.9.5" +pytest-socket = "^0.6.0" [tool.coverage.run] source = ["aws_lambda_powertools"] -omit = ["tests/*", "aws_lambda_powertools/exceptions/*", "aws_lambda_powertools/utilities/parser/types.py", "aws_lambda_powertools/utilities/jmespath_utils/envelopes.py"] +omit = [ + "tests/*", + "aws_lambda_powertools/exceptions/*", + "aws_lambda_powertools/utilities/parser/types.py", + "aws_lambda_powertools/utilities/jmespath_utils/envelopes.py", +] branch = true [tool.coverage.html] @@ -112,26 +128,26 @@ title = "Powertools for AWS Lambda (Python) Test Coverage" [tool.coverage.report] fail_under = 90 exclude_lines = [ - # Have to re-enable the standard pragma - "pragma: no cover", + # Have to re-enable the standard pragma + "pragma: no cover", - # Don't complain about missing debug-only code: - "def __repr__", - "if self.debug", + # Don't complain about missing debug-only code: + "def __repr__", + "if self.debug", - # Don't complain if tests don't hit defensive assertion code: - "raise AssertionError", - "raise NotImplementedError", + # Don't complain if tests don't hit defensive assertion code: + "raise AssertionError", + "raise NotImplementedError", - # Don't complain if non-runnable code isn't run: - "if 0:", - "if __name__ == .__main__.:", + # Don't complain if non-runnable code isn't run: + "if 0:", + "if __name__ == .__main__.:", - # Ignore runtime type checking - "if TYPE_CHECKING:", + # Ignore runtime type checking + "if TYPE_CHECKING:", - # Ignore type function overload - "@overload", + # Ignore type function overload + "@overload", ] [tool.isort] @@ -164,16 +180,16 @@ minversion = "6.0" addopts = "-ra -vv" testpaths = "./tests" markers = [ - "perf: marks perf tests to be deselected (deselect with '-m \"not perf\"')", + "perf: marks perf tests to be deselected (deselect with '-m \"not perf\"')", ] # MAINTENANCE: Remove these lines when drop support to Pydantic v1 -filterwarnings=[ +filterwarnings = [ "ignore:.*The `parse_obj` method is deprecated*:DeprecationWarning", "ignore:.*The `parse_raw` method is deprecated*:DeprecationWarning", "ignore:.*load_str_bytes is deprecated*:DeprecationWarning", "ignore:.*The `dict` method is deprecated; use `model_dump` instead*:DeprecationWarning", - "ignore:.*Pydantic V1 style `@validator` validators are deprecated*:DeprecationWarning" + "ignore:.*Pydantic V1 style `@validator` validators are deprecated*:DeprecationWarning", ] [build-system] diff --git a/tests/functional/data_masking/conftest.py b/tests/functional/data_masking/conftest.py new file mode 100644 index 00000000000..6127858d6b3 --- /dev/null +++ b/tests/functional/data_masking/conftest.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from typing import Tuple + +from pytest_socket import disable_socket + + +def pytest_runtest_setup(): + """Disable Unix and TCP sockets for Data masking tests""" + disable_socket() + + +class FakeEncryptionClient: + ENCRYPTION_HEADER = "test" + + def encrypt(self, source: bytes | str, **kwargs) -> Tuple[bytes, str]: + if isinstance(source, str): + return source.encode(), self.ENCRYPTION_HEADER + + return source, self.ENCRYPTION_HEADER + + def decrypt(self, source: bytes, **kwargs) -> Tuple[bytes, str]: + return source, "dummy_decryption_header" diff --git a/tests/functional/data_masking/test_aws_encryption_sdk.py b/tests/functional/data_masking/test_aws_encryption_sdk.py index ed2a356f2f1..52d6b90654f 100644 --- a/tests/functional/data_masking/test_aws_encryption_sdk.py +++ b/tests/functional/data_masking/test_aws_encryption_sdk.py @@ -1,15 +1,32 @@ +from __future__ import annotations + import pytest from aws_lambda_powertools.utilities.data_masking.base import DataMasking -from aws_lambda_powertools.utilities.data_masking.providers.aws_encryption_sdk import AwsEncryptionSdkProvider -from tests.unit.data_masking.setup import * - -AWS_SDK_KEY = "arn:aws:kms:us-west-2:683517028648:key/269301eb-81eb-4067-ac72-98e8e49bf2b3" +from aws_lambda_powertools.utilities.data_masking.providers.aws_encryption_sdk import ( + AwsEncryptionSdkProvider, +) +from tests.functional.data_masking.conftest import FakeEncryptionClient +from tests.unit.data_masking.setup import ( + aws_encrypted_with_fields, + data_types, + data_types_and_masks, + dict_fields, + dictionaries, + fields_to_mask, + json_blob, + json_dict, + masked_with_fields, + python_dict, +) @pytest.fixture -def data_masker(): - return DataMasking(provider=AwsEncryptionSdkProvider(keys=[AWS_SDK_KEY])) +def data_masker() -> DataMasking: + """DataMasking using AWS Encryption SDK Provider with a fake client""" + fake_client = FakeEncryptionClient() + provider = AwsEncryptionSdkProvider(keys=["dummy"], client=fake_client) + return DataMasking(provider=provider) @pytest.mark.parametrize("value, value_masked", data_types_and_masks) @@ -36,7 +53,7 @@ def test_mask_with_fields(data_masker): @pytest.mark.parametrize("value", data_types) -def test_encrypt_decrypt(value, data_masker): +def test_encrypt_decrypt(value, data_masker: DataMasking): # GIVEN an instantiation of DataMasking with the AWS encryption provider # WHEN encrypting and then decrypting the encrypted data @@ -62,7 +79,4 @@ def test_encrypt_decrypt_with_fields(value, fields, data_masker): print("json blob!!!!") assert decrypted_data == value else: - print("json_blob_fields!!!!") - assert decrypted_data == str(value) - print("decrypted_data:", decrypted_data) - print("aws_encrypted_with_fields:", aws_encrypted_with_fields) + assert decrypted_data == aws_encrypted_with_fields diff --git a/tests/performance/test_data_masking.py b/tests/performance/test_data_masking.py index 33ebe42733f..9dcf041b81f 100644 --- a/tests/performance/test_data_masking.py +++ b/tests/performance/test_data_masking.py @@ -4,9 +4,6 @@ import pytest from aws_lambda_powertools.utilities.data_masking.base import DataMasking -from aws_lambda_powertools.utilities.data_masking.providers.itsdangerous import ( - ItsDangerousProvider, -) DATA_MASKING_PACKAGE = "aws_lambda_powertools.utilities.data_masking" DATA_MASKING_INIT_SLA: float = 0.002 @@ -58,16 +55,15 @@ def test_data_masking_init(benchmark): pytest.fail(f"High level imports should be below {DATA_MASKING_INIT_SLA}s: {stat}") -def encrypt_json_blob(): - data_masker = DataMasking(provider=ItsDangerousProvider("mykey")) - encrypted = data_masker.encrypt(json_blob, json_blob_fields) - data_masker.decrypt(encrypted, json_blob_fields) +def mask_json_blob(): + data_masker = DataMasking() + data_masker.mask(json_blob, json_blob_fields) @pytest.mark.perf @pytest.mark.benchmark(group="core", disable_gc=True, warmup=False) def test_data_masking_encrypt_with_json_blob(benchmark): - benchmark.pedantic(encrypt_json_blob) + benchmark.pedantic(mask_json_blob) stat = benchmark.stats.stats.max if stat > DATA_MASKING_NESTED_ENCRYPT_SLA: pytest.fail(f"High level imports should be below {DATA_MASKING_NESTED_ENCRYPT_SLA}s: {stat}") From 371ea051e91614c4c2ae6ea1edd627991613bd6b Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Sat, 9 Sep 2023 03:11:47 +0000 Subject: [PATCH 37/56] Building data within func tests instead of using setup.py --- .../providers/aws_encryption_sdk.py | 18 +- .../data_masking/test_aws_encryption_sdk.py | 285 +++++++++++++++--- tests/unit/data_masking/setup.py | 88 ------ 3 files changed, 253 insertions(+), 138 deletions(-) delete mode 100644 tests/unit/data_masking/setup.py diff --git a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py index 25c306e85d6..724472c27d4 100644 --- a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py +++ b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py @@ -1,4 +1,5 @@ import base64 +from collections.abc import Iterable from typing import Any, Dict, List, Optional, Union import botocore @@ -23,14 +24,19 @@ class Singleton: _instances: Dict[Any, "AwsEncryptionSdkProvider"] = {} def __new__(cls, *args, **kwargs): - # Generate a unique key based on the configuration + # Generate a unique key based on the configuration. # Create a tuple by iterating through the values in kwargs, sorting them, # and then adding them to the tuple. - config_key = tuple(v for value in kwargs.values() for v in sorted(value)) + config_key = tuple() + for value in kwargs.values(): + if isinstance(value, Iterable): + for val in sorted(value): + config_key += (val,) + else: + config_key += (value,) if config_key not in cls._instances: cls._instances[config_key] = super(Singleton, cls).__new__(cls, *args) - print("in if class instances:", cls._instances) return cls._instances[config_key] @@ -75,10 +81,10 @@ def __init__( max_messages_encrypted=max_messages_encrypted, ) - def _serialize(self, data: Any): + def _serialize(self, data: Any) -> bytes: return bytes(str(data), "utf-8") - def _deserialize(self, data: bytes): + def _deserialize(self, data: bytes) -> str: return data.decode("utf-8") def encrypt(self, data: Union[bytes, str], **provider_options) -> str: @@ -102,7 +108,7 @@ def encrypt(self, data: Union[bytes, str], **provider_options) -> str: ciphertext = base64.b64encode(ciphertext).decode() return ciphertext - def decrypt(self, data: str, **provider_options) -> bytes: + def decrypt(self, data: str, **provider_options) -> str: """ Decrypt data using AwsEncryptionSdkProvider. diff --git a/tests/functional/data_masking/test_aws_encryption_sdk.py b/tests/functional/data_masking/test_aws_encryption_sdk.py index 52d6b90654f..4d0cd4e8cfd 100644 --- a/tests/functional/data_masking/test_aws_encryption_sdk.py +++ b/tests/functional/data_masking/test_aws_encryption_sdk.py @@ -1,24 +1,15 @@ from __future__ import annotations +import json + import pytest from aws_lambda_powertools.utilities.data_masking.base import DataMasking +from aws_lambda_powertools.utilities.data_masking.constants import DATA_MASKING_STRING from aws_lambda_powertools.utilities.data_masking.providers.aws_encryption_sdk import ( AwsEncryptionSdkProvider, ) from tests.functional.data_masking.conftest import FakeEncryptionClient -from tests.unit.data_masking.setup import ( - aws_encrypted_with_fields, - data_types, - data_types_and_masks, - dict_fields, - dictionaries, - fields_to_mask, - json_blob, - json_dict, - masked_with_fields, - python_dict, -) @pytest.fixture @@ -29,54 +20,260 @@ def data_masker() -> DataMasking: return DataMasking(provider=provider) -@pytest.mark.parametrize("value, value_masked", data_types_and_masks) -def test_mask_types(value, value_masked, data_masker): - # GIVEN any data type +def test_mask_int(data_masker): + # GIVEN an int data type + + # WHEN mask is called with no fields argument + masked_string = data_masker.mask(42) + + # THEN the result is the data masked + assert masked_string == DATA_MASKING_STRING + + +def test_mask_float(data_masker): + # GIVEN a float data type + + # WHEN mask is called with no fields argument + masked_string = data_masker.mask(4.2) + + # THEN the result is the data masked + assert masked_string == DATA_MASKING_STRING + + +def test_mask_bool(data_masker): + # GIVEN a bool data type + + # WHEN mask is called with no fields argument + masked_string = data_masker.mask(True) + + # THEN the result is the data masked + assert masked_string == DATA_MASKING_STRING + + +def test_mask_none(data_masker): + # GIVEN a None data type + + # WHEN mask is called with no fields argument + masked_string = data_masker.mask(None) + + # THEN the result is the data masked + assert masked_string == DATA_MASKING_STRING + + +def test_mask_str(data_masker): + # GIVEN a str data type + + # WHEN mask is called with no fields argument + masked_string = data_masker.mask("this is a string") + + # THEN the result is the data masked + assert masked_string == DATA_MASKING_STRING + + +def test_mask_list(data_masker): + # GIVEN a list data type - # WHEN the AWS encryption provider's mask method is called with no fields argument - masked_string = data_masker.mask(value) + # WHEN mask is called with no fields argument + masked_string = data_masker.mask([1, 2, "string", 3]) - # THEN the result is the full input data masked - assert masked_string == value_masked + # THEN the result is the data masked, while maintaining type list + assert masked_string == [DATA_MASKING_STRING, DATA_MASKING_STRING, DATA_MASKING_STRING, DATA_MASKING_STRING] -def test_mask_with_fields(data_masker): - # GIVEN the data type is a dictionary, or a json representation of a dictionary +def test_mask_dict(data_masker): + # GIVEN a dict data type + data = { + "a": { + "1": {"None": "hello", "four": "world"}, + "b": {"3": {"4": "goodbye", "e": "world"}}, + }, + } - # WHEN the AWS encryption provider's mask is called with a list of fields specified - masked_string = data_masker.mask(python_dict, dict_fields) - masked_json_string = data_masker.mask(json_dict, dict_fields) + # WHEN mask is called with no fields argument + masked_string = data_masker.mask(data) + + # THEN the result is the data masked + assert masked_string == DATA_MASKING_STRING + + +def test_mask_dict_with_fields(data_masker): + # GIVEN the data type is a dictionary + data = { + "a": { + "1": {"None": "hello", "four": "world"}, + "b": {"3": {"4": "goodbye", "e": "world"}}, + }, + } + + # WHEN mask is called with a list of fields specified + masked_string = data_masker.mask(data, fields=["a.1.None", "a.b.3.4"]) # THEN the result is only the specified fields are masked - assert masked_string == masked_with_fields - assert masked_json_string == masked_with_fields + assert masked_string == { + "a": { + "1": {"None": DATA_MASKING_STRING, "four": "world"}, + "b": {"3": {"4": DATA_MASKING_STRING, "e": "world"}}, + }, + } + + +def test_mask_json_dict_with_fields(data_masker): + # GIVEN the data type is a json representation of a dictionary + data = json.dumps( + { + "a": { + "1": {"None": "hello", "four": "world"}, + "b": {"3": {"4": "goodbye", "e": "world"}}, + }, + }, + ) + + # WHEN mask is called with a list of fields specified + masked_json_string = data_masker.mask(data, fields=["a.1.None", "a.b.3.4"]) + + # THEN the result is only the specified fields are masked + assert masked_json_string == { + "a": { + "1": {"None": DATA_MASKING_STRING, "four": "world"}, + "b": {"3": {"4": DATA_MASKING_STRING, "e": "world"}}, + }, + } + + +def test_encrypt_int(data_masker): + # GIVEN an int data type + + # WHEN encrypting and then decrypting the encrypted data + encrypted_data = data_masker.encrypt(-1) + decrypted_data = data_masker.decrypt(encrypted_data) + + # THEN the result is the original input data + assert decrypted_data == str(-1) -@pytest.mark.parametrize("value", data_types) -def test_encrypt_decrypt(value, data_masker: DataMasking): - # GIVEN an instantiation of DataMasking with the AWS encryption provider +def test_encrypt_float(data_masker): + # GIVEN an float data type # WHEN encrypting and then decrypting the encrypted data - encrypted_data = data_masker.encrypt(value) + encrypted_data = data_masker.encrypt(-1.11) decrypted_data = data_masker.decrypt(encrypted_data) # THEN the result is the original input data - assert decrypted_data == str(value) + assert decrypted_data == str(-1.11) -@pytest.mark.parametrize("value, fields", zip(dictionaries, fields_to_mask)) -def test_encrypt_decrypt_with_fields(value, fields, data_masker): - # GIVEN an instantiation of DataMasking with the AWS encryption provider +def test_encrypt_bool(data_masker): + # GIVEN an bool data type - # WHEN encrypting and then decrypting the encrypted data with a list of fields - encrypted_data = data_masker.encrypt(value, fields) - decrypted_data = data_masker.decrypt(encrypted_data, fields) + # WHEN encrypting and then decrypting the encrypted data + encrypted_data = data_masker.encrypt(True) + decrypted_data = data_masker.decrypt(encrypted_data) # THEN the result is the original input data - # AWS Encryption SDK decrypt method only returns bytes - print("value:", value) - if value == json_blob: - print("json blob!!!!") - assert decrypted_data == value - else: - assert decrypted_data == aws_encrypted_with_fields + assert decrypted_data == str(True) + + +def test_encrypt_none(data_masker): + # GIVEN an none data type + + # WHEN encrypting and then decrypting the encrypted data + encrypted_data = data_masker.encrypt(None) + decrypted_data = data_masker.decrypt(encrypted_data) + + # THEN the result is the original input data + assert decrypted_data == str(None) + + +def test_encrypt_str(data_masker): + # GIVEN an str data type + + # WHEN encrypting and then decrypting the encrypted data + encrypted_data = data_masker.encrypt("this is a string") + decrypted_data = data_masker.decrypt(encrypted_data) + + # THEN the result is the original input data + assert decrypted_data == str("this is a string") + + +def test_encrypt_list(data_masker): + # GIVEN an list data type + + # WHEN encrypting and then decrypting the encrypted data + encrypted_data = data_masker.encrypt([1, 2, "a string", 3.4]) + decrypted_data = data_masker.decrypt(encrypted_data) + + # THEN the result is the original input data + assert decrypted_data == str([1, 2, "a string", 3.4]) + + +def test_encrypt_dict(data_masker): + # GIVEN an dict data type + + data = { + "a": { + "1": {"None": "hello", "four": "world"}, + "b": {"3": {"4": "goodbye", "e": "world"}}, + }, + } + + # WHEN encrypting and then decrypting the encrypted data + encrypted_data = data_masker.encrypt(data) + decrypted_data = data_masker.decrypt(encrypted_data) + + # THEN the result is the original input data + assert decrypted_data == str(data) + + +def test_encrypt_dict_with_fields(data_masker): + # GIVEN the data type is a dictionary + data = { + "a": { + "1": {"None": "hello", "four": "world"}, + "b": {"3": {"4": "goodbye", "e": "world"}}, + }, + } + + # WHEN encrypting and then decrypting the encrypted data + encrypted_data = data_masker.encrypt(data, fields=["a.1.None", "a.b.3.4"]) + decrypted_data = data_masker.decrypt(encrypted_data, fields=["a.1.None", "a.b.3.4"]) + + # THEN the result is only the specified fields are masked + assert decrypted_data == data + + +def test_encrypt_json_dict_with_fields(data_masker): + # GIVEN the data type is a json representation of a dictionary + data = json.dumps( + { + "a": { + "1": {"None": "hello", "four": "world"}, + "b": {"3": {"4": "goodbye", "e": "world"}}, + }, + }, + ) + + # WHEN encrypting and then decrypting the encrypted data + encrypted_data = data_masker.encrypt(data, fields=["a.1.None", "a.b.3.4"]) + decrypted_data = data_masker.decrypt(encrypted_data, fields=["a.1.None", "a.b.3.4"]) + + # THEN the result is only the specified fields are masked + assert decrypted_data == json.loads(data) + + +def test_encrypt_json_blob_with_fields(data_masker): + # GIVEN the data type is a json representation of a dictionary + data = json.dumps( + { + "a": { + "1": {"None": "hello", "four": "world"}, + "b": {"3": {"4": "goodbye", "e": "world"}}, + }, + }, + ) + + # WHEN encrypting and then decrypting the encrypted data + encrypted_data = data_masker.encrypt(data, fields=["a.1.None", "a.b.3.4"]) + decrypted_data = data_masker.decrypt(encrypted_data, fields=["a.1.None", "a.b.3.4"]) + + # THEN the result is only the specified fields are masked + assert decrypted_data == json.loads(data) diff --git a/tests/unit/data_masking/setup.py b/tests/unit/data_masking/setup.py deleted file mode 100644 index be79712fd21..00000000000 --- a/tests/unit/data_masking/setup.py +++ /dev/null @@ -1,88 +0,0 @@ -import copy -import json - -from aws_lambda_powertools.utilities.data_masking.constants import DATA_MASKING_STRING - -python_dict = { - "a": { - "1": {"None": "hello", "four": "world"}, # None type key doesn't work - "b": {"3": {"4": "goodbye", "e": "world"}}, # key "4.5" doesn't work - }, -} - - -json_dict = json.dumps(python_dict) - - -dict_fields = ["a.1.None", "a.b.3.4"] - - -masked_with_fields = { - "a": {"1": {"None": DATA_MASKING_STRING, "four": "world"}, "b": {"3": {"4": DATA_MASKING_STRING, "e": "world"}}}, -} - -aws_encrypted_with_fields = { - "a": { - "1": {"None": bytes("hello", "utf-8"), "four": "world"}, - "b": {"3": {"4": bytes("goodbye", "utf-8"), "e": "world"}}, - }, -} - -# 10kb JSON blob for latency testing -json_blob = { - "id": 1, - "name": "John Doe", - "age": 30, - "email": "johndoe@example.com", - "address": {"street": "123 Main St", "city": "Anytown", "state": "CA", "zip": "12345"}, - "phone_numbers": ["+1-555-555-1234", "+1-555-555-5678"], - "interests": ["Hiking", "Traveling", "Photography", "Reading"], - "job_history": { - "company": "Acme Inc.", - "position": "Software Engineer", - "start_date": "2015-01-01", - "end_date": "2017-12-31", - }, - "about_me": """ - Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla tincidunt velit quis - sapien mollis, at egestas massa tincidunt. Suspendisse ultrices arcu a dolor dapibus, - ut pretium turpis volutpat. Vestibulum at sapien quis sapien dignissim volutpat ut a enim. - Praesent fringilla sem eu dui convallis luctus. Donec ullamcorper, sapien ut convallis congue, - risus mauris pretium tortor, nec dignissim arcu urna a nisl. Vivamus non fermentum ex. Proin - interdum nisi id sagittis egestas. Nam sit amet nisi nec quam pharetra sagittis. Aliquam erat - volutpat. Donec nec luctus sem, nec ornare lorem. Vivamus vitae orci quis enim faucibus placerat. - Nulla facilisi. Proin in turpis orci. Donec imperdiet velit ac tellus gravida, eget laoreet tellus - malesuada. Praesent venenatis tellus ac urna blandit, at varius felis posuere. Integer a commodo nunc. - """, -} - - -json_blob_fields = ["address.street", "job_history.company"] -aws_encrypted_json_blob = copy.deepcopy(json_blob) -aws_encrypted_json_blob["address"]["street"] = bytes("123 Main St", "utf-8") -aws_encrypted_json_blob["job_history"]["company"] = bytes("Acme Inc.", "utf-8") - -dictionaries = [python_dict, json_dict, json_blob] -fields_to_mask = [dict_fields, dict_fields, json_blob_fields] - - -data_types_and_masks = [ - # simple data types - [42, DATA_MASKING_STRING], - [4.22, DATA_MASKING_STRING], - [True, DATA_MASKING_STRING], - [None, DATA_MASKING_STRING], - ["this is a string", DATA_MASKING_STRING], - # iterables - [[1, 2, 3, 4], [DATA_MASKING_STRING, DATA_MASKING_STRING, DATA_MASKING_STRING, DATA_MASKING_STRING]], - [ - ["hello", 1, 2, 3, "world"], - [DATA_MASKING_STRING, DATA_MASKING_STRING, DATA_MASKING_STRING, DATA_MASKING_STRING, DATA_MASKING_STRING], - ], - # dictionaries - [python_dict, DATA_MASKING_STRING], - [json_dict, DATA_MASKING_STRING], -] - - -data_types = [item[0] for item in data_types_and_masks] From b3d123d1141618d14d2c17476bbc20e3dab8d19b Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Mon, 11 Sep 2023 22:20:17 +0000 Subject: [PATCH 38/56] Updated json serializer for aws encrypt sdk to return original data type --- .../providers/aws_encryption_sdk.py | 23 +++++++++++-------- .../data_masking/test_aws_encryption_sdk.py | 14 +++++------ 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py index 724472c27d4..beb06e8a9eb 100644 --- a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py +++ b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py @@ -1,6 +1,7 @@ import base64 +import json from collections.abc import Iterable -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Tuple, Union import botocore from aws_encryption_sdk import ( @@ -21,13 +22,13 @@ def __init__(self, key): class Singleton: - _instances: Dict[Any, "AwsEncryptionSdkProvider"] = {} + _instances: Dict[Tuple, "AwsEncryptionSdkProvider"] = {} - def __new__(cls, *args, **kwargs): + def __new__(cls, *args, force_new_instance=False, **kwargs): # Generate a unique key based on the configuration. # Create a tuple by iterating through the values in kwargs, sorting them, # and then adding them to the tuple. - config_key = tuple() + config_key = () for value in kwargs.values(): if isinstance(value, Iterable): for val in sorted(value): @@ -35,7 +36,7 @@ def __new__(cls, *args, **kwargs): else: config_key += (value,) - if config_key not in cls._instances: + if force_new_instance or config_key not in cls._instances: cls._instances[config_key] = super(Singleton, cls).__new__(cls, *args) return cls._instances[config_key] @@ -82,12 +83,14 @@ def __init__( ) def _serialize(self, data: Any) -> bytes: - return bytes(str(data), "utf-8") + json_data = json.dumps(data) + return json_data.encode("utf-8") - def _deserialize(self, data: bytes) -> str: - return data.decode("utf-8") + def _deserialize(self, data: bytes) -> Any: + json_data = data.decode("utf-8") + return json.loads(json_data) - def encrypt(self, data: Union[bytes, str], **provider_options) -> str: + def encrypt(self, data: Union[bytes, str], **provider_options) -> bytes: """ Encrypt data using the AwsEncryptionSdkProvider. @@ -108,7 +111,7 @@ def encrypt(self, data: Union[bytes, str], **provider_options) -> str: ciphertext = base64.b64encode(ciphertext).decode() return ciphertext - def decrypt(self, data: str, **provider_options) -> str: + def decrypt(self, data: str, **provider_options) -> Any: """ Decrypt data using AwsEncryptionSdkProvider. diff --git a/tests/functional/data_masking/test_aws_encryption_sdk.py b/tests/functional/data_masking/test_aws_encryption_sdk.py index 4d0cd4e8cfd..0cab8d153b2 100644 --- a/tests/functional/data_masking/test_aws_encryption_sdk.py +++ b/tests/functional/data_masking/test_aws_encryption_sdk.py @@ -148,7 +148,7 @@ def test_encrypt_int(data_masker): decrypted_data = data_masker.decrypt(encrypted_data) # THEN the result is the original input data - assert decrypted_data == str(-1) + assert decrypted_data == -1 def test_encrypt_float(data_masker): @@ -159,7 +159,7 @@ def test_encrypt_float(data_masker): decrypted_data = data_masker.decrypt(encrypted_data) # THEN the result is the original input data - assert decrypted_data == str(-1.11) + assert decrypted_data == -1.11 def test_encrypt_bool(data_masker): @@ -170,7 +170,7 @@ def test_encrypt_bool(data_masker): decrypted_data = data_masker.decrypt(encrypted_data) # THEN the result is the original input data - assert decrypted_data == str(True) + assert decrypted_data is True def test_encrypt_none(data_masker): @@ -181,7 +181,7 @@ def test_encrypt_none(data_masker): decrypted_data = data_masker.decrypt(encrypted_data) # THEN the result is the original input data - assert decrypted_data == str(None) + assert decrypted_data is None def test_encrypt_str(data_masker): @@ -192,7 +192,7 @@ def test_encrypt_str(data_masker): decrypted_data = data_masker.decrypt(encrypted_data) # THEN the result is the original input data - assert decrypted_data == str("this is a string") + assert decrypted_data == "this is a string" def test_encrypt_list(data_masker): @@ -203,7 +203,7 @@ def test_encrypt_list(data_masker): decrypted_data = data_masker.decrypt(encrypted_data) # THEN the result is the original input data - assert decrypted_data == str([1, 2, "a string", 3.4]) + assert decrypted_data == [1, 2, "a string", 3.4] def test_encrypt_dict(data_masker): @@ -221,7 +221,7 @@ def test_encrypt_dict(data_masker): decrypted_data = data_masker.decrypt(encrypted_data) # THEN the result is the original input data - assert decrypted_data == str(data) + assert decrypted_data == data def test_encrypt_dict_with_fields(data_masker): From c0c3f2f6ba226ad5989a693d55e1075f88b3ea91 Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Tue, 12 Sep 2023 20:28:11 +0000 Subject: [PATCH 39/56] Added ability for user input custom json de/serializer in base class --- .../utilities/data_masking/constants.py | 4 +++ .../utilities/data_masking/provider.py | 15 ++++++-- .../providers/aws_encryption_sdk.py | 35 ++++++++----------- .../data_masking/test_aws_encryption_sdk.py | 22 +----------- tests/unit/data_masking/test_data_masking.py | 28 ++++++--------- 5 files changed, 43 insertions(+), 61 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_masking/constants.py b/aws_lambda_powertools/utilities/data_masking/constants.py index cd493378629..47e74f472cf 100644 --- a/aws_lambda_powertools/utilities/data_masking/constants.py +++ b/aws_lambda_powertools/utilities/data_masking/constants.py @@ -1 +1,5 @@ DATA_MASKING_STRING: str = "*****" +CACHE_CAPACITY: int = 100 +MAX_CACHE_AGE_SECONDS: float = 300.0 +MAX_MESSAGES_ENCRYPTED: int = 200 +# NOTE: You can also set max messages/bytes per data key diff --git a/aws_lambda_powertools/utilities/data_masking/provider.py b/aws_lambda_powertools/utilities/data_masking/provider.py index 2542a258b81..a4515e47183 100644 --- a/aws_lambda_powertools/utilities/data_masking/provider.py +++ b/aws_lambda_powertools/utilities/data_masking/provider.py @@ -1,16 +1,27 @@ -from abc import ABC +import json +from abc import ABCMeta from collections.abc import Iterable from typing import Union from aws_lambda_powertools.utilities.data_masking.constants import DATA_MASKING_STRING -class BaseProvider(ABC): +class BaseProvider(metaclass=ABCMeta): """ When you try to create an instance of a subclass that does not implement the encrypt method, you will get a NotImplementedError with a message that says the method is not implemented: """ + def __init__(self, json_serializer=None, json_deserializer=None) -> None: + self.json_serializer = json_serializer or self.default_json_serializer + self.json_deserializer = json_deserializer or self.default_json_deserializer + + def default_json_serializer(self, data): + return json.dumps(data).encode("utf-8") + + def default_json_deserializer(self, data): + return json.loads(data.decode("utf-8")) + def encrypt(self, data) -> Union[bytes, str]: raise NotImplementedError("Subclasses must implement encrypt()") diff --git a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py index beb06e8a9eb..1fb7b80b01a 100644 --- a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py +++ b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py @@ -1,7 +1,6 @@ import base64 -import json from collections.abc import Iterable -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, List, Optional, Tuple, Union import botocore from aws_encryption_sdk import ( @@ -12,6 +11,11 @@ ) from aws_lambda_powertools.shared.user_agent import register_feature_to_botocore_session +from aws_lambda_powertools.utilities.data_masking.constants import ( + CACHE_CAPACITY, + MAX_CACHE_AGE_SECONDS, + MAX_MESSAGES_ENCRYPTED, +) from aws_lambda_powertools.utilities.data_masking.provider import BaseProvider @@ -41,12 +45,6 @@ def __new__(cls, *args, force_new_instance=False, **kwargs): return cls._instances[config_key] -CACHE_CAPACITY: int = 100 -MAX_CACHE_AGE_SECONDS: float = 300.0 -MAX_MESSAGES_ENCRYPTED: int = 200 -# NOTE: You can also set max messages/bytes per data key - - class AwsEncryptionSdkProvider(BaseProvider, Singleton): """ The AwsEncryptionSdkProvider is to be used as a Provider for the Datamasking class. @@ -67,10 +65,13 @@ def __init__( self, keys: List[str], client: Optional[EncryptionSDKClient] = None, - local_cache_capacity: Optional[int] = CACHE_CAPACITY, - max_cache_age_seconds: Optional[float] = MAX_CACHE_AGE_SECONDS, - max_messages_encrypted: Optional[int] = MAX_MESSAGES_ENCRYPTED, + local_cache_capacity: int = CACHE_CAPACITY, + max_cache_age_seconds: float = MAX_CACHE_AGE_SECONDS, + max_messages_encrypted: int = MAX_MESSAGES_ENCRYPTED, + json_serializer: Optional[Callable[[Dict], str]] = None, + json_deserializer: Optional[Callable[[Union[Dict, str, bool, int, float]], str]] = None, ): + super().__init__(json_serializer=json_serializer, json_deserializer=json_deserializer) self.client = client or EncryptionSDKClient() self.keys = keys self.cache = LocalCryptoMaterialsCache(local_cache_capacity) @@ -82,14 +83,6 @@ def __init__( max_messages_encrypted=max_messages_encrypted, ) - def _serialize(self, data: Any) -> bytes: - json_data = json.dumps(data) - return json_data.encode("utf-8") - - def _deserialize(self, data: bytes) -> Any: - json_data = data.decode("utf-8") - return json.loads(json_data) - def encrypt(self, data: Union[bytes, str], **provider_options) -> bytes: """ Encrypt data using the AwsEncryptionSdkProvider. @@ -106,7 +99,7 @@ def encrypt(self, data: Union[bytes, str], **provider_options) -> bytes: ciphertext : str The encrypted data, as a base64-encoded string. """ - data = self._serialize(data) + data = self.json_serializer(data) ciphertext, _ = self.client.encrypt(source=data, materials_manager=self.cache_cmm, **provider_options) ciphertext = base64.b64encode(ciphertext).decode() return ciphertext @@ -141,5 +134,5 @@ def decrypt(self, data: str, **provider_options) -> Any: if decryptor_header.encryption_context.get(key) != value: raise ContextMismatchError(key) - ciphertext = self._deserialize(ciphertext) + ciphertext = self.json_deserializer(ciphertext) return ciphertext diff --git a/tests/functional/data_masking/test_aws_encryption_sdk.py b/tests/functional/data_masking/test_aws_encryption_sdk.py index 0cab8d153b2..27771e6f4e3 100644 --- a/tests/functional/data_masking/test_aws_encryption_sdk.py +++ b/tests/functional/data_masking/test_aws_encryption_sdk.py @@ -97,7 +97,7 @@ def test_mask_dict(data_masker): def test_mask_dict_with_fields(data_masker): - # GIVEN the data type is a dictionary + # GIVEN a dict data type data = { "a": { "1": {"None": "hello", "four": "world"}, @@ -208,7 +208,6 @@ def test_encrypt_list(data_masker): def test_encrypt_dict(data_masker): # GIVEN an dict data type - data = { "a": { "1": {"None": "hello", "four": "world"}, @@ -258,22 +257,3 @@ def test_encrypt_json_dict_with_fields(data_masker): # THEN the result is only the specified fields are masked assert decrypted_data == json.loads(data) - - -def test_encrypt_json_blob_with_fields(data_masker): - # GIVEN the data type is a json representation of a dictionary - data = json.dumps( - { - "a": { - "1": {"None": "hello", "four": "world"}, - "b": {"3": {"4": "goodbye", "e": "world"}}, - }, - }, - ) - - # WHEN encrypting and then decrypting the encrypted data - encrypted_data = data_masker.encrypt(data, fields=["a.1.None", "a.b.3.4"]) - decrypted_data = data_masker.decrypt(encrypted_data, fields=["a.1.None", "a.b.3.4"]) - - # THEN the result is only the specified fields are masked - assert decrypted_data == json.loads(data) diff --git a/tests/unit/data_masking/test_data_masking.py b/tests/unit/data_masking/test_data_masking.py index b227f5eba6f..096eaf0bb6b 100644 --- a/tests/unit/data_masking/test_data_masking.py +++ b/tests/unit/data_masking/test_data_masking.py @@ -88,7 +88,7 @@ def test_mask_dict(data_masker): def test_mask_dict_with_fields(data_masker): - # GIVEN the data type is a dictionary + # GIVEN a dict data type data = { "a": { "1": {"None": "hello", "four": "world"}, @@ -135,9 +135,8 @@ def test_encrypt_not_implemented(data_masker): # GIVEN DataMasking is not initialized with a Provider # WHEN attempting to call the encrypt method on the data - - # THEN the result is a NotImplementedError with pytest.raises(NotImplementedError): + # THEN the result is a NotImplementedError data_masker.encrypt("hello world") @@ -145,9 +144,8 @@ def test_decrypt_not_implemented(data_masker): # GIVEN DataMasking is not initialized with a Provider # WHEN attempting to call the decrypt method on the data - - # THEN the result is a NotImplementedError with pytest.raises(NotImplementedError): + # THEN the result is a NotImplementedError data_masker.decrypt("hello world") @@ -155,15 +153,13 @@ def test_parsing_unsupported_data_type(data_masker): # GIVEN an initialization of the DataMasking class # WHEN attempting to pass in a list of fields with input data that is not a dict - - # THEN the result is a TypeError with pytest.raises(TypeError): + # THEN the result is a TypeError data_masker.mask(42, ["this.field"]) def test_parsing_nonexistent_fields(data_masker): - # GIVEN an initialization of the DataMasking class - + # GIVEN a dict data type data = { "3": { "1": {"None": "hello", "four": "world"}, @@ -172,15 +168,13 @@ def test_parsing_nonexistent_fields(data_masker): } # WHEN attempting to pass in fields that do not exist in the input data - - # THEN the result is a KeyError with pytest.raises(KeyError): + # THEN the result is a KeyError data_masker.mask(data, ["3.1.True"]) def test_parsing_nonstring_fields(data_masker): - # GIVEN an initialization of the DataMasking class - + # GIVEN a dict data type data = { "3": { "1": {"None": "hello", "four": "world"}, @@ -196,16 +190,16 @@ def test_parsing_nonstring_fields(data_masker): def test_parsing_nonstring_keys_and_fields(data_masker): - # GIVEN an initialization of the DataMasking class - - # WHEN the input data is a dictionary with integer keys + # GIVEN a dict data type with integer keys data = { 3: { "1": {"None": "hello", "four": "world"}, 4: {"33": {"5": "goodbye", "e": "world"}}, }, } + + # WHEN masked with a list of fields that are integer keys masked = data_masker.mask(data, fields=[3.4]) - # THEN the result is the value of the nested field should be masked as normal + # THEN the result is the value of the nested field should be masked assert masked == {"3": {"1": {"None": "hello", "four": "world"}, "4": DATA_MASKING_STRING}} From c5233af6fbfc133c36907f3c6a07b998dc341087 Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Tue, 12 Sep 2023 21:29:43 +0000 Subject: [PATCH 40/56] Apply patch for use latest manylinux --- .../utils/lambda_layer/powertools_layer.py | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/tests/e2e/utils/lambda_layer/powertools_layer.py b/tests/e2e/utils/lambda_layer/powertools_layer.py index 70870af200e..05147048676 100644 --- a/tests/e2e/utils/lambda_layer/powertools_layer.py +++ b/tests/e2e/utils/lambda_layer/powertools_layer.py @@ -1,6 +1,6 @@ -import logging import subprocess from pathlib import Path +from typing import List from aws_cdk.aws_lambda import Architecture from checksumdir import dirhash @@ -9,18 +9,20 @@ from tests.e2e.utils.constants import CDK_OUT_PATH, SOURCE_CODE_ROOT_PATH from tests.e2e.utils.lambda_layer.base import BaseLocalLambdaLayer -logger = logging.getLogger(__name__) - class LocalLambdaPowertoolsLayer(BaseLocalLambdaLayer): IGNORE_EXTENSIONS = ["pyc"] + ARCHITECTURE_PLATFORM_MAPPING = { + Architecture.X86_64.name: ("manylinux_2_17_x86_64", "manylinux_2_28_x86_64"), + Architecture.ARM_64.name: ("manylinux_2_17_aarch64", "manylinux_2_28_aarch64"), + } def __init__(self, output_dir: Path = CDK_OUT_PATH, architecture: Architecture = Architecture.X86_64): super().__init__(output_dir) self.package = f"{SOURCE_CODE_ROOT_PATH}[all]" - platform_name = self._resolve_platform(architecture) - self.build_args = f"--platform {platform_name} --only-binary=:all: --upgrade" + self.platform_args = self._resolve_platform(architecture) + self.build_args = f"{self.platform_args} --only-binary=:all: --upgrade" self.build_command = f"python -m pip install {self.package} {self.build_args} --target {self.target_dir}" self.cleanup_command = ( f"rm -rf {self.target_dir}/boto* {self.target_dir}/s3transfer* && " @@ -62,16 +64,20 @@ def _has_source_changed(self) -> bool: return False def _resolve_platform(self, architecture: Architecture) -> str: - """Returns the correct plaform name for the manylinux project (see PEP 599) + """Returns the correct pip platform tag argument for the manylinux project (see PEP 599) Returns ------- - platform_name : str - The platform tag + str + pip's platform argument, e.g., --platform manylinux_2_17_x86_64 --platform manylinux_2_28_x86_64 """ - if architecture.name == Architecture.X86_64.name: - return "manylinux1_x86_64" - elif architecture.name == Architecture.ARM_64.name: - return "manylinux2014_aarch64" - else: - raise ValueError(f"unknown architecture {architecture.name}") + platforms = self.ARCHITECTURE_PLATFORM_MAPPING.get(architecture.name) + if not platforms: + raise ValueError( + f"unknown architecture {architecture.name}. Supported: {self.ARCHITECTURE_PLATFORM_MAPPING.keys()}", + ) + + return self._build_platform_args(platforms) + + def _build_platform_args(self, platforms: List[str]): + return " ".join([f"--platform {platform}" for platform in platforms]) From bcc735a82ad27af9796c6edf0a3c62e16184d7fe Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Wed, 13 Sep 2023 15:52:28 -0700 Subject: [PATCH 41/56] Added KMS permissions to lambda handler for e2e tests --- .../data_masking/handlers/basic_handler.py | 3 ++- tests/e2e/data_masking/infrastructure.py | 7 +++++- tests/e2e/data_masking/test_data_masking.py | 23 ++++++++++--------- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/tests/e2e/data_masking/handlers/basic_handler.py b/tests/e2e/data_masking/handlers/basic_handler.py index 8df533f377d..69d99ea8256 100644 --- a/tests/e2e/data_masking/handlers/basic_handler.py +++ b/tests/e2e/data_masking/handlers/basic_handler.py @@ -12,7 +12,8 @@ def lambda_handler(event, context): logger.append_keys(**append_keys) logger.info(message) - kms_key = event.get("kms_key") + # Encrypting data for test_encryption_in_handler test + kms_key = event.get("kms_key", "") data_masker = DataMasking(provider=AwsEncryptionSdkProvider(keys=[kms_key])) value = [1, 2, "string", 4.5] encrypted_data = data_masker.encrypt(value) diff --git a/tests/e2e/data_masking/infrastructure.py b/tests/e2e/data_masking/infrastructure.py index f6590622b49..ee18b272450 100644 --- a/tests/e2e/data_masking/infrastructure.py +++ b/tests/e2e/data_masking/infrastructure.py @@ -1,15 +1,20 @@ import aws_cdk.aws_kms as kms from aws_cdk import CfnOutput, Duration +from aws_cdk import aws_iam as iam from tests.e2e.utils.infrastructure import BaseInfrastructure class DataMaskingStack(BaseInfrastructure): def create_resources(self): - self.create_lambda_functions(function_props={"timeout": Duration.seconds(10)}) + functions = self.create_lambda_functions(function_props={"timeout": Duration.seconds(10)}) key1 = kms.Key(self.stack, "MyKMSKey1", description="My KMS Key1") CfnOutput(self.stack, "KMSKey1Arn", value=key1.key_arn, description="ARN of the created KMS Key1") key2 = kms.Key(self.stack, "MyKMSKey2", description="My KMS Key2") CfnOutput(self.stack, "KMSKey2Arn", value=key2.key_arn, description="ARN of the created KMS Key2") + + functions["BasicHandler"].add_to_role_policy( + iam.PolicyStatement(effect=iam.Effect.ALLOW, actions=["kms:*"], resources=[key1.key_arn, key2.key_arn]), + ) diff --git a/tests/e2e/data_masking/test_data_masking.py b/tests/e2e/data_masking/test_data_masking.py index 9b5451f7018..78cb3d041a6 100644 --- a/tests/e2e/data_masking/test_data_masking.py +++ b/tests/e2e/data_masking/test_data_masking.py @@ -49,7 +49,7 @@ def test_encryption(data_masker): decrypted_data = data_masker.decrypt(encrypted_data) # THEN the result is the original input data - assert decrypted_data == str(value) + assert decrypted_data == value @pytest.mark.xdist_group(name="data_masking") @@ -64,7 +64,7 @@ def test_encryption_context(data_masker): decrypted_data = data_masker.decrypt(encrypted_data, encryption_context=context) # THEN the result is the original input data - assert decrypted_data == str(value) + assert decrypted_data == value @pytest.mark.xdist_group(name="data_masking") @@ -121,14 +121,14 @@ def test_encryption_provider_singleton(data_masker, kms_key1_arn, kms_key2_arn): decrypted_data = data_masker_2.decrypt(encrypted_data) # THEN the result is the original input data - assert decrypted_data == str(value) + assert decrypted_data == value data_masker_3 = DataMasking(provider=AwsEncryptionSdkProvider(keys=[kms_key2_arn])) assert data_masker_2.provider is not data_masker_3.provider @pytest.mark.xdist_group(name="data_masking") -def test_encryption_in_logs(data_masker, basic_handler_fn, basic_handler_fn_arn): +def test_encryption_in_logs(data_masker, basic_handler_fn, basic_handler_fn_arn, kms_key1_arn): # GIVEN an instantiation of DataMasking with the AWS encryption provider # WHEN encrypting a value and logging it @@ -137,7 +137,7 @@ def test_encryption_in_logs(data_masker, basic_handler_fn, basic_handler_fn_arn) message = encrypted_data custom_key = "order_id" additional_keys = {custom_key: f"{uuid4()}"} - payload = json.dumps({"message": message, "append_keys": additional_keys}) + payload = json.dumps({"message": message, "kms_key": kms_key1_arn, "append_keys": additional_keys}) _, execution_time = data_fetcher.get_lambda_response(lambda_arn=basic_handler_fn_arn, payload=payload) data_fetcher.get_lambda_response(lambda_arn=basic_handler_fn_arn, payload=payload) @@ -148,15 +148,16 @@ def test_encryption_in_logs(data_masker, basic_handler_fn, basic_handler_fn_arn) for log in logs.get_log(key=custom_key): encrypted_data = log.message decrypted_data = data_masker.decrypt(encrypted_data) - assert decrypted_data == str(value) + assert decrypted_data == value -# NOTE: This test is failing currently, need to find a fix for building correct dependencies @pytest.mark.xdist_group(name="data_masking") -def test_encryption_in_handler(basic_handler_fn_arn, kms_key1_arn): - payload = {"kms_key": kms_key1_arn, "append_keys": {"order_id": f"{uuid4()}"}} +def test_encryption_in_handler(data_masker, basic_handler_fn_arn, kms_key1_arn): + # GIVEN a lambda_handler with an instantiation the AWS encryption provider data masker - # WHEN a lambda handler for encryption is invoked + payload = {"kms_key": kms_key1_arn} + + # WHEN the handler is invoked to encrypt data handler_result, _ = data_fetcher.get_lambda_response(lambda_arn=basic_handler_fn_arn, payload=json.dumps(payload)) response = json.loads(handler_result["Payload"].read()) @@ -164,4 +165,4 @@ def test_encryption_in_handler(basic_handler_fn_arn, kms_key1_arn): decrypted_data = data_masker.decrypt(encrypted_data) # THEN decrypting the encrypted data from the response should result in the original value - assert decrypted_data == str([1, 2, "string", 4.5]) + assert decrypted_data == [1, 2, "string", 4.5] From eee4c86597457a82a73b91c2e794645007bcf509 Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Thu, 14 Sep 2023 13:48:53 -0700 Subject: [PATCH 42/56] Clarified variable names and documented logic (wip still need to discuss conventions) --- .../utilities/data_masking/base.py | 64 ++++++++++++++----- 1 file changed, 47 insertions(+), 17 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_masking/base.py b/aws_lambda_powertools/utilities/data_masking/base.py index 356dee91c9f..846527577b4 100644 --- a/aws_lambda_powertools/utilities/data_masking/base.py +++ b/aws_lambda_powertools/utilities/data_masking/base.py @@ -14,7 +14,7 @@ def encrypt(self, data, fields=None, **provider_options): def decrypt(self, data, fields=None, **provider_options): return self._apply_action(data, fields, self.provider.decrypt, **provider_options) - def mask(self, data, fields=None, **provider_options) -> str: + def mask(self, data, fields=None, **provider_options): return self._apply_action(data, fields, self.provider.mask, **provider_options) def _apply_action(self, data, fields, action, **provider_options): @@ -43,16 +43,16 @@ def _apply_action(self, data, fields, action, **provider_options): else: return action(data, **provider_options) - def _apply_action_to_fields(self, data: Union[dict, str], fields, action, **provider_options) -> Union[dict, str]: + def _apply_action_to_fields(self, data: Union[dict, str], fields: list, + action, **provider_options) -> Union[dict, str]: """ - This method takes the input data, which can be either a dictionary or a JSON string representation - of a dictionary, and applies a mask, an encryption, or a decryption to the specified fields. + This method takes the input data, which can be either a dictionary or a JSON string, + and applies a mask, an encryption, or a decryption to the specified fields. Parameters ---------- data : Union[dict, str]) - The input data to process. It can be either a dictionary or a JSON string - representation of a dictionary. + The input data to process. It can be either a dictionary or a JSON string. fields : List A list of fields to apply the action to. Each field can be specified as a string or a list of strings representing nested keys in the dictionary. @@ -66,8 +66,8 @@ def _apply_action_to_fields(self, data: Union[dict, str], fields, action, **prov Returns ------- - str - A JSON string representation of the modified dictionary after applying the action to the + dict + The modified dictionary after applying the action to the specified fields. Raises @@ -75,9 +75,16 @@ def _apply_action_to_fields(self, data: Union[dict, str], fields, action, **prov ValueError If 'fields' parameter is None. TypeError - If the 'data' parameter is not a dictionary or a JSON string representation of a dictionary. - KeyError - If specified 'fields' do not exist in input data + If the 'data' parameter is not a dictionary or a JSON string. + + # TODO: document for field in fields logic (more in code) + Example + ------- + >>> data = {'a': {'b': {'c': 1}}, 'x': {'y': 2}} + >>> fields = ['a.b.c', 'a.x.y'] + The function will transform the value at 'a.b.c' (1) and 'a.x.y' (2) + and store the result as: + new_dict = {'a': {'b': {'c': 'transformed_value'}}, 'x': {'y': 'transformed_value'}} """ if fields is None: @@ -87,28 +94,51 @@ def _apply_action_to_fields(self, data: Union[dict, str], fields, action, **prov # Parse JSON string as dictionary my_dict_parsed = json.loads(data) elif isinstance(data, dict): + #TODO: Revisit Turn into json string so everything has quotes around it. + # Think this is only so integer keys work on str and vice versa? + # so if we don't care abt that then can remove this + # Turn into json string so everything has quotes around it my_dict_parsed = json.dumps(data) # Turn back into dict so can parse it my_dict_parsed = json.loads(my_dict_parsed) else: + #TODO: Revisit error handling in _apply_action_to_fields for string data that isn't a JSON string. + # This is an internal method and can't be called by anything but just in case? + # this method is only called if there's a list of fields, and if the list of + # fields doesn't match the input data + # then they get a key error, would that be enough? raise TypeError( "Unsupported data type. The 'data' parameter must be a dictionary or a JSON string " "representation of a dictionary.", ) - for field in fields: - # prevent overriding loop variable - current_field = field - if not isinstance(field, str): - current_field = json.dumps(field) + #TODO: Revisit variable names in for field in fields to reduce cognitive load + # is nested_key ok? field_path? + for nested_key in fields: + # Prevent overriding loop variable + curr_nested_key = nested_key + + # this is also not necessary if we can ensure the list of fields will be a string + # If the nested_key is not a string, convert it to a string representation + if not isinstance(curr_nested_key, str): + curr_nested_key = json.dumps(curr_nested_key) - keys = current_field.split(".") + # Split the nested key string into a list of nested keys + # ['a.b.c'] -> ['a', 'b', 'c'] + keys = curr_nested_key.split(".") + # Initialize a current dictionary to the root dictionary curr_dict = my_dict_parsed + + # Traverse the dictionary hierarchy by iterating through the list of nested keys for key in keys[:-1]: curr_dict = curr_dict[key] + + # Retrieve the final value of the nested field valtochange = curr_dict[(keys[-1])] + + # Apply the specified 'action' to the target value curr_dict[keys[-1]] = action(valtochange, **provider_options) return my_dict_parsed From ab15acdc3f9f89e639bc85371998af851f96f580 Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Tue, 19 Sep 2023 10:26:31 -0700 Subject: [PATCH 43/56] Polished var names, error strings, documentation, etc --- .../utilities/data_masking/base.py | 36 ++++++++----------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_masking/base.py b/aws_lambda_powertools/utilities/data_masking/base.py index 846527577b4..9019eff3b1d 100644 --- a/aws_lambda_powertools/utilities/data_masking/base.py +++ b/aws_lambda_powertools/utilities/data_masking/base.py @@ -43,8 +43,13 @@ def _apply_action(self, data, fields, action, **provider_options): else: return action(data, **provider_options) - def _apply_action_to_fields(self, data: Union[dict, str], fields: list, - action, **provider_options) -> Union[dict, str]: + def _apply_action_to_fields( + self, + data: Union[dict, str], + fields: list, + action, + **provider_options, + ) -> Union[dict, str]: """ This method takes the input data, which can be either a dictionary or a JSON string, and applies a mask, an encryption, or a decryption to the specified fields. @@ -75,16 +80,17 @@ def _apply_action_to_fields(self, data: Union[dict, str], fields: list, ValueError If 'fields' parameter is None. TypeError - If the 'data' parameter is not a dictionary or a JSON string. + If the 'data' parameter is not a traversable type - # TODO: document for field in fields logic (more in code) Example ------- + ```python >>> data = {'a': {'b': {'c': 1}}, 'x': {'y': 2}} >>> fields = ['a.b.c', 'a.x.y'] - The function will transform the value at 'a.b.c' (1) and 'a.x.y' (2) - and store the result as: + # The function will transform the value at 'a.b.c' (1) and 'a.x.y' (2) + # and store the result as: new_dict = {'a': {'b': {'c': 'transformed_value'}}, 'x': {'y': 'transformed_value'}} + ``` """ if fields is None: @@ -94,32 +100,20 @@ def _apply_action_to_fields(self, data: Union[dict, str], fields: list, # Parse JSON string as dictionary my_dict_parsed = json.loads(data) elif isinstance(data, dict): - #TODO: Revisit Turn into json string so everything has quotes around it. - # Think this is only so integer keys work on str and vice versa? - # so if we don't care abt that then can remove this - - # Turn into json string so everything has quotes around it + # In case their data has keys that are not strings (i.e. ints), convert it all into a JSON string my_dict_parsed = json.dumps(data) # Turn back into dict so can parse it my_dict_parsed = json.loads(my_dict_parsed) else: - #TODO: Revisit error handling in _apply_action_to_fields for string data that isn't a JSON string. - # This is an internal method and can't be called by anything but just in case? - # this method is only called if there's a list of fields, and if the list of - # fields doesn't match the input data - # then they get a key error, would that be enough? raise TypeError( - "Unsupported data type. The 'data' parameter must be a dictionary or a JSON string " - "representation of a dictionary.", + f"Unsupported data type for 'data' parameter. Expected a traversable type, but got {type(data)}.", ) - #TODO: Revisit variable names in for field in fields to reduce cognitive load - # is nested_key ok? field_path? + # ['a.b.c'] in ['a.b.c', 'a.x.y'] for nested_key in fields: # Prevent overriding loop variable curr_nested_key = nested_key - # this is also not necessary if we can ensure the list of fields will be a string # If the nested_key is not a string, convert it to a string representation if not isinstance(curr_nested_key, str): curr_nested_key = json.dumps(curr_nested_key) From 73ae38213c89200a57d4d2f28c1664a9f83476ae Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Thu, 21 Sep 2023 17:49:22 -0700 Subject: [PATCH 44/56] Added a stack for load testing data masking and added artillery config file to run load tests --- .../data_masking/data_masking_load_test.yaml | 40 +++ .../pt-load-test-stack/.gitignore | 243 ++++++++++++++++++ .../pt-load-test-stack/README.md | 163 ++++++++++++ .../pt-load-test-stack/__init__.py | 0 .../pt-load-test-stack/events/hello.json | 111 ++++++++ .../nonsingleton_128/__init__.py | 0 .../nonsingleton_128/app.py | 59 +++++ .../nonsingleton_128/requirements.txt | 3 + .../nonsingleton_1769/__init__.py | 0 .../nonsingleton_1769/app.py | 60 +++++ .../nonsingleton_1769/requirements.txt | 3 + .../pt-load-test-stack/samconfig.toml | 34 +++ .../singleton_128/__init__.py | 0 .../pt-load-test-stack/singleton_128/app.py | 59 +++++ .../singleton_128/requirements.txt | 3 + .../singleton_1769/__init__.py | 0 .../pt-load-test-stack/singleton_1769/app.py | 60 +++++ .../singleton_1769/requirements.txt | 3 + .../pt-load-test-stack/template.yaml | 191 ++++++++++++++ .../pt-load-test-stack/tests/__init__.py | 0 .../pt-load-test-stack/tests/requirements.txt | 3 + .../pt-load-test-stack/tests/unit/__init__.py | 0 .../tests/unit/test_handler.py | 145 +++++++++++ .../{ => data_masking}/test_data_masking.py | 0 24 files changed, 1180 insertions(+) create mode 100644 tests/performance/data_masking/data_masking_load_test.yaml create mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/.gitignore create mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/README.md create mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/__init__.py create mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/events/hello.json create mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_128/__init__.py create mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_128/app.py create mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_128/requirements.txt create mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1769/__init__.py create mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1769/app.py create mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1769/requirements.txt create mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/samconfig.toml create mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_128/__init__.py create mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_128/app.py create mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_128/requirements.txt create mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1769/__init__.py create mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1769/app.py create mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1769/requirements.txt create mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/template.yaml create mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/tests/__init__.py create mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/tests/requirements.txt create mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/tests/unit/__init__.py create mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/tests/unit/test_handler.py rename tests/performance/{ => data_masking}/test_data_masking.py (100%) diff --git a/tests/performance/data_masking/data_masking_load_test.yaml b/tests/performance/data_masking/data_masking_load_test.yaml new file mode 100644 index 00000000000..6adcdf5bc6b --- /dev/null +++ b/tests/performance/data_masking/data_masking_load_test.yaml @@ -0,0 +1,40 @@ +config: + # This is a test server run by team Artillery + # It's designed to be highly scalable + + target: https://sebwc2y7gh.execute-api.us-west-2.amazonaws.com/Prod/singleton128 + phases: + - duration: 60 + arrivalRate: 1 + rampTo: 5 + name: Warm up phase + - duration: 60 + arrivalRate: 5 + rampTo: 10 + name: Ramp up load + - duration: 30 + arrivalRate: 10 + rampTo: 30 + name: Spike phase + # Load & configure a couple of useful plugins + # https://docs.art/reference/extensions + plugins: + ensure: {} + apdex: {} + metrics-by-endpoint: {} + apdex: + threshold: 100 + ensure: + thresholds: + - http.response_time.p99: 100 + - http.response_time.p95: 75 +scenarios: + - flow: + - loop: + - get: + url: "https://sebwc2y7gh.execute-api.us-west-2.amazonaws.com/Prod/singleton1769" + - get: + url: "https://sebwc2y7gh.execute-api.us-west-2.amazonaws.com/Prod/nonsingleton128" + - get: + url: "https://sebwc2y7gh.execute-api.us-west-2.amazonaws.com/Prod/nonsingleton1769" + count: 100 \ No newline at end of file diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/.gitignore b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/.gitignore new file mode 100644 index 00000000000..4c7a643c028 --- /dev/null +++ b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/.gitignore @@ -0,0 +1,243 @@ +# Created by https://www.gitignore.io/api/osx,linux,python,windows,pycharm,visualstudiocode + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### OSX ### +*.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### PyCharm ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff: +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/dictionaries + +# Sensitive or high-churn files: +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.xml +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml + +# Gradle: +.idea/**/gradle.xml +.idea/**/libraries + +# CMake +cmake-build-debug/ + +# Mongo Explorer plugin: +.idea/**/mongoSettings.xml + +## File-based project format: +*.iws + +## Plugin-specific files: + +# IntelliJ +/out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Ruby plugin and RubyMine +/.rakeTasks + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +### PyCharm Patch ### +# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 + +# *.iml +# modules.xml +# .idea/misc.xml +# *.ipr + +# Sonarlint plugin +.idea/sonarlint + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +.pytest_cache/ +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule.* + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +### VisualStudioCode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +.history + +### Windows ### +# Windows thumbnail cache files +Thumbs.db +ehthumbs.db +ehthumbs_vista.db + +# Folder config file +Desktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msm +*.msp + +# Windows shortcuts +*.lnk + +# Build folder + +*/build/* + +# End of https://www.gitignore.io/api/osx,linux,python,windows,pycharm,visualstudiocode diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/README.md b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/README.md new file mode 100644 index 00000000000..aed9d43976d --- /dev/null +++ b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/README.md @@ -0,0 +1,163 @@ +# pt-load-test-stack + +Congratulations, you have just created a Serverless "Hello World" application using the AWS Serverless Application Model (AWS SAM) for the `python3.10` runtime, and options to bootstrap it with [**Powertools for AWS Lambda (Python)**](https://awslabs.github.io/aws-lambda-powertools-python/latest/) (Powertools for AWS Lambda (Python)) utilities for Logging, Tracing and Metrics. + +Powertools for AWS Lambda (Python) is a developer toolkit to implement Serverless best practices and increase developer velocity. + +## Powertools for AWS Lambda (Python) features + +Powertools for AWS Lambda (Python) provides three core utilities: + +* **[Tracing](https://awslabs.github.io/aws-lambda-powertools-python/latest/core/tracer/)** - Decorators and utilities to trace Lambda function handlers, and both synchronous and asynchronous functions +* **[Logging](https://awslabs.github.io/aws-lambda-powertools-python/latest/core/logger/)** - Structured logging made easier, and decorator to enrich structured logging with key Lambda context details +* **[Metrics](https://awslabs.github.io/aws-lambda-powertools-python/latest/core/metrics/)** - Custom Metrics created asynchronously via CloudWatch Embedded Metric Format (EMF) + +Find the complete project's [documentation here](https://awslabs.github.io/aws-lambda-powertools-python). + +### Installing Powertools for AWS Lambda (Python)n + +With [pip](https://pip.pypa.io/en/latest/index.html) installed, run: + +```bash +pip install aws-lambda-powertools +``` + +### Powertools for AWS Lambda (Python) Examples + +* [Tutorial](https://awslabs.github.io/aws-lambda-powertools-python/latest/tutorial) +* [Serverless Shopping cart](https://github.com/aws-samples/aws-serverless-shopping-cart) +* [Serverless Airline](https://github.com/aws-samples/aws-serverless-airline-booking) +* [Serverless E-commerce platform](https://github.com/aws-samples/aws-serverless-ecommerce-platform) +* [Serverless GraphQL Nanny Booking Api](https://github.com/trey-rosius/babysitter_api) + +## Working with this project + +This project contains source code and supporting files for a serverless application that you can deploy with the SAM CLI. It includes the following files and folders. + +* hello_world - Code for the application's Lambda function. +* events - Invocation events that you can use to invoke the function. +* tests - Unit tests for the application code. +* template.yaml - A template that defines the application's AWS resources. + +The application uses several AWS resources, including Lambda functions and an API Gateway API. These resources are defined in the `template.yaml` file in this project. You can update the template to add AWS resources through the same deployment process that updates your application code. + +If you prefer to use an integrated development environment (IDE) to build and test your application, you can use the AWS Toolkit. +The AWS Toolkit is an open source plug-in for popular IDEs that uses the SAM CLI to build and deploy serverless applications on AWS. The AWS Toolkit also adds a simplified step-through debugging experience for Lambda function code. See the following links to get started. + +* [CLion](https://docs.aws.amazon.com/toolkit-for-jetbrains/latest/userguide/welcome.html) +* [GoLand](https://docs.aws.amazon.com/toolkit-for-jetbrains/latest/userguide/welcome.html) +* [IntelliJ](https://docs.aws.amazon.com/toolkit-for-jetbrains/latest/userguide/welcome.html) +* [WebStorm](https://docs.aws.amazon.com/toolkit-for-jetbrains/latest/userguide/welcome.html) +* [Rider](https://docs.aws.amazon.com/toolkit-for-jetbrains/latest/userguide/welcome.html) +* [PhpStorm](https://docs.aws.amazon.com/toolkit-for-jetbrains/latest/userguide/welcome.html) +* [PyCharm](https://docs.aws.amazon.com/toolkit-for-jetbrains/latest/userguide/welcome.html) +* [RubyMine](https://docs.aws.amazon.com/toolkit-for-jetbrains/latest/userguide/welcome.html) +* [DataGrip](https://docs.aws.amazon.com/toolkit-for-jetbrains/latest/userguide/welcome.html) +* [VS Code](https://docs.aws.amazon.com/toolkit-for-vscode/latest/userguide/welcome.html) +* [Visual Studio](https://docs.aws.amazon.com/toolkit-for-visual-studio/latest/user-guide/welcome.html) + +### Deploy the sample application + +The Serverless Application Model Command Line Interface (SAM CLI) is an extension of the AWS CLI that adds functionality for building and testing Lambda applications. It uses Docker to run your functions in an Amazon Linux environment that matches Lambda. It can also emulate your application's build environment and API. + +To use the SAM CLI, you need the following tools. + +* SAM CLI - [Install the SAM CLI](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-cli-install.html) +* [Python 3 installed](https://www.python.org/downloads/) +* Docker - [Install Docker community edition](https://hub.docker.com/search/?type=edition&offering=community) + +To build and deploy your application for the first time, run the following in your shell: + +```bash +sam build --use-container +sam deploy --guided +``` + +The first command will build the source of your application. The second command will package and deploy your application to AWS, with a series of prompts: + +* **Stack Name**: The name of the stack to deploy to CloudFormation. This should be unique to your account and region, and a good starting point would be something matching your project name. +* **AWS Region**: The AWS region you want to deploy your app to. +* **Confirm changes before deploy**: If set to yes, any change sets will be shown to you before execution for manual review. If set to no, the AWS SAM CLI will automatically deploy application changes. +* **Allow SAM CLI IAM role creation**: Many AWS SAM templates, including this example, create AWS IAM roles required for the AWS Lambda function(s) included to access AWS services. By default, these are scoped down to minimum required permissions. To deploy an AWS CloudFormation stack which creates or modifies IAM roles, the `CAPABILITY_IAM` value for `capabilities` must be provided. If permission isn't provided through this prompt, to deploy this example you must explicitly pass `--capabilities CAPABILITY_IAM` to the `sam deploy` command. +* **Save arguments to samconfig.toml**: If set to yes, your choices will be saved to a configuration file inside the project, so that in the future you can just re-run `sam deploy` without parameters to deploy changes to your application. + +You can find your API Gateway Endpoint URL in the output values displayed after deployment. + +### Use the SAM CLI to build and test locally + +Build your application with the `sam build --use-container` command. + +```bash +pt-load-test-stack$ sam build --use-container +``` + +The SAM CLI installs dependencies defined in `hello_world/requirements.txt`, creates a deployment package, and saves it in the `.aws-sam/build` folder. + +Test a single function by invoking it directly with a test event. An event is a JSON document that represents the input that the function receives from the event source. Test events are included in the `events` folder in this project. + +Run functions locally and invoke them with the `sam local invoke` command. + +```bash +pt-load-test-stack$ sam local invoke HelloWorldFunction --event events/event.json +``` + +The SAM CLI can also emulate your application's API. Use the `sam local start-api` to run the API locally on port 3000. + +```bash +pt-load-test-stack$ sam local start-api +pt-load-test-stack$ curl http://localhost:3000/ +``` + +The SAM CLI reads the application template to determine the API's routes and the functions that they invoke. The `Events` property on each function's definition includes the route and method for each path. + +```yaml + Events: + HelloWorld: + Type: Api + Properties: + Path: /hello + Method: get +``` + +### Add a resource to your application + +The application template uses AWS Serverless Application Model (AWS SAM) to define application resources. AWS SAM is an extension of AWS CloudFormation with a simpler syntax for configuring common serverless application resources such as functions, triggers, and APIs. For resources not included in [the SAM specification](https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md), you can use standard [AWS CloudFormation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-template-resource-type-ref.html) resource types. + +### Fetch, tail, and filter Lambda function logs + +To simplify troubleshooting, SAM CLI has a command called `sam logs`. `sam logs` lets you fetch logs generated by your deployed Lambda function from the command line. In addition to printing the logs on the terminal, this command has several nifty features to help you quickly find the bug. + +`NOTE`: This command works for all AWS Lambda functions; not just the ones you deploy using SAM. + +```bash +pt-load-test-stack$ sam logs -n HelloWorldFunction --stack-name pt-load-test-stack --tail +``` + +You can find more information and examples about filtering Lambda function logs in the [SAM CLI Documentation](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-cli-logging.html). + +### Tests + +Tests are defined in the `tests` folder in this project. Use PIP to install the test dependencies and run tests. + +```bash +pt-load-test-stack$ pip install -r tests/requirements.txt --user +# unit test +pt-load-test-stack$ python -m pytest tests/unit -v +# integration test, requiring deploying the stack first. +# Create the env variable AWS_SAM_STACK_NAME with the name of the stack we are testing +pt-load-test-stack$ AWS_SAM_STACK_NAME="pt-load-test-stack" python -m pytest tests/integration -v +``` + +### Cleanup + +To delete the sample application that you created, use the AWS CLI. Assuming you used your project name for the stack name, you can run the following: + +```bash +sam delete --stack-name "pt-load-test-stack" +``` + +## Resources + +See the [AWS SAM developer guide](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/what-is-sam.html) for an introduction to SAM specification, the SAM CLI, and serverless application concepts. + +Next, you can use AWS Serverless Application Repository to deploy ready to use Apps that go beyond hello world samples and learn how authors developed their applications: [AWS Serverless Application Repository main page](https://aws.amazon.com/serverless/serverlessrepo/) \ No newline at end of file diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/__init__.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/events/hello.json b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/events/hello.json new file mode 100644 index 00000000000..fdb5180fe0a --- /dev/null +++ b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/events/hello.json @@ -0,0 +1,111 @@ +{ + "body":"", + "headers":{ + "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", + "Accept-Encoding":"gzip, deflate, br", + "Accept-Language":"pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7", + "Cache-Control":"max-age=0", + "Connection":"keep-alive", + "Host":"127.0.0.1:3000", + "Sec-Ch-Ua":"\"Google Chrome\";v=\"105\", \"Not)A;Brand\";v=\"8\", \"Chromium\";v=\"105\"", + "Sec-Ch-Ua-Mobile":"?0", + "Sec-Ch-Ua-Platform":"\"Linux\"", + "Sec-Fetch-Dest":"document", + "Sec-Fetch-Mode":"navigate", + "Sec-Fetch-Site":"none", + "Sec-Fetch-User":"?1", + "Upgrade-Insecure-Requests":"1", + "User-Agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36", + "X-Forwarded-Port":"3000", + "X-Forwarded-Proto":"http" + }, + "httpMethod":"GET", + "isBase64Encoded": false, + "multiValueHeaders":{ + "Accept":[ + "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" + ], + "Accept-Encoding":[ + "gzip, deflate, br" + ], + "Accept-Language":[ + "pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7" + ], + "Cache-Control":[ + "max-age=0" + ], + "Connection":[ + "keep-alive" + ], + "Host":[ + "127.0.0.1:3000" + ], + "Sec-Ch-Ua":[ + "\"Google Chrome\";v=\"105\", \"Not)A;Brand\";v=\"8\", \"Chromium\";v=\"105\"" + ], + "Sec-Ch-Ua-Mobile":[ + "?0" + ], + "Sec-Ch-Ua-Platform":[ + "\"Linux\"" + ], + "Sec-Fetch-Dest":[ + "document" + ], + "Sec-Fetch-Mode":[ + "navigate" + ], + "Sec-Fetch-Site":[ + "none" + ], + "Sec-Fetch-User":[ + "?1" + ], + "Upgrade-Insecure-Requests":[ + "1" + ], + "User-Agent":[ + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36" + ], + "X-Forwarded-Port":[ + "3000" + ], + "X-Forwarded-Proto":[ + "http" + ] + }, + "multiValueQueryStringParameters":"", + "path":"/hello", + "pathParameters":"", + "queryStringParameters":"", + "requestContext":{ + "accountId":"123456789012", + "apiId":"1234567890", + "domainName":"127.0.0.1:3000", + "extendedRequestId":"", + "httpMethod":"GET", + "identity":{ + "accountId":"", + "apiKey":"", + "caller":"", + "cognitoAuthenticationProvider":"", + "cognitoAuthenticationType":"", + "cognitoIdentityPoolId":"", + "sourceIp":"127.0.0.1", + "user":"", + "userAgent":"Custom User Agent String", + "userArn":"" + }, + "path":"/hello", + "protocol":"HTTP/1.1", + "requestId":"a3590457-cac2-4f10-8fc9-e47114bf7c62", + "requestTime":"02/Feb/2023:11:45:26 +0000", + "requestTimeEpoch":1675338326, + "resourceId":"123456", + "resourcePath":"/hello", + "stage":"Prod" + }, + "resource":"/hello", + "stageVariables":"", + "version":"1.0" + } diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_128/__init__.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_128/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_128/app.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_128/app.py new file mode 100644 index 00000000000..cb785ea38a0 --- /dev/null +++ b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_128/app.py @@ -0,0 +1,59 @@ +import os +from aws_lambda_powertools.event_handler import APIGatewayRestResolver +from aws_lambda_powertools.utilities.typing import LambdaContext +from aws_lambda_powertools.logging import correlation_paths +from aws_lambda_powertools import Logger +from aws_lambda_powertools import Tracer +from aws_lambda_powertools.utilities.data_masking.base import DataMasking +from aws_lambda_powertools.utilities.data_masking.providers.aws_encryption_sdk import AwsEncryptionSdkProviderWithoutSingleton + +KMS_KEY_ARN = os.environ['KMS_KEY_ARN'] + +json_blob = { + "id": 1, + "name": "John Doe", + "age": 30, + "email": "johndoe@example.com", + "address": {"street": "123 Main St", "city": "Anytown", "state": "CA", "zip": "12345"}, + "phone_numbers": ["+1-555-555-1234", "+1-555-555-5678"], + "interests": ["Hiking", "Traveling", "Photography", "Reading"], + "job_history": { + "company": { + "company_name": "Acme Inc.", + "company_address": "5678 Interview Dr.", + }, + "position": "Software Engineer", + "start_date": "2015-01-01", + "end_date": "2017-12-31", + }, + "about_me": """ + Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla tincidunt velit quis + sapien mollis, at egestas massa tincidunt. Suspendisse ultrices arcu a dolor dapibus, + ut pretium turpis volutpat. Vestibulum at sapien quis sapien dignissim volutpat ut a enim. + Praesent fringilla sem eu dui convallis luctus. Donec ullamcorper, sapien ut convallis congue, + risus mauris pretium tortor, nec dignissim arcu urna a nisl. Vivamus non fermentum ex. Proin + interdum nisi id sagittis egestas. Nam sit amet nisi nec quam pharetra sagittis. Aliquam erat + volutpat. Donec nec luctus sem, nec ornare lorem. Vivamus vitae orci quis enim faucibus placerat. + Nulla facilisi. Proin in turpis orci. Donec imperdiet velit ac tellus gravida, eget laoreet tellus + malesuada. Praesent venenatis tellus ac urna blandit, at varius felis posuere. Integer a commodo nunc. + """, +} + +app = APIGatewayRestResolver() +tracer = Tracer() +logger = Logger() + +@app.get("/nonsingleton128") +@tracer.capture_method +def nonsingleton128(): + logger.info("Hello world non singleton128 - HTTP 200") + data_masker = DataMasking(provider=AwsEncryptionSdkProviderWithoutSingleton(keys=[KMS_KEY_ARN])) + encrypted = data_masker.encrypt(json_blob, fields=["address.street", "job_history.company.company_name"]) + decrypted = data_masker.decrypt(encrypted, fields=["address.street", "job_history.company.company_name"]) + #TODO: decrypt in another function + return {"Decrypted_json_blob_non_singleton_128": decrypted} + +@logger.inject_lambda_context(correlation_id_path=correlation_paths.API_GATEWAY_REST) +@tracer.capture_lambda_handler +def lambda_handler(event: dict, context: LambdaContext) -> dict: + return app.resolve(event, context) diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_128/requirements.txt b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_128/requirements.txt new file mode 100644 index 00000000000..b74b60fc263 --- /dev/null +++ b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_128/requirements.txt @@ -0,0 +1,3 @@ +requests +aws-lambda-powertools[tracer] +aws-encryption-sdk diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1769/__init__.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1769/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1769/app.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1769/app.py new file mode 100644 index 00000000000..16cb9ab4b91 --- /dev/null +++ b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1769/app.py @@ -0,0 +1,60 @@ +import os +from aws_lambda_powertools.event_handler import APIGatewayRestResolver +from aws_lambda_powertools.utilities.typing import LambdaContext +from aws_lambda_powertools.logging import correlation_paths +from aws_lambda_powertools import Logger +from aws_lambda_powertools import Tracer +from aws_lambda_powertools.utilities.data_masking.base import DataMasking +from aws_lambda_powertools.utilities.data_masking.providers.aws_encryption_sdk import AwsEncryptionSdkProviderWithoutSingleton + +KMS_KEY_ARN = os.environ['KMS_KEY_ARN'] + +json_blob = { + "id": 1, + "name": "John Doe", + "age": 30, + "email": "johndoe@example.com", + "address": {"street": "123 Main St", "city": "Anytown", "state": "CA", "zip": "12345"}, + "phone_numbers": ["+1-555-555-1234", "+1-555-555-5678"], + "interests": ["Hiking", "Traveling", "Photography", "Reading"], + "job_history": { + "company": { + "company_name": "Acme Inc.", + "company_address": "5678 Interview Dr.", + }, + "position": "Software Engineer", + "start_date": "2015-01-01", + "end_date": "2017-12-31", + }, + "about_me": """ + Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla tincidunt velit quis + sapien mollis, at egestas massa tincidunt. Suspendisse ultrices arcu a dolor dapibus, + ut pretium turpis volutpat. Vestibulum at sapien quis sapien dignissim volutpat ut a enim. + Praesent fringilla sem eu dui convallis luctus. Donec ullamcorper, sapien ut convallis congue, + risus mauris pretium tortor, nec dignissim arcu urna a nisl. Vivamus non fermentum ex. Proin + interdum nisi id sagittis egestas. Nam sit amet nisi nec quam pharetra sagittis. Aliquam erat + volutpat. Donec nec luctus sem, nec ornare lorem. Vivamus vitae orci quis enim faucibus placerat. + Nulla facilisi. Proin in turpis orci. Donec imperdiet velit ac tellus gravida, eget laoreet tellus + malesuada. Praesent venenatis tellus ac urna blandit, at varius felis posuere. Integer a commodo nunc. + """, +} + +app = APIGatewayRestResolver() +tracer = Tracer() +logger = Logger() + +@app.get("/nonsingleton1769") +@tracer.capture_method +def nonsingleton1769(): + logger.info("Hello world non singleton1769 - HTTP 200") + data_masker = DataMasking(provider=AwsEncryptionSdkProviderWithoutSingleton(keys=[KMS_KEY_ARN])) + encrypted = data_masker.encrypt(json_blob, fields=["address.street", "job_history.company.company_name"]) + decrypted = data_masker.decrypt(encrypted, fields=["address.street", "job_history.company.company_name"]) + #TODO: decrypt in another function + return {"Decrypted_json_blob_non_singleton_1769": decrypted} + + +@logger.inject_lambda_context(correlation_id_path=correlation_paths.API_GATEWAY_REST) +@tracer.capture_lambda_handler +def lambda_handler(event: dict, context: LambdaContext) -> dict: + return app.resolve(event, context) diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1769/requirements.txt b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1769/requirements.txt new file mode 100644 index 00000000000..b74b60fc263 --- /dev/null +++ b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1769/requirements.txt @@ -0,0 +1,3 @@ +requests +aws-lambda-powertools[tracer] +aws-encryption-sdk diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/samconfig.toml b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/samconfig.toml new file mode 100644 index 00000000000..82f9cdc06d9 --- /dev/null +++ b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/samconfig.toml @@ -0,0 +1,34 @@ +# More information about the configuration file can be found here: +# https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-cli-config.html +version = 0.1 + +[default] +[default.global.parameters] +stack_name = "pt-load-test-stack" + +[default.build.parameters] +cached = true +parallel = true + +[default.validate.parameters] +lint = true + +[default.deploy.parameters] +capabilities = "CAPABILITY_IAM" +confirm_changeset = true +resolve_s3 = true +s3_prefix = "pt-load-test-stack" +region = "us-west-2" +image_repositories = [] + +[default.package.parameters] +resolve_s3 = true + +[default.sync.parameters] +watch = true + +[default.local_start_api.parameters] +warm_containers = "EAGER" + +[default.local_start_lambda.parameters] +warm_containers = "EAGER" diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_128/__init__.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_128/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_128/app.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_128/app.py new file mode 100644 index 00000000000..44c22c3575f --- /dev/null +++ b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_128/app.py @@ -0,0 +1,59 @@ +import os +from aws_lambda_powertools.event_handler import APIGatewayRestResolver +from aws_lambda_powertools.utilities.typing import LambdaContext +from aws_lambda_powertools.logging import correlation_paths +from aws_lambda_powertools import Logger +from aws_lambda_powertools import Tracer +from aws_lambda_powertools.utilities.data_masking.base import DataMasking +from aws_lambda_powertools.utilities.data_masking.providers.aws_encryption_sdk import AwsEncryptionSdkProvider + +KMS_KEY_ARN = os.environ['KMS_KEY_ARN'] + +json_blob = { + "id": 1, + "name": "John Doe", + "age": 30, + "email": "johndoe@example.com", + "address": {"street": "123 Main St", "city": "Anytown", "state": "CA", "zip": "12345"}, + "phone_numbers": ["+1-555-555-1234", "+1-555-555-5678"], + "interests": ["Hiking", "Traveling", "Photography", "Reading"], + "job_history": { + "company": { + "company_name": "Acme Inc.", + "company_address": "5678 Interview Dr.", + }, + "position": "Software Engineer", + "start_date": "2015-01-01", + "end_date": "2017-12-31", + }, + "about_me": """ + Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla tincidunt velit quis + sapien mollis, at egestas massa tincidunt. Suspendisse ultrices arcu a dolor dapibus, + ut pretium turpis volutpat. Vestibulum at sapien quis sapien dignissim volutpat ut a enim. + Praesent fringilla sem eu dui convallis luctus. Donec ullamcorper, sapien ut convallis congue, + risus mauris pretium tortor, nec dignissim arcu urna a nisl. Vivamus non fermentum ex. Proin + interdum nisi id sagittis egestas. Nam sit amet nisi nec quam pharetra sagittis. Aliquam erat + volutpat. Donec nec luctus sem, nec ornare lorem. Vivamus vitae orci quis enim faucibus placerat. + Nulla facilisi. Proin in turpis orci. Donec imperdiet velit ac tellus gravida, eget laoreet tellus + malesuada. Praesent venenatis tellus ac urna blandit, at varius felis posuere. Integer a commodo nunc. + """, +} + +app = APIGatewayRestResolver() +tracer = Tracer() +logger = Logger() + +@app.get("/singleton128") +@tracer.capture_method +def singleton128(): + logger.info("Hello world singleton128 - HTTP 200") + data_masker = DataMasking(provider=AwsEncryptionSdkProvider(keys=[KMS_KEY_ARN])) + encrypted = data_masker.encrypt(json_blob, fields=["address.street", "job_history.company.company_name"]) + decrypted = data_masker.decrypt(encrypted, fields=["address.street", "job_history.company.company_name"]) + #TODO: decrypt in another function + return {"Decrypted_json_blob_singleton_128": decrypted} + +@logger.inject_lambda_context(correlation_id_path=correlation_paths.API_GATEWAY_REST) +@tracer.capture_lambda_handler +def lambda_handler(event: dict, context: LambdaContext) -> dict: + return app.resolve(event, context) diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_128/requirements.txt b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_128/requirements.txt new file mode 100644 index 00000000000..b74b60fc263 --- /dev/null +++ b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_128/requirements.txt @@ -0,0 +1,3 @@ +requests +aws-lambda-powertools[tracer] +aws-encryption-sdk diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1769/__init__.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1769/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1769/app.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1769/app.py new file mode 100644 index 00000000000..6661f193564 --- /dev/null +++ b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1769/app.py @@ -0,0 +1,60 @@ +import os +from aws_lambda_powertools.event_handler import APIGatewayRestResolver +from aws_lambda_powertools.utilities.typing import LambdaContext +from aws_lambda_powertools.logging import correlation_paths +from aws_lambda_powertools import Logger +from aws_lambda_powertools import Tracer +from aws_lambda_powertools.utilities.data_masking.base import DataMasking +from aws_lambda_powertools.utilities.data_masking.providers.aws_encryption_sdk import AwsEncryptionSdkProvider + +KMS_KEY_ARN = os.environ['KMS_KEY_ARN'] + +json_blob = { + "id": 1, + "name": "John Doe", + "age": 30, + "email": "johndoe@example.com", + "address": {"street": "123 Main St", "city": "Anytown", "state": "CA", "zip": "12345"}, + "phone_numbers": ["+1-555-555-1234", "+1-555-555-5678"], + "interests": ["Hiking", "Traveling", "Photography", "Reading"], + "job_history": { + "company": { + "company_name": "Acme Inc.", + "company_address": "5678 Interview Dr.", + }, + "position": "Software Engineer", + "start_date": "2015-01-01", + "end_date": "2017-12-31", + }, + "about_me": """ + Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla tincidunt velit quis + sapien mollis, at egestas massa tincidunt. Suspendisse ultrices arcu a dolor dapibus, + ut pretium turpis volutpat. Vestibulum at sapien quis sapien dignissim volutpat ut a enim. + Praesent fringilla sem eu dui convallis luctus. Donec ullamcorper, sapien ut convallis congue, + risus mauris pretium tortor, nec dignissim arcu urna a nisl. Vivamus non fermentum ex. Proin + interdum nisi id sagittis egestas. Nam sit amet nisi nec quam pharetra sagittis. Aliquam erat + volutpat. Donec nec luctus sem, nec ornare lorem. Vivamus vitae orci quis enim faucibus placerat. + Nulla facilisi. Proin in turpis orci. Donec imperdiet velit ac tellus gravida, eget laoreet tellus + malesuada. Praesent venenatis tellus ac urna blandit, at varius felis posuere. Integer a commodo nunc. + """, +} + +app = APIGatewayRestResolver() +tracer = Tracer() +logger = Logger() + +@app.get("/singleton1769") +@tracer.capture_method +def singleton1769(): + logger.info("Hello world singleton1769 - HTTP 200") + data_masker = DataMasking(provider=AwsEncryptionSdkProvider(keys=[KMS_KEY_ARN])) + encrypted = data_masker.encrypt(json_blob, fields=["address.street", "job_history.company.company_name"]) + decrypted = data_masker.decrypt(encrypted, fields=["address.street", "job_history.company.company_name"]) + #TODO: decrypt in another function + return {"Decrypted_json_blob_singleton_1769": decrypted} + + +@logger.inject_lambda_context(correlation_id_path=correlation_paths.API_GATEWAY_REST) +@tracer.capture_lambda_handler +def lambda_handler(event: dict, context: LambdaContext) -> dict: + return app.resolve(event, context) diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1769/requirements.txt b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1769/requirements.txt new file mode 100644 index 00000000000..b74b60fc263 --- /dev/null +++ b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1769/requirements.txt @@ -0,0 +1,3 @@ +requests +aws-lambda-powertools[tracer] +aws-encryption-sdk diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/template.yaml b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/template.yaml new file mode 100644 index 00000000000..22593c35392 --- /dev/null +++ b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/template.yaml @@ -0,0 +1,191 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: > + pt-load-test-stack + + Powertools for AWS Lambda (Python) example + +Globals: # https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-specification-template-anatomy-globals.html + Function: + Timeout: 5 + Runtime: python3.10 + + Tracing: Active + Api: + TracingEnabled: true +Resources: + MyKMSKey: + Type: AWS::KMS::Key + Properties: + Enabled: true + KeyPolicy: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: kms:* + Resource: "*" + Principal: + AWS: !Join [ "", [ "arn:aws:iam::", !Ref "AWS::AccountId", ":root" ] ] + Function128Singleton: + Type: AWS::Serverless::Function # More info about Function Resource: https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-resource-function.html + Properties: + Handler: app.lambda_handler + CodeUri: singleton_128 + Description: function 128 MB Singleton + MemorySize: 128 + Architectures: + - x86_64 + Policies: + Statement: + - Effect: Allow + Action: kms:* + Resource: !GetAtt MyKMSKey.Arn + Tracing: Active + Events: + HelloPath: + Type: Api # More info about API Event Source: https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-property-function-api.html + Properties: + Path: /singleton128 + Method: GET + # Powertools for AWS Lambda (Python) env vars: https://awslabs.github.io/aws-lambda-powertools-python/#environment-variables + Environment: + Variables: + POWERTOOLS_SERVICE_NAME: PowertoolsHelloWorld + POWERTOOLS_METRICS_NAMESPACE: Powertools + LOG_LEVEL: INFO + KMS_KEY_ARN: !GetAtt MyKMSKey.Arn + Tags: + LambdaPowertools: python + + Function1769Singleton: + Type: AWS::Serverless::Function # More info about Function Resource: https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-resource-function.html + Properties: + Handler: app.lambda_handler + CodeUri: singleton_1769 + Description: function 1769 MB Singleton + MemorySize: 1769 + Architectures: + - x86_64 + Policies: + Statement: + - Effect: Allow + Action: kms:* + Resource: !GetAtt MyKMSKey.Arn + Tracing: Active + Events: + HelloPath: + Type: Api # More info about API Event Source: https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-property-function-api.html + Properties: + Path: /singleton1769 + Method: GET + # Powertools for AWS Lambda (Python) env vars: https://awslabs.github.io/aws-lambda-powertools-python/#environment-variables + Environment: + Variables: + POWERTOOLS_SERVICE_NAME: PowertoolsHelloWorld + POWERTOOLS_METRICS_NAMESPACE: Powertools + LOG_LEVEL: INFO + KMS_KEY_ARN: !GetAtt MyKMSKey.Arn + Tags: + LambdaPowertools: python + + Function128NonSingleton: + Type: AWS::Serverless::Function # More info about Function Resource: https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-resource-function.html + Properties: + Handler: app.lambda_handler + CodeUri: nonsingleton_128 + Description: function 128 MB Non Singleton + MemorySize: 128 + Architectures: + - x86_64 + Policies: + Statement: + - Effect: Allow + Action: kms:* + Resource: !GetAtt MyKMSKey.Arn + Tracing: Active + Events: + HelloPath: + Type: Api # More info about API Event Source: https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-property-function-api.html + Properties: + Path: /nonsingleton128 + Method: GET + # Powertools for AWS Lambda (Python) env vars: https://awslabs.github.io/aws-lambda-powertools-python/#environment-variables + Environment: + Variables: + POWERTOOLS_SERVICE_NAME: PowertoolsHelloWorld + POWERTOOLS_METRICS_NAMESPACE: Powertools + LOG_LEVEL: INFO + KMS_KEY_ARN: !GetAtt MyKMSKey.Arn + Tags: + LambdaPowertools: python + + Function1769NonSingleton: + Type: AWS::Serverless::Function # More info about Function Resource: https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-resource-function.html + Properties: + Handler: app.lambda_handler + CodeUri: nonsingleton_1769 + Description: function 1769 MB Non Singleton + MemorySize: 1769 + Architectures: + - x86_64 + Policies: + Statement: + - Effect: Allow + Action: kms:* + Resource: !GetAtt MyKMSKey.Arn + Tracing: Active + Events: + HelloPath: + Type: Api # More info about API Event Source: https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-property-function-api.html + Properties: + Path: /nonsingleton1769 + Method: GET + # Powertools for AWS Lambda (Python) env vars: https://awslabs.github.io/aws-lambda-powertools-python/#environment-variables + Environment: + Variables: + POWERTOOLS_SERVICE_NAME: PowertoolsHelloWorld + POWERTOOLS_METRICS_NAMESPACE: Powertools + LOG_LEVEL: INFO + KMS_KEY_ARN: !GetAtt MyKMSKey.Arn + Tags: + LambdaPowertools: python + +Outputs: + KMSKeyArn: + Description: ARN of the KMS Key + Value: !GetAtt MyKMSKey.Arn + + + 128SingletonApi: + Description: API Gateway endpoint URL for Prod environment for Function 128 MB Singleton + Value: !Sub "https://${ServerlessRestApi}.execute-api.${AWS::Region}.amazonaws.com/Prod/singleton128" + + 1769SingletonApi: + Description: API Gateway endpoint URL for Prod environment for Function 1769 MB Singleton + Value: !Sub "https://${ServerlessRestApi}.execute-api.${AWS::Region}.amazonaws.com/Prod/singleton1769" + + Function128Singleton: + Description: Lambda Function 128 MB Singleton ARN + Value: !GetAtt Function128Singleton.Arn + + Function1769Singleton: + Description: Lambda Function 1769 MB Singleton ARN + Value: !GetAtt Function1769Singleton.Arn + + + 128NonSingletonApi: + Description: API Gateway endpoint URL for Prod environment for Function 128 MB Non Singleton + Value: !Sub "https://${ServerlessRestApi}.execute-api.${AWS::Region}.amazonaws.com/Prod/nonsingleton128" + + 1769NonSingletonApi: + Description: API Gateway endpoint URL for Prod environment for Function 1769 MB Non Singleton + Value: !Sub "https://${ServerlessRestApi}.execute-api.${AWS::Region}.amazonaws.com/Prod/nonsingleton1769" + + Function128NonSingleton: + Description: Lambda Function 128 MB Non Singleton ARN + Value: !GetAtt Function128NonSingleton.Arn + + Function1769NonSingleton: + Description: Lambda Function 1769 MB Non Singleton ARN + Value: !GetAtt Function1769NonSingleton.Arn + diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/tests/__init__.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/tests/requirements.txt b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/tests/requirements.txt new file mode 100644 index 00000000000..b9cf27ab27a --- /dev/null +++ b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/tests/requirements.txt @@ -0,0 +1,3 @@ +pytest +boto3 +requests diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/tests/unit/__init__.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/tests/unit/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/tests/unit/test_handler.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/tests/unit/test_handler.py new file mode 100644 index 00000000000..d192c1b1b34 --- /dev/null +++ b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/tests/unit/test_handler.py @@ -0,0 +1,145 @@ +import json + +import pytest + +from hello_world import app + +def lambda_context(): + class LambdaContext: + def __init__(self): + self.function_name = "test-func" + self.memory_limit_in_mb = 128 + self.invoked_function_arn = "arn:aws:lambda:eu-west-1:809313241234:function:test-func" + self.aws_request_id = "52fdfc07-2182-154f-163f-5f0f9a621d72" + + def get_remaining_time_in_millis(self) -> int: + return 1000 + + return LambdaContext() + + +@pytest.fixture() +def apigw_event(): + """ Generates API GW Event""" + + return { + "body":"", + "headers":{ + "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", + "Accept-Encoding":"gzip, deflate, br", + "Accept-Language":"pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7", + "Cache-Control":"max-age=0", + "Connection":"keep-alive", + "Host":"127.0.0.1:3000", + "Sec-Ch-Ua":"\"Google Chrome\";v=\"105\", \"Not)A;Brand\";v=\"8\", \"Chromium\";v=\"105\"", + "Sec-Ch-Ua-Mobile":"?0", + "Sec-Ch-Ua-Platform":"\"Linux\"", + "Sec-Fetch-Dest":"document", + "Sec-Fetch-Mode":"navigate", + "Sec-Fetch-Site":"none", + "Sec-Fetch-User":"?1", + "Upgrade-Insecure-Requests":"1", + "User-Agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36", + "X-Forwarded-Port":"3000", + "X-Forwarded-Proto":"http" + }, + "httpMethod":"GET", + "isBase64Encoded":False, + "multiValueHeaders":{ + "Accept":[ + "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" + ], + "Accept-Encoding":[ + "gzip, deflate, br" + ], + "Accept-Language":[ + "pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7" + ], + "Cache-Control":[ + "max-age=0" + ], + "Connection":[ + "keep-alive" + ], + "Host":[ + "127.0.0.1:3000" + ], + "Sec-Ch-Ua":[ + "\"Google Chrome\";v=\"105\", \"Not)A;Brand\";v=\"8\", \"Chromium\";v=\"105\"" + ], + "Sec-Ch-Ua-Mobile":[ + "?0" + ], + "Sec-Ch-Ua-Platform":[ + "\"Linux\"" + ], + "Sec-Fetch-Dest":[ + "document" + ], + "Sec-Fetch-Mode":[ + "navigate" + ], + "Sec-Fetch-Site":[ + "none" + ], + "Sec-Fetch-User":[ + "?1" + ], + "Upgrade-Insecure-Requests":[ + "1" + ], + "User-Agent":[ + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36" + ], + "X-Forwarded-Port":[ + "3000" + ], + "X-Forwarded-Proto":[ + "http" + ] + }, + "multiValueQueryStringParameters":"", + "path":"/hello", + "pathParameters":"", + "queryStringParameters":"", + "requestContext":{ + "accountId":"123456789012", + "apiId":"1234567890", + "domainName":"127.0.0.1:3000", + "extendedRequestId":"", + "httpMethod":"GET", + "identity":{ + "accountId":"", + "apiKey":"", + "caller":"", + "cognitoAuthenticationProvider":"", + "cognitoAuthenticationType":"", + "cognitoIdentityPoolId":"", + "sourceIp":"127.0.0.1", + "user":"", + "userAgent":"Custom User Agent String", + "userArn":"" + }, + "path":"/hello", + "protocol":"HTTP/1.1", + "requestId":"a3590457-cac2-4f10-8fc9-e47114bf7c62", + "requestTime":"02/Feb/2023:11:45:26 +0000", + "requestTimeEpoch":1675338326, + "resourceId":"123456", + "resourcePath":"/hello", + "stage":"Prod" + }, + "resource":"/hello", + "stageVariables":"", + "version":"1.0" +} + + +def test_lambda_handler(apigw_event): + + ret = app.lambda_handler(apigw_event, lambda_context()) + data = json.loads(ret["body"]) + + assert ret["statusCode"] == 200 + assert "message" in ret["body"] + assert data["message"] == "hello world" diff --git a/tests/performance/test_data_masking.py b/tests/performance/data_masking/test_data_masking.py similarity index 100% rename from tests/performance/test_data_masking.py rename to tests/performance/data_masking/test_data_masking.py From 39a835e8cd3a4ec9dca52e3ad28bf9f927aea361 Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Fri, 22 Sep 2023 14:23:37 -0700 Subject: [PATCH 45/56] Added 1024MB funcs and load tested with them --- .../data_masking/data_masking_load_test.yaml | 6 ++ .../nonsingleton_1024/__init__.py | 0 .../nonsingleton_1024/app.py | 59 ++++++++++++++ .../nonsingleton_1024/requirements.txt | 3 + .../singleton_1024/__init__.py | 0 .../pt-load-test-stack/singleton_1024/app.py | 59 ++++++++++++++ .../singleton_1024/requirements.txt | 3 + .../pt-load-test-stack/template.yaml | 81 ++++++++++++++++++- 8 files changed, 209 insertions(+), 2 deletions(-) create mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1024/__init__.py create mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1024/app.py create mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1024/requirements.txt create mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1024/__init__.py create mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1024/app.py create mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1024/requirements.txt diff --git a/tests/performance/data_masking/data_masking_load_test.yaml b/tests/performance/data_masking/data_masking_load_test.yaml index 6adcdf5bc6b..8ca11cb997f 100644 --- a/tests/performance/data_masking/data_masking_load_test.yaml +++ b/tests/performance/data_masking/data_masking_load_test.yaml @@ -31,10 +31,16 @@ config: scenarios: - flow: - loop: + - get: + url: "https://sebwc2y7gh.execute-api.us-west-2.amazonaws.com/Prod/singleton128" + - get: + url: "https://sebwc2y7gh.execute-api.us-west-2.amazonaws.com/Prod/singleton1024" - get: url: "https://sebwc2y7gh.execute-api.us-west-2.amazonaws.com/Prod/singleton1769" - get: url: "https://sebwc2y7gh.execute-api.us-west-2.amazonaws.com/Prod/nonsingleton128" + - get: + url: "https://sebwc2y7gh.execute-api.us-west-2.amazonaws.com/Prod/nonsingleton1024" - get: url: "https://sebwc2y7gh.execute-api.us-west-2.amazonaws.com/Prod/nonsingleton1769" count: 100 \ No newline at end of file diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1024/__init__.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1024/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1024/app.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1024/app.py new file mode 100644 index 00000000000..9cf25de0c10 --- /dev/null +++ b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1024/app.py @@ -0,0 +1,59 @@ +import os +from aws_lambda_powertools.event_handler import APIGatewayRestResolver +from aws_lambda_powertools.utilities.typing import LambdaContext +from aws_lambda_powertools.logging import correlation_paths +from aws_lambda_powertools import Logger +from aws_lambda_powertools import Tracer +from aws_lambda_powertools.utilities.data_masking.base import DataMasking +from aws_lambda_powertools.utilities.data_masking.providers.aws_encryption_sdk import AwsEncryptionSdkProviderWithoutSingleton + +KMS_KEY_ARN = os.environ['KMS_KEY_ARN'] + +json_blob = { + "id": 1, + "name": "John Doe", + "age": 30, + "email": "johndoe@example.com", + "address": {"street": "123 Main St", "city": "Anytown", "state": "CA", "zip": "12345"}, + "phone_numbers": ["+1-555-555-1234", "+1-555-555-5678"], + "interests": ["Hiking", "Traveling", "Photography", "Reading"], + "job_history": { + "company": { + "company_name": "Acme Inc.", + "company_address": "5678 Interview Dr.", + }, + "position": "Software Engineer", + "start_date": "2015-01-01", + "end_date": "2017-12-31", + }, + "about_me": """ + Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla tincidunt velit quis + sapien mollis, at egestas massa tincidunt. Suspendisse ultrices arcu a dolor dapibus, + ut pretium turpis volutpat. Vestibulum at sapien quis sapien dignissim volutpat ut a enim. + Praesent fringilla sem eu dui convallis luctus. Donec ullamcorper, sapien ut convallis congue, + risus mauris pretium tortor, nec dignissim arcu urna a nisl. Vivamus non fermentum ex. Proin + interdum nisi id sagittis egestas. Nam sit amet nisi nec quam pharetra sagittis. Aliquam erat + volutpat. Donec nec luctus sem, nec ornare lorem. Vivamus vitae orci quis enim faucibus placerat. + Nulla facilisi. Proin in turpis orci. Donec imperdiet velit ac tellus gravida, eget laoreet tellus + malesuada. Praesent venenatis tellus ac urna blandit, at varius felis posuere. Integer a commodo nunc. + """, +} + +app = APIGatewayRestResolver() +tracer = Tracer() +logger = Logger() + +@app.get("/nonsingleton1024") +@tracer.capture_method +def nonsingleton1024(): + logger.info("Hello world non singleton1024 - HTTP 200") + data_masker = DataMasking(provider=AwsEncryptionSdkProviderWithoutSingleton(keys=[KMS_KEY_ARN])) + encrypted = data_masker.encrypt(json_blob, fields=["address.street", "job_history.company.company_name"]) + decrypted = data_masker.decrypt(encrypted, fields=["address.street", "job_history.company.company_name"]) + #TODO: decrypt in another function + return {"Decrypted_json_blob_non_singleton_1024": decrypted} + +@logger.inject_lambda_context(correlation_id_path=correlation_paths.API_GATEWAY_REST) +@tracer.capture_lambda_handler +def lambda_handler(event: dict, context: LambdaContext) -> dict: + return app.resolve(event, context) diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1024/requirements.txt b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1024/requirements.txt new file mode 100644 index 00000000000..b74b60fc263 --- /dev/null +++ b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1024/requirements.txt @@ -0,0 +1,3 @@ +requests +aws-lambda-powertools[tracer] +aws-encryption-sdk diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1024/__init__.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1024/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1024/app.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1024/app.py new file mode 100644 index 00000000000..6844b1a22e1 --- /dev/null +++ b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1024/app.py @@ -0,0 +1,59 @@ +import os +from aws_lambda_powertools.event_handler import APIGatewayRestResolver +from aws_lambda_powertools.utilities.typing import LambdaContext +from aws_lambda_powertools.logging import correlation_paths +from aws_lambda_powertools import Logger +from aws_lambda_powertools import Tracer +from aws_lambda_powertools.utilities.data_masking.base import DataMasking +from aws_lambda_powertools.utilities.data_masking.providers.aws_encryption_sdk import AwsEncryptionSdkProvider + +KMS_KEY_ARN = os.environ['KMS_KEY_ARN'] + +json_blob = { + "id": 1, + "name": "John Doe", + "age": 30, + "email": "johndoe@example.com", + "address": {"street": "123 Main St", "city": "Anytown", "state": "CA", "zip": "12345"}, + "phone_numbers": ["+1-555-555-1234", "+1-555-555-5678"], + "interests": ["Hiking", "Traveling", "Photography", "Reading"], + "job_history": { + "company": { + "company_name": "Acme Inc.", + "company_address": "5678 Interview Dr.", + }, + "position": "Software Engineer", + "start_date": "2015-01-01", + "end_date": "2017-12-31", + }, + "about_me": """ + Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla tincidunt velit quis + sapien mollis, at egestas massa tincidunt. Suspendisse ultrices arcu a dolor dapibus, + ut pretium turpis volutpat. Vestibulum at sapien quis sapien dignissim volutpat ut a enim. + Praesent fringilla sem eu dui convallis luctus. Donec ullamcorper, sapien ut convallis congue, + risus mauris pretium tortor, nec dignissim arcu urna a nisl. Vivamus non fermentum ex. Proin + interdum nisi id sagittis egestas. Nam sit amet nisi nec quam pharetra sagittis. Aliquam erat + volutpat. Donec nec luctus sem, nec ornare lorem. Vivamus vitae orci quis enim faucibus placerat. + Nulla facilisi. Proin in turpis orci. Donec imperdiet velit ac tellus gravida, eget laoreet tellus + malesuada. Praesent venenatis tellus ac urna blandit, at varius felis posuere. Integer a commodo nunc. + """, +} + +app = APIGatewayRestResolver() +tracer = Tracer() +logger = Logger() + +@app.get("/singleton1024") +@tracer.capture_method +def singleton1024(): + logger.info("Hello world singleton1024 - HTTP 200") + data_masker = DataMasking(provider=AwsEncryptionSdkProvider(keys=[KMS_KEY_ARN])) + encrypted = data_masker.encrypt(json_blob, fields=["address.street", "job_history.company.company_name"]) + decrypted = data_masker.decrypt(encrypted, fields=["address.street", "job_history.company.company_name"]) + #TODO: decrypt in another function + return {"Decrypted_json_blob_singleton_1024": decrypted} + +@logger.inject_lambda_context(correlation_id_path=correlation_paths.API_GATEWAY_REST) +@tracer.capture_lambda_handler +def lambda_handler(event: dict, context: LambdaContext) -> dict: + return app.resolve(event, context) diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1024/requirements.txt b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1024/requirements.txt new file mode 100644 index 00000000000..b74b60fc263 --- /dev/null +++ b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1024/requirements.txt @@ -0,0 +1,3 @@ +requests +aws-lambda-powertools[tracer] +aws-encryption-sdk diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/template.yaml b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/template.yaml index 22593c35392..695ca0b2a70 100644 --- a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/template.yaml +++ b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/template.yaml @@ -57,6 +57,38 @@ Resources: Tags: LambdaPowertools: python + Function1024Singleton: + Type: AWS::Serverless::Function # More info about Function Resource: https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-resource-function.html + Properties: + Handler: app.lambda_handler + CodeUri: singleton_1024 + Description: function 1024 MB Singleton + MemorySize: 1024 + Architectures: + - x86_64 + Policies: + Statement: + - Effect: Allow + Action: kms:* + Resource: !GetAtt MyKMSKey.Arn + Tracing: Active + Events: + HelloPath: + Type: Api # More info about API Event Source: https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-property-function-api.html + Properties: + Path: /singleton1024 + Method: GET + # Powertools for AWS Lambda (Python) env vars: https://awslabs.github.io/aws-lambda-powertools-python/#environment-variables + Environment: + Variables: + POWERTOOLS_SERVICE_NAME: PowertoolsHelloWorld + POWERTOOLS_METRICS_NAMESPACE: Powertools + LOG_LEVEL: INFO + KMS_KEY_ARN: !GetAtt MyKMSKey.Arn + Tags: + LambdaPowertools: python + + Function1769Singleton: Type: AWS::Serverless::Function # More info about Function Resource: https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-resource-function.html Properties: @@ -119,6 +151,37 @@ Resources: Tags: LambdaPowertools: python + Function1024NonSingleton: + Type: AWS::Serverless::Function # More info about Function Resource: https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-resource-function.html + Properties: + Handler: app.lambda_handler + CodeUri: nonsingleton_1024 + Description: function 1024 MB Non Singleton + MemorySize: 1024 + Architectures: + - x86_64 + Policies: + Statement: + - Effect: Allow + Action: kms:* + Resource: !GetAtt MyKMSKey.Arn + Tracing: Active + Events: + HelloPath: + Type: Api # More info about API Event Source: https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-property-function-api.html + Properties: + Path: /nonsingleton1024 + Method: GET + # Powertools for AWS Lambda (Python) env vars: https://awslabs.github.io/aws-lambda-powertools-python/#environment-variables + Environment: + Variables: + POWERTOOLS_SERVICE_NAME: PowertoolsHelloWorld + POWERTOOLS_METRICS_NAMESPACE: Powertools + LOG_LEVEL: INFO + KMS_KEY_ARN: !GetAtt MyKMSKey.Arn + Tags: + LambdaPowertools: python + Function1769NonSingleton: Type: AWS::Serverless::Function # More info about Function Resource: https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-resource-function.html Properties: @@ -155,11 +218,14 @@ Outputs: Description: ARN of the KMS Key Value: !GetAtt MyKMSKey.Arn - 128SingletonApi: Description: API Gateway endpoint URL for Prod environment for Function 128 MB Singleton Value: !Sub "https://${ServerlessRestApi}.execute-api.${AWS::Region}.amazonaws.com/Prod/singleton128" + 1024SingletonApi: + Description: API Gateway endpoint URL for Prod environment for Function 1024 MB Singleton + Value: !Sub "https://${ServerlessRestApi}.execute-api.${AWS::Region}.amazonaws.com/Prod/singleton1024" + 1769SingletonApi: Description: API Gateway endpoint URL for Prod environment for Function 1769 MB Singleton Value: !Sub "https://${ServerlessRestApi}.execute-api.${AWS::Region}.amazonaws.com/Prod/singleton1769" @@ -168,15 +234,22 @@ Outputs: Description: Lambda Function 128 MB Singleton ARN Value: !GetAtt Function128Singleton.Arn + Function1024Singleton: + Description: Lambda Function 1024 MB Singleton ARN + Value: !GetAtt Function1024Singleton.Arn + Function1769Singleton: Description: Lambda Function 1769 MB Singleton ARN Value: !GetAtt Function1769Singleton.Arn - 128NonSingletonApi: Description: API Gateway endpoint URL for Prod environment for Function 128 MB Non Singleton Value: !Sub "https://${ServerlessRestApi}.execute-api.${AWS::Region}.amazonaws.com/Prod/nonsingleton128" + 1024NonSingletonApi: + Description: API Gateway endpoint URL for Prod environment for Function 1024 MB Non Singleton + Value: !Sub "https://${ServerlessRestApi}.execute-api.${AWS::Region}.amazonaws.com/Prod/nonsingleton1024" + 1769NonSingletonApi: Description: API Gateway endpoint URL for Prod environment for Function 1769 MB Non Singleton Value: !Sub "https://${ServerlessRestApi}.execute-api.${AWS::Region}.amazonaws.com/Prod/nonsingleton1769" @@ -185,6 +258,10 @@ Outputs: Description: Lambda Function 128 MB Non Singleton ARN Value: !GetAtt Function128NonSingleton.Arn + Function1024NonSingleton: + Description: Lambda Function 1024 MB Non Singleton ARN + Value: !GetAtt Function1024NonSingleton.Arn + Function1769NonSingleton: Description: Lambda Function 1769 MB Non Singleton ARN Value: !GetAtt Function1769NonSingleton.Arn From da24bcf021ec4e79a97854f3c461fcf85ce5f887 Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Mon, 25 Sep 2023 20:47:04 -0700 Subject: [PATCH 46/56] Removed orchestrator function and test since same test in E2E --- tests/performance/data_masking/data_masking_load_test.yaml | 5 +---- .../pt-load-test-stack/singleton_1024/app.py | 1 - .../pt-load-test-stack/singleton_128/app.py | 1 - .../load_test_data_masking/pt-load-test-stack/template.yaml | 2 -- 4 files changed, 1 insertion(+), 8 deletions(-) diff --git a/tests/performance/data_masking/data_masking_load_test.yaml b/tests/performance/data_masking/data_masking_load_test.yaml index 8ca11cb997f..58dee96b1bd 100644 --- a/tests/performance/data_masking/data_masking_load_test.yaml +++ b/tests/performance/data_masking/data_masking_load_test.yaml @@ -1,7 +1,4 @@ config: - # This is a test server run by team Artillery - # It's designed to be highly scalable - target: https://sebwc2y7gh.execute-api.us-west-2.amazonaws.com/Prod/singleton128 phases: - duration: 60 @@ -23,7 +20,7 @@ config: apdex: {} metrics-by-endpoint: {} apdex: - threshold: 100 + threshold: 500 ensure: thresholds: - http.response_time.p99: 100 diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1024/app.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1024/app.py index 6844b1a22e1..0e69bf07f9c 100644 --- a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1024/app.py +++ b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1024/app.py @@ -50,7 +50,6 @@ def singleton1024(): data_masker = DataMasking(provider=AwsEncryptionSdkProvider(keys=[KMS_KEY_ARN])) encrypted = data_masker.encrypt(json_blob, fields=["address.street", "job_history.company.company_name"]) decrypted = data_masker.decrypt(encrypted, fields=["address.street", "job_history.company.company_name"]) - #TODO: decrypt in another function return {"Decrypted_json_blob_singleton_1024": decrypted} @logger.inject_lambda_context(correlation_id_path=correlation_paths.API_GATEWAY_REST) diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_128/app.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_128/app.py index 44c22c3575f..9b53a27347c 100644 --- a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_128/app.py +++ b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_128/app.py @@ -50,7 +50,6 @@ def singleton128(): data_masker = DataMasking(provider=AwsEncryptionSdkProvider(keys=[KMS_KEY_ARN])) encrypted = data_masker.encrypt(json_blob, fields=["address.street", "job_history.company.company_name"]) decrypted = data_masker.decrypt(encrypted, fields=["address.street", "job_history.company.company_name"]) - #TODO: decrypt in another function return {"Decrypted_json_blob_singleton_128": decrypted} @logger.inject_lambda_context(correlation_id_path=correlation_paths.API_GATEWAY_REST) diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/template.yaml b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/template.yaml index 695ca0b2a70..27b0f573908 100644 --- a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/template.yaml +++ b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/template.yaml @@ -87,8 +87,6 @@ Resources: KMS_KEY_ARN: !GetAtt MyKMSKey.Arn Tags: LambdaPowertools: python - - Function1769Singleton: Type: AWS::Serverless::Function # More info about Function Resource: https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-resource-function.html Properties: From 970df5cdb28ef0ae3c2d0474919dc94ff53aa990 Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Tue, 26 Sep 2023 12:03:24 -0700 Subject: [PATCH 47/56] Removed singleton class from code and load and e2e tests --- .../providers/aws_encryption_sdk.py | 25 +-- tests/e2e/data_masking/test_data_masking.py | 17 -- .../data_masking/data_masking_load_test.yaml | 19 +- .../__init__.py | 0 .../{singleton_128 => function_1024}/app.py | 18 +- .../requirements.txt | 0 .../__init__.py | 0 .../{singleton_1024 => function_128}/app.py | 18 +- .../requirements.txt | 0 .../__init__.py | 0 .../{singleton_1769 => function_1769}/app.py | 18 +- .../requirements.txt | 0 .../nonsingleton_1024/app.py | 59 ------ .../nonsingleton_128/app.py | 59 ------ .../nonsingleton_1769/app.py | 60 ------ .../singleton_1024/__init__.py | 0 .../singleton_1024/requirements.txt | 3 - .../singleton_128/__init__.py | 0 .../singleton_128/requirements.txt | 3 - .../singleton_1769/__init__.py | 0 .../singleton_1769/requirements.txt | 3 - .../pt-load-test-stack/template.yaml | 179 +++--------------- .../pt-load-test-stack/tests/__init__.py | 0 .../pt-load-test-stack/tests/requirements.txt | 3 - .../pt-load-test-stack/tests/unit/__init__.py | 0 .../tests/unit/test_handler.py | 145 -------------- 26 files changed, 65 insertions(+), 564 deletions(-) rename tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/{nonsingleton_1024 => function_1024}/__init__.py (100%) rename tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/{singleton_128 => function_1024}/app.py (89%) rename tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/{nonsingleton_1024 => function_1024}/requirements.txt (100%) rename tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/{nonsingleton_128 => function_128}/__init__.py (100%) rename tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/{singleton_1024 => function_128}/app.py (89%) rename tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/{nonsingleton_128 => function_128}/requirements.txt (100%) rename tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/{nonsingleton_1769 => function_1769}/__init__.py (100%) rename tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/{singleton_1769 => function_1769}/app.py (88%) rename tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/{nonsingleton_1769 => function_1769}/requirements.txt (100%) delete mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1024/app.py delete mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_128/app.py delete mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1769/app.py delete mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1024/__init__.py delete mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1024/requirements.txt delete mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_128/__init__.py delete mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_128/requirements.txt delete mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1769/__init__.py delete mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1769/requirements.txt delete mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/tests/__init__.py delete mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/tests/requirements.txt delete mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/tests/unit/__init__.py delete mode 100644 tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/tests/unit/test_handler.py diff --git a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py index 1fb7b80b01a..39852ae2d12 100644 --- a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py +++ b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py @@ -1,6 +1,5 @@ import base64 -from collections.abc import Iterable -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, List, Optional, Union import botocore from aws_encryption_sdk import ( @@ -25,27 +24,7 @@ def __init__(self, key): self.key = key -class Singleton: - _instances: Dict[Tuple, "AwsEncryptionSdkProvider"] = {} - - def __new__(cls, *args, force_new_instance=False, **kwargs): - # Generate a unique key based on the configuration. - # Create a tuple by iterating through the values in kwargs, sorting them, - # and then adding them to the tuple. - config_key = () - for value in kwargs.values(): - if isinstance(value, Iterable): - for val in sorted(value): - config_key += (val,) - else: - config_key += (value,) - - if force_new_instance or config_key not in cls._instances: - cls._instances[config_key] = super(Singleton, cls).__new__(cls, *args) - return cls._instances[config_key] - - -class AwsEncryptionSdkProvider(BaseProvider, Singleton): +class AwsEncryptionSdkProvider(BaseProvider): """ The AwsEncryptionSdkProvider is to be used as a Provider for the Datamasking class. diff --git a/tests/e2e/data_masking/test_data_masking.py b/tests/e2e/data_masking/test_data_masking.py index 78cb3d041a6..dbc4817ba71 100644 --- a/tests/e2e/data_masking/test_data_masking.py +++ b/tests/e2e/data_masking/test_data_masking.py @@ -110,23 +110,6 @@ def test_encryption_decryption_key_mismatch(data_masker, kms_key2_arn): data_masker_key2.decrypt(encrypted_data) -def test_encryption_provider_singleton(data_masker, kms_key1_arn, kms_key2_arn): - data_masker_2 = DataMasking(provider=AwsEncryptionSdkProvider(keys=[kms_key1_arn])) - assert data_masker.provider is data_masker_2.provider - - value = [1, 2, "string", 4.5] - - # WHEN encrypting and then decrypting the encrypted data - encrypted_data = data_masker.encrypt(value) - decrypted_data = data_masker_2.decrypt(encrypted_data) - - # THEN the result is the original input data - assert decrypted_data == value - - data_masker_3 = DataMasking(provider=AwsEncryptionSdkProvider(keys=[kms_key2_arn])) - assert data_masker_2.provider is not data_masker_3.provider - - @pytest.mark.xdist_group(name="data_masking") def test_encryption_in_logs(data_masker, basic_handler_fn, basic_handler_fn_arn, kms_key1_arn): # GIVEN an instantiation of DataMasking with the AWS encryption provider diff --git a/tests/performance/data_masking/data_masking_load_test.yaml b/tests/performance/data_masking/data_masking_load_test.yaml index 58dee96b1bd..5f696d57114 100644 --- a/tests/performance/data_masking/data_masking_load_test.yaml +++ b/tests/performance/data_masking/data_masking_load_test.yaml @@ -1,5 +1,5 @@ config: - target: https://sebwc2y7gh.execute-api.us-west-2.amazonaws.com/Prod/singleton128 + target: https://sebwc2y7gh.execute-api.us-west-2.amazonaws.com/Prod/function128 phases: - duration: 60 arrivalRate: 1 @@ -16,28 +16,17 @@ config: # Load & configure a couple of useful plugins # https://docs.art/reference/extensions plugins: - ensure: {} apdex: {} metrics-by-endpoint: {} apdex: threshold: 500 - ensure: - thresholds: - - http.response_time.p99: 100 - - http.response_time.p95: 75 scenarios: - flow: - loop: - get: - url: "https://sebwc2y7gh.execute-api.us-west-2.amazonaws.com/Prod/singleton128" + url: "https://sebwc2y7gh.execute-api.us-west-2.amazonaws.com/Prod/function128" - get: - url: "https://sebwc2y7gh.execute-api.us-west-2.amazonaws.com/Prod/singleton1024" + url: "https://sebwc2y7gh.execute-api.us-west-2.amazonaws.com/Prod/function1024" - get: - url: "https://sebwc2y7gh.execute-api.us-west-2.amazonaws.com/Prod/singleton1769" - - get: - url: "https://sebwc2y7gh.execute-api.us-west-2.amazonaws.com/Prod/nonsingleton128" - - get: - url: "https://sebwc2y7gh.execute-api.us-west-2.amazonaws.com/Prod/nonsingleton1024" - - get: - url: "https://sebwc2y7gh.execute-api.us-west-2.amazonaws.com/Prod/nonsingleton1769" + url: "https://sebwc2y7gh.execute-api.us-west-2.amazonaws.com/Prod/function1769" count: 100 \ No newline at end of file diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1024/__init__.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/function_1024/__init__.py similarity index 100% rename from tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1024/__init__.py rename to tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/function_1024/__init__.py diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_128/app.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/function_1024/app.py similarity index 89% rename from tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_128/app.py rename to tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/function_1024/app.py index 9b53a27347c..e311fbf9b8f 100644 --- a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_128/app.py +++ b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/function_1024/app.py @@ -1,13 +1,13 @@ import os + +from aws_lambda_powertools import Logger, Tracer from aws_lambda_powertools.event_handler import APIGatewayRestResolver -from aws_lambda_powertools.utilities.typing import LambdaContext from aws_lambda_powertools.logging import correlation_paths -from aws_lambda_powertools import Logger -from aws_lambda_powertools import Tracer from aws_lambda_powertools.utilities.data_masking.base import DataMasking from aws_lambda_powertools.utilities.data_masking.providers.aws_encryption_sdk import AwsEncryptionSdkProvider +from aws_lambda_powertools.utilities.typing import LambdaContext -KMS_KEY_ARN = os.environ['KMS_KEY_ARN'] +KMS_KEY_ARN = os.environ["KMS_KEY_ARN"] json_blob = { "id": 1, @@ -43,14 +43,16 @@ tracer = Tracer() logger = Logger() -@app.get("/singleton128") + +@app.get("/function1024") @tracer.capture_method -def singleton128(): - logger.info("Hello world singleton128 - HTTP 200") +def function1024(): + logger.info("Hello world function1024 - HTTP 200") data_masker = DataMasking(provider=AwsEncryptionSdkProvider(keys=[KMS_KEY_ARN])) encrypted = data_masker.encrypt(json_blob, fields=["address.street", "job_history.company.company_name"]) decrypted = data_masker.decrypt(encrypted, fields=["address.street", "job_history.company.company_name"]) - return {"Decrypted_json_blob_singleton_128": decrypted} + return {"Decrypted_json_blob_function_1024": decrypted} + @logger.inject_lambda_context(correlation_id_path=correlation_paths.API_GATEWAY_REST) @tracer.capture_lambda_handler diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1024/requirements.txt b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/function_1024/requirements.txt similarity index 100% rename from tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1024/requirements.txt rename to tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/function_1024/requirements.txt diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_128/__init__.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/function_128/__init__.py similarity index 100% rename from tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_128/__init__.py rename to tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/function_128/__init__.py diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1024/app.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/function_128/app.py similarity index 89% rename from tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1024/app.py rename to tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/function_128/app.py index 0e69bf07f9c..574aa8fd583 100644 --- a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1024/app.py +++ b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/function_128/app.py @@ -1,13 +1,13 @@ import os + +from aws_lambda_powertools import Logger, Tracer from aws_lambda_powertools.event_handler import APIGatewayRestResolver -from aws_lambda_powertools.utilities.typing import LambdaContext from aws_lambda_powertools.logging import correlation_paths -from aws_lambda_powertools import Logger -from aws_lambda_powertools import Tracer from aws_lambda_powertools.utilities.data_masking.base import DataMasking from aws_lambda_powertools.utilities.data_masking.providers.aws_encryption_sdk import AwsEncryptionSdkProvider +from aws_lambda_powertools.utilities.typing import LambdaContext -KMS_KEY_ARN = os.environ['KMS_KEY_ARN'] +KMS_KEY_ARN = os.environ["KMS_KEY_ARN"] json_blob = { "id": 1, @@ -43,14 +43,16 @@ tracer = Tracer() logger = Logger() -@app.get("/singleton1024") + +@app.get("/function128") @tracer.capture_method -def singleton1024(): - logger.info("Hello world singleton1024 - HTTP 200") +def function128(): + logger.info("Hello world function128 - HTTP 200") data_masker = DataMasking(provider=AwsEncryptionSdkProvider(keys=[KMS_KEY_ARN])) encrypted = data_masker.encrypt(json_blob, fields=["address.street", "job_history.company.company_name"]) decrypted = data_masker.decrypt(encrypted, fields=["address.street", "job_history.company.company_name"]) - return {"Decrypted_json_blob_singleton_1024": decrypted} + return {"Decrypted_json_blob_function_128": decrypted} + @logger.inject_lambda_context(correlation_id_path=correlation_paths.API_GATEWAY_REST) @tracer.capture_lambda_handler diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_128/requirements.txt b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/function_128/requirements.txt similarity index 100% rename from tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_128/requirements.txt rename to tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/function_128/requirements.txt diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1769/__init__.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/function_1769/__init__.py similarity index 100% rename from tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1769/__init__.py rename to tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/function_1769/__init__.py diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1769/app.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/function_1769/app.py similarity index 88% rename from tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1769/app.py rename to tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/function_1769/app.py index 6661f193564..76580e6706f 100644 --- a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1769/app.py +++ b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/function_1769/app.py @@ -1,13 +1,13 @@ import os + +from aws_lambda_powertools import Logger, Tracer from aws_lambda_powertools.event_handler import APIGatewayRestResolver -from aws_lambda_powertools.utilities.typing import LambdaContext from aws_lambda_powertools.logging import correlation_paths -from aws_lambda_powertools import Logger -from aws_lambda_powertools import Tracer from aws_lambda_powertools.utilities.data_masking.base import DataMasking from aws_lambda_powertools.utilities.data_masking.providers.aws_encryption_sdk import AwsEncryptionSdkProvider +from aws_lambda_powertools.utilities.typing import LambdaContext -KMS_KEY_ARN = os.environ['KMS_KEY_ARN'] +KMS_KEY_ARN = os.environ["KMS_KEY_ARN"] json_blob = { "id": 1, @@ -43,15 +43,15 @@ tracer = Tracer() logger = Logger() -@app.get("/singleton1769") + +@app.get("/function1769") @tracer.capture_method -def singleton1769(): - logger.info("Hello world singleton1769 - HTTP 200") +def function1769(): + logger.info("Hello world function1769 - HTTP 200") data_masker = DataMasking(provider=AwsEncryptionSdkProvider(keys=[KMS_KEY_ARN])) encrypted = data_masker.encrypt(json_blob, fields=["address.street", "job_history.company.company_name"]) decrypted = data_masker.decrypt(encrypted, fields=["address.street", "job_history.company.company_name"]) - #TODO: decrypt in another function - return {"Decrypted_json_blob_singleton_1769": decrypted} + return {"Decrypted_json_blob_function_1769": decrypted} @logger.inject_lambda_context(correlation_id_path=correlation_paths.API_GATEWAY_REST) diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1769/requirements.txt b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/function_1769/requirements.txt similarity index 100% rename from tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1769/requirements.txt rename to tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/function_1769/requirements.txt diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1024/app.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1024/app.py deleted file mode 100644 index 9cf25de0c10..00000000000 --- a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1024/app.py +++ /dev/null @@ -1,59 +0,0 @@ -import os -from aws_lambda_powertools.event_handler import APIGatewayRestResolver -from aws_lambda_powertools.utilities.typing import LambdaContext -from aws_lambda_powertools.logging import correlation_paths -from aws_lambda_powertools import Logger -from aws_lambda_powertools import Tracer -from aws_lambda_powertools.utilities.data_masking.base import DataMasking -from aws_lambda_powertools.utilities.data_masking.providers.aws_encryption_sdk import AwsEncryptionSdkProviderWithoutSingleton - -KMS_KEY_ARN = os.environ['KMS_KEY_ARN'] - -json_blob = { - "id": 1, - "name": "John Doe", - "age": 30, - "email": "johndoe@example.com", - "address": {"street": "123 Main St", "city": "Anytown", "state": "CA", "zip": "12345"}, - "phone_numbers": ["+1-555-555-1234", "+1-555-555-5678"], - "interests": ["Hiking", "Traveling", "Photography", "Reading"], - "job_history": { - "company": { - "company_name": "Acme Inc.", - "company_address": "5678 Interview Dr.", - }, - "position": "Software Engineer", - "start_date": "2015-01-01", - "end_date": "2017-12-31", - }, - "about_me": """ - Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla tincidunt velit quis - sapien mollis, at egestas massa tincidunt. Suspendisse ultrices arcu a dolor dapibus, - ut pretium turpis volutpat. Vestibulum at sapien quis sapien dignissim volutpat ut a enim. - Praesent fringilla sem eu dui convallis luctus. Donec ullamcorper, sapien ut convallis congue, - risus mauris pretium tortor, nec dignissim arcu urna a nisl. Vivamus non fermentum ex. Proin - interdum nisi id sagittis egestas. Nam sit amet nisi nec quam pharetra sagittis. Aliquam erat - volutpat. Donec nec luctus sem, nec ornare lorem. Vivamus vitae orci quis enim faucibus placerat. - Nulla facilisi. Proin in turpis orci. Donec imperdiet velit ac tellus gravida, eget laoreet tellus - malesuada. Praesent venenatis tellus ac urna blandit, at varius felis posuere. Integer a commodo nunc. - """, -} - -app = APIGatewayRestResolver() -tracer = Tracer() -logger = Logger() - -@app.get("/nonsingleton1024") -@tracer.capture_method -def nonsingleton1024(): - logger.info("Hello world non singleton1024 - HTTP 200") - data_masker = DataMasking(provider=AwsEncryptionSdkProviderWithoutSingleton(keys=[KMS_KEY_ARN])) - encrypted = data_masker.encrypt(json_blob, fields=["address.street", "job_history.company.company_name"]) - decrypted = data_masker.decrypt(encrypted, fields=["address.street", "job_history.company.company_name"]) - #TODO: decrypt in another function - return {"Decrypted_json_blob_non_singleton_1024": decrypted} - -@logger.inject_lambda_context(correlation_id_path=correlation_paths.API_GATEWAY_REST) -@tracer.capture_lambda_handler -def lambda_handler(event: dict, context: LambdaContext) -> dict: - return app.resolve(event, context) diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_128/app.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_128/app.py deleted file mode 100644 index cb785ea38a0..00000000000 --- a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_128/app.py +++ /dev/null @@ -1,59 +0,0 @@ -import os -from aws_lambda_powertools.event_handler import APIGatewayRestResolver -from aws_lambda_powertools.utilities.typing import LambdaContext -from aws_lambda_powertools.logging import correlation_paths -from aws_lambda_powertools import Logger -from aws_lambda_powertools import Tracer -from aws_lambda_powertools.utilities.data_masking.base import DataMasking -from aws_lambda_powertools.utilities.data_masking.providers.aws_encryption_sdk import AwsEncryptionSdkProviderWithoutSingleton - -KMS_KEY_ARN = os.environ['KMS_KEY_ARN'] - -json_blob = { - "id": 1, - "name": "John Doe", - "age": 30, - "email": "johndoe@example.com", - "address": {"street": "123 Main St", "city": "Anytown", "state": "CA", "zip": "12345"}, - "phone_numbers": ["+1-555-555-1234", "+1-555-555-5678"], - "interests": ["Hiking", "Traveling", "Photography", "Reading"], - "job_history": { - "company": { - "company_name": "Acme Inc.", - "company_address": "5678 Interview Dr.", - }, - "position": "Software Engineer", - "start_date": "2015-01-01", - "end_date": "2017-12-31", - }, - "about_me": """ - Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla tincidunt velit quis - sapien mollis, at egestas massa tincidunt. Suspendisse ultrices arcu a dolor dapibus, - ut pretium turpis volutpat. Vestibulum at sapien quis sapien dignissim volutpat ut a enim. - Praesent fringilla sem eu dui convallis luctus. Donec ullamcorper, sapien ut convallis congue, - risus mauris pretium tortor, nec dignissim arcu urna a nisl. Vivamus non fermentum ex. Proin - interdum nisi id sagittis egestas. Nam sit amet nisi nec quam pharetra sagittis. Aliquam erat - volutpat. Donec nec luctus sem, nec ornare lorem. Vivamus vitae orci quis enim faucibus placerat. - Nulla facilisi. Proin in turpis orci. Donec imperdiet velit ac tellus gravida, eget laoreet tellus - malesuada. Praesent venenatis tellus ac urna blandit, at varius felis posuere. Integer a commodo nunc. - """, -} - -app = APIGatewayRestResolver() -tracer = Tracer() -logger = Logger() - -@app.get("/nonsingleton128") -@tracer.capture_method -def nonsingleton128(): - logger.info("Hello world non singleton128 - HTTP 200") - data_masker = DataMasking(provider=AwsEncryptionSdkProviderWithoutSingleton(keys=[KMS_KEY_ARN])) - encrypted = data_masker.encrypt(json_blob, fields=["address.street", "job_history.company.company_name"]) - decrypted = data_masker.decrypt(encrypted, fields=["address.street", "job_history.company.company_name"]) - #TODO: decrypt in another function - return {"Decrypted_json_blob_non_singleton_128": decrypted} - -@logger.inject_lambda_context(correlation_id_path=correlation_paths.API_GATEWAY_REST) -@tracer.capture_lambda_handler -def lambda_handler(event: dict, context: LambdaContext) -> dict: - return app.resolve(event, context) diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1769/app.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1769/app.py deleted file mode 100644 index 16cb9ab4b91..00000000000 --- a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/nonsingleton_1769/app.py +++ /dev/null @@ -1,60 +0,0 @@ -import os -from aws_lambda_powertools.event_handler import APIGatewayRestResolver -from aws_lambda_powertools.utilities.typing import LambdaContext -from aws_lambda_powertools.logging import correlation_paths -from aws_lambda_powertools import Logger -from aws_lambda_powertools import Tracer -from aws_lambda_powertools.utilities.data_masking.base import DataMasking -from aws_lambda_powertools.utilities.data_masking.providers.aws_encryption_sdk import AwsEncryptionSdkProviderWithoutSingleton - -KMS_KEY_ARN = os.environ['KMS_KEY_ARN'] - -json_blob = { - "id": 1, - "name": "John Doe", - "age": 30, - "email": "johndoe@example.com", - "address": {"street": "123 Main St", "city": "Anytown", "state": "CA", "zip": "12345"}, - "phone_numbers": ["+1-555-555-1234", "+1-555-555-5678"], - "interests": ["Hiking", "Traveling", "Photography", "Reading"], - "job_history": { - "company": { - "company_name": "Acme Inc.", - "company_address": "5678 Interview Dr.", - }, - "position": "Software Engineer", - "start_date": "2015-01-01", - "end_date": "2017-12-31", - }, - "about_me": """ - Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla tincidunt velit quis - sapien mollis, at egestas massa tincidunt. Suspendisse ultrices arcu a dolor dapibus, - ut pretium turpis volutpat. Vestibulum at sapien quis sapien dignissim volutpat ut a enim. - Praesent fringilla sem eu dui convallis luctus. Donec ullamcorper, sapien ut convallis congue, - risus mauris pretium tortor, nec dignissim arcu urna a nisl. Vivamus non fermentum ex. Proin - interdum nisi id sagittis egestas. Nam sit amet nisi nec quam pharetra sagittis. Aliquam erat - volutpat. Donec nec luctus sem, nec ornare lorem. Vivamus vitae orci quis enim faucibus placerat. - Nulla facilisi. Proin in turpis orci. Donec imperdiet velit ac tellus gravida, eget laoreet tellus - malesuada. Praesent venenatis tellus ac urna blandit, at varius felis posuere. Integer a commodo nunc. - """, -} - -app = APIGatewayRestResolver() -tracer = Tracer() -logger = Logger() - -@app.get("/nonsingleton1769") -@tracer.capture_method -def nonsingleton1769(): - logger.info("Hello world non singleton1769 - HTTP 200") - data_masker = DataMasking(provider=AwsEncryptionSdkProviderWithoutSingleton(keys=[KMS_KEY_ARN])) - encrypted = data_masker.encrypt(json_blob, fields=["address.street", "job_history.company.company_name"]) - decrypted = data_masker.decrypt(encrypted, fields=["address.street", "job_history.company.company_name"]) - #TODO: decrypt in another function - return {"Decrypted_json_blob_non_singleton_1769": decrypted} - - -@logger.inject_lambda_context(correlation_id_path=correlation_paths.API_GATEWAY_REST) -@tracer.capture_lambda_handler -def lambda_handler(event: dict, context: LambdaContext) -> dict: - return app.resolve(event, context) diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1024/__init__.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1024/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1024/requirements.txt b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1024/requirements.txt deleted file mode 100644 index b74b60fc263..00000000000 --- a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1024/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -requests -aws-lambda-powertools[tracer] -aws-encryption-sdk diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_128/__init__.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_128/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_128/requirements.txt b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_128/requirements.txt deleted file mode 100644 index b74b60fc263..00000000000 --- a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_128/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -requests -aws-lambda-powertools[tracer] -aws-encryption-sdk diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1769/__init__.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1769/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1769/requirements.txt b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1769/requirements.txt deleted file mode 100644 index b74b60fc263..00000000000 --- a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/singleton_1769/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -requests -aws-lambda-powertools[tracer] -aws-encryption-sdk diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/template.yaml b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/template.yaml index 27b0f573908..f2a6540c267 100644 --- a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/template.yaml +++ b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/template.yaml @@ -26,12 +26,12 @@ Resources: Resource: "*" Principal: AWS: !Join [ "", [ "arn:aws:iam::", !Ref "AWS::AccountId", ":root" ] ] - Function128Singleton: + Function128: Type: AWS::Serverless::Function # More info about Function Resource: https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-resource-function.html Properties: Handler: app.lambda_handler - CodeUri: singleton_128 - Description: function 128 MB Singleton + CodeUri: function_128 + Description: function 128 MB MemorySize: 128 Architectures: - x86_64 @@ -45,7 +45,7 @@ Resources: HelloPath: Type: Api # More info about API Event Source: https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-property-function-api.html Properties: - Path: /singleton128 + Path: /function128 Method: GET # Powertools for AWS Lambda (Python) env vars: https://awslabs.github.io/aws-lambda-powertools-python/#environment-variables Environment: @@ -56,105 +56,12 @@ Resources: KMS_KEY_ARN: !GetAtt MyKMSKey.Arn Tags: LambdaPowertools: python - - Function1024Singleton: - Type: AWS::Serverless::Function # More info about Function Resource: https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-resource-function.html - Properties: - Handler: app.lambda_handler - CodeUri: singleton_1024 - Description: function 1024 MB Singleton - MemorySize: 1024 - Architectures: - - x86_64 - Policies: - Statement: - - Effect: Allow - Action: kms:* - Resource: !GetAtt MyKMSKey.Arn - Tracing: Active - Events: - HelloPath: - Type: Api # More info about API Event Source: https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-property-function-api.html - Properties: - Path: /singleton1024 - Method: GET - # Powertools for AWS Lambda (Python) env vars: https://awslabs.github.io/aws-lambda-powertools-python/#environment-variables - Environment: - Variables: - POWERTOOLS_SERVICE_NAME: PowertoolsHelloWorld - POWERTOOLS_METRICS_NAMESPACE: Powertools - LOG_LEVEL: INFO - KMS_KEY_ARN: !GetAtt MyKMSKey.Arn - Tags: - LambdaPowertools: python - Function1769Singleton: - Type: AWS::Serverless::Function # More info about Function Resource: https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-resource-function.html - Properties: - Handler: app.lambda_handler - CodeUri: singleton_1769 - Description: function 1769 MB Singleton - MemorySize: 1769 - Architectures: - - x86_64 - Policies: - Statement: - - Effect: Allow - Action: kms:* - Resource: !GetAtt MyKMSKey.Arn - Tracing: Active - Events: - HelloPath: - Type: Api # More info about API Event Source: https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-property-function-api.html - Properties: - Path: /singleton1769 - Method: GET - # Powertools for AWS Lambda (Python) env vars: https://awslabs.github.io/aws-lambda-powertools-python/#environment-variables - Environment: - Variables: - POWERTOOLS_SERVICE_NAME: PowertoolsHelloWorld - POWERTOOLS_METRICS_NAMESPACE: Powertools - LOG_LEVEL: INFO - KMS_KEY_ARN: !GetAtt MyKMSKey.Arn - Tags: - LambdaPowertools: python - - Function128NonSingleton: - Type: AWS::Serverless::Function # More info about Function Resource: https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-resource-function.html - Properties: - Handler: app.lambda_handler - CodeUri: nonsingleton_128 - Description: function 128 MB Non Singleton - MemorySize: 128 - Architectures: - - x86_64 - Policies: - Statement: - - Effect: Allow - Action: kms:* - Resource: !GetAtt MyKMSKey.Arn - Tracing: Active - Events: - HelloPath: - Type: Api # More info about API Event Source: https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-property-function-api.html - Properties: - Path: /nonsingleton128 - Method: GET - # Powertools for AWS Lambda (Python) env vars: https://awslabs.github.io/aws-lambda-powertools-python/#environment-variables - Environment: - Variables: - POWERTOOLS_SERVICE_NAME: PowertoolsHelloWorld - POWERTOOLS_METRICS_NAMESPACE: Powertools - LOG_LEVEL: INFO - KMS_KEY_ARN: !GetAtt MyKMSKey.Arn - Tags: - LambdaPowertools: python - - Function1024NonSingleton: + Function1024: Type: AWS::Serverless::Function # More info about Function Resource: https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-resource-function.html Properties: Handler: app.lambda_handler - CodeUri: nonsingleton_1024 - Description: function 1024 MB Non Singleton + CodeUri: function_1024 + Description: function 1024 MB MemorySize: 1024 Architectures: - x86_64 @@ -168,7 +75,7 @@ Resources: HelloPath: Type: Api # More info about API Event Source: https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-property-function-api.html Properties: - Path: /nonsingleton1024 + Path: /function1024 Method: GET # Powertools for AWS Lambda (Python) env vars: https://awslabs.github.io/aws-lambda-powertools-python/#environment-variables Environment: @@ -179,13 +86,12 @@ Resources: KMS_KEY_ARN: !GetAtt MyKMSKey.Arn Tags: LambdaPowertools: python - - Function1769NonSingleton: + Function1769: Type: AWS::Serverless::Function # More info about Function Resource: https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-resource-function.html Properties: Handler: app.lambda_handler - CodeUri: nonsingleton_1769 - Description: function 1769 MB Non Singleton + CodeUri: function_1769 + Description: function 1769 MB MemorySize: 1769 Architectures: - x86_64 @@ -199,7 +105,7 @@ Resources: HelloPath: Type: Api # More info about API Event Source: https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-property-function-api.html Properties: - Path: /nonsingleton1769 + Path: /function1769 Method: GET # Powertools for AWS Lambda (Python) env vars: https://awslabs.github.io/aws-lambda-powertools-python/#environment-variables Environment: @@ -216,51 +122,26 @@ Outputs: Description: ARN of the KMS Key Value: !GetAtt MyKMSKey.Arn - 128SingletonApi: - Description: API Gateway endpoint URL for Prod environment for Function 128 MB Singleton - Value: !Sub "https://${ServerlessRestApi}.execute-api.${AWS::Region}.amazonaws.com/Prod/singleton128" - - 1024SingletonApi: - Description: API Gateway endpoint URL for Prod environment for Function 1024 MB Singleton - Value: !Sub "https://${ServerlessRestApi}.execute-api.${AWS::Region}.amazonaws.com/Prod/singleton1024" - - 1769SingletonApi: - Description: API Gateway endpoint URL for Prod environment for Function 1769 MB Singleton - Value: !Sub "https://${ServerlessRestApi}.execute-api.${AWS::Region}.amazonaws.com/Prod/singleton1769" - - Function128Singleton: - Description: Lambda Function 128 MB Singleton ARN - Value: !GetAtt Function128Singleton.Arn - - Function1024Singleton: - Description: Lambda Function 1024 MB Singleton ARN - Value: !GetAtt Function1024Singleton.Arn - - Function1769Singleton: - Description: Lambda Function 1769 MB Singleton ARN - Value: !GetAtt Function1769Singleton.Arn - - 128NonSingletonApi: - Description: API Gateway endpoint URL for Prod environment for Function 128 MB Non Singleton - Value: !Sub "https://${ServerlessRestApi}.execute-api.${AWS::Region}.amazonaws.com/Prod/nonsingleton128" - - 1024NonSingletonApi: - Description: API Gateway endpoint URL for Prod environment for Function 1024 MB Non Singleton - Value: !Sub "https://${ServerlessRestApi}.execute-api.${AWS::Region}.amazonaws.com/Prod/nonsingleton1024" + 128FunctionApi: + Description: API Gateway endpoint URL for Prod environment for Function 128 MB + Value: !Sub "https://${ServerlessRestApi}.execute-api.${AWS::Region}.amazonaws.com/Prod/function128" - 1769NonSingletonApi: - Description: API Gateway endpoint URL for Prod environment for Function 1769 MB Non Singleton - Value: !Sub "https://${ServerlessRestApi}.execute-api.${AWS::Region}.amazonaws.com/Prod/nonsingleton1769" + 1024FunctionApi: + Description: API Gateway endpoint URL for Prod environment for Function 1024 MB + Value: !Sub "https://${ServerlessRestApi}.execute-api.${AWS::Region}.amazonaws.com/Prod/function1024" - Function128NonSingleton: - Description: Lambda Function 128 MB Non Singleton ARN - Value: !GetAtt Function128NonSingleton.Arn + 1769FunctionApi: + Description: API Gateway endpoint URL for Prod environment for Function 1769 MB + Value: !Sub "https://${ServerlessRestApi}.execute-api.${AWS::Region}.amazonaws.com/Prod/function1769" - Function1024NonSingleton: - Description: Lambda Function 1024 MB Non Singleton ARN - Value: !GetAtt Function1024NonSingleton.Arn + Function128: + Description: Lambda Function 128 MB ARN + Value: !GetAtt Function128.Arn - Function1769NonSingleton: - Description: Lambda Function 1769 MB Non Singleton ARN - Value: !GetAtt Function1769NonSingleton.Arn + Function1024: + Description: Lambda Function 1024 MB ARN + Value: !GetAtt Function1024.Arn + Function1769: + Description: Lambda Function 1769 MB ARN + Value: !GetAtt Function1769.Arn diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/tests/__init__.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/tests/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/tests/requirements.txt b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/tests/requirements.txt deleted file mode 100644 index b9cf27ab27a..00000000000 --- a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/tests/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -pytest -boto3 -requests diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/tests/unit/__init__.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/tests/unit/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/tests/unit/test_handler.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/tests/unit/test_handler.py deleted file mode 100644 index d192c1b1b34..00000000000 --- a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/tests/unit/test_handler.py +++ /dev/null @@ -1,145 +0,0 @@ -import json - -import pytest - -from hello_world import app - -def lambda_context(): - class LambdaContext: - def __init__(self): - self.function_name = "test-func" - self.memory_limit_in_mb = 128 - self.invoked_function_arn = "arn:aws:lambda:eu-west-1:809313241234:function:test-func" - self.aws_request_id = "52fdfc07-2182-154f-163f-5f0f9a621d72" - - def get_remaining_time_in_millis(self) -> int: - return 1000 - - return LambdaContext() - - -@pytest.fixture() -def apigw_event(): - """ Generates API GW Event""" - - return { - "body":"", - "headers":{ - "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", - "Accept-Encoding":"gzip, deflate, br", - "Accept-Language":"pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7", - "Cache-Control":"max-age=0", - "Connection":"keep-alive", - "Host":"127.0.0.1:3000", - "Sec-Ch-Ua":"\"Google Chrome\";v=\"105\", \"Not)A;Brand\";v=\"8\", \"Chromium\";v=\"105\"", - "Sec-Ch-Ua-Mobile":"?0", - "Sec-Ch-Ua-Platform":"\"Linux\"", - "Sec-Fetch-Dest":"document", - "Sec-Fetch-Mode":"navigate", - "Sec-Fetch-Site":"none", - "Sec-Fetch-User":"?1", - "Upgrade-Insecure-Requests":"1", - "User-Agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36", - "X-Forwarded-Port":"3000", - "X-Forwarded-Proto":"http" - }, - "httpMethod":"GET", - "isBase64Encoded":False, - "multiValueHeaders":{ - "Accept":[ - "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" - ], - "Accept-Encoding":[ - "gzip, deflate, br" - ], - "Accept-Language":[ - "pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7" - ], - "Cache-Control":[ - "max-age=0" - ], - "Connection":[ - "keep-alive" - ], - "Host":[ - "127.0.0.1:3000" - ], - "Sec-Ch-Ua":[ - "\"Google Chrome\";v=\"105\", \"Not)A;Brand\";v=\"8\", \"Chromium\";v=\"105\"" - ], - "Sec-Ch-Ua-Mobile":[ - "?0" - ], - "Sec-Ch-Ua-Platform":[ - "\"Linux\"" - ], - "Sec-Fetch-Dest":[ - "document" - ], - "Sec-Fetch-Mode":[ - "navigate" - ], - "Sec-Fetch-Site":[ - "none" - ], - "Sec-Fetch-User":[ - "?1" - ], - "Upgrade-Insecure-Requests":[ - "1" - ], - "User-Agent":[ - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36" - ], - "X-Forwarded-Port":[ - "3000" - ], - "X-Forwarded-Proto":[ - "http" - ] - }, - "multiValueQueryStringParameters":"", - "path":"/hello", - "pathParameters":"", - "queryStringParameters":"", - "requestContext":{ - "accountId":"123456789012", - "apiId":"1234567890", - "domainName":"127.0.0.1:3000", - "extendedRequestId":"", - "httpMethod":"GET", - "identity":{ - "accountId":"", - "apiKey":"", - "caller":"", - "cognitoAuthenticationProvider":"", - "cognitoAuthenticationType":"", - "cognitoIdentityPoolId":"", - "sourceIp":"127.0.0.1", - "user":"", - "userAgent":"Custom User Agent String", - "userArn":"" - }, - "path":"/hello", - "protocol":"HTTP/1.1", - "requestId":"a3590457-cac2-4f10-8fc9-e47114bf7c62", - "requestTime":"02/Feb/2023:11:45:26 +0000", - "requestTimeEpoch":1675338326, - "resourceId":"123456", - "resourcePath":"/hello", - "stage":"Prod" - }, - "resource":"/hello", - "stageVariables":"", - "version":"1.0" -} - - -def test_lambda_handler(apigw_event): - - ret = app.lambda_handler(apigw_event, lambda_context()) - data = json.loads(ret["body"]) - - assert ret["statusCode"] == 200 - assert "message" in ret["body"] - assert data["message"] == "hello world" From 069aa9417dc95711057e809fdf2ee5cfc5d5a531 Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Tue, 26 Sep 2023 13:13:47 -0700 Subject: [PATCH 48/56] Fix linting errors --- aws_lambda_powertools/utilities/data_masking/base.py | 2 +- aws_lambda_powertools/utilities/data_masking/provider.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_masking/base.py b/aws_lambda_powertools/utilities/data_masking/base.py index 9019eff3b1d..a7f2f820761 100644 --- a/aws_lambda_powertools/utilities/data_masking/base.py +++ b/aws_lambda_powertools/utilities/data_masking/base.py @@ -109,7 +109,7 @@ def _apply_action_to_fields( f"Unsupported data type for 'data' parameter. Expected a traversable type, but got {type(data)}.", ) - # ['a.b.c'] in ['a.b.c', 'a.x.y'] + # For example: ['a.b.c'] in ['a.b.c', 'a.x.y'] for nested_key in fields: # Prevent overriding loop variable curr_nested_key = nested_key diff --git a/aws_lambda_powertools/utilities/data_masking/provider.py b/aws_lambda_powertools/utilities/data_masking/provider.py index a4515e47183..3032a40f150 100644 --- a/aws_lambda_powertools/utilities/data_masking/provider.py +++ b/aws_lambda_powertools/utilities/data_masking/provider.py @@ -1,12 +1,11 @@ import json -from abc import ABCMeta from collections.abc import Iterable from typing import Union from aws_lambda_powertools.utilities.data_masking.constants import DATA_MASKING_STRING -class BaseProvider(metaclass=ABCMeta): +class BaseProvider: """ When you try to create an instance of a subclass that does not implement the encrypt method, you will get a NotImplementedError with a message that says the method is not implemented: From ee325f44c759f46823e60c066dbceab4952478ef Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Tue, 26 Sep 2023 13:43:00 -0700 Subject: [PATCH 49/56] Fix mypy errors --- aws_lambda_powertools/utilities/data_masking/provider.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_masking/provider.py b/aws_lambda_powertools/utilities/data_masking/provider.py index 3032a40f150..f1c3bb869f9 100644 --- a/aws_lambda_powertools/utilities/data_masking/provider.py +++ b/aws_lambda_powertools/utilities/data_masking/provider.py @@ -1,6 +1,5 @@ import json -from collections.abc import Iterable -from typing import Union +from typing import Any, Union from aws_lambda_powertools.utilities.data_masking.constants import DATA_MASKING_STRING @@ -24,12 +23,12 @@ def default_json_deserializer(self, data): def encrypt(self, data) -> Union[bytes, str]: raise NotImplementedError("Subclasses must implement encrypt()") - def decrypt(self, data) -> any: + def decrypt(self, data) -> Any: raise NotImplementedError("Subclasses must implement decrypt()") - def mask(self, data) -> str: + def mask(self, data) -> Any: if isinstance(data, (str, dict, bytes)): return DATA_MASKING_STRING - elif isinstance(data, Iterable): + elif isinstance(data, (list, tuple, set)): return type(data)([DATA_MASKING_STRING] * len(data)) return DATA_MASKING_STRING From 49afeedf60a633d035b61f047c0f173a9822bd15 Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Tue, 26 Sep 2023 13:58:38 -0700 Subject: [PATCH 50/56] Modified data masking test names --- .../{test_data_masking.py => test_e2e_data_masking.py} | 0 .../{test_data_masking.py => test_perf_data_masking.py} | 0 .../{test_data_masking.py => test_unit_data_masking.py} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename tests/e2e/data_masking/{test_data_masking.py => test_e2e_data_masking.py} (100%) rename tests/performance/data_masking/{test_data_masking.py => test_perf_data_masking.py} (100%) rename tests/unit/data_masking/{test_data_masking.py => test_unit_data_masking.py} (100%) diff --git a/tests/e2e/data_masking/test_data_masking.py b/tests/e2e/data_masking/test_e2e_data_masking.py similarity index 100% rename from tests/e2e/data_masking/test_data_masking.py rename to tests/e2e/data_masking/test_e2e_data_masking.py diff --git a/tests/performance/data_masking/test_data_masking.py b/tests/performance/data_masking/test_perf_data_masking.py similarity index 100% rename from tests/performance/data_masking/test_data_masking.py rename to tests/performance/data_masking/test_perf_data_masking.py diff --git a/tests/unit/data_masking/test_data_masking.py b/tests/unit/data_masking/test_unit_data_masking.py similarity index 100% rename from tests/unit/data_masking/test_data_masking.py rename to tests/unit/data_masking/test_unit_data_masking.py From 73df808d1cea295db617c0093f1bd065f9cfb683 Mon Sep 17 00:00:00 2001 From: Seshu Brahma Date: Tue, 26 Sep 2023 14:15:27 -0700 Subject: [PATCH 51/56] Fix dummy KMS key for correct parsing --- tests/functional/data_masking/test_aws_encryption_sdk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/data_masking/test_aws_encryption_sdk.py b/tests/functional/data_masking/test_aws_encryption_sdk.py index 27771e6f4e3..e37c43cec90 100644 --- a/tests/functional/data_masking/test_aws_encryption_sdk.py +++ b/tests/functional/data_masking/test_aws_encryption_sdk.py @@ -16,7 +16,7 @@ def data_masker() -> DataMasking: """DataMasking using AWS Encryption SDK Provider with a fake client""" fake_client = FakeEncryptionClient() - provider = AwsEncryptionSdkProvider(keys=["dummy"], client=fake_client) + provider = AwsEncryptionSdkProvider(keys=["arn:aws:kms:us-east-1:0123456789012:key/dummy"], client=fake_client) return DataMasking(provider=provider) From 1ea59f039a76e14faf1cb2f12a5432fdef66d4e7 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Tue, 26 Sep 2023 22:29:01 +0100 Subject: [PATCH 52/56] Bumping cryptography library --- poetry.lock | 185 ++++++++++++++++++++++++---------------------------- 1 file changed, 84 insertions(+), 101 deletions(-) diff --git a/poetry.lock b/poetry.lock index b4e38335f41..46d0d7bef43 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "anyio" @@ -93,17 +93,17 @@ typeguard = ">=2.13.3,<2.14.0" [[package]] name = "aws-cdk-aws-apigatewayv2-alpha" -version = "2.95.0a0" +version = "2.98.0a0" description = "The CDK Construct Library for AWS::APIGatewayv2" optional = false python-versions = "~=3.7" files = [ - {file = "aws-cdk.aws-apigatewayv2-alpha-2.95.0a0.tar.gz", hash = "sha256:ae6eb5fe0a06910a6aa0ac2fcc708585dba9cb288991811c8dd65520f121169b"}, - {file = "aws_cdk.aws_apigatewayv2_alpha-2.95.0a0-py3-none-any.whl", hash = "sha256:fab88cb71c2a3bec81851981c004c915114521944364b80f8a2e63cbaa79cbe1"}, + {file = "aws-cdk.aws-apigatewayv2-alpha-2.98.0a0.tar.gz", hash = "sha256:c2786ad0c2f409a7215ad4d923f2a36977aad9d109910352dacbf2082857ca51"}, + {file = "aws_cdk.aws_apigatewayv2_alpha-2.98.0a0-py3-none-any.whl", hash = "sha256:c460b7928c82997e666eefb7344f2c536d9b985a7892d5d191725c606c2f0e28"}, ] [package.dependencies] -aws-cdk-lib = "2.95.0" +aws-cdk-lib = "2.98.0" constructs = ">=10.0.0,<11.0.0" jsii = ">=1.88.0,<2.0.0" publication = ">=0.0.3" @@ -111,18 +111,18 @@ typeguard = ">=2.13.3,<2.14.0" [[package]] name = "aws-cdk-aws-apigatewayv2-authorizers-alpha" -version = "2.95.0a0" +version = "2.98.0a0" description = "Authorizers for AWS APIGateway V2" optional = false python-versions = "~=3.7" files = [ - {file = "aws-cdk.aws-apigatewayv2-authorizers-alpha-2.95.0a0.tar.gz", hash = "sha256:416b73ed01611d6a44578395b9236c1cba1f0878e88c0f6a84d0f43f00a7d15c"}, - {file = "aws_cdk.aws_apigatewayv2_authorizers_alpha-2.95.0a0-py3-none-any.whl", hash = "sha256:1ec30cf64211d8ee39dd6c51604faa01a448fe9a5611bd315172373dd6f18e85"}, + {file = "aws-cdk.aws-apigatewayv2-authorizers-alpha-2.98.0a0.tar.gz", hash = "sha256:06a2013641bb053acd803f57e6efbf22ccb43ff71a9f8b93bb3a3b5b065007eb"}, + {file = "aws_cdk.aws_apigatewayv2_authorizers_alpha-2.98.0a0-py3-none-any.whl", hash = "sha256:d2c3186f023d564f5081f894d75eaffbe2181098c32412e38b75821ea817ca1a"}, ] [package.dependencies] -"aws-cdk.aws-apigatewayv2-alpha" = "2.95.0.a0" -aws-cdk-lib = "2.95.0" +"aws-cdk.aws-apigatewayv2-alpha" = "2.98.0.a0" +aws-cdk-lib = "2.98.0" constructs = ">=10.0.0,<11.0.0" jsii = ">=1.88.0,<2.0.0" publication = ">=0.0.3" @@ -130,18 +130,18 @@ typeguard = ">=2.13.3,<2.14.0" [[package]] name = "aws-cdk-aws-apigatewayv2-integrations-alpha" -version = "2.95.0a0" +version = "2.98.0a0" description = "Integrations for AWS APIGateway V2" optional = false python-versions = "~=3.7" files = [ - {file = "aws-cdk.aws-apigatewayv2-integrations-alpha-2.95.0a0.tar.gz", hash = "sha256:59f41a6e4002687d20f01bcbc4249887f8b29d3dc27255976a0348e5a43cf158"}, - {file = "aws_cdk.aws_apigatewayv2_integrations_alpha-2.95.0a0-py3-none-any.whl", hash = "sha256:cc2236b2d901e2907ac523739e0d59be7dbf8c187801d3ef52ae2cb305402119"}, + {file = "aws-cdk.aws-apigatewayv2-integrations-alpha-2.98.0a0.tar.gz", hash = "sha256:e7f70c663fbf2140737f6f97cf7611a30494ad011c6988020b126349a88b3185"}, + {file = "aws_cdk.aws_apigatewayv2_integrations_alpha-2.98.0a0-py3-none-any.whl", hash = "sha256:4175b0fba4436e52e5e1b08527ef5097a41dd1c59960c3dcf3514752bbc214eb"}, ] [package.dependencies] -"aws-cdk.aws-apigatewayv2-alpha" = "2.95.0.a0" -aws-cdk-lib = "2.95.0" +"aws-cdk.aws-apigatewayv2-alpha" = "2.98.0.a0" +aws-cdk-lib = "2.98.0" constructs = ">=10.0.0,<11.0.0" jsii = ">=1.88.0,<2.0.0" publication = ">=0.0.3" @@ -149,13 +149,13 @@ typeguard = ">=2.13.3,<2.14.0" [[package]] name = "aws-cdk-lib" -version = "2.95.0" +version = "2.98.0" description = "Version 2 of the AWS Cloud Development Kit library" optional = false python-versions = "~=3.7" files = [ - {file = "aws-cdk-lib-2.95.0.tar.gz", hash = "sha256:db87e5dc5cc2d7ea709f8618aa90fc815d43e750dc2e807b0a10801522d7a40b"}, - {file = "aws_cdk_lib-2.95.0-py3-none-any.whl", hash = "sha256:10c276e889eb7d036cd34be9076cc5f92e0960763b14dc79f2600caf5af13c4b"}, + {file = "aws-cdk-lib-2.98.0.tar.gz", hash = "sha256:4a1d13a9bd2b7ec0efa41d3d70291ff21cac02e9d3028253c08c53261d49bfa2"}, + {file = "aws_cdk_lib-2.98.0-py3-none-any.whl", hash = "sha256:8e5ae2ecd74e249e6cfdc0e53a1a71be516d2408d2bcd96c49c1718f51cdb16b"}, ] [package.dependencies] @@ -200,23 +200,23 @@ requests = ">=0.14.0" [[package]] name = "aws-sam-translator" -version = "1.74.0" +version = "1.75.0" description = "AWS SAM Translator is a library that transform SAM templates into AWS CloudFormation templates" optional = false python-versions = ">=3.7, <=4.0, !=4.0" files = [ - {file = "aws-sam-translator-1.74.0.tar.gz", hash = "sha256:6975ddf0798f45952554d6ea2d5e95de8f321ecb7d3299bb127073d4ffb53c90"}, - {file = "aws_sam_translator-1.74.0-py3-none-any.whl", hash = "sha256:74b386ed3780786a8b08d122c77c462f1872db4049617b63c497953b51008bf0"}, + {file = "aws-sam-translator-1.75.0.tar.gz", hash = "sha256:18c83abcae594de084947befb9c80f689f8b99ece2d38729d27a9cea634da15c"}, + {file = "aws_sam_translator-1.75.0-py3-none-any.whl", hash = "sha256:02bad7636356438b439c8e0ef0195618e3b7b67b6dfbf675b1627d6fd84b2910"}, ] [package.dependencies] boto3 = ">=1.19.5,<2.dev0" jsonschema = ">=3.2,<5" -pydantic = ">=1.8,<2.0" +pydantic = ">=1.8,<3" typing-extensions = ">=4.4,<5" [package.extras] -dev = ["black (==23.1.0)", "boto3 (>=1.23,<2)", "boto3-stubs[appconfig,serverlessrepo] (>=1.19.5,<2.dev0)", "coverage (>=5.3,<8)", "dateparser (>=1.1,<2.0)", "importlib-metadata", "mypy (>=1.1.0,<1.2.0)", "parameterized (>=0.7,<1.0)", "pytest (>=6.2,<8)", "pytest-cov (>=2.10,<5)", "pytest-env (>=0.6,<1)", "pytest-rerunfailures (>=9.1,<12)", "pytest-xdist (>=2.5,<4)", "pyyaml (>=6.0,<7.0)", "requests (>=2.28,<3.0)", "ruamel.yaml (==0.17.21)", "ruff (==0.0.263)", "tenacity (>=8.0,<9.0)", "types-PyYAML (>=6.0,<7.0)", "types-jsonschema (>=3.2,<4.0)"] +dev = ["black (==23.3.0)", "boto3 (>=1.23,<2)", "boto3-stubs[appconfig,serverlessrepo] (>=1.19.5,<2.dev0)", "coverage (>=5.3,<8)", "dateparser (>=1.1,<2.0)", "importlib-metadata", "mypy (>=1.3.0,<1.4.0)", "parameterized (>=0.7,<1.0)", "pytest (>=6.2,<8)", "pytest-cov (>=2.10,<5)", "pytest-env (>=0.6,<1)", "pytest-rerunfailures (>=9.1,<12)", "pytest-xdist (>=2.5,<4)", "pyyaml (>=6.0,<7.0)", "requests (>=2.28,<3.0)", "ruamel.yaml (==0.17.21)", "ruff (==0.0.284)", "tenacity (>=8.0,<9.0)", "types-PyYAML (>=6.0,<7.0)", "types-jsonschema (>=3.2,<4.0)"] [[package]] name = "aws-xray-sdk" @@ -322,32 +322,32 @@ uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "boto3" -version = "1.28.43" +version = "1.28.55" description = "The AWS SDK for Python" optional = false python-versions = ">= 3.7" files = [ - {file = "boto3-1.28.43-py3-none-any.whl", hash = "sha256:4cd3e96900fb50bddc9f48007176c80d15396d08c5248b25a41220f3570e014f"}, - {file = "boto3-1.28.43.tar.gz", hash = "sha256:c0211a3e830432851c73fa1e136b14dbb6d02b5c9a5e1272c557e63538620b88"}, + {file = "boto3-1.28.55-py3-none-any.whl", hash = "sha256:2680c0e36167e672777110ccef5303d59fa4a6a4f10086f9c14158c5cb008d5c"}, + {file = "boto3-1.28.55.tar.gz", hash = "sha256:2ceb644b1df7c3c8907913ab367a9900af79e271b4cfca37b542ec1fa142faf8"}, ] [package.dependencies] -botocore = ">=1.31.43,<1.32.0" +botocore = ">=1.31.55,<1.32.0" jmespath = ">=0.7.1,<2.0.0" -s3transfer = ">=0.6.0,<0.7.0" +s3transfer = ">=0.7.0,<0.8.0" [package.extras] crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.31.43" +version = "1.31.55" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">= 3.7" files = [ - {file = "botocore-1.31.43-py3-none-any.whl", hash = "sha256:d8b0c41c8c75d82f15fee57f7d54a852a99810faacbeb9d6f3f022558a2c330e"}, - {file = "botocore-1.31.43.tar.gz", hash = "sha256:b4a3a1fcf75011351e2b0d3eb991f51f8d44a375d3e065f907dac67db232fc97"}, + {file = "botocore-1.31.55-py3-none-any.whl", hash = "sha256:5ec27caa440257619712af0a71524cc2e56110fc502853c3e4046f87b65e42e9"}, + {file = "botocore-1.31.55.tar.gz", hash = "sha256:21ba89c4df083338ec463d9c2a8cffca42a99f9ad5f24bcac1870393b216c5a7"}, ] [package.dependencies] @@ -732,34 +732,34 @@ toml = ["tomli"] [[package]] name = "cryptography" -version = "41.0.3" +version = "41.0.4" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = true python-versions = ">=3.7" files = [ - {file = "cryptography-41.0.3-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:652627a055cb52a84f8c448185922241dd5217443ca194d5739b44612c5e6507"}, - {file = "cryptography-41.0.3-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:8f09daa483aedea50d249ef98ed500569841d6498aa9c9f4b0531b9964658922"}, - {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4fd871184321100fb400d759ad0cddddf284c4b696568204d281c902fc7b0d81"}, - {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84537453d57f55a50a5b6835622ee405816999a7113267739a1b4581f83535bd"}, - {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3fb248989b6363906827284cd20cca63bb1a757e0a2864d4c1682a985e3dca47"}, - {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:42cb413e01a5d36da9929baa9d70ca90d90b969269e5a12d39c1e0d475010116"}, - {file = "cryptography-41.0.3-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:aeb57c421b34af8f9fe830e1955bf493a86a7996cc1338fe41b30047d16e962c"}, - {file = "cryptography-41.0.3-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:6af1c6387c531cd364b72c28daa29232162010d952ceb7e5ca8e2827526aceae"}, - {file = "cryptography-41.0.3-cp37-abi3-win32.whl", hash = "sha256:0d09fb5356f975974dbcb595ad2d178305e5050656affb7890a1583f5e02a306"}, - {file = "cryptography-41.0.3-cp37-abi3-win_amd64.whl", hash = "sha256:a983e441a00a9d57a4d7c91b3116a37ae602907a7618b882c8013b5762e80574"}, - {file = "cryptography-41.0.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5259cb659aa43005eb55a0e4ff2c825ca111a0da1814202c64d28a985d33b087"}, - {file = "cryptography-41.0.3-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:67e120e9a577c64fe1f611e53b30b3e69744e5910ff3b6e97e935aeb96005858"}, - {file = "cryptography-41.0.3-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:7efe8041897fe7a50863e51b77789b657a133c75c3b094e51b5e4b5cec7bf906"}, - {file = "cryptography-41.0.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ce785cf81a7bdade534297ef9e490ddff800d956625020ab2ec2780a556c313e"}, - {file = "cryptography-41.0.3-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:57a51b89f954f216a81c9d057bf1a24e2f36e764a1ca9a501a6964eb4a6800dd"}, - {file = "cryptography-41.0.3-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:4c2f0d35703d61002a2bbdcf15548ebb701cfdd83cdc12471d2bae80878a4207"}, - {file = "cryptography-41.0.3-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:23c2d778cf829f7d0ae180600b17e9fceea3c2ef8b31a99e3c694cbbf3a24b84"}, - {file = "cryptography-41.0.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:95dd7f261bb76948b52a5330ba5202b91a26fbac13ad0e9fc8a3ac04752058c7"}, - {file = "cryptography-41.0.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:41d7aa7cdfded09b3d73a47f429c298e80796c8e825ddfadc84c8a7f12df212d"}, - {file = "cryptography-41.0.3-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d0d651aa754ef58d75cec6edfbd21259d93810b73f6ec246436a21b7841908de"}, - {file = "cryptography-41.0.3-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:ab8de0d091acbf778f74286f4989cf3d1528336af1b59f3e5d2ebca8b5fe49e1"}, - {file = "cryptography-41.0.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a74fbcdb2a0d46fe00504f571a2a540532f4c188e6ccf26f1f178480117b33c4"}, - {file = "cryptography-41.0.3.tar.gz", hash = "sha256:6d192741113ef5e30d89dcb5b956ef4e1578f304708701b8b73d38e3e1461f34"}, + {file = "cryptography-41.0.4-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:80907d3faa55dc5434a16579952ac6da800935cd98d14dbd62f6f042c7f5e839"}, + {file = "cryptography-41.0.4-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:35c00f637cd0b9d5b6c6bd11b6c3359194a8eba9c46d4e875a3660e3b400005f"}, + {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cecfefa17042941f94ab54f769c8ce0fe14beff2694e9ac684176a2535bf9714"}, + {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e40211b4923ba5a6dc9769eab704bdb3fbb58d56c5b336d30996c24fcf12aadb"}, + {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:23a25c09dfd0d9f28da2352503b23e086f8e78096b9fd585d1d14eca01613e13"}, + {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2ed09183922d66c4ec5fdaa59b4d14e105c084dd0febd27452de8f6f74704143"}, + {file = "cryptography-41.0.4-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:5a0f09cefded00e648a127048119f77bc2b2ec61e736660b5789e638f43cc397"}, + {file = "cryptography-41.0.4-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:9eeb77214afae972a00dee47382d2591abe77bdae166bda672fb1e24702a3860"}, + {file = "cryptography-41.0.4-cp37-abi3-win32.whl", hash = "sha256:3b224890962a2d7b57cf5eeb16ccaafba6083f7b811829f00476309bce2fe0fd"}, + {file = "cryptography-41.0.4-cp37-abi3-win_amd64.whl", hash = "sha256:c880eba5175f4307129784eca96f4e70b88e57aa3f680aeba3bab0e980b0f37d"}, + {file = "cryptography-41.0.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:004b6ccc95943f6a9ad3142cfabcc769d7ee38a3f60fb0dddbfb431f818c3a67"}, + {file = "cryptography-41.0.4-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:86defa8d248c3fa029da68ce61fe735432b047e32179883bdb1e79ed9bb8195e"}, + {file = "cryptography-41.0.4-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:37480760ae08065437e6573d14be973112c9e6dcaf5f11d00147ee74f37a3829"}, + {file = "cryptography-41.0.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:b5f4dfe950ff0479f1f00eda09c18798d4f49b98f4e2006d644b3301682ebdca"}, + {file = "cryptography-41.0.4-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7e53db173370dea832190870e975a1e09c86a879b613948f09eb49324218c14d"}, + {file = "cryptography-41.0.4-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:5b72205a360f3b6176485a333256b9bcd48700fc755fef51c8e7e67c4b63e3ac"}, + {file = "cryptography-41.0.4-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:93530900d14c37a46ce3d6c9e6fd35dbe5f5601bf6b3a5c325c7bffc030344d9"}, + {file = "cryptography-41.0.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:efc8ad4e6fc4f1752ebfb58aefece8b4e3c4cae940b0994d43649bdfce8d0d4f"}, + {file = "cryptography-41.0.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c3391bd8e6de35f6f1140e50aaeb3e2b3d6a9012536ca23ab0d9c35ec18c8a91"}, + {file = "cryptography-41.0.4-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:0d9409894f495d465fe6fda92cb70e8323e9648af912d5b9141d616df40a87b8"}, + {file = "cryptography-41.0.4-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:8ac4f9ead4bbd0bc8ab2d318f97d85147167a488be0e08814a37eb2f439d5cf6"}, + {file = "cryptography-41.0.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:047c4603aeb4bbd8db2756e38f5b8bd7e94318c047cfe4efeb5d715e08b49311"}, + {file = "cryptography-41.0.4.tar.gz", hash = "sha256:7febc3094125fc126a7f6fb1f420d0da639f3f32cb15c8ff0dc3997c4549f51a"}, ] [package.dependencies] @@ -791,13 +791,13 @@ requests = ">=2.6.0" [[package]] name = "datadog-lambda" -version = "4.79.0" +version = "4.80.0" description = "The Datadog AWS Lambda Library" optional = false python-versions = ">=3.7.0,<4" files = [ - {file = "datadog_lambda-4.79.0-py3-none-any.whl", hash = "sha256:b297ff278a453dd0900e5cbe1332b9381bd2624d9ad8f21c8b464ef3c2dc85cc"}, - {file = "datadog_lambda-4.79.0.tar.gz", hash = "sha256:b0cf2d727edf9d652b75ab3f6b9e9eff3e5549394fcff7aa02f2d7cd955435da"}, + {file = "datadog_lambda-4.80.0-py3-none-any.whl", hash = "sha256:506b8964567230d87e2bfd323420854d37b4d7c7a9bfab7e192389f9b4c8150c"}, + {file = "datadog_lambda-4.80.0.tar.gz", hash = "sha256:ddd3ed20592df97523ae26ba552b69de239520c37e31804ca9949b010f90b461"}, ] [package.dependencies] @@ -1055,19 +1055,22 @@ smmap = ">=3.0.1,<6" [[package]] name = "gitpython" -version = "3.1.35" +version = "3.1.37" description = "GitPython is a Python library used to interact with Git repositories" optional = false python-versions = ">=3.7" files = [ - {file = "GitPython-3.1.35-py3-none-any.whl", hash = "sha256:c19b4292d7a1d3c0f653858db273ff8a6614100d1eb1528b014ec97286193c09"}, - {file = "GitPython-3.1.35.tar.gz", hash = "sha256:9cbefbd1789a5fe9bcf621bb34d3f441f3a90c8461d377f84eda73e721d9b06b"}, + {file = "GitPython-3.1.37-py3-none-any.whl", hash = "sha256:5f4c4187de49616d710a77e98ddf17b4782060a1788df441846bddefbb89ab33"}, + {file = "GitPython-3.1.37.tar.gz", hash = "sha256:f9b9ddc0761c125d5780eab2d64be4873fc6817c2899cbcb34b02344bdc7bc54"}, ] [package.dependencies] gitdb = ">=4.0.1,<5" typing-extensions = {version = ">=3.7.4.3", markers = "python_version < \"3.8\""} +[package.extras] +test = ["black", "coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mypy", "pre-commit", "pytest", "pytest-cov", "pytest-sugar"] + [[package]] name = "h11" version = "0.14.0" @@ -1351,13 +1354,13 @@ pbr = "*" [[package]] name = "jsii" -version = "1.88.0" +version = "1.89.0" description = "Python client for jsii runtime" optional = false python-versions = "~=3.7" files = [ - {file = "jsii-1.88.0-py3-none-any.whl", hash = "sha256:b3888141c30b83a30bfbe03a877bbf8ae42f957b6ccca02bae448853debffaf8"}, - {file = "jsii-1.88.0.tar.gz", hash = "sha256:a59e0f962589dcc741d2bcf2a7b4c4a927a29d3f9a2804a192c734e2e3275018"}, + {file = "jsii-1.89.0-py3-none-any.whl", hash = "sha256:20a463e8533eded656b285f532e5468a414c48ab083cf0cf93a86d593f0c36b8"}, + {file = "jsii-1.89.0.tar.gz", hash = "sha256:6edbb79afc0b7407cb64e9dd0f27b512279201307c16dd9ae72462b3cbd09970"}, ] [package.dependencies] @@ -1557,16 +1560,6 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -1644,13 +1637,13 @@ test = ["coverage", "flake8 (>=3.0)", "shtab"] [[package]] name = "mkdocs" -version = "1.5.2" +version = "1.5.3" description = "Project documentation with Markdown." optional = false python-versions = ">=3.7" files = [ - {file = "mkdocs-1.5.2-py3-none-any.whl", hash = "sha256:60a62538519c2e96fe8426654a67ee177350451616118a41596ae7c876bb7eac"}, - {file = "mkdocs-1.5.2.tar.gz", hash = "sha256:70d0da09c26cff288852471be03c23f0f521fc15cf16ac89c7a3bfb9ae8d24f9"}, + {file = "mkdocs-1.5.3-py3-none-any.whl", hash = "sha256:3b3a78e736b31158d64dbb2f8ba29bd46a379d0c6e324c2246c3bc3d2189cfc1"}, + {file = "mkdocs-1.5.3.tar.gz", hash = "sha256:eb7c99214dcb945313ba30426c2451b735992c73c2e10838f76d09e39ff4d0e2"}, ] [package.dependencies] @@ -1715,13 +1708,13 @@ requests = ">=2.26,<3.0" [[package]] name = "mkdocs-material-extensions" -version = "1.1.1" +version = "1.2" description = "Extension pack for Python Markdown and MkDocs Material." optional = false python-versions = ">=3.7" files = [ - {file = "mkdocs_material_extensions-1.1.1-py3-none-any.whl", hash = "sha256:e41d9f38e4798b6617ad98ca8f7f1157b1e4385ac1459ca1e4ea219b556df945"}, - {file = "mkdocs_material_extensions-1.1.1.tar.gz", hash = "sha256:9c003da71e2cc2493d910237448c672e00cefc800d3d6ae93d2fc69979e3bd93"}, + {file = "mkdocs_material_extensions-1.2-py3-none-any.whl", hash = "sha256:c767bd6d6305f6420a50f0b541b0c9966d52068839af97029be14443849fb8a1"}, + {file = "mkdocs_material_extensions-1.2.tar.gz", hash = "sha256:27e2d1ed2d031426a6e10d5ea06989d67e90bb02acd588bc5673106b5ee5eedf"}, ] [[package]] @@ -2445,7 +2438,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -2453,15 +2445,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -2478,7 +2463,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -2486,7 +2470,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -2657,13 +2640,13 @@ decorator = ">=3.4.2" [[package]] name = "rich" -version = "13.5.2" +version = "13.5.3" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false python-versions = ">=3.7.0" files = [ - {file = "rich-13.5.2-py3-none-any.whl", hash = "sha256:146a90b3b6b47cac4a73c12866a499e9817426423f57c5a66949c086191a8808"}, - {file = "rich-13.5.2.tar.gz", hash = "sha256:fb9d6c0a0f643c99eed3875b5377a184132ba9be4d61516a55273d3554d75a39"}, + {file = "rich-13.5.3-py3-none-any.whl", hash = "sha256:9257b468badc3d347e146a4faa268ff229039d4c2d176ab0cffb4c4fbc73d5d9"}, + {file = "rich-13.5.3.tar.gz", hash = "sha256:87b43e0543149efa1253f485cd845bb7ee54df16c9617b8a893650ab84b4acb6"}, ] [package.dependencies] @@ -2702,13 +2685,13 @@ files = [ [[package]] name = "s3transfer" -version = "0.6.2" +version = "0.7.0" description = "An Amazon S3 Transfer Manager" optional = false python-versions = ">= 3.7" files = [ - {file = "s3transfer-0.6.2-py3-none-any.whl", hash = "sha256:b014be3a8a2aab98cfe1abc7229cc5a9a0cf05eb9c1f2b86b230fd8df3f78084"}, - {file = "s3transfer-0.6.2.tar.gz", hash = "sha256:cab66d3380cca3e70939ef2255d01cd8aece6a4907a9528740f668c4b0611861"}, + {file = "s3transfer-0.7.0-py3-none-any.whl", hash = "sha256:10d6923c6359175f264811ef4bf6161a3156ce8e350e705396a7557d6293c33a"}, + {file = "s3transfer-0.7.0.tar.gz", hash = "sha256:fd3889a66f5fe17299fe75b82eae6cf722554edca744ca5d5fe308b104883d2e"}, ] [package.dependencies] @@ -2790,13 +2773,13 @@ files = [ [[package]] name = "smmap" -version = "5.0.0" +version = "5.0.1" description = "A pure Python implementation of a sliding window memory map manager" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "smmap-5.0.0-py3-none-any.whl", hash = "sha256:2aba19d6a040e78d8b09de5c57e96207b09ed71d8e55ce0959eeee6c8e190d94"}, - {file = "smmap-5.0.0.tar.gz", hash = "sha256:c840e62059cd3be204b0c9c9f74be2c09d5648eddd4580d9314c3ecde0b30936"}, + {file = "smmap-5.0.1-py3-none-any.whl", hash = "sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da"}, + {file = "smmap-5.0.1.tar.gz", hash = "sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62"}, ] [[package]] @@ -3170,4 +3153,4 @@ validation = ["fastjsonschema"] [metadata] lock-version = "2.0" python-versions = "^3.7.4" -content-hash = "4e756acb30cdd11d63e3116d54cf19027f0fd3f1dd39e85f32efdec908c95660" +content-hash = "6b80bba30034ca42bb4d53689cd73b9802333f6de3e194d01086243d0085c817" From ba534ed5a4d70fd0cf09543999628caeb9cb0f90 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Tue, 26 Sep 2023 22:52:20 +0100 Subject: [PATCH 53/56] Setting default region to avoid HTTP connection --- tests/functional/data_masking/test_aws_encryption_sdk.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/functional/data_masking/test_aws_encryption_sdk.py b/tests/functional/data_masking/test_aws_encryption_sdk.py index e37c43cec90..d0d218b337b 100644 --- a/tests/functional/data_masking/test_aws_encryption_sdk.py +++ b/tests/functional/data_masking/test_aws_encryption_sdk.py @@ -13,10 +13,13 @@ @pytest.fixture -def data_masker() -> DataMasking: +def data_masker(monkeypatch) -> DataMasking: + # Setting a default region + monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1") + """DataMasking using AWS Encryption SDK Provider with a fake client""" fake_client = FakeEncryptionClient() - provider = AwsEncryptionSdkProvider(keys=["arn:aws:kms:us-east-1:0123456789012:key/dummy"], client=fake_client) + provider = AwsEncryptionSdkProvider(keys=["dummy"], client=fake_client) return DataMasking(provider=provider) From ceb6131db560c850b3a132745edb81ad8103eac7 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Tue, 26 Sep 2023 23:19:42 +0100 Subject: [PATCH 54/56] Removing user agent tracking --- .../utilities/data_masking/providers/aws_encryption_sdk.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py index 39852ae2d12..6886c3470fc 100644 --- a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py +++ b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py @@ -1,7 +1,6 @@ import base64 from typing import Any, Callable, Dict, List, Optional, Union -import botocore from aws_encryption_sdk import ( CachingCryptoMaterialsManager, EncryptionSDKClient, @@ -9,7 +8,6 @@ StrictAwsKmsMasterKeyProvider, ) -from aws_lambda_powertools.shared.user_agent import register_feature_to_botocore_session from aws_lambda_powertools.utilities.data_masking.constants import ( CACHE_CAPACITY, MAX_CACHE_AGE_SECONDS, @@ -37,9 +35,6 @@ class AwsEncryptionSdkProvider(BaseProvider): "a string" """ - session = botocore.session.Session() - register_feature_to_botocore_session(session, "data-masking") - def __init__( self, keys: List[str], @@ -54,7 +49,7 @@ def __init__( self.client = client or EncryptionSDKClient() self.keys = keys self.cache = LocalCryptoMaterialsCache(local_cache_capacity) - self.key_provider = StrictAwsKmsMasterKeyProvider(key_ids=self.keys, botocore_session=self.session) + self.key_provider = StrictAwsKmsMasterKeyProvider(key_ids=self.keys) self.cache_cmm = CachingCryptoMaterialsManager( master_key_provider=self.key_provider, cache=self.cache, From bf0e4ed078e6376a0047e930484d35c12258c20f Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Tue, 26 Sep 2023 23:24:19 +0100 Subject: [PATCH 55/56] Reverting --- .../utilities/data_masking/providers/aws_encryption_sdk.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py index 6886c3470fc..39852ae2d12 100644 --- a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py +++ b/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py @@ -1,6 +1,7 @@ import base64 from typing import Any, Callable, Dict, List, Optional, Union +import botocore from aws_encryption_sdk import ( CachingCryptoMaterialsManager, EncryptionSDKClient, @@ -8,6 +9,7 @@ StrictAwsKmsMasterKeyProvider, ) +from aws_lambda_powertools.shared.user_agent import register_feature_to_botocore_session from aws_lambda_powertools.utilities.data_masking.constants import ( CACHE_CAPACITY, MAX_CACHE_AGE_SECONDS, @@ -35,6 +37,9 @@ class AwsEncryptionSdkProvider(BaseProvider): "a string" """ + session = botocore.session.Session() + register_feature_to_botocore_session(session, "data-masking") + def __init__( self, keys: List[str], @@ -49,7 +54,7 @@ def __init__( self.client = client or EncryptionSDKClient() self.keys = keys self.cache = LocalCryptoMaterialsCache(local_cache_capacity) - self.key_provider = StrictAwsKmsMasterKeyProvider(key_ids=self.keys) + self.key_provider = StrictAwsKmsMasterKeyProvider(key_ids=self.keys, botocore_session=self.session) self.cache_cmm = CachingCryptoMaterialsManager( master_key_provider=self.key_provider, cache=self.cache, From 6a064b14ce45564188e2983704d81932f659ca45 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Wed, 27 Sep 2023 19:48:56 +0100 Subject: [PATCH 56/56] Creating a specific provider instead a client to avoid any http call from AWS SDK Encryption lib --- Makefile | 4 +- .../utilities/data_masking/__init__.py | 5 + .../utilities/data_masking/base.py | 32 ++++++ .../data_masking/provider/__init__.py | 5 + .../{provider.py => provider/base.py} | 4 +- .../data_masking/provider/kms/__init__.py | 5 + .../kms}/aws_encryption_sdk.py | 102 ++++++++++++++---- .../data_masking/providers/__init__.py | 0 pyproject.toml | 5 +- .../data_masking/handlers/basic_handler.py | 4 +- .../e2e/data_masking/test_e2e_data_masking.py | 4 +- tests/functional/data_masking/conftest.py | 17 --- .../data_masking/test_aws_encryption_sdk.py | 37 +++++-- .../pt-load-test-stack/function_1024/app.py | 4 +- .../pt-load-test-stack/function_128/app.py | 4 +- .../pt-load-test-stack/function_1769/app.py | 4 +- 16 files changed, 173 insertions(+), 63 deletions(-) create mode 100644 aws_lambda_powertools/utilities/data_masking/provider/__init__.py rename aws_lambda_powertools/utilities/data_masking/{provider.py => provider/base.py} (93%) create mode 100644 aws_lambda_powertools/utilities/data_masking/provider/kms/__init__.py rename aws_lambda_powertools/utilities/data_masking/{providers => provider/kms}/aws_encryption_sdk.py (51%) delete mode 100644 aws_lambda_powertools/utilities/data_masking/providers/__init__.py diff --git a/Makefile b/Makefile index 5bfbf031949..584ed361ed9 100644 --- a/Makefile +++ b/Makefile @@ -7,13 +7,13 @@ target: dev: pip install --upgrade pip pre-commit poetry @$(MAKE) dev-version-plugin - poetry install --extras "all" + poetry install --extras "all datamasking-aws-sdk" pre-commit install dev-gitpod: pip install --upgrade pip poetry @$(MAKE) dev-version-plugin - poetry install --extras "all" + poetry install --extras "all datamasking-aws-sdk" pre-commit install format: diff --git a/aws_lambda_powertools/utilities/data_masking/__init__.py b/aws_lambda_powertools/utilities/data_masking/__init__.py index e69de29bb2d..428cea6635d 100644 --- a/aws_lambda_powertools/utilities/data_masking/__init__.py +++ b/aws_lambda_powertools/utilities/data_masking/__init__.py @@ -0,0 +1,5 @@ +from aws_lambda_powertools.utilities.data_masking.base import DataMasking + +__all__ = [ + "DataMasking", +] diff --git a/aws_lambda_powertools/utilities/data_masking/base.py b/aws_lambda_powertools/utilities/data_masking/base.py index a7f2f820761..06b383ce4a4 100644 --- a/aws_lambda_powertools/utilities/data_masking/base.py +++ b/aws_lambda_powertools/utilities/data_masking/base.py @@ -5,6 +5,38 @@ class DataMasking: + """ + A utility class for masking sensitive data within various data types. + + This class provides methods for masking sensitive information, such as personal + identifiers or confidential data, within different data types such as strings, + dictionaries, lists, and more. It helps protect sensitive information while + preserving the structure of the original data. + + Usage: + Instantiate an object of this class and use its methods to mask sensitive data + based on the data type. Supported data types include strings, dictionaries, + and more. + + Example: + ``` + from aws_lambda_powertools.utilities.data_masking.base import DataMasking + + def lambda_handler(event, context): + masker = DataMasking() + + data = { + "project": "powertools", + "sensitive": "xxxxxxxxxx" + } + + masked = masker.mask(data,fields=["sensitive"]) + + return masked + + ``` + """ + def __init__(self, provider: Optional[BaseProvider] = None): self.provider = provider or BaseProvider() diff --git a/aws_lambda_powertools/utilities/data_masking/provider/__init__.py b/aws_lambda_powertools/utilities/data_masking/provider/__init__.py new file mode 100644 index 00000000000..5a0180eb82b --- /dev/null +++ b/aws_lambda_powertools/utilities/data_masking/provider/__init__.py @@ -0,0 +1,5 @@ +from aws_lambda_powertools.utilities.data_masking.provider.base import BaseProvider + +__all__ = [ + "BaseProvider", +] diff --git a/aws_lambda_powertools/utilities/data_masking/provider.py b/aws_lambda_powertools/utilities/data_masking/provider/base.py similarity index 93% rename from aws_lambda_powertools/utilities/data_masking/provider.py rename to aws_lambda_powertools/utilities/data_masking/provider/base.py index f1c3bb869f9..ceb222aa7f8 100644 --- a/aws_lambda_powertools/utilities/data_masking/provider.py +++ b/aws_lambda_powertools/utilities/data_masking/provider/base.py @@ -1,5 +1,5 @@ import json -from typing import Any, Union +from typing import Any from aws_lambda_powertools.utilities.data_masking.constants import DATA_MASKING_STRING @@ -20,7 +20,7 @@ def default_json_serializer(self, data): def default_json_deserializer(self, data): return json.loads(data.decode("utf-8")) - def encrypt(self, data) -> Union[bytes, str]: + def encrypt(self, data) -> str: raise NotImplementedError("Subclasses must implement encrypt()") def decrypt(self, data) -> Any: diff --git a/aws_lambda_powertools/utilities/data_masking/provider/kms/__init__.py b/aws_lambda_powertools/utilities/data_masking/provider/kms/__init__.py new file mode 100644 index 00000000000..8cc33b5e075 --- /dev/null +++ b/aws_lambda_powertools/utilities/data_masking/provider/kms/__init__.py @@ -0,0 +1,5 @@ +from aws_lambda_powertools.utilities.data_masking.provider.kms.aws_encryption_sdk import AwsEncryptionSdkProvider + +__all__ = [ + "AwsEncryptionSdkProvider", +] diff --git a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py b/aws_lambda_powertools/utilities/data_masking/provider/kms/aws_encryption_sdk.py similarity index 51% rename from aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py rename to aws_lambda_powertools/utilities/data_masking/provider/kms/aws_encryption_sdk.py index 39852ae2d12..9e4c7ef19cb 100644 --- a/aws_lambda_powertools/utilities/data_masking/providers/aws_encryption_sdk.py +++ b/aws_lambda_powertools/utilities/data_masking/provider/kms/aws_encryption_sdk.py @@ -1,5 +1,7 @@ +from __future__ import annotations + import base64 -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Callable, Dict, List import botocore from aws_encryption_sdk import ( @@ -26,35 +28,89 @@ def __init__(self, key): class AwsEncryptionSdkProvider(BaseProvider): """ - The AwsEncryptionSdkProvider is to be used as a Provider for the Datamasking class. - - Example - ------- - >>> data_masker = DataMasking(provider=AwsEncryptionSdkProvider(keys=[keyARN1, keyARN2,...,])) - >>> encrypted_data = data_masker.encrypt("a string") - "encrptedBase64String" - >>> decrypted_data = data_masker.decrypt(encrypted_data) - "a string" - """ + The AwsEncryptionSdkProvider is used as a provider for the DataMasking class. + + This provider allows you to perform data masking using the AWS Encryption SDK + for encryption and decryption. It integrates with the DataMasking class to + securely encrypt and decrypt sensitive data. + + Usage Example: + ``` + from aws_lambda_powertools.utilities.data_masking import DataMasking + from aws_lambda_powertools.utilities.data_masking.providers.kms.aws_encryption_sdk import ( + AwsEncryptionSdkProvider, + ) + + + def lambda_handler(event, context): + provider = AwsEncryptionSdkProvider(["arn:aws:kms:us-east-1:0123456789012:key/key-id"]) + masker = DataMasking(provider=provider) + + data = { + "project": "powertools", + "sensitive": "xxxxxxxxxx" + } + + masked = masker.encrypt(data,fields=["sensitive"]) - session = botocore.session.Session() - register_feature_to_botocore_session(session, "data-masking") + return masked + + ``` + """ def __init__( self, keys: List[str], - client: Optional[EncryptionSDKClient] = None, + key_provider=None, local_cache_capacity: int = CACHE_CAPACITY, max_cache_age_seconds: float = MAX_CACHE_AGE_SECONDS, max_messages_encrypted: int = MAX_MESSAGES_ENCRYPTED, - json_serializer: Optional[Callable[[Dict], str]] = None, - json_deserializer: Optional[Callable[[Union[Dict, str, bool, int, float]], str]] = None, + json_serializer: Callable | None = None, + json_deserializer: Callable | None = None, ): super().__init__(json_serializer=json_serializer, json_deserializer=json_deserializer) - self.client = client or EncryptionSDKClient() + + self._key_provider = key_provider or KMSKeyProvider( + keys=keys, + local_cache_capacity=local_cache_capacity, + max_cache_age_seconds=max_cache_age_seconds, + max_messages_encrypted=max_messages_encrypted, + json_serializer=self.json_serializer, + json_deserializer=self.json_deserializer, + ) + + def encrypt(self, data: bytes | str | Dict | int, **provider_options) -> str: + return self._key_provider.encrypt(data=data, **provider_options) + + def decrypt(self, data: str, **provider_options) -> Any: + return self._key_provider.decrypt(data=data, **provider_options) + + +class KMSKeyProvider: + + """ + The KMSKeyProvider is responsible for assembling an AWS Key Management Service (KMS) + client, a caching mechanism, and a keyring for secure key management and data encryption. + """ + + def __init__( + self, + keys: List[str], + json_serializer: Callable, + json_deserializer: Callable, + local_cache_capacity: int = CACHE_CAPACITY, + max_cache_age_seconds: float = MAX_CACHE_AGE_SECONDS, + max_messages_encrypted: int = MAX_MESSAGES_ENCRYPTED, + ): + session = botocore.session.Session() + register_feature_to_botocore_session(session, "data-masking") + + self.json_serializer = json_serializer + self.json_deserializer = json_deserializer + self.client = EncryptionSDKClient() self.keys = keys self.cache = LocalCryptoMaterialsCache(local_cache_capacity) - self.key_provider = StrictAwsKmsMasterKeyProvider(key_ids=self.keys, botocore_session=self.session) + self.key_provider = StrictAwsKmsMasterKeyProvider(key_ids=self.keys, botocore_session=session) self.cache_cmm = CachingCryptoMaterialsManager( master_key_provider=self.key_provider, cache=self.cache, @@ -62,7 +118,7 @@ def __init__( max_messages_encrypted=max_messages_encrypted, ) - def encrypt(self, data: Union[bytes, str], **provider_options) -> bytes: + def encrypt(self, data: bytes | str | Dict | float, **provider_options) -> str: """ Encrypt data using the AwsEncryptionSdkProvider. @@ -78,8 +134,12 @@ def encrypt(self, data: Union[bytes, str], **provider_options) -> bytes: ciphertext : str The encrypted data, as a base64-encoded string. """ - data = self.json_serializer(data) - ciphertext, _ = self.client.encrypt(source=data, materials_manager=self.cache_cmm, **provider_options) + data_encoded = self.json_serializer(data) + ciphertext, _ = self.client.encrypt( + source=data_encoded, + materials_manager=self.cache_cmm, + **provider_options, + ) ciphertext = base64.b64encode(ciphertext).decode() return ciphertext diff --git a/aws_lambda_powertools/utilities/data_masking/providers/__init__.py b/aws_lambda_powertools/utilities/data_masking/providers/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/pyproject.toml b/pyproject.toml index d8c5db9b553..1cad7a87b1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,14 +92,13 @@ datadog-lambda = "^4.77.0" [tool.poetry.extras] parser = ["pydantic"] -datamasking-aws-sdk= ["aws-encryption-sdk"] -datamasking-all = ["aws-encryption-sdk"] validation = ["fastjsonschema"] tracer = ["aws-xray-sdk"] -all = ["pydantic", "aws-xray-sdk", "fastjsonschema", "aws-encryption-sdk"] +all = ["pydantic", "aws-xray-sdk", "fastjsonschema"] # allow customers to run code locally without emulators (SAM CLI, etc.) aws-sdk = ["boto3"] datadog = ["datadog-lambda"] +datamasking-aws-sdk = ["aws-encryption-sdk"] [tool.poetry.group.dev.dependencies] cfn-lint = "0.80.3" diff --git a/tests/e2e/data_masking/handlers/basic_handler.py b/tests/e2e/data_masking/handlers/basic_handler.py index 69d99ea8256..7a5d965fa38 100644 --- a/tests/e2e/data_masking/handlers/basic_handler.py +++ b/tests/e2e/data_masking/handlers/basic_handler.py @@ -1,6 +1,6 @@ from aws_lambda_powertools import Logger -from aws_lambda_powertools.utilities.data_masking.base import DataMasking -from aws_lambda_powertools.utilities.data_masking.providers.aws_encryption_sdk import AwsEncryptionSdkProvider +from aws_lambda_powertools.utilities.data_masking import DataMasking +from aws_lambda_powertools.utilities.data_masking.provider.kms.aws_encryption_sdk import AwsEncryptionSdkProvider logger = Logger() diff --git a/tests/e2e/data_masking/test_e2e_data_masking.py b/tests/e2e/data_masking/test_e2e_data_masking.py index dbc4817ba71..c15b4fb0d38 100644 --- a/tests/e2e/data_masking/test_e2e_data_masking.py +++ b/tests/e2e/data_masking/test_e2e_data_masking.py @@ -4,8 +4,8 @@ import pytest from aws_encryption_sdk.exceptions import DecryptKeyError -from aws_lambda_powertools.utilities.data_masking.base import DataMasking -from aws_lambda_powertools.utilities.data_masking.providers.aws_encryption_sdk import ( +from aws_lambda_powertools.utilities.data_masking import DataMasking +from aws_lambda_powertools.utilities.data_masking.provider.kms.aws_encryption_sdk import ( AwsEncryptionSdkProvider, ContextMismatchError, ) diff --git a/tests/functional/data_masking/conftest.py b/tests/functional/data_masking/conftest.py index 6127858d6b3..f73ccca4113 100644 --- a/tests/functional/data_masking/conftest.py +++ b/tests/functional/data_masking/conftest.py @@ -1,23 +1,6 @@ -from __future__ import annotations - -from typing import Tuple - from pytest_socket import disable_socket def pytest_runtest_setup(): """Disable Unix and TCP sockets for Data masking tests""" disable_socket() - - -class FakeEncryptionClient: - ENCRYPTION_HEADER = "test" - - def encrypt(self, source: bytes | str, **kwargs) -> Tuple[bytes, str]: - if isinstance(source, str): - return source.encode(), self.ENCRYPTION_HEADER - - return source, self.ENCRYPTION_HEADER - - def decrypt(self, source: bytes, **kwargs) -> Tuple[bytes, str]: - return source, "dummy_decryption_header" diff --git a/tests/functional/data_masking/test_aws_encryption_sdk.py b/tests/functional/data_masking/test_aws_encryption_sdk.py index d0d218b337b..49ed775c10d 100644 --- a/tests/functional/data_masking/test_aws_encryption_sdk.py +++ b/tests/functional/data_masking/test_aws_encryption_sdk.py @@ -1,25 +1,46 @@ from __future__ import annotations +import base64 import json +from typing import Any, Callable, Dict, Union import pytest -from aws_lambda_powertools.utilities.data_masking.base import DataMasking +from aws_lambda_powertools.utilities.data_masking import DataMasking from aws_lambda_powertools.utilities.data_masking.constants import DATA_MASKING_STRING -from aws_lambda_powertools.utilities.data_masking.providers.aws_encryption_sdk import ( +from aws_lambda_powertools.utilities.data_masking.provider import BaseProvider +from aws_lambda_powertools.utilities.data_masking.provider.kms import ( AwsEncryptionSdkProvider, ) -from tests.functional.data_masking.conftest import FakeEncryptionClient + + +class FakeEncryptionKeyProvider(BaseProvider): + def __init__( + self, + json_serializer: Callable[[Dict], str] | None = None, + json_deserializer: Callable[[Union[Dict, str, bool, int, float]], str] | None = None, + ): + super().__init__(json_serializer=json_serializer, json_deserializer=json_deserializer) + + def encrypt(self, data: bytes | str, **kwargs) -> str: + data = self.json_serializer(data) + ciphertext = base64.b64encode(data).decode() + return ciphertext + + def decrypt(self, data: bytes, **kwargs) -> Any: + ciphertext_decoded = base64.b64decode(data) + ciphertext = self.json_deserializer(ciphertext_decoded) + return ciphertext @pytest.fixture def data_masker(monkeypatch) -> DataMasking: - # Setting a default region - monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1") - """DataMasking using AWS Encryption SDK Provider with a fake client""" - fake_client = FakeEncryptionClient() - provider = AwsEncryptionSdkProvider(keys=["dummy"], client=fake_client) + fake_key_provider = FakeEncryptionKeyProvider() + provider = AwsEncryptionSdkProvider( + keys=["dummy"], + key_provider=fake_key_provider, + ) return DataMasking(provider=provider) diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/function_1024/app.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/function_1024/app.py index e311fbf9b8f..f6988ea66ac 100644 --- a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/function_1024/app.py +++ b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/function_1024/app.py @@ -3,8 +3,8 @@ from aws_lambda_powertools import Logger, Tracer from aws_lambda_powertools.event_handler import APIGatewayRestResolver from aws_lambda_powertools.logging import correlation_paths -from aws_lambda_powertools.utilities.data_masking.base import DataMasking -from aws_lambda_powertools.utilities.data_masking.providers.aws_encryption_sdk import AwsEncryptionSdkProvider +from aws_lambda_powertools.utilities.data_masking import DataMasking +from aws_lambda_powertools.utilities.data_masking.provider.kms.aws_encryption_sdk import AwsEncryptionSdkProvider from aws_lambda_powertools.utilities.typing import LambdaContext KMS_KEY_ARN = os.environ["KMS_KEY_ARN"] diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/function_128/app.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/function_128/app.py index 574aa8fd583..463ed7ca1a9 100644 --- a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/function_128/app.py +++ b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/function_128/app.py @@ -3,8 +3,8 @@ from aws_lambda_powertools import Logger, Tracer from aws_lambda_powertools.event_handler import APIGatewayRestResolver from aws_lambda_powertools.logging import correlation_paths -from aws_lambda_powertools.utilities.data_masking.base import DataMasking -from aws_lambda_powertools.utilities.data_masking.providers.aws_encryption_sdk import AwsEncryptionSdkProvider +from aws_lambda_powertools.utilities.data_masking import DataMasking +from aws_lambda_powertools.utilities.data_masking.provider.kms.aws_encryption_sdk import AwsEncryptionSdkProvider from aws_lambda_powertools.utilities.typing import LambdaContext KMS_KEY_ARN = os.environ["KMS_KEY_ARN"] diff --git a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/function_1769/app.py b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/function_1769/app.py index 76580e6706f..44ddeab189b 100644 --- a/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/function_1769/app.py +++ b/tests/performance/data_masking/load_test_data_masking/pt-load-test-stack/function_1769/app.py @@ -3,8 +3,8 @@ from aws_lambda_powertools import Logger, Tracer from aws_lambda_powertools.event_handler import APIGatewayRestResolver from aws_lambda_powertools.logging import correlation_paths -from aws_lambda_powertools.utilities.data_masking.base import DataMasking -from aws_lambda_powertools.utilities.data_masking.providers.aws_encryption_sdk import AwsEncryptionSdkProvider +from aws_lambda_powertools.utilities.data_masking import DataMasking +from aws_lambda_powertools.utilities.data_masking.provider.kms.aws_encryption_sdk import AwsEncryptionSdkProvider from aws_lambda_powertools.utilities.typing import LambdaContext KMS_KEY_ARN = os.environ["KMS_KEY_ARN"]