From 6cc810955ac1557d62c34e86a9d099746c40df90 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Mon, 26 Aug 2019 16:35:26 -0700 Subject: [PATCH 01/10] rename paginators tests to specify scan paginator --- test/functional/encrypted/test_client.py | 8 ++++---- test/functional/functional_test_utils.py | 18 ++++++++++++++---- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/test/functional/encrypted/test_client.py b/test/functional/encrypted/test_client.py index 4498f954..42921867 100644 --- a/test/functional/encrypted/test_client.py +++ b/test/functional/encrypted/test_client.py @@ -20,7 +20,7 @@ build_static_jce_cmp, client_batch_items_unprocessed_check, client_cycle_batch_items_check, - client_cycle_batch_items_check_paginators, + client_cycle_batch_items_check_scan_paginator, client_cycle_single_item_check, set_parametrized_actions, set_parametrized_cmp, @@ -49,8 +49,8 @@ def _client_cycle_batch_items_check(materials_provider, initial_actions, initial ) -def _client_cycle_batch_items_check_paginators(materials_provider, initial_actions, initial_item): - return client_cycle_batch_items_check_paginators( +def _client_cycle_batch_items_check_scan_paginator(materials_provider, initial_actions, initial_item): + return client_cycle_batch_items_check_scan_paginator( materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, "us-west-2" ) @@ -73,7 +73,7 @@ def test_ephemeral_batch_item_cycle(example_table, some_cmps, parametrized_actio def test_ephemeral_batch_item_cycle_paginators(example_table, some_cmps, parametrized_actions, parametrized_item): """Test a small number of curated CMPs against a small number of curated items using paginators.""" - _client_cycle_batch_items_check_paginators(some_cmps, parametrized_actions, parametrized_item) + _client_cycle_batch_items_check_scan_paginator(some_cmps, parametrized_actions, parametrized_item) def test_batch_item_unprocessed(example_table, parametrized_actions, parametrized_item): diff --git a/test/functional/functional_test_utils.py b/test/functional/functional_test_utils.py index 86ebf196..efb6cc50 100644 --- a/test/functional/functional_test_utils.py +++ b/test/functional/functional_test_utils.py @@ -437,6 +437,7 @@ def cycle_batch_item_check( check_attribute_actions = initial_actions.copy() check_attribute_actions.set_index_keys(*list(TEST_KEY.keys())) items = _generate_items(initial_item, write_transformer) + items_in_table = len(items) _put_result = encrypted.batch_write_item( # noqa RequestItems={table_name: [{"PutRequest": {"Item": _item}} for _item in items]} @@ -458,9 +459,11 @@ def cycle_batch_item_check( if delete_items: _cleanup_items(encrypted, write_transformer, table_name) + items_in_table = 0 del check_attribute_actions del items + return items_in_table def cycle_batch_writer_check(raw_table, encrypted_table, initial_actions, initial_item): @@ -692,16 +695,23 @@ def client_batch_items_unprocessed_check( ) -def client_cycle_batch_items_check_paginators( +def client_cycle_batch_items_check_scan_paginator( materials_provider, initial_actions, initial_item, table_name, region_name=None ): + """Helper function for testing the "scan" paginator. + + Populate the specified table with encrypted items, + scan the table with raw client paginator to get encrypted items, + scan the table with encrypted client paginator to get decrypted items, + then verify that all items appear to have been encrypted correctly. + """ kwargs = {} if region_name is not None: kwargs["region_name"] = region_name client = boto3.client("dynamodb", **kwargs) e_client = EncryptedClient(client=client, materials_provider=materials_provider, attribute_actions=initial_actions) - cycle_batch_item_check( + items_in_table = cycle_batch_item_check( raw=client, encrypted=e_client, initial_actions=initial_actions, @@ -722,8 +732,8 @@ def client_cycle_batch_items_check_paginators( for page in encrypted_paginator.paginate(TableName=table_name, ConsistentRead=True): decrypted_items.extend(page["Items"]) - print(encrypted_items) - print(decrypted_items) + assert encrypted_items and decrypted_items + assert len(encrypted_items) == len(decrypted_items) == items_in_table check_attribute_actions = initial_actions.copy() check_attribute_actions.set_index_keys(*list(TEST_KEY.keys())) From 96e6c5127c966a0ecd97d5fd84bfe7864c2b717f Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Mon, 26 Aug 2019 17:26:52 -0700 Subject: [PATCH 02/10] add scan paginator integration tests --- test/integration/encrypted/test_client.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/integration/encrypted/test_client.py b/test/integration/encrypted/test_client.py index 4d75d248..7d577c55 100644 --- a/test/integration/encrypted/test_client.py +++ b/test/integration/encrypted/test_client.py @@ -54,3 +54,17 @@ def test_ephemeral_batch_item_cycle_kms(ddb_table_name, aws_kms_cmp, parametrize functional_test_utils.client_cycle_batch_items_check( aws_kms_cmp, parametrized_actions, parametrized_item, ddb_table_name ) + + +def test_ephemeral_batch_item_cycle_scan_paginator(ddb_table_name, some_cmps, parametrized_actions, parametrized_item): + """Test a small number of curated CMPs against a small number of curated items using the scan paginator.""" + functional_test_utils.client_cycle_batch_items_check_scan_paginator( + some_cmps, parametrized_actions, parametrized_item, ddb_table_name + ) + + +def test_ephemeral_batch_item_cycle_scan_paginator_kms(ddb_table_name, aws_kms_cmp, parametrized_actions, parametrized_item): + """Test a the AWS KMS CMP against a small number of curated items using the scan paginator.""" + functional_test_utils.client_cycle_batch_items_check_scan_paginator( + aws_kms_cmp, parametrized_actions, parametrized_item, ddb_table_name + ) From 3a12e54c07c5c28e1dce9161719458df1bb42d05 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Mon, 26 Aug 2019 17:27:15 -0700 Subject: [PATCH 03/10] continue paginator tests renaming --- test/functional/encrypted/test_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/functional/encrypted/test_client.py b/test/functional/encrypted/test_client.py index 42921867..83f0f250 100644 --- a/test/functional/encrypted/test_client.py +++ b/test/functional/encrypted/test_client.py @@ -71,8 +71,8 @@ def test_ephemeral_batch_item_cycle(example_table, some_cmps, parametrized_actio _client_cycle_batch_items_check(some_cmps, parametrized_actions, parametrized_item) -def test_ephemeral_batch_item_cycle_paginators(example_table, some_cmps, parametrized_actions, parametrized_item): - """Test a small number of curated CMPs against a small number of curated items using paginators.""" +def test_ephemeral_batch_item_cycle_scan_paginator(example_table, some_cmps, parametrized_actions, parametrized_item): + """Test a small number of curated CMPs against a small number of curated items using the scan paginator.""" _client_cycle_batch_items_check_scan_paginator(some_cmps, parametrized_actions, parametrized_item) From 55e723ba1c244b87ffcb33acb146cdf26647c1e8 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Mon, 26 Aug 2019 17:27:47 -0700 Subject: [PATCH 04/10] fix test helpers to actually clean up items if there is an error --- test/functional/functional_test_utils.py | 79 ++++++++++++------------ 1 file changed, 41 insertions(+), 38 deletions(-) diff --git a/test/functional/functional_test_utils.py b/test/functional/functional_test_utils.py index efb6cc50..a5192c83 100644 --- a/test/functional/functional_test_utils.py +++ b/test/functional/functional_test_utils.py @@ -443,23 +443,24 @@ def cycle_batch_item_check( RequestItems={table_name: [{"PutRequest": {"Item": _item}} for _item in items]} ) - ddb_keys = [write_transformer(key) for key in TEST_BATCH_KEYS] - encrypted_result = raw.batch_get_item(RequestItems={table_name: {"Keys": ddb_keys}}) - check_many_encrypted_items( - actual=encrypted_result["Responses"][table_name], - expected=items, - attribute_actions=check_attribute_actions, - transformer=read_transformer, - ) - - decrypted_result = encrypted.batch_get_item(RequestItems={table_name: {"Keys": ddb_keys}}) - assert_equal_lists_of_items( - actual=decrypted_result["Responses"][table_name], expected=items, transformer=read_transformer - ) + try: + ddb_keys = [write_transformer(key) for key in TEST_BATCH_KEYS] + encrypted_result = raw.batch_get_item(RequestItems={table_name: {"Keys": ddb_keys}}) + check_many_encrypted_items( + actual=encrypted_result["Responses"][table_name], + expected=items, + attribute_actions=check_attribute_actions, + transformer=read_transformer, + ) - if delete_items: - _cleanup_items(encrypted, write_transformer, table_name) - items_in_table = 0 + decrypted_result = encrypted.batch_get_item(RequestItems={table_name: {"Keys": ddb_keys}}) + assert_equal_lists_of_items( + actual=decrypted_result["Responses"][table_name], expected=items, transformer=read_transformer + ) + finally: + if delete_items: + _cleanup_items(encrypted, write_transformer, table_name) + items_in_table = 0 del check_attribute_actions del items @@ -722,29 +723,31 @@ def client_cycle_batch_items_check_scan_paginator( delete_items=False, ) - encrypted_items = [] - raw_paginator = client.get_paginator("scan") - for page in raw_paginator.paginate(TableName=table_name, ConsistentRead=True): - encrypted_items.extend(page["Items"]) - - decrypted_items = [] - encrypted_paginator = e_client.get_paginator("scan") - for page in encrypted_paginator.paginate(TableName=table_name, ConsistentRead=True): - decrypted_items.extend(page["Items"]) - - assert encrypted_items and decrypted_items - assert len(encrypted_items) == len(decrypted_items) == items_in_table - - check_attribute_actions = initial_actions.copy() - check_attribute_actions.set_index_keys(*list(TEST_KEY.keys())) - check_many_encrypted_items( - actual=encrypted_items, - expected=decrypted_items, - attribute_actions=check_attribute_actions, - transformer=ddb_to_dict, - ) + try: + encrypted_items = [] + raw_paginator = client.get_paginator("scan") + for page in raw_paginator.paginate(TableName=table_name, ConsistentRead=True): + encrypted_items.extend(page["Items"]) + + decrypted_items = [] + encrypted_paginator = e_client.get_paginator("scan") + for page in encrypted_paginator.paginate(TableName=table_name, ConsistentRead=True): + decrypted_items.extend(page["Items"]) + + assert encrypted_items and decrypted_items + assert len(encrypted_items) == len(decrypted_items) == items_in_table + + check_attribute_actions = initial_actions.copy() + check_attribute_actions.set_index_keys(*list(TEST_KEY.keys())) + check_many_encrypted_items( + actual=encrypted_items, + expected=decrypted_items, + attribute_actions=check_attribute_actions, + transformer=ddb_to_dict, + ) - _cleanup_items(encrypted=e_client, write_transformer=dict_to_ddb, table_name=table_name) + finally: + _cleanup_items(encrypted=e_client, write_transformer=dict_to_ddb, table_name=table_name) raw_scan_result = client.scan(TableName=table_name, ConsistentRead=True) e_scan_result = e_client.scan(TableName=table_name, ConsistentRead=True) From 54fe37696d62d993dd30ed59368f39b0286ce802 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Mon, 26 Aug 2019 19:22:44 -0700 Subject: [PATCH 05/10] autoformat --- test/integration/encrypted/test_client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/integration/encrypted/test_client.py b/test/integration/encrypted/test_client.py index 7d577c55..7b8de423 100644 --- a/test/integration/encrypted/test_client.py +++ b/test/integration/encrypted/test_client.py @@ -63,7 +63,9 @@ def test_ephemeral_batch_item_cycle_scan_paginator(ddb_table_name, some_cmps, pa ) -def test_ephemeral_batch_item_cycle_scan_paginator_kms(ddb_table_name, aws_kms_cmp, parametrized_actions, parametrized_item): +def test_ephemeral_batch_item_cycle_scan_paginator_kms( + ddb_table_name, aws_kms_cmp, parametrized_actions, parametrized_item +): """Test a the AWS KMS CMP against a small number of curated items using the scan paginator.""" functional_test_utils.client_cycle_batch_items_check_scan_paginator( aws_kms_cmp, parametrized_actions, parametrized_item, ddb_table_name From 751a27acd2158227238800800c4b05ca088e1ade Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Mon, 26 Aug 2019 19:23:50 -0700 Subject: [PATCH 06/10] single-source logic for decrypting a list of items used in both decrypt_multi_get and EncryptedPaginator --- .../encrypted/client.py | 8 ++++-- src/dynamodb_encryption_sdk/internal/utils.py | 27 ++++++++++++++----- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/src/dynamodb_encryption_sdk/encrypted/client.py b/src/dynamodb_encryption_sdk/encrypted/client.py index 21720a40..ebee5d90 100644 --- a/src/dynamodb_encryption_sdk/encrypted/client.py +++ b/src/dynamodb_encryption_sdk/encrypted/client.py @@ -22,6 +22,7 @@ crypto_config_from_kwargs, decrypt_batch_get_item, decrypt_get_item, + decrypt_list_of_items, decrypt_multi_get, encrypt_batch_write_item, encrypt_put_item, @@ -104,8 +105,11 @@ def paginate(self, **kwargs): crypto_config, ddb_kwargs = self._crypto_config_method(**kwargs) for page in self._paginator.paginate(**ddb_kwargs): - for pos, value in enumerate(page["Items"]): - page["Items"][pos] = self._decrypt_method(item=value, crypto_config=crypto_config) + page["Items"] = list( + decrypt_list_of_items( + crypto_config=crypto_config, decrypt_method=self._decrypt_method, items=page["Items"] + ) + ) yield page diff --git a/src/dynamodb_encryption_sdk/internal/utils.py b/src/dynamodb_encryption_sdk/internal/utils.py index 93bdbca3..ae8ebe16 100644 --- a/src/dynamodb_encryption_sdk/internal/utils.py +++ b/src/dynamodb_encryption_sdk/internal/utils.py @@ -29,7 +29,7 @@ from dynamodb_encryption_sdk.transform import dict_to_ddb try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Any, Bool, Callable, Dict, Text # noqa pylint: disable=unused-import + from typing import Any, Bool, Callable, Dict, Iterable, Text # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass @@ -41,6 +41,7 @@ "crypto_config_from_cache", "decrypt_get_item", "decrypt_multi_get", + "decrypt_list_of_items", "decrypt_batch_get_item", "encrypt_put_item", "encrypt_batch_write_item", @@ -171,6 +172,22 @@ def _item_transformer(crypto_transformer): return lambda x: x +def decrypt_list_of_items(crypto_config, decrypt_method, items): + # type: (CryptoConfig, Callable, Iterable[Any]) -> Iterable[Any] + # TODO: narrow this down + """Iterate through a list of encrypted items, decrypting each item and yielding the plaintext item. + + :param CryptoConfig crypto_config: :class:`CryptoConfig` to use + :param callable decrypt_method: Method to use to decrypt items + :param items: Iterable of encrypted items + :return: Iterable of plaintext items + """ + for value in items: + yield decrypt_method( + item=value, crypto_config=crypto_config.with_item(_item_transformer(decrypt_method)(value)) + ) + + def decrypt_multi_get(decrypt_method, crypto_config_method, read_method, **kwargs): # type: (Callable, Callable, Callable, **Any) -> Dict # TODO: narrow this down @@ -186,11 +203,9 @@ def decrypt_multi_get(decrypt_method, crypto_config_method, read_method, **kwarg validate_get_arguments(kwargs) crypto_config, ddb_kwargs = crypto_config_method(**kwargs) response = read_method(**ddb_kwargs) - for pos in range(len(response["Items"])): - response["Items"][pos] = decrypt_method( - item=response["Items"][pos], - crypto_config=crypto_config.with_item(_item_transformer(decrypt_method)(response["Items"][pos])), - ) + response["Items"] = list( + decrypt_list_of_items(crypto_config=crypto_config, decrypt_method=decrypt_method, items=response["Items"]) + ) return response From 200c5a6d4ecd8c2b043d83e6fa99561344c4194c Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Mon, 26 Aug 2019 19:28:02 -0700 Subject: [PATCH 07/10] update changelog with EncryptedPaginator fix --- CHANGELOG.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d65af4ad..eb9ca779 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,14 @@ Changelog ********* +1.1.1 -- 2019-08-xx +=================== + +Bugfixes +-------- +* Fix :class:`EncryptedPaginator` to successfully decrypt when using :class:`AwsKmsCryptographicMaterialsProvider` + `#105 `_ + 1.1.0 -- 2019-03-13 =================== From e3a96addffd33a06634d5cecddfe71d789bb97cd Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Tue, 27 Aug 2019 12:08:17 -0700 Subject: [PATCH 08/10] fix link name in changelog --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index eb9ca779..812e345a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,7 +8,7 @@ Changelog Bugfixes -------- * Fix :class:`EncryptedPaginator` to successfully decrypt when using :class:`AwsKmsCryptographicMaterialsProvider` - `#105 `_ + `#118 `_ 1.1.0 -- 2019-03-13 =================== From 51f3da447c265ff54fb9547bd7da15248967abbe Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Tue, 27 Aug 2019 15:53:11 -0700 Subject: [PATCH 09/10] verify that attributes are always set for all non-KMS CMPs --- test/functional/encrypted/test_item.py | 2 +- test/functional/functional_test_utils.py | 69 +++++++++++++++++++++--- 2 files changed, 63 insertions(+), 8 deletions(-) diff --git a/test/functional/encrypted/test_item.py b/test/functional/encrypted/test_item.py index c51e4851..f4c411ad 100644 --- a/test/functional/encrypted/test_item.py +++ b/test/functional/encrypted/test_item.py @@ -38,7 +38,7 @@ def pytest_generate_tests(metafunc): set_parametrized_actions(metafunc) - set_parametrized_cmp(metafunc) + set_parametrized_cmp(metafunc, require_attributes=False) set_parametrized_item(metafunc) diff --git a/test/functional/functional_test_utils.py b/test/functional/functional_test_utils.py index a5192c83..b1237af4 100644 --- a/test/functional/functional_test_utils.py +++ b/test/functional/functional_test_utils.py @@ -34,12 +34,14 @@ from dynamodb_encryption_sdk.encrypted.table import EncryptedTable from dynamodb_encryption_sdk.identifiers import CryptoAction from dynamodb_encryption_sdk.internal.identifiers import ReservedAttributes +from dynamodb_encryption_sdk.material_providers import CryptographicMaterialsProvider from dynamodb_encryption_sdk.material_providers.most_recent import MostRecentProvider from dynamodb_encryption_sdk.material_providers.static import StaticCryptographicMaterialsProvider from dynamodb_encryption_sdk.material_providers.store.meta import MetaStore from dynamodb_encryption_sdk.material_providers.wrapped import WrappedCryptographicMaterialsProvider +from dynamodb_encryption_sdk.materials import CryptographicMaterials from dynamodb_encryption_sdk.materials.raw import RawDecryptionMaterials, RawEncryptionMaterials -from dynamodb_encryption_sdk.structures import AttributeActions +from dynamodb_encryption_sdk.structures import AttributeActions, EncryptionContext from dynamodb_encryption_sdk.transform import ddb_to_dict, dict_to_ddb RUNNING_IN_TRAVIS = "TRAVIS" in os.environ @@ -164,6 +166,38 @@ def table_with_global_secondary_indexes(): mock_dynamodb2().stop() +class PassThroughCryptographicMaterialsProviderThatRequiresAttributes(CryptographicMaterialsProvider): + """Cryptographic materials provider that passes through to another, but requires that attributes are set. + + If the EncryptionContext passed to decryption_materials or encryption_materials + ever does not have attributes set, + a ValueError is raised. + Otherwise, it passes through to the passthrough CMP normally. + """ + + def __init__(self, passthrough_cmp): + self._passthrough_cmp = passthrough_cmp + + def _assert_attributes_set(self, encryption_context): + # type: (EncryptionContext) -> None + if not encryption_context.attributes: + raise ValueError("Encryption context attributes MUST be set!") + + def decryption_materials(self, encryption_context): + # type: (EncryptionContext) -> CryptographicMaterials + self._assert_attributes_set(encryption_context) + return self._passthrough_cmp.decryption_materials(encryption_context) + + def encryption_materials(self, encryption_context): + # type: (EncryptionContext) -> CryptographicMaterials + self._assert_attributes_set(encryption_context) + return self._passthrough_cmp.encryption_materials(encryption_context) + + def refresh(self): + # type: () -> None + self._passthrough_cmp.refresh() + + def _get_from_cache(dk_class, algorithm, key_length): """Don't generate new keys every time. All we care about is that they are valid keys, not that they are unique.""" try: @@ -221,8 +255,15 @@ def _some_algorithm_pairs(): _cmp_builders = {"static": build_static_jce_cmp, "wrapped": _build_wrapped_jce_cmp} -def _all_possible_cmps(algorithm_generator): - """Generate all possible cryptographic materials providers based on the supplied generator.""" +def _all_possible_cmps(algorithm_generator, require_attributes): + """Generate all possible cryptographic materials providers based on the supplied generator. + + require_attributes determines whether the CMP will be wrapped in + PassThroughCryptographicMaterialsProviderThatRequiresAttributes + to require that attributes are set on every request. + This should ONLY be disabled on the item encryptor tests. + All high-level helper clients MUST set the attributes before passing the encryption context down. + """ # The AES combinations do the same thing, but this makes sure that the AESWrap name works as expected. yield _build_wrapped_jce_cmp("AESWrap", 256, "HmacSHA256", 256) @@ -242,17 +283,31 @@ def _all_possible_cmps(algorithm_generator): sig_key_length=signing_key_length, ) + inner_cmp = builder_func(encryption_algorithm, encryption_key_length, signing_algorithm, signing_key_length) + + if require_attributes: + outer_cmp = PassThroughCryptographicMaterialsProviderThatRequiresAttributes(inner_cmp) + else: + outer_cmp = inner_cmp + yield pytest.param( - builder_func(encryption_algorithm, encryption_key_length, signing_algorithm, signing_key_length), + outer_cmp, id=id_string, ) -def set_parametrized_cmp(metafunc): - """Set paramatrized values for cryptographic materials providers.""" +def set_parametrized_cmp(metafunc, require_attributes=True): + """Set paramatrized values for cryptographic materials providers. + + require_attributes determines whether the CMP will be wrapped in + PassThroughCryptographicMaterialsProviderThatRequiresAttributes + to require that attributes are set on every request. + This should ONLY be disabled on the item encryptor tests. + All high-level helper clients MUST set the attributes before passing the encryption context down. + """ for name, algorithm_generator in (("all_the_cmps", _all_algorithm_pairs), ("some_cmps", _some_algorithm_pairs)): if name in metafunc.fixturenames: - metafunc.parametrize(name, _all_possible_cmps(algorithm_generator)) + metafunc.parametrize(name, _all_possible_cmps(algorithm_generator, require_attributes)) _ACTIONS = { From 12ea0e8b185b804573b061b2d09baaa25950df70 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Tue, 27 Aug 2019 16:20:20 -0700 Subject: [PATCH 10/10] verify that attributes are always set for all KMS CMPs --- test/functional/functional_test_utils.py | 5 +--- test/integration/encrypted/test_client.py | 15 +++++----- test/integration/encrypted/test_resource.py | 7 +++-- test/integration/encrypted/test_table.py | 7 +++-- test/integration/integration_test_utils.py | 12 ++++++-- .../material_providers/test_aws_kms.py | 30 +++++++++++-------- 6 files changed, 44 insertions(+), 32 deletions(-) diff --git a/test/functional/functional_test_utils.py b/test/functional/functional_test_utils.py index b1237af4..950aee47 100644 --- a/test/functional/functional_test_utils.py +++ b/test/functional/functional_test_utils.py @@ -290,10 +290,7 @@ def _all_possible_cmps(algorithm_generator, require_attributes): else: outer_cmp = inner_cmp - yield pytest.param( - outer_cmp, - id=id_string, - ) + yield pytest.param(outer_cmp, id=id_string) def set_parametrized_cmp(metafunc, require_attributes=True): diff --git a/test/integration/encrypted/test_client.py b/test/integration/encrypted/test_client.py index 7b8de423..1d72e122 100644 --- a/test/integration/encrypted/test_client.py +++ b/test/integration/encrypted/test_client.py @@ -14,9 +14,9 @@ import pytest from ..integration_test_utils import ( # noqa pylint: disable=unused-import - aws_kms_cmp, ddb_table_name, functional_test_utils, + set_parameterized_kms_cmps, ) pytestmark = [pytest.mark.integ, pytest.mark.ddb_integ] @@ -26,6 +26,7 @@ def pytest_generate_tests(metafunc): functional_test_utils.set_parametrized_actions(metafunc) functional_test_utils.set_parametrized_cmp(metafunc) functional_test_utils.set_parametrized_item(metafunc) + set_parameterized_kms_cmps(metafunc) def test_ephemeral_item_cycle(ddb_table_name, some_cmps, parametrized_actions, parametrized_item): @@ -35,10 +36,10 @@ def test_ephemeral_item_cycle(ddb_table_name, some_cmps, parametrized_actions, p ) -def test_ephemeral_item_cycle_kms(ddb_table_name, aws_kms_cmp, parametrized_actions, parametrized_item): +def test_ephemeral_item_cycle_kms(ddb_table_name, all_aws_kms_cmps, parametrized_actions, parametrized_item): """Test the AWS KMS CMP against a small number of curated items.""" functional_test_utils.client_cycle_single_item_check( - aws_kms_cmp, parametrized_actions, parametrized_item, ddb_table_name + all_aws_kms_cmps, parametrized_actions, parametrized_item, ddb_table_name ) @@ -49,10 +50,10 @@ def test_ephemeral_batch_item_cycle(ddb_table_name, some_cmps, parametrized_acti ) -def test_ephemeral_batch_item_cycle_kms(ddb_table_name, aws_kms_cmp, parametrized_actions, parametrized_item): +def test_ephemeral_batch_item_cycle_kms(ddb_table_name, all_aws_kms_cmps, parametrized_actions, parametrized_item): """Test the AWS KMS CMP against a small number of curated items.""" functional_test_utils.client_cycle_batch_items_check( - aws_kms_cmp, parametrized_actions, parametrized_item, ddb_table_name + all_aws_kms_cmps, parametrized_actions, parametrized_item, ddb_table_name ) @@ -64,9 +65,9 @@ def test_ephemeral_batch_item_cycle_scan_paginator(ddb_table_name, some_cmps, pa def test_ephemeral_batch_item_cycle_scan_paginator_kms( - ddb_table_name, aws_kms_cmp, parametrized_actions, parametrized_item + ddb_table_name, all_aws_kms_cmps, parametrized_actions, parametrized_item ): """Test a the AWS KMS CMP against a small number of curated items using the scan paginator.""" functional_test_utils.client_cycle_batch_items_check_scan_paginator( - aws_kms_cmp, parametrized_actions, parametrized_item, ddb_table_name + all_aws_kms_cmps, parametrized_actions, parametrized_item, ddb_table_name ) diff --git a/test/integration/encrypted/test_resource.py b/test/integration/encrypted/test_resource.py index a96ef764..1c84cd81 100644 --- a/test/integration/encrypted/test_resource.py +++ b/test/integration/encrypted/test_resource.py @@ -14,9 +14,9 @@ import pytest from ..integration_test_utils import ( # noqa pylint: disable=unused-import - aws_kms_cmp, ddb_table_name, functional_test_utils, + set_parameterized_kms_cmps, ) pytestmark = [pytest.mark.integ, pytest.mark.ddb_integ] @@ -26,6 +26,7 @@ def pytest_generate_tests(metafunc): functional_test_utils.set_parametrized_actions(metafunc) functional_test_utils.set_parametrized_cmp(metafunc) functional_test_utils.set_parametrized_item(metafunc) + set_parameterized_kms_cmps(metafunc) def test_ephemeral_batch_item_cycle(ddb_table_name, some_cmps, parametrized_actions, parametrized_item): @@ -35,8 +36,8 @@ def test_ephemeral_batch_item_cycle(ddb_table_name, some_cmps, parametrized_acti ) -def test_ephemeral_batch_item_cycle_kms(ddb_table_name, aws_kms_cmp, parametrized_actions, parametrized_item): +def test_ephemeral_batch_item_cycle_kms(ddb_table_name, all_aws_kms_cmps, parametrized_actions, parametrized_item): """Test the AWS KMS CMP against a small number of curated items.""" functional_test_utils.resource_cycle_batch_items_check( - aws_kms_cmp, parametrized_actions, parametrized_item, ddb_table_name + all_aws_kms_cmps, parametrized_actions, parametrized_item, ddb_table_name ) diff --git a/test/integration/encrypted/test_table.py b/test/integration/encrypted/test_table.py index 5bb17661..f857f7be 100644 --- a/test/integration/encrypted/test_table.py +++ b/test/integration/encrypted/test_table.py @@ -14,9 +14,9 @@ import pytest from ..integration_test_utils import ( # noqa pylint: disable=unused-import - aws_kms_cmp, ddb_table_name, functional_test_utils, + set_parameterized_kms_cmps, ) pytestmark = [pytest.mark.integ, pytest.mark.ddb_integ] @@ -26,6 +26,7 @@ def pytest_generate_tests(metafunc): functional_test_utils.set_parametrized_actions(metafunc) functional_test_utils.set_parametrized_cmp(metafunc) functional_test_utils.set_parametrized_item(metafunc) + set_parameterized_kms_cmps(metafunc) def test_ephemeral_item_cycle(ddb_table_name, some_cmps, parametrized_actions, parametrized_item): @@ -40,6 +41,6 @@ def test_ephemeral_item_cycle_batch_writer(ddb_table_name, some_cmps, parametriz ) -def test_ephemeral_item_cycle_kms(ddb_table_name, aws_kms_cmp, parametrized_actions, parametrized_item): +def test_ephemeral_item_cycle_kms(ddb_table_name, all_aws_kms_cmps, parametrized_actions, parametrized_item): """Test the AWS KMS CMP against a small number of curated items.""" - functional_test_utils.table_cycle_check(aws_kms_cmp, parametrized_actions, parametrized_item, ddb_table_name) + functional_test_utils.table_cycle_check(all_aws_kms_cmps, parametrized_actions, parametrized_item, ddb_table_name) diff --git a/test/integration/integration_test_utils.py b/test/integration/integration_test_utils.py index c5b24ea2..63aa5ca5 100644 --- a/test/integration/integration_test_utils.py +++ b/test/integration/integration_test_utils.py @@ -48,9 +48,15 @@ def cmk_arn(): return cmk_arn_value() -@pytest.fixture -def aws_kms_cmp(): - return AwsKmsCryptographicMaterialsProvider(key_id=cmk_arn_value()) +def set_parameterized_kms_cmps(metafunc, require_attributes=True): + inner_cmp = AwsKmsCryptographicMaterialsProvider(key_id=cmk_arn_value()) + if require_attributes: + outer_cmp = functional_test_utils.PassThroughCryptographicMaterialsProviderThatRequiresAttributes(inner_cmp) + else: + outer_cmp = inner_cmp + + if "all_aws_kms_cmps" in metafunc.fixturenames: + metafunc.parametrize("all_aws_kms_cmps", (pytest.param(outer_cmp, id="Standard KMS CMP"),)) @pytest.fixture diff --git a/test/integration/material_providers/test_aws_kms.py b/test/integration/material_providers/test_aws_kms.py index 0a6e93d9..059ad6dd 100644 --- a/test/integration/material_providers/test_aws_kms.py +++ b/test/integration/material_providers/test_aws_kms.py @@ -23,8 +23,7 @@ from dynamodb_encryption_sdk.structures import AttributeActions, EncryptionContext from dynamodb_encryption_sdk.transform import dict_to_ddb -from ..integration_test_utils import aws_kms_cmp # noqa pylint: disable=unused-import -from ..integration_test_utils import functional_test_utils, hypothesis_strategies +from ..integration_test_utils import functional_test_utils, hypothesis_strategies, set_parameterized_kms_cmps pytestmark = pytest.mark.integ @@ -34,12 +33,13 @@ def pytest_generate_tests(metafunc): functional_test_utils.set_parametrized_actions(metafunc) functional_test_utils.set_parametrized_item(metafunc) + set_parameterized_kms_cmps(metafunc, require_attributes=False) -def test_verify_user_agent(aws_kms_cmp, caplog): +def test_verify_user_agent(all_aws_kms_cmps, caplog): caplog.set_level(level=logging.DEBUG) - aws_kms_cmp.encryption_materials(EncryptionContext()) + all_aws_kms_cmps.encryption_materials(EncryptionContext()) assert USER_AGENT_SUFFIX in caplog.text @@ -54,10 +54,10 @@ def _many_items(): @pytest.mark.parametrize("item", _many_items()) -def test_aws_kms_diverse_indexes(aws_kms_cmp, item): +def test_aws_kms_diverse_indexes(all_aws_kms_cmps, item): """Verify that AWS KMS cycle works for items with all possible combinations for primary index attribute types.""" crypto_config = CryptoConfig( - materials_provider=aws_kms_cmp, + materials_provider=all_aws_kms_cmps, encryption_context=EncryptionContext( partition_key_name="partition_key", sort_key_name="sort_key", attributes=dict_to_ddb(item) ), @@ -68,9 +68,11 @@ def test_aws_kms_diverse_indexes(aws_kms_cmp, item): functional_test_utils.cycle_item_check(item, crypto_config) -def test_aws_kms_item_cycle(aws_kms_cmp, parametrized_actions, parametrized_item): +def test_aws_kms_item_cycle(all_aws_kms_cmps, parametrized_actions, parametrized_item): crypto_config = CryptoConfig( - materials_provider=aws_kms_cmp, encryption_context=EncryptionContext(), attribute_actions=parametrized_actions + materials_provider=all_aws_kms_cmps, + encryption_context=EncryptionContext(), + attribute_actions=parametrized_actions, ) functional_test_utils.cycle_item_check(parametrized_item, crypto_config) @@ -78,9 +80,11 @@ def test_aws_kms_item_cycle(aws_kms_cmp, parametrized_actions, parametrized_item @pytest.mark.slow @hypothesis_strategies.SLOW_SETTINGS @hypothesis.given(item=hypothesis_strategies.ddb_items) -def test_aws_kms_item_cycle_hypothesis_slow(aws_kms_cmp, hypothesis_actions, item): +def test_aws_kms_item_cycle_hypothesis_slow(all_aws_kms_cmps, hypothesis_actions, item): crypto_config = CryptoConfig( - materials_provider=aws_kms_cmp, encryption_context=EncryptionContext(), attribute_actions=hypothesis_actions + materials_provider=all_aws_kms_cmps, + encryption_context=EncryptionContext(), + attribute_actions=hypothesis_actions, ) functional_test_utils.cycle_item_check(item, crypto_config) @@ -88,8 +92,10 @@ def test_aws_kms_item_cycle_hypothesis_slow(aws_kms_cmp, hypothesis_actions, ite @pytest.mark.veryslow @hypothesis_strategies.VERY_SLOW_SETTINGS @hypothesis.given(item=hypothesis_strategies.ddb_items) -def test_aws_kms_item_cycle_hypothesis_veryslow(aws_kms_cmp, hypothesis_actions, item): +def test_aws_kms_item_cycle_hypothesis_veryslow(all_aws_kms_cmps, hypothesis_actions, item): crypto_config = CryptoConfig( - materials_provider=aws_kms_cmp, encryption_context=EncryptionContext(), attribute_actions=hypothesis_actions + materials_provider=all_aws_kms_cmps, + encryption_context=EncryptionContext(), + attribute_actions=hypothesis_actions, ) functional_test_utils.cycle_item_check(item, crypto_config)