From 3036f3b37124d8cd783f9d0a1c853e65003a80f6 Mon Sep 17 00:00:00 2001 From: Gary Page-Wood Date: Fri, 18 Jan 2019 14:42:30 +1300 Subject: [PATCH 1/7] Replace unprocessed items in batch write response with the original plaintext versions --- src/dynamodb_encryption_sdk/internal/utils.py | 91 +++++++++++++- test/functional/internal/test_utils.py | 119 ++++++++++++++++++ 2 files changed, 206 insertions(+), 4 deletions(-) create mode 100644 test/functional/internal/test_utils.py diff --git a/src/dynamodb_encryption_sdk/internal/utils.py b/src/dynamodb_encryption_sdk/internal/utils.py index bb133766..589b73df 100644 --- a/src/dynamodb_encryption_sdk/internal/utils.py +++ b/src/dynamodb_encryption_sdk/internal/utils.py @@ -16,17 +16,20 @@ No guarantee is provided on the modules and APIs within this namespace staying consistent. Directly reference at your own risk. """ +import copy +from functools import partial + import attr import botocore.client from dynamodb_encryption_sdk.encrypted import CryptoConfig from dynamodb_encryption_sdk.encrypted.item import decrypt_python_item, encrypt_python_item from dynamodb_encryption_sdk.exceptions import InvalidArgumentError -from dynamodb_encryption_sdk.structures import EncryptionContext, TableInfo +from dynamodb_encryption_sdk.structures import CryptoAction, EncryptionContext, TableInfo from dynamodb_encryption_sdk.transform import dict_to_ddb try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Any, Callable, Dict, Text # noqa pylint: disable=unused-import + from typing import Any, Bool, Callable, Dict, Text # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass @@ -271,19 +274,22 @@ def encrypt_batch_write_item(encrypt_method, crypto_config_method, write_method, """Transparently encrypt multiple items before putting them in a batch request. :param callable encrypt_method: Method to use to encrypt items - :param callable crypto_config_method: Method that accepts ``kwargs`` and provides a :class:`CryptoConfig` + :param callable crypto_config_method: Method that accepts a table name string and provides a :class:`CryptoConfig` :param callable write_method: Method that writes to the table :param **kwargs: Keyword arguments to pass to ``write_method`` :return: DynamoDB response :rtype: dict """ request_crypto_config = kwargs.pop("crypto_config", None) + table_cryptos = {} + plaintext_items = copy.deepcopy(kwargs["RequestItems"]) for table_name, items in kwargs["RequestItems"].items(): if request_crypto_config is not None: crypto_config = request_crypto_config else: crypto_config = crypto_config_method(table_name=table_name) + table_cryptos[table_name] = crypto_config for pos, value in enumerate(items): for request_type, item in value.items(): @@ -293,4 +299,81 @@ def encrypt_batch_write_item(encrypt_method, crypto_config_method, write_method, item=item["Item"], crypto_config=crypto_config.with_item(_item_transformer(encrypt_method)(item["Item"])), ) - return write_method(**kwargs) + + response = write_method(**kwargs) + return _process_batch_write_response(plaintext_items, response, table_cryptos) + + +def _process_batch_write_response(request, response, table_crypto_config): + # type: (Dict, Dict, Dict[Text, CryptoConfig]) -> Dict + """Handle unprocessed items in the response from a transparently encrypted write. + + :param dict request: The DynamoDB plaintext request dictionary + :param dict response: The DynamoDB response from the batch operation + :param Dict[Text, CryptoConfig] table_crypto_config: table level CryptoConfig used in encrypting the request items + :return: DynamoDB response, with any unprocessed items reverted back to the original plaintext values + :rtype: dict + """ + if not (response and response.get("UnprocessedItems")): + return response + + # Unprocessed items need to be returned in their original state + for table_name, unprocessed in response["UnprocessedItems"].items(): + original_items = request[table_name] + crypto_config = table_crypto_config[table_name] + + if crypto_config.encryption_context.partition_key_name: + items_match = partial(_item_keys_match, crypto_config) + else: + items_match = partial(_item_attributes_match, crypto_config) + + for pos, operation in enumerate(unprocessed): + for request_type, item in operation.items(): + for plaintext_item in original_items: + if plaintext_item.get(request_type) and items_match( + plaintext_item[request_type]["Item"], item["Item"] + ): + unprocessed[pos] = plaintext_item.copy() + break + + return response + + +def _item_keys_match(crypto_config, item1, item2): + # type: (CryptoConfig, Dict, Dict) -> Bool + """Determines whether the values in the primary and sort keys (if they exist) are the same + + :param CryptoConfig crypto_config: CryptoConfig used in encrypting the given items + :param dict item1: The first item to compare + :param dict item2: The second item to compare + :return: Bool response, True if the key attributes match + :rtype: bool + """ + encryption_context = crypto_config.encryption_context + + return item1[encryption_context.partition_key_name] == item2[encryption_context.partition_key_name] \ + and item1.get(encryption_context.sort_key_name) == item2.get(encryption_context.sort_key_name) + + +def _item_attributes_match(crypto_config, plaintext_item, encrypted_item): + # type: (CryptoConfig, Dict, Dict) -> Bool + """Determines whether the unencrypted values in the plaintext items attributes are the same as those in the + encrypted item. Essentially this uses brute force to cover when we don't know the primary and sort + index attribute names, since they can't be encrypted. + + :param CryptoConfig crypto_config: CryptoConfig used in encrypting the given items + :param dict plaintext_item: The plaintext item + :param dict encrypted_item: The encrypted item + :return: Bool response, True if the unencrypted attributes in the plaintext item match those in + the encrypted item + :rtype: bool + """ + + for name, value in plaintext_item.items(): + if crypto_config.attribute_actions.action(name) != CryptoAction.DO_NOTHING: + continue + + if encrypted_item.get(name) != value: + return False + + return True diff --git a/test/functional/internal/test_utils.py b/test/functional/internal/test_utils.py new file mode 100644 index 00000000..dbb05d66 --- /dev/null +++ b/test/functional/internal/test_utils.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Test suite for ``dynamodb_encryption_sdk.internal.utils``.""" +import copy + +import pytest + +from dynamodb_encryption_sdk.encrypted import CryptoConfig +from dynamodb_encryption_sdk.identifiers import CryptoAction +from dynamodb_encryption_sdk.internal.utils import encrypt_batch_write_item +from dynamodb_encryption_sdk.material_providers import CryptographicMaterialsProvider +from dynamodb_encryption_sdk.structures import AttributeActions, EncryptionContext +from dynamodb_encryption_sdk.transform import ddb_to_dict + +from ..functional_test_vector_generators import attribute_test_vectors + + +def get_test_item(standard_dict_format, partition_key, sort_key): + attributes = attribute_test_vectors("serialize") + + attributes = {"attr_" + str(pos): attribute[0] for pos, attribute in enumerate(attributes)} + attributes["partition-key"] = {"S": partition_key} + if sort_key: + attributes["sort-key"] = {"S": sort_key} + + if standard_dict_format: + attributes = ddb_to_dict(attributes) + return attributes + + +def get_test_items(standard_dict_format, table_name="table"): + items = [ + get_test_item(standard_dict_format, partition_key="key-1", sort_key=None), + get_test_item(standard_dict_format, partition_key="key-2", sort_key=None), + get_test_item(standard_dict_format, partition_key="key-3", sort_key="sort-1"), + get_test_item(standard_dict_format, partition_key="key-4", sort_key="sort-1"), + get_test_item(standard_dict_format, partition_key="key-4", sort_key="sort-2"), + ] + + for pos, item in enumerate(items): + item["encrypt-me"] = table_name + str(pos) + + return {table_name: [{"PutRequest": {"Item": item}} for item in items]} + + +def get_dummy_crypto_config(partition_key_name, sort_key_name, encrypted_attribute_name): + context = EncryptionContext(partition_key_name=partition_key_name, sort_key_name=sort_key_name) + actions = AttributeActions( + default_action=CryptoAction.DO_NOTHING, + attribute_actions={encrypted_attribute_name: CryptoAction.ENCRYPT_AND_SIGN}, + ) + materials = CryptographicMaterialsProvider() + return CryptoConfig(materials_provider=materials, encryption_context=context, attribute_actions=actions) + + +def check_encrypt_batch_write_item_call(request_items, crypto_config, encrypted_attribute_name): + def dummy_encrypt(item, **kwargs): + result = item.copy() + result[encrypted_attribute_name] = "pretend Im encrypted" + return result + + result = encrypt_batch_write_item( + encrypt_method=dummy_encrypt, + write_method=lambda **kwargs: {"UnprocessedItems": kwargs["RequestItems"]}, + crypto_config_method=lambda **kwargs: crypto_config, + RequestItems=copy.deepcopy(request_items), + ) + + # assert the returned items equal the submitted items + unprocessed = result["UnprocessedItems"] + + assert unprocessed == request_items + + +@pytest.mark.parametrize( + "items", + ( + (get_test_items(standard_dict_format=True)), + (get_test_items(standard_dict_format=False)) + ) +) +def test_encrypt_batch_write_returns_plaintext_unprocessed_items_with_known_keys(items): + crypto_config = get_dummy_crypto_config("partition-key", "sort-key", encrypted_attribute_name="encrypt-me") + + check_encrypt_batch_write_item_call(items, crypto_config, encrypted_attribute_name="encrypt-me") + + +@pytest.mark.parametrize( + "items", + ( + (get_test_items(standard_dict_format=True)), + (get_test_items(standard_dict_format=False)) + ) +) +def test_encrypt_batch_write_returns_plaintext_unprocessed_items_with_unknown_keys(items): + crypto_config = get_dummy_crypto_config(None, None, encrypted_attribute_name="encrypt-me") + + check_encrypt_batch_write_item_call(items, crypto_config, encrypted_attribute_name="encrypt-me") + + +def test_encrypt_batch_write_returns_plaintext_unprocessed_items_over_multiple_tables(): + crypto_config = get_dummy_crypto_config("partition-key", "sort-key", encrypted_attribute_name="encrypt-me") + + items = get_test_items(False, "table-one") + more_items = get_test_items(False, "table-two") + items.update(more_items) + + check_encrypt_batch_write_item_call(items, crypto_config, encrypted_attribute_name="encrypt-me") From 8bb2054baf7ff868b09a78217f22bfde627be1f5 Mon Sep 17 00:00:00 2001 From: Gary Page-Wood Date: Tue, 22 Jan 2019 16:21:06 +1300 Subject: [PATCH 2/7] Code review changes --- src/dynamodb_encryption_sdk/internal/utils.py | 28 +++-- test/functional/internal/test_utils.py | 112 ++++++++++++------ 2 files changed, 96 insertions(+), 44 deletions(-) diff --git a/src/dynamodb_encryption_sdk/internal/utils.py b/src/dynamodb_encryption_sdk/internal/utils.py index 589b73df..93bdbca3 100644 --- a/src/dynamodb_encryption_sdk/internal/utils.py +++ b/src/dynamodb_encryption_sdk/internal/utils.py @@ -281,7 +281,7 @@ def encrypt_batch_write_item(encrypt_method, crypto_config_method, write_method, :rtype: dict """ request_crypto_config = kwargs.pop("crypto_config", None) - table_cryptos = {} + table_crypto_configs = {} plaintext_items = copy.deepcopy(kwargs["RequestItems"]) for table_name, items in kwargs["RequestItems"].items(): @@ -289,7 +289,7 @@ def encrypt_batch_write_item(encrypt_method, crypto_config_method, write_method, crypto_config = request_crypto_config else: crypto_config = crypto_config_method(table_name=table_name) - table_cryptos[table_name] = crypto_config + table_crypto_configs[table_name] = crypto_config for pos, value in enumerate(items): for request_type, item in value.items(): @@ -301,7 +301,7 @@ def encrypt_batch_write_item(encrypt_method, crypto_config_method, write_method, ) response = write_method(**kwargs) - return _process_batch_write_response(plaintext_items, response, table_cryptos) + return _process_batch_write_response(plaintext_items, response, table_crypto_configs) def _process_batch_write_response(request, response, table_crypto_config): @@ -314,11 +314,13 @@ def _process_batch_write_response(request, response, table_crypto_config): :return: DynamoDB response, with any unprocessed items reverted back to the original plaintext values :rtype: dict """ - if not (response and response.get("UnprocessedItems")): + try: + unprocessed_items = response["UnprocessedItems"] + except KeyError: return response # Unprocessed items need to be returned in their original state - for table_name, unprocessed in response["UnprocessedItems"].items(): + for table_name, unprocessed in unprocessed_items.items(): original_items = request[table_name] crypto_config = table_crypto_config[table_name] @@ -329,6 +331,9 @@ def _process_batch_write_response(request, response, table_crypto_config): for pos, operation in enumerate(unprocessed): for request_type, item in operation.items(): + if request_type != "PutRequest": + continue + for plaintext_item in original_items: if plaintext_item.get(request_type) and items_match( plaintext_item[request_type]["Item"], item["Item"] @@ -349,10 +354,15 @@ def _item_keys_match(crypto_config, item1, item2): :return: Bool response, True if the key attributes match :rtype: bool """ - encryption_context = crypto_config.encryption_context + partition_key_name = crypto_config.encryption_context.partition_key_name + sort_key_name = crypto_config.encryption_context.sort_key_name + + partition_keys_match = item1[partition_key_name] == item2[partition_key_name] + + if sort_key_name is None: + return partition_keys_match - return item1[encryption_context.partition_key_name] == item2[encryption_context.partition_key_name] \ - and item1.get(encryption_context.sort_key_name) == item2.get(encryption_context.sort_key_name) + return partition_keys_match and item1[sort_key_name] == item2[sort_key_name] def _item_attributes_match(crypto_config, plaintext_item, encrypted_item): @@ -370,7 +380,7 @@ def _item_attributes_match(crypto_config, plaintext_item, encrypted_item): """ for name, value in plaintext_item.items(): - if crypto_config.attribute_actions.action(name) != CryptoAction.DO_NOTHING: + if crypto_config.attribute_actions.action(name) == CryptoAction.ENCRYPT_AND_SIGN: continue if encrypted_item.get(name) != value: diff --git a/test/functional/internal/test_utils.py b/test/functional/internal/test_utils.py index dbb05d66..c1c20575 100644 --- a/test/functional/internal/test_utils.py +++ b/test/functional/internal/test_utils.py @@ -15,38 +15,44 @@ import copy import pytest +from mock import Mock from dynamodb_encryption_sdk.encrypted import CryptoConfig from dynamodb_encryption_sdk.identifiers import CryptoAction from dynamodb_encryption_sdk.internal.utils import encrypt_batch_write_item from dynamodb_encryption_sdk.material_providers import CryptographicMaterialsProvider from dynamodb_encryption_sdk.structures import AttributeActions, EncryptionContext -from dynamodb_encryption_sdk.transform import ddb_to_dict +from dynamodb_encryption_sdk.transform import dict_to_ddb -from ..functional_test_vector_generators import attribute_test_vectors +from ..functional_test_utils import diverse_item -def get_test_item(standard_dict_format, partition_key, sort_key): - attributes = attribute_test_vectors("serialize") +def get_test_item(standard_dict_format, partition_key, sort_key=None): + attributes = diverse_item() attributes = {"attr_" + str(pos): attribute[0] for pos, attribute in enumerate(attributes)} - attributes["partition-key"] = {"S": partition_key} - if sort_key: - attributes["sort-key"] = {"S": sort_key} + attributes["partition-key"] = partition_key + if sort_key is not None: + attributes["sort-key"] = sort_key - if standard_dict_format: - attributes = ddb_to_dict(attributes) + if not standard_dict_format: + attributes = dict_to_ddb(attributes) return attributes -def get_test_items(standard_dict_format, table_name="table"): - items = [ - get_test_item(standard_dict_format, partition_key="key-1", sort_key=None), - get_test_item(standard_dict_format, partition_key="key-2", sort_key=None), - get_test_item(standard_dict_format, partition_key="key-3", sort_key="sort-1"), - get_test_item(standard_dict_format, partition_key="key-4", sort_key="sort-1"), - get_test_item(standard_dict_format, partition_key="key-4", sort_key="sort-2"), - ] +def get_test_items(standard_dict_format, table_name="table", with_sort_keys=False): + + if with_sort_keys: + items = [ + get_test_item(standard_dict_format, partition_key="key-1", sort_key="sort-1"), + get_test_item(standard_dict_format, partition_key="key-2", sort_key="sort-1"), + get_test_item(standard_dict_format, partition_key="key-2", sort_key="sort-2"), + ] + else: + items = [ + get_test_item(standard_dict_format, partition_key="key-1"), + get_test_item(standard_dict_format, partition_key="key-2"), + ] for pos, item in enumerate(items): item["encrypt-me"] = table_name + str(pos) @@ -54,22 +60,30 @@ def get_test_items(standard_dict_format, table_name="table"): return {table_name: [{"PutRequest": {"Item": item}} for item in items]} -def get_dummy_crypto_config(partition_key_name, sort_key_name, encrypted_attribute_name): +def get_dummy_crypto_config(partition_key_name=None, sort_key_name=None, sign_keys=False): context = EncryptionContext(partition_key_name=partition_key_name, sort_key_name=sort_key_name) actions = AttributeActions( default_action=CryptoAction.DO_NOTHING, - attribute_actions={encrypted_attribute_name: CryptoAction.ENCRYPT_AND_SIGN}, + attribute_actions={ + "encrypt-me": CryptoAction.ENCRYPT_AND_SIGN + }, ) - materials = CryptographicMaterialsProvider() + if sign_keys: + actions.attribute_actions['partition-key'] = CryptoAction.SIGN_ONLY + if sort_key_name is not None: + actions.attribute_actions['sort-key'] = CryptoAction.SIGN_ONLY + + materials = Mock(spec=CryptographicMaterialsProvider) # type: CryptographicMaterialsProvider return CryptoConfig(materials_provider=materials, encryption_context=context, attribute_actions=actions) -def check_encrypt_batch_write_item_call(request_items, crypto_config, encrypted_attribute_name): +def check_encrypt_batch_write_item_call(request_items, crypto_config): def dummy_encrypt(item, **kwargs): result = item.copy() - result[encrypted_attribute_name] = "pretend Im encrypted" + result['encrypt-me'] = "pretend Im encrypted" return result + # execute a batch write, but make the write method return ALL the provided items as unprocessed result = encrypt_batch_write_item( encrypt_method=dummy_encrypt, write_method=lambda **kwargs: {"UnprocessedItems": kwargs["RequestItems"]}, @@ -86,34 +100,62 @@ def dummy_encrypt(item, **kwargs): @pytest.mark.parametrize( "items", ( - (get_test_items(standard_dict_format=True)), - (get_test_items(standard_dict_format=False)) + get_test_items(standard_dict_format=True), + get_test_items(standard_dict_format=False), ) ) -def test_encrypt_batch_write_returns_plaintext_unprocessed_items_with_known_keys(items): - crypto_config = get_dummy_crypto_config("partition-key", "sort-key", encrypted_attribute_name="encrypt-me") +def test_encrypt_batch_write_returns_plaintext_unprocessed_items_with_known_partition_key(items): + crypto_config = get_dummy_crypto_config("partition-key") + check_encrypt_batch_write_item_call(items, crypto_config) - check_encrypt_batch_write_item_call(items, crypto_config, encrypted_attribute_name="encrypt-me") + +@pytest.mark.parametrize( + "items", + ( + get_test_items(standard_dict_format=True, with_sort_keys=True), + get_test_items(standard_dict_format=False, with_sort_keys=True), + ) +) +def test_encrypt_batch_write_returns_plaintext_unprocessed_items_with_known_partition_and_sort_keys(items): + crypto_config = get_dummy_crypto_config("partition-key", "sort-key") + check_encrypt_batch_write_item_call(items, crypto_config) @pytest.mark.parametrize( "items", ( - (get_test_items(standard_dict_format=True)), - (get_test_items(standard_dict_format=False)) + get_test_items(standard_dict_format=True), + get_test_items(standard_dict_format=False), + get_test_items(standard_dict_format=True, with_sort_keys=True), + get_test_items(standard_dict_format=False, with_sort_keys=True), ) ) def test_encrypt_batch_write_returns_plaintext_unprocessed_items_with_unknown_keys(items): - crypto_config = get_dummy_crypto_config(None, None, encrypted_attribute_name="encrypt-me") + crypto_config = get_dummy_crypto_config(None, None) + + check_encrypt_batch_write_item_call(items, crypto_config) + + +@pytest.mark.parametrize( + "items", + ( + get_test_items(standard_dict_format=True), + get_test_items(standard_dict_format=False), + get_test_items(standard_dict_format=True, with_sort_keys=True), + get_test_items(standard_dict_format=False, with_sort_keys=True), + ) +) +def test_encrypt_batch_write_returns_plaintext_unprocessed_items_with_unknown_signed_keys(items): + crypto_config = get_dummy_crypto_config(None, None, sign_keys=True) - check_encrypt_batch_write_item_call(items, crypto_config, encrypted_attribute_name="encrypt-me") + check_encrypt_batch_write_item_call(items, crypto_config) def test_encrypt_batch_write_returns_plaintext_unprocessed_items_over_multiple_tables(): - crypto_config = get_dummy_crypto_config("partition-key", "sort-key", encrypted_attribute_name="encrypt-me") + crypto_config = get_dummy_crypto_config("partition-key", "sort-key") - items = get_test_items(False, "table-one") - more_items = get_test_items(False, "table-two") + items = get_test_items(standard_dict_format=True, table_name="table-one", with_sort_keys=True) + more_items = get_test_items(standard_dict_format=False, table_name="table-two", with_sort_keys=True) items.update(more_items) - check_encrypt_batch_write_item_call(items, crypto_config, encrypted_attribute_name="encrypt-me") + check_encrypt_batch_write_item_call(items, crypto_config) From fd63b16034ccf1edd82bbe3bf0869e40e49ac1b7 Mon Sep 17 00:00:00 2001 From: Gary Page-Wood Date: Wed, 23 Jan 2019 10:19:53 +1300 Subject: [PATCH 3/7] fix item generation bug in utils tests --- test/functional/internal/test_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/functional/internal/test_utils.py b/test/functional/internal/test_utils.py index c1c20575..e9077244 100644 --- a/test/functional/internal/test_utils.py +++ b/test/functional/internal/test_utils.py @@ -30,7 +30,6 @@ def get_test_item(standard_dict_format, partition_key, sort_key=None): attributes = diverse_item() - attributes = {"attr_" + str(pos): attribute[0] for pos, attribute in enumerate(attributes)} attributes["partition-key"] = partition_key if sort_key is not None: attributes["sort-key"] = sort_key From ec4eca1741b630b646f36cbfd2b46e9415aadf58 Mon Sep 17 00:00:00 2001 From: Gary Page-Wood Date: Wed, 23 Jan 2019 10:20:27 +1300 Subject: [PATCH 4/7] simplify utils tests a little --- test/functional/internal/test_utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/functional/internal/test_utils.py b/test/functional/internal/test_utils.py index e9077244..3357b17d 100644 --- a/test/functional/internal/test_utils.py +++ b/test/functional/internal/test_utils.py @@ -69,8 +69,7 @@ def get_dummy_crypto_config(partition_key_name=None, sort_key_name=None, sign_ke ) if sign_keys: actions.attribute_actions['partition-key'] = CryptoAction.SIGN_ONLY - if sort_key_name is not None: - actions.attribute_actions['sort-key'] = CryptoAction.SIGN_ONLY + actions.attribute_actions['sort-key'] = CryptoAction.SIGN_ONLY materials = Mock(spec=CryptographicMaterialsProvider) # type: CryptographicMaterialsProvider return CryptoConfig(materials_provider=materials, encryption_context=context, attribute_actions=actions) From ca65b2598a684ed57c41de7ef79718746c7f2c13 Mon Sep 17 00:00:00 2001 From: Gary Page-Wood Date: Wed, 23 Jan 2019 20:02:27 +1300 Subject: [PATCH 5/7] add high level functional tests to cover UnprocessedItems in batch writes --- test/functional/encrypted/test_client.py | 14 +++ test/functional/encrypted/test_resource.py | 17 ++- test/functional/encrypted/test_table.py | 12 +- test/functional/functional_test_utils.py | 121 ++++++++++++++++++++- 4 files changed, 159 insertions(+), 5 deletions(-) diff --git a/test/functional/encrypted/test_client.py b/test/functional/encrypted/test_client.py index c928a55e..4498f954 100644 --- a/test/functional/encrypted/test_client.py +++ b/test/functional/encrypted/test_client.py @@ -17,6 +17,8 @@ from ..functional_test_utils import example_table # noqa pylint: disable=unused-import from ..functional_test_utils import ( TEST_TABLE_NAME, + build_static_jce_cmp, + client_batch_items_unprocessed_check, client_cycle_batch_items_check, client_cycle_batch_items_check_paginators, client_cycle_single_item_check, @@ -53,6 +55,12 @@ def _client_cycle_batch_items_check_paginators(materials_provider, initial_actio ) +def _client_batch_items_unprocessed_check(materials_provider, initial_actions, initial_item): + client_batch_items_unprocessed_check( + materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, "us-west-2" + ) + + def test_ephemeral_item_cycle(example_table, some_cmps, parametrized_actions, parametrized_item): """Test a small number of curated CMPs against a small number of curated items.""" _client_cycle_single_item_check(some_cmps, parametrized_actions, parametrized_item) @@ -68,6 +76,12 @@ def test_ephemeral_batch_item_cycle_paginators(example_table, some_cmps, paramet _client_cycle_batch_items_check_paginators(some_cmps, parametrized_actions, parametrized_item) +def test_batch_item_unprocessed(example_table, parametrized_actions, parametrized_item): + """Test Unprocessed Items handling with a single ephemeral static CMP against a small number of curated items.""" + cmp = build_static_jce_cmp("AES", 256, "HmacSHA256", 256) + _client_batch_items_unprocessed_check(cmp, parametrized_actions, parametrized_item) + + @pytest.mark.slow def test_ephemeral_item_cycle_slow(example_table, all_the_cmps, parametrized_actions, parametrized_item): """Test ALL THE CMPS against a small number of curated items.""" diff --git a/test/functional/encrypted/test_resource.py b/test/functional/encrypted/test_resource.py index 3400906f..3ee7223a 100644 --- a/test/functional/encrypted/test_resource.py +++ b/test/functional/encrypted/test_resource.py @@ -13,9 +13,11 @@ """Functional tests for ``dynamodb_encryption_sdk.encrypted.resource``.""" import pytest -from ..functional_test_utils import example_table # noqa pylint: disable=unused-import +from ..functional_test_utils import example_table # noqa pylint: disable=unused-import from ..functional_test_utils import ( TEST_TABLE_NAME, + build_static_jce_cmp, + resource_batch_items_unprocessed_check, resource_cycle_batch_items_check, set_parametrized_actions, set_parametrized_cmp, @@ -35,11 +37,24 @@ def _resource_cycle_batch_items_check(materials_provider, initial_actions, initi resource_cycle_batch_items_check(materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, "us-west-2") +def _resource_batch_items_unprocessed_check(materials_provider, initial_actions, initial_item): + resource_batch_items_unprocessed_check( + materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, "us-west-2" + ) + + def test_ephemeral_batch_item_cycle(example_table, some_cmps, parametrized_actions, parametrized_item): """Test a small number of curated CMPs against a small number of curated items.""" _resource_cycle_batch_items_check(some_cmps, parametrized_actions, parametrized_item) +def test_batch_item_unprocessed(example_table, parametrized_actions, parametrized_item): + """Test Unprocessed Items handling with a single ephemeral static CMP against a small number of curated items.""" + _resource_batch_items_unprocessed_check( + build_static_jce_cmp("AES", 256, "HmacSHA256", 256), parametrized_actions, parametrized_item + ) + + @pytest.mark.travis_isolation @pytest.mark.slow def test_ephemeral_batch_item_cycle_slow(example_table, all_the_cmps, parametrized_actions, parametrized_item): diff --git a/test/functional/encrypted/test_table.py b/test/functional/encrypted/test_table.py index 4759edfe..c9e23b7e 100644 --- a/test/functional/encrypted/test_table.py +++ b/test/functional/encrypted/test_table.py @@ -14,12 +14,14 @@ import hypothesis import pytest -from ..functional_test_utils import example_table # noqa pylint: disable=unused-import +from ..functional_test_utils import example_table # noqa pylint: disable=unused-import from ..functional_test_utils import ( TEST_TABLE_NAME, + build_static_jce_cmp, set_parametrized_actions, set_parametrized_cmp, set_parametrized_item, + table_batch_writer_unprocessed_items_check, table_cycle_batch_writer_check, table_cycle_check, ) @@ -48,6 +50,14 @@ def test_ephemeral_item_cycle_batch_writer(example_table, some_cmps, parametrize table_cycle_batch_writer_check(some_cmps, parametrized_actions, parametrized_item, TEST_TABLE_NAME, "us-west-2") +def test_batch_writer_unprocessed(example_table, parametrized_actions, parametrized_item): + """Test Unprocessed Items handling with a single ephemeral static CMP against a small number of curated items.""" + cmp = build_static_jce_cmp("AES", 256, "HmacSHA256", 256) + table_batch_writer_unprocessed_items_check( + cmp, parametrized_actions, parametrized_item, TEST_TABLE_NAME, "us-west-2" + ) + + @pytest.mark.slow def test_ephemeral_item_cycle_slow(example_table, all_the_cmps, parametrized_actions, parametrized_item): """Test ALL THE CMPS against a small number of curated items.""" diff --git a/test/functional/functional_test_utils.py b/test/functional/functional_test_utils.py index 9353408a..9edf5ff8 100644 --- a/test/functional/functional_test_utils.py +++ b/test/functional/functional_test_utils.py @@ -24,6 +24,7 @@ import pytest from boto3.dynamodb.types import Binary from botocore.exceptions import NoRegionError +from mock import patch from moto import mock_dynamodb2 from dynamodb_encryption_sdk.delegated_keys.jce import JceNameLocalDelegatedKey @@ -336,6 +337,12 @@ def diverse_item(): _reserved_attributes = set([attr.value for attr in ReservedAttributes]) +def return_requestitems_as_unprocessed(*args, **kwargs): + return { + "UnprocessedItems": kwargs['RequestItems'] + } + + def check_encrypted_item(plaintext_item, ciphertext_item, attribute_actions): # Verify that all expected attributes are present ciphertext_attributes = set(ciphertext_item.keys()) @@ -374,12 +381,20 @@ def _nop_transformer(item): return item +def assert_items_exist_in_list(source, expected, transformer): + for actual_item in source: + expected_item = _matching_key(actual_item, expected) + assert transformer(actual_item) == transformer(expected_item) + + def assert_equal_lists_of_items(actual, expected, transformer=_nop_transformer): assert len(actual) == len(expected) + assert_items_exist_in_list(actual, expected, transformer) - for actual_item in actual: - expected_item = _matching_key(actual_item, expected) - assert transformer(actual_item) == transformer(expected_item) + +def assert_list_of_items_contains(full, subset, transformer=_nop_transformer): + assert len(full) >= len(subset) + assert_items_exist_in_list(subset, full, transformer) def check_many_encrypted_items(actual, expected, attribute_actions, transformer=_nop_transformer): @@ -479,6 +494,28 @@ def cycle_batch_writer_check(raw_table, encrypted_table, initial_actions, initia del items +def batch_write_item_unprocessed_check( + encrypted, + initial_item, + write_transformer=_nop_transformer, + table_name=TEST_TABLE_NAME, +): + """Check that unprocessed items in a batch result are unencrypted.""" + items = _generate_items(initial_item, write_transformer) + + request_items = {table_name: [{"PutRequest": {"Item": _item}} for _item in items]} + _put_result = encrypted.batch_write_item(RequestItems=request_items) + + # we expect results to include Unprocessed items, or the test case is invalid! + unprocessed_items = _put_result["UnprocessedItems"] + assert unprocessed_items != {} + + unprocessed = [operation["PutRequest"]["Item"] for operation in _put_result["UnprocessedItems"][TEST_TABLE_NAME]] + assert_list_of_items_contains(items, unprocessed, transformer=_nop_transformer) + + del items + + def cycle_item_check(plaintext_item, crypto_config): """Check that cycling (plaintext->encrypted->decrypted) an item has the expected results.""" ciphertext_item = encrypt_python_item(plaintext_item, crypto_config) @@ -527,6 +564,34 @@ def table_cycle_batch_writer_check(materials_provider, initial_actions, initial_ cycle_batch_writer_check(table, e_table, initial_actions, initial_item) +def table_batch_writer_unprocessed_items_check( + materials_provider, + initial_actions, + initial_item, + table_name, + region_name=None +): + kwargs = {} + if region_name is not None: + kwargs["region_name"] = region_name + resource = boto3.resource("dynamodb", **kwargs) + table = resource.Table(table_name) + + items = _generate_items(initial_item, _nop_transformer) + request_items = {table_name: [{"PutRequest": {"Item": _item}} for _item in items]} + + with patch.object(table.meta.client, "batch_write_item") as batch_write_mock: + # Check that unprocessed items returned to a BatchWriter are successfully retried + batch_write_mock.side_effect = [{"UnprocessedItems": request_items}, {'UnprocessedItems': {}}] + e_table = EncryptedTable(table=table, materials_provider=materials_provider, attribute_actions=initial_actions) + + with e_table.batch_writer() as writer: + for item in items: + writer.put_item(item) + + del items + + def resource_cycle_batch_items_check(materials_provider, initial_actions, initial_item, table_name, region_name=None): kwargs = {} if region_name is not None: @@ -550,6 +615,31 @@ def resource_cycle_batch_items_check(materials_provider, initial_actions, initia assert not e_scan_result["Items"] +def resource_batch_items_unprocessed_check( + materials_provider, + initial_actions, + initial_item, + table_name, + region_name=None +): + kwargs = {} + if region_name is not None: + kwargs["region_name"] = region_name + resource = boto3.resource('dynamodb', **kwargs) + + with patch.object(resource, "batch_write_item", return_requestitems_as_unprocessed): + e_resource = EncryptedResource( + resource=resource, materials_provider=materials_provider, attribute_actions=initial_actions + ) + + batch_write_item_unprocessed_check( + encrypted=e_resource, + initial_item=initial_item, + write_transformer=dict_to_ddb, + table_name=table_name, + ) + + def client_cycle_single_item_check(materials_provider, initial_actions, initial_item, table_name, region_name=None): check_attribute_actions = initial_actions.copy() check_attribute_actions.set_index_keys(*list(TEST_KEY.keys())) @@ -600,6 +690,31 @@ def client_cycle_batch_items_check(materials_provider, initial_actions, initial_ assert not e_scan_result["Items"] +def client_batch_items_unprocessed_check( + materials_provider, + initial_actions, + initial_item, + table_name, + region_name=None +): + kwargs = {} + if region_name is not None: + kwargs["region_name"] = region_name + client = boto3.client('dynamodb', **kwargs) + + with patch.object(client, "batch_write_item", return_requestitems_as_unprocessed): + e_client = EncryptedClient( + client=client, materials_provider=materials_provider, attribute_actions=initial_actions + ) + + batch_write_item_unprocessed_check( + encrypted=e_client, + initial_item=initial_item, + write_transformer=dict_to_ddb, + table_name=table_name, + ) + + def client_cycle_batch_items_check_paginators( materials_provider, initial_actions, initial_item, table_name, region_name=None ): From 2a71472848d44f0a71e1aa6888c59487554dfb4d Mon Sep 17 00:00:00 2001 From: Gary Page-Wood Date: Thu, 24 Jan 2019 08:25:29 +1300 Subject: [PATCH 6/7] dry up test code a little --- test/functional/functional_test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/functional_test_utils.py b/test/functional/functional_test_utils.py index 9edf5ff8..9b4dea68 100644 --- a/test/functional/functional_test_utils.py +++ b/test/functional/functional_test_utils.py @@ -510,7 +510,7 @@ def batch_write_item_unprocessed_check( unprocessed_items = _put_result["UnprocessedItems"] assert unprocessed_items != {} - unprocessed = [operation["PutRequest"]["Item"] for operation in _put_result["UnprocessedItems"][TEST_TABLE_NAME]] + unprocessed = [operation["PutRequest"]["Item"] for operation in unprocessed_items[TEST_TABLE_NAME]] assert_list_of_items_contains(items, unprocessed, transformer=_nop_transformer) del items From 0e0ce65751fdd97b79c3838867bfd5844bf56a1d Mon Sep 17 00:00:00 2001 From: Gary Page-Wood Date: Fri, 25 Jan 2019 13:58:07 +1300 Subject: [PATCH 7/7] autoformat --- test/functional/encrypted/test_resource.py | 2 +- test/functional/encrypted/test_table.py | 2 +- test/functional/functional_test_utils.py | 43 +++++----------------- test/functional/internal/test_utils.py | 23 ++++-------- 4 files changed, 20 insertions(+), 50 deletions(-) diff --git a/test/functional/encrypted/test_resource.py b/test/functional/encrypted/test_resource.py index 3ee7223a..3c6a01e2 100644 --- a/test/functional/encrypted/test_resource.py +++ b/test/functional/encrypted/test_resource.py @@ -13,7 +13,7 @@ """Functional tests for ``dynamodb_encryption_sdk.encrypted.resource``.""" import pytest -from ..functional_test_utils import example_table # noqa pylint: disable=unused-import +from ..functional_test_utils import example_table # noqa pylint: disable=unused-import from ..functional_test_utils import ( TEST_TABLE_NAME, build_static_jce_cmp, diff --git a/test/functional/encrypted/test_table.py b/test/functional/encrypted/test_table.py index c9e23b7e..fe9339ad 100644 --- a/test/functional/encrypted/test_table.py +++ b/test/functional/encrypted/test_table.py @@ -14,7 +14,7 @@ import hypothesis import pytest -from ..functional_test_utils import example_table # noqa pylint: disable=unused-import +from ..functional_test_utils import example_table # noqa pylint: disable=unused-import from ..functional_test_utils import ( TEST_TABLE_NAME, build_static_jce_cmp, diff --git a/test/functional/functional_test_utils.py b/test/functional/functional_test_utils.py index 9b4dea68..86ebf196 100644 --- a/test/functional/functional_test_utils.py +++ b/test/functional/functional_test_utils.py @@ -338,9 +338,7 @@ def diverse_item(): def return_requestitems_as_unprocessed(*args, **kwargs): - return { - "UnprocessedItems": kwargs['RequestItems'] - } + return {"UnprocessedItems": kwargs["RequestItems"]} def check_encrypted_item(plaintext_item, ciphertext_item, attribute_actions): @@ -495,10 +493,7 @@ def cycle_batch_writer_check(raw_table, encrypted_table, initial_actions, initia def batch_write_item_unprocessed_check( - encrypted, - initial_item, - write_transformer=_nop_transformer, - table_name=TEST_TABLE_NAME, + encrypted, initial_item, write_transformer=_nop_transformer, table_name=TEST_TABLE_NAME ): """Check that unprocessed items in a batch result are unencrypted.""" items = _generate_items(initial_item, write_transformer) @@ -565,11 +560,7 @@ def table_cycle_batch_writer_check(materials_provider, initial_actions, initial_ def table_batch_writer_unprocessed_items_check( - materials_provider, - initial_actions, - initial_item, - table_name, - region_name=None + materials_provider, initial_actions, initial_item, table_name, region_name=None ): kwargs = {} if region_name is not None: @@ -582,7 +573,7 @@ def table_batch_writer_unprocessed_items_check( with patch.object(table.meta.client, "batch_write_item") as batch_write_mock: # Check that unprocessed items returned to a BatchWriter are successfully retried - batch_write_mock.side_effect = [{"UnprocessedItems": request_items}, {'UnprocessedItems': {}}] + batch_write_mock.side_effect = [{"UnprocessedItems": request_items}, {"UnprocessedItems": {}}] e_table = EncryptedTable(table=table, materials_provider=materials_provider, attribute_actions=initial_actions) with e_table.batch_writer() as writer: @@ -616,16 +607,12 @@ def resource_cycle_batch_items_check(materials_provider, initial_actions, initia def resource_batch_items_unprocessed_check( - materials_provider, - initial_actions, - initial_item, - table_name, - region_name=None + materials_provider, initial_actions, initial_item, table_name, region_name=None ): kwargs = {} if region_name is not None: kwargs["region_name"] = region_name - resource = boto3.resource('dynamodb', **kwargs) + resource = boto3.resource("dynamodb", **kwargs) with patch.object(resource, "batch_write_item", return_requestitems_as_unprocessed): e_resource = EncryptedResource( @@ -633,10 +620,7 @@ def resource_batch_items_unprocessed_check( ) batch_write_item_unprocessed_check( - encrypted=e_resource, - initial_item=initial_item, - write_transformer=dict_to_ddb, - table_name=table_name, + encrypted=e_resource, initial_item=initial_item, write_transformer=dict_to_ddb, table_name=table_name ) @@ -691,16 +675,12 @@ def client_cycle_batch_items_check(materials_provider, initial_actions, initial_ def client_batch_items_unprocessed_check( - materials_provider, - initial_actions, - initial_item, - table_name, - region_name=None + materials_provider, initial_actions, initial_item, table_name, region_name=None ): kwargs = {} if region_name is not None: kwargs["region_name"] = region_name - client = boto3.client('dynamodb', **kwargs) + client = boto3.client("dynamodb", **kwargs) with patch.object(client, "batch_write_item", return_requestitems_as_unprocessed): e_client = EncryptedClient( @@ -708,10 +688,7 @@ def client_batch_items_unprocessed_check( ) batch_write_item_unprocessed_check( - encrypted=e_client, - initial_item=initial_item, - write_transformer=dict_to_ddb, - table_name=table_name, + encrypted=e_client, initial_item=initial_item, write_transformer=dict_to_ddb, table_name=table_name ) diff --git a/test/functional/internal/test_utils.py b/test/functional/internal/test_utils.py index 3357b17d..1e52c0fd 100644 --- a/test/functional/internal/test_utils.py +++ b/test/functional/internal/test_utils.py @@ -62,14 +62,11 @@ def get_test_items(standard_dict_format, table_name="table", with_sort_keys=Fals def get_dummy_crypto_config(partition_key_name=None, sort_key_name=None, sign_keys=False): context = EncryptionContext(partition_key_name=partition_key_name, sort_key_name=sort_key_name) actions = AttributeActions( - default_action=CryptoAction.DO_NOTHING, - attribute_actions={ - "encrypt-me": CryptoAction.ENCRYPT_AND_SIGN - }, + default_action=CryptoAction.DO_NOTHING, attribute_actions={"encrypt-me": CryptoAction.ENCRYPT_AND_SIGN} ) if sign_keys: - actions.attribute_actions['partition-key'] = CryptoAction.SIGN_ONLY - actions.attribute_actions['sort-key'] = CryptoAction.SIGN_ONLY + actions.attribute_actions["partition-key"] = CryptoAction.SIGN_ONLY + actions.attribute_actions["sort-key"] = CryptoAction.SIGN_ONLY materials = Mock(spec=CryptographicMaterialsProvider) # type: CryptographicMaterialsProvider return CryptoConfig(materials_provider=materials, encryption_context=context, attribute_actions=actions) @@ -78,7 +75,7 @@ def get_dummy_crypto_config(partition_key_name=None, sort_key_name=None, sign_ke def check_encrypt_batch_write_item_call(request_items, crypto_config): def dummy_encrypt(item, **kwargs): result = item.copy() - result['encrypt-me'] = "pretend Im encrypted" + result["encrypt-me"] = "pretend Im encrypted" return result # execute a batch write, but make the write method return ALL the provided items as unprocessed @@ -96,11 +93,7 @@ def dummy_encrypt(item, **kwargs): @pytest.mark.parametrize( - "items", - ( - get_test_items(standard_dict_format=True), - get_test_items(standard_dict_format=False), - ) + "items", (get_test_items(standard_dict_format=True), get_test_items(standard_dict_format=False)) ) def test_encrypt_batch_write_returns_plaintext_unprocessed_items_with_known_partition_key(items): crypto_config = get_dummy_crypto_config("partition-key") @@ -112,7 +105,7 @@ def test_encrypt_batch_write_returns_plaintext_unprocessed_items_with_known_part ( get_test_items(standard_dict_format=True, with_sort_keys=True), get_test_items(standard_dict_format=False, with_sort_keys=True), - ) + ), ) def test_encrypt_batch_write_returns_plaintext_unprocessed_items_with_known_partition_and_sort_keys(items): crypto_config = get_dummy_crypto_config("partition-key", "sort-key") @@ -126,7 +119,7 @@ def test_encrypt_batch_write_returns_plaintext_unprocessed_items_with_known_part get_test_items(standard_dict_format=False), get_test_items(standard_dict_format=True, with_sort_keys=True), get_test_items(standard_dict_format=False, with_sort_keys=True), - ) + ), ) def test_encrypt_batch_write_returns_plaintext_unprocessed_items_with_unknown_keys(items): crypto_config = get_dummy_crypto_config(None, None) @@ -141,7 +134,7 @@ def test_encrypt_batch_write_returns_plaintext_unprocessed_items_with_unknown_ke get_test_items(standard_dict_format=False), get_test_items(standard_dict_format=True, with_sort_keys=True), get_test_items(standard_dict_format=False, with_sort_keys=True), - ) + ), ) def test_encrypt_batch_write_returns_plaintext_unprocessed_items_with_unknown_signed_keys(items): crypto_config = get_dummy_crypto_config(None, None, sign_keys=True)