* helper clients need to set item attributes in encryption context used for processing item

* move sorted_key_map to serialize.attribute (only consumer) to avoid circular dependencies
* add acceptance tests for helper clients to verify that item values are being added to encryption context (used by AWS CMP)

Author: mattsb42-aws

src/dynamodb_encryption_sdk/encrypted/__init__.py

@@ -104,3 +104,14 @@ def copy(self):
             encryption_context=copy.copy(self.encryption_context),
             attribute_actions=self.attribute_actions
         )
+
+    def with_item(self, item):
+        """Return a copy of this instance with an encryption context that includes the provided item attributes.
+
+        :param dict item: DynamoDB item in DynamnoDB JSON format
+        :returns: New :class:`CryptoConfig` identical to this one
+        :rtype: CryptoConfig
+        """
+        new_config = self.copy()
+        new_config.encryption_context.attributes = item
+        return new_config

src/dynamodb_encryption_sdk/encrypted/client.py

@@ -136,8 +136,8 @@ class EncryptedClient(object):
 
         We do not currently support the ``update_item`` method.
 
-    :param table: Pre-configured boto3 DynamoDB client object
-    :type table: boto3.resources.base.BaseClient
+    :param client: Pre-configured boto3 DynamoDB client object
+    :type client: boto3.resources.base.BaseClient
     :param CryptographicMaterialsProvider materials_provider: Cryptographic materials provider
         to use
     :param AttributeActions attribute_actions: Table-level configuration of how to encrypt/sign

src/dynamodb_encryption_sdk/internal/formatting/serialize/attribute.py

@@ -33,13 +33,28 @@
 from dynamodb_encryption_sdk.internal.formatting.serialize import encode_length, encode_value
 from dynamodb_encryption_sdk.internal.identifiers import Tag, TagValues
 from dynamodb_encryption_sdk.internal.str_ops import to_bytes
-from dynamodb_encryption_sdk.internal.utils import sorted_key_map
 
 __all__ = ('serialize_attribute',)
 _LOGGER = logging.getLogger(LOGGER_NAME)
 _RESERVED = b'\x00'
 
 
+def _sorted_key_map(item, transform=to_bytes):
+    """Creates a list of the item's key/value pairs as tuples, sorted by the keys transformed by transform.
+
+    :param dict item: Source dictionary
+    :param function transform: Transform function
+    :returns: List of tuples containing transformed key, original value, and original key for each entry
+    :rtype: list(tuple)
+    """
+    sorted_items = []
+    for key, value in item.items():
+        _key = transform(key)
+        sorted_items.append((_key, value, key))
+    sorted_items = sorted(sorted_items, key=lambda x: x[0])
+    return sorted_items
+
+
 def serialize_attribute(attribute):  # noqa: C901 pylint: disable=too-many-locals
     # type: (dynamodb_types.RAW_ATTRIBUTE) -> bytes
     """Serializes a raw attribute to a byte string as defined for the DynamoDB Client-Side Encryption Standard.
@@ -213,7 +228,7 @@ def _serialize_map(_attribute):
         serialized_attribute.write(Tag.MAP.tag)
         serialized_attribute.write(encode_length(_attribute))
 
-        sorted_items = sorted_key_map(
+        sorted_items = _sorted_key_map(
             item=_attribute,
             transform=_transform_string_value
         )

src/dynamodb_encryption_sdk/internal/utils.py

@@ -20,9 +20,10 @@
 import botocore.client
 
 from dynamodb_encryption_sdk.encrypted import CryptoConfig
+from dynamodb_encryption_sdk.encrypted.item import decrypt_python_item, encrypt_python_item
 from dynamodb_encryption_sdk.exceptions import InvalidArgumentError
-from dynamodb_encryption_sdk.internal.str_ops import to_bytes
 from dynamodb_encryption_sdk.structures import EncryptionContext, TableInfo
+from dynamodb_encryption_sdk.transform import dict_to_ddb
 
 try:  # Python 3.5.0 and 3.5.1 have incompatible typing modules
     from typing import Any, Callable, Dict, Text  # noqa pylint: disable=unused-import
@@ -31,30 +32,14 @@
     pass
 
 __all__ = (
-    'sorted_key_map', 'TableInfoCache',
+    'TableInfoCache',
     'crypto_config_from_kwargs', 'crypto_config_from_table_info', 'crypto_config_from_cache',
     'decrypt_get_item', 'decrypt_multi_get', 'decrypt_batch_get_item',
     'encrypt_put_item', 'encrypt_batch_write_item',
     'validate_get_arguments'
 )
 
 
-def sorted_key_map(item, transform=to_bytes):
-    """Creates a list of the item's key/value pairs as tuples, sorted by the keys transformed by transform.
-
-    :param dict item: Source dictionary
-    :param function transform: Transform function
-    :returns: List of tuples containing transformed key, original value, and original key for each entry
-    :rtype: list(tuple)
-    """
-    sorted_items = []
-    for key, value in item.items():
-        _key = transform(key)
-        sorted_items.append((_key, value, key))
-    sorted_items = sorted(sorted_items, key=lambda x: x[0])
-    return sorted_items
-
-
 @attr.s(init=False)
 class TableInfoCache(object):
     # pylint: disable=too-few-public-methods
@@ -142,9 +127,16 @@ def crypto_config_from_table_info(materials_provider, attribute_actions, table_i
     :returns: crypto config and updated kwargs
     :rtype: tuple(CryptoConfig, dict)
     """
+    ec_kwargs = table_info.encryption_context_values
+    if table_info.primary_index is not None:
+        ec_kwargs.update({
+            'partition_key_name': table_info.primary_index.partition,
+            'sort_key_name': table_info.primary_index.sort
+        })
+
     return CryptoConfig(
         materials_provider=materials_provider,
-        encryption_context=EncryptionContext(**table_info.encryption_context_values),
+        encryption_context=EncryptionContext(**ec_kwargs),
         attribute_actions=attribute_actions
     )
 
@@ -163,10 +155,23 @@ def crypto_config_from_cache(materials_provider, attribute_actions, table_info_c
     return crypto_config_from_table_info(materials_provider, attribute_actions, table_info)
 
 
+def _item_transformer(crypto_transformer):
+    """Supply an item transformer to go from an item that the provided ``crypto_transformer``
+    can understand to a DynamoDB JSON object.
+
+    :param crypto_transformer: An item encryptor or decryptor function
+    :returns: Item transformer function
+    """
+    if crypto_transformer in (decrypt_python_item, encrypt_python_item):
+        return dict_to_ddb
+
+    return lambda x: x
+
+
 def decrypt_multi_get(decrypt_method, crypto_config_method, read_method, **kwargs):
     # type: (Callable, Callable, Callable, **Any) -> Dict
     # TODO: narrow this down
-    """Transparently decrypt multiple items after getting them from the table.
+    """Transparently decrypt multiple items after getting them from the table with a scan or query method.
 
     :param callable decrypt_method: Method to use to decrypt items
     :param callable crypto_config_method: Method that accepts ``kwargs`` and provides a :class:`CryptoConfig`
@@ -181,7 +186,7 @@ def decrypt_multi_get(decrypt_method, crypto_config_method, read_method, **kwarg
     for pos in range(len(response['Items'])):
         response['Items'][pos] = decrypt_method(
             item=response['Items'][pos],
-            crypto_config=crypto_config
+            crypto_config=crypto_config.with_item(_item_transformer(decrypt_method)(response['Items'][pos]))
         )
     return response
 
@@ -204,7 +209,7 @@ def decrypt_get_item(decrypt_method, crypto_config_method, read_method, **kwargs
     if 'Item' in response:
         response['Item'] = decrypt_method(
             item=response['Item'],
-            crypto_config=crypto_config
+            crypto_config=crypto_config.with_item(_item_transformer(decrypt_method)(response['Item']))
         )
     return response
 
@@ -236,7 +241,7 @@ def decrypt_batch_get_item(decrypt_method, crypto_config_method, read_method, **
         for pos, value in enumerate(items):
             items[pos] = decrypt_method(
                 item=value,
-                crypto_config=crypto_config
+                crypto_config=crypto_config.with_item(_item_transformer(decrypt_method)(items[pos]))
             )
     return response
 
@@ -256,7 +261,7 @@ def encrypt_put_item(encrypt_method, crypto_config_method, write_method, **kwarg
     crypto_config, ddb_kwargs = crypto_config_method(**kwargs)
     ddb_kwargs['Item'] = encrypt_method(
         item=ddb_kwargs['Item'],
-        crypto_config=crypto_config
+        crypto_config=crypto_config.with_item(_item_transformer(encrypt_method)(ddb_kwargs['Item']))
     )
     return write_method(**ddb_kwargs)
 
@@ -287,6 +292,6 @@ def encrypt_batch_write_item(encrypt_method, crypto_config_method, write_method,
                 if request_type == 'PutRequest':
                     items[pos][request_type]['Item'] = encrypt_method(
                         item=item['Item'],
-                        crypto_config=crypto_config
+                        crypto_config=crypto_config.with_item(_item_transformer(encrypt_method)(item['Item']))
                     )
     return write_method(**kwargs)

test/README.rst

@@ -0,0 +1,16 @@
+********************************************
+dynamodb-encryption-client Integration Tests
+********************************************
+
+In order to run these integration tests successfully, these things which must be configured.
+
+#. These tests assume that AWS credentials are available in one of the
+   `automatically discoverable credential locations`_.
+#. The ``AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID`` environment variable
+   must be set to a valid `AWS KMS CMK ARN`_ that can be used by the available credentials.
+#. The ``DDB_ENCRYPTION_CLIENT_TEST_TABLE_NAME`` environment variable must be set to a valid
+   DynamoDB table name, in the default region, to which the discoverable credentials have
+   read, write, and describe permissions.
+
+.. _automatically discoverable credential locations: http://boto3.readthedocs.io/en/latest/guide/configuration.html
+.. _AWS KMS CMK ARN: http://docs.aws.amazon.com/kms/latest/APIReference/API_Encrypt.html

test/acceptance/acceptance_test_utils.py

@@ -19,7 +19,6 @@
 import sys
 
 import boto3
-from moto import mock_dynamodb2
 import pytest
 from six.moves.urllib.parse import urlparse  # moves confuse pylint: disable=wrong-import-order
 

test/acceptance/encrypted/test_client.py

@@ -0,0 +1,175 @@
+# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""Acceptance tests for ``dynamodb_encryption_sdk.encrypted.client``."""
+import botocore
+from mock import MagicMock
+from moto import mock_dynamodb2
+import pytest
+
+from dynamodb_encryption_sdk.encrypted.client import EncryptedClient
+from dynamodb_encryption_sdk.structures import TableIndex, TableInfo
+from dynamodb_encryption_sdk.transform import dict_to_ddb
+from ..acceptance_test_utils import load_scenarios
+
+pytestmark = [pytest.mark.accept]
+
+
+def fake_client(table_name, item):
+    client = MagicMock(__class__=botocore.client.BaseClient)
+    client.get_item.return_value = {'Item': item.copy()}
+    client.batch_get_item.return_value = {'Responses': {table_name: [item.copy()]}}
+    return client
+
+
+def _compare_item(plaintext_item, decrypted_item):
+    assert set(decrypted_item.keys()) == set(plaintext_item.keys())
+    for key in decrypted_item:
+        if key == 'version':
+            continue
+        assert decrypted_item[key] == plaintext_item[key]
+
+
+def _client_setup(
+        materials_provider,
+        table_name,
+        table_index,
+        ciphertext_item,
+        attribute_actions,
+        prep
+):
+    prep()  # Test scenario setup that needs to happen inside the test
+    cmp = materials_provider()  # Some of the materials providers need to be constructed inside the test
+    client = fake_client(table_name, ciphertext_item)
+    table_info = TableInfo(
+        name=table_name,
+        primary_index=TableIndex(
+            partition=table_index['partition'],
+            sort=table_index.get('sort', None)
+        )
+    )
+    item_key = {table_info.primary_index.partition: ciphertext_item[table_info.primary_index.partition]}
+    if table_info.primary_index.sort is not None:
+        item_key[table_info.primary_index.sort] = ciphertext_item[table_info.primary_index.sort]
+
+    e_client = EncryptedClient(
+        client=client,
+        materials_provider=cmp,
+        attribute_actions=attribute_actions,
+        auto_refresh_table_indexes=False
+    )
+    e_client._table_info_cache._all_tables_info[table_name] = table_info
+    return e_client, dict_to_ddb(item_key)
+
+
+def _item_check(
+        materials_provider,
+        table_name,
+        table_index,
+        ciphertext_item,
+        plaintext_item,
+        attribute_actions,
+        prep
+):
+    e_client, item_key = _client_setup(
+        materials_provider,
+        table_name,
+        table_index,
+        ciphertext_item,
+        attribute_actions,
+        prep
+    )
+    decrypted_item = e_client.get_item(TableName=table_name, Key=item_key)['Item']
+    _compare_item(plaintext_item, decrypted_item)
+
+
+def _batch_items_check(
+        materials_provider,
+        table_name,
+        table_index,
+        ciphertext_item,
+        plaintext_item,
+        attribute_actions,
+        prep
+):
+    e_client, item_key = _client_setup(
+        materials_provider,
+        table_name,
+        table_index,
+        ciphertext_item,
+        attribute_actions,
+        prep
+    )
+    response = e_client.batch_get_item(RequestItems={table_name: {'Keys': [item_key]}})
+    decrypted_item = response['Responses'][table_name][0]
+    _compare_item(plaintext_item, decrypted_item)
+
+
+def pytest_generate_tests(metafunc):
+    if 'item_checker' in metafunc.fixturenames:
+        metafunc.parametrize('item_checker', (
+            pytest.param(_item_check, id='single-item'),
+            pytest.param(_batch_items_check, id='batch-items')
+        ))
+
+
+@mock_dynamodb2
+@pytest.mark.local
+@pytest.mark.parametrize(
+    'materials_provider, table_name, table_index, ciphertext_item, plaintext_item, attribute_actions, prep',
+    load_scenarios(online=False)
+)
+def test_client_get_offline(
+        materials_provider,
+        table_name,
+        table_index,
+        ciphertext_item,
+        plaintext_item,
+        attribute_actions,
+        prep,
+        item_checker
+):
+    return item_checker(
+        materials_provider,
+        table_name,
+        table_index,
+        ciphertext_item,
+        plaintext_item,
+        attribute_actions,
+        prep
+    )
+
+
+@pytest.mark.integ
+@pytest.mark.parametrize(
+    'materials_provider, table_name, table_index, ciphertext_item, plaintext_item, attribute_actions, prep',
+    load_scenarios(online=True)
+)
+def test_client_get_online(
+        materials_provider,
+        table_name,
+        table_index,
+        ciphertext_item,
+        plaintext_item,
+        attribute_actions,
+        prep,
+        item_checker
+):
+    return item_checker(
+        materials_provider,
+        table_name,
+        table_index,
+        ciphertext_item,
+        plaintext_item,
+        attribute_actions,
+        prep
+    )

test/acceptance/encrypted/test_resource.py

@@ -0,0 +1,149 @@
+# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""Acceptance tests for ``dynamodb_encryption_sdk.encrypted.resource``."""
+from boto3.resources.base import ServiceResource
+from boto3.resources.collection import CollectionManager
+import botocore
+from mock import MagicMock
+from moto import mock_dynamodb2
+import pytest
+
+from dynamodb_encryption_sdk.encrypted.resource import EncryptedResource
+from dynamodb_encryption_sdk.structures import TableIndex, TableInfo
+from dynamodb_encryption_sdk.transform import ddb_to_dict
+from ..acceptance_test_utils import load_scenarios
+
+pytestmark = [pytest.mark.accept]
+
+
+def fake_resource(table_name, item):
+    resource = MagicMock(__class__=ServiceResource)
+    resource.batch_get_item.return_value = {'Responses': {table_name: [item.copy()]}}
+    resource.meta.client = MagicMock(__class__=botocore.client.BaseClient)
+    resource.tables = MagicMock(__class__=CollectionManager)
+    return resource
+
+
+def _compare_item(plaintext_item, decrypted_item):
+    assert set(decrypted_item.keys()) == set(plaintext_item.keys())
+    for key in decrypted_item:
+        if key == 'version':
+            continue
+        assert decrypted_item[key] == plaintext_item[key]
+
+
+def _resource_setup(
+        materials_provider,
+        table_name,
+        table_index,
+        ciphertext_item,
+        attribute_actions,
+        prep
+):
+    prep()  # Test scenario setup that needs to happen inside the test
+    cmp = materials_provider()  # Some of the materials providers need to be constructed inside the test
+    resource = fake_resource(table_name, ciphertext_item)
+    table_info = TableInfo(
+        name=table_name,
+        primary_index=TableIndex(
+            partition=table_index['partition'],
+            sort=table_index.get('sort', None)
+        )
+    )
+    item_key = {table_info.primary_index.partition: ciphertext_item[table_info.primary_index.partition]}
+    if table_info.primary_index.sort is not None:
+        item_key[table_info.primary_index.sort] = ciphertext_item[table_info.primary_index.sort]
+
+    e_resource = EncryptedResource(
+        resource=resource,
+        materials_provider=cmp,
+        attribute_actions=attribute_actions,
+        auto_refresh_table_indexes=False
+    )
+    e_resource._table_info_cache._all_tables_info[table_name] = table_info
+    return e_resource, item_key
+
+
+def _batch_items_check(
+        materials_provider,
+        table_name,
+        table_index,
+        ciphertext_item,
+        plaintext_item,
+        attribute_actions,
+        prep
+):
+    plaintext_item = ddb_to_dict(plaintext_item)
+    ciphertext_item = ddb_to_dict(ciphertext_item)
+    e_resource, item_key = _resource_setup(
+        materials_provider,
+        table_name,
+        table_index,
+        ciphertext_item,
+        attribute_actions,
+        prep
+    )
+    response = e_resource.batch_get_item(RequestItems={table_name: {'Keys': [item_key]}})
+    decrypted_item = response['Responses'][table_name][0]
+    _compare_item(plaintext_item, decrypted_item)
+
+
+@mock_dynamodb2
+@pytest.mark.local
+@pytest.mark.parametrize(
+    'materials_provider, table_name, table_index, ciphertext_item, plaintext_item, attribute_actions, prep',
+    load_scenarios(online=False)
+)
+def test_client_get_offline(
+        materials_provider,
+        table_name,
+        table_index,
+        ciphertext_item,
+        plaintext_item,
+        attribute_actions,
+        prep
+):
+    return _batch_items_check(
+        materials_provider,
+        table_name,
+        table_index,
+        ciphertext_item,
+        plaintext_item,
+        attribute_actions,
+        prep
+    )
+
+
+@pytest.mark.integ
+@pytest.mark.parametrize(
+    'materials_provider, table_name, table_index, ciphertext_item, plaintext_item, attribute_actions, prep',
+    load_scenarios(online=True)
+)
+def test_client_get_online(
+        materials_provider,
+        table_name,
+        table_index,
+        ciphertext_item,
+        plaintext_item,
+        attribute_actions,
+        prep
+):
+    return _batch_items_check(
+        materials_provider,
+        table_name,
+        table_index,
+        ciphertext_item,
+        plaintext_item,
+        attribute_actions,
+        prep
+    )

test/acceptance/encrypted/test_table.py

@@ -0,0 +1,121 @@
+# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""Acceptance tests for ``dynamodb_encryption_sdk.encrypted.table``."""
+from boto3.resources.base import ServiceResource
+from mock import MagicMock
+from moto import mock_dynamodb2
+import pytest
+
+from dynamodb_encryption_sdk.encrypted.table import EncryptedTable
+from dynamodb_encryption_sdk.structures import TableIndex, TableInfo
+from dynamodb_encryption_sdk.transform import ddb_to_dict
+from ..acceptance_test_utils import load_scenarios
+
+pytestmark = [pytest.mark.accept]
+
+
+def fake_table(item):
+    table = MagicMock(__class__=ServiceResource)
+    table.get_item.return_value = {'Item': item}
+    return table
+
+
+def _item_check(
+        materials_provider,
+        table_name,
+        table_index,
+        ciphertext_item,
+        plaintext_item,
+        attribute_actions,
+        prep
+):
+    ciphertext_item = ddb_to_dict(ciphertext_item)
+    plaintext_item = ddb_to_dict(plaintext_item)
+    prep()  # Test scenario setup that needs to happen inside the test
+    cmp = materials_provider()  # Some of the materials providers need to be constructed inside the test
+    table = fake_table(ciphertext_item)
+    table_info = TableInfo(
+        name=table_name,
+        primary_index=TableIndex(
+            partition=table_index['partition'],
+            sort=table_index.get('sort', None)
+        )
+    )
+    item_key = {table_info.primary_index.partition: ciphertext_item[table_info.primary_index.partition]}
+    if table_info.primary_index.sort is not None:
+        item_key[table_info.primary_index.sort] = ciphertext_item[table_info.primary_index.sort]
+
+    e_table = EncryptedTable(
+        table=table,
+        materials_provider=cmp,
+        table_info=table_info,
+        attribute_actions=attribute_actions,
+        auto_refresh_table_indexes=False
+    )
+    decrypted_item = e_table.get_item(Key=item_key)['Item']
+    assert set(decrypted_item.keys()) == set(plaintext_item.keys())
+    for key in decrypted_item:
+        if key == 'version':
+            continue
+        assert decrypted_item[key] == plaintext_item[key]
+
+
+@mock_dynamodb2
+@pytest.mark.local
+@pytest.mark.parametrize(
+    'materials_provider, table_name, table_index, ciphertext_item, plaintext_item, attribute_actions, prep',
+    load_scenarios(online=False)
+)
+def test_table_get_offline(
+        materials_provider,
+        table_name,
+        table_index,
+        ciphertext_item,
+        plaintext_item,
+        attribute_actions,
+        prep
+):
+    return _item_check(
+        materials_provider,
+        table_name,
+        table_index,
+        ciphertext_item,
+        plaintext_item,
+        attribute_actions,
+        prep
+    )
+
+
+@pytest.mark.integ
+@pytest.mark.parametrize(
+    'materials_provider, table_name, table_index, ciphertext_item, plaintext_item, attribute_actions, prep',
+    load_scenarios(online=True)
+)
+def test_table_get_online(
+        materials_provider,
+        table_name,
+        table_index,
+        ciphertext_item,
+        plaintext_item,
+        attribute_actions,
+        prep
+):
+    return _item_check(
+        materials_provider,
+        table_name,
+        table_index,
+        ciphertext_item,
+        plaintext_item,
+        attribute_actions,
+        prep
+    )