diff --git a/doc/index.rst b/doc/index.rst index 30794af9..b0de37c2 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -12,6 +12,8 @@ Modules dynamodb_encryption_sdk.exceptions dynamodb_encryption_sdk.identifiers dynamodb_encryption_sdk.structures + dynamodb_encryption_sdk.delegated_keys + dynamodb_encryption_sdk.delegated_keys.jce dynamodb_encryption_sdk.encrypted dynamodb_encryption_sdk.encrypted.client dynamodb_encryption_sdk.encrypted.item @@ -21,29 +23,28 @@ Modules dynamodb_encryption_sdk.material_providers.aws_kms dynamodb_encryption_sdk.material_providers.static dynamodb_encryption_sdk.material_providers.wrapped - dynamodb_encryption_sdk.material_providers.store dynamodb_encryption_sdk.materials dynamodb_encryption_sdk.materials.raw dynamodb_encryption_sdk.materials.wrapped dynamodb_encryption_sdk.internal - dynamodb_encryption_sdk.internal.defaults dynamodb_encryption_sdk.internal.dynamodb_types dynamodb_encryption_sdk.internal.identifiers dynamodb_encryption_sdk.internal.str_ops dynamodb_encryption_sdk.internal.utils + dynamodb_encryption_sdk.internal.validators dynamodb_encryption_sdk.internal.crypto + dynamodb_encryption_sdk.internal.crypto.authentication + dynamodb_encryption_sdk.internal.crypto.encryption dynamodb_encryption_sdk.internal.crypto.jce_bridge dynamodb_encryption_sdk.internal.crypto.jce_bridge.authentication dynamodb_encryption_sdk.internal.crypto.jce_bridge.encryption dynamodb_encryption_sdk.internal.crypto.jce_bridge.primitives - dynamodb_encryption_sdk.internal.crypto.authentication - dynamodb_encryption_sdk.internal.crypto.encryption dynamodb_encryption_sdk.internal.formatting + dynamodb_encryption_sdk.internal.formatting.material_description + dynamodb_encryption_sdk.internal.formatting.transform dynamodb_encryption_sdk.internal.formatting.deserialize dynamodb_encryption_sdk.internal.formatting.deserialize.attribute dynamodb_encryption_sdk.internal.formatting.serialize dynamodb_encryption_sdk.internal.formatting.serialize.attribute - dynamodb_encryption_sdk.internal.formatting.material_description - dynamodb_encryption_sdk.internal.formatting.transform .. include:: ../CHANGELOG.rst diff --git a/src/dynamodb_encryption_sdk/internal/defaults.py b/examples/src/__init__.py similarity index 83% rename from src/dynamodb_encryption_sdk/internal/defaults.py rename to examples/src/__init__.py index d262409a..b08a227c 100644 --- a/src/dynamodb_encryption_sdk/internal/defaults.py +++ b/examples/src/__init__.py @@ -10,8 +10,4 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""""" - -ENCODING = 'utf-8' -LOGGING_NAME = 'dynamodb_encryption_sdk' -MATERIAL_DESCRIPTION_VERSION = b'\00' * 4 +"""Stub module indicator to make linter configuration simpler.""" diff --git a/examples/src/aws_kms_encrypted_client.py b/examples/src/aws_kms_encrypted_client.py new file mode 100644 index 00000000..46913c62 --- /dev/null +++ b/examples/src/aws_kms_encrypted_client.py @@ -0,0 +1,183 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Example showing use of AWS KMS CMP with EncryptedClient.""" +import boto3 +from dynamodb_encryption_sdk.encrypted.client import EncryptedClient +from dynamodb_encryption_sdk.identifiers import CryptoAction +from dynamodb_encryption_sdk.material_providers.aws_kms import AwsKmsCryptographicMaterialsProvider +from dynamodb_encryption_sdk.structures import AttributeActions + + +def encrypt_item(table_name, aws_cmk_id): + """Demonstrate use of EncryptedClient to transparently encrypt an item.""" + index_key = { + 'partition_attribute': {'S': 'is this'}, + 'sort_attribute': {'N': '55'} + } + plaintext_item = { + 'example': {'S': 'data'}, + 'some numbers': {'N': '99'}, + 'and some binary': {'B': b'\x00\x01\x02'}, + 'leave me': {'S': 'alone'} # We want to ignore this attribute + } + # Collect all of the attributes that will be encrypted (used later). + encrypted_attributes = set(plaintext_item.keys()) + encrypted_attributes.remove('leave me') + # Collect all of the attributes that will not be encrypted (used later). + unencrypted_attributes = set(index_key.keys()) + unencrypted_attributes.add('leave me') + # Add the index pairs to the item. + plaintext_item.update(index_key) + + # Create a normal client. + client = boto3.client('dynamodb') + # Create a crypto materials provider using the specified AWS KMS key. + aws_kms_cmp = AwsKmsCryptographicMaterialsProvider(key_id=aws_cmk_id) + # Create attribute actions that tells the encrypted client to encrypt all attributes except one. + actions = AttributeActions( + default_action=CryptoAction.ENCRYPT_AND_SIGN, + attribute_actions={ + 'leave me': CryptoAction.DO_NOTHING + } + ) + # Use these objects to create an encrypted client. + encrypted_table = EncryptedClient( + client=client, + materials_provider=aws_kms_cmp, + attribute_actions=actions + ) + + # Put the item to the table, using the encrypted client to transparently encrypt it. + encrypted_table.put_item(TableName=table_name, Item=plaintext_item) + + # Get the encrypted item using the standard client. + encrypted_item = client.get_item(TableName=table_name, Key=index_key)['Item'] + + # Get the item using the encrypted client, transparently decyrpting it. + decrypted_item = encrypted_table.get_item(TableName=table_name, Key=index_key)['Item'] + + # Verify that all of the attributes are different in the encrypted item + for name in encrypted_attributes: + assert encrypted_item[name] != plaintext_item[name] + assert decrypted_item[name] == plaintext_item[name] + + # Verify that all of the attributes that should not be encrypted were not. + for name in unencrypted_attributes: + assert decrypted_item[name] == encrypted_item[name] == plaintext_item[name] + + # Clean up the item + encrypted_table.delete_item(TableName=table_name, Key=index_key) + + +def encrypt_batch_items(table_name, aws_cmk_id): + """Demonstrate use of EncryptedClient to transparently encrypt multiple items in a batch request.""" + index_keys = [ + { + 'partition_attribute': {'S': 'is this'}, + 'sort_attribute': {'N': '55'} + }, + { + 'partition_attribute': {'S': 'is this'}, + 'sort_attribute': {'N': '56'} + }, + { + 'partition_attribute': {'S': 'is this'}, + 'sort_attribute': {'N': '57'} + }, + { + 'partition_attribute': {'S': 'another'}, + 'sort_attribute': {'N': '55'} + } + ] + plaintext_additional_attributes = { + 'example': {'S': 'data'}, + 'some numbers': {'N': '99'}, + 'and some binary': {'B': b'\x00\x01\x02'}, + 'leave me': {'S': 'alone'} # We want to ignore this attribute + } + plaintext_items = [] + for key in index_keys: + _attributes = key.copy() + _attributes.update(plaintext_additional_attributes) + plaintext_items.append(_attributes) + + # Collect all of the attributes that will be encrypted (used later). + encrypted_attributes = set(plaintext_additional_attributes.keys()) + encrypted_attributes.remove('leave me') + # Collect all of the attributes that will not be encrypted (used later). + unencrypted_attributes = set(index_keys[0].keys()) + unencrypted_attributes.add('leave me') + + # Create a normal client. + client = boto3.client('dynamodb') + # Create a crypto materials provider using the specified AWS KMS key. + aws_kms_cmp = AwsKmsCryptographicMaterialsProvider(key_id=aws_cmk_id) + # Create attribute actions that tells the encrypted client to encrypt all attributes except one. + actions = AttributeActions( + default_action=CryptoAction.ENCRYPT_AND_SIGN, + attribute_actions={ + 'leave me': CryptoAction.DO_NOTHING + } + ) + # Use these objects to create an encrypted client. + encrypted_client = EncryptedClient( + client=client, + materials_provider=aws_kms_cmp, + attribute_actions=actions + ) + + # Put the items to the table, using the encrypted client to transparently encrypt them. + encrypted_client.batch_write_item(RequestItems={ + table_name: [{'PutRequest': {'Item': item}} for item in plaintext_items] + }) + + # Get the encrypted item using the standard client. + encrypted_items = client.batch_get_item( + RequestItems={table_name: {'Keys': index_keys}} + )['Responses'][table_name] + + # Get the item using the encrypted client, transparently decyrpting it. + decrypted_items = encrypted_client.batch_get_item( + RequestItems={table_name: {'Keys': index_keys}} + )['Responses'][table_name] + + def _select_index_from_item(item): + """Find the index keys that match this item.""" + for index in index_keys: + if all([item[key] == value for key, value in index.items()]): + return index + + def _select_item_from_index(index, all_items): + """Find the item that matches these index keys.""" + for item in all_items: + if all([item[key] == value for key, value in index.items()]): + return item + + for encrypted_item in encrypted_items: + key = _select_index_from_item(encrypted_item) + plaintext_item = _select_item_from_index(key, plaintext_items) + decrypted_item = _select_item_from_index(key, decrypted_items) + + # Verify that all of the attributes are different in the encrypted item + for name in encrypted_attributes: + assert encrypted_item[name] != plaintext_item[name] + assert decrypted_item[name] == plaintext_item[name] + + # Verify that all of the attributes that should not be encrypted were not. + for name in unencrypted_attributes: + assert decrypted_item[name] == encrypted_item[name] == plaintext_item[name] + + # Clean up the item + encrypted_client.batch_write_item(RequestItems={ + table_name: [{'DeleteRequest': {'Key': key}} for key in index_keys] + }) diff --git a/examples/src/aws_kms_encrypted_item.py b/examples/src/aws_kms_encrypted_item.py new file mode 100644 index 00000000..30f04b6a --- /dev/null +++ b/examples/src/aws_kms_encrypted_item.py @@ -0,0 +1,101 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Example showing use of AWS KMS CMP with item encryption functions directly.""" +import boto3 +from boto3.dynamodb.types import Binary +from dynamodb_encryption_sdk.encrypted import CryptoConfig +from dynamodb_encryption_sdk.encrypted.item import decrypt_python_item, encrypt_python_item +from dynamodb_encryption_sdk.identifiers import CryptoAction +from dynamodb_encryption_sdk.material_providers.aws_kms import AwsKmsCryptographicMaterialsProvider +from dynamodb_encryption_sdk.structures import AttributeActions, EncryptionContext, TableInfo +from dynamodb_encryption_sdk.transform import dict_to_ddb + + +def encrypt_item(table_name, aws_cmk_id): + """Demonstrate use of EncryptedTable to transparently encrypt an item.""" + index_key = { + 'partition_attribute': 'is this', + 'sort_attribute': 55 + } + plaintext_item = { + 'example': 'data', + 'some numbers': 99, + 'and some binary': Binary(b'\x00\x01\x02'), + 'leave me': 'alone' # We want to ignore this attribute + } + # Collect all of the attributes that will be encrypted (used later). + encrypted_attributes = set(plaintext_item.keys()) + encrypted_attributes.remove('leave me') + # Collect all of the attributes that will not be encrypted (used later). + unencrypted_attributes = set(index_key.keys()) + unencrypted_attributes.add('leave me') + # Add the index pairs to the item. + plaintext_item.update(index_key) + + # Create a normal table resource. + table = boto3.resource('dynamodb').Table(table_name) + + # Use the TableInfo helper to collect information about the indexes. + table_info = TableInfo(name=table_name) + table_info.refresh_indexed_attributes(table.meta.client) + + # Create a crypto materials provider using the specified AWS KMS key. + aws_kms_cmp = AwsKmsCryptographicMaterialsProvider(key_id=aws_cmk_id) + + encryption_context = EncryptionContext( + table_name=table_name, + partition_key_name=table_info.primary_index.partition, + sort_key_name=table_info.primary_index.sort, + # The only attributes that are used by the AWS KMS cryptographic materials providers + # are the primary index attributes. + # These attributes need to be in the form of a DynamoDB JSON structure, so first + # convert the standard dictionary. + attributes=dict_to_ddb(index_key) + ) + + # Create attribute actions that tells the encrypted table to encrypt all attributes + # except the primary index attributes and the one identified attribute to ignore. + actions = AttributeActions( + default_action=CryptoAction.ENCRYPT_AND_SIGN, + attribute_actions={ + name: CryptoAction.DO_NOTHING + for name in set(['leave me']).union(table_info.protected_index_keys()) + } + ) + + # Build the crypto config to use for this item. + # When using the higher-level helpers, this is handled for you. + crypto_config = CryptoConfig( + materials_provider=aws_kms_cmp, + encryption_context=encryption_context, + attribute_actions=actions + ) + + # Encrypt the plaintext item directly + encrypted_item = encrypt_python_item(plaintext_item, crypto_config) + + # You could now put the encrypted item to DynamoDB just as you would any other item. + # table.put_item(Item=encrypted_item) + # We will skip this for the purposes of this example. + + # Decrypt the encrypted item directly + decrypted_item = decrypt_python_item(encrypted_item, crypto_config) + + # Verify that all of the attributes are different in the encrypted item + for name in encrypted_attributes: + assert encrypted_item[name] != plaintext_item[name] + assert decrypted_item[name] == plaintext_item[name] + + # Verify that all of the attributes that should not be encrypted were not. + for name in unencrypted_attributes: + assert decrypted_item[name] == encrypted_item[name] == plaintext_item[name] diff --git a/examples/src/aws_kms_encrypted_resource.py b/examples/src/aws_kms_encrypted_resource.py new file mode 100644 index 00000000..42f20460 --- /dev/null +++ b/examples/src/aws_kms_encrypted_resource.py @@ -0,0 +1,123 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Example showing use of AWS KMS CMP with EncryptedResource.""" +import boto3 +from boto3.dynamodb.types import Binary +from dynamodb_encryption_sdk.encrypted.resource import EncryptedResource +from dynamodb_encryption_sdk.identifiers import CryptoAction +from dynamodb_encryption_sdk.material_providers.aws_kms import AwsKmsCryptographicMaterialsProvider +from dynamodb_encryption_sdk.structures import AttributeActions + + +def encrypt_batch_items(table_name, aws_cmk_id): + """Demonstrate use of EncryptedResource to transparently encrypt multiple items in a batch request.""" + index_keys = [ + { + 'partition_attribute': 'is this', + 'sort_attribute': 55 + }, + { + 'partition_attribute': 'is this', + 'sort_attribute': 56 + }, + { + 'partition_attribute': 'is this', + 'sort_attribute': 57 + }, + { + 'partition_attribute': 'another', + 'sort_attribute': 55 + } + ] + plaintext_additional_attributes = { + 'example': 'data', + 'some numbers': 99, + 'and some binary': Binary(b'\x00\x01\x02'), + 'leave me': 'alone' # We want to ignore this attribute + } + plaintext_items = [] + for key in index_keys: + _attributes = key.copy() + _attributes.update(plaintext_additional_attributes) + plaintext_items.append(_attributes) + + # Collect all of the attributes that will be encrypted (used later). + encrypted_attributes = set(plaintext_additional_attributes.keys()) + encrypted_attributes.remove('leave me') + # Collect all of the attributes that will not be encrypted (used later). + unencrypted_attributes = set(index_keys[0].keys()) + unencrypted_attributes.add('leave me') + + # Create a normal service resource. + resource = boto3.resource('dynamodb') + # Create a crypto materials provider using the specified AWS KMS key. + aws_kms_cmp = AwsKmsCryptographicMaterialsProvider(key_id=aws_cmk_id) + # Create attribute actions that tells the encrypted resource to encrypt all attributes except one. + actions = AttributeActions( + default_action=CryptoAction.ENCRYPT_AND_SIGN, + attribute_actions={ + 'leave me': CryptoAction.DO_NOTHING + } + ) + # Use these objects to create an encrypted service resource. + encrypted_resource = EncryptedResource( + resource=resource, + materials_provider=aws_kms_cmp, + attribute_actions=actions + ) + + # Put the items to the table, using the encrypted service resource to transparently encrypt them. + encrypted_resource.batch_write_item(RequestItems={ + table_name: [{'PutRequest': {'Item': item}} for item in plaintext_items] + }) + + # Get the encrypted item using the standard service resource. + encrypted_items = resource.batch_get_item( + RequestItems={table_name: {'Keys': index_keys}} + )['Responses'][table_name] + + # Get the item using the encrypted service resource, transparently decyrpting it. + decrypted_items = encrypted_resource.batch_get_item( + RequestItems={table_name: {'Keys': index_keys}} + )['Responses'][table_name] + + def _select_index_from_item(item): + """Find the index keys that match this item.""" + for index in index_keys: + if all([item[key] == value for key, value in index.items()]): + return index + + def _select_item_from_index(index, all_items): + """Find the item that matches these index keys.""" + for item in all_items: + if all([item[key] == value for key, value in index.items()]): + return item + + for encrypted_item in encrypted_items: + key = _select_index_from_item(encrypted_item) + plaintext_item = _select_item_from_index(key, plaintext_items) + decrypted_item = _select_item_from_index(key, decrypted_items) + + # Verify that all of the attributes are different in the encrypted item + for name in encrypted_attributes: + assert encrypted_item[name] != plaintext_item[name] + assert decrypted_item[name] == plaintext_item[name] + + # Verify that all of the attributes that should not be encrypted were not. + for name in unencrypted_attributes: + assert decrypted_item[name] == encrypted_item[name] == plaintext_item[name] + + # Clean up the item + encrypted_resource.batch_write_item(RequestItems={ + table_name: [{'DeleteRequest': {'Key': key}} for key in index_keys] + }) diff --git a/examples/src/aws_kms_encrypted_table.py b/examples/src/aws_kms_encrypted_table.py new file mode 100644 index 00000000..9607a258 --- /dev/null +++ b/examples/src/aws_kms_encrypted_table.py @@ -0,0 +1,80 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Example showing use of AWS KMS CMP with EncryptedTable.""" +import boto3 +from boto3.dynamodb.types import Binary +from dynamodb_encryption_sdk.encrypted.table import EncryptedTable +from dynamodb_encryption_sdk.identifiers import CryptoAction +from dynamodb_encryption_sdk.material_providers.aws_kms import AwsKmsCryptographicMaterialsProvider +from dynamodb_encryption_sdk.structures import AttributeActions + + +def encrypt_item(table_name, aws_cmk_id): + """Demonstrate use of EncryptedTable to transparently encrypt an item.""" + index_key = { + 'partition_attribute': 'is this', + 'sort_attribute': 55 + } + plaintext_item = { + 'example': 'data', + 'some numbers': 99, + 'and some binary': Binary(b'\x00\x01\x02'), + 'leave me': 'alone' # We want to ignore this attribute + } + # Collect all of the attributes that will be encrypted (used later). + encrypted_attributes = set(plaintext_item.keys()) + encrypted_attributes.remove('leave me') + # Collect all of the attributes that will not be encrypted (used later). + unencrypted_attributes = set(index_key.keys()) + unencrypted_attributes.add('leave me') + # Add the index pairs to the item. + plaintext_item.update(index_key) + + # Create a normal table resource. + table = boto3.resource('dynamodb').Table(table_name) + # Create a crypto materials provider using the specified AWS KMS key. + aws_kms_cmp = AwsKmsCryptographicMaterialsProvider(key_id=aws_cmk_id) + # Create attribute actions that tells the encrypted table to encrypt all attributes except one. + actions = AttributeActions( + default_action=CryptoAction.ENCRYPT_AND_SIGN, + attribute_actions={ + 'leave me': CryptoAction.DO_NOTHING + } + ) + # Use these objects to create an encrypted table resource. + encrypted_table = EncryptedTable( + table=table, + materials_provider=aws_kms_cmp, + attribute_actions=actions + ) + + # Put the item to the table, using the encrypted table resource to transparently encrypt it. + encrypted_table.put_item(Item=plaintext_item) + + # Get the encrypted item using the standard table resource. + encrypted_item = table.get_item(Key=index_key)['Item'] + + # Get the item using the encrypted table resource, transparently decyrpting it. + decrypted_item = encrypted_table.get_item(Key=index_key)['Item'] + + # Verify that all of the attributes are different in the encrypted item + for name in encrypted_attributes: + assert encrypted_item[name] != plaintext_item[name] + assert decrypted_item[name] == plaintext_item[name] + + # Verify that all of the attributes that should not be encrypted were not. + for name in unencrypted_attributes: + assert decrypted_item[name] == encrypted_item[name] == plaintext_item[name] + + # Clean up the item + encrypted_table.delete_item(Key=index_key) diff --git a/examples/src/pylintrc b/examples/src/pylintrc new file mode 100644 index 00000000..de56ef0f --- /dev/null +++ b/examples/src/pylintrc @@ -0,0 +1,12 @@ +[BASIC] +# Allow function names up to 50 characters +function-rgx = [a-z_][a-z0-9_]{2,50}$ + +[DESIGN] +max-args = 10 + +[FORMAT] +max-line-length = 120 + +[REPORTS] +msg-template = {path}:{line}: [{msg_id}({symbol}), {obj}] {msg} diff --git a/examples/test/__init__.py b/examples/test/__init__.py new file mode 100644 index 00000000..b08a227c --- /dev/null +++ b/examples/test/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Stub module indicator to make linter configuration simpler.""" diff --git a/examples/test/pylintrc b/examples/test/pylintrc new file mode 100644 index 00000000..ed659e0d --- /dev/null +++ b/examples/test/pylintrc @@ -0,0 +1,23 @@ +[MESSAGES CONTROL] +# Disabling messages that we either don't care about +# for tests or are necessary to break for tests. +# +# C0103 : invalid-name (we prefer long, descriptive, names for tests) +# C0111 : missing-docstring (we generally don't write docstrings for tests) +# C0413 : wrong-import-position (similar to E0401, pylint does not appear to identify +# unknown modules as non-standard-library. flake8 tests for this as well +# and does treat them properly) +# E0401 : import-error (because the examples are not actually in a module, sys.path +# is patched to find tests and test utils. pylint does not recognize this) +# R0801 : duplicate-code (tests for similar things tend to be similar) +# W0621 : redefined-outer-name (raises false positives with fixtures) +disable = C0103, C0111, C0413, E0401, R0801, W0621 + +[DESIGN] +max-args = 10 + +[FORMAT] +max-line-length = 120 + +[REPORTS] +msg-template = {path}:{line}: [{msg_id}({symbol}), {obj}] {msg} diff --git a/examples/test/test_aws_kms_encrypted_examples.py b/examples/test/test_aws_kms_encrypted_examples.py new file mode 100644 index 00000000..3427a592 --- /dev/null +++ b/examples/test/test_aws_kms_encrypted_examples.py @@ -0,0 +1,49 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Test ``aws_kms_encrypted_*`` examples.""" +import os +import sys +sys.path.extend([ # noqa + os.sep.join([os.path.dirname(__file__), '..', '..', 'test', 'integration']), + os.sep.join([os.path.dirname(__file__), '..', 'src']) +]) + +import pytest + +import aws_kms_encrypted_client # noqa +import aws_kms_encrypted_item # noqa +import aws_kms_encrypted_resource # noqa +import aws_kms_encrypted_table # noqa +from integration_test_utils import cmk_arn, ddb_table_name # noqa pylint: disable=unused-import + +pytestmark = [pytest.mark.examples] + + +def test_aws_kms_encrypted_table(ddb_table_name, cmk_arn): + aws_kms_encrypted_table.encrypt_item(ddb_table_name, cmk_arn) + + +def test_aws_kms_encrypted_client_item(ddb_table_name, cmk_arn): + aws_kms_encrypted_client.encrypt_item(ddb_table_name, cmk_arn) + + +def test_aws_kms_encrypted_client_batch_items(ddb_table_name, cmk_arn): + aws_kms_encrypted_client.encrypt_batch_items(ddb_table_name, cmk_arn) + + +def test_aws_kms_encrypted_item(ddb_table_name, cmk_arn): + aws_kms_encrypted_item.encrypt_item(ddb_table_name, cmk_arn) + + +def test_aws_kms_encrypted_resource(ddb_table_name, cmk_arn): + aws_kms_encrypted_resource.encrypt_batch_items(ddb_table_name, cmk_arn) diff --git a/setup.cfg b/setup.cfg index 0a1d775e..d1c69221 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,6 +19,9 @@ markers = unit: mark test as a unit test (does not require network access) functional: mark test as a functional test (does not require network access) integ: mark a test as an integration test (requires network access) + accept: mark a test as an acceptance test (requires network access) + examples: mark a test as an examples test (requires network access) + hypothesis: mark a test as using hypothesis (will run many times for each pytest call) slow: mark a test as being known to take a long time to complete (order 5s < t < 60s) veryslow: mark a test as being known to take a very long time to complete (order t > 60s) nope: mark a test as being so slow that it should only be very infrequently (order t > 30m) diff --git a/src/dynamodb_encryption_sdk/__init__.py b/src/dynamodb_encryption_sdk/__init__.py index a7c66779..82a7c208 100644 --- a/src/dynamodb_encryption_sdk/__init__.py +++ b/src/dynamodb_encryption_sdk/__init__.py @@ -10,19 +10,14 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""""" +"""DynamoDB Encryption Client.""" +from dynamodb_encryption_sdk.encrypted.client import EncryptedClient from dynamodb_encryption_sdk.encrypted.item import ( decrypt_dynamodb_item, decrypt_python_item, encrypt_dynamodb_item, encrypt_python_item ) - -# encrypt_item -# encrypt_raw_item -# decrypt_item -# decrypt_raw_item -# EncryptedTable -# EncryptedResource -# EncryptedClient +from dynamodb_encryption_sdk.encrypted.resource import EncryptedResource +from dynamodb_encryption_sdk.encrypted.table import EncryptedTable # TableConfiguration # MaterialDescription @@ -30,5 +25,6 @@ __all__ = ( 'decrypt_dynamodb_item', 'decrypt_python_item', - 'encrypt_dynamodb_item', 'encrypt_python_item' + 'encrypt_dynamodb_item', 'encrypt_python_item', + 'EncryptedClient', 'EncryptedResource', 'EncryptedTable' ) diff --git a/src/dynamodb_encryption_sdk/delegated_keys/__init__.py b/src/dynamodb_encryption_sdk/delegated_keys/__init__.py index 89bf3d69..9c2ad28e 100644 --- a/src/dynamodb_encryption_sdk/delegated_keys/__init__.py +++ b/src/dynamodb_encryption_sdk/delegated_keys/__init__.py @@ -13,14 +13,25 @@ """Delegated keys.""" import abc try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Dict, Text # pylint: disable=unused-import + from typing import Dict, Text # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass import six -from dynamodb_encryption_sdk.identifiers import EncryptionKeyTypes +from dynamodb_encryption_sdk.identifiers import EncryptionKeyType # noqa pylint: disable=unused-import + +__all__ = ('DelegatedKey',) + + +def _raise_not_implemented(method_name): + """Raises a standardized ``NotImplementedError`` to report that the specified method + is not supported. + + :raises NotImplementedError: when called + """ + raise NotImplementedError('"{}" is not supported by this DelegatedKey'.format(method_name)) @six.add_metaclass(abc.ABCMeta) @@ -31,6 +42,7 @@ class DelegatedKey(object): Unless overridden by a subclass, any method that a delegated key does not implement raises a ``NotImplementedError`` detailing this. """ + #: Most delegated keys should not be used with RawCryptographicMaterials. allowed_for_raw_materials = False @@ -39,16 +51,10 @@ def algorithm(self): # type: () -> Text """Text description of algorithm used by this delegated key.""" - def _raise_not_implemented(self, method_name): - """Raises a standardized ``NotImplementedError`` to report that the specified method - is not supported. - - :raises NotImplementedError: when called - """ - raise NotImplementedError('"{}" is not supported by this DelegatedKey'.format(method_name)) - @classmethod def generate(cls, algorithm, key_length): + # type: (Text, int) -> None + # pylint: disable=unused-argument,no-self-use """Generate an instance of this DelegatedKey using the specified algorithm and key length. :param str algorithm: Text description of algorithm to be used @@ -56,10 +62,11 @@ def generate(cls, algorithm, key_length): :returns: Generated delegated key :rtype: dynamodb_encryption_sdk.delegated_keys.DelegatedKey """ - cls._raise_not_implemented('generate') + _raise_not_implemented('generate') def encrypt(self, algorithm, name, plaintext, additional_associated_data=None): # type: (Text, Text, bytes, Dict[Text, Text]) -> bytes + # pylint: disable=unused-argument,no-self-use """Encrypt data. :param str algorithm: Text description of algorithm to use to encrypt data @@ -70,10 +77,11 @@ def encrypt(self, algorithm, name, plaintext, additional_associated_data=None): :returns: Encrypted ciphertext :rtype: bytes """ - self._raise_not_implemented('encrypt') + _raise_not_implemented('encrypt') def decrypt(self, algorithm, name, ciphertext, additional_associated_data=None): # type: (Text, Text, bytes, Dict[Text, Text]) -> bytes + # pylint: disable=unused-argument,no-self-use """Encrypt data. :param str algorithm: Text description of algorithm to use to decrypt data @@ -84,10 +92,11 @@ def decrypt(self, algorithm, name, ciphertext, additional_associated_data=None): :returns: Decrypted plaintext :rtype: bytes """ - self._raise_not_implemented('decrypt') + _raise_not_implemented('decrypt') def wrap(self, algorithm, content_key, additional_associated_data=None): # type: (Text, bytes, Dict[Text, Text]) -> bytes + # pylint: disable=unused-argument,no-self-use """Wrap content key. :param str algorithm: Text description of algorithm to use to wrap key @@ -97,26 +106,28 @@ def wrap(self, algorithm, content_key, additional_associated_data=None): :returns: Wrapped key :rtype: bytes """ - self._raise_not_implemented('wrap') + _raise_not_implemented('wrap') def unwrap(self, algorithm, wrapped_key, wrapped_key_algorithm, wrapped_key_type, additional_associated_data=None): - # type: (Text, bytes, Text, EncryptionKeyTypes, Dict[Text, Text]) -> DelegatedKey + # type: (Text, bytes, Text, EncryptionKeyType, Dict[Text, Text]) -> DelegatedKey + # pylint: disable=unused-argument,no-self-use """Wrap content key. :param str algorithm: Text description of algorithm to use to unwrap key :param bytes content_key: Raw content key to wrap :param str wrapped_key_algorithm: Text description of algorithm for unwrapped key to use :param wrapped_key_type: Type of key to treat key as once unwrapped - :type wrapped_key_type: dynamodb_encryption_sdk.identifiers.EncryptionKeyTypes + :type wrapped_key_type: dynamodb_encryption_sdk.identifiers.EncryptionKeyType :param dict additional_associated_data: Not used by all delegated keys, but if it is, then if it is provided on wrap it must be required on unwrap. :returns: Delegated key using unwrapped key :rtype: dynamodb_encryption_sdk.delegated_keys.DelegatedKey """ - self._raise_not_implemented('unwrap') + _raise_not_implemented('unwrap') def sign(self, algorithm, data): # type: (Text, bytes) -> bytes + # pylint: disable=unused-argument,no-self-use """Sign data. :param str algorithm: Text description of algorithm to use to sign data @@ -124,20 +135,22 @@ def sign(self, algorithm, data): :returns: Signature value :rtype: bytes """ - self._raise_not_implemented('sign') + _raise_not_implemented('sign') def verify(self, algorithm, signature, data): # type: (Text, bytes, bytes) -> None + # pylint: disable=unused-argument,no-self-use """Sign data. :param str algorithm: Text description of algorithm to use to verify signature :param bytes signature: Signature to verify :param bytes data: Data over which to verify signature """ - self._raise_not_implemented('verify') + _raise_not_implemented('verify') def signing_algorithm(self): # type: () -> Text + # pylint: disable=no-self-use """Provides a description that can inform an appropriate cryptographic materials provider about how to build a DelegatedKey for signature verification. If implemented, the return value of this method is included in the material description written to @@ -146,4 +159,4 @@ def signing_algorithm(self): :returns: Signing algorithm identifier :rtype: str """ - self._raise_not_implemented('signing_algorithm') + _raise_not_implemented('signing_algorithm') diff --git a/src/dynamodb_encryption_sdk/delegated_keys/jce.py b/src/dynamodb_encryption_sdk/delegated_keys/jce.py index f57027b0..ad71fdc3 100644 --- a/src/dynamodb_encryption_sdk/delegated_keys/jce.py +++ b/src/dynamodb_encryption_sdk/delegated_keys/jce.py @@ -11,6 +11,8 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. """Delegated key that JCE StandardName algorithm values to determine behavior.""" +from __future__ import division + import logging import os @@ -20,30 +22,31 @@ from cryptography.hazmat.primitives.asymmetric import rsa import six -from . import DelegatedKey from dynamodb_encryption_sdk.exceptions import JceTransformationError, UnwrappingError -from dynamodb_encryption_sdk.identifiers import EncryptionKeyTypes, KeyEncodingType, LOGGER_NAME +from dynamodb_encryption_sdk.identifiers import EncryptionKeyType, KeyEncodingType, LOGGER_NAME from dynamodb_encryption_sdk.internal.crypto.jce_bridge import authentication, encryption, primitives +from . import DelegatedKey +__all__ = ('JceNameLocalDelegatedKey',) _LOGGER = logging.getLogger(LOGGER_NAME) def _generate_symmetric_key(key_length): """Generate a new AES key. - :param int key_length: Required key length in bytes + :param int key_length: Required key length in bits :returns: raw key, symmetric key identifier, and RAW encoding identifier - :rtype: tuple of bytes, EncryptionKeyTypes, and KeyEncodingType + :rtype: tuple of bytes, EncryptionKeyType, and KeyEncodingType """ - return os.urandom(key_length), EncryptionKeyTypes.SYMMETRIC, KeyEncodingType.RAW + return os.urandom(key_length // 8), EncryptionKeyType.SYMMETRIC, KeyEncodingType.RAW def _generate_rsa_key(key_length): """Generate a new RSA private key. - :param int key_length: Required key length in bytes + :param int key_length: Required key length in bits :returns: DER-encoded private key, private key identifier, and DER encoding identifier - :rtype: tuple of bytes, EncryptionKeyTypes, and KeyEncodingType + :rtype: tuple of bytes, EncryptionKeyType, and KeyEncodingType """ private_key = rsa.generate_private_key( public_exponent=65537, @@ -55,7 +58,7 @@ def _generate_rsa_key(key_length): format=serialization.PrivateFormat.PKCS8, encryption_algorithm=serialization.NoEncryption() ) - return key_bytes, EncryptionKeyTypes.PRIVATE, KeyEncodingType.DER + return key_bytes, EncryptionKeyType.PRIVATE, KeyEncodingType.DER _ALGORITHM_GENERATE_MAP = { @@ -64,20 +67,51 @@ def _generate_rsa_key(key_length): } -@attr.s(hash=False) +@attr.s class JceNameLocalDelegatedKey(DelegatedKey): + # pylint: disable=too-many-instance-attributes """Delegated key that uses JCE StandardName algorithm values to determine behavior. + Accepted algorithm names for this include: + + * `JCE Mac names`_ (for a signing key) + + * **HmacSHA512** + * **HmacSHA256** + * **HmacSHA384** + * **HmacSHA224** + + * `JCE Signature names`_ (for a signing key) + + * **SHA512withRSA** + * **SHA256withRSA** + * **SHA384withRSA** + * **SHA224withRSA** + + * `JCE Cipher names`_ (for an encryption key) + + * **RSA** + * **AES** + * **AESWrap** + + .. _JCE Mac names: + https://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html#Mac + .. _JCE Signature names: + https://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html#Signature + .. _JCE Cipher names: + https://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html#Cipher + :param bytes key: Raw key bytes :param str algorithm: JCE Standard Algorithm Name :param key_type: Identifies what type of key is being provided - :type key_type: dynamodb_encryption_sdk.identifiers.EncryptionKeyTypes + :type key_type: dynamodb_encryption_sdk.identifiers.EncryptionKeyType :param key_encoding: Identifies how the provided key is encoded :type key_encoding: dynamodb_encryption_sdk.identifiers.KeyEncodingTypes """ + key = attr.ib(validator=attr.validators.instance_of(bytes), repr=False) _algorithm = attr.ib(validator=attr.validators.instance_of(six.string_types)) - _key_type = attr.ib(validator=attr.validators.instance_of(EncryptionKeyTypes)) + _key_type = attr.ib(validator=attr.validators.instance_of(EncryptionKeyType)) _key_encoding = attr.ib(validator=attr.validators.instance_of(KeyEncodingType)) @property @@ -115,7 +149,11 @@ def __attrs_post_init__(self): except KeyError: pass else: - self.__key = key_transformer.load_key(self.key, self._key_type, self._key_encoding) + self.__key = key_transformer.load_key( # attrs confuses pylint: disable=attribute-defined-outside-init + self.key, + self._key_type, + self._key_encoding + ) self._enable_encryption() self._enable_wrap() return @@ -128,7 +166,11 @@ def __attrs_post_init__(self): except KeyError: pass else: - self.__key = key_transformer.load_key(self.key, self._key_type, self._key_encoding) + self.__key = key_transformer.load_key( # attrs confuses pylint: disable=attribute-defined-outside-init + self.key, + self._key_type, + self._key_encoding + ) self._enable_authentication() return @@ -140,7 +182,7 @@ def generate(cls, algorithm, key_length=None): """Generate an instance of this DelegatedKey using the specified algorithm and key length. :param str algorithm: Text description of algorithm to be used - :param int key_length: Size of key to generate + :param int key_length: Size in bits of key to generate :returns: Generated delegated key :rtype: dynamodb_encryption_sdk.delegated_keys.DelegatedKey """ @@ -171,6 +213,7 @@ def allowed_for_raw_materials(self): def _encrypt(self, algorithm, name, plaintext, additional_associated_data=None): # type: (Text, Text, bytes, Dict[Text, Text]) -> bytes + # pylint: disable=unused-argument """ Encrypt data. @@ -188,6 +231,7 @@ def _encrypt(self, algorithm, name, plaintext, additional_associated_data=None): def _decrypt(self, algorithm, name, ciphertext, additional_associated_data=None): # type: (Text, Text, bytes, Dict[Text, Text]) -> bytes + # pylint: disable=unused-argument """Encrypt data. :param str algorithm: Java StandardName transformation string of algorithm to use to decrypt data @@ -203,6 +247,7 @@ def _decrypt(self, algorithm, name, ciphertext, additional_associated_data=None) def _wrap(self, algorithm, content_key, additional_associated_data=None): # type: (Text, bytes, Dict[Text, Text]) -> bytes + # pylint: disable=unused-argument """Wrap content key. :param str algorithm: Text description of algorithm to use to wrap key @@ -218,19 +263,20 @@ def _wrap(self, algorithm, content_key, additional_associated_data=None): ) def _unwrap(self, algorithm, wrapped_key, wrapped_key_algorithm, wrapped_key_type, additional_associated_data=None): - # type: (Text, bytes, Text, EncryptionKeyTypes, Dict[Text, Text]) -> DelegatedKey + # type: (Text, bytes, Text, EncryptionKeyType, Dict[Text, Text]) -> DelegatedKey + # pylint: disable=unused-argument """Wrap content key. :param str algorithm: Text description of algorithm to use to unwrap key :param bytes content_key: Raw content key to wrap :param str wrapped_key_algorithm: Text description of algorithm for unwrapped key to use :param wrapped_key_type: Type of key to treat key as once unwrapped - :type wrapped_key_type: dynamodb_encryption_sdk.identifiers.EncryptionKeyTypes + :type wrapped_key_type: dynamodb_encryption_sdk.identifiers.EncryptionKeyType :param dict additional_associated_data: Not used by ``JceNameLocalDelegatedKey`` :returns: Delegated key using unwrapped key :rtype: dynamodb_encryption_sdk.delegated_keys.DelegatedKey """ - if wrapped_key_type is not EncryptionKeyTypes.SYMMETRIC: + if wrapped_key_type is not EncryptionKeyType.SYMMETRIC: raise UnwrappingError('Unsupported wrapped key type: "{}"'.format(wrapped_key_type)) unwrapper = encryption.JavaCipher.from_transformation(algorithm) diff --git a/src/dynamodb_encryption_sdk/encrypted/__init__.py b/src/dynamodb_encryption_sdk/encrypted/__init__.py index 1f717dd4..ec2051de 100644 --- a/src/dynamodb_encryption_sdk/encrypted/__init__.py +++ b/src/dynamodb_encryption_sdk/encrypted/__init__.py @@ -10,17 +10,27 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -import attr +"""Resources for encrypting items.""" import copy -import six -from dynamodb_encryption_sdk.identifiers import ItemAction +import attr + +try: # Python 3.5.0 and 3.5.1 have incompatible typing modules + from typing import Dict # noqa pylint: disable=unused-import +except ImportError: # pragma: no cover + # We only actually need these imports when running the mypy checks + pass + +from dynamodb_encryption_sdk.exceptions import InvalidArgumentError +from dynamodb_encryption_sdk.identifiers import CryptoAction from dynamodb_encryption_sdk.material_providers import CryptographicMaterialsProvider -from dynamodb_encryption_sdk.materials import DecryptionMaterials, EncryptionMaterials +from dynamodb_encryption_sdk.materials import DecryptionMaterials, EncryptionMaterials # noqa pylint: disable=unused-import from dynamodb_encryption_sdk.structures import AttributeActions, EncryptionContext +__all__ = ('CryptoConfig',) -@attr.s(hash=False) + +@attr.s class CryptoConfig(object): """Container for all configuration needed to encrypt or decrypt an item. @@ -31,24 +41,24 @@ class CryptoConfig(object): :param attribute_actions: Description of what action should be taken for each attribute :type attribute_actions: dynamodb_encryption_sdk.structures.AttributeActions """ + materials_provider = attr.ib(validator=attr.validators.instance_of(CryptographicMaterialsProvider)) encryption_context = attr.ib(validator=attr.validators.instance_of(EncryptionContext)) attribute_actions = attr.ib(validator=attr.validators.instance_of(AttributeActions)) def __attrs_post_init__(self): - """Make sure that restricted, indexed, attributes are not being encrypted.""" + # type: () -> None + """Make sure that primary index attributes are not being encrypted.""" if self.encryption_context.partition_key_name is not None: - if self.attribute_actions.action(self.encryption_context.partition_key_name) is ItemAction.ENCRYPT_AND_SIGN: - raise Exception('TODO:Cannot encrypt partition key') + if self.attribute_actions.action(self.encryption_context.partition_key_name) is CryptoAction.ENCRYPT_AND_SIGN: + raise InvalidArgumentError('Cannot encrypt partition key') if self.encryption_context.sort_key_name is not None: - if self.attribute_actions.action(self.encryption_context.sort_key_name) is ItemAction.ENCRYPT_AND_SIGN: - raise Exception('TODO:Cannot encrypt sort key') - - # TODO: secondary indexes? - # TODO: our own restricted attributes? + if self.attribute_actions.action(self.encryption_context.sort_key_name) is CryptoAction.ENCRYPT_AND_SIGN: + raise InvalidArgumentError('Cannot encrypt sort key') def decryption_materials(self): + # type: () -> DecryptionMaterials """Load decryption materials from instance resources. :returns: Decryption materials @@ -57,6 +67,7 @@ def decryption_materials(self): return self.materials_provider.decryption_materials(self.encryption_context) def encryption_materials(self): + # type: () -> EncryptionMaterials """Load encryption materials from instance resources. :returns: Encryption materials @@ -65,6 +76,7 @@ def encryption_materials(self): return self.materials_provider.encryption_materials(self.encryption_context) def copy(self): + # type: () -> CryptoConfig """Return a copy of this instance with a copied instance of its encryption context. :returns: New CryptoConfig identical to this one diff --git a/src/dynamodb_encryption_sdk/encrypted/client.py b/src/dynamodb_encryption_sdk/encrypted/client.py new file mode 100644 index 00000000..1a96bb98 --- /dev/null +++ b/src/dynamodb_encryption_sdk/encrypted/client.py @@ -0,0 +1,230 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""High-level helper class to provide a familiar interface to encrypted tables.""" +from functools import partial + +import attr +import botocore + +from dynamodb_encryption_sdk.internal.utils import ( + crypto_config_from_cache, crypto_config_from_kwargs, + decrypt_batch_get_item, decrypt_get_item, decrypt_multi_get, + encrypt_batch_write_item, encrypt_put_item, TableInfoCache, + validate_get_arguments +) +from dynamodb_encryption_sdk.internal.validators import callable_validator +from dynamodb_encryption_sdk.material_providers import CryptographicMaterialsProvider +from dynamodb_encryption_sdk.structures import AttributeActions +from .item import decrypt_dynamodb_item, decrypt_python_item, encrypt_dynamodb_item, encrypt_python_item + +__all__ = ('EncryptedClient',) + + +@attr.s +class EncryptedPaginator(object): + """Paginator that decrypts returned items before returning them. + + :param paginator: Pre-configured boto3 DynamoDB paginator object + :type paginator: botocore.paginate.Paginator + :param decrypt_method: Item decryptor method from ``dynamodb_encryption_sdk.encrypted.item`` + :param callable crypto_config_method: Callable that returns a crypto config + """ + + _paginator = attr.ib(validator=attr.validators.instance_of(botocore.paginate.Paginator)) + _decrypt_method = attr.ib() + _crypto_config_method = attr.ib(validator=callable_validator) + + @_decrypt_method.validator + def validate_decrypt_method(self, attribute, value): + # pylint: disable=unused-argument + """Validate that _decrypt_method is one of the item encryptors.""" + if self._decrypt_method not in (decrypt_python_item, decrypt_dynamodb_item): + raise ValueError( + '"{name}" must be an item decryptor from dynamodb_encryption_sdk.encrypted.item'.format( + name=attribute.name + ) + ) + + def __getattr__(self, name): + """Catch any method/attribute lookups that are not defined in this class and try + to find them on the provided client object. + + :param str name: Attribute name + :returns: Result of asking the provided client object for that attribute name + :raises AttributeError: if attribute is not found on provided client object + """ + return getattr(self._paginator, name) + + def paginate(self, **kwargs): + # type: (**Any) -> Dict + # TODO: narrow this down + """Create an iterator that will paginate through responses from the underlying paginator, + transparently decrypting any returned items. + """ + validate_get_arguments(kwargs) + + crypto_config, ddb_kwargs = self._crypto_config_method(**kwargs) + + for page in self._paginator.paginate(**ddb_kwargs): + for pos, value in enumerate(page['Items']): + page['Items'][pos] = self._decrypt_method( + item=value, + crypto_config=crypto_config + ) + yield page + + +@attr.s +class EncryptedClient(object): + # pylint: disable=too-few-public-methods,too-many-instance-attributes + """High-level helper class to provide a familiar interface to encrypted tables. + + >>> import boto3 + >>> from dynamodb_encryption_sdk.encrypted.client import EncryptedClient + >>> from dynamodb_encryption_sdk.material_providers.aws_kms import AwsKmsCryptographicMaterialsProvider + >>> client = boto3.client('dynamodb') + >>> aws_kms_cmp = AwsKmsCryptographicMaterialsProvider('alias/MyKmsAlias') + >>> encrypted_client = EncryptedClient( + ... client=client, + ... materials_provider=aws_kms_cmp + ... ) + + .. note:: + + This class provides a superset of the boto3 DynamoDB client API, so should work as + a drop-in replacement once configured. + + https://boto3.readthedocs.io/en/latest/reference/services/dynamodb.html#client + + If you want to provide per-request cryptographic details, the ``put_item``, ``get_item``, + ``query``, ``scan``, ``batch_write_item``, and ``batch_get_item`` methods will also + accept a ``crypto_config`` parameter, defining a custom ``CryptoConfig`` instance + for this request. + + .. warning:: + + We do not currently support the ``update_item`` method. + + :param table: Pre-configured boto3 DynamoDB client object + :type table: boto3.resources.base.BaseClient + :param materials_provider: Cryptographic materials provider to use + :type materials_provider: dynamodb_encryption_sdk.material_providers.CryptographicMaterialsProvider + :param attribute_actions: Table-level configuration of how to encrypt/sign attributes + :type attribute_actions: dynamodb_encryption_sdk.structures.AttributeActions + :param bool auto_refresh_table_indexes: Should we attempt to refresh information about table indexes? + Requires ``dynamodb:DescribeTable`` permissions on each table. (default: True) + :param bool expect_standard_dictionaries: Should we expect items to be standard Python + dictionaries? This should only be set to True if you are using a client obtained + from a service resource or table resource (ex: ``table.meta.client``). (default: False) + """ + + _client = attr.ib(validator=attr.validators.instance_of(botocore.client.BaseClient)) + _materials_provider = attr.ib(validator=attr.validators.instance_of(CryptographicMaterialsProvider)) + _attribute_actions = attr.ib( + validator=attr.validators.instance_of(AttributeActions), + default=attr.Factory(AttributeActions) + ) + _auto_refresh_table_indexes = attr.ib( + validator=attr.validators.instance_of(bool), + default=True + ) + _expect_standard_dictionaries = attr.ib( + validator=attr.validators.instance_of(bool), + default=False + ) + + def __attrs_post_init__(self): + """Set up the table info cache and translation methods.""" + if self._expect_standard_dictionaries: + self._encrypt_item = encrypt_python_item + self._decrypt_item = decrypt_python_item + else: + self._encrypt_item = encrypt_dynamodb_item + self._decrypt_item = decrypt_dynamodb_item + self._table_info_cache = TableInfoCache( # attrs confuses pylint: disable=attribute-defined-outside-init + client=self._client, + auto_refresh_table_indexes=self._auto_refresh_table_indexes + ) + self._table_crypto_config = partial( # attrs confuses pylint: disable=attribute-defined-outside-init + crypto_config_from_cache, + materials_provider=self._materials_provider, + attribute_actions=self._attribute_actions, + table_info_cache=self._table_info_cache + ) + self._item_crypto_config = partial( # attrs confuses pylint: disable=attribute-defined-outside-init + crypto_config_from_kwargs, + fallback=self._table_crypto_config + ) + self.get_item = partial( # attrs confuses pylint: disable=attribute-defined-outside-init + decrypt_get_item, + decrypt_method=self._decrypt_item, + crypto_config_method=self._item_crypto_config, + read_method=self._client.get_item + ) + self.put_item = partial( # attrs confuses pylint: disable=attribute-defined-outside-init + encrypt_put_item, + encrypt_method=self._encrypt_item, + crypto_config_method=self._item_crypto_config, + write_method=self._client.put_item + ) + self.query = partial( # attrs confuses pylint: disable=attribute-defined-outside-init + decrypt_multi_get, + decrypt_method=self._decrypt_item, + crypto_config_method=self._item_crypto_config, + read_method=self._client.query + ) + self.scan = partial( # attrs confuses pylint: disable=attribute-defined-outside-init + decrypt_multi_get, + decrypt_method=self._decrypt_item, + crypto_config_method=self._item_crypto_config, + read_method=self._client.scan + ) + self.batch_get_item = partial( # attrs confuses pylint: disable=attribute-defined-outside-init + decrypt_batch_get_item, + decrypt_method=self._decrypt_item, + crypto_config_method=self._table_crypto_config, + read_method=self._client.batch_get_item + ) + self.batch_write_item = partial( # attrs confuses pylint: disable=attribute-defined-outside-init + encrypt_batch_write_item, + encrypt_method=self._encrypt_item, + crypto_config_method=self._table_crypto_config, + write_method=self._client.batch_write_item + ) + + def __getattr__(self, name): + """Catch any method/attribute lookups that are not defined in this class and try + to find them on the provided client object. + + :param str name: Attribute name + :returns: Result of asking the provided client object for that attribute name + :raises AttributeError: if attribute is not found on provided client object + """ + return getattr(self._client, name) + + def update_item(self, **kwargs): + """Update item is not yet supported.""" + raise NotImplementedError('"update_item" is not yet implemented') + + def get_paginator(self, operation_name): + """""" + paginator = self._client.get_paginator(operation_name) + + if operation_name in ('scan', 'query'): + return EncryptedPaginator( + paginator=paginator, + decrypt_method=self._decrypt_item, + crypto_config_method=self._item_crypto_config + ) + + return paginator diff --git a/src/dynamodb_encryption_sdk/encrypted/item.py b/src/dynamodb_encryption_sdk/encrypted/item.py index ba8dbb9f..1847642f 100644 --- a/src/dynamodb_encryption_sdk/encrypted/item.py +++ b/src/dynamodb_encryption_sdk/encrypted/item.py @@ -12,23 +12,23 @@ # language governing permissions and limitations under the License. """Top-level functions for encrypting and decrypting DynamoDB items.""" try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Any, Callable, Dict # pylint: disable=unused-import - from dynamodb_encryption_sdk.internal import dynamodb_types # pylint: disable=unused-import + from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass -from . import CryptoConfig -from dynamodb_encryption_sdk.exceptions import DecryptionError -from dynamodb_encryption_sdk.identifiers import ItemAction +from dynamodb_encryption_sdk.exceptions import DecryptionError, EncryptionError +from dynamodb_encryption_sdk.identifiers import CryptoAction from dynamodb_encryption_sdk.internal.crypto.authentication import sign_item, verify_item_signature from dynamodb_encryption_sdk.internal.crypto.encryption import decrypt_attribute, encrypt_attribute from dynamodb_encryption_sdk.internal.formatting.material_description import ( deserialize as deserialize_material_description, serialize as serialize_material_description ) -from dynamodb_encryption_sdk.internal.identifiers import MaterialDescriptionKeys, MaterialDescriptionValues -from dynamodb_encryption_sdk.internal.formatting.transform import ddb_to_dict, dict_to_ddb -from dynamodb_encryption_sdk.internal.identifiers import ReservedAttributes +from dynamodb_encryption_sdk.internal.identifiers import ( + MaterialDescriptionKeys, MaterialDescriptionValues, ReservedAttributes +) +from dynamodb_encryption_sdk.transform import ddb_to_dict, dict_to_ddb +from . import CryptoConfig # noqa pylint: disable=unused-import __all__ = ('encrypt_dynamodb_item', 'encrypt_python_item', 'decrypt_dynamodb_item', 'decrypt_python_item') @@ -51,7 +51,12 @@ def encrypt_dynamodb_item(item, crypto_config): # If we explicitly have been told not to do anything to this item, just copy it. return item.copy() - # TODO: Check for attributes that we write + for reserved_name in ReservedAttributes: + if reserved_name.value in item: + raise EncryptionError('Reserved attribute name "{}" is not allowed in plaintext item.'.format( + reserved_name.value + )) + crypto_config.materials_provider.refresh() encryption_materials = crypto_config.encryption_materials() @@ -67,7 +72,7 @@ def encrypt_dynamodb_item(item, crypto_config): encrypted_item = {} for name, attribute in item.items(): - if crypto_config.attribute_actions.action(name) is not ItemAction.ENCRYPT_AND_SIGN: + if crypto_config.attribute_actions.action(name) is not CryptoAction.ENCRYPT_AND_SIGN: encrypted_item[name] = attribute.copy() continue @@ -167,7 +172,7 @@ def decrypt_dynamodb_item(item, crypto_config): # Once the signature has been verified, actually decrypt the item attributes. decrypted_item = {} for name, attribute in item.items(): - if inner_crypto_config.attribute_actions.action(name) is not ItemAction.ENCRYPT_AND_SIGN: + if inner_crypto_config.attribute_actions.action(name) is not CryptoAction.ENCRYPT_AND_SIGN: decrypted_item[name] = attribute.copy() continue diff --git a/src/dynamodb_encryption_sdk/encrypted/resource.py b/src/dynamodb_encryption_sdk/encrypted/resource.py new file mode 100644 index 00000000..ca8ff769 --- /dev/null +++ b/src/dynamodb_encryption_sdk/encrypted/resource.py @@ -0,0 +1,211 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""High-level helper class to provide a familiar interface to encrypted tables.""" +from functools import partial + +import attr +from boto3.resources.base import ServiceResource +from boto3.resources.collection import CollectionManager + +from dynamodb_encryption_sdk.internal.utils import ( + crypto_config_from_cache, decrypt_batch_get_item, encrypt_batch_write_item, TableInfoCache +) +from dynamodb_encryption_sdk.material_providers import CryptographicMaterialsProvider +from dynamodb_encryption_sdk.structures import AttributeActions +from .item import decrypt_python_item, encrypt_python_item +from .table import EncryptedTable + +__all__ = ('EncryptedResource',) + + +@attr.s +class EncryptedTablesCollectionManager(object): + # pylint: disable=too-few-public-methods + """Tables collection manager that provides EncryptedTable objects. + + https://boto3.readthedocs.io/en/latest/reference/services/dynamodb.html#DynamoDB.ServiceResource.tables + + :param collection: Pre-configured boto3 DynamoDB table collection manager + :type collection: boto3.resources.collection.CollectionManager + :param materials_provider: Cryptographic materials provider to use + :type materials_provider: dynamodb_encryption_sdk.material_providers.CryptographicMaterialsProvider + :param attribute_actions: Table-level configuration of how to encrypt/sign attributes + :type attribute_actions: dynamodb_encryption_sdk.structures.AttributeActions + :param table_info_cache: Local cache from which to obtain TableInfo data + :type table_info_cache: dynamodb_encryption_sdk.internal.utils.TableInfoCache + """ + + _collection = attr.ib(validator=attr.validators.instance_of(CollectionManager)) + _materials_provider = attr.ib(validator=attr.validators.instance_of(CryptographicMaterialsProvider)) + _attribute_actions = attr.ib(validator=attr.validators.instance_of(AttributeActions)) + _table_info_cache = attr.ib(validator=attr.validators.instance_of(TableInfoCache)) + + def __attrs_post_init__(self): + """Set up the translation methods.""" + self.all = partial( # attrs confuses pylint: disable=attribute-defined-outside-init + self._transform_table, + self._collection.all + ) + self.filter = partial( # attrs confuses pylint: disable=attribute-defined-outside-init + self._transform_table, + self._collection.filter + ) + self.limit = partial( # attrs confuses pylint: disable=attribute-defined-outside-init + self._transform_table, + self._collection.limit + ) + self.page_size = partial( # attrs confuses pylint: disable=attribute-defined-outside-init + self._transform_table, + self._collection.page_size + ) + + def __getattr__(self, name): + """Catch any method/attribute lookups that are not defined in this class and try + to find them on the provided collection object. + + :param str name: Attribute name + :returns: Result of asking the provided collection object for that attribute name + :raises AttributeError: if attribute is not found on provided collection object + """ + return getattr(self._collection, name) + + def _transform_table(self, method, **kwargs): + """Transform a Table from the underlying collection manager to an EncryptedTable. + + :param method: Method on underlying collection manager to call + :type method: callable + :param **kwargs: Keyword arguments to pass to ``method`` + """ + for table in method(**kwargs): + yield EncryptedTable( + table=table, + materials_provider=self._materials_provider, + table_info=self._table_info_cache.table_info(table.name), + attribute_actions=self._attribute_actions + ) + + +@attr.s +class EncryptedResource(object): + # pylint: disable=too-few-public-methods + """High-level helper class to provide a familiar interface to encrypted tables. + + >>> import boto3 + >>> from dynamodb_encryption_sdk.encrypted.resource import EncryptedResource + >>> from dynamodb_encryption_sdk.material_providers.aws_kms import AwsKmsCryptographicMaterialsProvider + >>> resource = boto3.resource('dynamodb') + >>> aws_kms_cmp = AwsKmsCryptographicMaterialsProvider('alias/MyKmsAlias') + >>> encrypted_resource = EncryptedResource( + ... resource=resource, + ... materials_provider=aws_kms_cmp + ... ) + + .. note:: + + This class provides a superset of the boto3 DynamoDB service resource API, so should + work as a drop-in replacement once configured. + + https://boto3.readthedocs.io/en/latest/reference/services/dynamodb.html#service-resource + + If you want to provide per-request cryptographic details, the ``batch_write_item`` + and ``batch_get_item`` methods will also accept a ``crypto_config`` parameter, defining + a custom ``CryptoConfig`` instance for this request. + + :param resource: Pre-configured boto3 DynamoDB service resource object + :type resource: boto3.resources.base.ServiceResource + :param materials_provider: Cryptographic materials provider to use + :type materials_provider: dynamodb_encryption_sdk.material_providers.CryptographicMaterialsProvider + :param attribute_actions: Table-level configuration of how to encrypt/sign attributes + :type attribute_actions: dynamodb_encryption_sdk.structures.AttributeActions + :param bool auto_refresh_table_indexes: Should we attempt to refresh information about table indexes? + Requires ``dynamodb:DescribeTable`` permissions on each table. (default: True) + """ + + _resource = attr.ib(validator=attr.validators.instance_of(ServiceResource)) + _materials_provider = attr.ib(validator=attr.validators.instance_of(CryptographicMaterialsProvider)) + _attribute_actions = attr.ib( + validator=attr.validators.instance_of(AttributeActions), + default=attr.Factory(AttributeActions) + ) + _auto_refresh_table_indexes = attr.ib( + validator=attr.validators.instance_of(bool), + default=True + ) + + def __attrs_post_init__(self): + """Set up the table info cache, encrypted tables collection manager, and translation methods.""" + self._table_info_cache = TableInfoCache( # attrs confuses pylint: disable=attribute-defined-outside-init + client=self._resource.meta.client, + auto_refresh_table_indexes=self._auto_refresh_table_indexes + ) + self._crypto_config = partial( # attrs confuses pylint: disable=attribute-defined-outside-init + crypto_config_from_cache, + materials_provider=self._materials_provider, + attribute_actions=self._attribute_actions, + table_info_cache=self._table_info_cache + ) + self.tables = EncryptedTablesCollectionManager( # attrs confuses pylint: disable=attribute-defined-outside-init + collection=self._resource.tables, + materials_provider=self._materials_provider, + attribute_actions=self._attribute_actions, + table_info_cache=self._table_info_cache + ) + self.batch_get_item = partial( # attrs confuses pylint: disable=attribute-defined-outside-init + decrypt_batch_get_item, + decrypt_method=decrypt_python_item, + crypto_config_method=self._crypto_config, + read_method=self._resource.batch_get_item + ) + self.batch_write_item = partial( # attrs confuses pylint: disable=attribute-defined-outside-init + encrypt_batch_write_item, + encrypt_method=encrypt_python_item, + crypto_config_method=self._crypto_config, + write_method=self._resource.batch_write_item + ) + + def __getattr__(self, name): + """Catch any method/attribute lookups that are not defined in this class and try + to find them on the provided resource object. + + :param str name: Attribute name + :returns: Result of asking the provided resource object for that attribute name + :raises AttributeError: if attribute is not found on provided resource object + """ + return getattr(self._resource, name) + + def Table(self, name, **kwargs): + # naming chosen to align with boto3 resource name, so pylint: disable=invalid-name + """Creates an EncryptedTable resource. + + If any of the optional configuration values are not provided, the corresponding values + for this ``EncryptedResource`` will be used. + + https://boto3.readthedocs.io/en/latest/reference/services/dynamodb.html#DynamoDB.ServiceResource.Table + + :param name: The table name. + :param materials_provider: Cryptographic materials provider to use (optional) + :type materials_provider: dynamodb_encryption_sdk.material_providers.CryptographicMaterialsProvider + :param table_info: Information about the target DynamoDB table (optional) + :type table_info: dynamodb_encryption_sdk.structures.TableInfo + :param attribute_actions: Table-level configuration of how to encrypt/sign attributes (optional) + :type attribute_actions: dynamodb_encryption_sdk.structures.AttributeActions + """ + table_kwargs = dict( + table=self._resource.Table(name), + materials_provider=kwargs.get('materials_provider', self._materials_provider), + attribute_actions=kwargs.get('attribute_actions', self._attribute_actions), + auto_refresh_table_indexes=kwargs.get('auto_refresh_table_indexes', self._auto_refresh_table_indexes), + table_info=self._table_info_cache.table_info(name) + ) + + return EncryptedTable(**table_kwargs) diff --git a/src/dynamodb_encryption_sdk/encrypted/table.py b/src/dynamodb_encryption_sdk/encrypted/table.py new file mode 100644 index 00000000..ee6fdcee --- /dev/null +++ b/src/dynamodb_encryption_sdk/encrypted/table.py @@ -0,0 +1,169 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""High-level helper class to provide a familiar interface to encrypted tables.""" +from functools import partial + +import attr +from boto3.dynamodb.table import BatchWriter +from boto3.resources.base import ServiceResource + +from dynamodb_encryption_sdk.internal.utils import ( + crypto_config_from_kwargs, crypto_config_from_table_info, + decrypt_get_item, decrypt_multi_get, encrypt_put_item +) +from dynamodb_encryption_sdk.material_providers import CryptographicMaterialsProvider +from dynamodb_encryption_sdk.structures import AttributeActions, TableInfo +from .client import EncryptedClient +from .item import decrypt_python_item, encrypt_python_item + +__all__ = ('EncryptedTable',) + + +@attr.s +class EncryptedTable(object): + # pylint: disable=too-few-public-methods + """High-level helper class to provide a familiar interface to encrypted tables. + + >>> import boto3 + >>> from dynamodb_encryption_sdk.encrypted.table import EncryptedTable + >>> from dynamodb_encryption_sdk.material_providers.aws_kms import AwsKmsCryptographicMaterialsProvider + >>> table = boto3.resource('dynamodb').Table('my_table') + >>> aws_kms_cmp = AwsKmsCryptographicMaterialsProvider('alias/MyKmsAlias') + >>> encrypted_table = EncryptedTable( + ... table=table, + ... materials_provider=aws_kms_cmp + ... ) + + .. note:: + + This class provides a superset of the boto3 DynamoDB Table API, so should work as + a drop-in replacement once configured. + + https://boto3.readthedocs.io/en/latest/reference/services/dynamodb.html#table + + If you want to provide per-request cryptographic details, the ``put_item``, ``get_item``, + ``query``, and ``scan`` methods will also accept a ``crypto_config`` parameter, defining + a custom ``CryptoConfig`` instance for this request. + + .. warning:: + + We do not currently support the ``update_item`` method. + + :param table: Pre-configured boto3 DynamoDB Table object + :type table: boto3.resources.base.ServiceResource + :param materials_provider: Cryptographic materials provider to use + :type materials_provider: dynamodb_encryption_sdk.material_providers.CryptographicMaterialsProvider + :param table_info: Information about the target DynamoDB table + :type table_info: dynamodb_encryption_sdk.structures.TableInfo + :param attribute_actions: Table-level configuration of how to encrypt/sign attributes + :type attribute_actions: dynamodb_encryption_sdk.structures.AttributeActions + :param bool auto_refresh_table_indexes: Should we attempt to refresh information about table indexes? + Requires ``dynamodb:DescribeTable`` permissions on each table. (default: True) + """ + + _table = attr.ib(validator=attr.validators.instance_of(ServiceResource)) + _materials_provider = attr.ib(validator=attr.validators.instance_of(CryptographicMaterialsProvider)) + _table_info = attr.ib( + validator=attr.validators.optional(attr.validators.instance_of(TableInfo)), + default=None + ) + _attribute_actions = attr.ib( + validator=attr.validators.instance_of(AttributeActions), + default=attr.Factory(AttributeActions) + ) + _auto_refresh_table_indexes = attr.ib( + validator=attr.validators.instance_of(bool), + default=True + ) + + def __attrs_post_init__(self): + """Prepare table info is it was not set and set up translation methods.""" + if self._table_info is None: + self._table_info = TableInfo(name=self._table.name) + + if self._auto_refresh_table_indexes: + self._table_info.refresh_indexed_attributes(self._table.meta.client) + + # Clone the attribute actions before we modify them + self._attribute_actions = self._attribute_actions.copy() + self._attribute_actions.set_index_keys(*self._table_info.protected_index_keys()) + + self._crypto_config = partial( # attrs confuses pylint: disable=attribute-defined-outside-init + crypto_config_from_kwargs, + fallback=partial( + crypto_config_from_table_info, + materials_provider=self._materials_provider, + attribute_actions=self._attribute_actions, + table_info=self._table_info + ) + ) + self.get_item = partial( # attrs confuses pylint: disable=attribute-defined-outside-init + decrypt_get_item, + decrypt_method=decrypt_python_item, + crypto_config_method=self._crypto_config, + read_method=self._table.get_item + ) + self.put_item = partial( # attrs confuses pylint: disable=attribute-defined-outside-init + encrypt_put_item, + encrypt_method=encrypt_python_item, + crypto_config_method=self._crypto_config, + write_method=self._table.put_item + ) + self.query = partial( # attrs confuses pylint: disable=attribute-defined-outside-init + decrypt_multi_get, + decrypt_method=decrypt_python_item, + crypto_config_method=self._crypto_config, + read_method=self._table.query + ) + self.scan = partial( # attrs confuses pylint: disable=attribute-defined-outside-init + decrypt_multi_get, + decrypt_method=decrypt_python_item, + crypto_config_method=self._crypto_config, + read_method=self._table.scan + ) + + def __getattr__(self, name): + """Catch any method/attribute lookups that are not defined in this class and try + to find them on the provided bridge object. + + :param str name: Attribute name + :returns: Result of asking the provided table object for that attribute name + :raises AttributeError: if attribute is not found on provided bridge object + """ + return getattr(self._table, name) + + def update_item(self, **kwargs): + """Update item is not yet supported.""" + raise NotImplementedError('"update_item" is not yet implemented') + + def batch_writer(self, overwrite_by_pkeys=None): + """Create a batch writer object. + + https://boto3.readthedocs.io/en/latest/reference/services/dynamodb.html#DynamoDB.Table.batch_writer + + :type overwrite_by_pkeys: list(string) + :param overwrite_by_pkeys: De-duplicate request items in buffer if match new request + item on specified primary keys. i.e ``["partition_key1", "sort_key2", "sort_key3"]`` + """ + encrypted_client = EncryptedClient( + client=self._table.meta.client, + materials_provider=self._materials_provider, + attribute_actions=self._attribute_actions, + auto_refresh_table_indexes=self._auto_refresh_table_indexes, + expect_standard_dictionaries=True + ) + return BatchWriter( + table_name=self._table.name, + client=encrypted_client, + overwrite_by_pkeys=overwrite_by_pkeys + ) diff --git a/src/dynamodb_encryption_sdk/exceptions.py b/src/dynamodb_encryption_sdk/exceptions.py index 5cb1fa83..1191f09f 100644 --- a/src/dynamodb_encryption_sdk/exceptions.py +++ b/src/dynamodb_encryption_sdk/exceptions.py @@ -10,12 +10,17 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +"""Exception classed for use in the DynamoDB Encryption Client.""" class DynamodbEncryptionSdkError(Exception): """Base class for all custom exceptions.""" +class InvalidArgumentError(DynamodbEncryptionSdkError): + """Raised when a general invalid argument is provided.""" + + class SerializationError(DynamodbEncryptionSdkError): """Otherwise undifferentiated errors encountered while serializing data.""" @@ -24,63 +29,61 @@ class DeserializationError(DynamodbEncryptionSdkError): """Otherwise undifferentiated errors encountered while deserializing data.""" -class InvalidMaterialsetError(DeserializationError): +class InvalidMaterialDescriptionError(DeserializationError): """Raised when errors are encountered processing a material description.""" - # TODO: MaterialDescription, not Materialset... -class InvalidMaterialsetVersionError(DeserializationError): +class InvalidMaterialDescriptionVersionError(DeserializationError): """Raised when a material description is encountered with an invalid version.""" - # TODO: MaterialDescription, not Materialset... -class InvalidAlgorithmError(DynamodbEncryptionSdkError): +class InvalidAlgorithmError(InvalidArgumentError): """Raised when an invalid algorithm identifier is encountered.""" class JceTransformationError(DynamodbEncryptionSdkError): - """""" + """Otherwise undifferentiated errors encountered when attempting to read a JCE transformation.""" class DelegatedKeyError(DynamodbEncryptionSdkError): - """""" + """Otherwise undifferentiated errors encountered by a DelegatedKey.""" class DelegatedKeyEncryptionError(DelegatedKeyError): - """""" + """Raised when a DelegatedKey encounters an error during encryption.""" class DelegatedKeyDecryptionError(DelegatedKeyError): - """""" + """Raised when a DelegatedKey encounters an error during decryption.""" class AwsKmsMaterialsProviderError(DynamodbEncryptionSdkError): - """""" + """Otherwise undifferentiated errors encountered by the AwsKmsCryptographicMaterialsProvider.""" class UnknownRegionError(AwsKmsMaterialsProviderError): - """""" + """Raised when the AwsKmsCryptographicMaterialsProvider is asked for an unknown region.""" class DecryptionError(DynamodbEncryptionSdkError): - """""" + """Otherwise undifferentiated error encountered while decrypting data.""" class UnwrappingError(DynamodbEncryptionSdkError): - """""" + """Otherwise undifferentiated error encountered while unwrapping a key.""" class EncryptionError(DynamodbEncryptionSdkError): - """""" + """Otherwise undifferentiated error encountered while encrypting data.""" class WrappingError(DynamodbEncryptionSdkError): - """""" + """Otherwise undifferentiated error encountered while wrapping a key.""" class SigningError(DynamodbEncryptionSdkError): - """""" + """Otherwise undifferentiated error encountered while signing data.""" class SignatureVerificationError(DynamodbEncryptionSdkError): - """""" + """Otherwise undifferentiated error encountered while verifying a signature.""" diff --git a/src/dynamodb_encryption_sdk/identifiers.py b/src/dynamodb_encryption_sdk/identifiers.py index e5ecb165..ce1e14ee 100644 --- a/src/dynamodb_encryption_sdk/identifiers.py +++ b/src/dynamodb_encryption_sdk/identifiers.py @@ -10,31 +10,41 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +"""Unique identifiers used by the DynamoDB Encryption Client.""" from enum import Enum +__all__ = ('LOGGER_NAME', 'CryptoAction', 'EncryptionKeyType', 'KeyEncodingType') __version__ = '0.0.0' LOGGER_NAME = 'dynamodb_encryption_sdk' -class ItemAction(Enum): +class CryptoAction(Enum): """Possible actions to take on an item attribute.""" + DO_NOTHING = 0 SIGN_ONLY = 1 ENCRYPT_AND_SIGN = 2 def __gt__(self, other): + # type: (CryptoAction) -> bool + """Define CryptoAction equality.""" return not self.__lt__(other) and not self.__eq__(other) def __lt__(self, other): + # type: (CryptoAction) -> bool + """Define CryptoAction equality.""" return self.value < other.value def __eq__(self, other): + # type: (CryptoAction) -> bool + """Define CryptoAction equality.""" return self.value == other.value -class EncryptionKeyTypes(Enum): +class EncryptionKeyType(Enum): """Supported types of encryption keys.""" + SYMMETRIC = 0 PRIVATE = 1 PUBLIC = 2 @@ -42,6 +52,7 @@ class EncryptionKeyTypes(Enum): class KeyEncodingType(Enum): """Supported key encoding schemes.""" + RAW = 0 DER = 1 PEM = 2 diff --git a/src/dynamodb_encryption_sdk/internal/crypto/__init__.py b/src/dynamodb_encryption_sdk/internal/crypto/__init__.py index 1ccc7fa1..f8e2c233 100644 --- a/src/dynamodb_encryption_sdk/internal/crypto/__init__.py +++ b/src/dynamodb_encryption_sdk/internal/crypto/__init__.py @@ -10,3 +10,4 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +"""Inner cryptographic components.""" diff --git a/src/dynamodb_encryption_sdk/internal/crypto/authentication.py b/src/dynamodb_encryption_sdk/internal/crypto/authentication.py index e0e3031b..176444cf 100644 --- a/src/dynamodb_encryption_sdk/internal/crypto/authentication.py +++ b/src/dynamodb_encryption_sdk/internal/crypto/authentication.py @@ -14,12 +14,12 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes -from dynamodb_encryption_sdk.delegated_keys import DelegatedKey -from dynamodb_encryption_sdk.encrypted import CryptoConfig -from dynamodb_encryption_sdk.identifiers import ItemAction +from dynamodb_encryption_sdk.delegated_keys import DelegatedKey # noqa pylint: disable=unused-import +from dynamodb_encryption_sdk.encrypted import CryptoConfig # noqa pylint: disable=unused-import +from dynamodb_encryption_sdk.identifiers import CryptoAction from dynamodb_encryption_sdk.internal.formatting.serialize.attribute import serialize_attribute from dynamodb_encryption_sdk.internal.identifiers import SignatureValues, Tag -from dynamodb_encryption_sdk.structures import AttributeActions +from dynamodb_encryption_sdk.structures import AttributeActions # noqa pylint: disable=unused-import __all__ = ('sign_item', 'verify_item_signature') @@ -89,7 +89,7 @@ def _string_to_sign(item, table_name, attribute_actions): )) for key in sorted(item.keys()): action = attribute_actions.action(key) - if action is ItemAction.DO_NOTHING: + if action is CryptoAction.DO_NOTHING: continue data_to_sign.extend(_hash_data( @@ -97,7 +97,7 @@ def _string_to_sign(item, table_name, attribute_actions): data=key.encode('utf-8') )) - if action is ItemAction.SIGN_ONLY: + if action is CryptoAction.SIGN_ONLY: data_to_sign.extend(SignatureValues.PLAINTEXT.sha256) else: data_to_sign.extend(SignatureValues.ENCRYPTED.sha256) diff --git a/src/dynamodb_encryption_sdk/internal/crypto/encryption.py b/src/dynamodb_encryption_sdk/internal/crypto/encryption.py index 31cd2b37..5caf71f6 100644 --- a/src/dynamodb_encryption_sdk/internal/crypto/encryption.py +++ b/src/dynamodb_encryption_sdk/internal/crypto/encryption.py @@ -12,13 +12,13 @@ # language governing permissions and limitations under the License. """Functions to handle encrypting and decrypting DynamoDB attributes.""" try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import text # pylint: disable=unused-import - from dynamodb_encryption_sdk.internal import dynamodb_types # pylint: disable=unused-import + from typing import Text # noqa pylint: disable=unused-import + from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass -from dynamodb_encryption_sdk.delegated_keys import DelegatedKey +from dynamodb_encryption_sdk.delegated_keys import DelegatedKey # noqa pylint: disable=unused-import from dynamodb_encryption_sdk.internal.formatting.deserialize.attribute import deserialize_attribute from dynamodb_encryption_sdk.internal.formatting.serialize.attribute import serialize_attribute from dynamodb_encryption_sdk.internal.identifiers import Tag diff --git a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/__init__.py b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/__init__.py index 22070dbc..3275667b 100644 --- a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/__init__.py +++ b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/__init__.py @@ -10,4 +10,6 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""""" +"""Components to provide cryptographic primitives based on JCE Standard Names. +https://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html +""" diff --git a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/authentication.py b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/authentication.py index 04a3469d..fa587bbc 100644 --- a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/authentication.py +++ b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/authentication.py @@ -12,6 +12,7 @@ # language governing permissions and limitations under the License. """Cryptographic authentication resources for JCE bridge.""" import abc +import logging import attr from cryptography.hazmat.backends import default_backend @@ -19,10 +20,19 @@ from cryptography.hazmat.primitives.asymmetric import padding, rsa import six -from .primitives import load_rsa_key +try: # Python 3.5.0 and 3.5.1 have incompatible typing modules + from typing import Text # noqa pylint: disable=unused-import +except ImportError: # pragma: no cover + # We only actually need these imports when running the mypy checks + pass + from dynamodb_encryption_sdk.exceptions import InvalidAlgorithmError, SignatureVerificationError, SigningError +from dynamodb_encryption_sdk.identifiers import EncryptionKeyType, KeyEncodingType, LOGGER_NAME +from dynamodb_encryption_sdk.internal.validators import callable_validator +from .primitives import load_rsa_key __all__ = ('JavaAuthenticator', 'JavaMac', 'JavaSignature', 'JAVA_AUTHENTICATOR') +_LOGGER = logging.getLogger(LOGGER_NAME) @six.add_metaclass(abc.ABCMeta) @@ -31,34 +41,70 @@ class JavaAuthenticator(object): @abc.abstractmethod def load_key(self, key, key_type, key_encoding): - """""" + # (bytes, EncryptionKeyType, KeyEncodingType) -> Any + # TODO: narrow down the output type + """Load a key from bytes. + + :param bytes key: Raw key bytes to load + :param key_type: Type of key to load + :type key_type: dynamodb_encryption_sdk.identifiers.EncryptionKeyType + :param key_encoding: Encoding used to serialize ``key`` + :type key_encoding: dynamodb_encryption_sdk.identifiers.KeyEncodingType + :returns: Loaded key + :rtype: bytes + """ @abc.abstractmethod def validate_algorithm(self, algorithm): - """""" + # type: (Text) -> None + """Determine whether the requested algorithm name is compatible with this authenticator. + + :param str algorithm: Algorithm name + :raises InvalidAlgorithmError: if specified algorithm name is not compatible with this authenticator + """ @abc.abstractmethod def sign(self, key, data): - """""" + # type: (Any, bytes) -> bytes + """Sign ``data`` using loaded ``key``. + + :param key: Loaded key + :param bytes data: Data to sign + :returns: Calculated signature + :rtype: bytes + :raises SigningError: if unable to sign ``data`` with ``key`` + """ @abc.abstractmethod def verify(self, key, signature, data): - """""" + # type: (Any, bytes, bytes) -> None + """Verify ``signature`` over ``data`` using ``key``. + + :param key: Loaded key + :param bytes signature: Signature to verify + :param bytes data: Data over which to verify signature + :raises SignatureVerificationError: if unable to verify ``signature`` + """ -@attr.s(hash=False) +@attr.s class JavaMac(JavaAuthenticator): """Symmetric MAC authenticators. https://docs.oracle.com/javase/8/docs/api/javax/crypto/Mac.html https://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html#Mac """ + java_name = attr.ib(validator=attr.validators.instance_of(six.string_types)) - algorithm_type = attr.ib() - hash_type = attr.ib() + algorithm_type = attr.ib(validator=callable_validator) + hash_type = attr.ib(validator=callable_validator) def _build_hmac_signer(self, key): - """""" + # type: (bytes) -> Any + """Build HMAC signer using instance algorithm and hash type and ``key``. + + :param bytes key: Key to use in signer + """ return self.algorithm_type( key, self.hash_type(), @@ -66,13 +112,28 @@ def _build_hmac_signer(self, key): ) def load_key(self, key, key_type, key_encoding): - """""" + # (bytes, EncryptionKeyType, KeyEncodingType) -> bytes + """Load a raw key from bytes. + + :param bytes key: Raw key bytes to load + :param key_type: Type of key to load + :type key_type: dynamodb_encryption_sdk.identifiers.EncryptionKeyType + :param key_encoding: Encoding used to serialize ``key`` + :type key_encoding: dynamodb_encryption_sdk.identifiers.KeyEncodingType + :returns: Loaded key + :rtype: bytes + :raises ValueError: if ``key_type`` is not symmetric or ``key_encoding`` is not raw + """ + if not (key_type is EncryptionKeyType.SYMMETRIC and key_encoding is KeyEncodingType.RAW): + raise ValueError('Key type must be symmetric and encoding must be raw.') + return key def validate_algorithm(self, algorithm): # type: (Text) -> None - """Determine whether the requested algorithm name is compatible with this signature. + """Determine whether the requested algorithm name is compatible with this authenticator. + :param str algorithm: Algorithm name :raises InvalidAlgorithmError: if specified algorithm name is not compatible with this authenticator """ if not algorithm.startswith(self.java_name): @@ -87,43 +148,58 @@ def sign(self, key, data): # type: (bytes, bytes) -> bytes """Sign ``data`` using loaded ``key``. - :param bytes key: Raw HMAC key + :param bytes key: Loaded key :param bytes data: Data to sign :returns: Calculated signature :rtype: bytes + :raises SigningError: if unable to sign ``data`` with ``key`` """ - signer = self._build_hmac_signer(key) - signer.update(data) - return signer.finalize() + try: + signer = self._build_hmac_signer(key) + signer.update(data) + return signer.finalize() + except Exception: + message = 'Unable to sign data' + _LOGGER.exception(message) + raise SigningError(message) def verify(self, key, signature, data): - """ + # type: (bytes, bytes, bytes) -> None + """Verify ``signature`` over ``data`` using ``key``. - :param bytes key: Raw HMAC key + :param bytes key: Loaded key :param bytes signature: Signature to verify :param bytes data: Data over which to verify signature + :raises SignatureVerificationError: if unable to verify ``signature`` """ - verifier = self._build_hmac_signer(key) - verifier.update(data) - verifier.verify(signature) + try: + verifier = self._build_hmac_signer(key) + verifier.update(data) + verifier.verify(signature) + except Exception: + message = 'Unable to verify signature' + _LOGGER.exception(message) + raise SignatureVerificationError(message) -@attr.s(hash=False) +@attr.s class JavaSignature(JavaAuthenticator): """Asymmetric signature authenticators. https://docs.oracle.com/javase/8/docs/api/java/security/Signature.html https://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html#Signature """ + java_name = attr.ib(validator=attr.validators.instance_of(six.string_types)) algorithm_type = attr.ib() - hash_type = attr.ib() - padding_type = attr.ib() + hash_type = attr.ib(validator=callable_validator) + padding_type = attr.ib(validator=callable_validator) def validate_algorithm(self, algorithm): # type: (Text) -> None - """Determine whether the requested algorithm name is compatible with this signature. + """Determine whether the requested algorithm name is compatible with this authenticator. + :param str algorithm: Algorithm name :raises InvalidAlgorithmError: if specified algorithm name is not compatible with this authenticator """ if not algorithm.endswith(self.java_name): @@ -135,35 +211,78 @@ def validate_algorithm(self, algorithm): ) def load_key(self, key, key_type, key_encoding): - """""" + # (bytes, EncryptionKeyType, KeyEncodingType) -> Any + # TODO: narrow down the output type + """Load a key object from the provided raw key bytes. + + :param bytes key: Raw key bytes to load + :param key_type: Type of key to load + :type key_type: dynamodb_encryption_sdk.identifiers.EncryptionKeyType + :param key_encoding: Encoding used to serialize ``key`` + :type key_encoding: dynamodb_encryption_sdk.identifiers.KeyEncodingType + :returns: Loaded key + :rtype: TODO: + :raises ValueError: if ``key_type`` and ``key_encoding`` are not a valid pairing + """ return load_rsa_key(key, key_type, key_encoding) def sign(self, key, data): - """""" + # type: (Any, bytes) -> bytes + # TODO: narrow down the key type + """Sign ``data`` using loaded ``key``. + + :param key: Loaded key + :type key: TODO: + :param bytes data: Data to sign + :returns: Calculated signature + :rtype: bytes + :raises SigningError: if unable to sign ``data`` with ``key`` + """ if hasattr(key, 'public_bytes'): raise SigningError('"sign" is not supported by public keys') - # TODO: normalize to SigningError - return key.sign( - data, - self.padding_type(), - self.hash_type() - ) + try: + return key.sign( + data, + self.padding_type(), + self.hash_type() + ) + except Exception: + message = 'Unable to sign data' + _LOGGER.exception(message) + raise SigningError(message) def verify(self, key, signature, data): - """""" + # type: (Any, bytes, bytes) -> None + # TODO: narrow down the key type + """Verify ``signature`` over ``data`` using ``key``. + + :param key: Loaded key + :type key: TODO: + :param bytes signature: Signature to verify + :param bytes data: Data over which to verify signature + :raises SignatureVerificationError: if unable to verify ``signature`` + """ if hasattr(key, 'private_bytes'): _key = key.public_key() else: _key = key - # TODO: normalize to SignatureVerificationError - _key.verify( - signature, - data, - self.padding_type(), - self.hash_type() - ) + try: + _key.verify( + signature, + data, + self.padding_type(), + self.hash_type() + ) + except Exception: + message = 'Unable to verify signature' + _LOGGER.exception(message) + raise SignatureVerificationError(message) +# Additional possible JCE names that we might support in the future if needed +# HmacSHA1 +# SHA(1|224|256|384|512)with(|EC)DSA +# If this changes, remember to update the JceNameLocalDelegatedKey docs. JAVA_AUTHENTICATOR = { 'HmacSHA224': JavaMac('HmacSHA224', hmac.HMAC, hashes.SHA224), 'HmacSHA256': JavaMac('HmacSHA256', hmac.HMAC, hashes.SHA256), @@ -173,9 +292,4 @@ def verify(self, key, signature, data): 'SHA256withRSA': JavaSignature('SHA256withRSA', rsa, hashes.SHA256, padding.PKCS1v15), 'SHA384withRSA': JavaSignature('SHA384withRSA', rsa, hashes.SHA384, padding.PKCS1v15), 'SHA512withRSA': JavaSignature('SHA512withRSA', rsa, hashes.SHA512, padding.PKCS1v15) - # TODO: should we support these? - # HmacMD5 - # HmacSHA1 - # (NONE|SHA(1|224|256|384|512))with(|EC)DSA - # (NONE|SHA1)withRSA } diff --git a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/encryption.py b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/encryption.py index cd7bace7..fa23e03f 100644 --- a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/encryption.py +++ b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/encryption.py @@ -13,15 +13,15 @@ """Cipher resource for JCE bridge.""" import attr +from dynamodb_encryption_sdk.exceptions import JceTransformationError from .primitives import ( - JAVA_ENCRYPTION_ALGORITHM, JavaEncryptionAlgorithm, JAVA_MODE, JavaMode, JAVA_PADDING, JavaPadding + JAVA_ENCRYPTION_ALGORITHM, JAVA_MODE, JAVA_PADDING, JavaEncryptionAlgorithm, JavaMode, JavaPadding ) -from dynamodb_encryption_sdk.exceptions import JceTransformationError __all__ = ('JavaCipher',) -@attr.s(hash=False) +@attr.s class JavaCipher(object): """Defines the encryption cipher, mode, and padding type to use for encryption. @@ -31,6 +31,7 @@ class JavaCipher(object): :param mode: TODO: :param padding: TODO: """ + cipher = attr.ib(validator=attr.validators.instance_of(JavaEncryptionAlgorithm)) mode = attr.ib(validator=attr.validators.instance_of(JavaMode)) padding = attr.ib(validator=attr.validators.instance_of(JavaPadding)) diff --git a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/primitives.py b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/primitives.py index 69a12ce2..675349cc 100644 --- a/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/primitives.py +++ b/src/dynamodb_encryption_sdk/internal/crypto/jce_bridge/primitives.py @@ -12,21 +12,28 @@ # language governing permissions and limitations under the License. """Cryptographic primitive resources for JCE bridge.""" import abc -import attr import logging import os +import attr from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import padding as symmetric_padding, hashes, serialization, keywrap +from cryptography.hazmat.primitives import hashes, keywrap, padding as symmetric_padding, serialization from cryptography.hazmat.primitives.asymmetric import padding as asymmetric_padding, rsa -from cryptography.hazmat.primitives.ciphers import algorithms, modes, Cipher +from cryptography.hazmat.primitives.ciphers import algorithms, Cipher, modes import six from dynamodb_encryption_sdk.exceptions import ( DecryptionError, EncryptionError, InvalidAlgorithmError, UnwrappingError, WrappingError ) -from dynamodb_encryption_sdk.identifiers import EncryptionKeyTypes, KeyEncodingType, LOGGER_NAME - +from dynamodb_encryption_sdk.identifiers import EncryptionKeyType, KeyEncodingType, LOGGER_NAME +from dynamodb_encryption_sdk.internal.validators import callable_validator + +__all__ = ( + 'JavaPadding', 'SimplePadding', 'BlockSizePadding', 'OaepPadding', + 'JavaMode', + 'JavaEncryptionAlgorithm', 'JavaSymmetricEncryptionAlgorithm', 'JavaAsymmetricEncryptionAlgorithm', + 'JAVA_ENCRYPTION_ALGORITHM', 'JAVA_MODE', 'JAVA_PADDING' +) _LOGGER = logging.getLogger(LOGGER_NAME) @@ -72,8 +79,9 @@ def unpadder(self): @six.add_metaclass(abc.ABCMeta) class JavaPadding(object): + # pylint: disable=too-few-public-methods """Bridge the gap from the Java padding names and Python resources. - https://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html#Cipher + https://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html#Cipher """ @abc.abstractmethod @@ -81,11 +89,13 @@ def build(self, block_size): """Build an instance of this padding type.""" -@attr.s(hash=False) +@attr.s class SimplePadding(JavaPadding): + # pylint: disable=too-few-public-methods """Padding types that do not require any preparation.""" + java_name = attr.ib(validator=attr.validators.instance_of(six.string_types)) - padding = attr.ib() + padding = attr.ib(validator=callable_validator) def build(self, block_size=None): # type: (int) -> ANY @@ -97,11 +107,13 @@ def build(self, block_size=None): return self.padding() -@attr.s(hash=False) +@attr.s class BlockSizePadding(JavaPadding): + # pylint: disable=too-few-public-methods """Padding types that require a block size input.""" + java_name = attr.ib(validator=attr.validators.instance_of(six.string_types)) - padding = attr.ib() + padding = attr.ib(validator=callable_validator) def build(self, block_size): # type: (int) -> ANY @@ -113,8 +125,9 @@ def build(self, block_size): return self.padding(block_size) -@attr.s(hash=False) +@attr.s class OaepPadding(JavaPadding): + # pylint: disable=too-few-public-methods """OAEP padding types. These require more complex setup. .. warning:: @@ -123,11 +136,12 @@ class OaepPadding(JavaPadding): The same hashing algorithm should be used by both OAEP and the MGF, but by default Java always uses SHA1 for the MGF. """ + java_name = attr.ib(validator=attr.validators.instance_of(six.string_types)) - padding = attr.ib() - digest = attr.ib() - mgf = attr.ib() - mgf_digest = attr.ib() + padding = attr.ib(validator=callable_validator) + digest = attr.ib(validator=callable_validator) + mgf = attr.ib(validator=callable_validator) + mgf_digest = attr.ib(validator=callable_validator) def build(self, block_size=None): # type: (int) -> ANY @@ -143,13 +157,15 @@ def build(self, block_size=None): ) -@attr.s(hash=False) +@attr.s class JavaMode(object): + # pylint: disable=too-few-public-methods """Bridge the gap from the Java encryption mode names and Python resources. - https://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html#Cipher + https://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html#Cipher """ + java_name = attr.ib(validator=attr.validators.instance_of(six.string_types)) - mode = attr.ib() + mode = attr.ib(validator=callable_validator) def build(self, iv): # type: (int) -> ANY @@ -161,11 +177,13 @@ def build(self, iv): return self.mode(iv) -@attr.s(hash=False) +@attr.s class JavaEncryptionAlgorithm(object): + # pylint: disable=too-few-public-methods """Bridge the gap from the Java encryption algorithm names and Python resources. https://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html#Cipher """ + java_name = attr.ib(validator=attr.validators.instance_of(six.string_types)) cipher = attr.ib() @@ -211,12 +229,12 @@ def load_key(self, key, key_type, key_encoding): :param bytes key: Key bytes :param key_type: Type of key - :type key_type: dynamodb_encryption_sdk.identifiers.EncryptionKeyTypes + :type key_type: dynamodb_encryption_sdk.identifiers.EncryptionKeyType :param key_encoding: Encoding used to serialize key :type key_encoding: dynamodb_encryption_sdk.identifiers.KeyEncodingType :returns: Loaded key """ - if key_type is not EncryptionKeyTypes.SYMMETRIC: + if key_type is not EncryptionKeyType.SYMMETRIC: raise ValueError('Invalid key type "{key_type}" for cipher "{cipher}"'.format( key_type=key_type, cipher=self.java_name @@ -277,6 +295,7 @@ def unwrap(self, wrapping_key, wrapped_key): raise UnwrappingError(error_message) def encrypt(self, key, data, mode, padding): + # this can be disabled by _disable_encryption, so pylint: disable=method-hidden """Encrypt data using the supplied values. :param bytes key: Loaded encryption key @@ -308,6 +327,7 @@ def encrypt(self, key, data, mode, padding): raise EncryptionError(error_message) def decrypt(self, key, data, mode, padding): + # this can be disabled by _disable_encryption, so pylint: disable=method-hidden """Decrypt data using the supplied values. :param bytes key: Loaded decryption key @@ -341,11 +361,11 @@ def decrypt(self, key, data, mode, padding): _RSA_KEY_LOADING = { - EncryptionKeyTypes.PRIVATE: { + EncryptionKeyType.PRIVATE: { KeyEncodingType.DER: serialization.load_der_private_key, KeyEncodingType.PEM: serialization.load_pem_private_key }, - EncryptionKeyTypes.PUBLIC: { + EncryptionKeyType.PUBLIC: { KeyEncodingType.DER: serialization.load_der_public_key, KeyEncodingType.PEM: serialization.load_pem_public_key } @@ -353,21 +373,33 @@ def decrypt(self, key, data, mode, padding): def load_rsa_key(key, key_type, key_encoding): - """""" + # (bytes, EncryptionKeyType, KeyEncodingType) -> Any + # TODO: narrow down the output type + """Load an RSA key object from the provided raw key bytes. + + :param bytes key: Raw key bytes to load + :param key_type: Type of key to load + :type key_type: dynamodb_encryption_sdk.identifiers.EncryptionKeyType + :param key_encoding: Encoding used to serialize ``key`` + :type key_encoding: dynamodb_encryption_sdk.identifiers.KeyEncodingType + :returns: Loaded key + :rtype: TODO: + :raises ValueError: if ``key_type`` and ``key_encoding`` are not a valid pairing + """ try: loader = _RSA_KEY_LOADING[key_type][key_encoding] except KeyError: - raise Exception('Invalid key type: {}'.format(key_type)) + raise ValueError('Invalid key type and encoding: {} and {}'.format(key_type, key_encoding)) kwargs = dict(data=key, backend=default_backend()) - if key_type is EncryptionKeyTypes.PRIVATE: + if key_type is EncryptionKeyType.PRIVATE: kwargs['password'] = None return loader(**kwargs) _KEY_LOADERS = { - rsa: load_rsa_key + rsa: load_rsa_key } @@ -382,12 +414,12 @@ def load_key(self, key, key_type, key_encoding): :param bytes key: Key bytes :param key_type: Type of key - :type key_type: dynamodb_encryption_sdk.identifiers.EncryptionKeyTypes + :type key_type: dynamodb_encryption_sdk.identifiers.EncryptionKeyType :param key_encoding: Encoding used to serialize key :type key_encoding: dynamodb_encryption_sdk.identifiers.KeyEncodingType :returns: Loaded key """ - if key_type not in (EncryptionKeyTypes.PRIVATE, EncryptionKeyTypes.PUBLIC): + if key_type not in (EncryptionKeyType.PRIVATE, EncryptionKeyType.PUBLIC): raise ValueError('Invalid key type "{key_type}" for cipher "{cipher}"'.format( key_type=key_type, cipher=self.java_name @@ -402,6 +434,7 @@ def load_key(self, key, key_type, key_encoding): return _KEY_LOADERS[self.cipher](key, key_type, key_encoding) def encrypt(self, key, data, mode, padding): + # pylint: disable=unused-argument,no-self-use """Encrypt data using the supplied values. :param bytes key: Loaded encryption key @@ -425,6 +458,7 @@ def encrypt(self, key, data, mode, padding): raise EncryptionError(error_message) def decrypt(self, key, data, mode, padding): + # pylint: disable=unused-argument,no-self-use """Decrypt data using the supplied values. :param bytes key: Loaded decryption key @@ -437,7 +471,7 @@ def decrypt(self, key, data, mode, padding): :rtype: bytes """ if hasattr(key, 'public_bytes'): - raise NotImplementedError('TODO:"decrypt" is not supported by public keys') + raise NotImplementedError('"decrypt" is not supported by public keys') try: return key.decrypt(data, padding.build()) except Exception: @@ -446,24 +480,17 @@ def decrypt(self, key, data, mode, padding): raise DecryptionError(error_message) +# If this changes, remember to update the JceNameLocalDelegatedKey docs. JAVA_ENCRYPTION_ALGORITHM = { 'RSA': JavaAsymmetricEncryptionAlgorithm('RSA', rsa), 'AES': JavaSymmetricEncryptionAlgorithm('AES', algorithms.AES), 'AESWrap': JavaSymmetricEncryptionAlgorithm('AESWrap', algorithms.AES) - # TODO: Should we support these? - # DES : pretty sure we don't want to support this - # DESede : pretty sure we don't want to support this - # 'BLOWFISH': JavaSymmetricEncryptionAlgorithm('Blowfish', algorithms.Blowfish) } JAVA_MODE = { 'ECB': JavaMode('ECB', modes.ECB), 'CBC': JavaMode('CBC', modes.CBC), 'CTR': JavaMode('CTR', modes.CTR), 'GCM': JavaMode('GCM', modes.GCM) - # TODO: Should we support these? - # 'OFB': JavaMode('OFB', modes.OFB) - # 'CFB': JavaMode('CFB', modes.CFB) - # 'CFB8': JavaMode('CFB8', modes.CFB8) } JAVA_PADDING = { 'NoPadding': SimplePadding('NoPadding', _NoPadding), diff --git a/src/dynamodb_encryption_sdk/internal/formatting/__init__.py b/src/dynamodb_encryption_sdk/internal/formatting/__init__.py index 1ccc7fa1..eec2d539 100644 --- a/src/dynamodb_encryption_sdk/internal/formatting/__init__.py +++ b/src/dynamodb_encryption_sdk/internal/formatting/__init__.py @@ -10,3 +10,4 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +"""Data formatting utilities for the DynamoDB Encryption Client.""" diff --git a/src/dynamodb_encryption_sdk/internal/formatting/deserialize/attribute.py b/src/dynamodb_encryption_sdk/internal/formatting/deserialize/attribute.py index 32bd10a6..af0532f5 100644 --- a/src/dynamodb_encryption_sdk/internal/formatting/deserialize/attribute.py +++ b/src/dynamodb_encryption_sdk/internal/formatting/deserialize/attribute.py @@ -19,7 +19,7 @@ try: # Python 3.5.0 and 3.5.1 have incompatible typing modules from typing import Callable, Dict, List, Union # noqa pylint: disable=unused-import - from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import + from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import,ungrouped-imports except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass @@ -27,13 +27,13 @@ from boto3.dynamodb.types import Binary from dynamodb_encryption_sdk.exceptions import DeserializationError -from dynamodb_encryption_sdk.internal.defaults import ENCODING, LOGGING_NAME +from dynamodb_encryption_sdk.identifiers import LOGGER_NAME from dynamodb_encryption_sdk.internal.formatting.deserialize import decode_byte, decode_length, decode_tag, decode_value from dynamodb_encryption_sdk.internal.identifiers import Tag, TagValues from dynamodb_encryption_sdk.internal.str_ops import to_str __all__ = ('deserialize_attribute',) -_LOGGER = logging.getLogger(LOGGING_NAME) +_LOGGER = logging.getLogger(LOGGER_NAME) def deserialize_attribute(serialized_attribute): # noqa: C901 pylint: disable=too-many-locals @@ -69,7 +69,7 @@ def _transform_string_value(value): :param bytes value: Raw deserialized value :rtype: dynamodb_encryption_sdk.internal.dynamodb_types.STRING """ - return codecs.decode(value, ENCODING) + return codecs.decode(value, 'utf-8') def _deserialize_string(stream): # type: (io.BytesIO) -> Dict[str, dynamodb_types.STRING] @@ -89,9 +89,9 @@ def _transform_number_value(value): :param bytes value: Raw deserialized value :rtype: dynamodb_encryption_sdk.internal.dynamodb_types.STRING """ - raw_value = codecs.decode(value, ENCODING) - decimal_value = Decimal(to_str(raw_value)) - return str(decimal_value.normalize()) + raw_value = codecs.decode(value, 'utf-8') + decimal_value = Decimal(to_str(raw_value)).normalize() + return '{0:f}'.format(decimal_value) def _deserialize_number(stream): # type: (io.BytesIO) -> Dict[str, dynamodb_types.STRING] diff --git a/src/dynamodb_encryption_sdk/internal/formatting/material_description.py b/src/dynamodb_encryption_sdk/internal/formatting/material_description.py index ac00426a..ed3d76fa 100644 --- a/src/dynamodb_encryption_sdk/internal/formatting/material_description.py +++ b/src/dynamodb_encryption_sdk/internal/formatting/material_description.py @@ -15,15 +15,16 @@ import logging import struct -from .deserialize import decode_value, unpack_value -from .serialize import encode_value -from dynamodb_encryption_sdk.exceptions import InvalidMaterialsetError, InvalidMaterialsetVersionError -from dynamodb_encryption_sdk.internal.defaults import LOGGING_NAME, MATERIAL_DESCRIPTION_VERSION +from dynamodb_encryption_sdk.exceptions import InvalidMaterialDescriptionError, InvalidMaterialDescriptionVersionError +from dynamodb_encryption_sdk.identifiers import LOGGER_NAME from dynamodb_encryption_sdk.internal.identifiers import Tag from dynamodb_encryption_sdk.internal.str_ops import to_bytes, to_str +from .deserialize import decode_value, unpack_value +from .serialize import encode_value __all__ = ('serialize', 'deserialize') -_LOGGER = logging.getLogger(LOGGING_NAME) +_LOGGER = logging.getLogger(LOGGER_NAME) +_MATERIAL_DESCRIPTION_VERSION = b'\00' * 4 def serialize(material_description): @@ -34,7 +35,7 @@ def serialize(material_description): :returns: Serialized material description as a DynamoDB binary attribute value :rtype: dict """ - material_description_bytes = bytearray(MATERIAL_DESCRIPTION_VERSION) + material_description_bytes = bytearray(_MATERIAL_DESCRIPTION_VERSION) # TODO: verify Java sorting order for name, value in sorted(material_description.items(), key=lambda x: x[0]): @@ -42,10 +43,12 @@ def serialize(material_description): material_description_bytes.extend(encode_value(to_bytes(name))) material_description_bytes.extend(encode_value(to_bytes(value))) except (TypeError, struct.error): - raise InvalidMaterialsetError('Invalid name or value in material description: "{name}"="{value}"'.format( - name=name, - value=value - )) + raise InvalidMaterialDescriptionError( + 'Invalid name or value in material description: "{name}"="{value}"'.format( + name=name, + value=value + ) + ) return {Tag.BINARY.dynamodb_tag: bytes(material_description_bytes)} @@ -57,7 +60,7 @@ def deserialize(serialized_material_description): :param dict serialized_material_description: DynamoDB attribute value containing serialized material description. :returns: Material description dictionary :rtype: dict - :raises InvalidMaterialsetError: if material description is invalid or malformed + :raises InvalidMaterialDescriptionError: if material description is invalid or malformed """ try: _raw_material_description = serialized_material_description[Tag.BINARY.dynamodb_tag] @@ -67,7 +70,7 @@ def deserialize(serialized_material_description): except (TypeError, KeyError): message = 'Invalid material description' _LOGGER.exception(message) - raise InvalidMaterialsetError(message) + raise InvalidMaterialDescriptionError(message) # We don't currently do anything with the version, but do check to make sure it is the one we know about. _read_version(material_description_bytes) @@ -80,7 +83,7 @@ def deserialize(serialized_material_description): except struct.error: message = 'Invalid material description' _LOGGER.exception(message) - raise InvalidMaterialsetError(message) + raise InvalidMaterialDescriptionError(message) return material_description @@ -90,14 +93,14 @@ def _read_version(material_description_bytes): :param material_description_bytes: serializezd material description :type material_description_bytes: io.BytesIO - :raises InvalidMaterialsetError: if malformed version - :raises InvalidMaterialsetVersionError: if unknown version is found + :raises InvalidMaterialDescriptionError: if malformed version + :raises InvalidMaterialDescriptionVersionError: if unknown version is found """ try: (version,) = unpack_value('>4s', material_description_bytes) except struct.error: message = 'Malformed material description version' _LOGGER.exception(message) - raise InvalidMaterialsetError(message) - if version != MATERIAL_DESCRIPTION_VERSION: - raise InvalidMaterialsetVersionError('Invalid material description version: {}'.format(repr(version))) + raise InvalidMaterialDescriptionError(message) + if version != _MATERIAL_DESCRIPTION_VERSION: + raise InvalidMaterialDescriptionVersionError('Invalid material description version: {}'.format(repr(version))) diff --git a/src/dynamodb_encryption_sdk/internal/formatting/serialize/__init__.py b/src/dynamodb_encryption_sdk/internal/formatting/serialize/__init__.py index de34b905..0fcaa79e 100644 --- a/src/dynamodb_encryption_sdk/internal/formatting/serialize/__init__.py +++ b/src/dynamodb_encryption_sdk/internal/formatting/serialize/__init__.py @@ -14,7 +14,7 @@ import struct try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Sized # pylint: disable=unused-import + from typing import Sized # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass diff --git a/src/dynamodb_encryption_sdk/internal/formatting/serialize/attribute.py b/src/dynamodb_encryption_sdk/internal/formatting/serialize/attribute.py index 9ab0afd6..9d3d633f 100644 --- a/src/dynamodb_encryption_sdk/internal/formatting/serialize/attribute.py +++ b/src/dynamodb_encryption_sdk/internal/formatting/serialize/attribute.py @@ -16,7 +16,7 @@ try: # Python 3.5.0 and 3.5.1 have incompatible typing modules from typing import Callable # noqa pylint: disable=unused-import - from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import + from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import,ungrouped-imports except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass @@ -24,14 +24,14 @@ from boto3.dynamodb.types import Binary, DYNAMODB_CONTEXT from dynamodb_encryption_sdk.exceptions import SerializationError -from dynamodb_encryption_sdk.internal.defaults import LOGGING_NAME +from dynamodb_encryption_sdk.identifiers import LOGGER_NAME from dynamodb_encryption_sdk.internal.formatting.serialize import encode_length, encode_value from dynamodb_encryption_sdk.internal.identifiers import Tag, TagValues from dynamodb_encryption_sdk.internal.str_ops import to_bytes from dynamodb_encryption_sdk.internal.utils import sorted_key_map __all__ = ('serialize_attribute',) -_LOGGER = logging.getLogger(LOGGING_NAME) +_LOGGER = logging.getLogger(LOGGER_NAME) _RESERVED = b'\x00' @@ -78,9 +78,8 @@ def _transform_number_value(value): # by dynamodb.TypeSerializer, so all numbers are str. However, TypeSerializer # leaves trailing zeros if they are defined in the Decimal call, but we need to # strip all trailing zeros. - decimal_value = DYNAMODB_CONTEXT.create_decimal(value) - raw_value = '{:f}'.format(decimal_value.normalize()) - return to_bytes(raw_value) + decimal_value = DYNAMODB_CONTEXT.create_decimal(value).normalize() + return '{0:f}'.format(decimal_value).encode('utf-8') def _serialize_number(_attribute): # type: (str) -> bytes diff --git a/src/dynamodb_encryption_sdk/internal/identifiers.py b/src/dynamodb_encryption_sdk/internal/identifiers.py index f2b32a66..5a917361 100644 --- a/src/dynamodb_encryption_sdk/internal/identifiers.py +++ b/src/dynamodb_encryption_sdk/internal/identifiers.py @@ -10,18 +10,23 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""""" +"""Unique identifiers for internal use only.""" from enum import Enum try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Any, ByteString, Dict, List, Text, Union # pylint: disable=unused-import + from typing import Text # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass +__all__ = ( + 'ReservedAttributes', 'Tag', 'TagValues', 'SignatureValues', 'MaterialDescriptionKeys', 'MaterialDescriptionValues' +) + class ReservedAttributes(Enum): """Item attributes reserved for use by DynamoDBEncryptionClient""" + MATERIAL_DESCRIPTION = '*amzn-ddb-map-desc*' SIGNATURE = '*amzn-ddb-map-sig*' @@ -55,6 +60,7 @@ def __init__(self, tag, dynamodb_tag, element_tag=None): class TagValues(Enum): """Static values to use when serializing attribute values.""" + FALSE = b'\x00' TRUE = b'\x01' @@ -67,6 +73,7 @@ class SignatureValues(Enum): The only time we actually use these values, we use the SHA256 hash of the value, so we pre-compute these hashes here. """ + ENCRYPTED = ( b'ENCRYPTED', b"9A\x15\xacN\xb0\x9a\xa4\x94)4\x88\x16\xb2\x03\x81'\xb0\xf9\xe3\xa5 7*\xe1\x00\xca\x19\xfb\x08\xfdP" @@ -89,6 +96,7 @@ def __init__(self, raw, sha256): class MaterialDescriptionKeys(Enum): """Static keys for use when building and reading material descriptions.""" + ATTRIBUTE_ENCRYPTION_MODE = 'amzn-ddb-map-sym-mode' SIGNING_KEY_ALGORITHM = 'amzn-ddb-map-signingAlg' WRAPPED_DATA_KEY = 'amzn-ddb-env-key' @@ -99,4 +107,5 @@ class MaterialDescriptionKeys(Enum): class MaterialDescriptionValues(Enum): """Static default values for use when building material descriptions.""" + CBC_PKCS5_ATTRIBUTE_ENCRYPTION = '/CBC/PKCS5Padding' diff --git a/src/dynamodb_encryption_sdk/internal/str_ops.py b/src/dynamodb_encryption_sdk/internal/str_ops.py index 2a04719b..f1213eb5 100644 --- a/src/dynamodb_encryption_sdk/internal/str_ops.py +++ b/src/dynamodb_encryption_sdk/internal/str_ops.py @@ -15,6 +15,8 @@ import six +__all__ = ('to_str', 'to_bytes') + def to_str(data): """Takes an input str or bytes object and returns an equivalent str object. diff --git a/src/dynamodb_encryption_sdk/internal/utils.py b/src/dynamodb_encryption_sdk/internal/utils.py index 738bba5d..b188424f 100644 --- a/src/dynamodb_encryption_sdk/internal/utils.py +++ b/src/dynamodb_encryption_sdk/internal/utils.py @@ -10,8 +10,28 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""""" +"""Otherwise undifferentiated utility resources.""" +import attr +import botocore.client + +from dynamodb_encryption_sdk.encrypted import CryptoConfig +from dynamodb_encryption_sdk.exceptions import InvalidArgumentError from dynamodb_encryption_sdk.internal.str_ops import to_bytes +from dynamodb_encryption_sdk.structures import EncryptionContext, TableInfo + +try: # Python 3.5.0 and 3.5.1 have incompatible typing modules + from typing import Callable, Dict, Text # noqa pylint: disable=unused-import +except ImportError: # pragma: no cover + # We only actually need these imports when running the mypy checks + pass + +__all__ = ( + 'sorted_key_map', 'TableInfoCache', + 'crypto_config_from_kwargs', 'crypto_config_from_table_info', 'crypto_config_from_cache', + 'decrypt_get_item', 'decrypt_multi_get', 'decrypt_batch_get_item', + 'encrypt_put_item', 'encrypt_batch_write_item', + 'validate_get_arguments' +) def sorted_key_map(item, transform=to_bytes): @@ -28,3 +48,211 @@ def sorted_key_map(item, transform=to_bytes): sorted_items.append((_key, value, key)) sorted_items = sorted(sorted_items, key=lambda x: x[0]) return sorted_items + + +@attr.s +class TableInfoCache(object): + # pylint: disable=too-few-public-methods + """Very simple cache of TableInfo objects, providing configuration information about DynamoDB tables. + + :param client: Boto3 DynamoDB client + :type client: botocore.client.BaseClient + :param bool auto_refresh_table_indexes: Should we attempt to refresh information about table indexes? + Requires ``dynamodb:DescribeTable`` permissions on each table. + """ + + _client = attr.ib(validator=attr.validators.instance_of(botocore.client.BaseClient)) + _auto_refresh_table_indexes = attr.ib(validator=attr.validators.instance_of(bool)) + + def __attrs_post_init__(self): + """Set up the empty cache.""" + self._all_tables_info = {} # type: Dict[Text, TableInfo] # pylint: disable=attribute-defined-outside-init + + def table_info(self, table_name): + """Collect a TableInfo object for the specified table, creating and adding it to + the cache if not already present. + + :param str table_name: Name of table + :returns: TableInfo describing the requested table + :rtype: dynamodb_encryption_sdk.structures.TableInfo + """ + try: + return self._all_tables_info[table_name] + except KeyError: + _table_info = TableInfo(name=table_name) + if self._auto_refresh_table_indexes: + _table_info.refresh_indexed_attributes(self._client) + self._all_tables_info[table_name] = _table_info + return _table_info + + +def validate_get_arguments(kwargs): + # type: (Dict[Text, Any]) -> None + """Verify that attribute filtering parameters are not found in the request. + + :raises InvalidArgumentError: if banned parameters are found + """ + for arg in ('AttributesToGet', 'ProjectionExpression'): + if arg in kwargs: + raise InvalidArgumentError('"{}" is not supported for this operation'.format(arg)) + + if kwargs.get('Select', None) in ('SPECIFIC_ATTRIBUTES', 'ALL_PROJECTED_ATTRIBUTES', 'SPECIFIC_ATTRIBUTES'): + raise InvalidArgumentError('Scan "Select" value of "{}" is not supported'.format(kwargs['Select'])) + + +def crypto_config_from_kwargs(fallback, **kwargs): + """Pull all encryption-specific parameters from the request and use them to build a crypto config. + + :returns: crypto config and updated kwargs + :rtype: dynamodb_encryption_sdk.encrypted.CryptoConfig and dict + """ + try: + crypto_config = kwargs.pop('crypto_config') + except KeyError: + try: + fallback_kwargs = {'table_name': kwargs['TableName']} + except KeyError: + fallback_kwargs = {} + crypto_config = fallback(**fallback_kwargs) + return crypto_config, kwargs + + +def crypto_config_from_table_info(materials_provider, attribute_actions, table_info): + """Build a crypto config from the provided values and table info. + + :returns: crypto config and updated kwargs + :rtype: dynamodb_encryption_sdk.encrypted.CryptoConfig and dict + """ + return CryptoConfig( + materials_provider=materials_provider, + encryption_context=EncryptionContext(**table_info.encryption_context_values), + attribute_actions=attribute_actions + ) + + +def crypto_config_from_cache(materials_provider, attribute_actions, table_info_cache, table_name): + """Build a crypto config from the provided values, loading the table info from the provided cache. + + :returns: crypto config and updated kwargs + :rtype: dynamodb_encryption_sdk.encrypted.CryptoConfig and dict + """ + table_info = table_info_cache.table_info(table_name) + + attribute_actions = attribute_actions.copy() + attribute_actions.set_index_keys(*table_info.protected_index_keys()) + + return crypto_config_from_table_info(materials_provider, attribute_actions, table_info) + + +def decrypt_multi_get(decrypt_method, crypto_config_method, read_method, **kwargs): + # type: (Callable, Callable, Callable, **Any) -> Dict + # TODO: narrow this down + """Transparently decrypt multiple items after getting them from the table. + + :param callable decrypt_method: Method to use to decrypt items + :param callable crypto_config_method: Method that accepts ``kwargs`` and provides a ``CryptoConfig`` + :param callable read_method: Method that reads from the table + """ + validate_get_arguments(kwargs) + crypto_config, ddb_kwargs = crypto_config_method(**kwargs) + response = read_method(**ddb_kwargs) + for pos in range(len(response['Items'])): + response['Items'][pos] = decrypt_method( + item=response['Items'][pos], + crypto_config=crypto_config + ) + return response + + +def decrypt_get_item(decrypt_method, crypto_config_method, read_method, **kwargs): + # type: (Callable, Callable, Callable, **Any) -> Dict + # TODO: narrow this down + """Transparently decrypt an item after getting it from the table. + + :param callable decrypt_method: Method to use to decrypt item + :param callable crypto_config_method: Method that accepts ``kwargs`` and provides a ``CryptoConfig`` + :param callable read_method: Method that reads from the table + """ + validate_get_arguments(kwargs) + crypto_config, ddb_kwargs = crypto_config_method(**kwargs) + response = read_method(**ddb_kwargs) + if 'Item' in response: + response['Item'] = decrypt_method( + item=response['Item'], + crypto_config=crypto_config + ) + return response + + +def decrypt_batch_get_item(decrypt_method, crypto_config_method, read_method, **kwargs): + # type: (Callable, Callable, Callable, **Any) -> Dict + # TODO: narrow this down + """Transparently decrypt multiple items after getting them in a batch request. + + :param callable decrypt_method: Method to use to decrypt items + :param callable crypto_config_method: Method that accepts ``kwargs`` and provides a ``CryptoConfig`` + :param callable read_method: Method that reads from the table + """ + request_crypto_config = kwargs.pop('crypto_config', None) + + for _table_name, table_kwargs in kwargs['RequestItems'].items(): + validate_get_arguments(table_kwargs) + + response = read_method(**kwargs) + for table_name, items in response['Responses'].items(): + if request_crypto_config is not None: + crypto_config = request_crypto_config + else: + crypto_config = crypto_config_method(table_name=table_name) + + for pos, value in enumerate(items): + items[pos] = decrypt_method( + item=value, + crypto_config=crypto_config + ) + return response + + +def encrypt_put_item(encrypt_method, crypto_config_method, write_method, **kwargs): + # type: (Callable, Callable, Callable, **Any) -> Dict + # TODO: narrow this down + """Transparently encrypt an item before putting it to the table. + + :param callable encrypt_method: Method to use to encrypt items + :param callable crypto_config_method: Method that accepts ``kwargs`` and provides a ``CryptoConfig`` + :param callable read_method: Method that reads from the table + """ + crypto_config, ddb_kwargs = crypto_config_method(**kwargs) + ddb_kwargs['Item'] = encrypt_method( + item=ddb_kwargs['Item'], + crypto_config=crypto_config + ) + return write_method(**ddb_kwargs) + + +def encrypt_batch_write_item(encrypt_method, crypto_config_method, write_method, **kwargs): + # type: (Callable, Callable, Callable, **Any) -> Dict + # TODO: narrow this down + """Transparently encrypt multiple items before putting them in a batch request. + + :param callable encrypt_method: Method to use to encrypt items + :param callable crypto_config_method: Method that accepts ``kwargs`` and provides a ``CryptoConfig`` + :param callable read_method: Method that reads from the table + """ + request_crypto_config = kwargs.pop('crypto_config', None) + + for table_name, items in kwargs['RequestItems'].items(): + if request_crypto_config is not None: + crypto_config = request_crypto_config + else: + crypto_config = crypto_config_method(table_name=table_name) + + for pos, value in enumerate(items): + for request_type, item in value.items(): + # We don't encrypt primary indexes, so we can ignore DeleteItem requests + if request_type == 'PutRequest': + items[pos][request_type]['Item'] = encrypt_method( + item=item['Item'], + crypto_config=crypto_config + ) + return write_method(**kwargs) diff --git a/src/dynamodb_encryption_sdk/internal/validators.py b/src/dynamodb_encryption_sdk/internal/validators.py new file mode 100644 index 00000000..3eb1c4b7 --- /dev/null +++ b/src/dynamodb_encryption_sdk/internal/validators.py @@ -0,0 +1,84 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Custom validators for ``attrs``.""" + +__all__ = ('dictionary_validator', 'iterable_validator') + + +def dictionary_validator(key_type, value_type): + """Validator for ``attrs`` that performs deep type checking of dictionaries.""" + + def _validate_dictionary(instance, attribute, value): + # pylint: disable=unused-argument + """Validate that a dictionary is structured as expected. + + :raises TypeError: if ``value`` is not a dictionary + :raises TypeError: if ``value`` keys are not all of ``key_type`` type + :raises TypeError: if ``value`` values are not all of ``value_type`` type + """ + if not isinstance(value, dict): + raise TypeError('"{}" must be a dictionary'.format(attribute.name)) + + for key, data in value.items(): + if not isinstance(key, key_type): + raise TypeError('"{name}" dictionary keys must be of type "{type}"'.format( + name=attribute.name, + type=key_type + )) + + if not isinstance(data, value_type): + raise TypeError('"{name}" dictionary values must be of type "{type}"'.format( + name=attribute.name, + type=value_type + )) + + return _validate_dictionary + + +def iterable_validator(iterable_type, member_type): + """Validator for ``attrs`` that performs deep type checking of iterables.""" + + def _validate_tuple(instance, attribute, value): + # pylint: disable=unused-argument + """Validate that a dictionary is structured as expected. + + :raises TypeError: if ``value`` is not of ``iterable_type`` type + :raises TypeError: if ``value`` members are not all of ``member_type`` type + """ + if not isinstance(value, iterable_type): + raise TypeError('"{name}" must be a {type}'.format( + name=attribute.name, + type=iterable_type + )) + + for member in value: + if not isinstance(member, member_type): + raise TypeError('"{name}" members must all be of type "{type}"'.format( + name=attribute.name, + type=member_type + )) + + return _validate_tuple + + +def callable_validator(instance, attribute, value): + # pylint: disable=unused-argument + """Validate that an attribute value is callable. + + :raises TypeError: if ``value`` is not callable + """ + if not callable(value): + raise TypeError('"{name}" value "{value}" must be callable'.format( + name=attribute.name, + value=value + )) diff --git a/src/dynamodb_encryption_sdk/material_providers/__init__.py b/src/dynamodb_encryption_sdk/material_providers/__init__.py index e0981e18..82735d17 100644 --- a/src/dynamodb_encryption_sdk/material_providers/__init__.py +++ b/src/dynamodb_encryption_sdk/material_providers/__init__.py @@ -11,8 +11,10 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. """Cryptographic materials providers.""" -from dynamodb_encryption_sdk.materials import DecryptionMaterials, EncryptionMaterials -from dynamodb_encryption_sdk.structures import EncryptionContext +from dynamodb_encryption_sdk.materials import DecryptionMaterials, EncryptionMaterials # noqa pylint: disable=unused-import +from dynamodb_encryption_sdk.structures import EncryptionContext # noqa pylint: disable=unused-import + +__all__ = ('CryptographicMaterialsProvider',) class CryptographicMaterialsProvider(object): @@ -20,6 +22,7 @@ class CryptographicMaterialsProvider(object): def decryption_materials(self, encryption_context): # type: (EncryptionContext) -> DecryptionMaterials + # pylint: disable=unused-argument,no-self-use """Return decryption materials. :param encryption_context: Encryption context for request @@ -30,6 +33,7 @@ def decryption_materials(self, encryption_context): def encryption_materials(self, encryption_context): # type: (EncryptionContext) -> EncryptionMaterials + # pylint: disable=unused-argument,no-self-use """Return encryption materials. :param encryption_context: Encryption context for request @@ -39,6 +43,8 @@ def encryption_materials(self, encryption_context): raise AttributeError('No encryption materials available') def refresh(self): + # type: () -> None + # pylint: disable=unused-argument,no-self-use """Ask this instance to refresh the cryptographic materials. .. note:: diff --git a/src/dynamodb_encryption_sdk/material_providers/aws_kms.py b/src/dynamodb_encryption_sdk/material_providers/aws_kms.py index 72b3cdeb..d926fad3 100644 --- a/src/dynamodb_encryption_sdk/material_providers/aws_kms.py +++ b/src/dynamodb_encryption_sdk/material_providers/aws_kms.py @@ -12,28 +12,38 @@ # language governing permissions and limitations under the License. """Cryptographic materials provider for use with the AWS Key Management Service (KMS).""" from __future__ import division + import base64 from enum import Enum +import logging import attr -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import hashes -from cryptography.hazmat.primitives.kdf.hkdf import HKDF import boto3 import botocore.client import botocore.session +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.kdf.hkdf import HKDF import six -from . import CryptographicMaterialsProvider +try: # Python 3.5.0 and 3.5.1 have incompatible typing modules + from dynamodb_encryption_sdk.internal import dynamodb_types # noqa pylint: disable=unused-import +except ImportError: # pragma: no cover + # We only actually need these imports when running the mypy checks + pass + from dynamodb_encryption_sdk.delegated_keys.jce import JceNameLocalDelegatedKey from dynamodb_encryption_sdk.exceptions import UnknownRegionError, UnwrappingError, WrappingError -from dynamodb_encryption_sdk.identifiers import EncryptionKeyTypes, KeyEncodingType -from dynamodb_encryption_sdk.internal import dynamodb_types +from dynamodb_encryption_sdk.identifiers import EncryptionKeyType, KeyEncodingType, LOGGER_NAME from dynamodb_encryption_sdk.internal.identifiers import MaterialDescriptionKeys +from dynamodb_encryption_sdk.internal.str_ops import to_bytes, to_str +from dynamodb_encryption_sdk.internal.validators import dictionary_validator, iterable_validator from dynamodb_encryption_sdk.materials.raw import RawDecryptionMaterials, RawEncryptionMaterials -from dynamodb_encryption_sdk.structures import EncryptionContext +from dynamodb_encryption_sdk.structures import EncryptionContext # noqa pylint: disable=unused-import +from . import CryptographicMaterialsProvider __all__ = ('AwsKmsCryptographicMaterialsProvider',) +_LOGGER = logging.getLogger(LOGGER_NAME) _COVERED_ATTR_CTX_KEY = 'aws-kms-ec-attr' _TABLE_NAME_EC_KEY = '*aws-kms-table*' @@ -47,29 +57,56 @@ class HkdfInfo(Enum): """Info strings used for HKDF calculations.""" + ENCRYPTION = b'Encryption' SIGNING = b'Signing' class EncryptionContextKeys(Enum): """Special keys for use in the AWS KMS encryption context.""" + CONTENT_ENCRYPTION_ALGORITHM = '*' + MaterialDescriptionKeys.CONTENT_ENCRYPTION_ALGORITHM.value + '*' SIGNATURE_ALGORITHM = '*' + MaterialDescriptionKeys.ITEM_SIGNATURE_ALGORITHM.value + '*' TABLE_NAME = '*aws-kms-table*' -@attr.s(hash=False) +@attr.s class KeyInfo(object): + # pylint: disable=too-few-public-methods """Identifying information for a specific key and how it should be used. :param str description: algorithm identifier joined with key length in bits :param str algorithm: algorithm identifier :param int length: Key length in bits """ + description = attr.ib(validator=attr.validators.instance_of(six.string_types)) algorithm = attr.ib(validator=attr.validators.instance_of(six.string_types)) length = attr.ib(validator=attr.validators.instance_of(six.integer_types)) + @classmethod + def from_description(cls, description, default_key_length=None): + # type: (Text) -> KeyInfo + """Load key info from key info description. + + :param str description: Key info description + :param int default_key_length: Key length to use if not found in description + """ + description_parts = description.split('/', 1) + algorithm = description_parts[0] + try: + key_length = int(description_parts[1]) + except IndexError: + if default_key_length is None: + raise ValueError( + 'Description "{}" does not contain key length and no default key length is provided'.format( + description + ) + ) + + key_length = default_key_length + return cls(description, algorithm, key_length) + @classmethod def from_material_description(cls, material_description, description_key, default_algorithm, default_key_length): # type: (Dict[Text, Text], Text, Text, int) -> KeyInfo @@ -78,21 +115,15 @@ def from_material_description(cls, material_description, description_key, defaul :param dict material_description: Material description to read :param str description_key: Material description key containing desired key info description :param str default_algorithm: Algorithm name to use if not found in material description - :param int default_key_length: Key length to use if not found in material description + :param int default_key_length: Key length to use if not found in key info description :returns: Key info loaded from material description, with defaults applied if necessary :rtype: dynamodb_encryption_sdk.material_providers.aws_kms.KeyInfo """ description = material_description.get(description_key, default_algorithm) - description_parts = description.split('/', 1) - algorithm = description_parts[0] - try: - key_length = int(description_parts[1]) - except IndexError: - key_length = default_key_length - return cls(description, algorithm, key_length) + return cls.from_description(description, default_key_length) -@attr.s(hash=False) +@attr.s class AwsKmsCryptographicMaterialsProvider(CryptographicMaterialsProvider): """Cryptographic materials provider for use with the AWS Key Management Service (KMS). @@ -110,66 +141,41 @@ class AwsKmsCryptographicMaterialsProvider(CryptographicMaterialsProvider): :param dict regional_clients: Dictionary mapping AWS region names to pre-configured boto3 KMS clients (optional) """ + _key_id = attr.ib(validator=attr.validators.instance_of(six.string_types)) _botocore_session = attr.ib( validator=attr.validators.instance_of(botocore.session.Session), default=attr.Factory(botocore.session.Session) ) - _grant_tokens = attr.ib(default=attr.Factory(tuple)) - - @_grant_tokens.validator - def _grant_tokens_validator(self, attribute, value): - """Validate grant token values.""" - if not isinstance(value, tuple): - raise TypeError('"grant_tokens" must be a tuple') - - for token in value: - if not isinstance(token, six.string_types): - raise TypeError('"grant_tokens" must contain strings') - - _material_description = attr.ib(default=attr.Factory(dict)) - - @_material_description.validator - def _material_description_validator(self, attribute, value): - """Validate material description values.""" - if not isinstance(value, dict): - raise TypeError('"material_description" must be a dictionary') - - for key, data in value.items(): - if not (isinstance(key, six.string_types) and isinstance(data, six.string_types)): - raise TypeError('"material_description" must be a string-string dictionary') - - _regional_clients = attr.ib(default=attr.Factory(dict)) - - @_regional_clients.validator - def regional_clients_validator(self, attribute, value): - """Validate regional clients values.""" - if not isinstance(value, dict): - raise TypeError('"regional_clients" must be a dictionary') - - for key, client in value.items(): - if not isinstance(key, six.string_types): - raise TypeError('"regional_clients" region name must be a string') - - if not isinstance(client, botocore.client.BaseClient): - raise TypeError('"regional_clients" client must be a botocore client') + _grant_tokens = attr.ib( + validator=iterable_validator(tuple, six.string_types), + default=attr.Factory(tuple) + ) + _material_description = attr.ib( + validator=dictionary_validator(six.string_types, six.string_types), + default=attr.Factory(dict) + ) + _regional_clients = attr.ib( + validator=dictionary_validator(six.string_types, botocore.client.BaseClient), + default=attr.Factory(dict) + ) def __attrs_post_init__(self): # type: () -> None """Load the content and signing key info.""" - self._content_key_info = KeyInfo.from_material_description( + self._content_key_info = KeyInfo.from_material_description( # pylint: disable=attribute-defined-outside-init material_description=self._material_description, description_key=MaterialDescriptionKeys.CONTENT_ENCRYPTION_ALGORITHM.value, default_algorithm=_DEFAULT_CONTENT_ENCRYPTION_ALGORITHM, default_key_length=_DEFAULT_CONTENT_KEY_LENGTH ) - self._signing_key_info = KeyInfo.from_material_description( + self._signing_key_info = KeyInfo.from_material_description( # pylint: disable=attribute-defined-outside-init material_description=self._material_description, description_key=MaterialDescriptionKeys.ITEM_SIGNATURE_ALGORITHM.value, default_algorithm=_DEFAULT_SIGNING_ALGORITHM, default_key_length=_DEFAULT_SIGNING_KEY_LENGTH ) - self._regional_clients = {} + self._regional_clients = {} # type: Dict[Text, botocore.client.BaseClient] # noqa pylint: disable=attribute-defined-outside-init def _add_regional_client(self, region_name): # type: (Text) -> None @@ -182,6 +188,7 @@ def _add_regional_client(self, region_name): region_name=region_name, botocore_session=self._botocore_session ).client('kms') + return self._regional_clients[region_name] def _client(self, key_id): """Returns a boto3 KMS client for the appropriate region. @@ -205,6 +212,7 @@ def _client(self, key_id): def _select_key_id(self, encryption_context): # type: (EncryptionContext) -> Text + # pylint: disable=unused-argument """Select the desired key id. .. note:: @@ -221,6 +229,8 @@ def _select_key_id(self, encryption_context): return self._key_id def _validate_key_id(self, key_id, encryption_context): + # type: (EncryptionContext) -> None + # pylint: disable=unused-argument,no-self-use """Validate the selected key id. .. note:: @@ -244,7 +254,7 @@ def _attribute_to_value(self, attribute): attribute_type, attribute_value = list(attribute.items())[0] if attribute_type == 'B': return base64.b64encode(attribute_value.value).decode('utf-8') - if attribute_type == 'S': + if attribute_type in ('S', 'N'): return attribute_value raise ValueError('Attribute of type "{}" cannot be used in KMS encryption context.'.format(attribute_type)) @@ -265,14 +275,22 @@ def _kms_encryption_context(self, encryption_context, encryption_description, si } if encryption_context.partition_key_name is not None: - partition_key_attribute = encryption_context.attributes.get(encryption_context.partition_key_name) - kms_encryption_context[encryption_context.partition_key_name] = self._attribute_to_value( - partition_key_attribute - ) + try: + partition_key_attribute = encryption_context.attributes[encryption_context.partition_key_name] + except KeyError: + pass + else: + kms_encryption_context[encryption_context.partition_key_name] = self._attribute_to_value( + partition_key_attribute + ) if encryption_context.sort_key_name is not None: - sort_key_attribute = encryption_context.attributes.get(encryption_context.sort_key_name) - kms_encryption_context[encryption_context.sort_key_name] = self._attribute_to_value(sort_key_attribute) + try: + sort_key_attribute = encryption_context.attributes[encryption_context.sort_key_name] + except KeyError: + pass + else: + kms_encryption_context[encryption_context.sort_key_name] = self._attribute_to_value(sort_key_attribute) if encryption_context.table_name is not None: kms_encryption_context[_TABLE_NAME_EC_KEY] = encryption_context.table_name @@ -308,7 +326,9 @@ def _generate_initial_material(self, encryption_context): response = self._client(key_id).generate_data_key(**kms_params) return response['Plaintext'], response['CiphertextBlob'] except (botocore.exceptions.ClientError, KeyError): - raise WrappingError('TODO:SOMETHING') + message = 'Failed to generate materials using AWS KMS' + _LOGGER.exception(message) + raise WrappingError(message) def _decrypt_initial_material(self, encryption_context): # type: () -> bytes @@ -330,9 +350,9 @@ def _decrypt_initial_material(self, encryption_context): MaterialDescriptionKeys.ITEM_SIGNATURE_ALGORITHM.value ) ) - encrypted_initial_material = base64.b64decode(encryption_context.material_description.get( + encrypted_initial_material = base64.b64decode(to_bytes(encryption_context.material_description.get( MaterialDescriptionKeys.WRAPPED_DATA_KEY.value - )) + ))) kms_params = dict( CiphertextBlob=encrypted_initial_material, EncryptionContext=kms_encryption_context @@ -344,7 +364,9 @@ def _decrypt_initial_material(self, encryption_context): response = self._client(key_id).decrypt(**kms_params) return response['Plaintext'] except (botocore.exceptions.ClientError, KeyError): - raise UnwrappingError('TODO:SOMETHING') + message = 'Failed to unwrap AWS KMS protected materials' + _LOGGER.exception(message) + raise UnwrappingError(message) def _hkdf(self, initial_material, key_length, info): # type: (bytes, int, Text) -> bytes @@ -381,7 +403,7 @@ def _derive_delegated_key(self, initial_material, key_info, hkdf_info): return JceNameLocalDelegatedKey( key=raw_key, algorithm=key_info.algorithm, - key_type=EncryptionKeyTypes.SYMMETRIC, + key_type=EncryptionKeyType.SYMMETRIC, key_encoding=KeyEncodingType.RAW ) @@ -454,7 +476,7 @@ def encryption_materials(self, encryption_context): MaterialDescriptionKeys.CONTENT_KEY_WRAPPING_ALGORITHM.value: 'kms', MaterialDescriptionKeys.CONTENT_ENCRYPTION_ALGORITHM.value: self._content_key_info.description, MaterialDescriptionKeys.ITEM_SIGNATURE_ALGORITHM.value: self._signing_key_info.description, - MaterialDescriptionKeys.WRAPPED_DATA_KEY.value: base64.b64encode(encrypted_initial_material) + MaterialDescriptionKeys.WRAPPED_DATA_KEY.value: to_str(base64.b64encode(encrypted_initial_material)) }) return RawEncryptionMaterials( signing_key=self._mac_key(initial_material, self._signing_key_info), diff --git a/src/dynamodb_encryption_sdk/material_providers/static.py b/src/dynamodb_encryption_sdk/material_providers/static.py index 6bdd403b..3c1ab28a 100644 --- a/src/dynamodb_encryption_sdk/material_providers/static.py +++ b/src/dynamodb_encryption_sdk/material_providers/static.py @@ -13,12 +13,14 @@ """Cryptographic materials provider for use with pre-configured encryption and decryption materials.""" import attr -from . import CryptographicMaterialsProvider from dynamodb_encryption_sdk.materials import DecryptionMaterials, EncryptionMaterials -from dynamodb_encryption_sdk.structures import EncryptionContext +from dynamodb_encryption_sdk.structures import EncryptionContext # noqa pylint: disable=unused-import +from . import CryptographicMaterialsProvider +__all__ = ('StaticCryptographicMaterialsProvider',) -@attr.s(hash=False) + +@attr.s class StaticCryptographicMaterialsProvider(CryptographicMaterialsProvider): """Manually combine encryption and decryption materials for use as a cryptographic materials provider. @@ -27,6 +29,7 @@ class StaticCryptographicMaterialsProvider(CryptographicMaterialsProvider): :param encryption_materials: Encryption materials to provide (optional) :type encryption_materials: dynamodb_encryption_sdk.materials.EncryptionMaterials """ + _decryption_materials = attr.ib( validator=attr.validators.optional(attr.validators.instance_of(DecryptionMaterials)), default=None @@ -45,7 +48,7 @@ def decryption_materials(self, encryption_context): :raises AttributeError: if no decryption materials are available """ if self._decryption_materials is None: - super(StaticCryptographicMaterialsProvider, self).decryption_materials(encryption_context) + return super(StaticCryptographicMaterialsProvider, self).decryption_materials(encryption_context) return self._decryption_materials @@ -58,6 +61,6 @@ def encryption_materials(self, encryption_context): :raises AttributeError: if no encryption materials are available """ if self._encryption_materials is None: - super(StaticCryptographicMaterialsProvider, self).encryption_materials(encryption_context) + return super(StaticCryptographicMaterialsProvider, self).encryption_materials(encryption_context) return self._encryption_materials diff --git a/src/dynamodb_encryption_sdk/material_providers/wrapped.py b/src/dynamodb_encryption_sdk/material_providers/wrapped.py index 6e074df9..96c3c025 100644 --- a/src/dynamodb_encryption_sdk/material_providers/wrapped.py +++ b/src/dynamodb_encryption_sdk/material_providers/wrapped.py @@ -13,14 +13,16 @@ """Cryptographic materials provider to use ephemeral content encryption keys wrapped by delegated keys.""" import attr -from . import CryptographicMaterialsProvider from dynamodb_encryption_sdk.delegated_keys import DelegatedKey from dynamodb_encryption_sdk.exceptions import UnwrappingError, WrappingError from dynamodb_encryption_sdk.materials.wrapped import WrappedCryptographicMaterials -from dynamodb_encryption_sdk.structures import EncryptionContext +from dynamodb_encryption_sdk.structures import EncryptionContext # noqa pylint: disable=unused-import +from . import CryptographicMaterialsProvider + +__all__ = ('WrappedCryptographicMaterialsProvider',) -@attr.s(hash=False) +@attr.s class WrappedCryptographicMaterialsProvider(CryptographicMaterialsProvider): """Cryptographic materials provider to use ephemeral content encryption keys wrapped by delegated keys. @@ -41,6 +43,7 @@ class WrappedCryptographicMaterialsProvider(CryptographicMaterialsProvider): ``unwrapping_key`` must be provided if providing decryption materials or loading materials from material description """ + _signing_key = attr.ib(validator=attr.validators.instance_of(DelegatedKey)) _wrapping_key = attr.ib( validator=attr.validators.optional(attr.validators.instance_of(DelegatedKey)), @@ -52,6 +55,7 @@ class WrappedCryptographicMaterialsProvider(CryptographicMaterialsProvider): ) def _build_materials(self, encryption_context): + # type: (EncryptionContext) -> WrappedCryptographicMaterials """Construct :param encryption_context: Encryption context for request diff --git a/src/dynamodb_encryption_sdk/materials/__init__.py b/src/dynamodb_encryption_sdk/materials/__init__.py index f61c3553..a1c73e82 100644 --- a/src/dynamodb_encryption_sdk/materials/__init__.py +++ b/src/dynamodb_encryption_sdk/materials/__init__.py @@ -10,18 +10,20 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""""" +"""Cryptographic materials are containers that provide delegated keys for cryptographic operations.""" import abc try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from mypy_extensions import NoReturn + from mypy_extensions import NoReturn # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass import six -from dynamodb_encryption_sdk.delegated_keys import DelegatedKey +from dynamodb_encryption_sdk.delegated_keys import DelegatedKey # noqa pylint: disable=unused-import + +__all__ = ('CryptographicMaterials', 'EncryptionMaterials', 'DecryptionMaterials') @six.add_metaclass(abc.ABCMeta) @@ -80,18 +82,20 @@ class EncryptionMaterials(CryptographicMaterials): @property def decryption_key(self): # type: () -> NoReturn - """ + """Encryption materials do not provide decryption keys. + :raises NotImplementedError: because encryption materials do not contain decryption keys """ - raise NotImplementedError('EncryptionMaterials do not provide decryption keys.') + raise NotImplementedError('Encryption materials do not provide decryption keys.') @property def verification_key(self): # type: () -> NoReturn - """ + """Encryption materials do not provide verification keys. + :raises NotImplementedError: because encryption materials do not contain verification keys """ - raise NotImplementedError('EncryptionMaterials do not provide verification keys.') + raise NotImplementedError('Encryption materials do not provide verification keys.') class DecryptionMaterials(CryptographicMaterials): @@ -100,15 +104,17 @@ class DecryptionMaterials(CryptographicMaterials): @property def encryption_key(self): # type: () -> NoReturn - """ + """Decryption materials do not provide encryption keys. + :raises NotImplementedError: because decryption materials do not contain encryption keys """ - raise NotImplementedError('EncryptionMaterials do not provide encryption keys.') + raise NotImplementedError('Decryption materials do not provide encryption keys.') @property def signing_key(self): # type: () -> NoReturn - """ + """Decryption materials do not provide signing keys. + :raises NotImplementedError: because decryption materials do not contain signing keys """ - raise NotImplementedError('EncryptionMaterials do not provide signing keys.') + raise NotImplementedError('Decryption materials do not provide signing keys.') diff --git a/src/dynamodb_encryption_sdk/materials/raw.py b/src/dynamodb_encryption_sdk/materials/raw.py index 15fe188b..f8aea5b4 100644 --- a/src/dynamodb_encryption_sdk/materials/raw.py +++ b/src/dynamodb_encryption_sdk/materials/raw.py @@ -25,13 +25,18 @@ import copy import attr +import six from dynamodb_encryption_sdk.delegated_keys import DelegatedKey +from dynamodb_encryption_sdk.internal.validators import dictionary_validator from dynamodb_encryption_sdk.materials import DecryptionMaterials, EncryptionMaterials +__all__ = ('RawEncryptionMaterials', 'RawDecryptionMaterials') -@attr.s(hash=False) + +@attr.s class RawEncryptionMaterials(EncryptionMaterials): + # inheritance confuses pylint: disable=abstract-method """Encryption materials for use directly with delegated keys. .. note:: @@ -44,10 +49,11 @@ class RawEncryptionMaterials(EncryptionMaterials): :type encryption_key: dynamodb_encryption_sdk.delegated_keys.DelegatedKey :param dict material_description: Material description to use with these cryptographic materials """ + _signing_key = attr.ib(validator=attr.validators.instance_of(DelegatedKey)) _encryption_key = attr.ib(validator=attr.validators.instance_of(DelegatedKey)) _material_description = attr.ib( - validator=attr.validators.instance_of(dict), + validator=dictionary_validator(six.string_types, six.string_types), converter=copy.deepcopy, default=attr.Factory(dict) ) @@ -90,8 +96,9 @@ def encryption_key(self): return self._encryption_key -@attr.s(hash=False) +@attr.s class RawDecryptionMaterials(DecryptionMaterials): + # inheritance confuses pylint: disable=abstract-method """Encryption materials for use directly with delegated keys. .. note:: @@ -104,10 +111,11 @@ class RawDecryptionMaterials(DecryptionMaterials): :type decryption_key: dynamodb_encryption_sdk.delegated_keys.DelegatedKey :param dict material_description: Material description to use with these cryptographic materials """ + _verification_key = attr.ib(validator=attr.validators.instance_of(DelegatedKey)) _decryption_key = attr.ib(validator=attr.validators.instance_of(DelegatedKey)) _material_description = attr.ib( - validator=attr.validators.instance_of(dict), + validator=dictionary_validator(six.string_types, six.string_types), converter=copy.deepcopy, default=attr.Factory(dict) ) diff --git a/src/dynamodb_encryption_sdk/materials/wrapped.py b/src/dynamodb_encryption_sdk/materials/wrapped.py index e8baa440..937ec9f5 100644 --- a/src/dynamodb_encryption_sdk/materials/wrapped.py +++ b/src/dynamodb_encryption_sdk/materials/wrapped.py @@ -11,20 +11,21 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. """Cryptographic materials to use ephemeral content encryption keys wrapped by delegated keys.""" -from __future__ import division import base64 import copy import attr +import six from dynamodb_encryption_sdk.delegated_keys import DelegatedKey from dynamodb_encryption_sdk.delegated_keys.jce import JceNameLocalDelegatedKey from dynamodb_encryption_sdk.exceptions import UnwrappingError, WrappingError -from dynamodb_encryption_sdk.identifiers import EncryptionKeyTypes +from dynamodb_encryption_sdk.identifiers import EncryptionKeyType from dynamodb_encryption_sdk.internal.identifiers import MaterialDescriptionKeys +from dynamodb_encryption_sdk.internal.validators import dictionary_validator from dynamodb_encryption_sdk.materials import CryptographicMaterials -__all__ = ('WrappedRawCryptographicMaterials',) +__all__ = ('WrappedCryptographicMaterials',) _DEFAULT_CONTENT_ENCRYPTION_ALGORITHM = 'AES/256' _WRAPPING_TRANSFORMATION = { 'AES': 'AESWrap', @@ -32,7 +33,7 @@ } -@attr.s(hash=False) +@attr.s class WrappedCryptographicMaterials(CryptographicMaterials): """Encryption/decryption key is a content key stored in the material description, wrapped by the wrapping key. @@ -55,6 +56,7 @@ class WrappedCryptographicMaterials(CryptographicMaterials): :param dict material_description: Material description to use with these cryptographic materials """ + _signing_key = attr.ib(validator=attr.validators.instance_of(DelegatedKey)) _wrapping_key = attr.ib( validator=attr.validators.optional(attr.validators.instance_of(DelegatedKey)), @@ -65,24 +67,25 @@ class WrappedCryptographicMaterials(CryptographicMaterials): default=None ) _material_description = attr.ib( - validator=attr.validators.instance_of(dict), + validator=dictionary_validator(six.string_types, six.string_types), converter=copy.deepcopy, default=attr.Factory(dict) ) def __attrs_post_init__(self): """Prepare the content key.""" - self._content_key_algorithm = self.material_description.get( + self._content_key_algorithm = self.material_description.get( # pylint: disable=attribute-defined-outside-init MaterialDescriptionKeys.CONTENT_ENCRYPTION_ALGORITHM.value, _DEFAULT_CONTENT_ENCRYPTION_ALGORITHM ) if MaterialDescriptionKeys.WRAPPED_DATA_KEY.value in self.material_description: - self._content_key = self._content_key_from_material_description() + self._content_key = self._content_key_from_material_description() # noqa pylint: disable=attribute-defined-outside-init else: - self._content_key, self._material_description = self._generate_content_key() + self._content_key, self._material_description = self._generate_content_key() # noqa pylint: disable=attribute-defined-outside-init - def _wrapping_transformation(self, algorithm): + @staticmethod + def _wrapping_transformation(algorithm): """Convert the specified algorithm name to the desired wrapping algorithm transformation. :param str algorithm: Algorithm name @@ -114,7 +117,7 @@ def _content_key_from_material_description(self): algorithm=wrapping_algorithm, wrapped_key=wrapped_key, wrapped_key_algorithm=content_key_algorithm, - wrapped_key_type=EncryptionKeyTypes.SYMMETRIC, + wrapped_key_type=EncryptionKeyType.SYMMETRIC, additional_associated_data=None ) @@ -135,7 +138,7 @@ def _generate_content_key(self): args = self._content_key_algorithm.split('/', 1) content_algorithm = args[0] try: - content_key_length = int(args[1]) // 8 + content_key_length = int(args[1]) except IndexError: content_key_length = None content_key = JceNameLocalDelegatedKey.generate( diff --git a/src/dynamodb_encryption_sdk/structures.py b/src/dynamodb_encryption_sdk/structures.py index d50b4f8e..776f9685 100644 --- a/src/dynamodb_encryption_sdk/structures.py +++ b/src/dynamodb_encryption_sdk/structures.py @@ -10,25 +10,45 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""""" -import attr +"""Common structures used by the DynamoDB Encryption Client.""" import copy +import attr import six -from .identifiers import ItemAction +from dynamodb_encryption_sdk.internal.identifiers import ReservedAttributes +from dynamodb_encryption_sdk.internal.validators import dictionary_validator, iterable_validator +from .identifiers import CryptoAction + +__all__ = ('EncryptionContext', 'AttributeActions', 'TableIndex', 'TableInfo') + + +def _validate_attribute_values_are_ddb_items(instance, attribute, value): # pylint: disable=unused-argument + """Validate that dictionary values in ``value`` match the structure of DynamoDB JSON + items. + .. note:: -@attr.s(hash=False) + We are not trying to validate the full structure of the item with this validator. + This is just meant to verify that the values roughly match the correct format. + """ + for data in value.values(): + if len(list(data.values())) != 1: + raise TypeError('"{}" values do not look like DynamoDB items'.format(attribute.name)) + + +@attr.s class EncryptionContext(object): + # pylint: disable=too-few-public-methods """Additional information about an encryption request. :param str table_name: Table name :param str partition_key_name: Name of primary index partition attribute :param str sort_key_name: Name of primary index sort attribute - :param dict attributes: Plaintext item attributes + :param dict attributes: Plaintext item attributes as a DynamoDB JSON dictionary :param dict material_description: Material description to use with this request """ + table_name = attr.ib( validator=attr.validators.optional(attr.validators.instance_of(six.string_types)), default=None @@ -41,46 +61,54 @@ class EncryptionContext(object): validator=attr.validators.optional(attr.validators.instance_of(six.string_types)), default=None ) - # TODO: converter to make sure that attributes are in DDB form attributes = attr.ib( - validator=attr.validators.optional(attr.validators.instance_of(dict)), + validator=( + dictionary_validator(six.string_types, dict), + _validate_attribute_values_are_ddb_items + ), default=attr.Factory(dict) ) material_description = attr.ib( - validator=attr.validators.instance_of(dict), + validator=dictionary_validator(six.string_types, six.string_types), converter=copy.deepcopy, default=attr.Factory(dict) ) -@attr.s(hash=False) +@attr.s class AttributeActions(object): """Configuration resource used to determine what action should be taken for a specific attribute. :param default_action: Action to take if no specific action is defined in ``attribute_actions`` - :type default_action: dynamodb_encryption_sdk.identifiers.ItemAction + :type default_action: dynamodb_encryption_sdk.identifiers.CryptoAction :param dict attribute_actions: Dictionary mapping attribute names to specific actions """ + default_action = attr.ib( - validator=attr.validators.instance_of(ItemAction), - default=ItemAction.ENCRYPT_AND_SIGN + validator=attr.validators.instance_of(CryptoAction), + default=CryptoAction.ENCRYPT_AND_SIGN ) attribute_actions = attr.ib( - validator=attr.validators.instance_of(dict), + validator=dictionary_validator(six.string_types, CryptoAction), default=attr.Factory(dict) ) def __attrs_post_init__(self): # () -> None """Determine if any actions should ever be taken with this configuration and record that for reference.""" + for attribute in ReservedAttributes: + if attribute.value in self.attribute_actions: + raise ValueError('No override behavior can be set for reserved attribute "{}"'.format(attribute.value)) + # Enums are not hashable, but their names are unique _unique_actions = set([self.default_action.name]) _unique_actions.update(set([action.name for action in self.attribute_actions.values()])) - self.take_no_actions = _unique_actions == set([ItemAction.DO_NOTHING.name]) + no_actions = _unique_actions == set([CryptoAction.DO_NOTHING.name]) + self.take_no_actions = no_actions # attrs confuses pylint: disable=attribute-defined-outside-init def action(self, attribute_name): - # (text) -> ItemAction - """Determines the correct ItemAction to apply to a supplied attribute based on this config.""" + # (text) -> CryptoAction + """Determines the correct CryptoAction to apply to a supplied attribute based on this config.""" return self.attribute_actions.get(attribute_name, self.default_action) def copy(self): @@ -100,7 +128,7 @@ def set_index_keys(self, *keys): """ for key in keys: current_action = self.action(key) - self.attribute_actions[key] = min(current_action, ItemAction.SIGN_ONLY) + self.attribute_actions[key] = min(current_action, CryptoAction.SIGN_ONLY) def __add__(self, other): # (AttributeActions) -> AttributeActions @@ -118,101 +146,134 @@ def __add__(self, other): ) -@attr.s(hash=False) +@attr.s class TableIndex(object): + # pylint: disable=too-few-public-methods """Describes a table index. :param str partition: Name of the partition attribute :param str sort: Name of the sort attribute (optional) """ + partition = attr.ib(validator=attr.validators.instance_of(six.string_types)) sort = attr.ib( - default=None, - validator=attr.validators.optional(attr.validators.instance_of(six.string_types)) + validator=attr.validators.optional(attr.validators.instance_of(six.string_types)), + default=None ) def __attrs_post_init__(self): """Set the ``attributes`` attribute for ease of access later.""" - self.attributes = set([self.partition]) - if self.sort is None: + self.attributes = set([self.partition]) # attrs confuses pylint: disable=attribute-defined-outside-init + if self.sort is not None: self.attributes.add(self.sort) + @classmethod + def from_key_schema(cls, key_schema): + # type: (Iterable[Dict[Text, Text]]) -> TableIndex + """Build a TableIndex from the key schema returned by DescribeTable. + + [ + { + "KeyType": "HASH"|"RANGE", + "AttributeName": "" + }, + ] + + :param list key_schema: KeySchema from DescribeTable response + :returns: New TableIndex that describes the provided schema + :rtype: dynamodb_encryption_sdk.structures.TableIndex + """ + index = { + key['KeyType']: key['AttributeName'] + for key in key_schema + } + return cls( + partition=index['HASH'], + sort=index.get('RANGE', None) + ) + -@attr.s(hash=False) +@attr.s class TableInfo(object): - """Description of a DynamoDB table. + """Describes a DynamoDB table. :param str name: Table name :param bool all_encrypting_secondary_indexes: Should we allow secondary index attributes to be encrypted? :param primary_index: Description of primary index :type primary_index: dynamodb_encryption_sdk.structures.TableIndex - :param indexed_attributes: Listing of all indexes attribute names - :type indexed_attributes: set of str + :param secondary_indexes: Set of TableIndex objects describing any secondary indexes + :type secondary_indexes: set of dynamodb_encryption_sdk.structures.TableIndex """ + name = attr.ib(validator=attr.validators.instance_of(six.string_types)) - allow_encrypting_secondary_indexes = attr.ib( - validator=attr.validators.instance_of(bool), - default=False - ) _primary_index = attr.ib( validator=attr.validators.optional(attr.validators.instance_of(TableIndex)), default=None ) - _indexed_attributes = attr.ib( - validator=attr.validators.optional(attr.validators.instance_of(set)), + _secondary_indexes = attr.ib( + validator=attr.validators.optional(iterable_validator(set, TableIndex)), default=None ) @property def primary_index(self): # type: () -> TableIndex - """""" + """Return the primary TableIndex. + + :returns: primary index description + :rtype: TableIndex + :raises AttributeError: if primary index is unknown + """ if self._primary_index is None: - raise Exception('TODO:Indexes unknown. Run refresh_indexed_attributes') + raise AttributeError('Indexes unknown. Run refresh_indexed_attributes') return self._primary_index @property - def indexed_attributes(self): - # type: () -> TableIndex - # TODO: Think about merging this and all_index_keys - """""" - if self._indexed_attributes is None: - raise Exception('TODO:Indexes unknown. Run refresh_indexed_attributes') - return self._indexed_attributes - - def all_index_keys(self): - # type: () -> Set[str] + def secondary_indexes(self): + # type: () -> Set[TableIndex] + """Return the primary TableIndex. + + :returns: secondary index descriptions + :rtype: TableIndex + :raises AttributeError: if secondary indexes are unknown + """ + if self._secondary_indexes is None: + raise AttributeError('Indexes unknown. Run refresh_indexed_attributes') + return self._secondary_indexes + + def protected_index_keys(self): + # type: () -> Set[Text] """Provide a set containing the names of all indexed attributes that must not be encrypted.""" - if self._primary_index is None: - return set() + return self.primary_index.attributes - if self.allow_encrypting_secondary_indexes: - return self.primary_index.attributes + @property + def encryption_context_values(self): + # type: () -> Dict[Text, Text] + """Build parameters needed to inform an EncryptionContext constructor about this table. - return self.indexed_attributes + :rtype: dict + """ + values = {'table_name': self.name} + if self.primary_index is not None: + values.update({ + 'partition_key_name': self.primary_index.partition, + 'sort_key_name': self.primary_index.sort + }) + return values def refresh_indexed_attributes(self, client): """Use the provided boto3 DynamoDB client to determine all indexes for this table. :param client: Pre-configured boto3 DynamoDB client - :type client: TODO: + :type client: botocore.client.BaseClient """ table = client.describe_table(TableName=self.name)['Table'] - primary_index = { - key['KeyType']: key['AttributeName'] - for key in table['KeySchema'] - } - indexed_attributes = set(primary_index.values()) - self._primary_index = TableIndex( - partition=primary_index['HASH'], - sort=primary_index.get('RANGE', None) - ) + self._primary_index = TableIndex.from_key_schema(table['KeySchema']) + + self._secondary_indexes = set() for group in ('LocalSecondaryIndexes', 'GlobalSecondaryIndexes'): try: for index in table[group]: - indexed_attributes.update(set([ - key['AttributeName'] for key in index['KeySchema'] - ])) + self._secondary_indexes.add(TableIndex.from_key_schema(index['KeySchema'])) except KeyError: pass # Not all tables will have secondary indexes. - self._indexed_attributes = indexed_attributes diff --git a/src/dynamodb_encryption_sdk/internal/formatting/transform.py b/src/dynamodb_encryption_sdk/transform.py similarity index 90% rename from src/dynamodb_encryption_sdk/internal/formatting/transform.py rename to src/dynamodb_encryption_sdk/transform.py index 43c441e7..8541639a 100644 --- a/src/dynamodb_encryption_sdk/internal/formatting/transform.py +++ b/src/dynamodb_encryption_sdk/transform.py @@ -12,12 +12,14 @@ # language governing permissions and limitations under the License. """Helper tools for translating between native and DynamoDB items.""" try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Any, Dict # pylint: disable=unused-import + from typing import Any, Dict # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass -from boto3.dynamodb.types import TypeSerializer, TypeDeserializer +from boto3.dynamodb.types import TypeDeserializer, TypeSerializer + +__all__ = ('dict_to_ddb', 'ddb_to_dict') def dict_to_ddb(item): diff --git a/src/pylintrc b/src/pylintrc index de56ef0f..43812946 100644 --- a/src/pylintrc +++ b/src/pylintrc @@ -1,6 +1,10 @@ [BASIC] # Allow function names up to 50 characters function-rgx = [a-z_][a-z0-9_]{2,50}$ +# Whitelist argument names: iv +argument-rgx = ([a-z_][a-z0-9_]{2,30}$)|(^iv$) +# Whitelist variable names: iv +variable-rgx = ([a-z_][a-z0-9_]{2,30}$)|(^iv$) [DESIGN] max-args = 10 diff --git a/test/acceptance/__init__.py b/test/acceptance/__init__.py index 1ccc7fa1..2add15ef 100644 --- a/test/acceptance/__init__.py +++ b/test/acceptance/__init__.py @@ -10,3 +10,4 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +"""Dummy stub to make linters work better.""" diff --git a/test/acceptance/acceptance_test_utils.py b/test/acceptance/acceptance_test_utils.py index dae4cd9c..fc0ff1a5 100644 --- a/test/acceptance/acceptance_test_utils.py +++ b/test/acceptance/acceptance_test_utils.py @@ -10,29 +10,32 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +"""Helper tools for use with acceptance tests.""" import base64 from collections import defaultdict import json import os import sys -sys.path.append(os.path.join( - os.path.abspath(os.path.dirname(__file__)), - '..', - 'functional' -)) import pytest -from six.moves.urllib.parse import urlparse - +from six.moves.urllib.parse import urlparse # moves confuse pylint: disable=wrong-import-order +from dynamodb_encryption_sdk.delegated_keys.jce import JceNameLocalDelegatedKey +from dynamodb_encryption_sdk.identifiers import EncryptionKeyType, KeyEncodingType +from dynamodb_encryption_sdk.material_providers.aws_kms import AwsKmsCryptographicMaterialsProvider from dynamodb_encryption_sdk.material_providers.static import StaticCryptographicMaterialsProvider from dynamodb_encryption_sdk.material_providers.wrapped import WrappedCryptographicMaterialsProvider -from dynamodb_encryption_sdk.materials.raw import RawDecryptionMaterials, RawEncryptionMaterials -from dynamodb_encryption_sdk.delegated_keys.jce import JceNameLocalDelegatedKey -from dynamodb_encryption_sdk.identifiers import EncryptionKeyTypes, ItemAction, KeyEncodingType +from dynamodb_encryption_sdk.materials.raw import RawDecryptionMaterials from dynamodb_encryption_sdk.structures import AttributeActions -import functional_test_vector_generators +sys.path.append(os.path.join( + os.path.abspath(os.path.dirname(__file__)), + '..', + 'functional' +)) + +# Convenience imports +import functional_test_vector_generators # noqa: E402,I100 pylint: disable=import-error,wrong-import-position _ENCRYPTED_ITEM_VECTORS_DIR = os.path.join( os.path.abspath(os.path.dirname(__file__)), @@ -64,7 +67,7 @@ def _decode_item(item): def _build_plaintext_items(plaintext_file, version): - """""" + # pylint: disable=too-many-locals with open(plaintext_file) as f: plaintext_data = json.load(f) @@ -122,9 +125,9 @@ def _load_keys(keys_file): _KEY_TYPE = { - 'SYMMETRIC': EncryptionKeyTypes.SYMMETRIC, - 'PUBLIC': EncryptionKeyTypes.PUBLIC, - 'PRIVATE': EncryptionKeyTypes.PRIVATE + 'SYMMETRIC': EncryptionKeyType.SYMMETRIC, + 'PUBLIC': EncryptionKeyType.PUBLIC, + 'PRIVATE': EncryptionKeyType.PRIVATE } _KEY_ENCODING = { 'RAW': KeyEncodingType.RAW, @@ -169,9 +172,15 @@ def _build_wrapped_cmp(decrypt_key, verify_key): ) +def _build_aws_kms_cmp(decrypt_key, verify_key): + key_id = decrypt_key['keyId'] + return AwsKmsCryptographicMaterialsProvider(key_id=key_id) + + _CMP_TYPE_MAP = { 'STATIC': _build_static_cmp, - 'WRAPPED': _build_wrapped_cmp + 'WRAPPED': _build_wrapped_cmp, + 'AWSKMS': _build_aws_kms_cmp } @@ -208,6 +217,7 @@ def _expand_items(ciphertext_items, plaintext_items): def load_scenarios(): + # pylint: disable=too-many-locals with open(_SCENARIO_FILE) as f: scenarios = json.load(f) keys_file = _filename_from_uri(scenarios['keys']) diff --git a/test/acceptance/encrypted/__init__.py b/test/acceptance/encrypted/__init__.py new file mode 100644 index 00000000..2add15ef --- /dev/null +++ b/test/acceptance/encrypted/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Dummy stub to make linters work better.""" diff --git a/test/acceptance/test_a_encrypted_item.py b/test/acceptance/encrypted/test_item.py similarity index 86% rename from test/acceptance/test_a_encrypted_item.py rename to test/acceptance/encrypted/test_item.py index 273b91db..50fbbf98 100644 --- a/test/acceptance/test_a_encrypted_item.py +++ b/test/acceptance/encrypted/test_item.py @@ -10,15 +10,15 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +"""Acceptance tests for ``dynamodb_encryption_sdk.encrypted.item``.""" import pytest from dynamodb_encryption_sdk.encrypted import CryptoConfig from dynamodb_encryption_sdk.encrypted.item import decrypt_dynamodb_item from dynamodb_encryption_sdk.structures import EncryptionContext +from ..acceptance_test_utils import load_scenarios -from .acceptance_test_utils import load_scenarios - -pytestmark = [pytest.mark.acceptance, pytest.mark.local] +pytestmark = [pytest.mark.accept, pytest.mark.integ] @pytest.mark.parametrize( @@ -36,7 +36,8 @@ def test_item_encryptor( encryption_context = EncryptionContext( table_name=table_name, partition_key_name=table_index['partition'], - sort_key_name=table_index.get('sort', None) + sort_key_name=table_index.get('sort', None), + attributes=ciphertext_item ) crypto_config = CryptoConfig( materials_provider=materials_provider, diff --git a/test/cloudformation/tables.yaml b/test/cloudformation/tables.yaml new file mode 100644 index 00000000..f749599f --- /dev/null +++ b/test/cloudformation/tables.yaml @@ -0,0 +1,28 @@ +Description: Create DynamoDB tables for use in the DynamoDB Encryption Client for Python integration tests. +Resources: + TestTable: + Type: AWS::DynamoDB::Table + Properties: + AttributeDefinitions: + - + AttributeName: partition_attribute + AttributeType: S + - + AttributeName: sort_attribute + AttributeType: N + KeySchema: + - + AttributeName: partition_attribute + KeyType: HASH + - + AttributeName: sort_attribute + KeyType: RANGE + ProvisionedThroughput: + ReadCapacityUnits: 100 + WriteCapacityUnits: 100 +Outputs: + TestTableName: + Description: Name of test table. + Value: !Ref TestTable + Export: + Name: DynamoDB-Encryption-Client-Test-Table \ No newline at end of file diff --git a/test/functional/__init__.py b/test/functional/__init__.py index 1ccc7fa1..2add15ef 100644 --- a/test/functional/__init__.py +++ b/test/functional/__init__.py @@ -10,3 +10,4 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +"""Dummy stub to make linters work better.""" diff --git a/test/functional/delegated_keys/__init__.py b/test/functional/delegated_keys/__init__.py new file mode 100644 index 00000000..2add15ef --- /dev/null +++ b/test/functional/delegated_keys/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Dummy stub to make linters work better.""" diff --git a/test/functional/delegated_keys/test_jce.py b/test/functional/delegated_keys/test_jce.py new file mode 100644 index 00000000..2c88902e --- /dev/null +++ b/test/functional/delegated_keys/test_jce.py @@ -0,0 +1,48 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Functional test suite for ``dynamodb_encryption_sdk.delegated_keys.jce``.""" +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import serialization +import pytest + +from dynamodb_encryption_sdk.delegated_keys.jce import JceNameLocalDelegatedKey + +pytestmark = [pytest.mark.functional, pytest.mark.local] + + +def _find_aes_key_length(key): + return len(key) * 8 + + +def _find_rsa_key_length(key): + loaded_key = serialization.load_der_private_key(data=key, password=None, backend=default_backend()) + return loaded_key._key_size + + +@pytest.mark.parametrize('algorithm, requested_bits, expected_bits, length_finder', ( + ('AES', 256, 256, _find_aes_key_length), + ('AESWrap', 256, 256, _find_aes_key_length), + ('RSA', 4096, 4096, _find_rsa_key_length), + ('HmacSHA512', 256, 256, _find_aes_key_length), + ('HmacSHA256', 256, 256, _find_aes_key_length), + ('HmacSHA384', 256, 256, _find_aes_key_length), + ('HmacSHA224', 256, 256, _find_aes_key_length), + ('SHA512withRSA', 4096, 4096, _find_rsa_key_length), + ('SHA256withRSA', 4096, 4096, _find_rsa_key_length), + ('SHA384withRSA', 4096, 4096, _find_rsa_key_length), + ('SHA224withRSA', 4096, 4096, _find_rsa_key_length) +)) +def test_generate_correct_key_length(algorithm, requested_bits, expected_bits, length_finder): + test = JceNameLocalDelegatedKey.generate(algorithm, requested_bits) + + assert length_finder(test.key) == expected_bits diff --git a/test/functional/encrypted/__init__.py b/test/functional/encrypted/__init__.py new file mode 100644 index 00000000..2add15ef --- /dev/null +++ b/test/functional/encrypted/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Dummy stub to make linters work better.""" diff --git a/test/functional/encrypted/test_client.py b/test/functional/encrypted/test_client.py new file mode 100644 index 00000000..fca79487 --- /dev/null +++ b/test/functional/encrypted/test_client.py @@ -0,0 +1,115 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Functional tests for ``dynamodb_encryption_sdk.encrypted.client``.""" +import hypothesis +import pytest + +from ..functional_test_utils import ( + client_cycle_batch_items_check, client_cycle_batch_items_check_paginators, client_cycle_single_item_check, + set_parametrized_actions, set_parametrized_cmp, set_parametrized_item, + TEST_TABLE_NAME +) +from ..functional_test_utils import example_table # noqa pylint: disable=unused-import +from ..hypothesis_strategies import ddb_items, SLOW_SETTINGS, VERY_SLOW_SETTINGS + +pytestmark = [pytest.mark.functional, pytest.mark.local] + + +def pytest_generate_tests(metafunc): + set_parametrized_actions(metafunc) + set_parametrized_cmp(metafunc) + set_parametrized_item(metafunc) + + +def _client_cycle_single_item_check(materials_provider, initial_actions, initial_item): + return client_cycle_single_item_check( + materials_provider, + initial_actions, + initial_item, + TEST_TABLE_NAME, + 'us-west-2' + ) + + +def _client_cycle_batch_items_check(materials_provider, initial_actions, initial_item): + return client_cycle_batch_items_check( + materials_provider, + initial_actions, + initial_item, + TEST_TABLE_NAME, + 'us-west-2' + ) + + +def _client_cycle_batch_items_check_paginators(materials_provider, initial_actions, initial_item): + return client_cycle_batch_items_check_paginators( + materials_provider, + initial_actions, + initial_item, + TEST_TABLE_NAME, + 'us-west-2' + ) + + +def test_ephemeral_item_cycle(example_table, some_cmps, parametrized_actions, parametrized_item): + """Test a small number of curated CMPs against a small number of curated items.""" + _client_cycle_single_item_check(some_cmps, parametrized_actions, parametrized_item) + + +def test_ephemeral_batch_item_cycle(example_table, some_cmps, parametrized_actions, parametrized_item): + """Test a small number of curated CMPs against a small number of curated items.""" + _client_cycle_batch_items_check(some_cmps, parametrized_actions, parametrized_item) + + +def test_ephemeral_batch_item_cycle_paginators(example_table, some_cmps, parametrized_actions, parametrized_item): + """Test a small number of curated CMPs against a small number of curated items using paginators.""" + _client_cycle_batch_items_check_paginators(some_cmps, parametrized_actions, parametrized_item) + + +@pytest.mark.slow +def test_ephemeral_item_cycle_slow(example_table, all_the_cmps, parametrized_actions, parametrized_item): + """Test ALL THE CMPS against a small number of curated items.""" + _client_cycle_single_item_check(all_the_cmps, parametrized_actions, parametrized_item) + + +@pytest.mark.slow +def test_ephemeral_batch_item_cycle_slow(example_table, all_the_cmps, parametrized_actions, parametrized_item): + """Test ALL THE CMPS against a small number of curated items.""" + _client_cycle_batch_items_check(all_the_cmps, parametrized_actions, parametrized_item) + + +@pytest.mark.slow +@pytest.mark.hypothesis +@SLOW_SETTINGS +@hypothesis.given(item=ddb_items) +def test_ephemeral_item_cycle_hypothesis_slow(example_table, some_cmps, hypothesis_actions, item): + """Test a small number of curated CMPs against a large number of items.""" + _client_cycle_single_item_check(some_cmps, hypothesis_actions, item) + + +@pytest.mark.veryslow +@pytest.mark.hypothesis +@VERY_SLOW_SETTINGS +@hypothesis.given(item=ddb_items) +def test_ephemeral_item_cycle_hypothesis_veryslow(example_table, some_cmps, hypothesis_actions, item): + """Test a small number of curated CMPs against ALL THE ITEMS.""" + _client_cycle_single_item_check(some_cmps, hypothesis_actions, item) + + +@pytest.mark.nope +@pytest.mark.hypothesis +@VERY_SLOW_SETTINGS +@hypothesis.given(item=ddb_items) +def test_ephemeral_item_cycle_hypothesis_nope(example_table, all_the_cmps, hypothesis_actions, item): + """Test ALL THE CMPs against ALL THE ITEMS.""" + _client_cycle_single_item_check(all_the_cmps, hypothesis_actions, item) diff --git a/test/functional/test_f_encrypted_item.py b/test/functional/encrypted/test_item.py similarity index 57% rename from test/functional/test_f_encrypted_item.py rename to test/functional/encrypted/test_item.py index 0a39302a..856dc764 100644 --- a/test/functional/test_f_encrypted_item.py +++ b/test/functional/encrypted/test_item.py @@ -10,17 +10,19 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +"""Functional tests for ``dynamodb_encryption_sdk.encrypted.item``.""" import hypothesis import pytest -from .functional_test_utils import ( - build_static_jce_cmp, cycle_item_check, set_parametrized_actions, set_parametrized_cmp, set_parametrized_item -) -from .hypothesis_strategies import ddb_items, SLOW_SETTINGS, VERY_SLOW_SETTINGS from dynamodb_encryption_sdk.encrypted import CryptoConfig -from dynamodb_encryption_sdk.encrypted.item import decrypt_python_item -from dynamodb_encryption_sdk.exceptions import DecryptionError +from dynamodb_encryption_sdk.encrypted.item import decrypt_python_item, encrypt_python_item +from dynamodb_encryption_sdk.exceptions import DecryptionError, EncryptionError +from dynamodb_encryption_sdk.internal.identifiers import ReservedAttributes from dynamodb_encryption_sdk.structures import AttributeActions, EncryptionContext +from ..functional_test_utils import ( + build_static_jce_cmp, cycle_item_check, set_parametrized_actions, set_parametrized_cmp, set_parametrized_item +) +from ..hypothesis_strategies import ddb_items, SLOW_SETTINGS, VERY_SLOW_SETTINGS pytestmark = [pytest.mark.functional, pytest.mark.local] @@ -31,75 +33,77 @@ def pytest_generate_tests(metafunc): set_parametrized_item(metafunc) -def test_unsigned_item(): - crypto_config = CryptoConfig( +@pytest.fixture +def static_cmp_crypto_config(): + return CryptoConfig( materials_provider=build_static_jce_cmp('AES', 256, 'HmacSHA256', 256), encryption_context=EncryptionContext(), attribute_actions=AttributeActions() ) + + +def test_unsigned_item(static_cmp_crypto_config): item = {'test': 'no signature'} with pytest.raises(DecryptionError) as exc_info: - decrypt_python_item(item, crypto_config) + decrypt_python_item(item, static_cmp_crypto_config) exc_info.match(r'No signature attribute found in item') -def test_ephemeral_item_cycle(some_cmps, parametrized_actions, parametrized_item): - """Test a small number of curated CMPs against a small number of curated items.""" +@pytest.mark.parametrize('item', ( + {reserved.value: 'asdf'} + for reserved in ReservedAttributes +)) +def test_reserved_attributes_on_encrypt(static_cmp_crypto_config, item): + with pytest.raises(EncryptionError) as exc_info: + encrypt_python_item(item, static_cmp_crypto_config) + + exc_info.match(r'Reserved attribute name *') + + +def _item_cycle_check(materials_provider, attribute_actions, item): crypto_config = CryptoConfig( - materials_provider=some_cmps, + materials_provider=materials_provider, encryption_context=EncryptionContext(), - attribute_actions=parametrized_actions + attribute_actions=attribute_actions ) - cycle_item_check(parametrized_item, crypto_config) + cycle_item_check(item, crypto_config) + + +def test_ephemeral_item_cycle(some_cmps, parametrized_actions, parametrized_item): + """Test a small number of curated CMPs against a small number of curated items.""" + _item_cycle_check(some_cmps, parametrized_actions, parametrized_item) @pytest.mark.slow def test_ephemeral_item_cycle_slow(all_the_cmps, parametrized_actions, parametrized_item): """Test ALL THE CMPS against a small number of curated items.""" - crypto_config = CryptoConfig( - materials_provider=all_the_cmps, - encryption_context=EncryptionContext(), - attribute_actions=parametrized_actions - ) - cycle_item_check(parametrized_item, crypto_config) + _item_cycle_check(all_the_cmps, parametrized_actions, parametrized_item) @pytest.mark.slow +@pytest.mark.hypothesis @SLOW_SETTINGS @hypothesis.given(item=ddb_items) -def test_ephemeral_item_cycle_hypothesis_slow(some_cmps, parametrized_actions, item): +def test_ephemeral_item_cycle_hypothesis_slow(some_cmps, hypothesis_actions, item): """Test a small number of curated CMPs against a large number of items.""" - crypto_config = CryptoConfig( - materials_provider=some_cmps, - encryption_context=EncryptionContext(), - attribute_actions=parametrized_actions - ) - cycle_item_check(item, crypto_config) + _item_cycle_check(some_cmps, hypothesis_actions, item) @pytest.mark.veryslow +@pytest.mark.hypothesis @VERY_SLOW_SETTINGS @hypothesis.given(item=ddb_items) -def test_ephemeral_item_cycle_hypothesis_veryslow(some_cmps, parametrized_actions, item): +def test_ephemeral_item_cycle_hypothesis_veryslow(some_cmps, hypothesis_actions, item): """Test a small number of curated CMPs against ALL THE ITEMS.""" - crypto_config = CryptoConfig( - materials_provider=some_cmps, - encryption_context=EncryptionContext(), - attribute_actions=parametrized_actions - ) - cycle_item_check(item, crypto_config) + _item_cycle_check(some_cmps, hypothesis_actions, item) @pytest.mark.nope +@pytest.mark.hypothesis @VERY_SLOW_SETTINGS @hypothesis.given(item=ddb_items) -def test_ephemeral_item_cycle_hypothesis_nope(all_the_cmps, parametrized_actions, item): +def test_ephemeral_item_cycle_hypothesis_nope(all_the_cmps, hypothesis_actions, item): """Test ALL THE CMPs against ALL THE ITEMS.""" - crypto_config = CryptoConfig( - materials_provider=all_the_cmps, - encryption_context=EncryptionContext(), - attribute_actions=parametrized_actions - ) - cycle_item_check(item, crypto_config) + _item_cycle_check(all_the_cmps, hypothesis_actions, item) diff --git a/test/functional/encrypted/test_resource.py b/test/functional/encrypted/test_resource.py new file mode 100644 index 00000000..3b84e933 --- /dev/null +++ b/test/functional/encrypted/test_resource.py @@ -0,0 +1,43 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Functional tests for ``dynamodb_encryption_sdk.encrypted.resource``.""" +import pytest + +from ..functional_test_utils import ( + resource_cycle_batch_items_check, set_parametrized_actions, + set_parametrized_cmp, set_parametrized_item, TEST_TABLE_NAME +) +from ..functional_test_utils import example_table # noqa pylint: disable=unused-import + +pytestmark = [pytest.mark.functional, pytest.mark.local] + + +def pytest_generate_tests(metafunc): + set_parametrized_actions(metafunc) + set_parametrized_cmp(metafunc) + set_parametrized_item(metafunc) + + +def _resource_cycle_batch_items_check(materials_provider, initial_actions, initial_item): + resource_cycle_batch_items_check(materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, 'us-west-2') + + +def test_ephemeral_batch_item_cycle(example_table, some_cmps, parametrized_actions, parametrized_item): + """Test a small number of curated CMPs against a small number of curated items.""" + _resource_cycle_batch_items_check(some_cmps, parametrized_actions, parametrized_item) + + +@pytest.mark.slow +def test_ephemeral_batch_item_cycle_slow(example_table, all_the_cmps, parametrized_actions, parametrized_item): + """Test ALL THE CMPS against a small number of curated items.""" + _resource_cycle_batch_items_check(all_the_cmps, parametrized_actions, parametrized_item) diff --git a/test/functional/encrypted/test_table.py b/test/functional/encrypted/test_table.py new file mode 100644 index 00000000..e51a30ed --- /dev/null +++ b/test/functional/encrypted/test_table.py @@ -0,0 +1,83 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Functional tests for ``dynamodb_encryption_sdk.encrypted.table``.""" +import hypothesis +import pytest + +from ..functional_test_utils import ( + set_parametrized_actions, set_parametrized_cmp, set_parametrized_item, + table_cycle_batch_writer_check, table_cycle_check, TEST_TABLE_NAME +) +from ..functional_test_utils import example_table # noqa pylint: disable=unused-import +from ..hypothesis_strategies import ddb_items, SLOW_SETTINGS, VERY_SLOW_SETTINGS + +pytestmark = [pytest.mark.functional, pytest.mark.local] + + +def pytest_generate_tests(metafunc): + set_parametrized_actions(metafunc) + set_parametrized_cmp(metafunc) + set_parametrized_item(metafunc) + + +def _table_cycle_check(materials_provider, initial_actions, initial_item): + return table_cycle_check(materials_provider, initial_actions, initial_item, TEST_TABLE_NAME, 'us-west-2') + + +def test_ephemeral_item_cycle(example_table, some_cmps, parametrized_actions, parametrized_item): + """Test a small number of curated CMPs against a small number of curated items.""" + _table_cycle_check(some_cmps, parametrized_actions, parametrized_item) + + +def test_ephemeral_item_cycle_batch_writer(example_table, some_cmps, parametrized_actions, parametrized_item): + """Test a small number of curated CMPs against a small number of curated items.""" + table_cycle_batch_writer_check(some_cmps, parametrized_actions, parametrized_item, TEST_TABLE_NAME, 'us-west-2') + + +@pytest.mark.slow +def test_ephemeral_item_cycle_slow(example_table, all_the_cmps, parametrized_actions, parametrized_item): + """Test ALL THE CMPS against a small number of curated items.""" + _table_cycle_check(all_the_cmps, parametrized_actions, parametrized_item) + + +@pytest.mark.slow +def test_ephemeral_item_cycle_batch_writer_slow(example_table, all_the_cmps, parametrized_actions, parametrized_item): + """Test a small number of curated CMPs against a small number of curated items.""" + table_cycle_batch_writer_check(all_the_cmps, parametrized_actions, parametrized_item, TEST_TABLE_NAME, 'us-west-2') + + +@pytest.mark.slow +@pytest.mark.hypothesis +@SLOW_SETTINGS +@hypothesis.given(item=ddb_items) +def test_ephemeral_item_cycle_hypothesis_slow(example_table, some_cmps, hypothesis_actions, item): + """Test a small number of curated CMPs against a large number of items.""" + _table_cycle_check(some_cmps, hypothesis_actions, item) + + +@pytest.mark.veryslow +@pytest.mark.hypothesis +@VERY_SLOW_SETTINGS +@hypothesis.given(item=ddb_items) +def test_ephemeral_item_cycle_hypothesis_veryslow(example_table, some_cmps, hypothesis_actions, item): + """Test a small number of curated CMPs against ALL THE ITEMS.""" + _table_cycle_check(some_cmps, hypothesis_actions, item) + + +@pytest.mark.nope +@pytest.mark.hypothesis +@VERY_SLOW_SETTINGS +@hypothesis.given(item=ddb_items) +def test_ephemeral_item_cycle_hypothesis_nope(example_table, all_the_cmps, hypothesis_actions, item): + """Test ALL THE CMPs against ALL THE ITEMS.""" + _table_cycle_check(all_the_cmps, hypothesis_actions, item) diff --git a/test/functional/functional_test_utils.py b/test/functional/functional_test_utils.py index 4c4279e6..87b5946d 100644 --- a/test/functional/functional_test_utils.py +++ b/test/functional/functional_test_utils.py @@ -10,25 +10,124 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +"""Helper tools for use in tests.""" from __future__ import division -import copy + from collections import defaultdict +import copy from decimal import Decimal import itertools +import boto3 from boto3.dynamodb.types import Binary +from moto import mock_dynamodb2 import pytest from dynamodb_encryption_sdk.delegated_keys.jce import JceNameLocalDelegatedKey +from dynamodb_encryption_sdk.encrypted.client import EncryptedClient from dynamodb_encryption_sdk.encrypted.item import decrypt_python_item, encrypt_python_item -from dynamodb_encryption_sdk.identifiers import ItemAction +from dynamodb_encryption_sdk.encrypted.resource import EncryptedResource +from dynamodb_encryption_sdk.encrypted.table import EncryptedTable +from dynamodb_encryption_sdk.identifiers import CryptoAction from dynamodb_encryption_sdk.internal.identifiers import ReservedAttributes from dynamodb_encryption_sdk.material_providers.static import StaticCryptographicMaterialsProvider from dynamodb_encryption_sdk.material_providers.wrapped import WrappedCryptographicMaterialsProvider from dynamodb_encryption_sdk.materials.raw import RawDecryptionMaterials, RawEncryptionMaterials -from dynamodb_encryption_sdk.structures import AttributeActions, EncryptionContext +from dynamodb_encryption_sdk.structures import AttributeActions +from dynamodb_encryption_sdk.transform import ddb_to_dict, dict_to_ddb _DELEGATED_KEY_CACHE = defaultdict(lambda: defaultdict(dict)) +TEST_TABLE_NAME = 'my_table' +TEST_INDEX = { + 'partition_attribute': { + 'type': 'S', + 'value': 'test_value' + }, + 'sort_attribute': { + 'type': 'N', + 'value': Decimal('99.233') + } +} +TEST_KEY = {name: value['value'] for name, value in TEST_INDEX.items()} +TEST_BATCH_INDEXES = [ + { + 'partition_attribute': { + 'type': 'S', + 'value': 'test_value' + }, + 'sort_attribute': { + 'type': 'N', + 'value': Decimal('99.233') + } + }, + { + 'partition_attribute': { + 'type': 'S', + 'value': 'test_value' + }, + 'sort_attribute': { + 'type': 'N', + 'value': Decimal('92986745') + } + }, + { + 'partition_attribute': { + 'type': 'S', + 'value': 'test_value' + }, + 'sort_attribute': { + 'type': 'N', + 'value': Decimal('2231.0001') + } + }, + { + 'partition_attribute': { + 'type': 'S', + 'value': 'another_test_value' + }, + 'sort_attribute': { + 'type': 'N', + 'value': Decimal('732342') + } + } +] +TEST_BATCH_KEYS = [ + {name: value['value'] for name, value in key.items()} + for key in TEST_BATCH_INDEXES +] + + +@pytest.fixture +def example_table(): + mock_dynamodb2().start() + ddb = boto3.client('dynamodb', region_name='us-west-2') + ddb.create_table( + TableName=TEST_TABLE_NAME, + KeySchema=[ + { + 'AttributeName': 'partition_attribute', + 'KeyType': 'HASH' + }, + { + 'AttributeName': 'sort_attribute', + 'KeyType': 'RANGE' + } + ], + AttributeDefinitions=[ + { + 'AttributeName': name, + 'AttributeType': value['type'] + } + for name, value in TEST_INDEX.items() + ], + ProvisionedThroughput={ + 'ReadCapacityUnits': 100, + 'WriteCapacityUnits': 100 + } + ) + yield + ddb.delete_table(TableName=TEST_TABLE_NAME) + mock_dynamodb2().stop() def _get_from_cache(dk_class, algorithm, key_length): @@ -116,7 +215,7 @@ def _some_algorithm_pairs(): def _all_possible_cmps(algorithm_generator): """Generate all possible cryptographic materials providers based on the supplied generator.""" # The AES combinations do the same thing, but this makes sure that the AESWrap name works as expected. - yield _build_wrapped_jce_cmp('AESWrap', 32, 'HmacSHA256', 32) + yield _build_wrapped_jce_cmp('AESWrap', 256, 'HmacSHA256', 256) for builder_info, args in itertools.product(_cmp_builders.items(), algorithm_generator()): builder_type, builder_func = builder_info @@ -134,9 +233,6 @@ def _all_possible_cmps(algorithm_generator): sig_key_length=signing_key_length ) - if encryption_algorithm == 'AES': - encryption_key_length //= 8 - yield pytest.param( builder_func( encryption_algorithm, @@ -152,61 +248,66 @@ def set_parametrized_cmp(metafunc): """Set paramatrized values for cryptographic materials providers.""" for name, algorithm_generator in (('all_the_cmps', _all_algorithm_pairs), ('some_cmps', _some_algorithm_pairs)): if name in metafunc.fixturenames: - metafunc.parametrize(name, _all_possible_cmps(algorithm_generator), scope='module') + metafunc.parametrize(name, _all_possible_cmps(algorithm_generator)) + + +_ACTIONS = { + 'hypothesis_actions': ( + pytest.param(AttributeActions(default_action=CryptoAction.ENCRYPT_AND_SIGN), id='encrypt all'), + pytest.param(AttributeActions(default_action=CryptoAction.SIGN_ONLY), id='sign only all'), + pytest.param(AttributeActions(default_action=CryptoAction.DO_NOTHING), id='do nothing'), + ) +} +_ACTIONS['parametrized_actions'] = _ACTIONS['hypothesis_actions'] + ( + pytest.param( + AttributeActions( + default_action=CryptoAction.ENCRYPT_AND_SIGN, + attribute_actions={ + 'number_set': CryptoAction.SIGN_ONLY, + 'string_set': CryptoAction.SIGN_ONLY, + 'binary_set': CryptoAction.SIGN_ONLY + } + ), + id='sign sets, encrypt everything else' + ), + pytest.param( + AttributeActions( + default_action=CryptoAction.ENCRYPT_AND_SIGN, + attribute_actions={ + 'number_set': CryptoAction.DO_NOTHING, + 'string_set': CryptoAction.DO_NOTHING, + 'binary_set': CryptoAction.DO_NOTHING + } + ), + id='ignore sets, encrypt everything else' + ), + pytest.param( + AttributeActions( + default_action=CryptoAction.DO_NOTHING, + attribute_actions={'map': CryptoAction.ENCRYPT_AND_SIGN} + ), + id='encrypt map, ignore everything else' + ), + pytest.param( + AttributeActions( + default_action=CryptoAction.SIGN_ONLY, + attribute_actions={ + 'number_set': CryptoAction.DO_NOTHING, + 'string_set': CryptoAction.DO_NOTHING, + 'binary_set': CryptoAction.DO_NOTHING, + 'map': CryptoAction.ENCRYPT_AND_SIGN + } + ), + id='ignore sets, encrypt map, sign everything else' + ) +) def set_parametrized_actions(metafunc): - """Set parametrized values for attribute actions""" - if 'parametrized_actions' in metafunc.fixturenames: - metafunc.parametrize( - 'parametrized_actions', - ( - pytest.param(AttributeActions(default_action=ItemAction.ENCRYPT_AND_SIGN), id='encrypt all'), - pytest.param(AttributeActions(default_action=ItemAction.SIGN_ONLY), id='sign only all'), - pytest.param(AttributeActions(default_action=ItemAction.DO_NOTHING), id='do nothing'), - pytest.param( - AttributeActions( - default_action=ItemAction.ENCRYPT_AND_SIGN, - attribute_actions={ - 'number_set': ItemAction.SIGN_ONLY, - 'string_set': ItemAction.SIGN_ONLY, - 'binary_set': ItemAction.SIGN_ONLY - } - ), - id='sign sets, encrypt everything else' - ), - pytest.param( - AttributeActions( - default_action=ItemAction.ENCRYPT_AND_SIGN, - attribute_actions={ - 'number_set': ItemAction.DO_NOTHING, - 'string_set': ItemAction.DO_NOTHING, - 'binary_set': ItemAction.DO_NOTHING - } - ), - id='ignore sets, encrypt everything else' - ), - pytest.param( - AttributeActions( - default_action=ItemAction.DO_NOTHING, - attribute_actions={'map': ItemAction.ENCRYPT_AND_SIGN} - ), - id='encrypt map, ignore everything else' - ), - pytest.param( - AttributeActions( - default_action=ItemAction.SIGN_ONLY, - attribute_actions={ - 'number_set': ItemAction.DO_NOTHING, - 'string_set': ItemAction.DO_NOTHING, - 'binary_set': ItemAction.DO_NOTHING, - 'map': ItemAction.ENCRYPT_AND_SIGN - } - ), - id='ignore sets, encrypt map, sign everything else' - ) - ) - ) + """Set parametrized values for attribute actions.""" + for name, actions in _ACTIONS.items(): + if name in metafunc.fixturenames: + metafunc.parametrize(name, actions) def set_parametrized_item(metafunc): @@ -238,14 +339,11 @@ def diverse_item(): _reserved_attributes = set([attr.value for attr in ReservedAttributes]) -def cycle_item_check(plaintext_item, crypto_config): - """Common logic for cycled item (plaintext->encrypted->decrypted) tests: used by many test suites.""" - ciphertext_item = encrypt_python_item(plaintext_item, crypto_config) - +def check_encrypted_item(plaintext_item, ciphertext_item, attribute_actions): # Verify that all expected attributes are present ciphertext_attributes = set(ciphertext_item.keys()) plaintext_attributes = set(plaintext_item.keys()) - if crypto_config.attribute_actions.take_no_actions: + if attribute_actions.take_no_actions: assert ciphertext_attributes == plaintext_attributes else: assert ciphertext_attributes == plaintext_attributes.union(_reserved_attributes) @@ -256,12 +354,378 @@ def cycle_item_check(plaintext_item, crypto_config): continue # If the attribute should have been encrypted, verify that it is Binary and different from the original - if crypto_config.attribute_actions.action(name) is ItemAction.ENCRYPT_AND_SIGN: + if attribute_actions.action(name) is CryptoAction.ENCRYPT_AND_SIGN: assert isinstance(value, Binary) assert value != plaintext_item[name] # Otherwise, verify that it is the same as the original else: assert value == plaintext_item[name] + +def _matching_key(actual_item, expected): + expected_item = [ + i for i in expected + if i['partition_attribute'] == actual_item['partition_attribute'] and + i['sort_attribute'] == actual_item['sort_attribute'] + ] + assert len(expected_item) == 1 + return expected_item[0] + + +def _nop_transformer(item): + return item + + +def assert_equal_lists_of_items(actual, expected, transformer=_nop_transformer): + assert len(actual) == len(expected) + + for actual_item in actual: + expected_item = _matching_key(actual_item, expected) + assert transformer(actual_item) == transformer(expected_item) + + +def check_many_encrypted_items(actual, expected, attribute_actions, transformer=_nop_transformer): + assert len(actual) == len(expected) + + for actual_item in actual: + expected_item = _matching_key(actual_item, expected) + check_encrypted_item( + plaintext_item=transformer(expected_item), + ciphertext_item=transformer(actual_item), + attribute_actions=attribute_actions + ) + + +def _generate_items(initial_item, write_transformer): + items = [] + for key in TEST_BATCH_KEYS: + _item = initial_item.copy() + _item.update(key) + items.append(write_transformer(_item)) + return items + + +def _cleanup_items(encrypted, write_transformer, table_name=TEST_TABLE_NAME): + ddb_keys = [write_transformer(key) for key in TEST_BATCH_KEYS] + _delete_result = encrypted.batch_write_item( # noqa + RequestItems={ + table_name: [ + {'DeleteRequest': {'Key': _key}} + for _key in ddb_keys + ] + } + ) + + +def cycle_batch_item_check( + raw, + encrypted, + initial_actions, + initial_item, + write_transformer=_nop_transformer, + read_transformer=_nop_transformer, + table_name=TEST_TABLE_NAME, + delete_items=True +): + """Check that cycling (plaintext->encrypted->decrypted) item batch has the expected results.""" + check_attribute_actions = initial_actions.copy() + check_attribute_actions.set_index_keys(*list(TEST_KEY.keys())) + items = _generate_items(initial_item, write_transformer) + + _put_result = encrypted.batch_write_item( # noqa + RequestItems={ + table_name: [ + {'PutRequest': {'Item': _item}} + for _item in items + ] + } + ) + + ddb_keys = [write_transformer(key) for key in TEST_BATCH_KEYS] + encrypted_result = raw.batch_get_item( + RequestItems={ + table_name: { + 'Keys': ddb_keys + } + } + ) + check_many_encrypted_items( + actual=encrypted_result['Responses'][table_name], + expected=items, + attribute_actions=check_attribute_actions, + transformer=read_transformer + ) + + decrypted_result = encrypted.batch_get_item( + RequestItems={ + table_name: { + 'Keys': ddb_keys + } + } + ) + assert_equal_lists_of_items( + actual=decrypted_result['Responses'][table_name], + expected=items, + transformer=read_transformer + ) + + if delete_items: + _cleanup_items(encrypted, write_transformer, table_name) + + del check_attribute_actions + del items + + +def cycle_batch_writer_check(raw_table, encrypted_table, initial_actions, initial_item): + """Check that cycling (plaintext->encrypted->decrypted) items with the Table batch writer + has the expected results. + """ + check_attribute_actions = initial_actions.copy() + check_attribute_actions.set_index_keys(*list(TEST_KEY.keys())) + items = _generate_items(initial_item, _nop_transformer) + + with encrypted_table.batch_writer() as writer: + for item in items: + writer.put_item(item) + + ddb_keys = [key for key in TEST_BATCH_KEYS] + encrypted_items = [ + raw_table.get_item(Key=key)['Item'] + for key in ddb_keys + ] + check_many_encrypted_items( + actual=encrypted_items, + expected=items, + attribute_actions=check_attribute_actions, + transformer=_nop_transformer + ) + + decrypted_result = [ + encrypted_table.get_item(Key=key)['Item'] + for key in ddb_keys + ] + assert_equal_lists_of_items( + actual=decrypted_result, + expected=items, + transformer=_nop_transformer + ) + + with encrypted_table.batch_writer() as writer: + for key in ddb_keys: + writer.delete_item(key) + + del check_attribute_actions + del items + + +def cycle_item_check(plaintext_item, crypto_config): + """Check that cycling (plaintext->encrypted->decrypted) an item has the expected results.""" + ciphertext_item = encrypt_python_item(plaintext_item, crypto_config) + + check_encrypted_item(plaintext_item, ciphertext_item, crypto_config.attribute_actions) + cycled_item = decrypt_python_item(ciphertext_item, crypto_config) + assert cycled_item == plaintext_item + del ciphertext_item + del cycled_item + + +def table_cycle_check(materials_provider, initial_actions, initial_item, table_name, region_name=None): + check_attribute_actions = initial_actions.copy() + check_attribute_actions.set_index_keys(*list(TEST_KEY.keys())) + item = initial_item.copy() + item.update(TEST_KEY) + + kwargs = {} + if region_name is not None: + kwargs['region_name'] = region_name + table = boto3.resource('dynamodb', **kwargs).Table(table_name) + e_table = EncryptedTable( + table=table, + materials_provider=materials_provider, + attribute_actions=initial_actions, + ) + + _put_result = e_table.put_item(Item=item) # noqa + + encrypted_result = table.get_item(Key=TEST_KEY) + check_encrypted_item(item, encrypted_result['Item'], check_attribute_actions) + + decrypted_result = e_table.get_item(Key=TEST_KEY) + assert decrypted_result['Item'] == item + + e_table.delete_item(Key=TEST_KEY) + del item + del check_attribute_actions + + +def table_cycle_batch_writer_check(materials_provider, initial_actions, initial_item, table_name, region_name=None): + kwargs = {} + if region_name is not None: + kwargs['region_name'] = region_name + table = boto3.resource('dynamodb', **kwargs).Table(table_name) + e_table = EncryptedTable( + table=table, + materials_provider=materials_provider, + attribute_actions=initial_actions, + ) + + cycle_batch_writer_check(table, e_table, initial_actions, initial_item) + + +def resource_cycle_batch_items_check(materials_provider, initial_actions, initial_item, table_name, region_name=None): + kwargs = {} + if region_name is not None: + kwargs['region_name'] = region_name + resource = boto3.resource('dynamodb', **kwargs) + e_resource = EncryptedResource( + resource=resource, + materials_provider=materials_provider, + attribute_actions=initial_actions + ) + + cycle_batch_item_check( + raw=resource, + encrypted=e_resource, + initial_actions=initial_actions, + initial_item=initial_item, + table_name=table_name + ) + + raw_scan_result = resource.Table(table_name).scan() + e_scan_result = e_resource.Table(table_name).scan() + assert not raw_scan_result['Items'] + assert not e_scan_result['Items'] + + +def client_cycle_single_item_check(materials_provider, initial_actions, initial_item, table_name, region_name=None): + check_attribute_actions = initial_actions.copy() + check_attribute_actions.set_index_keys(*list(TEST_KEY.keys())) + item = initial_item.copy() + item.update(TEST_KEY) + ddb_item = dict_to_ddb(item) + ddb_key = dict_to_ddb(TEST_KEY) + + kwargs = {} + if region_name is not None: + kwargs['region_name'] = region_name + client = boto3.client('dynamodb', **kwargs) + e_client = EncryptedClient( + client=client, + materials_provider=materials_provider, + attribute_actions=initial_actions + ) + + _put_result = e_client.put_item( # noqa + TableName=table_name, + Item=ddb_item + ) + + encrypted_result = client.get_item( + TableName=table_name, + Key=ddb_key + ) + check_encrypted_item(item, ddb_to_dict(encrypted_result['Item']), check_attribute_actions) + + decrypted_result = e_client.get_item( + TableName=table_name, + Key=ddb_key + ) + assert ddb_to_dict(decrypted_result['Item']) == item + + e_client.delete_item( + TableName=table_name, + Key=ddb_key + ) + del item + del check_attribute_actions + + +def client_cycle_batch_items_check(materials_provider, initial_actions, initial_item, table_name, region_name=None): + kwargs = {} + if region_name is not None: + kwargs['region_name'] = region_name + client = boto3.client('dynamodb', **kwargs) + e_client = EncryptedClient( + client=client, + materials_provider=materials_provider, + attribute_actions=initial_actions + ) + + cycle_batch_item_check( + raw=client, + encrypted=e_client, + initial_actions=initial_actions, + initial_item=initial_item, + write_transformer=dict_to_ddb, + read_transformer=ddb_to_dict, + table_name=table_name + ) + + raw_scan_result = client.scan(TableName=table_name) + e_scan_result = e_client.scan(TableName=table_name) + assert not raw_scan_result['Items'] + assert not e_scan_result['Items'] + + +def client_cycle_batch_items_check_paginators( + materials_provider, + initial_actions, + initial_item, + table_name, + region_name=None +): + kwargs = {} + if region_name is not None: + kwargs['region_name'] = region_name + client = boto3.client('dynamodb', **kwargs) + e_client = EncryptedClient( + client=client, + materials_provider=materials_provider, + attribute_actions=initial_actions + ) + + cycle_batch_item_check( + raw=client, + encrypted=e_client, + initial_actions=initial_actions, + initial_item=initial_item, + write_transformer=dict_to_ddb, + read_transformer=ddb_to_dict, + table_name=table_name, + delete_items=False + ) + + encrypted_items = [] + raw_paginator = client.get_paginator('scan') + for page in raw_paginator.paginate(TableName=table_name): + encrypted_items.extend(page['Items']) + + decrypted_items = [] + encrypted_paginator = e_client.get_paginator('scan') + for page in encrypted_paginator.paginate(TableName=table_name): + decrypted_items.extend(page['Items']) + + print(encrypted_items) + print(decrypted_items) + + check_attribute_actions = initial_actions.copy() + check_attribute_actions.set_index_keys(*list(TEST_KEY.keys())) + check_many_encrypted_items( + actual=encrypted_items, + expected=decrypted_items, + attribute_actions=check_attribute_actions, + transformer=ddb_to_dict + ) + + _cleanup_items( + encrypted=e_client, + write_transformer=dict_to_ddb, + table_name=table_name + ) + + raw_scan_result = client.scan(TableName=table_name) + e_scan_result = e_client.scan(TableName=table_name) + assert not raw_scan_result['Items'] + assert not e_scan_result['Items'] diff --git a/test/functional/functional_test_vector_generators.py b/test/functional/functional_test_vector_generators.py index a425d2b2..88a7fd4e 100644 --- a/test/functional/functional_test_vector_generators.py +++ b/test/functional/functional_test_vector_generators.py @@ -10,15 +10,16 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Helper tools for attribute de/serialization functional tests.""" +"""Helper tools for collecting test vectors for use in functional tests.""" import base64 -from decimal import Decimal import codecs +from decimal import Decimal import json import os from boto3.dynamodb.types import Binary -from dynamodb_encryption_sdk.identifiers import ItemAction + +from dynamodb_encryption_sdk.identifiers import CryptoAction from dynamodb_encryption_sdk.structures import AttributeActions _ATTRIBUTE_TEST_VECTOR_FILE_TEMPLATE = os.path.join( @@ -41,12 +42,12 @@ ) -def decode_value(value, transform_binary=False): +def decode_value(value, transform_binary=False): # noqa: C901 def _decode_string(_value): return _value def _decode_number(_value): - return str(Decimal(_value).normalize()) + return '{0:f}'.format(Decimal(_value)) def _decode_binary(_value): raw = base64.b64decode(_value) @@ -132,9 +133,9 @@ def material_description_test_vectors(): ACTION_MAP = { - 'encrypt': ItemAction.ENCRYPT_AND_SIGN, - 'sign': ItemAction.SIGN_ONLY, - 'nothing': ItemAction.DO_NOTHING + 'encrypt': CryptoAction.ENCRYPT_AND_SIGN, + 'sign': CryptoAction.SIGN_ONLY, + 'nothing': CryptoAction.DO_NOTHING } @@ -148,7 +149,7 @@ def string_to_sign_test_vectors(): } bare_actions = {key: ACTION_MAP[value['action']] for key, value in vector['item'].items()} attribute_actions = AttributeActions( - default_action=ItemAction.DO_NOTHING, + default_action=CryptoAction.DO_NOTHING, attribute_actions=bare_actions ) yield ( diff --git a/test/functional/hypothesis_strategies.py b/test/functional/hypothesis_strategies.py index e4f9038d..27725de2 100644 --- a/test/functional/hypothesis_strategies.py +++ b/test/functional/hypothesis_strategies.py @@ -11,13 +11,15 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Helper resources for functional tests.""" +"""Hypothesis strategies for use in tests.""" from collections import namedtuple from decimal import Decimal from boto3.dynamodb.types import Binary, DYNAMODB_CONTEXT import hypothesis -from hypothesis.strategies import binary, booleans, dictionaries, deferred, fractions, just, lists, none, sets, text +from hypothesis.strategies import ( + binary, booleans, characters, deferred, dictionaries, fractions, just, lists, none, sets, text +) SLOW_SETTINGS = hypothesis.settings( suppress_health_check=( @@ -49,23 +51,39 @@ ) -ddb_string = text(min_size=1, max_size=MAX_ITEM_BYTES) +def _valid_ddb_number(value): + try: + DYNAMODB_CONTEXT.create_decimal(float(value)) + except Exception: + return False + else: + return True + + +ddb_string = text( + min_size=1, + max_size=MAX_ITEM_BYTES, + alphabet=characters( + blacklist_categories=('Cs',), + blacklist_characters=('"', "'") # Quotes break moto :( + ) +) ddb_string_set = sets(ddb_string, min_size=1) def _ddb_fraction_to_decimal(val): - """hypothesis does not support providing a custom Context, so working around that""" - return DYNAMODB_CONTEXT.create_decimal(Decimal(val.numerator) / Decimal(val.denominator)) + """Hypothesis does not support providing a custom Context, so working around that.""" + return Decimal(val.numerator) / Decimal(val.denominator) _ddb_positive_numbers = fractions( min_value=POSITIVE_NUMBER_RANGE.min, max_value=POSITIVE_NUMBER_RANGE.max -).map(_ddb_fraction_to_decimal) +).map(_ddb_fraction_to_decimal).filter(_valid_ddb_number) _ddb_negative_numbers = fractions( min_value=NEGATIVE_NUMBER_RANGE.min, max_value=NEGATIVE_NUMBER_RANGE.max -).map(_ddb_fraction_to_decimal) +).map(_ddb_fraction_to_decimal).filter(_valid_ddb_number) ddb_number = _ddb_negative_numbers | just(Decimal('0')) | _ddb_positive_numbers ddb_number_set = sets(ddb_number, min_size=1) @@ -77,34 +95,42 @@ def _ddb_fraction_to_decimal(val): ddb_null = none() ddb_scalar_types = ( - ddb_string - | ddb_number - | ddb_binary - | ddb_boolean - | ddb_null + ddb_string | + ddb_number | + ddb_binary | + ddb_boolean | + ddb_null ) ddb_set_types = ( - ddb_string_set - | ddb_number_set - | ddb_binary_set + ddb_string_set | + ddb_number_set | + ddb_binary_set +) +ddb_attribute_names = text( + min_size=1, + max_size=255, + alphabet=characters( + blacklist_categories=('Cs',), + blacklist_characters=('"', "'") # Quotes break moto :( + ) ) # TODO: List and Map types have a max depth of 32 ddb_map_type = deferred(lambda: dictionaries( - keys=text(), + keys=ddb_attribute_names, values=( - ddb_scalar_types - | ddb_set_types - | ddb_list_type - | ddb_map_type + ddb_scalar_types | + ddb_set_types | + ddb_list_type | + ddb_map_type ), min_size=1 )) ddb_list_type = deferred(lambda: lists( - ddb_scalar_types - | ddb_set_types - | ddb_list_type - | ddb_map_type, + ddb_scalar_types | + ddb_set_types | + ddb_list_type | + ddb_map_type, min_size=1 )) ddb_document_types = ddb_map_type | ddb_list_type @@ -112,7 +138,7 @@ def _ddb_fraction_to_decimal(val): ddb_attribute_values = ddb_scalar_types | ddb_set_types | ddb_list_type ddb_items = dictionaries( - keys=text(min_size=1, max_size=255), + keys=ddb_attribute_names, values=ddb_scalar_types | ddb_set_types | ddb_list_type ) diff --git a/test/functional/internal/__init__.py b/test/functional/internal/__init__.py new file mode 100644 index 00000000..2add15ef --- /dev/null +++ b/test/functional/internal/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Dummy stub to make linters work better.""" diff --git a/test/functional/internal/crypto/__init__.py b/test/functional/internal/crypto/__init__.py new file mode 100644 index 00000000..2add15ef --- /dev/null +++ b/test/functional/internal/crypto/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Dummy stub to make linters work better.""" diff --git a/test/functional/test_f_authentication_string_to_sign.py b/test/functional/internal/crypto/test_authentication.py similarity index 93% rename from test/functional/test_f_authentication_string_to_sign.py rename to test/functional/internal/crypto/test_authentication.py index 545919a5..c17a078c 100644 --- a/test/functional/test_f_authentication_string_to_sign.py +++ b/test/functional/internal/crypto/test_authentication.py @@ -13,8 +13,8 @@ """Functional tests for material description de/serialization.""" import pytest -from .functional_test_vector_generators import string_to_sign_test_vectors from dynamodb_encryption_sdk.internal.crypto.authentication import _string_to_sign +from ...functional_test_vector_generators import string_to_sign_test_vectors pytestmark = [pytest.mark.functional, pytest.mark.local] diff --git a/test/functional/internal/formatting/__init__.py b/test/functional/internal/formatting/__init__.py new file mode 100644 index 00000000..2add15ef --- /dev/null +++ b/test/functional/internal/formatting/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Dummy stub to make linters work better.""" diff --git a/test/functional/test_f_formatting_attribute_serialization.py b/test/functional/internal/formatting/test_attribute.py similarity index 92% rename from test/functional/test_f_formatting_attribute_serialization.py rename to test/functional/internal/formatting/test_attribute.py index d128758b..6b726f46 100644 --- a/test/functional/test_f_formatting_attribute_serialization.py +++ b/test/functional/internal/formatting/test_attribute.py @@ -18,9 +18,9 @@ from dynamodb_encryption_sdk.exceptions import DeserializationError, SerializationError from dynamodb_encryption_sdk.internal.formatting.deserialize.attribute import deserialize_attribute from dynamodb_encryption_sdk.internal.formatting.serialize.attribute import serialize_attribute -from dynamodb_encryption_sdk.internal.formatting.transform import ddb_to_dict, dict_to_ddb -from .functional_test_vector_generators import attribute_test_vectors -from .hypothesis_strategies import ddb_attribute_values, ddb_items, SLOW_SETTINGS, VERY_SLOW_SETTINGS +from dynamodb_encryption_sdk.transform import ddb_to_dict, dict_to_ddb +from ...functional_test_vector_generators import attribute_test_vectors +from ...hypothesis_strategies import ddb_attribute_values, ddb_items, SLOW_SETTINGS, VERY_SLOW_SETTINGS pytestmark = [pytest.mark.functional, pytest.mark.local] @@ -73,6 +73,7 @@ def _serialize_deserialize_cycle(attribute): @pytest.mark.slow +@pytest.mark.hypothesis @SLOW_SETTINGS @hypothesis.given(ddb_attribute_values) def test_serialize_deserialize_attribute_slow(attribute): @@ -80,6 +81,7 @@ def test_serialize_deserialize_attribute_slow(attribute): @pytest.mark.veryslow +@pytest.mark.hypothesis @VERY_SLOW_SETTINGS @hypothesis.given(ddb_attribute_values) def test_serialize_deserialize_attribute_vslow(attribute): @@ -93,6 +95,7 @@ def _ddb_dict_ddb_transform_cycle(item): @pytest.mark.slow +@pytest.mark.hypothesis @SLOW_SETTINGS @hypothesis.given(ddb_items) def test_dict_to_ddb_and_back_slow(item): @@ -100,6 +103,7 @@ def test_dict_to_ddb_and_back_slow(item): @pytest.mark.veryslow +@pytest.mark.hypothesis @VERY_SLOW_SETTINGS @hypothesis.given(ddb_items) def test_dict_to_ddb_and_back_vslow(item): diff --git a/test/functional/test_f_formatting_material_description_serialization.py b/test/functional/internal/formatting/test_material_description.py similarity index 76% rename from test/functional/test_f_formatting_material_description_serialization.py rename to test/functional/internal/formatting/test_material_description.py index 37d66b28..1fbf219d 100644 --- a/test/functional/test_f_formatting_material_description_serialization.py +++ b/test/functional/internal/formatting/test_material_description.py @@ -14,12 +14,12 @@ import hypothesis import pytest -from .functional_test_vector_generators import material_description_test_vectors -from .hypothesis_strategies import material_descriptions, SLOW_SETTINGS, VERY_SLOW_SETTINGS -from dynamodb_encryption_sdk.exceptions import InvalidMaterialsetError, InvalidMaterialsetVersionError +from dynamodb_encryption_sdk.exceptions import InvalidMaterialDescriptionError, InvalidMaterialDescriptionVersionError from dynamodb_encryption_sdk.internal.formatting.material_description import ( deserialize as deserialize_material_description, serialize as serialize_material_description ) +from ...functional_test_vector_generators import material_description_test_vectors +from ...hypothesis_strategies import material_descriptions, SLOW_SETTINGS, VERY_SLOW_SETTINGS pytestmark = [pytest.mark.functional, pytest.mark.local] @@ -31,8 +31,8 @@ def test_serialize_material_description(material_description, serialized): @pytest.mark.parametrize('data, expected_type, expected_message', ( - ({'test': 5}, InvalidMaterialsetError, 'Invalid name or value in material description: *'), - ({5: 'test'}, InvalidMaterialsetError, 'Invalid name or value in material description: *'), + ({'test': 5}, InvalidMaterialDescriptionError, 'Invalid name or value in material description: *'), + ({5: 'test'}, InvalidMaterialDescriptionError, 'Invalid name or value in material description: *'), )) def test_serialize_material_description_errors(data, expected_type, expected_message): with pytest.raises(expected_type) as exc_info: @@ -49,17 +49,17 @@ def test_deserialize_material_description(material_description, serialized): @pytest.mark.parametrize('data, expected_type, expected_message', ( # Invalid version - ({'B': b'\x00\x00\x00\x01'}, InvalidMaterialsetVersionError, r'Invalid material description version: *'), + ({'B': b'\x00\x00\x00\x01'}, InvalidMaterialDescriptionVersionError, r'Invalid material description version: *'), # Malformed version - ({'B': b'\x00\x00\x00'}, InvalidMaterialsetError, r'Malformed material description version'), + ({'B': b'\x00\x00\x00'}, InvalidMaterialDescriptionError, r'Malformed material description version'), # Invalid attribute type - ({'S': 'not bytes'}, InvalidMaterialsetError, r'Invalid material description'), + ({'S': 'not bytes'}, InvalidMaterialDescriptionError, r'Invalid material description'), # Invalid data: not a DDB attribute - (b'bare bytes', InvalidMaterialsetError, r'Invalid material description'), + (b'bare bytes', InvalidMaterialDescriptionError, r'Invalid material description'), # Partial entry ( {'B': b'\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x01A\x00\x00\x00\x01'}, - InvalidMaterialsetError, + InvalidMaterialDescriptionError, r'Invalid material description' ) )) @@ -77,6 +77,7 @@ def _serialize_deserialize_cycle(material_description): @pytest.mark.slow +@pytest.mark.hypothesis @SLOW_SETTINGS @hypothesis.given(material_descriptions) def test_serialize_deserialize_material_description_slow(material_description): @@ -84,6 +85,7 @@ def test_serialize_deserialize_material_description_slow(material_description): @pytest.mark.veryslow +@pytest.mark.hypothesis @VERY_SLOW_SETTINGS @hypothesis.given(material_descriptions) def test_serialize_deserialize_material_description_vslow(material_description): diff --git a/test/functional/test_f_str_ops.py b/test/functional/internal/test_str_ops.py similarity index 95% rename from test/functional/test_f_str_ops.py rename to test/functional/internal/test_str_ops.py index 08782367..785c5b93 100644 --- a/test/functional/test_f_str_ops.py +++ b/test/functional/internal/test_str_ops.py @@ -11,7 +11,7 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Test suite for dynamodb_encryption_sdk.internal.str_ops""" +"""Test suite for ``dynamodb_encryption_sdk.internal.str_ops``.""" import codecs import pytest diff --git a/test/functional/test_f_identifiers.py b/test/functional/test_f_identifiers.py deleted file mode 100644 index 61986780..00000000 --- a/test/functional/test_f_identifiers.py +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. -import operator - -import pytest - -from dynamodb_encryption_sdk.identifiers import ItemAction - -pytestmark = [pytest.mark.functional, pytest.mark.local] - - -@pytest.mark.parametrize('left, right, expected', ( - (ItemAction.ENCRYPT_AND_SIGN, ItemAction.ENCRYPT_AND_SIGN, ItemAction.ENCRYPT_AND_SIGN), - (ItemAction.ENCRYPT_AND_SIGN, ItemAction.SIGN_ONLY, ItemAction.ENCRYPT_AND_SIGN), - (ItemAction.ENCRYPT_AND_SIGN, ItemAction.DO_NOTHING, ItemAction.ENCRYPT_AND_SIGN), - (ItemAction.SIGN_ONLY, ItemAction.ENCRYPT_AND_SIGN, ItemAction.ENCRYPT_AND_SIGN), - (ItemAction.SIGN_ONLY, ItemAction.SIGN_ONLY, ItemAction.SIGN_ONLY), - (ItemAction.SIGN_ONLY, ItemAction.DO_NOTHING, ItemAction.SIGN_ONLY), - (ItemAction.DO_NOTHING, ItemAction.ENCRYPT_AND_SIGN, ItemAction.ENCRYPT_AND_SIGN), - (ItemAction.DO_NOTHING, ItemAction.SIGN_ONLY, ItemAction.SIGN_ONLY), - (ItemAction.DO_NOTHING, ItemAction.DO_NOTHING, ItemAction.DO_NOTHING), -)) -def test_item_action_max(left, right, expected): - assert max(left, right) == expected - - -@pytest.mark.parametrize('left, right, expected', ( - (ItemAction.ENCRYPT_AND_SIGN, ItemAction.ENCRYPT_AND_SIGN, ItemAction.ENCRYPT_AND_SIGN), - (ItemAction.ENCRYPT_AND_SIGN, ItemAction.SIGN_ONLY, ItemAction.SIGN_ONLY), - (ItemAction.ENCRYPT_AND_SIGN, ItemAction.DO_NOTHING, ItemAction.DO_NOTHING), - (ItemAction.SIGN_ONLY, ItemAction.ENCRYPT_AND_SIGN, ItemAction.SIGN_ONLY), - (ItemAction.SIGN_ONLY, ItemAction.SIGN_ONLY, ItemAction.SIGN_ONLY), - (ItemAction.SIGN_ONLY, ItemAction.DO_NOTHING, ItemAction.DO_NOTHING), - (ItemAction.DO_NOTHING, ItemAction.ENCRYPT_AND_SIGN, ItemAction.DO_NOTHING), - (ItemAction.DO_NOTHING, ItemAction.SIGN_ONLY, ItemAction.DO_NOTHING), - (ItemAction.DO_NOTHING, ItemAction.DO_NOTHING, ItemAction.DO_NOTHING), -)) -def test_item_action_min(left, right, expected): - assert min(left, right) == expected - - -@pytest.mark.parametrize('left, right, expected_comparison', ( - (ItemAction.ENCRYPT_AND_SIGN, ItemAction.ENCRYPT_AND_SIGN, operator.eq), - (ItemAction.ENCRYPT_AND_SIGN, ItemAction.SIGN_ONLY, operator.ne), - (ItemAction.ENCRYPT_AND_SIGN, ItemAction.SIGN_ONLY, operator.gt), - (ItemAction.ENCRYPT_AND_SIGN, ItemAction.DO_NOTHING, operator.gt), - (ItemAction.SIGN_ONLY, ItemAction.ENCRYPT_AND_SIGN, operator.lt), - (ItemAction.SIGN_ONLY, ItemAction.SIGN_ONLY, operator.eq), - (ItemAction.SIGN_ONLY, ItemAction.DO_NOTHING, operator.ne), - (ItemAction.SIGN_ONLY, ItemAction.DO_NOTHING, operator.gt), - (ItemAction.DO_NOTHING, ItemAction.ENCRYPT_AND_SIGN, operator.lt), - (ItemAction.DO_NOTHING, ItemAction.SIGN_ONLY, operator.lt), - (ItemAction.DO_NOTHING, ItemAction.DO_NOTHING, operator.eq), - (ItemAction.DO_NOTHING, ItemAction.ENCRYPT_AND_SIGN, operator.ne) -)) -def test_item_action_comp(left, right, expected_comparison): - assert expected_comparison(left, right) diff --git a/test/functional/test_identifiers.py b/test/functional/test_identifiers.py new file mode 100644 index 00000000..7baa7a59 --- /dev/null +++ b/test/functional/test_identifiers.py @@ -0,0 +1,68 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Functional tests for ``dynamodb_encryption_sdk.identifiers``.""" +import operator + +import pytest + +from dynamodb_encryption_sdk.identifiers import CryptoAction + +pytestmark = [pytest.mark.functional, pytest.mark.local] + + +@pytest.mark.parametrize('left, right, expected', ( + (CryptoAction.ENCRYPT_AND_SIGN, CryptoAction.ENCRYPT_AND_SIGN, CryptoAction.ENCRYPT_AND_SIGN), + (CryptoAction.ENCRYPT_AND_SIGN, CryptoAction.SIGN_ONLY, CryptoAction.ENCRYPT_AND_SIGN), + (CryptoAction.ENCRYPT_AND_SIGN, CryptoAction.DO_NOTHING, CryptoAction.ENCRYPT_AND_SIGN), + (CryptoAction.SIGN_ONLY, CryptoAction.ENCRYPT_AND_SIGN, CryptoAction.ENCRYPT_AND_SIGN), + (CryptoAction.SIGN_ONLY, CryptoAction.SIGN_ONLY, CryptoAction.SIGN_ONLY), + (CryptoAction.SIGN_ONLY, CryptoAction.DO_NOTHING, CryptoAction.SIGN_ONLY), + (CryptoAction.DO_NOTHING, CryptoAction.ENCRYPT_AND_SIGN, CryptoAction.ENCRYPT_AND_SIGN), + (CryptoAction.DO_NOTHING, CryptoAction.SIGN_ONLY, CryptoAction.SIGN_ONLY), + (CryptoAction.DO_NOTHING, CryptoAction.DO_NOTHING, CryptoAction.DO_NOTHING), +)) +def test_item_action_max(left, right, expected): + assert max(left, right) == expected + + +@pytest.mark.parametrize('left, right, expected', ( + (CryptoAction.ENCRYPT_AND_SIGN, CryptoAction.ENCRYPT_AND_SIGN, CryptoAction.ENCRYPT_AND_SIGN), + (CryptoAction.ENCRYPT_AND_SIGN, CryptoAction.SIGN_ONLY, CryptoAction.SIGN_ONLY), + (CryptoAction.ENCRYPT_AND_SIGN, CryptoAction.DO_NOTHING, CryptoAction.DO_NOTHING), + (CryptoAction.SIGN_ONLY, CryptoAction.ENCRYPT_AND_SIGN, CryptoAction.SIGN_ONLY), + (CryptoAction.SIGN_ONLY, CryptoAction.SIGN_ONLY, CryptoAction.SIGN_ONLY), + (CryptoAction.SIGN_ONLY, CryptoAction.DO_NOTHING, CryptoAction.DO_NOTHING), + (CryptoAction.DO_NOTHING, CryptoAction.ENCRYPT_AND_SIGN, CryptoAction.DO_NOTHING), + (CryptoAction.DO_NOTHING, CryptoAction.SIGN_ONLY, CryptoAction.DO_NOTHING), + (CryptoAction.DO_NOTHING, CryptoAction.DO_NOTHING, CryptoAction.DO_NOTHING), +)) +def test_item_action_min(left, right, expected): + assert min(left, right) == expected + + +@pytest.mark.parametrize('left, right, expected_comparison', ( + (CryptoAction.ENCRYPT_AND_SIGN, CryptoAction.ENCRYPT_AND_SIGN, operator.eq), + (CryptoAction.ENCRYPT_AND_SIGN, CryptoAction.SIGN_ONLY, operator.ne), + (CryptoAction.ENCRYPT_AND_SIGN, CryptoAction.SIGN_ONLY, operator.gt), + (CryptoAction.ENCRYPT_AND_SIGN, CryptoAction.DO_NOTHING, operator.gt), + (CryptoAction.SIGN_ONLY, CryptoAction.ENCRYPT_AND_SIGN, operator.lt), + (CryptoAction.SIGN_ONLY, CryptoAction.SIGN_ONLY, operator.eq), + (CryptoAction.SIGN_ONLY, CryptoAction.DO_NOTHING, operator.ne), + (CryptoAction.SIGN_ONLY, CryptoAction.DO_NOTHING, operator.gt), + (CryptoAction.DO_NOTHING, CryptoAction.ENCRYPT_AND_SIGN, operator.lt), + (CryptoAction.DO_NOTHING, CryptoAction.SIGN_ONLY, operator.lt), + (CryptoAction.DO_NOTHING, CryptoAction.DO_NOTHING, operator.eq), + (CryptoAction.DO_NOTHING, CryptoAction.ENCRYPT_AND_SIGN, operator.ne) +)) +def test_item_action_comp(left, right, expected_comparison): + assert expected_comparison(left, right) diff --git a/test/functional/test_structures.py b/test/functional/test_structures.py new file mode 100644 index 00000000..4fb62bb0 --- /dev/null +++ b/test/functional/test_structures.py @@ -0,0 +1,58 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Functional tests for ``dynamodb_encryption_sdk.structures``.""" +import pytest + +from dynamodb_encryption_sdk.structures import TableIndex + +pytestmark = [pytest.mark.functional, pytest.mark.local] + + +@pytest.mark.parametrize('kwargs, expected_attributes', ( + (dict(partition='partition_name'), set(['partition_name'])), + (dict(partition='partition_name', sort='sort_name'), set(['partition_name', 'sort_name'])) +)) +def test_tableindex_attributes(kwargs, expected_attributes): + index = TableIndex(**kwargs) + assert index.attributes == expected_attributes + + +@pytest.mark.parametrize('key_schema, expected_kwargs', ( + ( + [ + { + 'KeyType': 'HASH', + 'AttributeName': 'partition_name' + } + ], + dict(partition='partition_name') + ), + ( + [ + { + 'KeyType': 'HASH', + 'AttributeName': 'partition_name' + }, + { + 'KeyType': 'RANGE', + 'AttributeName': 'sort_name' + } + ], + dict(partition='partition_name', sort='sort_name') + ) +)) +def test_tableindex_from_key_schema(key_schema, expected_kwargs): + index = TableIndex.from_key_schema(key_schema) + expected_index = TableIndex(**expected_kwargs) + + assert index == expected_index diff --git a/test/integration/__init__.py b/test/integration/__init__.py index 1ccc7fa1..2add15ef 100644 --- a/test/integration/__init__.py +++ b/test/integration/__init__.py @@ -10,3 +10,4 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +"""Dummy stub to make linters work better.""" diff --git a/test/integration/encrypted/__init__.py b/test/integration/encrypted/__init__.py new file mode 100644 index 00000000..2add15ef --- /dev/null +++ b/test/integration/encrypted/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Dummy stub to make linters work better.""" diff --git a/test/integration/encrypted/test_client.py b/test/integration/encrypted/test_client.py new file mode 100644 index 00000000..41cb6b2b --- /dev/null +++ b/test/integration/encrypted/test_client.py @@ -0,0 +1,65 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Integration tests for ``dynamodb_encryption_sdk.encrypted.client``.""" +import pytest + +from ..integration_test_utils import aws_kms_cmp, ddb_table_name # noqa pylint: disable=unused-import +from ..integration_test_utils import functional_test_utils + +pytestmark = pytest.mark.integ + + +def pytest_generate_tests(metafunc): + functional_test_utils.set_parametrized_actions(metafunc) + functional_test_utils.set_parametrized_cmp(metafunc) + functional_test_utils.set_parametrized_item(metafunc) + + +def test_ephemeral_item_cycle(ddb_table_name, some_cmps, parametrized_actions, parametrized_item): + """Test a small number of curated CMPs against a small number of curated items.""" + functional_test_utils.client_cycle_single_item_check( + some_cmps, + parametrized_actions, + parametrized_item, + ddb_table_name + ) + + +def test_ephemeral_item_cycle_kms(ddb_table_name, aws_kms_cmp, parametrized_actions, parametrized_item): + """Test the AWS KMS CMP against a small number of curated items.""" + functional_test_utils.client_cycle_single_item_check( + aws_kms_cmp, + parametrized_actions, + parametrized_item, + ddb_table_name + ) + + +def test_ephemeral_batch_item_cycle(ddb_table_name, some_cmps, parametrized_actions, parametrized_item): + """Test a small number of curated CMPs against a small number of curated items.""" + functional_test_utils.client_cycle_batch_items_check( + some_cmps, + parametrized_actions, + parametrized_item, + ddb_table_name + ) + + +def test_ephemeral_batch_item_cycle_kms(ddb_table_name, aws_kms_cmp, parametrized_actions, parametrized_item): + """Test the AWS KMS CMP against a small number of curated items.""" + functional_test_utils.client_cycle_batch_items_check( + aws_kms_cmp, + parametrized_actions, + parametrized_item, + ddb_table_name + ) diff --git a/test/integration/encrypted/test_resource.py b/test/integration/encrypted/test_resource.py new file mode 100644 index 00000000..8dde4b15 --- /dev/null +++ b/test/integration/encrypted/test_resource.py @@ -0,0 +1,45 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Integration tests for ``dynamodb_encryption_sdk.encrypted.resource``.""" +import pytest + +from ..integration_test_utils import aws_kms_cmp, ddb_table_name # noqa pylint: disable=unused-import +from ..integration_test_utils import functional_test_utils + +pytestmark = pytest.mark.integ + + +def pytest_generate_tests(metafunc): + functional_test_utils.set_parametrized_actions(metafunc) + functional_test_utils.set_parametrized_cmp(metafunc) + functional_test_utils.set_parametrized_item(metafunc) + + +def test_ephemeral_batch_item_cycle(ddb_table_name, some_cmps, parametrized_actions, parametrized_item): + """Test a small number of curated CMPs against a small number of curated items.""" + functional_test_utils.resource_cycle_batch_items_check( + some_cmps, + parametrized_actions, + parametrized_item, + ddb_table_name + ) + + +def test_ephemeral_batch_item_cycle_kms(ddb_table_name, aws_kms_cmp, parametrized_actions, parametrized_item): + """Test the AWS KMS CMP against a small number of curated items.""" + functional_test_utils.resource_cycle_batch_items_check( + aws_kms_cmp, + parametrized_actions, + parametrized_item, + ddb_table_name + ) diff --git a/test/integration/encrypted/test_table.py b/test/integration/encrypted/test_table.py new file mode 100644 index 00000000..b69105ac --- /dev/null +++ b/test/integration/encrypted/test_table.py @@ -0,0 +1,45 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Integration tests for ``dynamodb_encryption_sdk.encrypted.table``.""" +import pytest + +from ..integration_test_utils import aws_kms_cmp, ddb_table_name # noqa pylint: disable=unused-import +from ..integration_test_utils import functional_test_utils + +pytestmark = pytest.mark.integ + + +def pytest_generate_tests(metafunc): + functional_test_utils.set_parametrized_actions(metafunc) + functional_test_utils.set_parametrized_cmp(metafunc) + functional_test_utils.set_parametrized_item(metafunc) + + +def test_ephemeral_item_cycle(ddb_table_name, some_cmps, parametrized_actions, parametrized_item): + """Test a small number of curated CMPs against a small number of curated items.""" + functional_test_utils.table_cycle_check(some_cmps, parametrized_actions, parametrized_item, ddb_table_name) + + +def test_ephemeral_item_cycle_batch_writer(ddb_table_name, some_cmps, parametrized_actions, parametrized_item): + """Test a small number of curated CMPs against a small number of curated items.""" + functional_test_utils.table_cycle_batch_writer_check( + some_cmps, + parametrized_actions, + parametrized_item, + ddb_table_name + ) + + +def test_ephemeral_item_cycle_kms(ddb_table_name, aws_kms_cmp, parametrized_actions, parametrized_item): + """Test the AWS KMS CMP against a small number of curated items.""" + functional_test_utils.table_cycle_check(aws_kms_cmp, parametrized_actions, parametrized_item, ddb_table_name) diff --git a/test/integration/integration_test_utils.py b/test/integration/integration_test_utils.py index f2673cd0..edd699f7 100644 --- a/test/integration/integration_test_utils.py +++ b/test/integration/integration_test_utils.py @@ -10,26 +10,31 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +"""Helper utilities for integration tests.""" import os import sys + +import pytest + +from dynamodb_encryption_sdk.material_providers.aws_kms import AwsKmsCryptographicMaterialsProvider + sys.path.append(os.path.join( os.path.abspath(os.path.dirname(__file__)), '..', 'functional' )) -import pytest - -from dynamodb_encryption_sdk.material_providers.aws_kms import AwsKmsCryptographicMaterialsProvider - -import functional_test_utils, hypothesis_strategies +# Convenience imports +import functional_test_utils # noqa: E402,F401,I100 pylint: disable=import-error,unused-import,wrong-import-position +import hypothesis_strategies # noqa: E402,F401,I100 pylint: disable=import-error,unused-import,wrong-import-position AWS_KMS_KEY_ID = 'AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID' +DDB_TABLE_NAME = 'DDB_ENCRYPTION_CLIENT_TEST_TABLE_NAME' @pytest.fixture def cmk_arn(): - """Retrieves the target CMK ARN from environment variable.""" + """Retrieve the target CMK ARN from environment variable.""" arn = os.environ.get(AWS_KMS_KEY_ID, None) if arn is None: raise ValueError( @@ -45,3 +50,19 @@ def cmk_arn(): @pytest.fixture def aws_kms_cmp(): return AwsKmsCryptographicMaterialsProvider(key_id=cmk_arn()) + + +@pytest.fixture +def ddb_table_name(): + """Retrieve the target DynamoDB table from environment variable.""" + try: + return os.environ[DDB_TABLE_NAME] + except KeyError: + raise ValueError( + ( + 'Environment variable "{}" must be set to the correct DynamoDB table name' + ' for integration tests to run' + ).format( + AWS_KMS_KEY_ID + ) + ) diff --git a/test/integration/material_providers/__init__.py b/test/integration/material_providers/__init__.py new file mode 100644 index 00000000..2add15ef --- /dev/null +++ b/test/integration/material_providers/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Dummy stub to make linters work better.""" diff --git a/test/integration/test_i_materials_provider_aws_kms.py b/test/integration/material_providers/test_aws_kms.py similarity index 78% rename from test/integration/test_i_materials_provider_aws_kms.py rename to test/integration/material_providers/test_aws_kms.py index e22c4810..3f239ede 100644 --- a/test/integration/test_i_materials_provider_aws_kms.py +++ b/test/integration/material_providers/test_aws_kms.py @@ -10,12 +10,14 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +"""Integration tests for ``dynamodb_encryption_sdk.material_providers.aws_kms``.""" import hypothesis import pytest -from .integration_test_utils import aws_kms_cmp, functional_test_utils, hypothesis_strategies from dynamodb_encryption_sdk.encrypted import CryptoConfig from dynamodb_encryption_sdk.structures import EncryptionContext +from ..integration_test_utils import aws_kms_cmp # noqa pylint: disable=unused-import +from ..integration_test_utils import functional_test_utils, hypothesis_strategies pytestmark = pytest.mark.integ @@ -37,11 +39,11 @@ def test_aws_kms_item_cycle(aws_kms_cmp, parametrized_actions, parametrized_item @pytest.mark.slow @hypothesis_strategies.SLOW_SETTINGS @hypothesis.given(item=hypothesis_strategies.ddb_items) -def test_aws_kms_item_cycle_hypothesis_slow(aws_kms_cmp, parametrized_actions, item): +def test_aws_kms_item_cycle_hypothesis_slow(aws_kms_cmp, hypothesis_actions, item): crypto_config = CryptoConfig( materials_provider=aws_kms_cmp, encryption_context=EncryptionContext(), - attribute_actions=parametrized_actions + attribute_actions=hypothesis_actions ) functional_test_utils.cycle_item_check(item, crypto_config) @@ -49,10 +51,10 @@ def test_aws_kms_item_cycle_hypothesis_slow(aws_kms_cmp, parametrized_actions, i @pytest.mark.veryslow @hypothesis_strategies.VERY_SLOW_SETTINGS @hypothesis.given(item=hypothesis_strategies.ddb_items) -def test_aws_kms_item_cycle_hypothesis_veryslow(aws_kms_cmp, parametrized_actions, item): +def test_aws_kms_item_cycle_hypothesis_veryslow(aws_kms_cmp, hypothesis_actions, item): crypto_config = CryptoConfig( materials_provider=aws_kms_cmp, encryption_context=EncryptionContext(), - attribute_actions=parametrized_actions + attribute_actions=hypothesis_actions ) functional_test_utils.cycle_item_check(item, crypto_config) diff --git a/test/pylintrc b/test/pylintrc index bc06e8de..27d82a9c 100644 --- a/test/pylintrc +++ b/test/pylintrc @@ -8,7 +8,7 @@ # R0801 : duplicate-code (unit tests for similar things tend to be similar) # W0212 : protected-access (raised when calling _ methods) # W0621 : redefined-outer-name (raised when using pytest-mock) -# W0613 : unused-argument (raised when patches are needed but not called) +# W0613 : unused-argument (raised when patches and fixtures are needed but not called) disable = C0103, C0111, E1101, R0801, W0212, W0621, W0613 [DESIGN] diff --git a/test/requirements.txt b/test/requirements.txt new file mode 100644 index 00000000..780cff82 --- /dev/null +++ b/test/requirements.txt @@ -0,0 +1,7 @@ +hypothesis +mock +moto +pytest>=3.3.1 +pytest-cov +pytest-mock +pytest-xdist \ No newline at end of file diff --git a/test/unit/__init__.py b/test/unit/__init__.py index 1ccc7fa1..2add15ef 100644 --- a/test/unit/__init__.py +++ b/test/unit/__init__.py @@ -10,3 +10,4 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +"""Dummy stub to make linters work better.""" diff --git a/test/unit/material_providers/__init__.py b/test/unit/material_providers/__init__.py index 1ccc7fa1..5eb0e67b 100644 --- a/test/unit/material_providers/__init__.py +++ b/test/unit/material_providers/__init__.py @@ -10,3 +10,25 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +"""Unit test for ``dynamodb_encryption_sdk.material_providers``.""" +import pytest + +from dynamodb_encryption_sdk.material_providers import CryptographicMaterialsProvider + +pytestmark = [pytest.mark.unit, pytest.mark.local] + + +@pytest.mark.parametrize('method, message', ( + ('decryption_materials', 'No decryption materials available'), + ('encryption_materials', 'No encryption materials available') +)) +def test_no_materials(method, message): + empty_cmp = CryptographicMaterialsProvider( + decryption_materials=None, + encryption_materials=None + ) + + with pytest.raises(AttributeError) as excinfo: + getattr(empty_cmp, method)(None) + + excinfo.match(message) diff --git a/test/unit/material_providers/test_aws_kms.py b/test/unit/material_providers/test_aws_kms.py index 701143ce..16ce3f3a 100644 --- a/test/unit/material_providers/test_aws_kms.py +++ b/test/unit/material_providers/test_aws_kms.py @@ -10,17 +10,130 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +"""Unit tests for ``dynamodb_encryption_sdk.material_providers.aws_kms``.""" +import base64 +from mock import MagicMock, sentinel + import boto3 -import botocore.session +from boto3.dynamodb.types import Binary +import botocore from moto import mock_kms import pytest +from pytest_mock import mocker # noqa pylint: disable=unused-import -from dynamodb_encryption_sdk.material_providers.aws_kms import AwsKmsCryptographicMaterialsProvider +from dynamodb_encryption_sdk.delegated_keys.jce import JceNameLocalDelegatedKey +from dynamodb_encryption_sdk.exceptions import UnknownRegionError, UnwrappingError, WrappingError +from dynamodb_encryption_sdk.identifiers import EncryptionKeyType, KeyEncodingType +import dynamodb_encryption_sdk.material_providers.aws_kms +from dynamodb_encryption_sdk.material_providers.aws_kms import ( + _DEFAULT_CONTENT_ENCRYPTION_ALGORITHM, _DEFAULT_SIGNING_ALGORITHM, + AwsKmsCryptographicMaterialsProvider, KeyInfo +) +from dynamodb_encryption_sdk.structures import EncryptionContext pytestmark = [pytest.mark.unit, pytest.mark.local] +_VALID_KEY_INFO_KWARGS = dict( + description='some string', + algorithm='algorithm name', + length=1234 +) +_REGION = 'fake-region' +_KEY_ID = 'arn:aws:kms:{}:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab'.format(_REGION) +_DERIVED_KEYS = { + 'initial_material': b'\xafx2"\xb5\xd5`\xc6\x8d\xaa\xfe\xc10E3x?D\x18\x93$<\x161\xcb\x99\xef\xc0Z\x1a\x1b]', + 'encrypted_initial_material': ( + b"\x01\x01\x02\x00x@\xf3\x8c'^1\tt\x16\xc1\x07)QPW\x19d\xad\xa3\xef\x1c!\xe9L\x8b\xa0\xbd\xbc\x9d\x0f\xb4\x14" + b"\x00\x00\x00~0|\x06\t*\x86H\x86\xf7\r\x01\x07\x06\xa0o0m\x02\x01\x000h\x06\t*\x86H\x86\xf7\r\x01\x07\x010" + b"\x1e\x06\t`\x86H\x01e\x03\x04\x01.0\x11\x04\x0c-\xc0&\x1f\xeb_\xdek\xca/$y\x02\x01\x10\x80;!\x99z\xbek3|\x8b" + b"\x98\x1b\xba\x91H<\xb1X\x8c\xc7vGv\x84*\xe1\xf1B\xd4\xe5&\xa2\xa3)\x04\x1f\xad\t\x07\x90\x14\xbeQo\xa0\xff" + b"\x1a\xc2\xa5(i\x0c4\x10\xe8\xe2\xf3\x17}\t\xd6" + ), # encrypted using our public-test CMK in us-west-2 + 'encryption_key': b'\xb3~{,Z\x80\x7f\x82I\xe5=3.3.1 - pytest-cov - pytest-mock - pytest-xdist +deps = -rtest/requirements.txt commands = + # Only run small test scenario sets local-fast: {[testenv:base-command]commands} -m "local and not slow and not veryslow and not nope" integ-fast: {[testenv:base-command]commands} -m "integ and not slow and not veryslow and not nope" all-fast: {[testenv:base-command]commands} -m "not slow and not veryslow and not nope" + # Also run moderately large test scenario sets local-slow: {[testenv:base-command]commands} -m "local and not veryslow and not nope" integ-slow: {[testenv:base-command]commands} -m "integ and not veryslow and not nope" all-slow: {[testenv:base-command]commands} -m "not veryslow and not nope" + # Also run very large test scenario sets local-full: {[testenv:base-command]commands} -m "local and not nope" integ-full: {[testenv:base-command]commands} -m "integ and not nope" all-full: {[testenv:base-command]commands} -m "not nope" + # Only run extremely large test scenario sets local-nope: {[testenv:base-command]commands} -m "local and nope" integ-nope: {[testenv:base-command]commands} -m "integ and nope" all-nope: {[testenv:base-command]commands} -m "nope" + # Do not select any specific markers + manual: {[testenv:base-command]commands} + # Only run examples tests + examples: {[testenv:base-command]commands} -m "examples" + +# Verify that local tests work without environment variables present +[testenv:nocmk] +basepython = python3 +sitepackages = False +deps = -rtest/requirements.txt +commands = {[testenv:base-command]commands} -m "local and not slow and not veryslow and not nope" # mypy [testenv:mypy-coverage] @@ -109,6 +122,20 @@ commands = --ignore F811,D103 \ test/ +[testenv:flake8-examples] +basepython = {[testenv:flake8]basepython} +deps = {[testenv:flake8]deps} +commands = + flake8 \ + # Examples should not treat any imports as application-local. + --application-import-names= \ + examples/src/ + flake8 \ + # Ignore F811 redefinition errors in tests (breaks with fixture use) + # Ignore D103 docstring requirements for tests + --ignore F811,D103 \ + examples/test/ + [testenv:pylint] basepython = python3 deps = @@ -128,7 +155,17 @@ deps = {[testenv:pylint]deps} commands = pylint \ --rcfile=test/pylintrc \ - test/unit/ + test/unit/ \ + test/acceptance/ \ + test/functional/ \ + test/integration/ + +[testenv:pylint-examples] +basepython = {[testenv:pylint]basepython} +deps = {[testenv:pylint]deps} +commands = + pylint --rcfile=examples/src/pylintrc examples/src/ + pylint --rcfile=examples/test/pylintrc examples/test/ [testenv:doc8] basepython = python3