From 060b060005bdeb6e9379a0c41ea7f08f1336c1a5 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Mon, 14 May 2018 09:18:59 -0700 Subject: [PATCH 01/10] fix mypy return value for _ttl_action --- src/dynamodb_encryption_sdk/material_providers/most_recent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dynamodb_encryption_sdk/material_providers/most_recent.py b/src/dynamodb_encryption_sdk/material_providers/most_recent.py index e534bc90..e01cc560 100644 --- a/src/dynamodb_encryption_sdk/material_providers/most_recent.py +++ b/src/dynamodb_encryption_sdk/material_providers/most_recent.py @@ -182,7 +182,7 @@ def decryption_materials(self, encryption_context): return provider.decryption_materials(encryption_context) def _ttl_action(self): - # type: () -> bool + # type: () -> TtlActions """Determine the correct action to take based on the local resources and TTL. :returns: decision From 1fdcb5c32b83fc9356149c8ceba591df68c32272 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Mon, 14 May 2018 09:20:13 -0700 Subject: [PATCH 02/10] _get_most_recent_version should not return the version number any more: if the lock was not acquired, this would hit an error --- src/dynamodb_encryption_sdk/material_providers/most_recent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dynamodb_encryption_sdk/material_providers/most_recent.py b/src/dynamodb_encryption_sdk/material_providers/most_recent.py index e01cc560..a5db9b17 100644 --- a/src/dynamodb_encryption_sdk/material_providers/most_recent.py +++ b/src/dynamodb_encryption_sdk/material_providers/most_recent.py @@ -247,7 +247,7 @@ def _get_most_recent_version(self, allow_local): # If blocking, we will never reach this point. # If not blocking, we want whatever the latest local version is. version = self._version - return version, self._cache.get(version) + return self._cache.get(version) try: max_version = self._get_max_version() From 608842adfbc1ed2dd5a595d64591aaccc7b919c3 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Wed, 16 May 2018 11:27:26 -0700 Subject: [PATCH 03/10] test to verify failed lock acquisition fix --- .../material_providers/store/test_meta.py | 2 +- .../material_providers/test_most_recent.py | 37 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 test/functional/material_providers/test_most_recent.py diff --git a/test/functional/material_providers/store/test_meta.py b/test/functional/material_providers/store/test_meta.py index 6eee7a13..7d2ea0c1 100644 --- a/test/functional/material_providers/store/test_meta.py +++ b/test/functional/material_providers/store/test_meta.py @@ -10,7 +10,7 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Integration tests for ``dynamodb_encryption_sdk.material_providers.store.meta``.""" +"""Functional tests for ``dynamodb_encryption_sdk.material_providers.store.meta``.""" import base64 import os diff --git a/test/functional/material_providers/test_most_recent.py b/test/functional/material_providers/test_most_recent.py new file mode 100644 index 00000000..399a7d11 --- /dev/null +++ b/test/functional/material_providers/test_most_recent.py @@ -0,0 +1,37 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Functional tests for ``dynamodb_encryption_sdk.material_providers.most_recent``.""" +from mock import MagicMock, sentinel +import pytest + +from dynamodb_encryption_sdk.material_providers.most_recent import MostRecentProvider +from dynamodb_encryption_sdk.material_providers.store import ProviderStore + +pytestmark = [pytest.mark.functional, pytest.mark.local] + + +def test_failed_lock_acquisition(): + store = MagicMock(__class__=ProviderStore) + provider = MostRecentProvider( + provider_store=store, + material_name='my material', + version_ttl=10.0 + ) + provider._version = 9 + provider._cache.put(provider._version, sentinel.nine) + + with provider._lock: + test = provider._get_most_recent_version(allow_local=True) + + assert test is sentinel.nine + assert not store.mock_calls From 5001aad19e7f42379293bbbeb38659854e43b630 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Wed, 16 May 2018 11:38:38 -0700 Subject: [PATCH 04/10] version bump to v1.0.4 and updating changelog --- CHANGELOG.rst | 14 ++++++++++++++ src/dynamodb_encryption_sdk/identifiers.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b70fb561..05ab77bd 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,12 +2,26 @@ Changelog ********* +1.0.4 -- 2018-05-16 +=================== + +Bugfixes +-------- +* Fix ``MostRecentProvider`` behavior when lock cannot be acquired. + `#72 `_ + 1.0.3 -- 2018-05-03 =================== + +Bugfixes +-------- * Finish fixing ``MANIFEST.in``. 1.0.2 -- 2018-05-03 =================== + +Bugfixes +-------- * Fill out ``MANIFEST.in`` to correctly include necessary files in source build. 1.0.1 -- 2018-05-02 diff --git a/src/dynamodb_encryption_sdk/identifiers.py b/src/dynamodb_encryption_sdk/identifiers.py index c49ec32a..89ca8d16 100644 --- a/src/dynamodb_encryption_sdk/identifiers.py +++ b/src/dynamodb_encryption_sdk/identifiers.py @@ -14,7 +14,7 @@ from enum import Enum __all__ = ('LOGGER_NAME', 'CryptoAction', 'EncryptionKeyType', 'KeyEncodingType') -__version__ = '1.0.3' +__version__ = '1.0.4' LOGGER_NAME = 'dynamodb_encryption_sdk' USER_AGENT_SUFFIX = 'DynamodbEncryptionSdkPython/{}'.format(__version__) From 2d3d2899cebca73107d6736a5991e259f339016b Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Fri, 18 May 2018 08:54:28 -0700 Subject: [PATCH 05/10] fix #73 : locking syntax error in Python 2.7 --- CHANGELOG.rst | 4 +++- src/dynamodb_encryption_sdk/material_providers/most_recent.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 05ab77bd..c2d8027d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,8 +7,10 @@ Changelog Bugfixes -------- -* Fix ``MostRecentProvider`` behavior when lock cannot be acquired. +* Fix :class:`MostRecentProvider` behavior when lock cannot be acquired. `#72 `_ +* Fix :class:`MostRecentProvider` lock acquisition for Python 2.7. + `#73 `_ 1.0.3 -- 2018-05-03 =================== diff --git a/src/dynamodb_encryption_sdk/material_providers/most_recent.py b/src/dynamodb_encryption_sdk/material_providers/most_recent.py index a5db9b17..e70379dc 100644 --- a/src/dynamodb_encryption_sdk/material_providers/most_recent.py +++ b/src/dynamodb_encryption_sdk/material_providers/most_recent.py @@ -240,7 +240,7 @@ def _get_most_recent_version(self, allow_local): :returns: version and corresponding cryptographic materials provider :rtype: CryptographicMaterialsProvider """ - acquired = self._lock.acquire(blocking=not allow_local) + acquired = self._lock.acquire(not allow_local) if not acquired: # We failed to acquire the lock. From 11e30394362f26a2d57a6d4c1982535b2d2c2283 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Fri, 18 May 2018 08:55:26 -0700 Subject: [PATCH 06/10] sphinx apparently does not play well with multiple build targets in a single command: change the "docs" build comamnd to run the html build and the link check in separate commands --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index fb0f6847..dc99adf7 100644 --- a/tox.ini +++ b/tox.ini @@ -259,7 +259,8 @@ commands = basepython = python3 deps = -rdoc/requirements.txt commands = - sphinx-build -E -c doc/ -b html -b linkcheck doc/ doc/build/html + sphinx-build -E -c doc/ -b html doc/ doc/build/html + sphinx-build -E -c doc/ -b linkcheck doc/ doc/build/html [testenv:serve-docs] basepython = python3 From 0ceb1b7ece1aa2c056509ec13fa73e6c262d0bba Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Fri, 18 May 2018 08:57:33 -0700 Subject: [PATCH 07/10] fix issue ID reference in changelog --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c2d8027d..af342b7a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,7 +10,7 @@ Bugfixes * Fix :class:`MostRecentProvider` behavior when lock cannot be acquired. `#72 `_ * Fix :class:`MostRecentProvider` lock acquisition for Python 2.7. - `#73 `_ + `#74 `_ 1.0.3 -- 2018-05-03 =================== From 27c1640919063f86b850aa08c427a95e26e67789 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Mon, 21 May 2018 09:09:05 -0700 Subject: [PATCH 08/10] add tests to cover #75 and convert TableInfo secondary index storage from set to list to fix #75 --- CHANGELOG.rst | 4 +- src/dynamodb_encryption_sdk/structures.py | 6 +- test/functional/functional_test_utils.py | 136 ++++++++++++++++++++++ test/functional/test_structures.py | 29 ++++- 4 files changed, 170 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index af342b7a..9f7eabca 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,7 +2,7 @@ Changelog ********* -1.0.4 -- 2018-05-16 +1.0.4 -- 2018-05-xx =================== Bugfixes @@ -11,6 +11,8 @@ Bugfixes `#72 `_ * Fix :class:`MostRecentProvider` lock acquisition for Python 2.7. `#74 `_ +* Fix :class:`TableInfo` secondary index storage. + `#75 `_ 1.0.3 -- 2018-05-03 =================== diff --git a/src/dynamodb_encryption_sdk/structures.py b/src/dynamodb_encryption_sdk/structures.py index cccf266a..3402d867 100644 --- a/src/dynamodb_encryption_sdk/structures.py +++ b/src/dynamodb_encryption_sdk/structures.py @@ -304,7 +304,7 @@ class TableInfo(object): default=None ) _secondary_indexes = attr.ib( - validator=attr.validators.optional(iterable_validator(set, TableIndex)), + validator=attr.validators.optional(iterable_validator(list, TableIndex)), default=None ) @@ -378,10 +378,10 @@ def refresh_indexed_attributes(self, client): table = client.describe_table(TableName=self.name)['Table'] self._primary_index = TableIndex.from_key_schema(table['KeySchema']) - self._secondary_indexes = set() + self._secondary_indexes = [] for group in ('LocalSecondaryIndexes', 'GlobalSecondaryIndexes'): try: for index in table[group]: - self._secondary_indexes.add(TableIndex.from_key_schema(index['KeySchema'])) + self._secondary_indexes.append(TableIndex.from_key_schema(index['KeySchema'])) except KeyError: pass # Not all tables will have secondary indexes. diff --git a/test/functional/functional_test_utils.py b/test/functional/functional_test_utils.py index 6d6a322b..b7fb3a2f 100644 --- a/test/functional/functional_test_utils.py +++ b/test/functional/functional_test_utils.py @@ -48,6 +48,16 @@ 'value': Decimal('99.233') } } +SECONARY_INDEX = { + 'secondary_index_1': { + 'type': 'B', + 'value': Binary(b'\x00\x01\x02') + }, + 'secondary_index_1': { + 'type': 'S', + 'value': 'another_value' + } +} TEST_KEY = {name: value['value'] for name, value in TEST_INDEX.items()} TEST_BATCH_INDEXES = [ { @@ -130,6 +140,132 @@ def example_table(): mock_dynamodb2().stop() +@pytest.fixture +def table_with_local_seconary_indexes(): + mock_dynamodb2().start() + ddb = boto3.client('dynamodb', region_name='us-west-2') + ddb.create_table( + TableName=TEST_TABLE_NAME, + KeySchema=[ + { + 'AttributeName': 'partition_attribute', + 'KeyType': 'HASH' + }, + { + 'AttributeName': 'sort_attribute', + 'KeyType': 'RANGE' + } + ], + LocalSecondaryIndexes=[ + { + 'IndexName': 'lsi-1', + 'KeySchema': [ + { + 'AttributeName': 'secondary_index_1', + 'KeyType': 'HASH' + } + ], + 'Projection': { + 'ProjectionType': 'ALL' + } + }, + { + 'IndexName': 'lsi-2', + 'KeySchema': [ + { + 'AttributeName': 'secondary_index_2', + 'KeyType': 'HASH' + } + ], + 'Projection': { + 'ProjectionType': 'ALL' + } + } + ], + AttributeDefinitions=[ + { + 'AttributeName': name, + 'AttributeType': value['type'] + } + for name, value in list(TEST_INDEX.items()) + list(SECONARY_INDEX.items()) + ], + ProvisionedThroughput={ + 'ReadCapacityUnits': 100, + 'WriteCapacityUnits': 100 + } + ) + yield + ddb.delete_table(TableName=TEST_TABLE_NAME) + mock_dynamodb2().stop() + + +@pytest.fixture +def table_with_global_seconary_indexes(): + mock_dynamodb2().start() + ddb = boto3.client('dynamodb', region_name='us-west-2') + ddb.create_table( + TableName=TEST_TABLE_NAME, + KeySchema=[ + { + 'AttributeName': 'partition_attribute', + 'KeyType': 'HASH' + }, + { + 'AttributeName': 'sort_attribute', + 'KeyType': 'RANGE' + } + ], + GlobalSecondaryIndexes=[ + { + 'IndexName': 'gsi-1', + 'KeySchema': [ + { + 'AttributeName': 'secondary_index_1', + 'KeyType': 'HASH' + } + ], + 'Projection': { + 'ProjectionType': 'ALL' + }, + 'ProvisionedThroughput': { + 'ReadCapacityUnits': 100, + 'WriteCapacityUnits': 100 + } + }, + { + 'IndexName': 'gsi-2', + 'KeySchema': [ + { + 'AttributeName': 'secondary_index_2', + 'KeyType': 'HASH' + } + ], + 'Projection': { + 'ProjectionType': 'ALL' + }, + 'ProvisionedThroughput': { + 'ReadCapacityUnits': 100, + 'WriteCapacityUnits': 100 + } + } + ], + AttributeDefinitions=[ + { + 'AttributeName': name, + 'AttributeType': value['type'] + } + for name, value in list(TEST_INDEX.items()) + list(SECONARY_INDEX.items()) + ], + ProvisionedThroughput={ + 'ReadCapacityUnits': 100, + 'WriteCapacityUnits': 100 + } + ) + yield + ddb.delete_table(TableName=TEST_TABLE_NAME) + mock_dynamodb2().stop() + + def _get_from_cache(dk_class, algorithm, key_length): """Don't generate new keys every time. All we care about is that they are valid keys, not that they are unique.""" try: diff --git a/test/functional/test_structures.py b/test/functional/test_structures.py index b025ca4f..60810961 100644 --- a/test/functional/test_structures.py +++ b/test/functional/test_structures.py @@ -11,15 +11,42 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. """Functional tests for ``dynamodb_encryption_sdk.structures``.""" +import boto3 import pytest from dynamodb_encryption_sdk.exceptions import InvalidArgumentError from dynamodb_encryption_sdk.identifiers import CryptoAction -from dynamodb_encryption_sdk.structures import AttributeActions, TableIndex +from dynamodb_encryption_sdk.structures import AttributeActions, TableIndex, TableInfo +from .functional_test_utils import ( + example_table, table_with_global_seconary_indexes, table_with_local_seconary_indexes, TEST_TABLE_NAME +) pytestmark = [pytest.mark.functional, pytest.mark.local] +# TODO: There is a way to parameterize fixtures, but the existing docs on that are very unclear. +# This will get us what we need for now, but we should come back to this to clean this up later. +def test_tableinfo_refresh_indexes_no_secondary_indexes(example_table): + client = boto3.client('dynamodb', region_name='us-west-2') + table = TableInfo(name=TEST_TABLE_NAME) + + table.refresh_indexed_attributes(client) + + +def test_tableinfo_refresh_indexes_with_gsis(table_with_global_seconary_indexes): + client = boto3.client('dynamodb', region_name='us-west-2') + table = TableInfo(name=TEST_TABLE_NAME) + + table.refresh_indexed_attributes(client) + + +def test_tableinfo_refresh_indexes_with_lsis(table_with_local_seconary_indexes): + client = boto3.client('dynamodb', region_name='us-west-2') + table = TableInfo(name=TEST_TABLE_NAME) + + table.refresh_indexed_attributes(client) + + @pytest.mark.parametrize('kwargs, expected_attributes', ( (dict(partition='partition_name'), set(['partition_name'])), (dict(partition='partition_name', sort='sort_name'), set(['partition_name', 'sort_name'])) From 1b8c922eeb961ec80f77c491c97821d183c4ab34 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Mon, 21 May 2018 15:00:32 -0700 Subject: [PATCH 09/10] fix type annotations following #75 fix --- src/dynamodb_encryption_sdk/structures.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/dynamodb_encryption_sdk/structures.py b/src/dynamodb_encryption_sdk/structures.py index 3402d867..22c38cdf 100644 --- a/src/dynamodb_encryption_sdk/structures.py +++ b/src/dynamodb_encryption_sdk/structures.py @@ -17,7 +17,7 @@ import six try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Dict, Iterable, Optional, Set, Text # noqa pylint: disable=unused-import + from typing import Dict, Iterable, List, Optional, Set, Text # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass @@ -295,7 +295,7 @@ class TableInfo(object): :param bool all_encrypting_secondary_indexes: Should we allow secondary index attributes to be encrypted? :param TableIndex primary_index: Description of primary index :param secondary_indexes: Set of TableIndex objects describing any secondary indexes - :type secondary_indexes: set(TableIndex) + :type secondary_indexes: list(TableIndex) """ name = attr.ib(validator=attr.validators.instance_of(six.string_types)) @@ -312,7 +312,7 @@ def __init__( self, name, # type: Text primary_index=None, # type: Optional[TableIndex] - secondary_indexes=None # type: Optional[Set[TableIndex]] + secondary_indexes=None # type: Optional[List[TableIndex]] ): # noqa=D107 # type: (...) -> None # Workaround pending resolution of attrs/mypy interaction. @@ -338,7 +338,7 @@ def primary_index(self): @property def secondary_indexes(self): - # type: () -> Set[TableIndex] + # type: () -> List[TableIndex] """Return the primary TableIndex. :returns: secondary index descriptions From 3bed1036aa89580be418723e7d9012170ced67ad Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Mon, 21 May 2018 15:00:51 -0700 Subject: [PATCH 10/10] set release date for 1.0.4 --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9f7eabca..6bfbb690 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,7 +2,7 @@ Changelog ********* -1.0.4 -- 2018-05-xx +1.0.4 -- 2018-05-22 =================== Bugfixes