From 0805e6cc9eef215241e2f529656091a2b8946387 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Tue, 15 Jan 2019 21:44:15 -0800 Subject: [PATCH 01/11] add ddb_integ tests for MostRecentProvider --- .../material_providers/test_most_recent.py | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 test/integration/material_providers/test_most_recent.py diff --git a/test/integration/material_providers/test_most_recent.py b/test/integration/material_providers/test_most_recent.py new file mode 100644 index 00000000..71af7206 --- /dev/null +++ b/test/integration/material_providers/test_most_recent.py @@ -0,0 +1,68 @@ +# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Load testing using MostRecentProvider and MetaStore.""" +import boto3 +import pytest + +from dynamodb_encryption_sdk.encrypted.table import EncryptedTable +from dynamodb_encryption_sdk.material_providers.most_recent import MostRecentProvider + +from ..integration_test_utils import ( # pylint: disable=unused-import + ddb_table_name, + functional_test_utils, + temp_metastore, +) + +pytestmark = [pytest.mark.integ, pytest.mark.ddb_integ] + + +def count_entries(records, *messages): + count = 0 + + for record in records: + if all((message in record.getMessage() for message in messages)): + count += 1 + + return count + + +def count_puts(records, table_name): + return count_entries(records, '"TableName": "{}"'.format(table_name), "OperationModel(name=PutItem)") + + +def count_gets(records, table_name): + return count_entries(records, '"TableName": "{}"'.format(table_name), "OperationModel(name=GetItem)") + + +def test_cache_use_encrypt(temp_metastore, ddb_table_name, caplog): + table = boto3.resource("dynamodb").Table(ddb_table_name) + + e_table = EncryptedTable( + table=table, + materials_provider=MostRecentProvider(provider_store=temp_metastore, material_name="test", version_ttl=600.0), + ) + + item = functional_test_utils.diverse_item() + item.update(functional_test_utils.TEST_KEY) + e_table.put_item(Item=item) + e_table.put_item(Item=item) + e_table.put_item(Item=item) + e_table.put_item(Item=item) + + e_table.delete_item(Key=functional_test_utils.TEST_KEY) + + primary_puts = count_puts(caplog.records, ddb_table_name) + metastore_puts = count_puts(caplog.records, temp_metastore._table.name) + + assert primary_puts == 4 + assert metastore_puts == 1 From da8e02a9f63912f7c8110a03cfa21453f5ac72ef Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Tue, 15 Jan 2019 21:44:36 -0800 Subject: [PATCH 02/11] rework metastore fixtures to actually run cleanup --- test/functional/functional_test_utils.py | 19 +++++++++++++------ test/integration/integration_test_utils.py | 4 +++- .../material_providers/store/test_meta.py | 3 --- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/test/functional/functional_test_utils.py b/test/functional/functional_test_utils.py index 202b0e02..df6e5194 100644 --- a/test/functional/functional_test_utils.py +++ b/test/functional/functional_test_utils.py @@ -16,14 +16,12 @@ import base64 import copy import itertools -import logging import os from collections import defaultdict from decimal import Decimal import boto3 import pytest -import six from boto3.dynamodb.types import Binary from moto import mock_dynamodb2 @@ -658,14 +656,23 @@ def build_metastore(): waiter.wait(TableName=table_name) table = boto3.resource("dynamodb", region_name="us-west-2").Table(table_name) - yield MetaStore(table, build_static_jce_cmp("AES", 256, "HmacSHA256", 256)) + return MetaStore(table, build_static_jce_cmp("AES", 256, "HmacSHA256", 256)), table_name + +def delete_metastore(table_name): + client = boto3.client("dynamodb", region_name="us-west-2") client.delete_table(TableName=table_name) - waiter = client.get_waiter("table_not_exists") - waiter.wait(TableName=table_name) + # It sometimes takes a long time to delete a table. + # If hanging, asynchronously deleting tables becomes an issue, + # come back to this. + # Otherwise, let's just let them take care of themselves. + # waiter = client.get_waiter("table_not_exists") + # waiter.wait(TableName=table_name) @pytest.fixture def mock_metastore(): with mock_dynamodb2(): - yield next(build_metastore()) + metastore, table_name = build_metastore() + yield metastore + delete_metastore(table_name) diff --git a/test/integration/integration_test_utils.py b/test/integration/integration_test_utils.py index 522cc130..c5b24ea2 100644 --- a/test/integration/integration_test_utils.py +++ b/test/integration/integration_test_utils.py @@ -69,4 +69,6 @@ def ddb_table_name(): @pytest.fixture def temp_metastore(): - yield next(functional_test_utils.build_metastore()) + metastore, table_name = functional_test_utils.build_metastore() + yield metastore + functional_test_utils.delete_metastore(table_name) diff --git a/test/integration/material_providers/store/test_meta.py b/test/integration/material_providers/store/test_meta.py index 50dadc71..c092ba21 100644 --- a/test/integration/material_providers/store/test_meta.py +++ b/test/integration/material_providers/store/test_meta.py @@ -11,9 +11,6 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. """Integration tests for ``dynamodb_encryption_sdk.material_providers.store.meta``.""" -import os - -import boto3 import pytest from dynamodb_encryption_sdk.exceptions import NoKnownVersionError From 6bb54da8d6c9701ce3bd2c815ccdc072ef7d115c Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Tue, 15 Jan 2019 21:45:07 -0800 Subject: [PATCH 03/11] add logging to MetaStore and MostRecentProvider --- .../material_providers/most_recent.py | 8 ++++++++ .../material_providers/store/meta.py | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/src/dynamodb_encryption_sdk/material_providers/most_recent.py b/src/dynamodb_encryption_sdk/material_providers/most_recent.py index 01c38607..37fee554 100644 --- a/src/dynamodb_encryption_sdk/material_providers/most_recent.py +++ b/src/dynamodb_encryption_sdk/material_providers/most_recent.py @@ -183,6 +183,7 @@ def _ttl_action(self): :rtype: TtlActions """ if self._version is None: + _LOGGER.debug("TTL Expired because no version is known") return TtlActions.EXPIRED time_since_updated = time.time() - self._last_updated @@ -193,6 +194,7 @@ def _ttl_action(self): elif time_since_updated < self._version_ttl + _GRACE_PERIOD: return TtlActions.GRACE_PERIOD + _LOGGER.debug("TTL Expired because known version has expired") return TtlActions.EXPIRED def _get_max_version(self): @@ -260,6 +262,8 @@ def _get_most_recent_version(self, allow_local): finally: self._lock.release() + _LOGGER.debug("New latest version is %d", self._version) + return provider def encryption_materials(self, encryption_context): @@ -271,6 +275,8 @@ def encryption_materials(self, encryption_context): """ ttl_action = self._ttl_action() + _LOGGER.debug('TTL Action "%s" when getting encryption materials', ttl_action.name) + provider = None if ttl_action is TtlActions.LIVE: @@ -286,6 +292,7 @@ def encryption_materials(self, encryption_context): # Otherwise, block until we can acquire the lock. allow_local = bool(ttl_action is TtlActions.GRACE_PERIOD) + _LOGGER.debug("Getting most recent materials provider version") provider = self._get_most_recent_version(allow_local) return provider.encryption_materials(encryption_context) @@ -293,6 +300,7 @@ def encryption_materials(self, encryption_context): def refresh(self): # type: () -> None """Clear all local caches for this provider.""" + _LOGGER.debug("Refreshing MostRecentProvider instance.") with self._lock: self._cache.clear() self._version = None # type: int # pylint: disable=attribute-defined-outside-init diff --git a/src/dynamodb_encryption_sdk/material_providers/store/meta.py b/src/dynamodb_encryption_sdk/material_providers/store/meta.py index 2dbfe227..bf0ffcc8 100644 --- a/src/dynamodb_encryption_sdk/material_providers/store/meta.py +++ b/src/dynamodb_encryption_sdk/material_providers/store/meta.py @@ -36,6 +36,7 @@ __all__ = ("MetaStore",) +_LOGGER = logging.getLogger(LOGGER_NAME) class MetaStoreAttributeNames(Enum): @@ -104,6 +105,7 @@ def create_table(cls, client, table_name, read_units, write_units): :param int read_units: Read capacity units to provision :param int write_units: Write capacity units to provision """ + _LOGGER.debug("Creating MetaStore table") try: client.create_table( TableName=table_name, @@ -127,6 +129,7 @@ def _load_materials(self, material_name, version): :returns: Materials loaded into delegated keys :rtype: tuple(JceNameLocalDelegatedKey) """ + _LOGGER.debug('Loading material "%s" version %d from MetaStore table', material_name, version) key = {MetaStoreAttributeNames.PARTITION.value: material_name, MetaStoreAttributeNames.SORT.value: version} response = self._encrypted_table.get_item(Key=key) try: @@ -168,6 +171,7 @@ def _save_materials(self, material_name, version, encryption_key, signing_key): :param int version: Version of material to locate :raises VersionAlreadyExistsError: if the specified version already exists """ + _LOGGER.debug('Saving material "%s" version %d to MetaStore table', material_name, version) item = { MetaStoreAttributeNames.PARTITION.value: material_name, MetaStoreAttributeNames.SORT.value: version, From a4836bf632d5a3f872e2dbe105017167d437eed6 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Tue, 15 Jan 2019 21:45:40 -0800 Subject: [PATCH 04/11] fix mypy signature --- src/dynamodb_encryption_sdk/internal/utils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/dynamodb_encryption_sdk/internal/utils.py b/src/dynamodb_encryption_sdk/internal/utils.py index 6b1ee127..bb133766 100644 --- a/src/dynamodb_encryption_sdk/internal/utils.py +++ b/src/dynamodb_encryption_sdk/internal/utils.py @@ -59,10 +59,8 @@ class TableInfoCache(object): _client = attr.ib(validator=attr.validators.instance_of(botocore.client.BaseClient)) _auto_refresh_table_indexes = attr.ib(validator=attr.validators.instance_of(bool)) - def __init__( - self, client, auto_refresh_table_indexes # type: botocore.client.BaseClient # type: bool - ): # noqa=D107 - # type: (...) -> None + def __init__(self, client, auto_refresh_table_indexes): # noqa=D107 + # type: (botocore.client.BaseClient, bool) -> None # Workaround pending resolution of attrs/mypy interaction. # https://github.com/python/mypy/issues/2088 # https://github.com/python-attrs/attrs/issues/215 From 76c0b93159453db86374dfd8b2687ac53827ca35 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Tue, 15 Jan 2019 21:46:35 -0800 Subject: [PATCH 05/11] fix bug where encrypt_ddb_item would call refresh() on the materials provider --- src/dynamodb_encryption_sdk/encrypted/item.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/dynamodb_encryption_sdk/encrypted/item.py b/src/dynamodb_encryption_sdk/encrypted/item.py index 8ef19631..25f1ef50 100644 --- a/src/dynamodb_encryption_sdk/encrypted/item.py +++ b/src/dynamodb_encryption_sdk/encrypted/item.py @@ -70,7 +70,6 @@ def encrypt_dynamodb_item(item, crypto_config): 'Reserved attribute name "{}" is not allowed in plaintext item.'.format(reserved_name.value) ) - crypto_config.materials_provider.refresh() encryption_materials = crypto_config.encryption_materials() inner_material_description = encryption_materials.material_description.copy() From 120a7ef4c8bc636fa7b19032388b057b225e74ac Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Tue, 15 Jan 2019 21:54:37 -0800 Subject: [PATCH 06/11] fix import fail --- src/dynamodb_encryption_sdk/material_providers/store/meta.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/dynamodb_encryption_sdk/material_providers/store/meta.py b/src/dynamodb_encryption_sdk/material_providers/store/meta.py index bf0ffcc8..c16cd5e7 100644 --- a/src/dynamodb_encryption_sdk/material_providers/store/meta.py +++ b/src/dynamodb_encryption_sdk/material_providers/store/meta.py @@ -11,6 +11,7 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. """Meta cryptographic provider store.""" +import logging from enum import Enum import attr @@ -22,7 +23,7 @@ from dynamodb_encryption_sdk.delegated_keys.jce import JceNameLocalDelegatedKey from dynamodb_encryption_sdk.encrypted.table import EncryptedTable from dynamodb_encryption_sdk.exceptions import InvalidVersionError, NoKnownVersionError, VersionAlreadyExistsError -from dynamodb_encryption_sdk.identifiers import EncryptionKeyType, KeyEncodingType +from dynamodb_encryption_sdk.identifiers import LOGGER_NAME, EncryptionKeyType, KeyEncodingType from dynamodb_encryption_sdk.material_providers import CryptographicMaterialsProvider from dynamodb_encryption_sdk.material_providers.wrapped import WrappedCryptographicMaterialsProvider From 0dbf8d0a1ef38153d855089c4f3d9da98da7b91c Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Wed, 16 Jan 2019 14:29:32 -0800 Subject: [PATCH 07/11] more logging for MostRecentProvider --- .../material_providers/most_recent.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/dynamodb_encryption_sdk/material_providers/most_recent.py b/src/dynamodb_encryption_sdk/material_providers/most_recent.py index 37fee554..80892882 100644 --- a/src/dynamodb_encryption_sdk/material_providers/most_recent.py +++ b/src/dynamodb_encryption_sdk/material_providers/most_recent.py @@ -112,6 +112,7 @@ def get(self, name): def clear(self): # type: () -> None """Clear the cache.""" + _LOGGER.debug("Clearing cache") with self._cache_lock: self._cache = OrderedDict() # type: OrderedDict[Any, Any] # pylint: disable=attribute-defined-outside-init @@ -163,8 +164,10 @@ def decryption_materials(self, encryption_context): """ version = self._provider_store.version_from_material_description(encryption_context.material_description) try: + _LOGGER.debug("Looking in cache for decryption materials provider version %d", version) provider = self._cache.get(version) except KeyError: + _LOGGER.debug("Decryption materials provider not found in cache") try: provider = self._provider_store.provider(self._material_name, version) except InvalidVersionError: @@ -281,10 +284,10 @@ def encryption_materials(self, encryption_context): if ttl_action is TtlActions.LIVE: try: - _LOGGER.debug("Looking in cache for materials provider version %d", self._version) + _LOGGER.debug("Looking in cache for encryption materials provider version %d", self._version) provider = self._cache.get(self._version) except KeyError: - _LOGGER.debug("Provider not found in cache") + _LOGGER.debug("Encryption materials provider not found in cache") ttl_action = TtlActions.EXPIRED if provider is None: From a7448abdbffdc3d06725d7adf77a342e08b5f288 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Wed, 16 Jan 2019 14:30:02 -0800 Subject: [PATCH 08/11] expand cache use testing and add functional tests relying on moto --- test/functional/functional_test_utils.py | 78 ++++++++++++++++++- .../material_providers/test_most_recent.py | 7 ++ test/functional/test_structures.py | 2 +- .../material_providers/test_most_recent.py | 46 +---------- 4 files changed, 84 insertions(+), 49 deletions(-) diff --git a/test/functional/functional_test_utils.py b/test/functional/functional_test_utils.py index df6e5194..6196f50f 100644 --- a/test/functional/functional_test_utils.py +++ b/test/functional/functional_test_utils.py @@ -32,6 +32,7 @@ from dynamodb_encryption_sdk.encrypted.table import EncryptedTable from dynamodb_encryption_sdk.identifiers import CryptoAction from dynamodb_encryption_sdk.internal.identifiers import ReservedAttributes +from dynamodb_encryption_sdk.material_providers.most_recent import MostRecentProvider from dynamodb_encryption_sdk.material_providers.static import StaticCryptographicMaterialsProvider from dynamodb_encryption_sdk.material_providers.store.meta import MetaStore from dynamodb_encryption_sdk.material_providers.wrapped import WrappedCryptographicMaterialsProvider @@ -74,7 +75,7 @@ @pytest.fixture def example_table(): - mock_dynamodb2().start() + mock_dynamodb2().start(reset=False) ddb = boto3.client("dynamodb", region_name="us-west-2") ddb.create_table( TableName=TEST_TABLE_NAME, @@ -94,7 +95,7 @@ def example_table(): @pytest.fixture def table_with_local_seconary_indexes(): - mock_dynamodb2().start() + mock_dynamodb2().start(reset=False) ddb = boto3.client("dynamodb", region_name="us-west-2") ddb.create_table( TableName=TEST_TABLE_NAME, @@ -126,8 +127,8 @@ def table_with_local_seconary_indexes(): @pytest.fixture -def table_with_global_seconary_indexes(): - mock_dynamodb2().start() +def table_with_global_secondary_indexes(): + mock_dynamodb2().start(reset=False) ddb = boto3.client("dynamodb", region_name="us-west-2") ddb.create_table( TableName=TEST_TABLE_NAME, @@ -676,3 +677,72 @@ def mock_metastore(): metastore, table_name = build_metastore() yield metastore delete_metastore(table_name) + + +def _count_entries(records, *messages): + count = 0 + + for record in records: + if all((message in record.getMessage() for message in messages)): + count += 1 + + return count + + +def _count_puts(records, table_name): + return _count_entries(records, '"TableName": "{}"'.format(table_name), "OperationModel(name=PutItem)") + + +def _count_gets(records, table_name): + return _count_entries(records, '"TableName": "{}"'.format(table_name), "OperationModel(name=GetItem)") + + +def check_metastore_cache_use_encrypt(metastore, table_name, log_capture): + table = boto3.resource("dynamodb").Table(table_name) + + most_recent_provider = MostRecentProvider(provider_store=metastore, material_name="test", version_ttl=600.0) + e_table = EncryptedTable( + table=table, + materials_provider=most_recent_provider, + ) + + item = diverse_item() + item.update(TEST_KEY) + e_table.put_item(Item=item) + e_table.put_item(Item=item) + e_table.put_item(Item=item) + e_table.put_item(Item=item) + + try: + primary_puts = _count_puts(log_capture.records, e_table.name) + metastore_puts = _count_puts(log_capture.records, metastore._table.name) + + assert primary_puts == 4 + assert metastore_puts == 1 + + e_table.get_item(Key=TEST_KEY) + e_table.get_item(Key=TEST_KEY) + e_table.get_item(Key=TEST_KEY) + + primary_gets = _count_gets(log_capture.records, e_table.name) + metastore_gets = _count_gets(log_capture.records, metastore._table.name) + metastore_puts = _count_puts(log_capture.records, metastore._table.name) + + assert primary_gets == 3 + assert metastore_gets == 0 + assert metastore_puts == 1 + + most_recent_provider.refresh() + + e_table.get_item(Key=TEST_KEY) + e_table.get_item(Key=TEST_KEY) + e_table.get_item(Key=TEST_KEY) + + primary_gets = _count_gets(log_capture.records, e_table.name) + metastore_gets = _count_gets(log_capture.records, metastore._table.name) + + assert primary_gets == 6 + assert metastore_gets == 1 + + finally: + e_table.delete_item(Key=TEST_KEY) diff --git a/test/functional/material_providers/test_most_recent.py b/test/functional/material_providers/test_most_recent.py index 4d20727a..d7ed32ec 100644 --- a/test/functional/material_providers/test_most_recent.py +++ b/test/functional/material_providers/test_most_recent.py @@ -21,6 +21,9 @@ from dynamodb_encryption_sdk.material_providers.most_recent import MostRecentProvider from dynamodb_encryption_sdk.material_providers.store import ProviderStore +from ..functional_test_utils import mock_metastore, example_table # pylint: disable=unused-import +from ..functional_test_utils import TEST_TABLE_NAME, check_metastore_cache_use_encrypt + pytestmark = [pytest.mark.functional, pytest.mark.local] @@ -130,3 +133,7 @@ def test_decryption_materials_cache_use(): expected_calls.append(("version_from_material_description", 0)) assert store.provider_calls == expected_calls + + +def test_cache_use_encrypt(mock_metastore, example_table, caplog): + check_metastore_cache_use_encrypt(mock_metastore, TEST_TABLE_NAME, caplog) diff --git a/test/functional/test_structures.py b/test/functional/test_structures.py index afe5c0f2..de28ab87 100644 --- a/test/functional/test_structures.py +++ b/test/functional/test_structures.py @@ -21,7 +21,7 @@ from .functional_test_utils import ( TEST_TABLE_NAME, example_table, - table_with_global_seconary_indexes, + table_with_global_secondary_indexes, table_with_local_seconary_indexes, ) diff --git a/test/integration/material_providers/test_most_recent.py b/test/integration/material_providers/test_most_recent.py index 71af7206..9d9c9ba3 100644 --- a/test/integration/material_providers/test_most_recent.py +++ b/test/integration/material_providers/test_most_recent.py @@ -11,58 +11,16 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. """Load testing using MostRecentProvider and MetaStore.""" -import boto3 import pytest -from dynamodb_encryption_sdk.encrypted.table import EncryptedTable -from dynamodb_encryption_sdk.material_providers.most_recent import MostRecentProvider - from ..integration_test_utils import ( # pylint: disable=unused-import ddb_table_name, - functional_test_utils, temp_metastore, ) +from ..integration_test_utils import functional_test_utils pytestmark = [pytest.mark.integ, pytest.mark.ddb_integ] -def count_entries(records, *messages): - count = 0 - - for record in records: - if all((message in record.getMessage() for message in messages)): - count += 1 - - return count - - -def count_puts(records, table_name): - return count_entries(records, '"TableName": "{}"'.format(table_name), "OperationModel(name=PutItem)") - - -def count_gets(records, table_name): - return count_entries(records, '"TableName": "{}"'.format(table_name), "OperationModel(name=GetItem)") - - def test_cache_use_encrypt(temp_metastore, ddb_table_name, caplog): - table = boto3.resource("dynamodb").Table(ddb_table_name) - - e_table = EncryptedTable( - table=table, - materials_provider=MostRecentProvider(provider_store=temp_metastore, material_name="test", version_ttl=600.0), - ) - - item = functional_test_utils.diverse_item() - item.update(functional_test_utils.TEST_KEY) - e_table.put_item(Item=item) - e_table.put_item(Item=item) - e_table.put_item(Item=item) - e_table.put_item(Item=item) - - e_table.delete_item(Key=functional_test_utils.TEST_KEY) - - primary_puts = count_puts(caplog.records, ddb_table_name) - metastore_puts = count_puts(caplog.records, temp_metastore._table.name) - - assert primary_puts == 4 - assert metastore_puts == 1 + functional_test_utils.check_metastore_cache_use_encrypt(temp_metastore, ddb_table_name, caplog) From b0500ab3c7fdc8be350e43c27b425bb6913044a4 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Wed, 16 Jan 2019 14:31:21 -0800 Subject: [PATCH 09/11] autoformat --- test/functional/functional_test_utils.py | 5 +---- test/functional/material_providers/test_most_recent.py | 8 ++++++-- test/integration/material_providers/test_most_recent.py | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/test/functional/functional_test_utils.py b/test/functional/functional_test_utils.py index 6196f50f..4dffa741 100644 --- a/test/functional/functional_test_utils.py +++ b/test/functional/functional_test_utils.py @@ -701,10 +701,7 @@ def check_metastore_cache_use_encrypt(metastore, table_name, log_capture): table = boto3.resource("dynamodb").Table(table_name) most_recent_provider = MostRecentProvider(provider_store=metastore, material_name="test", version_ttl=600.0) - e_table = EncryptedTable( - table=table, - materials_provider=most_recent_provider, - ) + e_table = EncryptedTable(table=table, materials_provider=most_recent_provider) item = diverse_item() item.update(TEST_KEY) diff --git a/test/functional/material_providers/test_most_recent.py b/test/functional/material_providers/test_most_recent.py index d7ed32ec..1d41ef83 100644 --- a/test/functional/material_providers/test_most_recent.py +++ b/test/functional/material_providers/test_most_recent.py @@ -21,8 +21,12 @@ from dynamodb_encryption_sdk.material_providers.most_recent import MostRecentProvider from dynamodb_encryption_sdk.material_providers.store import ProviderStore -from ..functional_test_utils import mock_metastore, example_table # pylint: disable=unused-import -from ..functional_test_utils import TEST_TABLE_NAME, check_metastore_cache_use_encrypt +from ..functional_test_utils import ( # pylint: disable=unused-import + TEST_TABLE_NAME, + check_metastore_cache_use_encrypt, + example_table, + mock_metastore, +) pytestmark = [pytest.mark.functional, pytest.mark.local] diff --git a/test/integration/material_providers/test_most_recent.py b/test/integration/material_providers/test_most_recent.py index 9d9c9ba3..90c2ccf8 100644 --- a/test/integration/material_providers/test_most_recent.py +++ b/test/integration/material_providers/test_most_recent.py @@ -15,9 +15,9 @@ from ..integration_test_utils import ( # pylint: disable=unused-import ddb_table_name, + functional_test_utils, temp_metastore, ) -from ..integration_test_utils import functional_test_utils pytestmark = [pytest.mark.integ, pytest.mark.ddb_integ] From d335a024087652537fb8cbe9311fe5c01060fd8d Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Wed, 16 Jan 2019 14:55:44 -0800 Subject: [PATCH 10/11] fix names and add handling for region selection --- test/functional/functional_test_utils.py | 27 ++++++++++++++---------- test/functional/test_structures.py | 6 +++--- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/test/functional/functional_test_utils.py b/test/functional/functional_test_utils.py index 4dffa741..9353408a 100644 --- a/test/functional/functional_test_utils.py +++ b/test/functional/functional_test_utils.py @@ -23,6 +23,7 @@ import boto3 import pytest from boto3.dynamodb.types import Binary +from botocore.exceptions import NoRegionError from moto import mock_dynamodb2 from dynamodb_encryption_sdk.delegated_keys.jce import JceNameLocalDelegatedKey @@ -43,11 +44,12 @@ RUNNING_IN_TRAVIS = "TRAVIS" in os.environ _DELEGATED_KEY_CACHE = defaultdict(lambda: defaultdict(dict)) TEST_TABLE_NAME = "my_table" +TEST_REGION_NAME = "us-west-2" TEST_INDEX = { "partition_attribute": {"type": "S", "value": "test_value"}, "sort_attribute": {"type": "N", "value": Decimal("99.233")}, } -SECONARY_INDEX = { +SECONDARY_INDEX = { "secondary_index_1": {"type": "B", "value": Binary(b"\x00\x01\x02")}, "secondary_index_2": {"type": "S", "value": "another_value"}, } @@ -76,7 +78,7 @@ @pytest.fixture def example_table(): mock_dynamodb2().start(reset=False) - ddb = boto3.client("dynamodb", region_name="us-west-2") + ddb = boto3.client("dynamodb", region_name=TEST_REGION_NAME) ddb.create_table( TableName=TEST_TABLE_NAME, KeySchema=[ @@ -94,9 +96,9 @@ def example_table(): @pytest.fixture -def table_with_local_seconary_indexes(): +def table_with_local_secondary_indexes(): mock_dynamodb2().start(reset=False) - ddb = boto3.client("dynamodb", region_name="us-west-2") + ddb = boto3.client("dynamodb", region_name=TEST_REGION_NAME) ddb.create_table( TableName=TEST_TABLE_NAME, KeySchema=[ @@ -117,7 +119,7 @@ def table_with_local_seconary_indexes(): ], AttributeDefinitions=[ {"AttributeName": name, "AttributeType": value["type"]} - for name, value in list(TEST_INDEX.items()) + list(SECONARY_INDEX.items()) + for name, value in list(TEST_INDEX.items()) + list(SECONDARY_INDEX.items()) ], ProvisionedThroughput={"ReadCapacityUnits": 100, "WriteCapacityUnits": 100}, ) @@ -129,7 +131,7 @@ def table_with_local_seconary_indexes(): @pytest.fixture def table_with_global_secondary_indexes(): mock_dynamodb2().start(reset=False) - ddb = boto3.client("dynamodb", region_name="us-west-2") + ddb = boto3.client("dynamodb", region_name=TEST_REGION_NAME) ddb.create_table( TableName=TEST_TABLE_NAME, KeySchema=[ @@ -152,7 +154,7 @@ def table_with_global_secondary_indexes(): ], AttributeDefinitions=[ {"AttributeName": name, "AttributeType": value["type"]} - for name, value in list(TEST_INDEX.items()) + list(SECONARY_INDEX.items()) + for name, value in list(TEST_INDEX.items()) + list(SECONDARY_INDEX.items()) ], ProvisionedThroughput={"ReadCapacityUnits": 100, "WriteCapacityUnits": 100}, ) @@ -649,19 +651,19 @@ def client_cycle_batch_items_check_paginators( def build_metastore(): - client = boto3.client("dynamodb", region_name="us-west-2") + client = boto3.client("dynamodb", region_name=TEST_REGION_NAME) table_name = base64.urlsafe_b64encode(os.urandom(32)).decode("utf-8").replace("=", ".") MetaStore.create_table(client, table_name, 1, 1) waiter = client.get_waiter("table_exists") waiter.wait(TableName=table_name) - table = boto3.resource("dynamodb", region_name="us-west-2").Table(table_name) + table = boto3.resource("dynamodb", region_name=TEST_REGION_NAME).Table(table_name) return MetaStore(table, build_static_jce_cmp("AES", 256, "HmacSHA256", 256)), table_name def delete_metastore(table_name): - client = boto3.client("dynamodb", region_name="us-west-2") + client = boto3.client("dynamodb", region_name=TEST_REGION_NAME) client.delete_table(TableName=table_name) # It sometimes takes a long time to delete a table. # If hanging, asynchronously deleting tables becomes an issue, @@ -698,7 +700,10 @@ def _count_gets(records, table_name): def check_metastore_cache_use_encrypt(metastore, table_name, log_capture): - table = boto3.resource("dynamodb").Table(table_name) + try: + table = boto3.resource("dynamodb").Table(table_name) + except NoRegionError: + table = boto3.resource("dynamodb", region_name=TEST_REGION_NAME).Table(table_name) most_recent_provider = MostRecentProvider(provider_store=metastore, material_name="test", version_ttl=600.0) e_table = EncryptedTable(table=table, materials_provider=most_recent_provider) diff --git a/test/functional/test_structures.py b/test/functional/test_structures.py index de28ab87..0c180fe9 100644 --- a/test/functional/test_structures.py +++ b/test/functional/test_structures.py @@ -22,7 +22,7 @@ TEST_TABLE_NAME, example_table, table_with_global_secondary_indexes, - table_with_local_seconary_indexes, + table_with_local_secondary_indexes, ) pytestmark = [pytest.mark.functional, pytest.mark.local] @@ -37,14 +37,14 @@ def test_tableinfo_refresh_indexes_no_secondary_indexes(example_table): table.refresh_indexed_attributes(client) -def test_tableinfo_refresh_indexes_with_gsis(table_with_global_seconary_indexes): +def test_tableinfo_refresh_indexes_with_gsis(table_with_global_secondary_indexes): client = boto3.client("dynamodb", region_name="us-west-2") table = TableInfo(name=TEST_TABLE_NAME) table.refresh_indexed_attributes(client) -def test_tableinfo_refresh_indexes_with_lsis(table_with_local_seconary_indexes): +def test_tableinfo_refresh_indexes_with_lsis(table_with_local_secondary_indexes): client = boto3.client("dynamodb", region_name="us-west-2") table = TableInfo(name=TEST_TABLE_NAME) From becfc89342f552c61b59017988fe66215ab1ef68 Mon Sep 17 00:00:00 2001 From: mattsb42-aws Date: Wed, 16 Jan 2019 14:57:36 -0800 Subject: [PATCH 11/11] add #105 to changelog --- CHANGELOG.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0d5009cf..e4476b60 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,14 @@ Changelog ********* +1.0.7 -- 2018-01-xx +=================== + +Bugfixes +-------- +* Fix :class:`MostRecentProvider` cache reuse bug. + `#105 `_ + 1.0.6 -- 2018-01-15 ===================