expand cache use testing and add functional tests relying on moto

Author: mattsb42-aws

test/functional/functional_test_utils.py

@@ -32,6 +32,7 @@
 from dynamodb_encryption_sdk.encrypted.table import EncryptedTable
 from dynamodb_encryption_sdk.identifiers import CryptoAction
 from dynamodb_encryption_sdk.internal.identifiers import ReservedAttributes
+from dynamodb_encryption_sdk.material_providers.most_recent import MostRecentProvider
 from dynamodb_encryption_sdk.material_providers.static import StaticCryptographicMaterialsProvider
 from dynamodb_encryption_sdk.material_providers.store.meta import MetaStore
 from dynamodb_encryption_sdk.material_providers.wrapped import WrappedCryptographicMaterialsProvider
@@ -74,7 +75,7 @@
 
 @pytest.fixture
 def example_table():
-    mock_dynamodb2().start()
+    mock_dynamodb2().start(reset=False)
     ddb = boto3.client("dynamodb", region_name="us-west-2")
     ddb.create_table(
         TableName=TEST_TABLE_NAME,
@@ -94,7 +95,7 @@ def example_table():
 
 @pytest.fixture
 def table_with_local_seconary_indexes():
-    mock_dynamodb2().start()
+    mock_dynamodb2().start(reset=False)
     ddb = boto3.client("dynamodb", region_name="us-west-2")
     ddb.create_table(
         TableName=TEST_TABLE_NAME,
@@ -126,8 +127,8 @@ def table_with_local_seconary_indexes():
 
 
 @pytest.fixture
-def table_with_global_seconary_indexes():
-    mock_dynamodb2().start()
+def table_with_global_secondary_indexes():
+    mock_dynamodb2().start(reset=False)
     ddb = boto3.client("dynamodb", region_name="us-west-2")
     ddb.create_table(
         TableName=TEST_TABLE_NAME,
@@ -676,3 +677,72 @@ def mock_metastore():
         metastore, table_name = build_metastore()
         yield metastore
         delete_metastore(table_name)
+
+
+def _count_entries(records, *messages):
+    count = 0
+
+    for record in records:
+        if all((message in record.getMessage() for message in messages)):
+            count += 1
+
+    return count
+
+
+def _count_puts(records, table_name):
+    return _count_entries(records, '"TableName": "{}"'.format(table_name), "OperationModel(name=PutItem)")
+
+
+def _count_gets(records, table_name):
+    return _count_entries(records, '"TableName": "{}"'.format(table_name), "OperationModel(name=GetItem)")
+
+
+def check_metastore_cache_use_encrypt(metastore, table_name, log_capture):
+    table = boto3.resource("dynamodb").Table(table_name)
+
+    most_recent_provider = MostRecentProvider(provider_store=metastore, material_name="test", version_ttl=600.0)
+    e_table = EncryptedTable(
+        table=table,
+        materials_provider=most_recent_provider,
+    )
+
+    item = diverse_item()
+    item.update(TEST_KEY)
+    e_table.put_item(Item=item)
+    e_table.put_item(Item=item)
+    e_table.put_item(Item=item)
+    e_table.put_item(Item=item)
+
+    try:
+        primary_puts = _count_puts(log_capture.records, e_table.name)
+        metastore_puts = _count_puts(log_capture.records, metastore._table.name)
+
+        assert primary_puts == 4
+        assert metastore_puts == 1
+
+        e_table.get_item(Key=TEST_KEY)
+        e_table.get_item(Key=TEST_KEY)
+        e_table.get_item(Key=TEST_KEY)
+
+        primary_gets = _count_gets(log_capture.records, e_table.name)
+        metastore_gets = _count_gets(log_capture.records, metastore._table.name)
+        metastore_puts = _count_puts(log_capture.records, metastore._table.name)
+
+        assert primary_gets == 3
+        assert metastore_gets == 0
+        assert metastore_puts == 1
+
+        most_recent_provider.refresh()
+
+        e_table.get_item(Key=TEST_KEY)
+        e_table.get_item(Key=TEST_KEY)
+        e_table.get_item(Key=TEST_KEY)
+
+        primary_gets = _count_gets(log_capture.records, e_table.name)
+        metastore_gets = _count_gets(log_capture.records, metastore._table.name)
+
+        assert primary_gets == 6
+        assert metastore_gets == 1
+
+    finally:
+        e_table.delete_item(Key=TEST_KEY)

test/functional/material_providers/test_most_recent.py

@@ -21,6 +21,9 @@
 from dynamodb_encryption_sdk.material_providers.most_recent import MostRecentProvider
 from dynamodb_encryption_sdk.material_providers.store import ProviderStore
 
+from ..functional_test_utils import mock_metastore, example_table  # pylint: disable=unused-import
+from ..functional_test_utils import TEST_TABLE_NAME, check_metastore_cache_use_encrypt
+
 pytestmark = [pytest.mark.functional, pytest.mark.local]
 
 
@@ -130,3 +133,7 @@ def test_decryption_materials_cache_use():
     expected_calls.append(("version_from_material_description", 0))
 
     assert store.provider_calls == expected_calls
+
+
+def test_cache_use_encrypt(mock_metastore, example_table, caplog):
+    check_metastore_cache_use_encrypt(mock_metastore, TEST_TABLE_NAME, caplog)

test/functional/test_structures.py

@@ -21,7 +21,7 @@
 from .functional_test_utils import (
     TEST_TABLE_NAME,
     example_table,
-    table_with_global_seconary_indexes,
+    table_with_global_secondary_indexes,
     table_with_local_seconary_indexes,
 )
 

test/integration/material_providers/test_most_recent.py

@@ -11,58 +11,16 @@
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
 """Load testing using MostRecentProvider and MetaStore."""
-import boto3
 import pytest
 
-from dynamodb_encryption_sdk.encrypted.table import EncryptedTable
-from dynamodb_encryption_sdk.material_providers.most_recent import MostRecentProvider
-
 from ..integration_test_utils import (  # pylint: disable=unused-import
     ddb_table_name,
-    functional_test_utils,
     temp_metastore,
 )
+from ..integration_test_utils import functional_test_utils
 
 pytestmark = [pytest.mark.integ, pytest.mark.ddb_integ]
 
 
-def count_entries(records, *messages):
-    count = 0
-
-    for record in records:
-        if all((message in record.getMessage() for message in messages)):
-            count += 1
-
-    return count
-
-
-def count_puts(records, table_name):
-    return count_entries(records, '"TableName": "{}"'.format(table_name), "OperationModel(name=PutItem)")
-
-
-def count_gets(records, table_name):
-    return count_entries(records, '"TableName": "{}"'.format(table_name), "OperationModel(name=GetItem)")
-
-
 def test_cache_use_encrypt(temp_metastore, ddb_table_name, caplog):
-    table = boto3.resource("dynamodb").Table(ddb_table_name)
-
-    e_table = EncryptedTable(
-        table=table,
-        materials_provider=MostRecentProvider(provider_store=temp_metastore, material_name="test", version_ttl=600.0),
-    )
-
-    item = functional_test_utils.diverse_item()
-    item.update(functional_test_utils.TEST_KEY)
-    e_table.put_item(Item=item)
-    e_table.put_item(Item=item)
-    e_table.put_item(Item=item)
-    e_table.put_item(Item=item)
-
-    e_table.delete_item(Key=functional_test_utils.TEST_KEY)
-
-    primary_puts = count_puts(caplog.records, ddb_table_name)
-    metastore_puts = count_puts(caplog.records, temp_metastore._table.name)
-
-    assert primary_puts == 4
-    assert metastore_puts == 1
+    functional_test_utils.check_metastore_cache_use_encrypt(temp_metastore, ddb_table_name, caplog)