Merge head of master into keyring (#195)

* Update PR template

* Added a check for max_age being greater than 0 (#172)

* Added a check for max_age being greater than 0

* Fixed flake8 by adding missing pydocstyle dependency

* Added the dependency to decrypt_oracle as well

* Added test for max_age<=0 ValueError

* Updated test for max_age<=0.0 ValueError

* Added negative test case

* Fixed KMS master key provider tests when default AWS region is configured (#179)

* Fixed KMS master key provider tests for users who have their default AWS region configured

* created fixture for botocore session with no region set

* add auto-used fixture in KMS master key provider unit tests to test against both with and without default region

* Wrote example and test for using one kms cmk with an unsigned algorithm

* Update one_kms_cmk_unsigned.py

* Update examples/src/one_kms_cmk_unsigned.py

Co-Authored-By: Matt Bullock <[email protected]>

* isort-check now succeeds

* [issue-190] Regional clients modify default botocore session (#193)

* [issue-190] Creation of regional clients modifies default botocore session's region

* update changelog with changes for 1.4.1 release

* bump version to 1.4.1

* Updates to handle new pylint requirements (#196)

* pylint max-attributes appears to be ratcheted down recently

* remove unnecessary comprehensions

* whitelist some pylint use-constant-test false-positives

* reorganize backwards compatibility test requirements definitions
attrs==19.2.0 removed a deprecated feature that aws-encryption-sdk==1.3.3 depended on.
This reorganization lets us define specific requirements bounds for old versions of aws-encryption-sdk
that will probably continue to be necessary as these old versions age.

* remove unnecessary comprehensions

* add newlines to the end of all requirements files

* help pylint ignore mypy type use

Author: mattsb42-aws

.github/PULL_REQUEST_TEMPLATE.md

@@ -4,3 +4,7 @@
 
 
 By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
+
+# Check any applicable:
+- [ ] Were any files moved? Moving files changes their URL, which breaks all hyperlinks to the files.
+

CHANGELOG.rst

@@ -2,6 +2,23 @@
 Changelog
 *********
 
+1.4.1 -- 2019-09-20
+===================
+
+Bugfixes
+--------
+
+* Fix region configuration override in botocore sessions.
+  `#190 <https://github.com/aws/aws-encryption-sdk-python/issues/190>`_
+  `#193 <https://github.com/aws/aws-encryption-sdk-python/pull/193>`_
+
+Minor
+-----
+
+* Caching CMM must require that max age configuration value is greater than 0.
+  `#147 <https://github.com/aws/aws-encryption-sdk-python/issues/147>`_
+  `#172 <https://github.com/aws/aws-encryption-sdk-python/pull/172>`_
+
 1.4.0 -- 2019-05-23
 ===================
 

decrypt_oracle/setup.py

@@ -22,7 +22,7 @@ def get_version():
 def get_requirements():
     """Read the requirements file."""
     requirements = read("requirements-actual.txt")
-    return [r for r in requirements.strip().splitlines()]
+    return list(requirements.strip().splitlines())
 
 
 setup(

decrypt_oracle/tox.ini

@@ -156,7 +156,7 @@ basepython = python3
 deps =
     flake8
     flake8-docstrings
-    pydocstyle < 4.0.0
+    pydocstyle<4.0.0
     # https://github.com/JBKahn/flake8-print/pull/30
     flake8-print>=3.1.0
 commands =

doc/requirements.txt

@@ -1,2 +1,2 @@
 sphinx>=1.3.0
-sphinx_rtd_theme
+sphinx_rtd_theme

examples/src/one_kms_cmk_unsigned.py

@@ -0,0 +1,51 @@
+# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""Example showing basic encryption and decryption of a value already in memory
+using one KMS CMK with an unsigned algorithm.
+"""
+from aws_encryption_sdk import KMSMasterKeyProvider, decrypt, encrypt
+from aws_encryption_sdk.identifiers import Algorithm
+
+
+def encrypt_decrypt(key_arn, source_plaintext, botocore_session=None):
+    """Encrypts and then decrypts a string under one KMS customer master key (CMK) with an unsigned algorithm.
+
+    :param str key_arn: Amazon Resource Name (ARN) of the KMS CMK
+    :param bytes source_plaintext: Data to encrypt
+    :param botocore_session: existing botocore session instance
+    :type botocore_session: botocore.session.Session
+    """
+    kwargs = dict(key_ids=[key_arn])
+
+    if botocore_session is not None:
+        kwargs["botocore_session"] = botocore_session
+
+    # Create master key provider using the ARN of the key and the session (botocore_session)
+    kms_key_provider = KMSMasterKeyProvider(**kwargs)
+
+    # Encrypt the plaintext using the AWS Encryption SDK. It returns the encrypted message and the header
+    ciphertext, encrypted_message_header = encrypt(
+        algorithm=Algorithm.AES_256_GCM_IV12_TAG16_HKDF_SHA256, source=source_plaintext, key_provider=kms_key_provider
+    )
+
+    # Decrypt the encrypted message using the AWS Encryption SDK. It returns the decrypted message and the header
+    plaintext, decrypted_message_header = decrypt(source=ciphertext, key_provider=kms_key_provider)
+
+    # Check if the original message and the decrypted message are the same
+    assert source_plaintext == plaintext
+
+    # Check if the headers of the encrypted message and decrypted message match
+    assert all(
+        pair in encrypted_message_header.encryption_context.items()
+        for pair in decrypted_message_header.encryption_context.items()
+    )

examples/test/test_i_one_kms_cmk_unsigned.py

@@ -0,0 +1,29 @@
+# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""Unit test suite for the encryption and decryption using one KMS CMK with an unsigned algorithm example."""
+
+import botocore.session
+import pytest
+
+from ..src.one_kms_cmk_unsigned import encrypt_decrypt
+from .examples_test_utils import get_cmk_arn
+from .examples_test_utils import static_plaintext
+
+
+pytestmark = [pytest.mark.examples]
+
+
+def test_one_kms_cmk_unsigned():
+    plaintext = static_plaintext
+    cmk_arn = get_cmk_arn()
+    encrypt_decrypt(key_arn=cmk_arn, source_plaintext=plaintext, botocore_session=botocore.session.Session())

requirements.txt

@@ -2,4 +2,4 @@ six
 boto3>=1.4.4
 cryptography>=1.8.1
 attrs>=19.1.0
-wrapt>=1.10.11
+wrapt>=1.10.11

setup.py

@@ -22,7 +22,7 @@ def get_version():
 def get_requirements():
     """Reads the requirements file."""
     requirements = read("requirements.txt")
-    return [r for r in requirements.strip().splitlines()]
+    return list(requirements.strip().splitlines())
 
 
 setup(

src/aws_encryption_sdk/caches/__init__.py

@@ -143,6 +143,7 @@ class CryptoMaterialsCacheEntryHints(object):
 
 @attr.s(hash=False)
 class CryptoMaterialsCacheEntry(object):
+    # pylint: disable=too-many-instance-attributes
     """Value and metadata store for cryptographic materials cache entries.
 
     :param bytes cache_key: Identifier for entries in cache

src/aws_encryption_sdk/identifiers.py

@@ -27,7 +27,7 @@
     # We only actually need these imports when running the mypy checks
     pass
 
-__version__ = "1.4.0"
+__version__ = "1.4.1"
 USER_AGENT_SUFFIX = "AwsEncryptionSdkPython/{}".format(__version__)
 
 

src/aws_encryption_sdk/key_providers/kms.py

@@ -161,8 +161,8 @@ def add_regional_client(self, region_name):
         :param str region_name: AWS Region ID (ex: us-east-1)
         """
         if region_name not in self._regional_clients:
-            session = boto3.session.Session(region_name=region_name, botocore_session=self.config.botocore_session)
-            client = session.client("kms", config=self._user_agent_adding_config)
+            session = boto3.session.Session(botocore_session=self.config.botocore_session)
+            client = session.client("kms", region_name=region_name, config=self._user_agent_adding_config)
             self._register_client(client, region_name)
             self._regional_clients[region_name] = client
 

src/aws_encryption_sdk/keyring/base.py

@@ -11,8 +11,13 @@
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
 """Base class interface for Keyrings."""
-from aws_encryption_sdk.materials_managers import DecryptionMaterials, EncryptionMaterials
-from aws_encryption_sdk.structures import EncryptedDataKey
+from aws_encryption_sdk.materials_managers import (  # only used for mypy; pylint: disable=unused-import,duplicate-code
+    DecryptionMaterials,
+    EncryptionMaterials,
+)
+from aws_encryption_sdk.structures import (  # only used for mypy; pylint: disable=unused-import,duplicate-code
+    EncryptedDataKey,
+)
 
 try:  # Python 3.5.0 and 3.5.1 have incompatible typing modules
     from typing import Iterable  # noqa pylint: disable=unused-import

src/aws_encryption_sdk/keyring/multi_keyring.py

@@ -17,7 +17,10 @@
 from attr.validators import deep_iterable, instance_of, optional
 
 from aws_encryption_sdk.exceptions import EncryptKeyError, GenerateKeyError
-from aws_encryption_sdk.keyring.base import DecryptionMaterials, EncryptedDataKey, EncryptionMaterials, Keyring
+from aws_encryption_sdk.keyring.base import DecryptionMaterials  # only used for mypy so pylint: disable=unused-import
+from aws_encryption_sdk.keyring.base import EncryptionMaterials  # only used for mypy so pylint: disable=unused-import
+from aws_encryption_sdk.keyring.base import Keyring
+from aws_encryption_sdk.structures import EncryptedDataKey  # only used for mypy so pylint: disable=unused-import
 
 try:  # Python 3.5.0 and 3.5.1 have incompatible typing modules
     from typing import Iterable  # noqa pylint: disable=unused-import

src/aws_encryption_sdk/keyring/raw_keyring.py

@@ -29,8 +29,12 @@
 from aws_encryption_sdk.internal.formatting.serialize import serialize_raw_master_key_prefix, serialize_wrapped_key
 from aws_encryption_sdk.key_providers.raw import RawMasterKey
 from aws_encryption_sdk.keyring.base import Keyring
-from aws_encryption_sdk.materials_managers import DecryptionMaterials, EncryptionMaterials
-from aws_encryption_sdk.structures import EncryptedDataKey, KeyringTrace, MasterKeyInfo, RawDataKey
+from aws_encryption_sdk.materials_managers import (  # only used for mypy so pylint: disable=unused-import
+    DecryptionMaterials,
+    EncryptionMaterials,
+)
+from aws_encryption_sdk.structures import EncryptedDataKey  # only used for mypy so pylint: disable=unused-import
+from aws_encryption_sdk.structures import KeyringTrace, MasterKeyInfo, RawDataKey
 
 try:  # Python 3.5.0 and 3.5.1 have incompatible typing modules
     from typing import Iterable  # noqa pylint: disable=unused-import

src/aws_encryption_sdk/materials_managers/caching.py

@@ -108,6 +108,9 @@ def __attrs_post_init__(self):
         if self.max_bytes_encrypted > MAX_BYTES_PER_KEY:
             raise ValueError("max_bytes_encrypted cannot exceed {}".format(MAX_BYTES_PER_KEY))
 
+        if self.max_age <= 0.0:
+            raise ValueError("max_age cannot be less than or equal to 0")
+
         if self.backing_materials_manager is None:
             if self.master_key_provider is None:
                 raise TypeError("Either backing_materials_manager or master_key_provider must be defined")

src/aws_encryption_sdk/streaming_client.py

@@ -235,7 +235,7 @@ def read(self, b=-1):
         if not self._message_prepped:
             self._prep_message()
 
-        if self.closed:
+        if self.closed:  # dynamic values confuse pylint: disable=using-constant-test
             raise ValueError("I/O operation on closed file")
 
         if b >= 0:
@@ -283,7 +283,7 @@ def readline(self):
 
     def readlines(self):
         """Reads all chunks of output, outputting a list as defined in the IOBase specification."""
-        return [line for line in self]
+        return list(self)
 
     def __iter__(self):
         """Make this class and subclasses identify as iterators."""
@@ -292,7 +292,7 @@ def __iter__(self):
     def next(self):
         """Provides hook for Python2 iterator functionality."""
         _LOGGER.debug("reading next")
-        if self.closed:
+        if self.closed:  # dynamic values confuse pylint: disable=using-constant-test
             _LOGGER.debug("stream is closed")
             raise StopIteration()
 

src/aws_encryption_sdk/structures.py

@@ -117,6 +117,7 @@ class KeyringTrace(object):
 
 @attr.s(hash=True)
 class MessageHeader(object):
+    # pylint: disable=too-many-instance-attributes
     """Deserialized message header object.
 
     :param version: Message format version, per spec

test/integration/integration_test_utils.py

@@ -13,10 +13,13 @@
 """Utility functions to handle configuration and credentials setup for integration tests."""
 import os
 
+import botocore.session
+
 from aws_encryption_sdk.key_providers.kms import KMSMasterKeyProvider
 
 AWS_KMS_KEY_ID = "AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID"
 _KMS_MKP = None
+_KMS_MKP_BOTO = None
 
 
 def get_cmk_arn():
@@ -47,3 +50,19 @@ def setup_kms_master_key_provider(cache=True):
         _KMS_MKP = kms_master_key_provider
 
     return kms_master_key_provider
+
+
+def setup_kms_master_key_provider_with_botocore_session(cache=True):
+    """Reads the test_values config file and builds the requested KMS Master Key Provider with botocore_session."""
+    global _KMS_MKP_BOTO  # pylint: disable=global-statement
+    if cache and _KMS_MKP_BOTO is not None:
+        return _KMS_MKP_BOTO
+
+    cmk_arn = get_cmk_arn()
+    kms_master_key_provider = KMSMasterKeyProvider(botocore_session=botocore.session.Session())
+    kms_master_key_provider.add_master_key(cmk_arn)
+
+    if cache:
+        _KMS_MKP_BOTO = kms_master_key_provider
+
+    return kms_master_key_provider

test/integration/test_i_aws_encrytion_sdk_client.py

@@ -21,7 +21,11 @@
 from aws_encryption_sdk.identifiers import USER_AGENT_SUFFIX, Algorithm
 from aws_encryption_sdk.key_providers.kms import KMSMasterKey, KMSMasterKeyProvider
 
-from .integration_test_utils import get_cmk_arn, setup_kms_master_key_provider
+from .integration_test_utils import (
+    get_cmk_arn,
+    setup_kms_master_key_provider,
+    setup_kms_master_key_provider_with_botocore_session,
+)
 
 pytestmark = [pytest.mark.integ]
 
@@ -68,6 +72,15 @@ def test_remove_bad_client():
     assert not test._regional_clients
 
 
+def test_regional_client_does_not_modify_botocore_session(caplog):
+    mkp = setup_kms_master_key_provider_with_botocore_session()
+    fake_region = "us-fakey-12"
+
+    assert mkp.config.botocore_session.get_config_variable("region") != fake_region
+    mkp.add_regional_client(fake_region)
+    assert mkp.config.botocore_session.get_config_variable("region") != fake_region
+
+
 class TestKMSThickClientIntegration(object):
     @pytest.fixture(autouse=True)
     def apply_fixtures(self):

test/requirements.txt

@@ -1,4 +1,4 @@
 mock
 pytest>=3.3.1
 pytest-cov
-pytest-mock
+pytest-mock

test/unit/test_identifiers.py

@@ -41,7 +41,7 @@ def test_algorithm_safe_to_cache(check_algorithm, safe_to_cache):
         assert not check_algorithm.safe_to_cache()
 
 
-@pytest.mark.parametrize("suite", [suite for suite in EncryptionSuite])
+@pytest.mark.parametrize("suite", list(EncryptionSuite))
 def test_encryption_suite_invalid_kdf(suite):
     mock_kdf = Mock()
     mock_kdf.input_length.return_value = 1

test/unit/test_material_managers_caching.py

@@ -117,6 +117,8 @@ def test_mkp_to_default_cmm(mocker):
             dict(max_bytes_encrypted=MAX_BYTES_PER_KEY + 1),
             r"max_bytes_encrypted cannot exceed {}".format(MAX_BYTES_PER_KEY),
         ),
+        (dict(max_age=0.0), r"max_age cannot be less than or equal to 0"),
+        (dict(max_age=-1.0), r"max_age cannot be less than or equal to 0"),
     ),
 )
 def test_invalid_values(invalid_kwargs, error_message):