From 7360edd46964ae0a0d7127a1cf51bd180973af33 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 31 Jan 2024 15:18:21 -0800 Subject: [PATCH 001/422] passing hierarchy keyring example --- examples/src/basic_encryption.py | 98 +++++++++++++++++-- requirements.txt | 2 +- setup.py | 8 ++ .../internal/crypto/authentication.py | 8 +- src/aws_encryption_sdk/streaming_client.py | 83 +++++++++++++--- 5 files changed, 179 insertions(+), 20 deletions(-) diff --git a/examples/src/basic_encryption.py b/examples/src/basic_encryption.py index cfe8ac791..f48f7e4a1 100644 --- a/examples/src/basic_encryption.py +++ b/examples/src/basic_encryption.py @@ -13,7 +13,40 @@ """Example showing basic encryption and decryption of a value already in memory.""" import aws_encryption_sdk from aws_encryption_sdk import CommitmentPolicy +import aws_cryptographic_materialproviders +import boto3 +from aws_encryption_sdk.cmm_handler import (CMMHandler) + +import sys + +module_root_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(module_root_dir) + +import aws_cryptographic_materialproviders + +from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.client import AwsCryptographicMaterialProviders +from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.config import MaterialProvidersConfig +from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.models import ( + CreateAwsKmsHierarchicalKeyringInput, + CacheTypeDefault, + DefaultCache, + GetBranchKeyIdInput, + GetBranchKeyIdOutput, + CreateDefaultCryptographicMaterialsManagerInput, +) +from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.references import ( + IKeyring, + IBranchKeyIdSupplier, +) + +from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_keystore.client import KeyStore +from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_keystore.config import KeyStoreConfig +from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_keystore.models import ( + CreateKeyInput, + KMSConfigurationKmsKeyArn, +) def cycle_string(key_arn, source_plaintext, botocore_session=None): """Encrypts and then decrypts a string under a KMS customer master key (CMK). @@ -25,23 +58,72 @@ def cycle_string(key_arn, source_plaintext, botocore_session=None): """ # Set up an encryption client with an explicit commitment policy. Note that if you do not explicitly choose a # commitment policy, REQUIRE_ENCRYPT_REQUIRE_DECRYPT is used by default. - client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + client = aws_encryption_sdk.EncryptionSDKClient() # Create a KMS master key provider. Note that because we are planning on decrypting using this same provider, # we MUST provide the ARN of the KMS Key. If we provide a raw key id or a key alias, decryption will fail. kms_kwargs = dict(key_ids=[key_arn]) if botocore_session is not None: kms_kwargs["botocore_session"] = botocore_session - master_key_provider = aws_encryption_sdk.StrictAwsKmsMasterKeyProvider(**kms_kwargs) + # master_key_provider = aws_encryption_sdk.StrictAwsKmsMasterKeyProvider(**kms_kwargs) + + ##### + + + key_store_table_name="KeyStoreDdbTable" + logical_key_store_name="KeyStoreDdbTable" + keystore_kms_key_id="arn:aws:kms:us-west-2:370957321024:key/9d989aa2-2f9c-438c-a745-cc57d3ad0126" + + ddb_client = boto3.client('dynamodb') + kms_client = boto3.client('kms') + + keystore: KeyStore = KeyStore( + config=KeyStoreConfig( + ddb_client=ddb_client, + ddb_table_name=key_store_table_name, + logical_key_store_name=logical_key_store_name, + kms_client=kms_client, + kms_configuration=KMSConfigurationKmsKeyArn(value=keystore_kms_key_id), + ) + ) + + new_branch_key_id: str = keystore.create_key(input=CreateKeyInput()).branch_key_identifier + print(f"DEBUG: {new_branch_key_id=}") + + mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + config=MaterialProvidersConfig() + ) + + keyring_input: CreateAwsKmsHierarchicalKeyringInput = CreateAwsKmsHierarchicalKeyringInput( + key_store=keystore, + branch_key_id=new_branch_key_id, + ttl_seconds=600, + cache=CacheTypeDefault(value=DefaultCache(entry_capacity=100)), + ) + + hierarchical_keyring: IKeyring = mat_prov.create_aws_kms_hierarchical_keyring( + input=keyring_input + ) + # This is as far as we can go in the linked Java example without the ESDK. + # We can't use this keyring until it's integrated with the ESDK :( + # Peek at it with print statement for now + print(f"DEBUG: {hierarchical_keyring=}") + + ##### + + cmm = mat_prov.create_default_cryptographic_materials_manager(CreateDefaultCryptographicMaterialsManagerInput(keyring=hierarchical_keyring)) + + cmm_handler: CMMHandler = CMMHandler(cmm) # Encrypt the plaintext source data - ciphertext, encryptor_header = client.encrypt(source=source_plaintext, key_provider=master_key_provider) + ciphertext, encryptor_header = client.encrypt(source=source_plaintext, materials_manager=cmm_handler) # Decrypt the ciphertext - cycled_plaintext, decrypted_header = client.decrypt(source=ciphertext, key_provider=master_key_provider) + cycled_plaintext, decrypted_header = client.decrypt(source=ciphertext, materials_manager=cmm_handler) + cycled_plaintext_str = str(cycled_plaintext, encoding="ascii") # Verify that the "cycled" (encrypted, then decrypted) plaintext is identical to the source plaintext - assert cycled_plaintext == source_plaintext + assert cycled_plaintext_str == source_plaintext # Verify that the encryption context used in the decrypt operation includes all key pairs from # the encrypt operation. (The SDK can add pairs, so don't require an exact match.) @@ -49,5 +131,9 @@ def cycle_string(key_arn, source_plaintext, botocore_session=None): # In production, always use a meaningful encryption context. In this sample, we omit the # encryption context (no key pairs). assert all( - pair in decrypted_header.encryption_context.items() for pair in encryptor_header.encryption_context.items() + (str(k, encoding="ascii"), str(v, encoding="ascii")) in decrypted_header.encryption_context.items() for (k, v) in encryptor_header.encryption_context.items() ) + +# hack in a test +import botocore +cycle_string("arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f", "abcdefg", botocore_session=botocore.session.Session()) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 2f4323845..13466216c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ boto3>=1.10.0 cryptography>=3.4.0 attrs>=17.4.0 -wrapt>=1.10.11 +wrapt>=1.16.0 diff --git a/setup.py b/setup.py index 7cc111bac..cafc16979 100644 --- a/setup.py +++ b/setup.py @@ -39,6 +39,11 @@ def get_requirements(): keywords="aws-encryption-sdk aws kms encryption", license="Apache License 2.0", install_requires=get_requirements(), + # TODO: Point at main once Python is merged into main. + # PyPI will not accept a package that declares dependencies using direct URLs. + extras_require={ + "MPL": ["aws-cryptographic-material-providers @ git+https://github.com/aws/aws-cryptographic-material-providers-library.git@lucmcdon/python-mpl#subdirectory=AwsCryptographicMaterialProviders/runtimes/python"], + }, classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", @@ -49,6 +54,9 @@ def get_requirements(): "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: Implementation :: CPython", "Topic :: Security", "Topic :: Security :: Cryptography", diff --git a/src/aws_encryption_sdk/internal/crypto/authentication.py b/src/aws_encryption_sdk/internal/crypto/authentication.py index f90ac77e0..ad5cf1b2a 100644 --- a/src/aws_encryption_sdk/internal/crypto/authentication.py +++ b/src/aws_encryption_sdk/internal/crypto/authentication.py @@ -76,7 +76,8 @@ def from_key_bytes(cls, algorithm, key_bytes): :param bytes key_bytes: Raw signing key :rtype: aws_encryption_sdk.internal.crypto.Signer """ - key = serialization.load_der_private_key(data=key_bytes, password=None, backend=default_backend()) + # key = serialization.load_der_private_key(data=key_bytes, password=None, backend=default_backend()) + key = serialization.load_pem_private_key(data=key_bytes, password=None, backend=default_backend()) return cls(algorithm, key) def key_bytes(self): @@ -140,6 +141,7 @@ def from_encoded_point(cls, algorithm, encoded_point): :returns: Instance of Verifier generated from encoded point :rtype: aws_encryption_sdk.internal.crypto.Verifier """ + print(f"from_encoded_point {encoded_point=}") return cls( algorithm=algorithm, key=_ecc_public_numbers_from_compressed_point( @@ -157,8 +159,10 @@ def from_key_bytes(cls, algorithm, key_bytes): :returns: Instance of Verifier generated from encoded point :rtype: aws_encryption_sdk.internal.crypto.Verifier """ + print(f"{algorithm=}") + print(f"{key_bytes=}") return cls( - algorithm=algorithm, key=serialization.load_der_public_key(data=key_bytes, backend=default_backend()) + algorithm=algorithm, key=serialization.load_pem_public_key(data=key_bytes, backend=default_backend()) ) def key_bytes(self): diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 1119cb740..afed52e0f 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -66,6 +66,21 @@ from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager from aws_encryption_sdk.materials_managers.default import DefaultCryptoMaterialsManager from aws_encryption_sdk.structures import MessageHeader +try: + import aws_cryptographic_materialproviders + from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.client import AwsCryptographicMaterialProviders + from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.config import MaterialProvidersConfig + from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.models import ( + CreateDefaultCryptographicMaterialsManagerInput + ) + from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.references import ( + IKeyring, + ) + from aws_encryption_sdk.cmm_handler import CMMHandler + + _has_mpl = True +except ImportError as e: + _has_mpl = False _LOGGER = logging.getLogger(__name__) @@ -113,6 +128,10 @@ class _ClientConfig(object): # pylint: disable=too-many-instance-attributes key_provider = attr.ib( hash=True, default=None, validator=attr.validators.optional(attr.validators.instance_of(MasterKeyProvider)) ) + if _has_mpl: + keyring = attr.ib( + hash=True, default=None, validator=attr.validators.optional(attr.validators.instance_of(IKeyring)) + ) source_length = attr.ib( hash=True, default=None, validator=attr.validators.optional(attr.validators.instance_of(six.integer_types)) ) @@ -122,13 +141,38 @@ class _ClientConfig(object): # pylint: disable=too-many-instance-attributes def __attrs_post_init__(self): """Normalize inputs to crypto material manager.""" - both_cmm_and_mkp_defined = self.materials_manager is not None and self.key_provider is not None - neither_cmm_nor_mkp_defined = self.materials_manager is None and self.key_provider is None + if _has_mpl: + all_cmm_and_mkp_and_keyring_defined = all([ + self.materials_manager is not None, + self.key_provider is not None, + self.keyring is not None, + ]) + none_cmm_nor_mkp_nor_keyring_defined = all([ + self.materials_manager is None, + self.key_provider is None, + self.keyring is None, + ]) + + if all_cmm_and_mkp_and_keyring_defined or none_cmm_nor_mkp_nor_keyring_defined: + raise TypeError("Exactly one of keyring, materials_manager, or key_provider must be provided") + if self.materials_manager is None: + if self.key_provider is not None: + self.materials_manager = DefaultCryptoMaterialsManager(master_key_provider=self.key_provider) + elif self.keyring is not None: + mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + config=MaterialProvidersConfig() + ) + cmm = mat_prov.create_default_cryptographic_materials_manager(CreateDefaultCryptographicMaterialsManagerInput(keyring=self.keyring)) + cmm_handler: CryptoMaterialsManager = CMMHandler(cmm) + self.materials_manager = cmm_handler + elif not _has_mpl: + both_cmm_and_mkp_defined = self.materials_manager is not None and self.key_provider is not None + neither_cmm_nor_mkp_defined = self.materials_manager is None and self.key_provider is None - if both_cmm_and_mkp_defined or neither_cmm_nor_mkp_defined: - raise TypeError("Exactly one of materials_manager or key_provider must be provided") - if self.materials_manager is None: - self.materials_manager = DefaultCryptoMaterialsManager(master_key_provider=self.key_provider) + if both_cmm_and_mkp_defined or neither_cmm_nor_mkp_defined: + raise TypeError("Exactly one of materials_manager or key_provider must be provided") + if self.materials_manager is None: + self.materials_manager = DefaultCryptoMaterialsManager(master_key_provider=self.key_provider) class _EncryptionStream(io.IOBase): @@ -343,6 +387,8 @@ class EncryptorConfig(_ClientConfig): :param key_provider: `MasterKeyProvider` from which to obtain data keys for encryption (either `materials_manager` or `key_provider` required) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider + :param keyring: `IKeyring` TODO-MPL content + :type keyring: TODO-MPL :param int source_length: Length of source data (optional) .. note:: @@ -394,6 +440,8 @@ class StreamEncryptor(_EncryptionStream): # pylint: disable=too-many-instance-a :param key_provider: `MasterKeyProvider` from which to obtain data keys for encryption (either `materials_manager` or `key_provider` required) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider + :param keyring: `IKeyring` TODO-MPL content + :type keyring: TODO-MPL :param int source_length: Length of source data (optional) .. note:: @@ -729,11 +777,13 @@ class DecryptorConfig(_ClientConfig): :param source: Source data to encrypt or decrypt :type source: str, bytes, io.IOBase, or file :param materials_manager: `CryptoMaterialsManager` from which to obtain cryptographic materials - (either `materials_manager` or `key_provider` required) + (either `keyring`, `materials_manager` or `key_provider` required) :type materials_manager: aws_encryption_sdk.materials_managers.base.CryptoMaterialsManager :param key_provider: `MasterKeyProvider` from which to obtain data keys for decryption - (either `materials_manager` or `key_provider` required) + (either `keyring`, `materials_manager` or `key_provider` required) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider + :param keyring: `IKeyring` TODO-MPL content + :type keyring: TODO-MPL :param int source_length: Length of source data (optional) .. note:: @@ -770,6 +820,8 @@ class StreamDecryptor(_EncryptionStream): # pylint: disable=too-many-instance-a :param key_provider: `MasterKeyProvider` from which to obtain data keys for decryption (either `materials_manager` or `key_provider` required) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider + :param keyring: `IKeyring` TODO-MPL content + :type keyring: TODO-MPL :param int source_length: Length of source data (optional) .. note:: @@ -831,9 +883,18 @@ def _read_header(self): if decryption_materials.verification_key is None: self.verifier = None else: - self.verifier = Verifier.from_key_bytes( - algorithm=header.algorithm, key_bytes=decryption_materials.verification_key - ) + # MPL verification key is NOT key bytes, it is bytes of the compressed point + # TODO-MPL: clean this up, least-privilege violation + import base64 + if hasattr(self.config.materials_manager, "mpl_cmm"): + self.verifier = Verifier.from_encoded_point( + algorithm=header.algorithm, + encoded_point=base64.b64encode(decryption_materials.verification_key) + ) + else: + self.verifier = Verifier.from_key_bytes( + algorithm=header.algorithm, key_bytes=decryption_materials.verification_key + ) if self.verifier is not None: self.verifier.update(raw_header) From 53c46ece22fec60a2d4d653a4720f66fe706ccca Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 2 Feb 2024 09:19:14 -0800 Subject: [PATCH 002/422] cleanup --- examples/src/basic_encryption.py | 98 +--------------- requirements.txt | 2 +- setup.py | 1 - .../internal/crypto/authentication.py | 3 - src/aws_encryption_sdk/streaming_client.py | 105 +++++++++++------- 5 files changed, 74 insertions(+), 135 deletions(-) diff --git a/examples/src/basic_encryption.py b/examples/src/basic_encryption.py index f48f7e4a1..cfe8ac791 100644 --- a/examples/src/basic_encryption.py +++ b/examples/src/basic_encryption.py @@ -13,40 +13,7 @@ """Example showing basic encryption and decryption of a value already in memory.""" import aws_encryption_sdk from aws_encryption_sdk import CommitmentPolicy -import aws_cryptographic_materialproviders -import boto3 -from aws_encryption_sdk.cmm_handler import (CMMHandler) - -import sys - -module_root_dir = '/'.join(__file__.split("/")[:-1]) - -sys.path.append(module_root_dir) - -import aws_cryptographic_materialproviders - -from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.client import AwsCryptographicMaterialProviders -from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.config import MaterialProvidersConfig -from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.models import ( - CreateAwsKmsHierarchicalKeyringInput, - CacheTypeDefault, - DefaultCache, - GetBranchKeyIdInput, - GetBranchKeyIdOutput, - CreateDefaultCryptographicMaterialsManagerInput, -) -from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.references import ( - IKeyring, - IBranchKeyIdSupplier, -) - -from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_keystore.client import KeyStore -from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_keystore.config import KeyStoreConfig -from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_keystore.models import ( - CreateKeyInput, - KMSConfigurationKmsKeyArn, -) def cycle_string(key_arn, source_plaintext, botocore_session=None): """Encrypts and then decrypts a string under a KMS customer master key (CMK). @@ -58,72 +25,23 @@ def cycle_string(key_arn, source_plaintext, botocore_session=None): """ # Set up an encryption client with an explicit commitment policy. Note that if you do not explicitly choose a # commitment policy, REQUIRE_ENCRYPT_REQUIRE_DECRYPT is used by default. - client = aws_encryption_sdk.EncryptionSDKClient() + client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) # Create a KMS master key provider. Note that because we are planning on decrypting using this same provider, # we MUST provide the ARN of the KMS Key. If we provide a raw key id or a key alias, decryption will fail. kms_kwargs = dict(key_ids=[key_arn]) if botocore_session is not None: kms_kwargs["botocore_session"] = botocore_session - # master_key_provider = aws_encryption_sdk.StrictAwsKmsMasterKeyProvider(**kms_kwargs) - - ##### - - - key_store_table_name="KeyStoreDdbTable" - logical_key_store_name="KeyStoreDdbTable" - keystore_kms_key_id="arn:aws:kms:us-west-2:370957321024:key/9d989aa2-2f9c-438c-a745-cc57d3ad0126" - - ddb_client = boto3.client('dynamodb') - kms_client = boto3.client('kms') - - keystore: KeyStore = KeyStore( - config=KeyStoreConfig( - ddb_client=ddb_client, - ddb_table_name=key_store_table_name, - logical_key_store_name=logical_key_store_name, - kms_client=kms_client, - kms_configuration=KMSConfigurationKmsKeyArn(value=keystore_kms_key_id), - ) - ) - - new_branch_key_id: str = keystore.create_key(input=CreateKeyInput()).branch_key_identifier - print(f"DEBUG: {new_branch_key_id=}") - - mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( - config=MaterialProvidersConfig() - ) - - keyring_input: CreateAwsKmsHierarchicalKeyringInput = CreateAwsKmsHierarchicalKeyringInput( - key_store=keystore, - branch_key_id=new_branch_key_id, - ttl_seconds=600, - cache=CacheTypeDefault(value=DefaultCache(entry_capacity=100)), - ) - - hierarchical_keyring: IKeyring = mat_prov.create_aws_kms_hierarchical_keyring( - input=keyring_input - ) - # This is as far as we can go in the linked Java example without the ESDK. - # We can't use this keyring until it's integrated with the ESDK :( - # Peek at it with print statement for now - print(f"DEBUG: {hierarchical_keyring=}") - - ##### - - cmm = mat_prov.create_default_cryptographic_materials_manager(CreateDefaultCryptographicMaterialsManagerInput(keyring=hierarchical_keyring)) - - cmm_handler: CMMHandler = CMMHandler(cmm) + master_key_provider = aws_encryption_sdk.StrictAwsKmsMasterKeyProvider(**kms_kwargs) # Encrypt the plaintext source data - ciphertext, encryptor_header = client.encrypt(source=source_plaintext, materials_manager=cmm_handler) + ciphertext, encryptor_header = client.encrypt(source=source_plaintext, key_provider=master_key_provider) # Decrypt the ciphertext - cycled_plaintext, decrypted_header = client.decrypt(source=ciphertext, materials_manager=cmm_handler) - cycled_plaintext_str = str(cycled_plaintext, encoding="ascii") + cycled_plaintext, decrypted_header = client.decrypt(source=ciphertext, key_provider=master_key_provider) # Verify that the "cycled" (encrypted, then decrypted) plaintext is identical to the source plaintext - assert cycled_plaintext_str == source_plaintext + assert cycled_plaintext == source_plaintext # Verify that the encryption context used in the decrypt operation includes all key pairs from # the encrypt operation. (The SDK can add pairs, so don't require an exact match.) @@ -131,9 +49,5 @@ def cycle_string(key_arn, source_plaintext, botocore_session=None): # In production, always use a meaningful encryption context. In this sample, we omit the # encryption context (no key pairs). assert all( - (str(k, encoding="ascii"), str(v, encoding="ascii")) in decrypted_header.encryption_context.items() for (k, v) in encryptor_header.encryption_context.items() + pair in decrypted_header.encryption_context.items() for pair in encryptor_header.encryption_context.items() ) - -# hack in a test -import botocore -cycle_string("arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f", "abcdefg", botocore_session=botocore.session.Session()) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 13466216c..2f4323845 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ boto3>=1.10.0 cryptography>=3.4.0 attrs>=17.4.0 -wrapt>=1.16.0 +wrapt>=1.10.11 diff --git a/setup.py b/setup.py index cafc16979..8ffd74015 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,6 @@ def get_requirements(): "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", "Programming Language :: Python :: Implementation :: CPython", "Topic :: Security", "Topic :: Security :: Cryptography", diff --git a/src/aws_encryption_sdk/internal/crypto/authentication.py b/src/aws_encryption_sdk/internal/crypto/authentication.py index ad5cf1b2a..b9692eb16 100644 --- a/src/aws_encryption_sdk/internal/crypto/authentication.py +++ b/src/aws_encryption_sdk/internal/crypto/authentication.py @@ -141,7 +141,6 @@ def from_encoded_point(cls, algorithm, encoded_point): :returns: Instance of Verifier generated from encoded point :rtype: aws_encryption_sdk.internal.crypto.Verifier """ - print(f"from_encoded_point {encoded_point=}") return cls( algorithm=algorithm, key=_ecc_public_numbers_from_compressed_point( @@ -159,8 +158,6 @@ def from_key_bytes(cls, algorithm, key_bytes): :returns: Instance of Verifier generated from encoded point :rtype: aws_encryption_sdk.internal.crypto.Verifier """ - print(f"{algorithm=}") - print(f"{key_bytes=}") return cls( algorithm=algorithm, key=serialization.load_pem_public_key(data=key_bytes, backend=default_backend()) ) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index afed52e0f..176b92334 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -139,40 +139,61 @@ class _ClientConfig(object): # pylint: disable=too-many-instance-attributes hash=True, default=LINE_LENGTH, validator=attr.validators.instance_of(six.integer_types) ) # DEPRECATED: Value is no longer configurable here. Parameter left here to avoid breaking consumers. + def _has_mpl_attrs_post_init(self): + + def _exactly_one_arg_is_not_None(*args): + ''' + Private helper function. + Returns `True` if exactly one item in the list is not `None`. + Returns `False` otherwise. + ''' + # Have not found any `not None` + found_one = False + for arg in args: + if arg is not None: + if found_one == False: + # Have not already found a `not None`, found a `not None` => only one `not None` (so far) + found_one = True + else: + # Already found a `not None`, found another `not None` => not exactly one `not None` + return False + return found_one + + if not _exactly_one_arg_is_not_None(self.materials_manager, self.key_provider, self.keyring): + raise TypeError("Exactly one of keyring, materials_manager, or key_provider must be provided") + if self.materials_manager is None: + if self.key_provider is not None: + # No CMM, provided (legacy) native `key_provider` => create (legacy) native DefaultCryptoMaterialsManager + self.materials_manager = DefaultCryptoMaterialsManager(master_key_provider=self.key_provider) + elif self.keyring is not None: + # No CMM, provided MPL keyring => create MPL's DefaultCryptographicMaterialsManager + try: + assert isinstance(self.keyring, IKeyring) + except AssertionError as e: + raise ValueError(f"Argument provided to keyring MUST be a {IKeyring}. Found {keyring.__class__.__name__=}") + + mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + config=MaterialProvidersConfig() + ) + cmm = mat_prov.create_default_cryptographic_materials_manager(CreateDefaultCryptographicMaterialsManagerInput(keyring=self.keyring)) + cmm_handler: CryptoMaterialsManager = CMMHandler(cmm) + self.materials_manager = cmm_handler + + def _no_mpl_attrs_post_init(self): + both_cmm_and_mkp_defined = self.materials_manager is not None and self.key_provider is not None + neither_cmm_nor_mkp_defined = self.materials_manager is None and self.key_provider is None + + if both_cmm_and_mkp_defined or neither_cmm_nor_mkp_defined: + raise TypeError("Exactly one of materials_manager or key_provider must be provided") + if self.materials_manager is None: + self.materials_manager = DefaultCryptoMaterialsManager(master_key_provider=self.key_provider) + def __attrs_post_init__(self): """Normalize inputs to crypto material manager.""" if _has_mpl: - all_cmm_and_mkp_and_keyring_defined = all([ - self.materials_manager is not None, - self.key_provider is not None, - self.keyring is not None, - ]) - none_cmm_nor_mkp_nor_keyring_defined = all([ - self.materials_manager is None, - self.key_provider is None, - self.keyring is None, - ]) - - if all_cmm_and_mkp_and_keyring_defined or none_cmm_nor_mkp_nor_keyring_defined: - raise TypeError("Exactly one of keyring, materials_manager, or key_provider must be provided") - if self.materials_manager is None: - if self.key_provider is not None: - self.materials_manager = DefaultCryptoMaterialsManager(master_key_provider=self.key_provider) - elif self.keyring is not None: - mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( - config=MaterialProvidersConfig() - ) - cmm = mat_prov.create_default_cryptographic_materials_manager(CreateDefaultCryptographicMaterialsManagerInput(keyring=self.keyring)) - cmm_handler: CryptoMaterialsManager = CMMHandler(cmm) - self.materials_manager = cmm_handler + self._has_mpl_attrs_post_init() elif not _has_mpl: - both_cmm_and_mkp_defined = self.materials_manager is not None and self.key_provider is not None - neither_cmm_nor_mkp_defined = self.materials_manager is None and self.key_provider is None - - if both_cmm_and_mkp_defined or neither_cmm_nor_mkp_defined: - raise TypeError("Exactly one of materials_manager or key_provider must be provided") - if self.materials_manager is None: - self.materials_manager = DefaultCryptoMaterialsManager(master_key_provider=self.key_provider) + self._no_mpl_attrs_post_init() class _EncryptionStream(io.IOBase): @@ -387,8 +408,10 @@ class EncryptorConfig(_ClientConfig): :param key_provider: `MasterKeyProvider` from which to obtain data keys for encryption (either `materials_manager` or `key_provider` required) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider - :param keyring: `IKeyring` TODO-MPL content - :type keyring: TODO-MPL + :param keyring: `IKeyring` from the aws_cryptographic_materialproviders library + which handles encryption and decryption + :type keyring: + aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.references.IKeyring :param int source_length: Length of source data (optional) .. note:: @@ -440,8 +463,10 @@ class StreamEncryptor(_EncryptionStream): # pylint: disable=too-many-instance-a :param key_provider: `MasterKeyProvider` from which to obtain data keys for encryption (either `materials_manager` or `key_provider` required) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider - :param keyring: `IKeyring` TODO-MPL content - :type keyring: TODO-MPL + :param keyring: `IKeyring` from the aws_cryptographic_materialproviders library + which handles encryption and decryption + :type keyring: + aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.references.IKeyring :param int source_length: Length of source data (optional) .. note:: @@ -782,8 +807,10 @@ class DecryptorConfig(_ClientConfig): :param key_provider: `MasterKeyProvider` from which to obtain data keys for decryption (either `keyring`, `materials_manager` or `key_provider` required) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider - :param keyring: `IKeyring` TODO-MPL content - :type keyring: TODO-MPL + :param keyring: `IKeyring` from the aws_cryptographic_materialproviders library + which handles encryption and decryption + :type keyring: + aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.references.IKeyring :param int source_length: Length of source data (optional) .. note:: @@ -820,8 +847,10 @@ class StreamDecryptor(_EncryptionStream): # pylint: disable=too-many-instance-a :param key_provider: `MasterKeyProvider` from which to obtain data keys for decryption (either `materials_manager` or `key_provider` required) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider - :param keyring: `IKeyring` TODO-MPL content - :type keyring: TODO-MPL + :param keyring: `IKeyring` from the aws_cryptographic_materialproviders library + which handles encryption and decryption + :type keyring: + aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.references.IKeyring :param int source_length: Length of source data (optional) .. note:: From 3f5a503ab7f866c7750cabdc79dbf7f8f75a34e6 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 2 Feb 2024 09:21:46 -0800 Subject: [PATCH 003/422] add --- setup.py | 3 +- src/aws_encryption_sdk/cmm_handler.py | 138 ++++++++++++++++ src/aws_encryption_sdk/materials_handlers.py | 165 +++++++++++++++++++ 3 files changed, 304 insertions(+), 2 deletions(-) create mode 100644 src/aws_encryption_sdk/cmm_handler.py create mode 100644 src/aws_encryption_sdk/materials_handlers.py diff --git a/setup.py b/setup.py index 8ffd74015..c4c277096 100644 --- a/setup.py +++ b/setup.py @@ -39,8 +39,7 @@ def get_requirements(): keywords="aws-encryption-sdk aws kms encryption", license="Apache License 2.0", install_requires=get_requirements(), - # TODO: Point at main once Python is merged into main. - # PyPI will not accept a package that declares dependencies using direct URLs. + # TODO: Point at MPL main branch once Python MPL is merged into main. extras_require={ "MPL": ["aws-cryptographic-material-providers @ git+https://github.com/aws/aws-cryptographic-material-providers-library.git@lucmcdon/python-mpl#subdirectory=AwsCryptographicMaterialProviders/runtimes/python"], }, diff --git a/src/aws_encryption_sdk/cmm_handler.py b/src/aws_encryption_sdk/cmm_handler.py new file mode 100644 index 000000000..82aad2248 --- /dev/null +++ b/src/aws_encryption_sdk/cmm_handler.py @@ -0,0 +1,138 @@ +# These dependencies are only loaded if you install the MPL. +try: + from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.references import ( + ICryptographicMaterialsManager, + ) + from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.models import ( + GetEncryptionMaterialsInput, + GetEncryptionMaterialsOutput, + DecryptMaterialsInput, + DecryptMaterialsOutput, + EncryptedDataKey as MPL_EncryptedDataKey, + CommitmentPolicyESDK, + AlgorithmSuiteIdESDK, + ) +except ImportError as e: + print(f"WARNING: MPL import failed with {e=}") + +from aws_encryption_sdk.materials_managers import ( + DecryptionMaterialsRequest, + EncryptionMaterialsRequest, +) +from aws_encryption_sdk.materials_managers.base import ( + CryptoMaterialsManager, +) +from aws_encryption_sdk.materials_handlers import ( + EncryptionMaterialsHandler, + DecryptionMaterialsHandler, +) +from aws_encryption_sdk.structures import ( + EncryptedDataKey as Native_EncryptedDataKey, +) +from aws_encryption_sdk.identifiers import ( + Algorithm, + AlgorithmSuite, + CommitmentPolicy, +) + +# TODO-MPL Should this implement interface..? seems like yes since it implements all of interface methods +class CMMHandler(CryptoMaterialsManager): + native_cmm: CryptoMaterialsManager + mpl_cmm: 'ICryptographicMaterialsManager' + + def __init__( + self, + cmm: 'CryptoMaterialsManager | ICryptographicMaterialsManager' + ): + if isinstance(cmm, CryptoMaterialsManager): + self.native_cmm = cmm + elif isinstance(cmm, ICryptographicMaterialsManager): + self.mpl_cmm = cmm + else: + raise ValueError(f"Invalid CMM passed to CMMHander: {cmm=}") + + def get_encryption_materials( + self, + request: EncryptionMaterialsRequest + ) -> EncryptionMaterialsHandler: + ''' + Returns an EncryptionMaterialsHandler based on the configured CMM. + ''' + if (hasattr(self, "native_cmm") and not hasattr(self, "mpl_cmm")): + return EncryptionMaterialsHandler(self.native_cmm.get_encryption_materials(request)) + else: + input: GetEncryptionMaterialsInput = CMMHandler._create_mpl_get_encryption_materials_input_from_request(request) + print(f"get_encryption_materials {input=}") + output: GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(input) + print(f"get_encryption_materials {output=}") + return EncryptionMaterialsHandler(output.encryption_materials) + + @staticmethod + def _create_mpl_get_encryption_materials_input_from_request( + request: EncryptionMaterialsRequest + ) -> 'GetEncryptionMaterialsInput': + print(f"_create_mpl_get_encryption_materials_input_from_request {request=}") + print(f"{CMMHandler._map_native_commitment_policy_to_mpl_commitment_policy(request.commitment_policy)=}") + print(f"_create_mpl_get_encryption_materials_input_from_request {request.encryption_context=}") + output: GetEncryptionMaterialsInput = GetEncryptionMaterialsInput( + encryption_context=request.encryption_context, + commitment_policy=CMMHandler._map_native_commitment_policy_to_mpl_commitment_policy(request.commitment_policy), + # TODO double check this + # optional... maybe this needs to be kwargs?? + # algorithm_suite_id=request.algorithm.algorithm_id, + max_plaintext_length=request.plaintext_length, + ) + print(f"_create_mpl_get_encryption_materials_input_from_request {output=}") + return output + + @staticmethod + def _map_native_commitment_policy_to_mpl_commitment_policy( + native_commitment_policy: CommitmentPolicy + ) -> CommitmentPolicyESDK: + if native_commitment_policy == CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT: + return CommitmentPolicyESDK(value="FORBID_ENCRYPT_ALLOW_DECRYPT") + elif native_commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT: + return CommitmentPolicyESDK(value="REQUIRE_ENCRYPT_ALLOW_DECRYPT") + elif native_commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT: + return CommitmentPolicyESDK(value="REQUIRE_ENCRYPT_REQUIRE_DECRYPT") + else: + raise ValueError(f"Invalid {native_commitment_policy=}") + + def decrypt_materials( + self, + request: DecryptionMaterialsRequest + ) -> DecryptionMaterialsHandler: + ''' + Returns a DecryptionMaterialsHandler based on the configured CMM. + ''' + print(f"decrypt_materials {request=}") + if (hasattr(self, "native_cmm") and not hasattr(self, "mpl_cmm")): + return DecryptionMaterialsHandler(self.native_cmm.decrypt_materials(request)) + else: + input: 'DecryptMaterialsInput' = CMMHandler._create_mpl_decrypt_materials_input_from_request(request) + output: 'DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(input) + print(f"decrypt_materials {output.decryption_materials.verification_key=}") + return DecryptionMaterialsHandler(output.decryption_materials) + + @staticmethod + def _native_algorithm_id_to_mpl_algorithm_id(native_algorithm_id: str) -> AlgorithmSuiteIdESDK: + # MPL algorithm suite ID = hexstr(native_algorithm_id) padded to 4 digits post-`x`. + return AlgorithmSuiteIdESDK(f"{native_algorithm_id:#0{6}x}") + + @staticmethod + def _create_mpl_decrypt_materials_input_from_request( + request: DecryptionMaterialsRequest + ) -> 'DecryptMaterialsInput': + key_blob_list: list[Native_EncryptedDataKey] = request.encrypted_data_keys + list_edks = [MPL_EncryptedDataKey( + key_provider_id=key_blob.key_provider.provider_id, + key_provider_info=key_blob.key_provider.key_info, + ciphertext=key_blob.encrypted_data_key, + ) for key_blob in key_blob_list] + output: DecryptMaterialsInput = DecryptMaterialsInput( + algorithm_suite_id=CMMHandler._native_algorithm_id_to_mpl_algorithm_id(request.algorithm.algorithm_id), + commitment_policy=CMMHandler._map_native_commitment_policy_to_mpl_commitment_policy(request.commitment_policy), + encrypted_data_keys=list_edks, + encryption_context=request.encryption_context, + ) + return output diff --git a/src/aws_encryption_sdk/materials_handlers.py b/src/aws_encryption_sdk/materials_handlers.py new file mode 100644 index 000000000..bf3073ad3 --- /dev/null +++ b/src/aws_encryption_sdk/materials_handlers.py @@ -0,0 +1,165 @@ +# These dependencies are only loaded if you install the MPL. +try: + from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.models import ( + DecryptionMaterials as MPL_DecryptionMaterials, + EncryptionMaterials as MPL_EncryptionMaterials, + EncryptedDataKey as MPL_EncryptedDataKey, + ) +except ImportError as e: + pass + +from aws_encryption_sdk.materials_managers import ( + DecryptionMaterials as Native_DecryptionMaterials, + EncryptionMaterials as Native_EncryptionMaterials, +) +from aws_encryption_sdk.identifiers import ( + Algorithm, + AlgorithmSuite, +) +from aws_encryption_sdk.structures import ( + DataKey, + EncryptedDataKey as Native_EncryptedDataKey, + MasterKeyInfo, +) +from aws_encryption_sdk.internal.crypto.authentication import ( + Signer +) + +class EncryptionMaterialsHandler: + native_materials: Native_EncryptionMaterials + mpl_materials: 'MPL_EncryptionMaterials' + + @staticmethod + def _mpl_algorithm_id_to_native_algorithm_id(mpl_algorithm_id: str): + # MPL algorithm suite ID == "ALG_" + native algorithm suite ID. + return int(mpl_algorithm_id, 16) + + def __init__( + self, + materials: 'Native_EncryptionMaterials | MPL_EncryptionMaterials' + ): + if isinstance(materials, Native_EncryptionMaterials): + self.native_materials = materials + elif isinstance(materials, MPL_EncryptionMaterials): + self.mpl_materials = materials + else: + raise ValueError(f"Invalid EncryptionMaterials passed to EncryptionMaterialsHandler: {materials=}") + @property + def algorithm(self) -> Algorithm: + if hasattr(self, "native_materials"): + return self.native_materials.algorithm + else: + print(f"algorithm {self.mpl_materials.algorithm_suite.id.value=}") + return AlgorithmSuite.get_by_id( + EncryptionMaterialsHandler._mpl_algorithm_id_to_native_algorithm_id( + self.mpl_materials.algorithm_suite.id.value + ) + ) + + @property + def encryption_context(self) -> dict[str, str]: + if hasattr(self, "native_materials"): + return self.native_materials.encryption_context + else: + return self.mpl_materials.encryption_context + + @property + def encrypted_data_keys(self) -> list[Native_EncryptedDataKey]: + if hasattr(self, "native_materials"): + return self.native_materials.encrypted_data_keys + else: + mpl_edk_list: list[MPL_EncryptedDataKey] = self.mpl_materials.encrypted_data_keys + key_blob_list: set[Native_EncryptedDataKey] = {Native_EncryptedDataKey( + key_provider=MasterKeyInfo( + provider_id=mpl_edk.key_provider_id, + key_info=mpl_edk.key_provider_info, + ), + encrypted_data_key=mpl_edk.ciphertext, + ) for mpl_edk in mpl_edk_list} + return key_blob_list + + @property + def data_encryption_key(self) -> DataKey: + if hasattr(self, "native_materials"): + return self.native_materials.data_encryption_key + else: + # TODO-MPL This impl is probably wrong + mpl_dek = self.mpl_materials.plaintext_data_key + return DataKey( + # key_provider=None, # No MasterKeyInfo object for plaintext data key + key_provider=MasterKeyInfo( + provider_id="", + key_info=b'' + ), + data_key=mpl_dek, + encrypted_data_key=b'', # No encrypted DEK + ) + + @property + def signing_key(self) -> bytes: + if hasattr(self, "native_materials"): + return self.native_materials.signing_key + else: + print(f"sign {self.mpl_materials.signing_key=}") + return self.mpl_materials.signing_key + # if self.mpl_materials.signing_key is None: + # return Signer.from_key_bytes( + # algorithm=AlgorithmSuite.get_by_id(self.mpl_materials.algorithm_suite.id.value), + # bytes=self.mpl_materials.signing_key + # ) + + def get_required_encryption_context_keys(self) -> list[str]: + if hasattr(self, "native_materials"): + return [] + else: + return self.mpl_materials.required_encryption_context_keys + +class DecryptionMaterialsHandler: + native_materials: Native_DecryptionMaterials + mpl_materials: 'MPL_DecryptionMaterials' + + def __init__( + self, + materials: 'Native_DecryptionMaterials | MPL_DecryptionMaterials' + ): + if isinstance(materials, Native_DecryptionMaterials): + self.native_materials = materials + elif isinstance(materials, MPL_DecryptionMaterials): + self.mpl_materials = materials + else: + raise ValueError(f"Invalid DecryptionMaterials passed to DecryptionMaterialsHandler: {materials=}") + + def get_encryption_context(self) -> dict[str, str]: + if hasattr(self, "native_materials"): + return {} # TODO-MPL This impl is probably wrong + else: + return self.mpl_materials.encryption_context + + @property + def data_key(self) -> DataKey: + if hasattr(self, "native_materials"): + return self.native_materials.data_key + else: + # TODO-MPL This impl is probably wrong + return DataKey( + key_provider=MasterKeyInfo( + provider_id="", + key_info=b'' + ), + data_key=self.mpl_materials.plaintext_data_key, + encrypted_data_key=b'', + ) + + @property + def verification_key(self) -> bytes: + if hasattr(self, "native_materials"): + return self.native_materials.verification_key + else: + print(f"ver {self.mpl_materials.verification_key=}") + return self.mpl_materials.verification_key + + def get_required_encryption_context_keys(self) -> list[str]: + if hasattr(self, "native_materials"): + return [] + else: + return self.mpl_materials.required_encryption_context_keys \ No newline at end of file From 16cf5c1f38ae1aaa0e7e5f68fc58cd0c6f3532b2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 2 Feb 2024 14:41:53 -0800 Subject: [PATCH 004/422] changes, cleanup: --- examples/src/basic_encryption.py | 5 + examples/src/keyrings/hierarchical_keyring.py | 248 ++++++++++++++++++ examples/src/keyrings/module_.py | 0 examples/src/module_.py | 0 src/aws_encryption_sdk/cmm_handler.py | 55 ++-- src/aws_encryption_sdk/materials_handlers.py | 56 ++-- src/aws_encryption_sdk/streaming_client.py | 18 +- .../test_streaming_client_stream_decryptor.py | 2 +- 8 files changed, 314 insertions(+), 70 deletions(-) create mode 100644 examples/src/keyrings/hierarchical_keyring.py create mode 100644 examples/src/keyrings/module_.py create mode 100644 examples/src/module_.py diff --git a/examples/src/basic_encryption.py b/examples/src/basic_encryption.py index cfe8ac791..7b729feab 100644 --- a/examples/src/basic_encryption.py +++ b/examples/src/basic_encryption.py @@ -51,3 +51,8 @@ def cycle_string(key_arn, source_plaintext, botocore_session=None): assert all( pair in decrypted_header.encryption_context.items() for pair in encryptor_header.encryption_context.items() ) + +cycle_string( + "arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f", + "abcdefg", +) \ No newline at end of file diff --git a/examples/src/keyrings/hierarchical_keyring.py b/examples/src/keyrings/hierarchical_keyring.py new file mode 100644 index 000000000..e8f662b73 --- /dev/null +++ b/examples/src/keyrings/hierarchical_keyring.py @@ -0,0 +1,248 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + + + +"""Example showing basic encryption and decryption of a value already in memory.""" +import aws_encryption_sdk +from aws_encryption_sdk import CommitmentPolicy +import boto3 + +import sys + +from aws_encryption_sdk.exceptions import ( + AWSEncryptionSDKClientError, + SerializationError, +) + +module_root_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(module_root_dir) + +import aws_cryptographic_materialproviders + +from aws_cryptographic_materialproviders.mpl.client import AwsCryptographicMaterialProviders +from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig +from aws_cryptographic_materialproviders.mpl.models import ( + CreateAwsKmsHierarchicalKeyringInput, + CacheTypeDefault, + DefaultCache, + GetBranchKeyIdInput, + GetBranchKeyIdOutput, +) +from aws_cryptographic_materialproviders.mpl.references import ( + IKeyring, + IBranchKeyIdSupplier, +) + +from aws_cryptographic_materialproviders.keystore.client import KeyStore +from aws_cryptographic_materialproviders.keystore.config import KeyStoreConfig +from aws_cryptographic_materialproviders.keystore.models import ( + CreateKeyInput, + KMSConfigurationKmsKeyArn, +) + +EXAMPLE_DATA: bytes = b"Hello World" + +def encrypt_and_decrypt_with_keyring( + key_store_table_name: str, + logical_key_store_name: str, + kms_key_id: str + ): + + # 1. Instantiate the encryption SDK client. + # This builds the client with the REQUIRE_ENCRYPT_REQUIRE_DECRYPT commitment policy, + # which enforces that this client only encrypts using committing algorithm suites and enforces + # that this client will only decrypt encrypted messages that were created with a committing + # algorithm suite. + # This is the default commitment policy if you were to build the client as + # `client = aws_encryption_sdk.EncryptionSDKClient()`. + + client = aws_encryption_sdk.EncryptionSDKClient( + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT + ) + + # 2. Create boto3 clients for DynamoDB and KMS. + ddb_client = boto3.client('dynamodb') + kms_client = boto3.client('kms') + + # 3. Configure your KeyStore resource. + # This SHOULD be the same configuration that you used + # to initially create and populate your KeyStore. + keystore: KeyStore = KeyStore( + config=KeyStoreConfig( + ddb_client=ddb_client, + ddb_table_name=key_store_table_name, + logical_key_store_name=logical_key_store_name, + kms_client=kms_client, + kms_configuration=KMSConfigurationKmsKeyArn( + value=kms_key_id + ), + ) + ) + + # 4. Call CreateKey to create two new active branch keys + branch_key_id_A: str = keystore.create_key(input=CreateKeyInput()).branch_key_identifier + branch_key_id_B: str = keystore.create_key(input=CreateKeyInput()).branch_key_identifier + + class ExampleBranchKeyIdSupplier(IBranchKeyIdSupplier): + branch_key_id_for_tenant_A: str + branch_key_id_for_tenant_B: str + + def __init__(self, tenant_1_id, tenant_2_id): + self.branch_key_id_for_tenant_A = tenant_1_id + self.branch_key_id_for_tenant_B = tenant_2_id + + def get_branch_key_id( + self, + input: GetBranchKeyIdInput + ) -> GetBranchKeyIdOutput: + encryption_context: dict[str, str] = input.encryption_context + + if b"tenant" not in encryption_context: + raise ValueError("EncryptionContext invalid, does not contain expected tenant key value pair.") + + tenant_key_id: str = encryption_context.get(b"tenant") + branch_key_id: str + + if tenant_key_id == b"TenantA": + branch_key_id = self.branch_key_id_for_tenant_A + elif tenant_key_id == b"TenantB": + branch_key_id = self.branch_key_id_for_tenant_B + else: + raise ValueError(f"Item does not contain valid tenant ID: {tenant_key_id=}") + + return GetBranchKeyIdOutput(branch_key_id=branch_key_id) + + # 5. Create a branch key supplier that maps the branch key id to a more readable format + branch_key_id_supplier: IBranchKeyIdSupplier = ExampleBranchKeyIdSupplier( + tenant_1_id=branch_key_id_A, + tenant_2_id=branch_key_id_B, + ) + + # 6. Create the Hierarchical Keyring. + mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + config=MaterialProvidersConfig() + ) + + keyring_input: CreateAwsKmsHierarchicalKeyringInput = CreateAwsKmsHierarchicalKeyringInput( + key_store=keystore, + branch_key_id_supplier=branch_key_id_supplier, + ttl_seconds=600, + cache=CacheTypeDefault( + value=DefaultCache( + entry_capacity=100 + ) + ), + ) + + hierarchical_keyring: IKeyring = mat_prov.create_aws_kms_hierarchical_keyring( + input=keyring_input + ) + + # The Branch Key Id supplier uses the encryption context to determine which branch key id will + # be used to encrypt data. + # Create encryption context for TenantA + encryption_context_A: dict[str, str] = { + "tenant": "TenantA", + "encryption": "context", + "is not": "secret", + "but adds": "useful metadata", + "that can help you": "be confident that", + "the data you are handling": "is what you think it is", + } + + # Create encryption context for TenantB + encryption_context_B: dict[str, str] = { + "tenant": "TenantB", + "encryption": "context", + "is not": "secret", + "but adds": "useful metadata", + "that can help you": "be confident that", + "the data you are handling": "is what you think it is", + } + + # Encrypt the data for encryptionContextA & encryptionContextB + ciphertext_A, _ = client.encrypt( + source=EXAMPLE_DATA, + keyring=hierarchical_keyring, + encryption_context=encryption_context_A + ) + ciphertext_B, _ = client.encrypt( + source=EXAMPLE_DATA, + keyring=hierarchical_keyring, + encryption_context=encryption_context_B + ) + + # To attest that TenantKeyB cannot decrypt a message written by TenantKeyA + # let's construct more restrictive hierarchical keyrings. + keyring_input_A: CreateAwsKmsHierarchicalKeyringInput = CreateAwsKmsHierarchicalKeyringInput( + key_store=keystore, + branch_key_id=branch_key_id_A, + ttl_seconds=600, + cache=CacheTypeDefault( + value=DefaultCache( + entry_capacity=100 + ) + ), + ) + + hierarchical_keyring_A: IKeyring = mat_prov.create_aws_kms_hierarchical_keyring( + input=keyring_input_A + ) + + keyring_input_B: CreateAwsKmsHierarchicalKeyringInput = CreateAwsKmsHierarchicalKeyringInput( + key_store=keystore, + branch_key_id=branch_key_id_B, + ttl_seconds=600, + cache=CacheTypeDefault( + value=DefaultCache( + entry_capacity=100 + ) + ), + ) + + hierarchical_keyring_B: IKeyring = mat_prov.create_aws_kms_hierarchical_keyring( + input=keyring_input_B + ) + + # TODO: Run the decrypt, get expected exception type + # This should fail + try: + client.decrypt( + source=ciphertext_A, + keyring=hierarchical_keyring_B + ) + except AWSEncryptionSDKClientError: + pass + + # # This should fail + try: + client.decrypt( + source=ciphertext_B, + keyring=hierarchical_keyring_A + ) + except AWSEncryptionSDKClientError: + pass + + # These should succeed + plaintext_bytes_A, _ = client.decrypt( + source=ciphertext_A, + keyring=hierarchical_keyring_A + ) + assert plaintext_bytes_A == EXAMPLE_DATA + plaintext_bytes_B, _ = client.decrypt( + source=ciphertext_B, + keyring=hierarchical_keyring_B + ) + assert plaintext_bytes_B == EXAMPLE_DATA + +# Also, a thread-safe example ig + +# hack in a test +import botocore +encrypt_and_decrypt_with_keyring( + "KeyStoreDdbTable", + "KeyStoreDdbTable", + "arn:aws:kms:us-west-2:370957321024:key/9d989aa2-2f9c-438c-a745-cc57d3ad0126" +) \ No newline at end of file diff --git a/examples/src/keyrings/module_.py b/examples/src/keyrings/module_.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/src/module_.py b/examples/src/module_.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/aws_encryption_sdk/cmm_handler.py b/src/aws_encryption_sdk/cmm_handler.py index 82aad2248..d634dd571 100644 --- a/src/aws_encryption_sdk/cmm_handler.py +++ b/src/aws_encryption_sdk/cmm_handler.py @@ -1,9 +1,12 @@ # These dependencies are only loaded if you install the MPL. try: - from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.references import ( + from aws_cryptographic_materialproviders.mpl.errors import ( + AwsCryptographicMaterialProvidersException + ) + from aws_cryptographic_materialproviders.mpl.references import ( ICryptographicMaterialsManager, ) - from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.models import ( + from aws_cryptographic_materialproviders.mpl.models import ( GetEncryptionMaterialsInput, GetEncryptionMaterialsOutput, DecryptMaterialsInput, @@ -13,8 +16,11 @@ AlgorithmSuiteIdESDK, ) except ImportError as e: - print(f"WARNING: MPL import failed with {e=}") + pass +from aws_encryption_sdk.exceptions import ( + AWSEncryptionSDKClientError, +) from aws_encryption_sdk.materials_managers import ( DecryptionMaterialsRequest, EncryptionMaterialsRequest, @@ -30,8 +36,6 @@ EncryptedDataKey as Native_EncryptedDataKey, ) from aws_encryption_sdk.identifiers import ( - Algorithm, - AlgorithmSuite, CommitmentPolicy, ) @@ -40,6 +44,9 @@ class CMMHandler(CryptoMaterialsManager): native_cmm: CryptoMaterialsManager mpl_cmm: 'ICryptographicMaterialsManager' + def _is_using_native_cmm(self): + return hasattr(self, "native_cmm") and not hasattr(self, "mpl_cmm") + def __init__( self, cmm: 'CryptoMaterialsManager | ICryptographicMaterialsManager' @@ -56,24 +63,24 @@ def get_encryption_materials( request: EncryptionMaterialsRequest ) -> EncryptionMaterialsHandler: ''' - Returns an EncryptionMaterialsHandler based on the configured CMM. + Returns an EncryptionMaterialsHandler for the configured CMM. ''' - if (hasattr(self, "native_cmm") and not hasattr(self, "mpl_cmm")): + if (self._is_using_native_cmm()): return EncryptionMaterialsHandler(self.native_cmm.get_encryption_materials(request)) else: - input: GetEncryptionMaterialsInput = CMMHandler._create_mpl_get_encryption_materials_input_from_request(request) - print(f"get_encryption_materials {input=}") - output: GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(input) - print(f"get_encryption_materials {output=}") - return EncryptionMaterialsHandler(output.encryption_materials) + try: + input: GetEncryptionMaterialsInput = CMMHandler._create_mpl_get_encryption_materials_input_from_request(request) + output: GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(input) + return EncryptionMaterialsHandler(output.encryption_materials) + except AwsCryptographicMaterialProvidersException as e: + # Wrap MPL error into the ESDK error type + # so customers only have to catch ESDK error types. + raise AWSEncryptionSDKClientError(e) @staticmethod def _create_mpl_get_encryption_materials_input_from_request( request: EncryptionMaterialsRequest ) -> 'GetEncryptionMaterialsInput': - print(f"_create_mpl_get_encryption_materials_input_from_request {request=}") - print(f"{CMMHandler._map_native_commitment_policy_to_mpl_commitment_policy(request.commitment_policy)=}") - print(f"_create_mpl_get_encryption_materials_input_from_request {request.encryption_context=}") output: GetEncryptionMaterialsInput = GetEncryptionMaterialsInput( encryption_context=request.encryption_context, commitment_policy=CMMHandler._map_native_commitment_policy_to_mpl_commitment_policy(request.commitment_policy), @@ -82,7 +89,6 @@ def _create_mpl_get_encryption_materials_input_from_request( # algorithm_suite_id=request.algorithm.algorithm_id, max_plaintext_length=request.plaintext_length, ) - print(f"_create_mpl_get_encryption_materials_input_from_request {output=}") return output @staticmethod @@ -103,16 +109,19 @@ def decrypt_materials( request: DecryptionMaterialsRequest ) -> DecryptionMaterialsHandler: ''' - Returns a DecryptionMaterialsHandler based on the configured CMM. + Returns a DecryptionMaterialsHandler for the configured CMM. ''' - print(f"decrypt_materials {request=}") - if (hasattr(self, "native_cmm") and not hasattr(self, "mpl_cmm")): + if (self._is_using_native_cmm()): return DecryptionMaterialsHandler(self.native_cmm.decrypt_materials(request)) else: - input: 'DecryptMaterialsInput' = CMMHandler._create_mpl_decrypt_materials_input_from_request(request) - output: 'DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(input) - print(f"decrypt_materials {output.decryption_materials.verification_key=}") - return DecryptionMaterialsHandler(output.decryption_materials) + try: + input: 'DecryptMaterialsInput' = CMMHandler._create_mpl_decrypt_materials_input_from_request(request) + output: 'DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(input) + return DecryptionMaterialsHandler(output.decryption_materials) + except AwsCryptographicMaterialProvidersException as e: + # Wrap MPL error into the ESDK error type + # so customers only have to catch ESDK error types. + raise AWSEncryptionSDKClientError(e) @staticmethod def _native_algorithm_id_to_mpl_algorithm_id(native_algorithm_id: str) -> AlgorithmSuiteIdESDK: diff --git a/src/aws_encryption_sdk/materials_handlers.py b/src/aws_encryption_sdk/materials_handlers.py index bf3073ad3..1f34eba03 100644 --- a/src/aws_encryption_sdk/materials_handlers.py +++ b/src/aws_encryption_sdk/materials_handlers.py @@ -1,6 +1,6 @@ # These dependencies are only loaded if you install the MPL. try: - from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.models import ( + from aws_cryptographic_materialproviders.mpl.models import ( DecryptionMaterials as MPL_DecryptionMaterials, EncryptionMaterials as MPL_EncryptionMaterials, EncryptedDataKey as MPL_EncryptedDataKey, @@ -25,15 +25,19 @@ Signer ) +def _mpl_algorithm_id_to_native_algorithm_id(mpl_algorithm_id: str): + # MPL algorithm suite ID == hex(native algorithm suite ID) + return int(mpl_algorithm_id, 16) + class EncryptionMaterialsHandler: + ''' + In instances where encryption materials may be provided by either + the native `aws_encryption_sdk.materials_managers.Native_EncryptionMaterials` + or the MPL's `aws_cryptographic_materialproviders.mpl.models` + ''' native_materials: Native_EncryptionMaterials mpl_materials: 'MPL_EncryptionMaterials' - @staticmethod - def _mpl_algorithm_id_to_native_algorithm_id(mpl_algorithm_id: str): - # MPL algorithm suite ID == "ALG_" + native algorithm suite ID. - return int(mpl_algorithm_id, 16) - def __init__( self, materials: 'Native_EncryptionMaterials | MPL_EncryptionMaterials' @@ -49,9 +53,8 @@ def algorithm(self) -> Algorithm: if hasattr(self, "native_materials"): return self.native_materials.algorithm else: - print(f"algorithm {self.mpl_materials.algorithm_suite.id.value=}") return AlgorithmSuite.get_by_id( - EncryptionMaterialsHandler._mpl_algorithm_id_to_native_algorithm_id( + _mpl_algorithm_id_to_native_algorithm_id( self.mpl_materials.algorithm_suite.id.value ) ) @@ -83,10 +86,12 @@ def data_encryption_key(self) -> DataKey: if hasattr(self, "native_materials"): return self.native_materials.data_encryption_key else: - # TODO-MPL This impl is probably wrong + # TODO-MPL This impl is probably wrong, but works for for now + # If this works for all features, great! Remove this comment before launch. + # Otherwise, fix the implementation. mpl_dek = self.mpl_materials.plaintext_data_key return DataKey( - # key_provider=None, # No MasterKeyInfo object for plaintext data key + # key_provider is unused, but the return type is DataKey key_provider=MasterKeyInfo( provider_id="", key_info=b'' @@ -100,20 +105,8 @@ def signing_key(self) -> bytes: if hasattr(self, "native_materials"): return self.native_materials.signing_key else: - print(f"sign {self.mpl_materials.signing_key=}") return self.mpl_materials.signing_key - # if self.mpl_materials.signing_key is None: - # return Signer.from_key_bytes( - # algorithm=AlgorithmSuite.get_by_id(self.mpl_materials.algorithm_suite.id.value), - # bytes=self.mpl_materials.signing_key - # ) - def get_required_encryption_context_keys(self) -> list[str]: - if hasattr(self, "native_materials"): - return [] - else: - return self.mpl_materials.required_encryption_context_keys - class DecryptionMaterialsHandler: native_materials: Native_DecryptionMaterials mpl_materials: 'MPL_DecryptionMaterials' @@ -128,19 +121,15 @@ def __init__( self.mpl_materials = materials else: raise ValueError(f"Invalid DecryptionMaterials passed to DecryptionMaterialsHandler: {materials=}") - - def get_encryption_context(self) -> dict[str, str]: - if hasattr(self, "native_materials"): - return {} # TODO-MPL This impl is probably wrong - else: - return self.mpl_materials.encryption_context @property def data_key(self) -> DataKey: if hasattr(self, "native_materials"): return self.native_materials.data_key else: - # TODO-MPL This impl is probably wrong + # TODO-MPL This impl is probably wrong, but works for for now + # If this works for all features, great! Remove this comment before launch. + # Otherwise, fix the implementation. return DataKey( key_provider=MasterKeyInfo( provider_id="", @@ -155,11 +144,4 @@ def verification_key(self) -> bytes: if hasattr(self, "native_materials"): return self.native_materials.verification_key else: - print(f"ver {self.mpl_materials.verification_key=}") - return self.mpl_materials.verification_key - - def get_required_encryption_context_keys(self) -> list[str]: - if hasattr(self, "native_materials"): - return [] - else: - return self.mpl_materials.required_encryption_context_keys \ No newline at end of file + return self.mpl_materials.verification_key \ No newline at end of file diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 176b92334..e6cf00635 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -68,12 +68,12 @@ from aws_encryption_sdk.structures import MessageHeader try: import aws_cryptographic_materialproviders - from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.client import AwsCryptographicMaterialProviders - from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.config import MaterialProvidersConfig - from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.models import ( + from aws_cryptographic_materialproviders.mpl.client import AwsCryptographicMaterialProviders + from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig + from aws_cryptographic_materialproviders.mpl.models import ( CreateDefaultCryptographicMaterialsManagerInput ) - from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.references import ( + from aws_cryptographic_materialproviders.mpl.references import ( IKeyring, ) from aws_encryption_sdk.cmm_handler import CMMHandler @@ -411,7 +411,7 @@ class EncryptorConfig(_ClientConfig): :param keyring: `IKeyring` from the aws_cryptographic_materialproviders library which handles encryption and decryption :type keyring: - aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.references.IKeyring + aws_cryptographic_materialproviders.mpl.references.IKeyring :param int source_length: Length of source data (optional) .. note:: @@ -466,7 +466,7 @@ class StreamEncryptor(_EncryptionStream): # pylint: disable=too-many-instance-a :param keyring: `IKeyring` from the aws_cryptographic_materialproviders library which handles encryption and decryption :type keyring: - aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.references.IKeyring + aws_cryptographic_materialproviders.mpl.references.IKeyring :param int source_length: Length of source data (optional) .. note:: @@ -810,7 +810,7 @@ class DecryptorConfig(_ClientConfig): :param keyring: `IKeyring` from the aws_cryptographic_materialproviders library which handles encryption and decryption :type keyring: - aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.references.IKeyring + aws_cryptographic_materialproviders.mpl.references.IKeyring :param int source_length: Length of source data (optional) .. note:: @@ -850,7 +850,7 @@ class StreamDecryptor(_EncryptionStream): # pylint: disable=too-many-instance-a :param keyring: `IKeyring` from the aws_cryptographic_materialproviders library which handles encryption and decryption :type keyring: - aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.references.IKeyring + aws_cryptographic_materialproviders.mpl.references.IKeyring :param int source_length: Length of source data (optional) .. note:: @@ -1082,7 +1082,7 @@ def close(self): """Closes out the stream.""" _LOGGER.debug("Closing stream") if not hasattr(self, "footer"): - raise SerializationError("Footer not read") + raise SerializationError("Footer not read, message may be corrupted or data key may be incorrect") super(StreamDecryptor, self).close() diff --git a/test/unit/test_streaming_client_stream_decryptor.py b/test/unit/test_streaming_client_stream_decryptor.py index 157755094..94b22b092 100644 --- a/test/unit/test_streaming_client_stream_decryptor.py +++ b/test/unit/test_streaming_client_stream_decryptor.py @@ -767,4 +767,4 @@ def test_close_no_footer(self, mock_close): ) with pytest.raises(SerializationError) as excinfo: test_decryptor.close() - excinfo.match("Footer not read") + excinfo.match("Footer not read, message may be corrupted or data key may be incorrect") From 5b5aa07af75e37d2ad42e7ba850345097d55fcc0 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 2 Feb 2024 15:04:57 -0800 Subject: [PATCH 005/422] changes, cleanup --- src/aws_encryption_sdk/streaming_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index e6cf00635..55be3b917 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -170,7 +170,7 @@ def _exactly_one_arg_is_not_None(*args): try: assert isinstance(self.keyring, IKeyring) except AssertionError as e: - raise ValueError(f"Argument provided to keyring MUST be a {IKeyring}. Found {keyring.__class__.__name__=}") + raise ValueError(f"Argument provided to keyring MUST be a {IKeyring}. Found {self.keyring.__class__.__name__=}") mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( config=MaterialProvidersConfig() From 03e19caff1eaa264873fab500beb8e62c890b583 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 2 Feb 2024 15:19:08 -0800 Subject: [PATCH 006/422] flake8 --- src/aws_encryption_sdk/cmm_handler.py | 31 ++++++---- src/aws_encryption_sdk/materials_handlers.py | 27 +++++---- src/aws_encryption_sdk/streaming_client.py | 62 +++++++++++--------- 3 files changed, 68 insertions(+), 52 deletions(-) diff --git a/src/aws_encryption_sdk/cmm_handler.py b/src/aws_encryption_sdk/cmm_handler.py index d634dd571..f7f95b0c9 100644 --- a/src/aws_encryption_sdk/cmm_handler.py +++ b/src/aws_encryption_sdk/cmm_handler.py @@ -15,7 +15,7 @@ CommitmentPolicyESDK, AlgorithmSuiteIdESDK, ) -except ImportError as e: +except ImportError: pass from aws_encryption_sdk.exceptions import ( @@ -39,6 +39,7 @@ CommitmentPolicy, ) + # TODO-MPL Should this implement interface..? seems like yes since it implements all of interface methods class CMMHandler(CryptoMaterialsManager): native_cmm: CryptoMaterialsManager @@ -57,7 +58,7 @@ def __init__( self.mpl_cmm = cmm else: raise ValueError(f"Invalid CMM passed to CMMHander: {cmm=}") - + def get_encryption_materials( self, request: EncryptionMaterialsRequest @@ -69,28 +70,32 @@ def get_encryption_materials( return EncryptionMaterialsHandler(self.native_cmm.get_encryption_materials(request)) else: try: - input: GetEncryptionMaterialsInput = CMMHandler._create_mpl_get_encryption_materials_input_from_request(request) + input: GetEncryptionMaterialsInput = CMMHandler._create_mpl_get_encryption_materials_input_from_request( + request + ) output: GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(input) return EncryptionMaterialsHandler(output.encryption_materials) except AwsCryptographicMaterialProvidersException as e: # Wrap MPL error into the ESDK error type # so customers only have to catch ESDK error types. raise AWSEncryptionSDKClientError(e) - + @staticmethod def _create_mpl_get_encryption_materials_input_from_request( request: EncryptionMaterialsRequest ) -> 'GetEncryptionMaterialsInput': output: GetEncryptionMaterialsInput = GetEncryptionMaterialsInput( encryption_context=request.encryption_context, - commitment_policy=CMMHandler._map_native_commitment_policy_to_mpl_commitment_policy(request.commitment_policy), + commitment_policy=CMMHandler._map_native_commitment_policy_to_mpl_commitment_policy( + request.commitment_policy + ), # TODO double check this # optional... maybe this needs to be kwargs?? # algorithm_suite_id=request.algorithm.algorithm_id, max_plaintext_length=request.plaintext_length, ) return output - + @staticmethod def _map_native_commitment_policy_to_mpl_commitment_policy( native_commitment_policy: CommitmentPolicy @@ -103,7 +108,7 @@ def _map_native_commitment_policy_to_mpl_commitment_policy( return CommitmentPolicyESDK(value="REQUIRE_ENCRYPT_REQUIRE_DECRYPT") else: raise ValueError(f"Invalid {native_commitment_policy=}") - + def decrypt_materials( self, request: DecryptionMaterialsRequest @@ -122,12 +127,12 @@ def decrypt_materials( # Wrap MPL error into the ESDK error type # so customers only have to catch ESDK error types. raise AWSEncryptionSDKClientError(e) - + @staticmethod def _native_algorithm_id_to_mpl_algorithm_id(native_algorithm_id: str) -> AlgorithmSuiteIdESDK: # MPL algorithm suite ID = hexstr(native_algorithm_id) padded to 4 digits post-`x`. return AlgorithmSuiteIdESDK(f"{native_algorithm_id:#0{6}x}") - + @staticmethod def _create_mpl_decrypt_materials_input_from_request( request: DecryptionMaterialsRequest @@ -139,8 +144,12 @@ def _create_mpl_decrypt_materials_input_from_request( ciphertext=key_blob.encrypted_data_key, ) for key_blob in key_blob_list] output: DecryptMaterialsInput = DecryptMaterialsInput( - algorithm_suite_id=CMMHandler._native_algorithm_id_to_mpl_algorithm_id(request.algorithm.algorithm_id), - commitment_policy=CMMHandler._map_native_commitment_policy_to_mpl_commitment_policy(request.commitment_policy), + algorithm_suite_id=CMMHandler._native_algorithm_id_to_mpl_algorithm_id( + request.algorithm.algorithm_id + ), + commitment_policy=CMMHandler._map_native_commitment_policy_to_mpl_commitment_policy( + request.commitment_policy + ), encrypted_data_keys=list_edks, encryption_context=request.encryption_context, ) diff --git a/src/aws_encryption_sdk/materials_handlers.py b/src/aws_encryption_sdk/materials_handlers.py index 1f34eba03..a03138e78 100644 --- a/src/aws_encryption_sdk/materials_handlers.py +++ b/src/aws_encryption_sdk/materials_handlers.py @@ -5,7 +5,7 @@ EncryptionMaterials as MPL_EncryptionMaterials, EncryptedDataKey as MPL_EncryptedDataKey, ) -except ImportError as e: +except ImportError: pass from aws_encryption_sdk.materials_managers import ( @@ -21,14 +21,13 @@ EncryptedDataKey as Native_EncryptedDataKey, MasterKeyInfo, ) -from aws_encryption_sdk.internal.crypto.authentication import ( - Signer -) + def _mpl_algorithm_id_to_native_algorithm_id(mpl_algorithm_id: str): # MPL algorithm suite ID == hex(native algorithm suite ID) return int(mpl_algorithm_id, 16) + class EncryptionMaterialsHandler: ''' In instances where encryption materials may be provided by either @@ -48,6 +47,7 @@ def __init__( self.mpl_materials = materials else: raise ValueError(f"Invalid EncryptionMaterials passed to EncryptionMaterialsHandler: {materials=}") + @property def algorithm(self) -> Algorithm: if hasattr(self, "native_materials"): @@ -58,14 +58,14 @@ def algorithm(self) -> Algorithm: self.mpl_materials.algorithm_suite.id.value ) ) - + @property def encryption_context(self) -> dict[str, str]: if hasattr(self, "native_materials"): return self.native_materials.encryption_context else: return self.mpl_materials.encryption_context - + @property def encrypted_data_keys(self) -> list[Native_EncryptedDataKey]: if hasattr(self, "native_materials"): @@ -80,7 +80,7 @@ def encrypted_data_keys(self) -> list[Native_EncryptedDataKey]: encrypted_data_key=mpl_edk.ciphertext, ) for mpl_edk in mpl_edk_list} return key_blob_list - + @property def data_encryption_key(self) -> DataKey: if hasattr(self, "native_materials"): @@ -97,16 +97,17 @@ def data_encryption_key(self) -> DataKey: key_info=b'' ), data_key=mpl_dek, - encrypted_data_key=b'', # No encrypted DEK + encrypted_data_key=b'', # No encrypted DEK ) - + @property def signing_key(self) -> bytes: if hasattr(self, "native_materials"): return self.native_materials.signing_key else: return self.mpl_materials.signing_key - + + class DecryptionMaterialsHandler: native_materials: Native_DecryptionMaterials mpl_materials: 'MPL_DecryptionMaterials' @@ -121,7 +122,7 @@ def __init__( self.mpl_materials = materials else: raise ValueError(f"Invalid DecryptionMaterials passed to DecryptionMaterialsHandler: {materials=}") - + @property def data_key(self) -> DataKey: if hasattr(self, "native_materials"): @@ -138,10 +139,10 @@ def data_key(self) -> DataKey: data_key=self.mpl_materials.plaintext_data_key, encrypted_data_key=b'', ) - + @property def verification_key(self) -> bytes: if hasattr(self, "native_materials"): return self.native_materials.verification_key else: - return self.mpl_materials.verification_key \ No newline at end of file + return self.mpl_materials.verification_key diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 55be3b917..661b3fa21 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -67,7 +67,6 @@ from aws_encryption_sdk.materials_managers.default import DefaultCryptoMaterialsManager from aws_encryption_sdk.structures import MessageHeader try: - import aws_cryptographic_materialproviders from aws_cryptographic_materialproviders.mpl.client import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig from aws_cryptographic_materialproviders.mpl.models import ( @@ -77,14 +76,33 @@ IKeyring, ) from aws_encryption_sdk.cmm_handler import CMMHandler - + _has_mpl = True -except ImportError as e: +except ImportError: _has_mpl = False _LOGGER = logging.getLogger(__name__) +def _exactly_one_arg_is_not_None(*args): + ''' + Private helper function. + Returns `True` if exactly one item in the list is not `None`. + Returns `False` otherwise. + ''' + # Have not found any `not None` + found_one = False + for arg in args: + if arg is not None: + if found_one is False: + # Have not already found a `not None`, found a `not None` => only one `not None` (so far) + found_one = True + else: + # Already found a `not None`, found another `not None` => not exactly one `not None` + return False + return found_one + + @attr.s(hash=True) # pylint: disable=too-many-instance-attributes @six.add_metaclass(abc.ABCMeta) class _ClientConfig(object): # pylint: disable=too-many-instance-attributes @@ -140,42 +158,30 @@ class _ClientConfig(object): # pylint: disable=too-many-instance-attributes ) # DEPRECATED: Value is no longer configurable here. Parameter left here to avoid breaking consumers. def _has_mpl_attrs_post_init(self): - - def _exactly_one_arg_is_not_None(*args): - ''' - Private helper function. - Returns `True` if exactly one item in the list is not `None`. - Returns `False` otherwise. - ''' - # Have not found any `not None` - found_one = False - for arg in args: - if arg is not None: - if found_one == False: - # Have not already found a `not None`, found a `not None` => only one `not None` (so far) - found_one = True - else: - # Already found a `not None`, found another `not None` => not exactly one `not None` - return False - return found_one - if not _exactly_one_arg_is_not_None(self.materials_manager, self.key_provider, self.keyring): raise TypeError("Exactly one of keyring, materials_manager, or key_provider must be provided") if self.materials_manager is None: if self.key_provider is not None: - # No CMM, provided (legacy) native `key_provider` => create (legacy) native DefaultCryptoMaterialsManager - self.materials_manager = DefaultCryptoMaterialsManager(master_key_provider=self.key_provider) + # No CMM, provided legacy native `key_provider` => create legacy native DefaultCryptoMaterialsManager + self.materials_manager = DefaultCryptoMaterialsManager( + master_key_provider=self.key_provider + ) elif self.keyring is not None: # No CMM, provided MPL keyring => create MPL's DefaultCryptographicMaterialsManager try: assert isinstance(self.keyring, IKeyring) - except AssertionError as e: - raise ValueError(f"Argument provided to keyring MUST be a {IKeyring}. Found {self.keyring.__class__.__name__=}") - + except AssertionError: + raise ValueError(f"Argument provided to keyring MUST be a {IKeyring}. \ + Found {self.keyring.__class__.__name__=}") + mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( config=MaterialProvidersConfig() ) - cmm = mat_prov.create_default_cryptographic_materials_manager(CreateDefaultCryptographicMaterialsManagerInput(keyring=self.keyring)) + cmm = mat_prov.create_default_cryptographic_materials_manager( + CreateDefaultCryptographicMaterialsManagerInput( + keyring=self.keyring + ) + ) cmm_handler: CryptoMaterialsManager = CMMHandler(cmm) self.materials_manager = cmm_handler From b5d33275a462e9311b08359a722496d0737a81be Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 2 Feb 2024 15:41:21 -0800 Subject: [PATCH 007/422] flake8 --- src/aws_encryption_sdk/cmm_handler.py | 25 +++++++-- src/aws_encryption_sdk/materials_handlers.py | 56 ++++++++++++++++++-- 2 files changed, 72 insertions(+), 9 deletions(-) diff --git a/src/aws_encryption_sdk/cmm_handler.py b/src/aws_encryption_sdk/cmm_handler.py index f7f95b0c9..bb60a4fa1 100644 --- a/src/aws_encryption_sdk/cmm_handler.py +++ b/src/aws_encryption_sdk/cmm_handler.py @@ -1,3 +1,5 @@ +"""Retrieves encryption/decryption materials from an underlying materials provider.""" + # These dependencies are only loaded if you install the MPL. try: from aws_cryptographic_materialproviders.mpl.errors import ( @@ -42,6 +44,15 @@ # TODO-MPL Should this implement interface..? seems like yes since it implements all of interface methods class CMMHandler(CryptoMaterialsManager): + """ + In instances where encryption materials may be provided by either + an implementation of the native + `aws_encryption_sdk.materials_managers.base.CryptoMaterialsManager` + or an implementation of the MPL's + `aws_cryptographic_materialproviders.mpl.references.ICryptographicMaterialsManager`, + this provides the correct materials based on the underlying materials manager. + """ + native_cmm: CryptoMaterialsManager mpl_cmm: 'ICryptographicMaterialsManager' @@ -57,15 +68,17 @@ def __init__( elif isinstance(cmm, ICryptographicMaterialsManager): self.mpl_cmm = cmm else: - raise ValueError(f"Invalid CMM passed to CMMHander: {cmm=}") + raise ValueError(f"Invalid CMM passed to CMMHandler: {cmm=}") def get_encryption_materials( self, request: EncryptionMaterialsRequest ) -> EncryptionMaterialsHandler: - ''' + """ Returns an EncryptionMaterialsHandler for the configured CMM. - ''' + :param request: Request for encryption materials + """ + if (self._is_using_native_cmm()): return EncryptionMaterialsHandler(self.native_cmm.get_encryption_materials(request)) else: @@ -113,9 +126,11 @@ def decrypt_materials( self, request: DecryptionMaterialsRequest ) -> DecryptionMaterialsHandler: - ''' + """ Returns a DecryptionMaterialsHandler for the configured CMM. - ''' + :param request: Request for decryption materials + """ + if (self._is_using_native_cmm()): return DecryptionMaterialsHandler(self.native_cmm.decrypt_materials(request)) else: diff --git a/src/aws_encryption_sdk/materials_handlers.py b/src/aws_encryption_sdk/materials_handlers.py index a03138e78..180dec3bb 100644 --- a/src/aws_encryption_sdk/materials_handlers.py +++ b/src/aws_encryption_sdk/materials_handlers.py @@ -1,3 +1,4 @@ +"""Provides encryption/decryption materials from an underlying materials provider.""" # These dependencies are only loaded if you install the MPL. try: from aws_cryptographic_materialproviders.mpl.models import ( @@ -29,11 +30,13 @@ def _mpl_algorithm_id_to_native_algorithm_id(mpl_algorithm_id: str): class EncryptionMaterialsHandler: - ''' + """ In instances where encryption materials may be provided by either - the native `aws_encryption_sdk.materials_managers.Native_EncryptionMaterials` - or the MPL's `aws_cryptographic_materialproviders.mpl.models` - ''' + the native `aws_encryption_sdk.materials_managers.EncryptionMaterials` + or the MPL's `aws_cryptographic_materialproviders.mpl.models.EncryptionMaterials`, + this provides the correct materials based on the configured materials provider. + """ + native_materials: Native_EncryptionMaterials mpl_materials: 'MPL_EncryptionMaterials' @@ -41,6 +44,11 @@ def __init__( self, materials: 'Native_EncryptionMaterials | MPL_EncryptionMaterials' ): + """ + Create EncryptionMaterialsHandler. + :param materials: Underlying encryption materials + """ + if isinstance(materials, Native_EncryptionMaterials): self.native_materials = materials elif isinstance(materials, MPL_EncryptionMaterials): @@ -50,6 +58,10 @@ def __init__( @property def algorithm(self) -> Algorithm: + """ + Materials' native Algorithm. + """ + if hasattr(self, "native_materials"): return self.native_materials.algorithm else: @@ -61,6 +73,10 @@ def algorithm(self) -> Algorithm: @property def encryption_context(self) -> dict[str, str]: + """ + Materials' encryption context. + """ + if hasattr(self, "native_materials"): return self.native_materials.encryption_context else: @@ -68,6 +84,10 @@ def encryption_context(self) -> dict[str, str]: @property def encrypted_data_keys(self) -> list[Native_EncryptedDataKey]: + """ + Materials' encrypted data keys. + """ + if hasattr(self, "native_materials"): return self.native_materials.encrypted_data_keys else: @@ -83,6 +103,10 @@ def encrypted_data_keys(self) -> list[Native_EncryptedDataKey]: @property def data_encryption_key(self) -> DataKey: + """ + Materials' data encryption key. + """ + if hasattr(self, "native_materials"): return self.native_materials.data_encryption_key else: @@ -102,6 +126,10 @@ def data_encryption_key(self) -> DataKey: @property def signing_key(self) -> bytes: + """ + Materials' signing key. + """ + if hasattr(self, "native_materials"): return self.native_materials.signing_key else: @@ -109,6 +137,13 @@ def signing_key(self) -> bytes: class DecryptionMaterialsHandler: + """ + In instances where decryption materials may be provided by either + the native `aws_encryption_sdk.materials_managers.DecryptionMaterials` + or the MPL's `aws_cryptographic_materialproviders.mpl.models.DecryptionMaterials`, + this provides the correct materials based on the configured materials provider. + """ + native_materials: Native_DecryptionMaterials mpl_materials: 'MPL_DecryptionMaterials' @@ -116,6 +151,11 @@ def __init__( self, materials: 'Native_DecryptionMaterials | MPL_DecryptionMaterials' ): + """ + Create DecryptionMaterialsHandler. + :param materials: Underlying decryption materials + """ + if isinstance(materials, Native_DecryptionMaterials): self.native_materials = materials elif isinstance(materials, MPL_DecryptionMaterials): @@ -125,6 +165,10 @@ def __init__( @property def data_key(self) -> DataKey: + """ + Materials' data key. + """ + if hasattr(self, "native_materials"): return self.native_materials.data_key else: @@ -142,6 +186,10 @@ def data_key(self) -> DataKey: @property def verification_key(self) -> bytes: + """ + Materials' verification key. + """ + if hasattr(self, "native_materials"): return self.native_materials.verification_key else: From b13cd191a8874dc0091e844c4cb0789d81591764 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 2 Feb 2024 15:45:55 -0800 Subject: [PATCH 008/422] flake8 --- src/aws_encryption_sdk/cmm_handler.py | 7 ++-- src/aws_encryption_sdk/materials_handlers.py | 37 ++++---------------- src/aws_encryption_sdk/streaming_client.py | 4 +-- 3 files changed, 14 insertions(+), 34 deletions(-) diff --git a/src/aws_encryption_sdk/cmm_handler.py b/src/aws_encryption_sdk/cmm_handler.py index bb60a4fa1..17d59792a 100644 --- a/src/aws_encryption_sdk/cmm_handler.py +++ b/src/aws_encryption_sdk/cmm_handler.py @@ -63,6 +63,11 @@ def __init__( self, cmm: 'CryptoMaterialsManager | ICryptographicMaterialsManager' ): + """ + Create DecryptionMaterialsHandler. + :param cmm: Underlying cryptographic materials manager + """ + if isinstance(cmm, CryptoMaterialsManager): self.native_cmm = cmm elif isinstance(cmm, ICryptographicMaterialsManager): @@ -78,7 +83,6 @@ def get_encryption_materials( Returns an EncryptionMaterialsHandler for the configured CMM. :param request: Request for encryption materials """ - if (self._is_using_native_cmm()): return EncryptionMaterialsHandler(self.native_cmm.get_encryption_materials(request)) else: @@ -130,7 +134,6 @@ def decrypt_materials( Returns a DecryptionMaterialsHandler for the configured CMM. :param request: Request for decryption materials """ - if (self._is_using_native_cmm()): return DecryptionMaterialsHandler(self.native_cmm.decrypt_materials(request)) else: diff --git a/src/aws_encryption_sdk/materials_handlers.py b/src/aws_encryption_sdk/materials_handlers.py index 180dec3bb..d54e4517b 100644 --- a/src/aws_encryption_sdk/materials_handlers.py +++ b/src/aws_encryption_sdk/materials_handlers.py @@ -48,7 +48,6 @@ def __init__( Create EncryptionMaterialsHandler. :param materials: Underlying encryption materials """ - if isinstance(materials, Native_EncryptionMaterials): self.native_materials = materials elif isinstance(materials, MPL_EncryptionMaterials): @@ -58,10 +57,7 @@ def __init__( @property def algorithm(self) -> Algorithm: - """ - Materials' native Algorithm. - """ - + """Materials' native Algorithm.""" if hasattr(self, "native_materials"): return self.native_materials.algorithm else: @@ -73,10 +69,7 @@ def algorithm(self) -> Algorithm: @property def encryption_context(self) -> dict[str, str]: - """ - Materials' encryption context. - """ - + """Materials' encryption context.""" if hasattr(self, "native_materials"): return self.native_materials.encryption_context else: @@ -84,10 +77,7 @@ def encryption_context(self) -> dict[str, str]: @property def encrypted_data_keys(self) -> list[Native_EncryptedDataKey]: - """ - Materials' encrypted data keys. - """ - + """Materials' encrypted data keys.""" if hasattr(self, "native_materials"): return self.native_materials.encrypted_data_keys else: @@ -103,10 +93,7 @@ def encrypted_data_keys(self) -> list[Native_EncryptedDataKey]: @property def data_encryption_key(self) -> DataKey: - """ - Materials' data encryption key. - """ - + """Materials' data encryption key.""" if hasattr(self, "native_materials"): return self.native_materials.data_encryption_key else: @@ -126,10 +113,7 @@ def data_encryption_key(self) -> DataKey: @property def signing_key(self) -> bytes: - """ - Materials' signing key. - """ - + """Materials' signing key.""" if hasattr(self, "native_materials"): return self.native_materials.signing_key else: @@ -155,7 +139,6 @@ def __init__( Create DecryptionMaterialsHandler. :param materials: Underlying decryption materials """ - if isinstance(materials, Native_DecryptionMaterials): self.native_materials = materials elif isinstance(materials, MPL_DecryptionMaterials): @@ -165,10 +148,7 @@ def __init__( @property def data_key(self) -> DataKey: - """ - Materials' data key. - """ - + """Materials' data key.""" if hasattr(self, "native_materials"): return self.native_materials.data_key else: @@ -186,10 +166,7 @@ def data_key(self) -> DataKey: @property def verification_key(self) -> bytes: - """ - Materials' verification key. - """ - + """Materials' verification key.""" if hasattr(self, "native_materials"): return self.native_materials.verification_key else: diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 661b3fa21..ec19b6dd5 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -85,11 +85,11 @@ def _exactly_one_arg_is_not_None(*args): - ''' + """ Private helper function. Returns `True` if exactly one item in the list is not `None`. Returns `False` otherwise. - ''' + """ # Have not found any `not None` found_one = False for arg in args: From 51065cb186bb3e649117a3a9871b2b5943066a52 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 2 Feb 2024 15:48:34 -0800 Subject: [PATCH 009/422] flake8 --- setup.py | 4 +++- src/aws_encryption_sdk/cmm_handler.py | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index c4c277096..353781800 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,9 @@ def get_requirements(): install_requires=get_requirements(), # TODO: Point at MPL main branch once Python MPL is merged into main. extras_require={ - "MPL": ["aws-cryptographic-material-providers @ git+https://github.com/aws/aws-cryptographic-material-providers-library.git@lucmcdon/python-mpl#subdirectory=AwsCryptographicMaterialProviders/runtimes/python"], + "MPL": ["aws-cryptographic-material-providers @\ + git+https://github.com/aws/aws-cryptographic-material-providers-library.git@\ + lucmcdon/python-mpl#subdirectory=AwsCryptographicMaterialProviders/runtimes/python"], }, classifiers=[ "Development Status :: 5 - Production/Stable", diff --git a/src/aws_encryption_sdk/cmm_handler.py b/src/aws_encryption_sdk/cmm_handler.py index 17d59792a..887d9d79e 100644 --- a/src/aws_encryption_sdk/cmm_handler.py +++ b/src/aws_encryption_sdk/cmm_handler.py @@ -67,7 +67,6 @@ def __init__( Create DecryptionMaterialsHandler. :param cmm: Underlying cryptographic materials manager """ - if isinstance(cmm, CryptoMaterialsManager): self.native_cmm = cmm elif isinstance(cmm, ICryptographicMaterialsManager): From fc4d254d7f7601d86fc9954ed69d0817869a43e2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 2 Feb 2024 15:51:15 -0800 Subject: [PATCH 010/422] flake8 --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 353781800..4cd8027cd 100644 --- a/setup.py +++ b/setup.py @@ -41,9 +41,9 @@ def get_requirements(): install_requires=get_requirements(), # TODO: Point at MPL main branch once Python MPL is merged into main. extras_require={ - "MPL": ["aws-cryptographic-material-providers @\ - git+https://github.com/aws/aws-cryptographic-material-providers-library.git@\ - lucmcdon/python-mpl#subdirectory=AwsCryptographicMaterialProviders/runtimes/python"], + "MPL": ["aws-cryptographic-material-providers @" \ + "git+https://github.com/aws/aws-cryptographic-material-providers-library.git@" \ + "lucmcdon/python-mpl#subdirectory=AwsCryptographicMaterialProviders/runtimes/python"], }, classifiers=[ "Development Status :: 5 - Production/Stable", From a8e52d310c777905326fc98cbf667217116de303 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 5 Feb 2024 13:23:21 -0800 Subject: [PATCH 011/422] fix pem/der --- .../internal/crypto/authentication.py | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/src/aws_encryption_sdk/internal/crypto/authentication.py b/src/aws_encryption_sdk/internal/crypto/authentication.py index b9692eb16..8c1b9af31 100644 --- a/src/aws_encryption_sdk/internal/crypto/authentication.py +++ b/src/aws_encryption_sdk/internal/crypto/authentication.py @@ -68,25 +68,31 @@ class Signer(_PrehashingAuthenticator): """ @classmethod - def from_key_bytes(cls, algorithm, key_bytes): + def from_key_bytes(cls, algorithm, key_bytes, encoding=serialization.Encoding.DER): """Builds a `Signer` from an algorithm suite and a raw signing key. :param algorithm: Algorithm on which to base signer :type algorithm: aws_encryption_sdk.identifiers.Algorithm :param bytes key_bytes: Raw signing key + :param encoding: Encoding used for key bytes + :type encoding: cryptography.hazmat.primitives.serialization.encoding :rtype: aws_encryption_sdk.internal.crypto.Signer """ - # key = serialization.load_der_private_key(data=key_bytes, password=None, backend=default_backend()) - key = serialization.load_pem_private_key(data=key_bytes, password=None, backend=default_backend()) + if encoding == serialization.Encoding.DER: + key = serialization.load_der_private_key(data=key_bytes, password=None, backend=default_backend()) + elif encoding == serialization.Encoding.PEM: + key = serialization.load_pem_private_key(data=key_bytes, password=None, backend=default_backend()) + else: + raise ValueError("Unsupported signing key encoding: {}".format(encoding)) return cls(algorithm, key) - def key_bytes(self): + def key_bytes(self, encoding=serialization.Encoding.DER): """Returns the raw signing key. :rtype: bytes """ return self.key.private_bytes( - encoding=serialization.Encoding.DER, + encoding=encoding, format=serialization.PrivateFormat.PKCS8, encryption_algorithm=serialization.NoEncryption(), ) @@ -149,19 +155,27 @@ def from_encoded_point(cls, algorithm, encoded_point): ) @classmethod - def from_key_bytes(cls, algorithm, key_bytes): + def from_key_bytes(cls, algorithm, key_bytes, encoding=serialization.Encoding.DER): """Creates a `Verifier` object based on the supplied algorithm and raw verification key. :param algorithm: Algorithm on which to base verifier :type algorithm: aws_encryption_sdk.identifiers.Algorithm :param bytes encoded_point: Raw verification key + :param encoding: Encoding used for key bytes + :type encoding: cryptography.hazmat.primitives.serialization.encoding :returns: Instance of Verifier generated from encoded point :rtype: aws_encryption_sdk.internal.crypto.Verifier """ + if encoding == serialization.Encoding.DER: + key = serialization.load_der_private_key(data=key_bytes, password=None, backend=default_backend()) + elif encoding == serialization.Encoding.PEM: + key = serialization.load_pem_private_key(data=key_bytes, password=None, backend=default_backend()) + else: + raise ValueError("Unsupported verification key encoding: {}".format(encoding)) return cls( algorithm=algorithm, key=serialization.load_pem_public_key(data=key_bytes, backend=default_backend()) ) - + def key_bytes(self): """Returns the raw verification key. From 6f5504741a4797ee1cce5986a5ba48718759b810 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 5 Feb 2024 13:30:49 -0800 Subject: [PATCH 012/422] fix pem/der --- src/aws_encryption_sdk/streaming_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index ec19b6dd5..f6b3529a9 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -172,7 +172,7 @@ def _has_mpl_attrs_post_init(self): assert isinstance(self.keyring, IKeyring) except AssertionError: raise ValueError(f"Argument provided to keyring MUST be a {IKeyring}. \ - Found {self.keyring.__class__.__name__=}") + Found {self.keyring.__class__.__name__}") mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( config=MaterialProvidersConfig() From 1b1b4e4bc61d7c74a3815d225e992c1cb7bce135 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 5 Feb 2024 16:11:08 -0800 Subject: [PATCH 013/422] debug --- src/aws_encryption_sdk/internal/crypto/authentication.py | 2 +- src/aws_encryption_sdk/streaming_client.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/aws_encryption_sdk/internal/crypto/authentication.py b/src/aws_encryption_sdk/internal/crypto/authentication.py index 8c1b9af31..5e8dcd10c 100644 --- a/src/aws_encryption_sdk/internal/crypto/authentication.py +++ b/src/aws_encryption_sdk/internal/crypto/authentication.py @@ -173,7 +173,7 @@ def from_key_bytes(cls, algorithm, key_bytes, encoding=serialization.Encoding.DE else: raise ValueError("Unsupported verification key encoding: {}".format(encoding)) return cls( - algorithm=algorithm, key=serialization.load_pem_public_key(data=key_bytes, backend=default_backend()) + algorithm=algorithm, key=key ) def key_bytes(self): diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index f6b3529a9..83e4a9ecd 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -165,12 +165,10 @@ def _has_mpl_attrs_post_init(self): # No CMM, provided legacy native `key_provider` => create legacy native DefaultCryptoMaterialsManager self.materials_manager = DefaultCryptoMaterialsManager( master_key_provider=self.key_provider - ) + ) elif self.keyring is not None: # No CMM, provided MPL keyring => create MPL's DefaultCryptographicMaterialsManager - try: - assert isinstance(self.keyring, IKeyring) - except AssertionError: + if not isinstance(self.keyring, IKeyring): raise ValueError(f"Argument provided to keyring MUST be a {IKeyring}. \ Found {self.keyring.__class__.__name__}") From 38a4cc9c6808d5da025822634e8ec9bb0b9f960f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 5 Feb 2024 16:14:15 -0800 Subject: [PATCH 014/422] debug --- src/aws_encryption_sdk/internal/crypto/authentication.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aws_encryption_sdk/internal/crypto/authentication.py b/src/aws_encryption_sdk/internal/crypto/authentication.py index 5e8dcd10c..80469ff1f 100644 --- a/src/aws_encryption_sdk/internal/crypto/authentication.py +++ b/src/aws_encryption_sdk/internal/crypto/authentication.py @@ -167,9 +167,9 @@ def from_key_bytes(cls, algorithm, key_bytes, encoding=serialization.Encoding.DE :rtype: aws_encryption_sdk.internal.crypto.Verifier """ if encoding == serialization.Encoding.DER: - key = serialization.load_der_private_key(data=key_bytes, password=None, backend=default_backend()) + key = serialization.load_der_private_key(data=key_bytes, backend=default_backend()) elif encoding == serialization.Encoding.PEM: - key = serialization.load_pem_private_key(data=key_bytes, password=None, backend=default_backend()) + key = serialization.load_pem_private_key(data=key_bytes, backend=default_backend()) else: raise ValueError("Unsupported verification key encoding: {}".format(encoding)) return cls( From 0cd0e2301fc7133e8023a825f128bc457af311aa Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 6 Feb 2024 15:47:48 -0800 Subject: [PATCH 015/422] fix --- .../internal/crypto/authentication.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/src/aws_encryption_sdk/internal/crypto/authentication.py b/src/aws_encryption_sdk/internal/crypto/authentication.py index 80469ff1f..88d21a2ef 100644 --- a/src/aws_encryption_sdk/internal/crypto/authentication.py +++ b/src/aws_encryption_sdk/internal/crypto/authentication.py @@ -78,12 +78,7 @@ def from_key_bytes(cls, algorithm, key_bytes, encoding=serialization.Encoding.DE :type encoding: cryptography.hazmat.primitives.serialization.encoding :rtype: aws_encryption_sdk.internal.crypto.Signer """ - if encoding == serialization.Encoding.DER: - key = serialization.load_der_private_key(data=key_bytes, password=None, backend=default_backend()) - elif encoding == serialization.Encoding.PEM: - key = serialization.load_pem_private_key(data=key_bytes, password=None, backend=default_backend()) - else: - raise ValueError("Unsupported signing key encoding: {}".format(encoding)) + key = serialization.load_der_private_key(data=key_bytes, password=None, backend=default_backend()) return cls(algorithm, key) def key_bytes(self, encoding=serialization.Encoding.DER): @@ -166,14 +161,8 @@ def from_key_bytes(cls, algorithm, key_bytes, encoding=serialization.Encoding.DE :returns: Instance of Verifier generated from encoded point :rtype: aws_encryption_sdk.internal.crypto.Verifier """ - if encoding == serialization.Encoding.DER: - key = serialization.load_der_private_key(data=key_bytes, backend=default_backend()) - elif encoding == serialization.Encoding.PEM: - key = serialization.load_pem_private_key(data=key_bytes, backend=default_backend()) - else: - raise ValueError("Unsupported verification key encoding: {}".format(encoding)) return cls( - algorithm=algorithm, key=key + algorithm=algorithm, key=serialization.load_der_public_key(data=key_bytes, backend=default_backend()) ) def key_bytes(self): From 44826a2568fd3fa86d3031d11837c707d17850e0 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 6 Feb 2024 16:09:14 -0800 Subject: [PATCH 016/422] fix --- .../internal/crypto/authentication.py | 14 +++++--------- src/aws_encryption_sdk/streaming_client.py | 3 ++- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/aws_encryption_sdk/internal/crypto/authentication.py b/src/aws_encryption_sdk/internal/crypto/authentication.py index 88d21a2ef..f90ac77e0 100644 --- a/src/aws_encryption_sdk/internal/crypto/authentication.py +++ b/src/aws_encryption_sdk/internal/crypto/authentication.py @@ -68,26 +68,24 @@ class Signer(_PrehashingAuthenticator): """ @classmethod - def from_key_bytes(cls, algorithm, key_bytes, encoding=serialization.Encoding.DER): + def from_key_bytes(cls, algorithm, key_bytes): """Builds a `Signer` from an algorithm suite and a raw signing key. :param algorithm: Algorithm on which to base signer :type algorithm: aws_encryption_sdk.identifiers.Algorithm :param bytes key_bytes: Raw signing key - :param encoding: Encoding used for key bytes - :type encoding: cryptography.hazmat.primitives.serialization.encoding :rtype: aws_encryption_sdk.internal.crypto.Signer """ key = serialization.load_der_private_key(data=key_bytes, password=None, backend=default_backend()) return cls(algorithm, key) - def key_bytes(self, encoding=serialization.Encoding.DER): + def key_bytes(self): """Returns the raw signing key. :rtype: bytes """ return self.key.private_bytes( - encoding=encoding, + encoding=serialization.Encoding.DER, format=serialization.PrivateFormat.PKCS8, encryption_algorithm=serialization.NoEncryption(), ) @@ -150,21 +148,19 @@ def from_encoded_point(cls, algorithm, encoded_point): ) @classmethod - def from_key_bytes(cls, algorithm, key_bytes, encoding=serialization.Encoding.DER): + def from_key_bytes(cls, algorithm, key_bytes): """Creates a `Verifier` object based on the supplied algorithm and raw verification key. :param algorithm: Algorithm on which to base verifier :type algorithm: aws_encryption_sdk.identifiers.Algorithm :param bytes encoded_point: Raw verification key - :param encoding: Encoding used for key bytes - :type encoding: cryptography.hazmat.primitives.serialization.encoding :returns: Instance of Verifier generated from encoded point :rtype: aws_encryption_sdk.internal.crypto.Verifier """ return cls( algorithm=algorithm, key=serialization.load_der_public_key(data=key_bytes, backend=default_backend()) ) - + def key_bytes(self): """Returns the raw verification key. diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 83e4a9ecd..582472025 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -919,7 +919,8 @@ def _read_header(self): # MPL verification key is NOT key bytes, it is bytes of the compressed point # TODO-MPL: clean this up, least-privilege violation import base64 - if hasattr(self.config.materials_manager, "mpl_cmm"): + if (isinstance(self.config.materials_manager, CMMHandler) + and hasattr(self.config.materials_manager, "mpl_cmm")): self.verifier = Verifier.from_encoded_point( algorithm=header.algorithm, encoded_point=base64.b64encode(decryption_materials.verification_key) From 02e9f843826506597ca03a3139da1b58a88da2f6 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 6 Feb 2024 16:27:51 -0800 Subject: [PATCH 017/422] fix --- src/aws_encryption_sdk/cmm_handler.py | 13 ++++++++----- src/aws_encryption_sdk/materials_handlers.py | 12 +++++++----- src/aws_encryption_sdk/streaming_client.py | 2 +- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/aws_encryption_sdk/cmm_handler.py b/src/aws_encryption_sdk/cmm_handler.py index 887d9d79e..fa1786837 100644 --- a/src/aws_encryption_sdk/cmm_handler.py +++ b/src/aws_encryption_sdk/cmm_handler.py @@ -17,9 +17,12 @@ CommitmentPolicyESDK, AlgorithmSuiteIdESDK, ) + except ImportError: pass +from typing import List + from aws_encryption_sdk.exceptions import ( AWSEncryptionSDKClientError, ) @@ -72,7 +75,7 @@ def __init__( elif isinstance(cmm, ICryptographicMaterialsManager): self.mpl_cmm = cmm else: - raise ValueError(f"Invalid CMM passed to CMMHandler: {cmm=}") + raise ValueError(f"Invalid CMM passed to CMMHandler. cmm: {cmm}") def get_encryption_materials( self, @@ -115,7 +118,7 @@ def _create_mpl_get_encryption_materials_input_from_request( @staticmethod def _map_native_commitment_policy_to_mpl_commitment_policy( native_commitment_policy: CommitmentPolicy - ) -> CommitmentPolicyESDK: + ) -> 'CommitmentPolicyESDK': if native_commitment_policy == CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT: return CommitmentPolicyESDK(value="FORBID_ENCRYPT_ALLOW_DECRYPT") elif native_commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT: @@ -123,7 +126,7 @@ def _map_native_commitment_policy_to_mpl_commitment_policy( elif native_commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT: return CommitmentPolicyESDK(value="REQUIRE_ENCRYPT_REQUIRE_DECRYPT") else: - raise ValueError(f"Invalid {native_commitment_policy=}") + raise ValueError(f"Invalid native_commitment_policy: {native_commitment_policy}") def decrypt_materials( self, @@ -146,7 +149,7 @@ def decrypt_materials( raise AWSEncryptionSDKClientError(e) @staticmethod - def _native_algorithm_id_to_mpl_algorithm_id(native_algorithm_id: str) -> AlgorithmSuiteIdESDK: + def _native_algorithm_id_to_mpl_algorithm_id(native_algorithm_id: str) -> 'AlgorithmSuiteIdESDK': # MPL algorithm suite ID = hexstr(native_algorithm_id) padded to 4 digits post-`x`. return AlgorithmSuiteIdESDK(f"{native_algorithm_id:#0{6}x}") @@ -154,7 +157,7 @@ def _native_algorithm_id_to_mpl_algorithm_id(native_algorithm_id: str) -> Algori def _create_mpl_decrypt_materials_input_from_request( request: DecryptionMaterialsRequest ) -> 'DecryptMaterialsInput': - key_blob_list: list[Native_EncryptedDataKey] = request.encrypted_data_keys + key_blob_list: List[Native_EncryptedDataKey] = request.encrypted_data_keys list_edks = [MPL_EncryptedDataKey( key_provider_id=key_blob.key_provider.provider_id, key_provider_info=key_blob.key_provider.key_info, diff --git a/src/aws_encryption_sdk/materials_handlers.py b/src/aws_encryption_sdk/materials_handlers.py index d54e4517b..970963e10 100644 --- a/src/aws_encryption_sdk/materials_handlers.py +++ b/src/aws_encryption_sdk/materials_handlers.py @@ -9,6 +9,8 @@ except ImportError: pass +from typing import Dict, List + from aws_encryption_sdk.materials_managers import ( DecryptionMaterials as Native_DecryptionMaterials, EncryptionMaterials as Native_EncryptionMaterials, @@ -53,7 +55,7 @@ def __init__( elif isinstance(materials, MPL_EncryptionMaterials): self.mpl_materials = materials else: - raise ValueError(f"Invalid EncryptionMaterials passed to EncryptionMaterialsHandler: {materials=}") + raise ValueError(f"Invalid EncryptionMaterials passed to EncryptionMaterialsHandler. materials: {materials}") @property def algorithm(self) -> Algorithm: @@ -68,7 +70,7 @@ def algorithm(self) -> Algorithm: ) @property - def encryption_context(self) -> dict[str, str]: + def encryption_context(self) -> Dict[str, str]: """Materials' encryption context.""" if hasattr(self, "native_materials"): return self.native_materials.encryption_context @@ -76,12 +78,12 @@ def encryption_context(self) -> dict[str, str]: return self.mpl_materials.encryption_context @property - def encrypted_data_keys(self) -> list[Native_EncryptedDataKey]: + def encrypted_data_keys(self) -> List[Native_EncryptedDataKey]: """Materials' encrypted data keys.""" if hasattr(self, "native_materials"): return self.native_materials.encrypted_data_keys else: - mpl_edk_list: list[MPL_EncryptedDataKey] = self.mpl_materials.encrypted_data_keys + mpl_edk_list: List[MPL_EncryptedDataKey] = self.mpl_materials.encrypted_data_keys key_blob_list: set[Native_EncryptedDataKey] = {Native_EncryptedDataKey( key_provider=MasterKeyInfo( provider_id=mpl_edk.key_provider_id, @@ -144,7 +146,7 @@ def __init__( elif isinstance(materials, MPL_DecryptionMaterials): self.mpl_materials = materials else: - raise ValueError(f"Invalid DecryptionMaterials passed to DecryptionMaterialsHandler: {materials=}") + raise ValueError(f"Invalid DecryptionMaterials passed to DecryptionMaterialsHandler. materials: {materials}") @property def data_key(self) -> DataKey: diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 582472025..6a2dc1d27 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -66,6 +66,7 @@ from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager from aws_encryption_sdk.materials_managers.default import DefaultCryptoMaterialsManager from aws_encryption_sdk.structures import MessageHeader +from aws_encryption_sdk.cmm_handler import CMMHandler try: from aws_cryptographic_materialproviders.mpl.client import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig @@ -75,7 +76,6 @@ from aws_cryptographic_materialproviders.mpl.references import ( IKeyring, ) - from aws_encryption_sdk.cmm_handler import CMMHandler _has_mpl = True except ImportError: From a3babfd2ef936cf4595bdaab041fc16e7a005868 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 6 Feb 2024 17:55:40 -0800 Subject: [PATCH 018/422] linter --- src/aws_encryption_sdk/cmm_handler.py | 36 +++++++++----------- src/aws_encryption_sdk/materials_handlers.py | 10 +++--- src/aws_encryption_sdk/streaming_client.py | 18 +++++----- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/aws_encryption_sdk/cmm_handler.py b/src/aws_encryption_sdk/cmm_handler.py index fa1786837..20298f801 100644 --- a/src/aws_encryption_sdk/cmm_handler.py +++ b/src/aws_encryption_sdk/cmm_handler.py @@ -85,38 +85,35 @@ def get_encryption_materials( Returns an EncryptionMaterialsHandler for the configured CMM. :param request: Request for encryption materials """ - if (self._is_using_native_cmm()): + if self._is_using_native_cmm(): return EncryptionMaterialsHandler(self.native_cmm.get_encryption_materials(request)) else: try: - input: GetEncryptionMaterialsInput = CMMHandler._create_mpl_get_encryption_materials_input_from_request( + mpl_input: GetEncryptionMaterialsInput = CMMHandler._native_to_mpl_get_encryption_materials( request ) - output: GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(input) - return EncryptionMaterialsHandler(output.encryption_materials) - except AwsCryptographicMaterialProvidersException as e: + mpl_output: GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) + return EncryptionMaterialsHandler(mpl_output.encryption_materials) + except AwsCryptographicMaterialProvidersException as mpl_exception: # Wrap MPL error into the ESDK error type # so customers only have to catch ESDK error types. - raise AWSEncryptionSDKClientError(e) + raise AWSEncryptionSDKClientError(mpl_exception) @staticmethod - def _create_mpl_get_encryption_materials_input_from_request( + def _native_to_mpl_get_encryption_materials( request: EncryptionMaterialsRequest ) -> 'GetEncryptionMaterialsInput': output: GetEncryptionMaterialsInput = GetEncryptionMaterialsInput( encryption_context=request.encryption_context, - commitment_policy=CMMHandler._map_native_commitment_policy_to_mpl_commitment_policy( + commitment_policy=CMMHandler._native_to_mpl_commmitment_policy( request.commitment_policy ), - # TODO double check this - # optional... maybe this needs to be kwargs?? - # algorithm_suite_id=request.algorithm.algorithm_id, max_plaintext_length=request.plaintext_length, ) return output @staticmethod - def _map_native_commitment_policy_to_mpl_commitment_policy( + def _native_to_mpl_commmitment_policy( native_commitment_policy: CommitmentPolicy ) -> 'CommitmentPolicyESDK': if native_commitment_policy == CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT: @@ -136,17 +133,18 @@ def decrypt_materials( Returns a DecryptionMaterialsHandler for the configured CMM. :param request: Request for decryption materials """ - if (self._is_using_native_cmm()): + if self._is_using_native_cmm(): return DecryptionMaterialsHandler(self.native_cmm.decrypt_materials(request)) else: try: - input: 'DecryptMaterialsInput' = CMMHandler._create_mpl_decrypt_materials_input_from_request(request) - output: 'DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(input) - return DecryptionMaterialsHandler(output.decryption_materials) - except AwsCryptographicMaterialProvidersException as e: + mpl_input: 'DecryptMaterialsInput' = \ + CMMHandler._create_mpl_decrypt_materials_input_from_request(request) + mpl_output: 'DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(mpl_input) + return DecryptionMaterialsHandler(mpl_output.decryption_materials) + except AwsCryptographicMaterialProvidersException as mpl_exception: # Wrap MPL error into the ESDK error type # so customers only have to catch ESDK error types. - raise AWSEncryptionSDKClientError(e) + raise AWSEncryptionSDKClientError(mpl_exception) @staticmethod def _native_algorithm_id_to_mpl_algorithm_id(native_algorithm_id: str) -> 'AlgorithmSuiteIdESDK': @@ -167,7 +165,7 @@ def _create_mpl_decrypt_materials_input_from_request( algorithm_suite_id=CMMHandler._native_algorithm_id_to_mpl_algorithm_id( request.algorithm.algorithm_id ), - commitment_policy=CMMHandler._map_native_commitment_policy_to_mpl_commitment_policy( + commitment_policy=CMMHandler._native_to_mpl_commmitment_policy( request.commitment_policy ), encrypted_data_keys=list_edks, diff --git a/src/aws_encryption_sdk/materials_handlers.py b/src/aws_encryption_sdk/materials_handlers.py index 970963e10..00d67ed71 100644 --- a/src/aws_encryption_sdk/materials_handlers.py +++ b/src/aws_encryption_sdk/materials_handlers.py @@ -9,7 +9,7 @@ except ImportError: pass -from typing import Dict, List +from typing import Dict, List, Set from aws_encryption_sdk.materials_managers import ( DecryptionMaterials as Native_DecryptionMaterials, @@ -55,7 +55,8 @@ def __init__( elif isinstance(materials, MPL_EncryptionMaterials): self.mpl_materials = materials else: - raise ValueError(f"Invalid EncryptionMaterials passed to EncryptionMaterialsHandler. materials: {materials}") + raise ValueError(f"Invalid EncryptionMaterials passed to EncryptionMaterialsHandler.\ + materials: {materials}") @property def algorithm(self) -> Algorithm: @@ -84,7 +85,7 @@ def encrypted_data_keys(self) -> List[Native_EncryptedDataKey]: return self.native_materials.encrypted_data_keys else: mpl_edk_list: List[MPL_EncryptedDataKey] = self.mpl_materials.encrypted_data_keys - key_blob_list: set[Native_EncryptedDataKey] = {Native_EncryptedDataKey( + key_blob_list: Set[Native_EncryptedDataKey] = {Native_EncryptedDataKey( key_provider=MasterKeyInfo( provider_id=mpl_edk.key_provider_id, key_info=mpl_edk.key_provider_info, @@ -146,7 +147,8 @@ def __init__( elif isinstance(materials, MPL_DecryptionMaterials): self.mpl_materials = materials else: - raise ValueError(f"Invalid DecryptionMaterials passed to DecryptionMaterialsHandler. materials: {materials}") + raise ValueError(f"Invalid DecryptionMaterials passed to DecryptionMaterialsHandler.\ + materials: {materials}") @property def data_key(self) -> DataKey: diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 6a2dc1d27..6b977e6e4 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -18,6 +18,7 @@ import io import logging import math +import base64 import attr import six @@ -77,14 +78,14 @@ IKeyring, ) - _has_mpl = True + HAS_MPL = True except ImportError: - _has_mpl = False + HAS_MPL = False _LOGGER = logging.getLogger(__name__) -def _exactly_one_arg_is_not_None(*args): +def _exactly_one_arg_is_not_none(*args): """ Private helper function. Returns `True` if exactly one item in the list is not `None`. @@ -146,7 +147,7 @@ class _ClientConfig(object): # pylint: disable=too-many-instance-attributes key_provider = attr.ib( hash=True, default=None, validator=attr.validators.optional(attr.validators.instance_of(MasterKeyProvider)) ) - if _has_mpl: + if HAS_MPL: keyring = attr.ib( hash=True, default=None, validator=attr.validators.optional(attr.validators.instance_of(IKeyring)) ) @@ -158,14 +159,14 @@ class _ClientConfig(object): # pylint: disable=too-many-instance-attributes ) # DEPRECATED: Value is no longer configurable here. Parameter left here to avoid breaking consumers. def _has_mpl_attrs_post_init(self): - if not _exactly_one_arg_is_not_None(self.materials_manager, self.key_provider, self.keyring): + if not _exactly_one_arg_is_not_none(self.materials_manager, self.key_provider, self.keyring): raise TypeError("Exactly one of keyring, materials_manager, or key_provider must be provided") if self.materials_manager is None: if self.key_provider is not None: # No CMM, provided legacy native `key_provider` => create legacy native DefaultCryptoMaterialsManager self.materials_manager = DefaultCryptoMaterialsManager( master_key_provider=self.key_provider - ) + ) elif self.keyring is not None: # No CMM, provided MPL keyring => create MPL's DefaultCryptographicMaterialsManager if not isinstance(self.keyring, IKeyring): @@ -194,9 +195,9 @@ def _no_mpl_attrs_post_init(self): def __attrs_post_init__(self): """Normalize inputs to crypto material manager.""" - if _has_mpl: + if HAS_MPL: self._has_mpl_attrs_post_init() - elif not _has_mpl: + elif not HAS_MPL: self._no_mpl_attrs_post_init() @@ -918,7 +919,6 @@ def _read_header(self): else: # MPL verification key is NOT key bytes, it is bytes of the compressed point # TODO-MPL: clean this up, least-privilege violation - import base64 if (isinstance(self.config.materials_manager, CMMHandler) and hasattr(self.config.materials_manager, "mpl_cmm")): self.verifier = Verifier.from_encoded_point( From d2c974afc11a0f11a3ab32ad44e17061d640c5d1 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 6 Feb 2024 17:59:21 -0800 Subject: [PATCH 019/422] linter --- setup.py | 1 + src/aws_encryption_sdk/cmm_handler.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/setup.py b/setup.py index 4cd8027cd..084edc09a 100644 --- a/setup.py +++ b/setup.py @@ -39,6 +39,7 @@ def get_requirements(): keywords="aws-encryption-sdk aws kms encryption", license="Apache License 2.0", install_requires=get_requirements(), + # pylint: disable=fixme # TODO: Point at MPL main branch once Python MPL is merged into main. extras_require={ "MPL": ["aws-cryptographic-material-providers @" \ diff --git a/src/aws_encryption_sdk/cmm_handler.py b/src/aws_encryption_sdk/cmm_handler.py index 20298f801..2479038a1 100644 --- a/src/aws_encryption_sdk/cmm_handler.py +++ b/src/aws_encryption_sdk/cmm_handler.py @@ -2,6 +2,8 @@ # These dependencies are only loaded if you install the MPL. try: + # pylint seems to struggle with this condition import + # pylint: disable=unused-import from aws_cryptographic_materialproviders.mpl.errors import ( AwsCryptographicMaterialProvidersException ) From 55b24a83580f880278d0689f57fbfe1bdca285f6 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 09:44:02 -0800 Subject: [PATCH 020/422] isort --- examples/src/basic_encryption.py | 5 --- examples/src/keyrings/hierarchical_keyring.py | 32 +++++--------- src/aws_encryption_sdk/cmm_handler.py | 44 ++++++------------- src/aws_encryption_sdk/materials_handlers.py | 13 ++---- src/aws_encryption_sdk/streaming_client.py | 13 +++--- 5 files changed, 32 insertions(+), 75 deletions(-) diff --git a/examples/src/basic_encryption.py b/examples/src/basic_encryption.py index 7b729feab..cfe8ac791 100644 --- a/examples/src/basic_encryption.py +++ b/examples/src/basic_encryption.py @@ -51,8 +51,3 @@ def cycle_string(key_arn, source_plaintext, botocore_session=None): assert all( pair in decrypted_header.encryption_context.items() for pair in encryptor_header.encryption_context.items() ) - -cycle_string( - "arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f", - "abcdefg", -) \ No newline at end of file diff --git a/examples/src/keyrings/hierarchical_keyring.py b/examples/src/keyrings/hierarchical_keyring.py index e8f662b73..20647bed6 100644 --- a/examples/src/keyrings/hierarchical_keyring.py +++ b/examples/src/keyrings/hierarchical_keyring.py @@ -4,43 +4,32 @@ """Example showing basic encryption and decryption of a value already in memory.""" -import aws_encryption_sdk -from aws_encryption_sdk import CommitmentPolicy -import boto3 - import sys -from aws_encryption_sdk.exceptions import ( - AWSEncryptionSDKClientError, - SerializationError, -) +import boto3 + +import aws_encryption_sdk +from aws_encryption_sdk import CommitmentPolicy +from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError, SerializationError module_root_dir = '/'.join(__file__.split("/")[:-1]) sys.path.append(module_root_dir) import aws_cryptographic_materialproviders - +from aws_cryptographic_materialproviders.keystore.client import KeyStore +from aws_cryptographic_materialproviders.keystore.config import KeyStoreConfig +from aws_cryptographic_materialproviders.keystore.models import CreateKeyInput, KMSConfigurationKmsKeyArn from aws_cryptographic_materialproviders.mpl.client import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig from aws_cryptographic_materialproviders.mpl.models import ( - CreateAwsKmsHierarchicalKeyringInput, CacheTypeDefault, + CreateAwsKmsHierarchicalKeyringInput, DefaultCache, GetBranchKeyIdInput, GetBranchKeyIdOutput, ) -from aws_cryptographic_materialproviders.mpl.references import ( - IKeyring, - IBranchKeyIdSupplier, -) - -from aws_cryptographic_materialproviders.keystore.client import KeyStore -from aws_cryptographic_materialproviders.keystore.config import KeyStoreConfig -from aws_cryptographic_materialproviders.keystore.models import ( - CreateKeyInput, - KMSConfigurationKmsKeyArn, -) +from aws_cryptographic_materialproviders.mpl.references import IBranchKeyIdSupplier, IKeyring EXAMPLE_DATA: bytes = b"Hello World" @@ -241,6 +230,7 @@ def get_branch_key_id( # hack in a test import botocore + encrypt_and_decrypt_with_keyring( "KeyStoreDdbTable", "KeyStoreDdbTable", diff --git a/src/aws_encryption_sdk/cmm_handler.py b/src/aws_encryption_sdk/cmm_handler.py index 2479038a1..5bac15b87 100644 --- a/src/aws_encryption_sdk/cmm_handler.py +++ b/src/aws_encryption_sdk/cmm_handler.py @@ -2,49 +2,31 @@ # These dependencies are only loaded if you install the MPL. try: - # pylint seems to struggle with this condition import + # pylint seems to struggle with this conditional import # pylint: disable=unused-import - from aws_cryptographic_materialproviders.mpl.errors import ( - AwsCryptographicMaterialProvidersException - ) - from aws_cryptographic_materialproviders.mpl.references import ( - ICryptographicMaterialsManager, - ) + from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException from aws_cryptographic_materialproviders.mpl.models import ( - GetEncryptionMaterialsInput, - GetEncryptionMaterialsOutput, + AlgorithmSuiteIdESDK, + CommitmentPolicyESDK, DecryptMaterialsInput, DecryptMaterialsOutput, EncryptedDataKey as MPL_EncryptedDataKey, - CommitmentPolicyESDK, - AlgorithmSuiteIdESDK, + GetEncryptionMaterialsInput, + GetEncryptionMaterialsOutput, ) + from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager except ImportError: pass from typing import List -from aws_encryption_sdk.exceptions import ( - AWSEncryptionSDKClientError, -) -from aws_encryption_sdk.materials_managers import ( - DecryptionMaterialsRequest, - EncryptionMaterialsRequest, -) -from aws_encryption_sdk.materials_managers.base import ( - CryptoMaterialsManager, -) -from aws_encryption_sdk.materials_handlers import ( - EncryptionMaterialsHandler, - DecryptionMaterialsHandler, -) -from aws_encryption_sdk.structures import ( - EncryptedDataKey as Native_EncryptedDataKey, -) -from aws_encryption_sdk.identifiers import ( - CommitmentPolicy, -) +from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError +from aws_encryption_sdk.identifiers import CommitmentPolicy +from aws_encryption_sdk.materials_handlers import DecryptionMaterialsHandler, EncryptionMaterialsHandler +from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest +from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager +from aws_encryption_sdk.structures import EncryptedDataKey as Native_EncryptedDataKey # TODO-MPL Should this implement interface..? seems like yes since it implements all of interface methods diff --git a/src/aws_encryption_sdk/materials_handlers.py b/src/aws_encryption_sdk/materials_handlers.py index 00d67ed71..57f54144e 100644 --- a/src/aws_encryption_sdk/materials_handlers.py +++ b/src/aws_encryption_sdk/materials_handlers.py @@ -3,27 +3,20 @@ try: from aws_cryptographic_materialproviders.mpl.models import ( DecryptionMaterials as MPL_DecryptionMaterials, - EncryptionMaterials as MPL_EncryptionMaterials, EncryptedDataKey as MPL_EncryptedDataKey, + EncryptionMaterials as MPL_EncryptionMaterials, ) except ImportError: pass from typing import Dict, List, Set +from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite from aws_encryption_sdk.materials_managers import ( DecryptionMaterials as Native_DecryptionMaterials, EncryptionMaterials as Native_EncryptionMaterials, ) -from aws_encryption_sdk.identifiers import ( - Algorithm, - AlgorithmSuite, -) -from aws_encryption_sdk.structures import ( - DataKey, - EncryptedDataKey as Native_EncryptedDataKey, - MasterKeyInfo, -) +from aws_encryption_sdk.structures import DataKey, EncryptedDataKey as Native_EncryptedDataKey, MasterKeyInfo def _mpl_algorithm_id_to_native_algorithm_id(mpl_algorithm_id: str): diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 6b977e6e4..afe9987ff 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -14,16 +14,17 @@ from __future__ import division import abc +import base64 import hmac import io import logging import math -import base64 import attr import six import aws_encryption_sdk.internal.utils +from aws_encryption_sdk.cmm_handler import CMMHandler from aws_encryption_sdk.exceptions import ( ActionNotAllowedError, AWSEncryptionSDKClientError, @@ -67,16 +68,12 @@ from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager from aws_encryption_sdk.materials_managers.default import DefaultCryptoMaterialsManager from aws_encryption_sdk.structures import MessageHeader -from aws_encryption_sdk.cmm_handler import CMMHandler + try: from aws_cryptographic_materialproviders.mpl.client import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig - from aws_cryptographic_materialproviders.mpl.models import ( - CreateDefaultCryptographicMaterialsManagerInput - ) - from aws_cryptographic_materialproviders.mpl.references import ( - IKeyring, - ) + from aws_cryptographic_materialproviders.mpl.models import CreateDefaultCryptographicMaterialsManagerInput + from aws_cryptographic_materialproviders.mpl.references import IKeyring HAS_MPL = True except ImportError: From 7e5fa4837f252f8b038efc51567d04f90d6510d8 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 09:54:14 -0800 Subject: [PATCH 021/422] flake8 examples --- examples/src/keyrings/hierarchical_keyring.py | 34 ++++++------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/examples/src/keyrings/hierarchical_keyring.py b/examples/src/keyrings/hierarchical_keyring.py index 20647bed6..81d02f786 100644 --- a/examples/src/keyrings/hierarchical_keyring.py +++ b/examples/src/keyrings/hierarchical_keyring.py @@ -1,8 +1,5 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 - - - """Example showing basic encryption and decryption of a value already in memory.""" import sys @@ -10,13 +7,8 @@ import aws_encryption_sdk from aws_encryption_sdk import CommitmentPolicy -from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError, SerializationError - -module_root_dir = '/'.join(__file__.split("/")[:-1]) - -sys.path.append(module_root_dir) +from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError -import aws_cryptographic_materialproviders from aws_cryptographic_materialproviders.keystore.client import KeyStore from aws_cryptographic_materialproviders.keystore.config import KeyStoreConfig from aws_cryptographic_materialproviders.keystore.models import CreateKeyInput, KMSConfigurationKmsKeyArn @@ -31,13 +23,18 @@ ) from aws_cryptographic_materialproviders.mpl.references import IBranchKeyIdSupplier, IKeyring +module_root_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(module_root_dir) + EXAMPLE_DATA: bytes = b"Hello World" + def encrypt_and_decrypt_with_keyring( - key_store_table_name: str, - logical_key_store_name: str, - kms_key_id: str - ): + key_store_table_name: str, + logical_key_store_name: str, + kms_key_id: str +): # 1. Instantiate the encryption SDK client. # This builds the client with the REQUIRE_ENCRYPT_REQUIRE_DECRYPT commitment policy, @@ -90,7 +87,7 @@ def get_branch_key_id( if b"tenant" not in encryption_context: raise ValueError("EncryptionContext invalid, does not contain expected tenant key value pair.") - + tenant_key_id: str = encryption_context.get(b"tenant") branch_key_id: str @@ -227,12 +224,3 @@ def get_branch_key_id( assert plaintext_bytes_B == EXAMPLE_DATA # Also, a thread-safe example ig - -# hack in a test -import botocore - -encrypt_and_decrypt_with_keyring( - "KeyStoreDdbTable", - "KeyStoreDdbTable", - "arn:aws:kms:us-west-2:370957321024:key/9d989aa2-2f9c-438c-a745-cc57d3ad0126" -) \ No newline at end of file From 055deabd332af255f40df7fb52b80063df268f06 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 09:58:05 -0800 Subject: [PATCH 022/422] isort + flake8 --- examples/src/keyrings/hierarchical_keyring.py | 10 +++++----- examples/src/keyrings/module_.py | 1 + examples/src/module_.py | 1 + examples/test/examples_test_utils.py | 2 +- examples/test/test_i_basic_encryption.py | 1 - ..._i_basic_file_encryption_with_multiple_providers.py | 4 +--- ...st_i_basic_file_encryption_with_raw_key_provider.py | 1 - examples/test/test_i_data_key_caching_basic.py | 1 - examples/test/test_i_discovery_kms_provider.py | 4 +--- examples/test/test_i_mrk_aware_kms_provider.py | 4 +--- examples/test/test_i_multiple_kms_cmk.py | 4 +--- examples/test/test_i_one_kms_cmk.py | 4 +--- examples/test/test_i_one_kms_cmk_streaming_data.py | 1 - examples/test/test_i_one_kms_cmk_unsigned.py | 4 +--- examples/test/test_i_set_commitment.py | 4 +--- 15 files changed, 15 insertions(+), 31 deletions(-) diff --git a/examples/src/keyrings/hierarchical_keyring.py b/examples/src/keyrings/hierarchical_keyring.py index 81d02f786..acc594cc3 100644 --- a/examples/src/keyrings/hierarchical_keyring.py +++ b/examples/src/keyrings/hierarchical_keyring.py @@ -4,11 +4,6 @@ import sys import boto3 - -import aws_encryption_sdk -from aws_encryption_sdk import CommitmentPolicy -from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError - from aws_cryptographic_materialproviders.keystore.client import KeyStore from aws_cryptographic_materialproviders.keystore.config import KeyStoreConfig from aws_cryptographic_materialproviders.keystore.models import CreateKeyInput, KMSConfigurationKmsKeyArn @@ -23,6 +18,10 @@ ) from aws_cryptographic_materialproviders.mpl.references import IBranchKeyIdSupplier, IKeyring +import aws_encryption_sdk +from aws_encryption_sdk import CommitmentPolicy +from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError + module_root_dir = '/'.join(__file__.split("/")[:-1]) sys.path.append(module_root_dir) @@ -35,6 +34,7 @@ def encrypt_and_decrypt_with_keyring( logical_key_store_name: str, kms_key_id: str ): + """Creates a hierarchical keyring using the provided resources, then encrypts and decrypts a string with it.""" # 1. Instantiate the encryption SDK client. # This builds the client with the REQUIRE_ENCRYPT_REQUIRE_DECRYPT commitment policy, diff --git a/examples/src/keyrings/module_.py b/examples/src/keyrings/module_.py index e69de29bb..2f64c8e0f 100644 --- a/examples/src/keyrings/module_.py +++ b/examples/src/keyrings/module_.py @@ -0,0 +1 @@ +"""Should remove this.""" \ No newline at end of file diff --git a/examples/src/module_.py b/examples/src/module_.py index e69de29bb..2f64c8e0f 100644 --- a/examples/src/module_.py +++ b/examples/src/module_.py @@ -0,0 +1 @@ +"""Should remove this.""" \ No newline at end of file diff --git a/examples/test/examples_test_utils.py b/examples/test/examples_test_utils.py index 8a51f21c8..08e8cf2f5 100644 --- a/examples/test/examples_test_utils.py +++ b/examples/test/examples_test_utils.py @@ -49,7 +49,7 @@ from integration_test_utils import ( # noqa pylint: disable=unused-import,import-error get_cmk_arn, - get_second_cmk_arn, get_mrk_arn, + get_second_cmk_arn, get_second_mrk_arn, ) diff --git a/examples/test/test_i_basic_encryption.py b/examples/test/test_i_basic_encryption.py index f2a4fab51..aa32d61fa 100644 --- a/examples/test/test_i_basic_encryption.py +++ b/examples/test/test_i_basic_encryption.py @@ -17,7 +17,6 @@ from ..src.basic_encryption import cycle_string from .examples_test_utils import get_cmk_arn, static_plaintext - pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_basic_file_encryption_with_multiple_providers.py b/examples/test/test_i_basic_file_encryption_with_multiple_providers.py index 282a272ab..0792f4958 100644 --- a/examples/test/test_i_basic_file_encryption_with_multiple_providers.py +++ b/examples/test/test_i_basic_file_encryption_with_multiple_providers.py @@ -18,9 +18,7 @@ import pytest from ..src.basic_file_encryption_with_multiple_providers import cycle_file -from .examples_test_utils import get_cmk_arn -from .examples_test_utils import static_plaintext - +from .examples_test_utils import get_cmk_arn, static_plaintext pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_basic_file_encryption_with_raw_key_provider.py b/examples/test/test_i_basic_file_encryption_with_raw_key_provider.py index 710c0ccac..046b7f964 100644 --- a/examples/test/test_i_basic_file_encryption_with_raw_key_provider.py +++ b/examples/test/test_i_basic_file_encryption_with_raw_key_provider.py @@ -19,7 +19,6 @@ from ..src.basic_file_encryption_with_raw_key_provider import cycle_file from .examples_test_utils import static_plaintext - pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_data_key_caching_basic.py b/examples/test/test_i_data_key_caching_basic.py index 734c35692..7a30f4e53 100644 --- a/examples/test/test_i_data_key_caching_basic.py +++ b/examples/test/test_i_data_key_caching_basic.py @@ -16,7 +16,6 @@ from ..src.data_key_caching_basic import encrypt_with_caching from .examples_test_utils import get_cmk_arn - pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_discovery_kms_provider.py b/examples/test/test_i_discovery_kms_provider.py index e9a1c6e71..0f64cbf59 100644 --- a/examples/test/test_i_discovery_kms_provider.py +++ b/examples/test/test_i_discovery_kms_provider.py @@ -16,9 +16,7 @@ import pytest from ..src.discovery_kms_provider import encrypt_decrypt -from .examples_test_utils import get_cmk_arn -from .examples_test_utils import static_plaintext - +from .examples_test_utils import get_cmk_arn, static_plaintext pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_mrk_aware_kms_provider.py b/examples/test/test_i_mrk_aware_kms_provider.py index 8e7a003f8..a90101fa8 100644 --- a/examples/test/test_i_mrk_aware_kms_provider.py +++ b/examples/test/test_i_mrk_aware_kms_provider.py @@ -15,9 +15,7 @@ import pytest from ..src.mrk_aware_kms_provider import encrypt_decrypt -from .examples_test_utils import get_mrk_arn, get_second_mrk_arn -from .examples_test_utils import static_plaintext - +from .examples_test_utils import get_mrk_arn, get_second_mrk_arn, static_plaintext pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_multiple_kms_cmk.py b/examples/test/test_i_multiple_kms_cmk.py index 39369cbc6..2915a0fd7 100644 --- a/examples/test/test_i_multiple_kms_cmk.py +++ b/examples/test/test_i_multiple_kms_cmk.py @@ -16,9 +16,7 @@ import pytest from ..src.multiple_kms_cmk import encrypt_decrypt -from .examples_test_utils import get_cmk_arn, get_second_cmk_arn -from .examples_test_utils import static_plaintext - +from .examples_test_utils import get_cmk_arn, get_second_cmk_arn, static_plaintext pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_one_kms_cmk.py b/examples/test/test_i_one_kms_cmk.py index 71ce74d3d..96dd48dae 100644 --- a/examples/test/test_i_one_kms_cmk.py +++ b/examples/test/test_i_one_kms_cmk.py @@ -16,9 +16,7 @@ import pytest from ..src.one_kms_cmk import encrypt_decrypt -from .examples_test_utils import get_cmk_arn -from .examples_test_utils import static_plaintext - +from .examples_test_utils import get_cmk_arn, static_plaintext pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_one_kms_cmk_streaming_data.py b/examples/test/test_i_one_kms_cmk_streaming_data.py index b22fa4232..f0a3094d0 100644 --- a/examples/test/test_i_one_kms_cmk_streaming_data.py +++ b/examples/test/test_i_one_kms_cmk_streaming_data.py @@ -20,7 +20,6 @@ from ..src.one_kms_cmk_streaming_data import encrypt_decrypt_stream from .examples_test_utils import get_cmk_arn, static_plaintext - pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_one_kms_cmk_unsigned.py b/examples/test/test_i_one_kms_cmk_unsigned.py index 8a2758c96..41f16473d 100644 --- a/examples/test/test_i_one_kms_cmk_unsigned.py +++ b/examples/test/test_i_one_kms_cmk_unsigned.py @@ -16,9 +16,7 @@ import pytest from ..src.one_kms_cmk_unsigned import encrypt_decrypt -from .examples_test_utils import get_cmk_arn -from .examples_test_utils import static_plaintext - +from .examples_test_utils import get_cmk_arn, static_plaintext pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_set_commitment.py b/examples/test/test_i_set_commitment.py index 96247334b..c14a379bf 100644 --- a/examples/test/test_i_set_commitment.py +++ b/examples/test/test_i_set_commitment.py @@ -16,9 +16,7 @@ import pytest from ..src.set_commitment import encrypt_decrypt -from .examples_test_utils import get_cmk_arn -from .examples_test_utils import static_plaintext - +from .examples_test_utils import get_cmk_arn, static_plaintext pytestmark = [pytest.mark.examples] From 6cf01d4c6cfd0b67656a1faba6af4894675caaba Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 10:08:03 -0800 Subject: [PATCH 023/422] flake8/pylint examples --- examples/src/keyrings/hierarchical_keyring.py | 1 - examples/src/keyrings/module_.py | 2 +- examples/src/module_.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/src/keyrings/hierarchical_keyring.py b/examples/src/keyrings/hierarchical_keyring.py index acc594cc3..76aef25e0 100644 --- a/examples/src/keyrings/hierarchical_keyring.py +++ b/examples/src/keyrings/hierarchical_keyring.py @@ -35,7 +35,6 @@ def encrypt_and_decrypt_with_keyring( kms_key_id: str ): """Creates a hierarchical keyring using the provided resources, then encrypts and decrypts a string with it.""" - # 1. Instantiate the encryption SDK client. # This builds the client with the REQUIRE_ENCRYPT_REQUIRE_DECRYPT commitment policy, # which enforces that this client only encrypts using committing algorithm suites and enforces diff --git a/examples/src/keyrings/module_.py b/examples/src/keyrings/module_.py index 2f64c8e0f..d9a8c058f 100644 --- a/examples/src/keyrings/module_.py +++ b/examples/src/keyrings/module_.py @@ -1 +1 @@ -"""Should remove this.""" \ No newline at end of file +"""Should remove this.""" diff --git a/examples/src/module_.py b/examples/src/module_.py index 2f64c8e0f..d9a8c058f 100644 --- a/examples/src/module_.py +++ b/examples/src/module_.py @@ -1 +1 @@ -"""Should remove this.""" \ No newline at end of file +"""Should remove this.""" From 00cfed1f368752b872ebb25631331e04e4660893 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 10:12:24 -0800 Subject: [PATCH 024/422] reset tests --- examples/test/examples_test_utils.py | 2 +- examples/test/test_i_basic_encryption.py | 1 + .../test_i_basic_file_encryption_with_multiple_providers.py | 4 +++- .../test_i_basic_file_encryption_with_raw_key_provider.py | 1 + examples/test/test_i_data_key_caching_basic.py | 1 + examples/test/test_i_discovery_kms_provider.py | 4 +++- examples/test/test_i_mrk_aware_kms_provider.py | 4 +++- examples/test/test_i_multiple_kms_cmk.py | 4 +++- examples/test/test_i_one_kms_cmk.py | 4 +++- examples/test/test_i_one_kms_cmk_streaming_data.py | 1 + examples/test/test_i_one_kms_cmk_unsigned.py | 4 +++- examples/test/test_i_set_commitment.py | 4 +++- 12 files changed, 26 insertions(+), 8 deletions(-) diff --git a/examples/test/examples_test_utils.py b/examples/test/examples_test_utils.py index 08e8cf2f5..8a51f21c8 100644 --- a/examples/test/examples_test_utils.py +++ b/examples/test/examples_test_utils.py @@ -49,7 +49,7 @@ from integration_test_utils import ( # noqa pylint: disable=unused-import,import-error get_cmk_arn, - get_mrk_arn, get_second_cmk_arn, + get_mrk_arn, get_second_mrk_arn, ) diff --git a/examples/test/test_i_basic_encryption.py b/examples/test/test_i_basic_encryption.py index aa32d61fa..f2a4fab51 100644 --- a/examples/test/test_i_basic_encryption.py +++ b/examples/test/test_i_basic_encryption.py @@ -17,6 +17,7 @@ from ..src.basic_encryption import cycle_string from .examples_test_utils import get_cmk_arn, static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_basic_file_encryption_with_multiple_providers.py b/examples/test/test_i_basic_file_encryption_with_multiple_providers.py index 0792f4958..282a272ab 100644 --- a/examples/test/test_i_basic_file_encryption_with_multiple_providers.py +++ b/examples/test/test_i_basic_file_encryption_with_multiple_providers.py @@ -18,7 +18,9 @@ import pytest from ..src.basic_file_encryption_with_multiple_providers import cycle_file -from .examples_test_utils import get_cmk_arn, static_plaintext +from .examples_test_utils import get_cmk_arn +from .examples_test_utils import static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_basic_file_encryption_with_raw_key_provider.py b/examples/test/test_i_basic_file_encryption_with_raw_key_provider.py index 046b7f964..710c0ccac 100644 --- a/examples/test/test_i_basic_file_encryption_with_raw_key_provider.py +++ b/examples/test/test_i_basic_file_encryption_with_raw_key_provider.py @@ -19,6 +19,7 @@ from ..src.basic_file_encryption_with_raw_key_provider import cycle_file from .examples_test_utils import static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_data_key_caching_basic.py b/examples/test/test_i_data_key_caching_basic.py index 7a30f4e53..734c35692 100644 --- a/examples/test/test_i_data_key_caching_basic.py +++ b/examples/test/test_i_data_key_caching_basic.py @@ -16,6 +16,7 @@ from ..src.data_key_caching_basic import encrypt_with_caching from .examples_test_utils import get_cmk_arn + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_discovery_kms_provider.py b/examples/test/test_i_discovery_kms_provider.py index 0f64cbf59..e9a1c6e71 100644 --- a/examples/test/test_i_discovery_kms_provider.py +++ b/examples/test/test_i_discovery_kms_provider.py @@ -16,7 +16,9 @@ import pytest from ..src.discovery_kms_provider import encrypt_decrypt -from .examples_test_utils import get_cmk_arn, static_plaintext +from .examples_test_utils import get_cmk_arn +from .examples_test_utils import static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_mrk_aware_kms_provider.py b/examples/test/test_i_mrk_aware_kms_provider.py index a90101fa8..8e7a003f8 100644 --- a/examples/test/test_i_mrk_aware_kms_provider.py +++ b/examples/test/test_i_mrk_aware_kms_provider.py @@ -15,7 +15,9 @@ import pytest from ..src.mrk_aware_kms_provider import encrypt_decrypt -from .examples_test_utils import get_mrk_arn, get_second_mrk_arn, static_plaintext +from .examples_test_utils import get_mrk_arn, get_second_mrk_arn +from .examples_test_utils import static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_multiple_kms_cmk.py b/examples/test/test_i_multiple_kms_cmk.py index 2915a0fd7..39369cbc6 100644 --- a/examples/test/test_i_multiple_kms_cmk.py +++ b/examples/test/test_i_multiple_kms_cmk.py @@ -16,7 +16,9 @@ import pytest from ..src.multiple_kms_cmk import encrypt_decrypt -from .examples_test_utils import get_cmk_arn, get_second_cmk_arn, static_plaintext +from .examples_test_utils import get_cmk_arn, get_second_cmk_arn +from .examples_test_utils import static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_one_kms_cmk.py b/examples/test/test_i_one_kms_cmk.py index 96dd48dae..71ce74d3d 100644 --- a/examples/test/test_i_one_kms_cmk.py +++ b/examples/test/test_i_one_kms_cmk.py @@ -16,7 +16,9 @@ import pytest from ..src.one_kms_cmk import encrypt_decrypt -from .examples_test_utils import get_cmk_arn, static_plaintext +from .examples_test_utils import get_cmk_arn +from .examples_test_utils import static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_one_kms_cmk_streaming_data.py b/examples/test/test_i_one_kms_cmk_streaming_data.py index f0a3094d0..b22fa4232 100644 --- a/examples/test/test_i_one_kms_cmk_streaming_data.py +++ b/examples/test/test_i_one_kms_cmk_streaming_data.py @@ -20,6 +20,7 @@ from ..src.one_kms_cmk_streaming_data import encrypt_decrypt_stream from .examples_test_utils import get_cmk_arn, static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_one_kms_cmk_unsigned.py b/examples/test/test_i_one_kms_cmk_unsigned.py index 41f16473d..8a2758c96 100644 --- a/examples/test/test_i_one_kms_cmk_unsigned.py +++ b/examples/test/test_i_one_kms_cmk_unsigned.py @@ -16,7 +16,9 @@ import pytest from ..src.one_kms_cmk_unsigned import encrypt_decrypt -from .examples_test_utils import get_cmk_arn, static_plaintext +from .examples_test_utils import get_cmk_arn +from .examples_test_utils import static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_set_commitment.py b/examples/test/test_i_set_commitment.py index c14a379bf..96247334b 100644 --- a/examples/test/test_i_set_commitment.py +++ b/examples/test/test_i_set_commitment.py @@ -16,7 +16,9 @@ import pytest from ..src.set_commitment import encrypt_decrypt -from .examples_test_utils import get_cmk_arn, static_plaintext +from .examples_test_utils import get_cmk_arn +from .examples_test_utils import static_plaintext + pytestmark = [pytest.mark.examples] From 61bbb3b474bbff6b360cbea867245a2c28405659 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 11:05:51 -0800 Subject: [PATCH 025/422] extend mpl --- .github/workflows/ci_tests.yaml | 12 ++++++++++ .../keyrings/test_i_hierarchical_keyring.py | 12 ++++++++++ tox.ini | 24 +++++++++++++++++-- 3 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 examples/test/keyrings/test_i_hierarchical_keyring.py diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 9d491203c..f1701de76 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -45,12 +45,24 @@ jobs: # Enable them once we sort how to provide them. # - integ # - examples + optional_dependency: + - "" + - mpl exclude: # x86 builds are only meaningful for Windows - os: ubuntu-latest architecture: x86 - os: macos-latest architecture: x86 + # MPL is not supported on <3.11 + - python: 3.7 + optional_dependency: mpl + - python: 3.8 + optional_dependency: mpl + - python: 3.9 + optional_dependency: mpl + - python: 3.10 + optional_dependency: mpl steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 diff --git a/examples/test/keyrings/test_i_hierarchical_keyring.py b/examples/test/keyrings/test_i_hierarchical_keyring.py new file mode 100644 index 000000000..5df72383f --- /dev/null +++ b/examples/test/keyrings/test_i_hierarchical_keyring.py @@ -0,0 +1,12 @@ +"""Unit test suite for the hierarchical keyring example.""" +import pytest + +from ..src.keyrings.hierarchical_keyring import encrypt_and_decrypt_with_keyring + +pytestmark = [pytest.mark.examples] + + +def test_encrypt_and_decrypt_with_keyring(): + key_store_table_name = "KeyStoreDdbTable" + key_arn = "arn:aws:kms:us-west-2:370957321024:key/9d989aa2-2f9c-438c-a745-cc57d3ad0126" + encrypt_and_decrypt_with_keyring(key_store_table_name, key_store_table_name, key_arn) diff --git a/tox.ini b/tox.ini index 9ca7a0cd6..c90a6fcd6 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,10 @@ [tox] envlist = - py{37,38,39,310,311,312}-{local,integ,accept,examples}, nocmk, + # <3.11: run all non-MPL tests + py{37,38,39,310}-{local,integ,accept,examples}, + # >=3.11: run all MPL tests and non-MPL tests + py{311,312}-{local,integ,accept,examples}{,-mpl}, + nocmk, bandit, doc8, readme, docs, {flake8,pylint}{,-tests,-examples}, isort-check, black-check, @@ -61,12 +65,17 @@ passenv = # Pass through custom pip config file settings PIP_CONFIG_FILE sitepackages = False -deps = -rdev_requirements/test-requirements.txt +deps = + -rdev_requirements/test-requirements.txt + # install the MPL if in environment + mpl: aws-cryptographic-material-providers>=0.0.1 commands = local: {[testenv:base-command]commands} test/ -m local integ: {[testenv:base-command]commands} test/ -m integ accept: {[testenv:base-command]commands} test/ -m accept examples: {[testenv:base-command]commands} examples/test/ -m examples + # append MPL examples to base examples command + examples-mpl: {[testenv:examples]commands} examples/mpl/test/ all: {[testenv:base-command]commands} test/ examples/test/ manual: {[testenv:base-command]commands} @@ -134,6 +143,17 @@ sitepackages = {[testenv:test-upstream-requirements-base]sitepackages} recreate = {[testenv:test-upstream-requirements-base]recreate} commands = {[testenv:test-upstream-requirements-base]commands} +# Test MPL +[testenv:py311-local-mpl] +basepython = {[testenv:pylint]basepython} +deps = {[testenv:pylint]deps} +commands = + pylint \ + --rcfile=test/pylintrc \ + test/unit/ \ + test/functional/ \ + test/integration/ + # Linters [testenv:flake8] basepython = python3 From 4d53ad695908384c6e3705fc7ea5982ed7be9d8f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 11:15:27 -0800 Subject: [PATCH 026/422] mpl gha --- .github/workflows/ci_tests.yaml | 16 ++++++++-------- tox.ini | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index f1701de76..3d22ab05f 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -45,9 +45,9 @@ jobs: # Enable them once we sort how to provide them. # - integ # - examples - optional_dependency: + optional_mpl_dependency: - "" - - mpl + - -mpl exclude: # x86 builds are only meaningful for Windows - os: ubuntu-latest @@ -56,13 +56,13 @@ jobs: architecture: x86 # MPL is not supported on <3.11 - python: 3.7 - optional_dependency: mpl + optional_mpl_dependency: mpl - python: 3.8 - optional_dependency: mpl + optional_mpl_dependency: mpl - python: 3.9 - optional_dependency: mpl + optional_mpl_dependency: mpl - python: 3.10 - optional_dependency: mpl + optional_mpl_dependency: mpl steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 @@ -74,7 +74,7 @@ jobs: pip install --upgrade -r dev_requirements/ci-requirements.txt - name: run test env: - TOXENV: ${{ matrix.category }} + TOXENV: ${{ matrix.category }}${{ matrix.optional_mpl_dependency }} run: tox -- -vv upstream-py37: runs-on: ubuntu-latest @@ -114,5 +114,5 @@ jobs: pip install --upgrade -r dev_requirements/ci-requirements.txt - name: run test env: - TOXENV: ${{ matrix.category }} + TOXENV: ${{ matrix.category }}${{ matrix.optional_mpl_dependency }} run: tox -- -vv diff --git a/tox.ini b/tox.ini index c90a6fcd6..8f12ab8e7 100644 --- a/tox.ini +++ b/tox.ini @@ -75,7 +75,7 @@ commands = accept: {[testenv:base-command]commands} test/ -m accept examples: {[testenv:base-command]commands} examples/test/ -m examples # append MPL examples to base examples command - examples-mpl: {[testenv:examples]commands} examples/mpl/test/ + examples-mpl: {[testenv:base-command]commands} examples/test/ examples/mpl/test -m examples all: {[testenv:base-command]commands} test/ examples/test/ manual: {[testenv:base-command]commands} From c1736d3e91d3d78cd3412ab5c9095eccd41340d6 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 11:26:21 -0800 Subject: [PATCH 027/422] debug --- .github/workflows/ci_tests.yaml | 8 ++++---- tox.ini | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 3d22ab05f..603f54371 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -56,13 +56,13 @@ jobs: architecture: x86 # MPL is not supported on <3.11 - python: 3.7 - optional_mpl_dependency: mpl + optional_mpl_dependency: -mpl - python: 3.8 - optional_mpl_dependency: mpl + optional_mpl_dependency: -mpl - python: 3.9 - optional_mpl_dependency: mpl + optional_mpl_dependency: -mpl - python: 3.10 - optional_mpl_dependency: mpl + optional_mpl_dependency: -mpl steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 diff --git a/tox.ini b/tox.ini index 8f12ab8e7..37fbbae51 100644 --- a/tox.ini +++ b/tox.ini @@ -68,7 +68,9 @@ sitepackages = False deps = -rdev_requirements/test-requirements.txt # install the MPL if in environment - mpl: aws-cryptographic-material-providers>=0.0.1 + mpl: "aws-cryptographic-material-providers @" \ + "git+https://github.com/aws/aws-cryptographic-material-providers-library.git@" \ + "lucmcdon/python-mpl#subdirectory=AwsCryptographicMaterialProviders/runtimes/python" commands = local: {[testenv:base-command]commands} test/ -m local integ: {[testenv:base-command]commands} test/ -m integ From 9991789b79842aa0812c84bcc3c33a6b36c2f182 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 12:40:53 -0800 Subject: [PATCH 028/422] debug --- tox.ini | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 37fbbae51..ef0f6fa29 100644 --- a/tox.ini +++ b/tox.ini @@ -68,9 +68,7 @@ sitepackages = False deps = -rdev_requirements/test-requirements.txt # install the MPL if in environment - mpl: "aws-cryptographic-material-providers @" \ - "git+https://github.com/aws/aws-cryptographic-material-providers-library.git@" \ - "lucmcdon/python-mpl#subdirectory=AwsCryptographicMaterialProviders/runtimes/python" + mpl: "aws-cryptographic-material-providers @git+https://github.com/aws/aws-cryptographic-material-providers-library.git@lucmcdon/python-mpl#subdirectory=AwsCryptographicMaterialProviders/runtimes/python" commands = local: {[testenv:base-command]commands} test/ -m local integ: {[testenv:base-command]commands} test/ -m integ From a501e8f07d1bd125fd77e6b5c28710eb19402e9a Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 12:53:41 -0800 Subject: [PATCH 029/422] debug --- .../keyrings/test_i_hierarchical_keyring.py | 2 +- tox.ini | 38 ++++++++++--------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/examples/test/keyrings/test_i_hierarchical_keyring.py b/examples/test/keyrings/test_i_hierarchical_keyring.py index 5df72383f..d80bb565d 100644 --- a/examples/test/keyrings/test_i_hierarchical_keyring.py +++ b/examples/test/keyrings/test_i_hierarchical_keyring.py @@ -1,7 +1,7 @@ """Unit test suite for the hierarchical keyring example.""" import pytest -from ..src.keyrings.hierarchical_keyring import encrypt_and_decrypt_with_keyring +from ...src.keyrings.hierarchical_keyring import encrypt_and_decrypt_with_keyring pytestmark = [pytest.mark.examples] diff --git a/tox.ini b/tox.ini index ef0f6fa29..d06cbab2e 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,7 @@ envlist = isort-check, black-check, # prone to false positives vulture +ignore_base_python_conflict = true # Additional test environments: # @@ -47,28 +48,29 @@ envlist = commands = pytest --basetemp={envtmpdir} -l {posargs} [testenv] -passenv = - # Identifies AWS KMS key id to use in integration tests - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID \ - # Identifies a second AWS KMS key id to use in integration tests - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2 \ - # Identifies AWS KMS MRK key id to use in integration tests - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1 \ - # Identifies a related AWS KMS MRK key id to use in integration tests - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2 \ - # Pass through AWS credentials - AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN \ - # AWS Role access in CodeBuild is via the contaner URI - AWS_CONTAINER_CREDENTIALS_RELATIVE_URI \ - # Pass through AWS profile name (useful for local testing) - AWS_PROFILE \ - # Pass through custom pip config file settings - PIP_CONFIG_FILE +passenv = AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2,AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_SESSION_TOKEN,AWS_CONTAINER_CREDENTIALS_RELATIVE_URI,AWS_PROFILE,PIP_CONFIG_FILE +; passenv = +; # Identifies AWS KMS key id to use in integration tests +; AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID \ +; # Identifies a second AWS KMS key id to use in integration tests +; AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2 \ +; # Identifies AWS KMS MRK key id to use in integration tests +; AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1 \ +; # Identifies a related AWS KMS MRK key id to use in integration tests +; AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2 \ +; # Pass through AWS credentials +; AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN \ +; # AWS Role access in CodeBuild is via the contaner URI +; AWS_CONTAINER_CREDENTIALS_RELATIVE_URI \ +; # Pass through AWS profile name (useful for local testing) +; AWS_PROFILE \ +; # Pass through custom pip config file settings +; PIP_CONFIG_FILE sitepackages = False deps = -rdev_requirements/test-requirements.txt # install the MPL if in environment - mpl: "aws-cryptographic-material-providers @git+https://github.com/aws/aws-cryptographic-material-providers-library.git@lucmcdon/python-mpl#subdirectory=AwsCryptographicMaterialProviders/runtimes/python" + mpl: -rrequirements_mpl.txt commands = local: {[testenv:base-command]commands} test/ -m local integ: {[testenv:base-command]commands} test/ -m integ From 6eb8f82edb6139e6df62ec21957ad89ef5efa58c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 12:53:55 -0800 Subject: [PATCH 030/422] debug --- examples/src/keyrings/__init__.py | 13 +++++++++++++ examples/test/keyrings/__init__.py | 13 +++++++++++++ requirements_mpl.txt | 1 + 3 files changed, 27 insertions(+) create mode 100644 examples/src/keyrings/__init__.py create mode 100644 examples/test/keyrings/__init__.py create mode 100644 requirements_mpl.txt diff --git a/examples/src/keyrings/__init__.py b/examples/src/keyrings/__init__.py new file mode 100644 index 000000000..e8fd618b1 --- /dev/null +++ b/examples/src/keyrings/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Stub module indicator to make linter configuration simpler.""" diff --git a/examples/test/keyrings/__init__.py b/examples/test/keyrings/__init__.py new file mode 100644 index 000000000..e8fd618b1 --- /dev/null +++ b/examples/test/keyrings/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Stub module indicator to make linter configuration simpler.""" diff --git a/requirements_mpl.txt b/requirements_mpl.txt new file mode 100644 index 000000000..209e10f2c --- /dev/null +++ b/requirements_mpl.txt @@ -0,0 +1 @@ +aws-cryptographic-material-providers @ git+https://github.com/aws/aws-cryptographic-material-providers-library.git@lucmcdon/python-mpl#subdirectory=AwsCryptographicMaterialProviders/runtimes/python \ No newline at end of file From 5ccfa0cce6afdde5e598ceb35719feb109da1d48 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 13:14:29 -0800 Subject: [PATCH 031/422] codebuild mpl --- tox.ini | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/tox.ini b/tox.ini index d06cbab2e..8eb141821 100644 --- a/tox.ini +++ b/tox.ini @@ -145,17 +145,6 @@ sitepackages = {[testenv:test-upstream-requirements-base]sitepackages} recreate = {[testenv:test-upstream-requirements-base]recreate} commands = {[testenv:test-upstream-requirements-base]commands} -# Test MPL -[testenv:py311-local-mpl] -basepython = {[testenv:pylint]basepython} -deps = {[testenv:pylint]deps} -commands = - pylint \ - --rcfile=test/pylintrc \ - test/unit/ \ - test/functional/ \ - test/integration/ - # Linters [testenv:flake8] basepython = python3 From 5e7ec9b94694ed4ace20aabd0ea87edd9e74f479 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 13:16:13 -0800 Subject: [PATCH 032/422] codebuild mpl --- codebuild/py311/examples_mpl.yml | 22 ++++++++++++++++++++++ codebuild/py311/integ_mpl.yml | 22 ++++++++++++++++++++++ codebuild/py312/examples_mpl.yml | 27 +++++++++++++++++++++++++++ codebuild/py312/integ_mpl.yml | 27 +++++++++++++++++++++++++++ 4 files changed, 98 insertions(+) create mode 100644 codebuild/py311/examples_mpl.yml create mode 100644 codebuild/py311/integ_mpl.yml create mode 100644 codebuild/py312/examples_mpl.yml create mode 100644 codebuild/py312/integ_mpl.yml diff --git a/codebuild/py311/examples_mpl.yml b/codebuild/py311/examples_mpl.yml new file mode 100644 index 000000000..abea2ad8c --- /dev/null +++ b/codebuild/py311/examples_mpl.yml @@ -0,0 +1,22 @@ +version: 0.2 + +env: + variables: + TOXENV: "py311-examples-mpl" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.11 + build: + commands: + - pip install "tox < 4.0" + - tox diff --git a/codebuild/py311/integ_mpl.yml b/codebuild/py311/integ_mpl.yml new file mode 100644 index 000000000..ad969c621 --- /dev/null +++ b/codebuild/py311/integ_mpl.yml @@ -0,0 +1,22 @@ +version: 0.2 + +env: + variables: + TOXENV: "py311-integ-mpl" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.11 + build: + commands: + - pip install "tox < 4.0" + - tox diff --git a/codebuild/py312/examples_mpl.yml b/codebuild/py312/examples_mpl.yml new file mode 100644 index 000000000..8ffd24964 --- /dev/null +++ b/codebuild/py312/examples_mpl.yml @@ -0,0 +1,27 @@ +version: 0.2 + +env: + variables: + TOXENV: "py312-examples-mpl" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: latest + build: + commands: + - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - + - pyenv install 3.12.0 + - pyenv local 3.12.0 + - pip install --upgrade pip + - pip install setuptools + - pip install "tox < 4.0" + - tox diff --git a/codebuild/py312/integ_mpl.yml b/codebuild/py312/integ_mpl.yml new file mode 100644 index 000000000..085cb4660 --- /dev/null +++ b/codebuild/py312/integ_mpl.yml @@ -0,0 +1,27 @@ +version: 0.2 + +env: + variables: + TOXENV: "py312-integ-mpl" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: latest + build: + commands: + - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - + - pyenv install 3.12.0 + - pyenv local 3.12.0 + - pip install --upgrade pip + - pip install setuptools + - pip install "tox < 4.0" + - tox From cc48697824accd00df18e71877a4f888d0f32125 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 13:20:54 -0800 Subject: [PATCH 033/422] codebuild mpl --- buildspec.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/buildspec.yml b/buildspec.yml index f92d203a0..c718c3df5 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -58,27 +58,51 @@ batch: buildspec: codebuild/py311/integ.yml env: image: aws/codebuild/standard:7.0 + - identifier: py311_integ_mpl + buildspec: codebuild/py311/integ_mpl.yml + env: + image: aws/codebuild/standard:7.0 - identifier: py311_examples buildspec: codebuild/py311/examples.yml env: image: aws/codebuild/standard:7.0 + - identifier: py311_examples_mpl + buildspec: codebuild/py311/examples_mpl.yml + env: + image: aws/codebuild/standard:7.0 - identifier: py311_awses_latest buildspec: codebuild/py311/awses_local.yml env: image: aws/codebuild/standard:7.0 + - identifier: py311_awses_latest_mpl + buildspec: codebuild/py311/awses_local_mpl.yml + env: + image: aws/codebuild/standard:7.0 - identifier: py312_integ buildspec: codebuild/py312/integ.yml env: image: aws/codebuild/standard:7.0 + - identifier: py312_integ_mpl + buildspec: codebuild/py312/integ_mpl.yml + env: + image: aws/codebuild/standard:7.0 - identifier: py312_examples buildspec: codebuild/py312/examples.yml env: image: aws/codebuild/standard:7.0 + - identifier: py312_examples_mpl + buildspec: codebuild/py312/examples_mpl.yml + env: + image: aws/codebuild/standard:7.0 - identifier: py312_awses_latest buildspec: codebuild/py312/awses_local.yml env: image: aws/codebuild/standard:7.0 + - identifier: py312_awses_latest_mpl + buildspec: codebuild/py312/awses_local_mpl.yml + env: + image: aws/codebuild/standard:7.0 - identifier: code_coverage buildspec: codebuild/coverage/coverage.yml From fae43d14db29780616356893d3ae7da9ce996dab Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 13:21:05 -0800 Subject: [PATCH 034/422] codebuild mpl --- codebuild/py311/awses_local_mpl.yml | 25 ++++++++++++++++++++++++ codebuild/py312/awses_local_mpl.yml | 30 +++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 codebuild/py311/awses_local_mpl.yml create mode 100644 codebuild/py312/awses_local_mpl.yml diff --git a/codebuild/py311/awses_local_mpl.yml b/codebuild/py311/awses_local_mpl.yml new file mode 100644 index 000000000..f98859b40 --- /dev/null +++ b/codebuild/py311/awses_local_mpl.yml @@ -0,0 +1,25 @@ +version: 0.2 + +env: + variables: + TOXENV: "py311-awses_local-mpl" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + python: 3.11 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - tox diff --git a/codebuild/py312/awses_local_mpl.yml b/codebuild/py312/awses_local_mpl.yml new file mode 100644 index 000000000..689d40da8 --- /dev/null +++ b/codebuild/py312/awses_local_mpl.yml @@ -0,0 +1,30 @@ +version: 0.2 + +env: + variables: + TOXENV: "py312-awses_local-mpl" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + python: latest + build: + commands: + - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - + - pyenv install 3.12.0 + - pyenv local 3.12.0 + - pip install --upgrade pip + - pip install setuptools + - pip install "tox < 4.0" + - cd test_vector_handlers + - tox From 263761678b201618c488874c040cfd7d50d0db7f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 13:32:27 -0800 Subject: [PATCH 035/422] debug --- tox.ini | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/tox.ini b/tox.ini index 8eb141821..8e6cf3f34 100644 --- a/tox.ini +++ b/tox.ini @@ -48,24 +48,24 @@ ignore_base_python_conflict = true commands = pytest --basetemp={envtmpdir} -l {posargs} [testenv] -passenv = AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2,AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_SESSION_TOKEN,AWS_CONTAINER_CREDENTIALS_RELATIVE_URI,AWS_PROFILE,PIP_CONFIG_FILE -; passenv = -; # Identifies AWS KMS key id to use in integration tests -; AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID \ -; # Identifies a second AWS KMS key id to use in integration tests -; AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2 \ -; # Identifies AWS KMS MRK key id to use in integration tests -; AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1 \ -; # Identifies a related AWS KMS MRK key id to use in integration tests -; AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2 \ -; # Pass through AWS credentials -; AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN \ -; # AWS Role access in CodeBuild is via the contaner URI -; AWS_CONTAINER_CREDENTIALS_RELATIVE_URI \ -; # Pass through AWS profile name (useful for local testing) -; AWS_PROFILE \ -; # Pass through custom pip config file settings -; PIP_CONFIG_FILE +; passenv = AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2,AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_SESSION_TOKEN,AWS_CONTAINER_CREDENTIALS_RELATIVE_URI,AWS_PROFILE,PIP_CONFIG_FILE +passenv = + # Identifies AWS KMS key id to use in integration tests + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID \ + # Identifies a second AWS KMS key id to use in integration tests + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2 \ + # Identifies AWS KMS MRK key id to use in integration tests + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1 \ + # Identifies a related AWS KMS MRK key id to use in integration tests + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2 \ + # Pass through AWS credentials + AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN \ + # AWS Role access in CodeBuild is via the contaner URI + AWS_CONTAINER_CREDENTIALS_RELATIVE_URI \ + # Pass through AWS profile name (useful for local testing) + AWS_PROFILE \ + # Pass through custom pip config file settings + PIP_CONFIG_FILE sitepackages = False deps = -rdev_requirements/test-requirements.txt From 2694932f5090404eaacf9c5d442b6acac98c0246 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 13:37:16 -0800 Subject: [PATCH 036/422] debug --- tox.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 8e6cf3f34..59a1dde74 100644 --- a/tox.ini +++ b/tox.ini @@ -65,7 +65,9 @@ passenv = # Pass through AWS profile name (useful for local testing) AWS_PROFILE \ # Pass through custom pip config file settings - PIP_CONFIG_FILE + PIP_CONFIG_FILE \ + # Pass through any configured AWS region + REGION sitepackages = False deps = -rdev_requirements/test-requirements.txt From f674d3e27a2f21f3a340c9c95b3ef60fc786dd3d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 13:40:29 -0800 Subject: [PATCH 037/422] debug --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 59a1dde74..26e0e5433 100644 --- a/tox.ini +++ b/tox.ini @@ -67,7 +67,7 @@ passenv = # Pass through custom pip config file settings PIP_CONFIG_FILE \ # Pass through any configured AWS region - REGION + AWS_REGION sitepackages = False deps = -rdev_requirements/test-requirements.txt From 0b5e655b1100f4b236a14dd47ab7e7451a90eed1 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 13:44:55 -0800 Subject: [PATCH 038/422] debug --- codebuild/py311/awses_local_mpl.yml | 1 + codebuild/py311/examples_mpl.yml | 1 + codebuild/py311/integ_mpl.yml | 1 + codebuild/py312/awses_local_mpl.yml | 1 + codebuild/py312/examples_mpl.yml | 1 + codebuild/py312/integ_mpl.yml | 1 + tox.ini | 4 ++-- 7 files changed, 8 insertions(+), 2 deletions(-) diff --git a/codebuild/py311/awses_local_mpl.yml b/codebuild/py311/awses_local_mpl.yml index f98859b40..04d268d5a 100644 --- a/codebuild/py311/awses_local_mpl.yml +++ b/codebuild/py311/awses_local_mpl.yml @@ -3,6 +3,7 @@ version: 0.2 env: variables: TOXENV: "py311-awses_local-mpl" + AWS_REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- diff --git a/codebuild/py311/examples_mpl.yml b/codebuild/py311/examples_mpl.yml index abea2ad8c..05bdc07c0 100644 --- a/codebuild/py311/examples_mpl.yml +++ b/codebuild/py311/examples_mpl.yml @@ -3,6 +3,7 @@ version: 0.2 env: variables: TOXENV: "py311-examples-mpl" + AWS_REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- diff --git a/codebuild/py311/integ_mpl.yml b/codebuild/py311/integ_mpl.yml index ad969c621..e6766619c 100644 --- a/codebuild/py311/integ_mpl.yml +++ b/codebuild/py311/integ_mpl.yml @@ -3,6 +3,7 @@ version: 0.2 env: variables: TOXENV: "py311-integ-mpl" + AWS_REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- diff --git a/codebuild/py312/awses_local_mpl.yml b/codebuild/py312/awses_local_mpl.yml index 689d40da8..a504696ec 100644 --- a/codebuild/py312/awses_local_mpl.yml +++ b/codebuild/py312/awses_local_mpl.yml @@ -3,6 +3,7 @@ version: 0.2 env: variables: TOXENV: "py312-awses_local-mpl" + AWS_REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- diff --git a/codebuild/py312/examples_mpl.yml b/codebuild/py312/examples_mpl.yml index 8ffd24964..a947c67b3 100644 --- a/codebuild/py312/examples_mpl.yml +++ b/codebuild/py312/examples_mpl.yml @@ -3,6 +3,7 @@ version: 0.2 env: variables: TOXENV: "py312-examples-mpl" + AWS_REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- diff --git a/codebuild/py312/integ_mpl.yml b/codebuild/py312/integ_mpl.yml index 085cb4660..3cf473d08 100644 --- a/codebuild/py312/integ_mpl.yml +++ b/codebuild/py312/integ_mpl.yml @@ -3,6 +3,7 @@ version: 0.2 env: variables: TOXENV: "py312-integ-mpl" + AWS_REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- diff --git a/tox.ini b/tox.ini index 26e0e5433..8b50e6b01 100644 --- a/tox.ini +++ b/tox.ini @@ -77,9 +77,9 @@ commands = local: {[testenv:base-command]commands} test/ -m local integ: {[testenv:base-command]commands} test/ -m integ accept: {[testenv:base-command]commands} test/ -m accept - examples: {[testenv:base-command]commands} examples/test/ -m examples + examples: {[testenv:base-command]commands} examples/test/ -m examples --ignore examples/test/keyrings/ # append MPL examples to base examples command - examples-mpl: {[testenv:base-command]commands} examples/test/ examples/mpl/test -m examples + examples-mpl: {[testenv:base-command]commands} examples/test/ -m examples all: {[testenv:base-command]commands} test/ examples/test/ manual: {[testenv:base-command]commands} From 831df1713823dd185883c87dfcab0e521b5fefcd Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 13:52:00 -0800 Subject: [PATCH 039/422] debug --- codebuild/py311/awses_local_mpl.yml | 2 +- codebuild/py311/examples_mpl.yml | 2 +- codebuild/py311/integ_mpl.yml | 2 +- codebuild/py312/awses_local_mpl.yml | 2 +- codebuild/py312/examples_mpl.yml | 2 +- codebuild/py312/integ_mpl.yml | 2 +- test_vector_handlers/tox.ini | 4 +++- tox.ini | 2 -- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/codebuild/py311/awses_local_mpl.yml b/codebuild/py311/awses_local_mpl.yml index 04d268d5a..859931aa3 100644 --- a/codebuild/py311/awses_local_mpl.yml +++ b/codebuild/py311/awses_local_mpl.yml @@ -3,7 +3,7 @@ version: 0.2 env: variables: TOXENV: "py311-awses_local-mpl" - AWS_REGION: "us-west-2" + REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- diff --git a/codebuild/py311/examples_mpl.yml b/codebuild/py311/examples_mpl.yml index 05bdc07c0..e29472507 100644 --- a/codebuild/py311/examples_mpl.yml +++ b/codebuild/py311/examples_mpl.yml @@ -3,7 +3,7 @@ version: 0.2 env: variables: TOXENV: "py311-examples-mpl" - AWS_REGION: "us-west-2" + REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- diff --git a/codebuild/py311/integ_mpl.yml b/codebuild/py311/integ_mpl.yml index e6766619c..694bc0850 100644 --- a/codebuild/py311/integ_mpl.yml +++ b/codebuild/py311/integ_mpl.yml @@ -3,7 +3,7 @@ version: 0.2 env: variables: TOXENV: "py311-integ-mpl" - AWS_REGION: "us-west-2" + REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- diff --git a/codebuild/py312/awses_local_mpl.yml b/codebuild/py312/awses_local_mpl.yml index a504696ec..f39bf8760 100644 --- a/codebuild/py312/awses_local_mpl.yml +++ b/codebuild/py312/awses_local_mpl.yml @@ -3,7 +3,7 @@ version: 0.2 env: variables: TOXENV: "py312-awses_local-mpl" - AWS_REGION: "us-west-2" + REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- diff --git a/codebuild/py312/examples_mpl.yml b/codebuild/py312/examples_mpl.yml index a947c67b3..d6bc3f440 100644 --- a/codebuild/py312/examples_mpl.yml +++ b/codebuild/py312/examples_mpl.yml @@ -3,7 +3,7 @@ version: 0.2 env: variables: TOXENV: "py312-examples-mpl" - AWS_REGION: "us-west-2" + REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- diff --git a/codebuild/py312/integ_mpl.yml b/codebuild/py312/integ_mpl.yml index 3cf473d08..8ffda4bd0 100644 --- a/codebuild/py312/integ_mpl.yml +++ b/codebuild/py312/integ_mpl.yml @@ -3,7 +3,7 @@ version: 0.2 env: variables: TOXENV: "py312-integ-mpl" - AWS_REGION: "us-west-2" + REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index 643750cd2..b6711361e 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -2,7 +2,7 @@ envlist = # The test vectors depend on new features now, # so until release we can only effectively test the local version of the ESDK. - py{37,38,39,310}-awses_local, + py{37,38,39,310}-awses_local{,-mpl}, # 1.2.0 and 1.2.max are being difficult because of attrs bandit, doc8, readme, {flake8,pylint}{,-tests}, @@ -48,6 +48,8 @@ passenv = sitepackages = False deps = -rtest/requirements.txt + # install the MPL if in environment + mpl: -rrequirements_mpl.txt .. commands = {[testenv:base-command]commands} diff --git a/tox.ini b/tox.ini index 8b50e6b01..903ea5170 100644 --- a/tox.ini +++ b/tox.ini @@ -66,8 +66,6 @@ passenv = AWS_PROFILE \ # Pass through custom pip config file settings PIP_CONFIG_FILE \ - # Pass through any configured AWS region - AWS_REGION sitepackages = False deps = -rdev_requirements/test-requirements.txt From 477e3a097da42fddfded9894d48fad6923d96144 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 13:56:57 -0800 Subject: [PATCH 040/422] debug --- examples/src/keyrings/hierarchical_keyring.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/src/keyrings/hierarchical_keyring.py b/examples/src/keyrings/hierarchical_keyring.py index 76aef25e0..a99728b6e 100644 --- a/examples/src/keyrings/hierarchical_keyring.py +++ b/examples/src/keyrings/hierarchical_keyring.py @@ -48,8 +48,8 @@ def encrypt_and_decrypt_with_keyring( ) # 2. Create boto3 clients for DynamoDB and KMS. - ddb_client = boto3.client('dynamodb') - kms_client = boto3.client('kms') + ddb_client = boto3.client('dynamodb', region_name="us-west-2") + kms_client = boto3.client('kms', region_name="us-west-2") # 3. Configure your KeyStore resource. # This SHOULD be the same configuration that you used From 166c5ab6ff339f8fd6dafa6ffea071704eb484a5 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 14:24:13 -0800 Subject: [PATCH 041/422] debug --- test_vector_handlers/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index b6711361e..7004080e3 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -49,7 +49,7 @@ sitepackages = False deps = -rtest/requirements.txt # install the MPL if in environment - mpl: -rrequirements_mpl.txt + mpl: -r../requirements_mpl.txt .. commands = {[testenv:base-command]commands} From 7ac88805b863add77147cabedc08c5968480ad55 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 14:35:06 -0800 Subject: [PATCH 042/422] debug --- tox.ini | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 903ea5170..4202979a4 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,6 @@ envlist = isort-check, black-check, # prone to false positives vulture -ignore_base_python_conflict = true # Additional test environments: # @@ -65,7 +64,7 @@ passenv = # Pass through AWS profile name (useful for local testing) AWS_PROFILE \ # Pass through custom pip config file settings - PIP_CONFIG_FILE \ + PIP_CONFIG_FILE sitepackages = False deps = -rdev_requirements/test-requirements.txt From 7e3ca151e85eb44d4d125f172f32ed70d3b6523b Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 14:41:54 -0800 Subject: [PATCH 043/422] fix --- codebuild/py312/awses_local_mpl.yml | 2 +- codebuild/py312/examples_mpl.yml | 2 +- codebuild/py312/integ_mpl.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/codebuild/py312/awses_local_mpl.yml b/codebuild/py312/awses_local_mpl.yml index f39bf8760..db25f4f57 100644 --- a/codebuild/py312/awses_local_mpl.yml +++ b/codebuild/py312/awses_local_mpl.yml @@ -22,7 +22,7 @@ phases: build: commands: - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - - - pyenv install 3.12.0 + - pyenv install --skip-existing 3.12.0 - pyenv local 3.12.0 - pip install --upgrade pip - pip install setuptools diff --git a/codebuild/py312/examples_mpl.yml b/codebuild/py312/examples_mpl.yml index d6bc3f440..ff2168cd5 100644 --- a/codebuild/py312/examples_mpl.yml +++ b/codebuild/py312/examples_mpl.yml @@ -20,7 +20,7 @@ phases: build: commands: - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - - - pyenv install 3.12.0 + - pyenv install --skip-existing 3.12.0 - pyenv local 3.12.0 - pip install --upgrade pip - pip install setuptools diff --git a/codebuild/py312/integ_mpl.yml b/codebuild/py312/integ_mpl.yml index 8ffda4bd0..553f41e8a 100644 --- a/codebuild/py312/integ_mpl.yml +++ b/codebuild/py312/integ_mpl.yml @@ -20,7 +20,7 @@ phases: build: commands: - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - - - pyenv install 3.12.0 + - pyenv install --skip-existing 3.12.0 - pyenv local 3.12.0 - pip install --upgrade pip - pip install setuptools From 4c6a1d00711352236d63b2adc44081dded3a1a65 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 15:08:53 -0800 Subject: [PATCH 044/422] fix --- src/aws_encryption_sdk/cmm_handler.py | 158 ----------------- src/aws_encryption_sdk/materials_handlers.py | 170 ------------------- 2 files changed, 328 deletions(-) delete mode 100644 src/aws_encryption_sdk/cmm_handler.py delete mode 100644 src/aws_encryption_sdk/materials_handlers.py diff --git a/src/aws_encryption_sdk/cmm_handler.py b/src/aws_encryption_sdk/cmm_handler.py deleted file mode 100644 index 5bac15b87..000000000 --- a/src/aws_encryption_sdk/cmm_handler.py +++ /dev/null @@ -1,158 +0,0 @@ -"""Retrieves encryption/decryption materials from an underlying materials provider.""" - -# These dependencies are only loaded if you install the MPL. -try: - # pylint seems to struggle with this conditional import - # pylint: disable=unused-import - from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException - from aws_cryptographic_materialproviders.mpl.models import ( - AlgorithmSuiteIdESDK, - CommitmentPolicyESDK, - DecryptMaterialsInput, - DecryptMaterialsOutput, - EncryptedDataKey as MPL_EncryptedDataKey, - GetEncryptionMaterialsInput, - GetEncryptionMaterialsOutput, - ) - from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager - -except ImportError: - pass - -from typing import List - -from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError -from aws_encryption_sdk.identifiers import CommitmentPolicy -from aws_encryption_sdk.materials_handlers import DecryptionMaterialsHandler, EncryptionMaterialsHandler -from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest -from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager -from aws_encryption_sdk.structures import EncryptedDataKey as Native_EncryptedDataKey - - -# TODO-MPL Should this implement interface..? seems like yes since it implements all of interface methods -class CMMHandler(CryptoMaterialsManager): - """ - In instances where encryption materials may be provided by either - an implementation of the native - `aws_encryption_sdk.materials_managers.base.CryptoMaterialsManager` - or an implementation of the MPL's - `aws_cryptographic_materialproviders.mpl.references.ICryptographicMaterialsManager`, - this provides the correct materials based on the underlying materials manager. - """ - - native_cmm: CryptoMaterialsManager - mpl_cmm: 'ICryptographicMaterialsManager' - - def _is_using_native_cmm(self): - return hasattr(self, "native_cmm") and not hasattr(self, "mpl_cmm") - - def __init__( - self, - cmm: 'CryptoMaterialsManager | ICryptographicMaterialsManager' - ): - """ - Create DecryptionMaterialsHandler. - :param cmm: Underlying cryptographic materials manager - """ - if isinstance(cmm, CryptoMaterialsManager): - self.native_cmm = cmm - elif isinstance(cmm, ICryptographicMaterialsManager): - self.mpl_cmm = cmm - else: - raise ValueError(f"Invalid CMM passed to CMMHandler. cmm: {cmm}") - - def get_encryption_materials( - self, - request: EncryptionMaterialsRequest - ) -> EncryptionMaterialsHandler: - """ - Returns an EncryptionMaterialsHandler for the configured CMM. - :param request: Request for encryption materials - """ - if self._is_using_native_cmm(): - return EncryptionMaterialsHandler(self.native_cmm.get_encryption_materials(request)) - else: - try: - mpl_input: GetEncryptionMaterialsInput = CMMHandler._native_to_mpl_get_encryption_materials( - request - ) - mpl_output: GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) - return EncryptionMaterialsHandler(mpl_output.encryption_materials) - except AwsCryptographicMaterialProvidersException as mpl_exception: - # Wrap MPL error into the ESDK error type - # so customers only have to catch ESDK error types. - raise AWSEncryptionSDKClientError(mpl_exception) - - @staticmethod - def _native_to_mpl_get_encryption_materials( - request: EncryptionMaterialsRequest - ) -> 'GetEncryptionMaterialsInput': - output: GetEncryptionMaterialsInput = GetEncryptionMaterialsInput( - encryption_context=request.encryption_context, - commitment_policy=CMMHandler._native_to_mpl_commmitment_policy( - request.commitment_policy - ), - max_plaintext_length=request.plaintext_length, - ) - return output - - @staticmethod - def _native_to_mpl_commmitment_policy( - native_commitment_policy: CommitmentPolicy - ) -> 'CommitmentPolicyESDK': - if native_commitment_policy == CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT: - return CommitmentPolicyESDK(value="FORBID_ENCRYPT_ALLOW_DECRYPT") - elif native_commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT: - return CommitmentPolicyESDK(value="REQUIRE_ENCRYPT_ALLOW_DECRYPT") - elif native_commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT: - return CommitmentPolicyESDK(value="REQUIRE_ENCRYPT_REQUIRE_DECRYPT") - else: - raise ValueError(f"Invalid native_commitment_policy: {native_commitment_policy}") - - def decrypt_materials( - self, - request: DecryptionMaterialsRequest - ) -> DecryptionMaterialsHandler: - """ - Returns a DecryptionMaterialsHandler for the configured CMM. - :param request: Request for decryption materials - """ - if self._is_using_native_cmm(): - return DecryptionMaterialsHandler(self.native_cmm.decrypt_materials(request)) - else: - try: - mpl_input: 'DecryptMaterialsInput' = \ - CMMHandler._create_mpl_decrypt_materials_input_from_request(request) - mpl_output: 'DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(mpl_input) - return DecryptionMaterialsHandler(mpl_output.decryption_materials) - except AwsCryptographicMaterialProvidersException as mpl_exception: - # Wrap MPL error into the ESDK error type - # so customers only have to catch ESDK error types. - raise AWSEncryptionSDKClientError(mpl_exception) - - @staticmethod - def _native_algorithm_id_to_mpl_algorithm_id(native_algorithm_id: str) -> 'AlgorithmSuiteIdESDK': - # MPL algorithm suite ID = hexstr(native_algorithm_id) padded to 4 digits post-`x`. - return AlgorithmSuiteIdESDK(f"{native_algorithm_id:#0{6}x}") - - @staticmethod - def _create_mpl_decrypt_materials_input_from_request( - request: DecryptionMaterialsRequest - ) -> 'DecryptMaterialsInput': - key_blob_list: List[Native_EncryptedDataKey] = request.encrypted_data_keys - list_edks = [MPL_EncryptedDataKey( - key_provider_id=key_blob.key_provider.provider_id, - key_provider_info=key_blob.key_provider.key_info, - ciphertext=key_blob.encrypted_data_key, - ) for key_blob in key_blob_list] - output: DecryptMaterialsInput = DecryptMaterialsInput( - algorithm_suite_id=CMMHandler._native_algorithm_id_to_mpl_algorithm_id( - request.algorithm.algorithm_id - ), - commitment_policy=CMMHandler._native_to_mpl_commmitment_policy( - request.commitment_policy - ), - encrypted_data_keys=list_edks, - encryption_context=request.encryption_context, - ) - return output diff --git a/src/aws_encryption_sdk/materials_handlers.py b/src/aws_encryption_sdk/materials_handlers.py deleted file mode 100644 index 57f54144e..000000000 --- a/src/aws_encryption_sdk/materials_handlers.py +++ /dev/null @@ -1,170 +0,0 @@ -"""Provides encryption/decryption materials from an underlying materials provider.""" -# These dependencies are only loaded if you install the MPL. -try: - from aws_cryptographic_materialproviders.mpl.models import ( - DecryptionMaterials as MPL_DecryptionMaterials, - EncryptedDataKey as MPL_EncryptedDataKey, - EncryptionMaterials as MPL_EncryptionMaterials, - ) -except ImportError: - pass - -from typing import Dict, List, Set - -from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite -from aws_encryption_sdk.materials_managers import ( - DecryptionMaterials as Native_DecryptionMaterials, - EncryptionMaterials as Native_EncryptionMaterials, -) -from aws_encryption_sdk.structures import DataKey, EncryptedDataKey as Native_EncryptedDataKey, MasterKeyInfo - - -def _mpl_algorithm_id_to_native_algorithm_id(mpl_algorithm_id: str): - # MPL algorithm suite ID == hex(native algorithm suite ID) - return int(mpl_algorithm_id, 16) - - -class EncryptionMaterialsHandler: - """ - In instances where encryption materials may be provided by either - the native `aws_encryption_sdk.materials_managers.EncryptionMaterials` - or the MPL's `aws_cryptographic_materialproviders.mpl.models.EncryptionMaterials`, - this provides the correct materials based on the configured materials provider. - """ - - native_materials: Native_EncryptionMaterials - mpl_materials: 'MPL_EncryptionMaterials' - - def __init__( - self, - materials: 'Native_EncryptionMaterials | MPL_EncryptionMaterials' - ): - """ - Create EncryptionMaterialsHandler. - :param materials: Underlying encryption materials - """ - if isinstance(materials, Native_EncryptionMaterials): - self.native_materials = materials - elif isinstance(materials, MPL_EncryptionMaterials): - self.mpl_materials = materials - else: - raise ValueError(f"Invalid EncryptionMaterials passed to EncryptionMaterialsHandler.\ - materials: {materials}") - - @property - def algorithm(self) -> Algorithm: - """Materials' native Algorithm.""" - if hasattr(self, "native_materials"): - return self.native_materials.algorithm - else: - return AlgorithmSuite.get_by_id( - _mpl_algorithm_id_to_native_algorithm_id( - self.mpl_materials.algorithm_suite.id.value - ) - ) - - @property - def encryption_context(self) -> Dict[str, str]: - """Materials' encryption context.""" - if hasattr(self, "native_materials"): - return self.native_materials.encryption_context - else: - return self.mpl_materials.encryption_context - - @property - def encrypted_data_keys(self) -> List[Native_EncryptedDataKey]: - """Materials' encrypted data keys.""" - if hasattr(self, "native_materials"): - return self.native_materials.encrypted_data_keys - else: - mpl_edk_list: List[MPL_EncryptedDataKey] = self.mpl_materials.encrypted_data_keys - key_blob_list: Set[Native_EncryptedDataKey] = {Native_EncryptedDataKey( - key_provider=MasterKeyInfo( - provider_id=mpl_edk.key_provider_id, - key_info=mpl_edk.key_provider_info, - ), - encrypted_data_key=mpl_edk.ciphertext, - ) for mpl_edk in mpl_edk_list} - return key_blob_list - - @property - def data_encryption_key(self) -> DataKey: - """Materials' data encryption key.""" - if hasattr(self, "native_materials"): - return self.native_materials.data_encryption_key - else: - # TODO-MPL This impl is probably wrong, but works for for now - # If this works for all features, great! Remove this comment before launch. - # Otherwise, fix the implementation. - mpl_dek = self.mpl_materials.plaintext_data_key - return DataKey( - # key_provider is unused, but the return type is DataKey - key_provider=MasterKeyInfo( - provider_id="", - key_info=b'' - ), - data_key=mpl_dek, - encrypted_data_key=b'', # No encrypted DEK - ) - - @property - def signing_key(self) -> bytes: - """Materials' signing key.""" - if hasattr(self, "native_materials"): - return self.native_materials.signing_key - else: - return self.mpl_materials.signing_key - - -class DecryptionMaterialsHandler: - """ - In instances where decryption materials may be provided by either - the native `aws_encryption_sdk.materials_managers.DecryptionMaterials` - or the MPL's `aws_cryptographic_materialproviders.mpl.models.DecryptionMaterials`, - this provides the correct materials based on the configured materials provider. - """ - - native_materials: Native_DecryptionMaterials - mpl_materials: 'MPL_DecryptionMaterials' - - def __init__( - self, - materials: 'Native_DecryptionMaterials | MPL_DecryptionMaterials' - ): - """ - Create DecryptionMaterialsHandler. - :param materials: Underlying decryption materials - """ - if isinstance(materials, Native_DecryptionMaterials): - self.native_materials = materials - elif isinstance(materials, MPL_DecryptionMaterials): - self.mpl_materials = materials - else: - raise ValueError(f"Invalid DecryptionMaterials passed to DecryptionMaterialsHandler.\ - materials: {materials}") - - @property - def data_key(self) -> DataKey: - """Materials' data key.""" - if hasattr(self, "native_materials"): - return self.native_materials.data_key - else: - # TODO-MPL This impl is probably wrong, but works for for now - # If this works for all features, great! Remove this comment before launch. - # Otherwise, fix the implementation. - return DataKey( - key_provider=MasterKeyInfo( - provider_id="", - key_info=b'' - ), - data_key=self.mpl_materials.plaintext_data_key, - encrypted_data_key=b'', - ) - - @property - def verification_key(self) -> bytes: - """Materials' verification key.""" - if hasattr(self, "native_materials"): - return self.native_materials.verification_key - else: - return self.mpl_materials.verification_key From e2e185844e0ac83f07dce346156ed9a7ab693275 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 15:09:10 -0800 Subject: [PATCH 045/422] fix --- src/aws_encryption_sdk/mpl/cmm_handler.py | 158 ++++++++++++++++ .../mpl/materials_handlers.py | 170 ++++++++++++++++++ 2 files changed, 328 insertions(+) create mode 100644 src/aws_encryption_sdk/mpl/cmm_handler.py create mode 100644 src/aws_encryption_sdk/mpl/materials_handlers.py diff --git a/src/aws_encryption_sdk/mpl/cmm_handler.py b/src/aws_encryption_sdk/mpl/cmm_handler.py new file mode 100644 index 000000000..5dfaab973 --- /dev/null +++ b/src/aws_encryption_sdk/mpl/cmm_handler.py @@ -0,0 +1,158 @@ +"""Retrieves encryption/decryption materials from an underlying materials provider.""" + +# These dependencies are only loaded if you install the MPL. +try: + # pylint seems to struggle with this conditional import + # pylint: disable=unused-import + from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException + from aws_cryptographic_materialproviders.mpl.models import ( + AlgorithmSuiteIdESDK, + CommitmentPolicyESDK, + DecryptMaterialsInput, + DecryptMaterialsOutput, + EncryptedDataKey as MPL_EncryptedDataKey, + GetEncryptionMaterialsInput, + GetEncryptionMaterialsOutput, + ) + from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager + +except ImportError: + pass + +from typing import List + +from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError +from aws_encryption_sdk.identifiers import CommitmentPolicy +from aws_encryption_sdk.mpl.materials_handlers import DecryptionMaterialsHandler, EncryptionMaterialsHandler +from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest +from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager +from aws_encryption_sdk.structures import EncryptedDataKey as Native_EncryptedDataKey + + +# TODO-MPL Should this implement interface...? seems like yes since it implements all of interface methods +class CMMHandler(CryptoMaterialsManager): + """ + In instances where encryption materials may be provided by either + an implementation of the native + `aws_encryption_sdk.materials_managers.base.CryptoMaterialsManager` + or an implementation of the MPL's + `aws_cryptographic_materialproviders.mpl.references.ICryptographicMaterialsManager`, + this provides the correct materials based on the underlying materials manager. + """ + + native_cmm: CryptoMaterialsManager + mpl_cmm: 'ICryptographicMaterialsManager' + + def _is_using_native_cmm(self): + return hasattr(self, "native_cmm") and not hasattr(self, "mpl_cmm") + + def __init__( + self, + cmm: 'CryptoMaterialsManager | ICryptographicMaterialsManager' + ): + """ + Create DecryptionMaterialsHandler. + :param cmm: Underlying cryptographic materials manager + """ + if isinstance(cmm, CryptoMaterialsManager): + self.native_cmm = cmm + elif isinstance(cmm, ICryptographicMaterialsManager): + self.mpl_cmm = cmm + else: + raise ValueError(f"Invalid CMM passed to CMMHandler. cmm: {cmm}") + + def get_encryption_materials( + self, + request: EncryptionMaterialsRequest + ) -> EncryptionMaterialsHandler: + """ + Returns an EncryptionMaterialsHandler for the configured CMM. + :param request: Request for encryption materials + """ + if self._is_using_native_cmm(): + return EncryptionMaterialsHandler(self.native_cmm.get_encryption_materials(request)) + else: + try: + mpl_input: GetEncryptionMaterialsInput = CMMHandler._native_to_mpl_get_encryption_materials( + request + ) + mpl_output: GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) + return EncryptionMaterialsHandler(mpl_output.encryption_materials) + except AwsCryptographicMaterialProvidersException as mpl_exception: + # Wrap MPL error into the ESDK error type + # so customers only have to catch ESDK error types. + raise AWSEncryptionSDKClientError(mpl_exception) + + @staticmethod + def _native_to_mpl_get_encryption_materials( + request: EncryptionMaterialsRequest + ) -> 'GetEncryptionMaterialsInput': + output: GetEncryptionMaterialsInput = GetEncryptionMaterialsInput( + encryption_context=request.encryption_context, + commitment_policy=CMMHandler._native_to_mpl_commmitment_policy( + request.commitment_policy + ), + max_plaintext_length=request.plaintext_length, + ) + return output + + @staticmethod + def _native_to_mpl_commmitment_policy( + native_commitment_policy: CommitmentPolicy + ) -> 'CommitmentPolicyESDK': + if native_commitment_policy == CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT: + return CommitmentPolicyESDK(value="FORBID_ENCRYPT_ALLOW_DECRYPT") + elif native_commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT: + return CommitmentPolicyESDK(value="REQUIRE_ENCRYPT_ALLOW_DECRYPT") + elif native_commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT: + return CommitmentPolicyESDK(value="REQUIRE_ENCRYPT_REQUIRE_DECRYPT") + else: + raise ValueError(f"Invalid native_commitment_policy: {native_commitment_policy}") + + def decrypt_materials( + self, + request: DecryptionMaterialsRequest + ) -> DecryptionMaterialsHandler: + """ + Returns a DecryptionMaterialsHandler for the configured CMM. + :param request: Request for decryption materials + """ + if self._is_using_native_cmm(): + return DecryptionMaterialsHandler(self.native_cmm.decrypt_materials(request)) + else: + try: + mpl_input: 'DecryptMaterialsInput' = \ + CMMHandler._create_mpl_decrypt_materials_input_from_request(request) + mpl_output: 'DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(mpl_input) + return DecryptionMaterialsHandler(mpl_output.decryption_materials) + except AwsCryptographicMaterialProvidersException as mpl_exception: + # Wrap MPL error into the ESDK error type + # so customers only have to catch ESDK error types. + raise AWSEncryptionSDKClientError(mpl_exception) + + @staticmethod + def _native_algorithm_id_to_mpl_algorithm_id(native_algorithm_id: str) -> 'AlgorithmSuiteIdESDK': + # MPL algorithm suite ID = hexstr(native_algorithm_id) padded to 4 digits post-`x`. + return AlgorithmSuiteIdESDK(f"{native_algorithm_id:#0{6}x}") + + @staticmethod + def _create_mpl_decrypt_materials_input_from_request( + request: DecryptionMaterialsRequest + ) -> 'DecryptMaterialsInput': + key_blob_list: List[Native_EncryptedDataKey] = request.encrypted_data_keys + list_edks = [MPL_EncryptedDataKey( + key_provider_id=key_blob.key_provider.provider_id, + key_provider_info=key_blob.key_provider.key_info, + ciphertext=key_blob.encrypted_data_key, + ) for key_blob in key_blob_list] + output: DecryptMaterialsInput = DecryptMaterialsInput( + algorithm_suite_id=CMMHandler._native_algorithm_id_to_mpl_algorithm_id( + request.algorithm.algorithm_id + ), + commitment_policy=CMMHandler._native_to_mpl_commmitment_policy( + request.commitment_policy + ), + encrypted_data_keys=list_edks, + encryption_context=request.encryption_context, + ) + return output diff --git a/src/aws_encryption_sdk/mpl/materials_handlers.py b/src/aws_encryption_sdk/mpl/materials_handlers.py new file mode 100644 index 000000000..57f54144e --- /dev/null +++ b/src/aws_encryption_sdk/mpl/materials_handlers.py @@ -0,0 +1,170 @@ +"""Provides encryption/decryption materials from an underlying materials provider.""" +# These dependencies are only loaded if you install the MPL. +try: + from aws_cryptographic_materialproviders.mpl.models import ( + DecryptionMaterials as MPL_DecryptionMaterials, + EncryptedDataKey as MPL_EncryptedDataKey, + EncryptionMaterials as MPL_EncryptionMaterials, + ) +except ImportError: + pass + +from typing import Dict, List, Set + +from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite +from aws_encryption_sdk.materials_managers import ( + DecryptionMaterials as Native_DecryptionMaterials, + EncryptionMaterials as Native_EncryptionMaterials, +) +from aws_encryption_sdk.structures import DataKey, EncryptedDataKey as Native_EncryptedDataKey, MasterKeyInfo + + +def _mpl_algorithm_id_to_native_algorithm_id(mpl_algorithm_id: str): + # MPL algorithm suite ID == hex(native algorithm suite ID) + return int(mpl_algorithm_id, 16) + + +class EncryptionMaterialsHandler: + """ + In instances where encryption materials may be provided by either + the native `aws_encryption_sdk.materials_managers.EncryptionMaterials` + or the MPL's `aws_cryptographic_materialproviders.mpl.models.EncryptionMaterials`, + this provides the correct materials based on the configured materials provider. + """ + + native_materials: Native_EncryptionMaterials + mpl_materials: 'MPL_EncryptionMaterials' + + def __init__( + self, + materials: 'Native_EncryptionMaterials | MPL_EncryptionMaterials' + ): + """ + Create EncryptionMaterialsHandler. + :param materials: Underlying encryption materials + """ + if isinstance(materials, Native_EncryptionMaterials): + self.native_materials = materials + elif isinstance(materials, MPL_EncryptionMaterials): + self.mpl_materials = materials + else: + raise ValueError(f"Invalid EncryptionMaterials passed to EncryptionMaterialsHandler.\ + materials: {materials}") + + @property + def algorithm(self) -> Algorithm: + """Materials' native Algorithm.""" + if hasattr(self, "native_materials"): + return self.native_materials.algorithm + else: + return AlgorithmSuite.get_by_id( + _mpl_algorithm_id_to_native_algorithm_id( + self.mpl_materials.algorithm_suite.id.value + ) + ) + + @property + def encryption_context(self) -> Dict[str, str]: + """Materials' encryption context.""" + if hasattr(self, "native_materials"): + return self.native_materials.encryption_context + else: + return self.mpl_materials.encryption_context + + @property + def encrypted_data_keys(self) -> List[Native_EncryptedDataKey]: + """Materials' encrypted data keys.""" + if hasattr(self, "native_materials"): + return self.native_materials.encrypted_data_keys + else: + mpl_edk_list: List[MPL_EncryptedDataKey] = self.mpl_materials.encrypted_data_keys + key_blob_list: Set[Native_EncryptedDataKey] = {Native_EncryptedDataKey( + key_provider=MasterKeyInfo( + provider_id=mpl_edk.key_provider_id, + key_info=mpl_edk.key_provider_info, + ), + encrypted_data_key=mpl_edk.ciphertext, + ) for mpl_edk in mpl_edk_list} + return key_blob_list + + @property + def data_encryption_key(self) -> DataKey: + """Materials' data encryption key.""" + if hasattr(self, "native_materials"): + return self.native_materials.data_encryption_key + else: + # TODO-MPL This impl is probably wrong, but works for for now + # If this works for all features, great! Remove this comment before launch. + # Otherwise, fix the implementation. + mpl_dek = self.mpl_materials.plaintext_data_key + return DataKey( + # key_provider is unused, but the return type is DataKey + key_provider=MasterKeyInfo( + provider_id="", + key_info=b'' + ), + data_key=mpl_dek, + encrypted_data_key=b'', # No encrypted DEK + ) + + @property + def signing_key(self) -> bytes: + """Materials' signing key.""" + if hasattr(self, "native_materials"): + return self.native_materials.signing_key + else: + return self.mpl_materials.signing_key + + +class DecryptionMaterialsHandler: + """ + In instances where decryption materials may be provided by either + the native `aws_encryption_sdk.materials_managers.DecryptionMaterials` + or the MPL's `aws_cryptographic_materialproviders.mpl.models.DecryptionMaterials`, + this provides the correct materials based on the configured materials provider. + """ + + native_materials: Native_DecryptionMaterials + mpl_materials: 'MPL_DecryptionMaterials' + + def __init__( + self, + materials: 'Native_DecryptionMaterials | MPL_DecryptionMaterials' + ): + """ + Create DecryptionMaterialsHandler. + :param materials: Underlying decryption materials + """ + if isinstance(materials, Native_DecryptionMaterials): + self.native_materials = materials + elif isinstance(materials, MPL_DecryptionMaterials): + self.mpl_materials = materials + else: + raise ValueError(f"Invalid DecryptionMaterials passed to DecryptionMaterialsHandler.\ + materials: {materials}") + + @property + def data_key(self) -> DataKey: + """Materials' data key.""" + if hasattr(self, "native_materials"): + return self.native_materials.data_key + else: + # TODO-MPL This impl is probably wrong, but works for for now + # If this works for all features, great! Remove this comment before launch. + # Otherwise, fix the implementation. + return DataKey( + key_provider=MasterKeyInfo( + provider_id="", + key_info=b'' + ), + data_key=self.mpl_materials.plaintext_data_key, + encrypted_data_key=b'', + ) + + @property + def verification_key(self) -> bytes: + """Materials' verification key.""" + if hasattr(self, "native_materials"): + return self.native_materials.verification_key + else: + return self.mpl_materials.verification_key From c790011ed67a16cfb6790ad56941c12099552bbe Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 15:12:52 -0800 Subject: [PATCH 046/422] mpl --- src/aws_encryption_sdk/streaming_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index afe9987ff..680784b1a 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -24,7 +24,7 @@ import six import aws_encryption_sdk.internal.utils -from aws_encryption_sdk.cmm_handler import CMMHandler +from aws_encryption_sdk.mpl.cmm_handler import CMMHandler from aws_encryption_sdk.exceptions import ( ActionNotAllowedError, AWSEncryptionSDKClientError, From 33ace5897bb532fbc15942c3317bee8e90215adc Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 15:18:27 -0800 Subject: [PATCH 047/422] fix --- src/aws_encryption_sdk/mpl/__init__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 src/aws_encryption_sdk/mpl/__init__.py diff --git a/src/aws_encryption_sdk/mpl/__init__.py b/src/aws_encryption_sdk/mpl/__init__.py new file mode 100644 index 000000000..41497cc20 --- /dev/null +++ b/src/aws_encryption_sdk/mpl/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Modules related to the MPL.""" From cbf2cdf79d107bb371ed76646307f9e3ec038d9e Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 15:32:30 -0800 Subject: [PATCH 048/422] fix --- src/aws_encryption_sdk/mpl/cmm_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/mpl/cmm_handler.py b/src/aws_encryption_sdk/mpl/cmm_handler.py index 5dfaab973..2ccbcb5f3 100644 --- a/src/aws_encryption_sdk/mpl/cmm_handler.py +++ b/src/aws_encryption_sdk/mpl/cmm_handler.py @@ -29,7 +29,7 @@ from aws_encryption_sdk.structures import EncryptedDataKey as Native_EncryptedDataKey -# TODO-MPL Should this implement interface...? seems like yes since it implements all of interface methods +# TODO-MPL Should this implement interface..? seems like yes since it implements all of interface methods class CMMHandler(CryptoMaterialsManager): """ In instances where encryption materials may be provided by either From b2594771818c265c6f6ce31a56a4339b86840f38 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Feb 2024 15:43:03 -0800 Subject: [PATCH 049/422] fix --- src/aws_encryption_sdk/mpl/cmm_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/mpl/cmm_handler.py b/src/aws_encryption_sdk/mpl/cmm_handler.py index 2ccbcb5f3..5dfaab973 100644 --- a/src/aws_encryption_sdk/mpl/cmm_handler.py +++ b/src/aws_encryption_sdk/mpl/cmm_handler.py @@ -29,7 +29,7 @@ from aws_encryption_sdk.structures import EncryptedDataKey as Native_EncryptedDataKey -# TODO-MPL Should this implement interface..? seems like yes since it implements all of interface methods +# TODO-MPL Should this implement interface...? seems like yes since it implements all of interface methods class CMMHandler(CryptoMaterialsManager): """ In instances where encryption materials may be provided by either From 9d52cf2ad37bf69fcc42bee1375365ea4e19bf15 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 8 Feb 2024 14:18:52 -0800 Subject: [PATCH 050/422] . --- src/aws_encryption_sdk/streaming_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 680784b1a..04d44334a 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -915,7 +915,7 @@ def _read_header(self): self.verifier = None else: # MPL verification key is NOT key bytes, it is bytes of the compressed point - # TODO-MPL: clean this up, least-privilege violation + # TODO-MPL: clean this up, least-privilege violation. if (isinstance(self.config.materials_manager, CMMHandler) and hasattr(self.config.materials_manager, "mpl_cmm")): self.verifier = Verifier.from_encoded_point( From 31b761616f7741293c8e6a33e2e9174c758f7c70 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 9 Feb 2024 09:45:18 -0800 Subject: [PATCH 051/422] debug tox mpl keystore env --- codebuild/py311/examples_mpl.yml | 6 ++++-- codebuild/py312/examples_mpl.yml | 5 +++-- tox.ini | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/codebuild/py311/examples_mpl.yml b/codebuild/py311/examples_mpl.yml index e29472507..b1afa5016 100644 --- a/codebuild/py311/examples_mpl.yml +++ b/codebuild/py311/examples_mpl.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - TOXENV: "py311-examples-mpl" + # No TOXENV; examples using the MPL switch envs REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f @@ -20,4 +20,6 @@ phases: build: commands: - pip install "tox < 4.0" - - tox + - tox -e py311-examples-mpl + - tox -e py311-examples-mpl-keystore + diff --git a/codebuild/py312/examples_mpl.yml b/codebuild/py312/examples_mpl.yml index ff2168cd5..cf53585b4 100644 --- a/codebuild/py312/examples_mpl.yml +++ b/codebuild/py312/examples_mpl.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - TOXENV: "py312-examples-mpl" + # No TOXENV; examples using the MPL switch envs REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f @@ -25,4 +25,5 @@ phases: - pip install --upgrade pip - pip install setuptools - pip install "tox < 4.0" - - tox + - tox -e py312-examples-mpl + - tox -e py312-examples-mpl-keystore diff --git a/tox.ini b/tox.ini index 4202979a4..d618cb030 100644 --- a/tox.ini +++ b/tox.ini @@ -75,8 +75,8 @@ commands = integ: {[testenv:base-command]commands} test/ -m integ accept: {[testenv:base-command]commands} test/ -m accept examples: {[testenv:base-command]commands} examples/test/ -m examples --ignore examples/test/keyrings/ - # append MPL examples to base examples command - examples-mpl: {[testenv:base-command]commands} examples/test/ -m examples + # MPL keyring examples require a special IAM role; run these separately under a separate set of permissions + examples-mpl-keyring: {[testenv:base-command]commands} examples/test/keyrings -m examples all: {[testenv:base-command]commands} test/ examples/test/ manual: {[testenv:base-command]commands} From 353b8cfc944b437bfb86f24390c0019693b5b76f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 9 Feb 2024 09:51:54 -0800 Subject: [PATCH 052/422] debug tox mpl keystore env --- codebuild/py311/examples_mpl.yml | 2 +- codebuild/py312/examples_mpl.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/codebuild/py311/examples_mpl.yml b/codebuild/py311/examples_mpl.yml index b1afa5016..c5b1e1872 100644 --- a/codebuild/py311/examples_mpl.yml +++ b/codebuild/py311/examples_mpl.yml @@ -21,5 +21,5 @@ phases: commands: - pip install "tox < 4.0" - tox -e py311-examples-mpl - - tox -e py311-examples-mpl-keystore + - tox -e py311-examples-mpl-keyring diff --git a/codebuild/py312/examples_mpl.yml b/codebuild/py312/examples_mpl.yml index cf53585b4..97a19ad50 100644 --- a/codebuild/py312/examples_mpl.yml +++ b/codebuild/py312/examples_mpl.yml @@ -26,4 +26,4 @@ phases: - pip install setuptools - pip install "tox < 4.0" - tox -e py312-examples-mpl - - tox -e py312-examples-mpl-keystore + - tox -e py312-examples-mpl-keyring From fb64d950a31a82169cbedaf289367dc3001a0c68 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 9 Feb 2024 09:57:15 -0800 Subject: [PATCH 053/422] debug tox mpl keystore env --- codebuild/py311/examples_mpl.yml | 8 ++++++++ codebuild/py312/examples_mpl.yml | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/codebuild/py311/examples_mpl.yml b/codebuild/py311/examples_mpl.yml index c5b1e1872..f7d705923 100644 --- a/codebuild/py311/examples_mpl.yml +++ b/codebuild/py311/examples_mpl.yml @@ -21,5 +21,13 @@ phases: commands: - pip install "tox < 4.0" - tox -e py311-examples-mpl + # Assume special role + - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-Py311ExamplesMpl") + - export TMP_ROLE + - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') + - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') + - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') + - aws sts get-caller-identity + # Run special role-specific examples - tox -e py311-examples-mpl-keyring diff --git a/codebuild/py312/examples_mpl.yml b/codebuild/py312/examples_mpl.yml index 97a19ad50..c95f606e6 100644 --- a/codebuild/py312/examples_mpl.yml +++ b/codebuild/py312/examples_mpl.yml @@ -26,4 +26,12 @@ phases: - pip install setuptools - pip install "tox < 4.0" - tox -e py312-examples-mpl + # Assume special role + - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-Py311ExamplesMpl") + - export TMP_ROLE + - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') + - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') + - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') + - aws sts get-caller-identity + # Run special role-specific examples - tox -e py312-examples-mpl-keyring From 916ae8e00b6195ed91dcbfa0448aa0e2c23c49bc Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 9 Feb 2024 10:19:26 -0800 Subject: [PATCH 054/422] debug tox mpl keystore env --- .../internal/crypto/authentication.py | 9 +++++++-- src/aws_encryption_sdk/streaming_client.py | 17 ++++++++++++++--- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/aws_encryption_sdk/internal/crypto/authentication.py b/src/aws_encryption_sdk/internal/crypto/authentication.py index f90ac77e0..a6446981e 100644 --- a/src/aws_encryption_sdk/internal/crypto/authentication.py +++ b/src/aws_encryption_sdk/internal/crypto/authentication.py @@ -68,7 +68,7 @@ class Signer(_PrehashingAuthenticator): """ @classmethod - def from_key_bytes(cls, algorithm, key_bytes): + def from_key_bytes(cls, algorithm, key_bytes, encoding=serialization.Encoding.DER): """Builds a `Signer` from an algorithm suite and a raw signing key. :param algorithm: Algorithm on which to base signer @@ -76,7 +76,12 @@ def from_key_bytes(cls, algorithm, key_bytes): :param bytes key_bytes: Raw signing key :rtype: aws_encryption_sdk.internal.crypto.Signer """ - key = serialization.load_der_private_key(data=key_bytes, password=None, backend=default_backend()) + if encoding == serialization.Encoding.DER: + key = serialization.load_der_private_key(data=key_bytes, password=None, backend=default_backend()) + elif serialization.Encoding.PEM: + key = serialization.load_pem_private_key(data=key_bytes, password=None, backend=default_backend()) + else: + raise ValueError(f"Unsupported encoding for Signer: {encoding}") return cls(algorithm, key) def key_bytes(self): diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 04d44334a..e514337f5 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -23,6 +23,8 @@ import attr import six +from cryptography.hazmat.primitives import serialization + import aws_encryption_sdk.internal.utils from aws_encryption_sdk.mpl.cmm_handler import CMMHandler from aws_encryption_sdk.exceptions import ( @@ -555,9 +557,18 @@ def _prep_message(self): if self._encryption_materials.signing_key is None: self.signer = None else: - self.signer = Signer.from_key_bytes( - algorithm=self._encryption_materials.algorithm, key_bytes=self._encryption_materials.signing_key - ) + # MPL verification key is NOT key bytes, it is bytes of the compressed point + # TODO-MPL: clean this up, least-privilege violation. + if (isinstance(self.config.materials_manager, CMMHandler) + and hasattr(self.config.materials_manager, "mpl_cmm")): + self.signer = Signer.from_key_bytes( + algorithm=self._encryption_materials.algorithm, key_bytes=self._encryption_materials.signing_key, + encoding=serialization.Encoding.PEM, + ) + else: + self.signer = Signer.from_key_bytes( + algorithm=self._encryption_materials.algorithm, key_bytes=self._encryption_materials.signing_key + ) aws_encryption_sdk.internal.utils.validate_frame_length( frame_length=self.config.frame_length, algorithm=self._encryption_materials.algorithm ) From 222b13549febbbfe144e7f865a89203b4f3789fd Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 9 Feb 2024 10:38:14 -0800 Subject: [PATCH 055/422] debug tox mpl keystore env --- codebuild/py311/examples_mpl.yml | 2 +- codebuild/py312/examples_mpl.yml | 2 +- tox.ini | 7 +++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/codebuild/py311/examples_mpl.yml b/codebuild/py311/examples_mpl.yml index f7d705923..f8f2a6a01 100644 --- a/codebuild/py311/examples_mpl.yml +++ b/codebuild/py311/examples_mpl.yml @@ -29,5 +29,5 @@ phases: - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') - aws sts get-caller-identity # Run special role-specific examples - - tox -e py311-examples-mpl-keyring + - tox -e py311-mplexamples-mpl diff --git a/codebuild/py312/examples_mpl.yml b/codebuild/py312/examples_mpl.yml index c95f606e6..ba0660024 100644 --- a/codebuild/py312/examples_mpl.yml +++ b/codebuild/py312/examples_mpl.yml @@ -34,4 +34,4 @@ phases: - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') - aws sts get-caller-identity # Run special role-specific examples - - tox -e py312-examples-mpl-keyring + - tox -e py312-mplexamples-mpl diff --git a/tox.ini b/tox.ini index d618cb030..61d65b11d 100644 --- a/tox.ini +++ b/tox.ini @@ -2,8 +2,11 @@ envlist = # <3.11: run all non-MPL tests py{37,38,39,310}-{local,integ,accept,examples}, - # >=3.11: run all MPL tests and non-MPL tests + # >=3.11: run all tests with MPL installed and without MPL installed + # The `-mpl` suffix tells tox to install the MPL py{311,312}-{local,integ,accept,examples}{,-mpl}, + # >=3.11: run ONLY the MPL-specific tests (requires a special IAM role) + py{311,312}-{mplexamples}-mpl nocmk, bandit, doc8, readme, docs, {flake8,pylint}{,-tests,-examples}, @@ -76,7 +79,7 @@ commands = accept: {[testenv:base-command]commands} test/ -m accept examples: {[testenv:base-command]commands} examples/test/ -m examples --ignore examples/test/keyrings/ # MPL keyring examples require a special IAM role; run these separately under a separate set of permissions - examples-mpl-keyring: {[testenv:base-command]commands} examples/test/keyrings -m examples + mplexamples: {[testenv:base-command]commands} examples/test/keyrings -m examples all: {[testenv:base-command]commands} test/ examples/test/ manual: {[testenv:base-command]commands} From cab60167b76eb142258b97f2493879ad0028e818 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 9 Feb 2024 16:03:12 -0800 Subject: [PATCH 056/422] some unit tests --- src/aws_encryption_sdk/mpl/cmm_handler.py | 2 + .../mpl/materials_handlers.py | 4 +- .../mpl/mpl_import_handler.py | 14 +++ src/aws_encryption_sdk/streaming_client.py | 16 +-- test/unit/mpl/README.md | 1 + test/unit/mpl/test_cmm_handler.py | 111 ++++++++++++++++++ test/unit/test_mpl_import_handler.py | 34 ++++++ test/unit/test_streaming_client_configs.py | 3 + test/unit/test_streaming_client_mpl_import.py | 42 +++++++ tox.ini | 6 +- 10 files changed, 221 insertions(+), 12 deletions(-) create mode 100644 src/aws_encryption_sdk/mpl/mpl_import_handler.py create mode 100644 test/unit/mpl/README.md create mode 100644 test/unit/mpl/test_cmm_handler.py create mode 100644 test/unit/test_mpl_import_handler.py create mode 100644 test/unit/test_streaming_client_mpl_import.py diff --git a/src/aws_encryption_sdk/mpl/cmm_handler.py b/src/aws_encryption_sdk/mpl/cmm_handler.py index 5dfaab973..1f6c9ff41 100644 --- a/src/aws_encryption_sdk/mpl/cmm_handler.py +++ b/src/aws_encryption_sdk/mpl/cmm_handler.py @@ -76,7 +76,9 @@ def get_encryption_materials( mpl_input: GetEncryptionMaterialsInput = CMMHandler._native_to_mpl_get_encryption_materials( request ) + print(f"mpl_input: {mpl_input}") mpl_output: GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) + print(f"mpl_output: {mpl_output}") return EncryptionMaterialsHandler(mpl_output.encryption_materials) except AwsCryptographicMaterialProvidersException as mpl_exception: # Wrap MPL error into the ESDK error type diff --git a/src/aws_encryption_sdk/mpl/materials_handlers.py b/src/aws_encryption_sdk/mpl/materials_handlers.py index 57f54144e..df5b57d53 100644 --- a/src/aws_encryption_sdk/mpl/materials_handlers.py +++ b/src/aws_encryption_sdk/mpl/materials_handlers.py @@ -48,8 +48,8 @@ def __init__( elif isinstance(materials, MPL_EncryptionMaterials): self.mpl_materials = materials else: - raise ValueError(f"Invalid EncryptionMaterials passed to EncryptionMaterialsHandler.\ - materials: {materials}") + raise ValueError("Invalid EncryptionMaterials passed to EncryptionMaterialsHandler. " \ + f"materials: {materials}") @property def algorithm(self) -> Algorithm: diff --git a/src/aws_encryption_sdk/mpl/mpl_import_handler.py b/src/aws_encryption_sdk/mpl/mpl_import_handler.py new file mode 100644 index 000000000..40669da1e --- /dev/null +++ b/src/aws_encryption_sdk/mpl/mpl_import_handler.py @@ -0,0 +1,14 @@ +def has_mpl(): + """Returns True if the aws_cryptographic_materialproviders library is installed, False otherwise.""" + try: + _import_mpl() + return True + except ImportError: + return False + +def _import_mpl(): + """Private wrapper for import to help with unit test coverage. + + This is not directly tested. + """ + import aws_cryptographic_materialproviders \ No newline at end of file diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index e514337f5..106121377 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -71,15 +71,15 @@ from aws_encryption_sdk.materials_managers.default import DefaultCryptoMaterialsManager from aws_encryption_sdk.structures import MessageHeader -try: +from aws_encryption_sdk.mpl import mpl_import_handler +if mpl_import_handler.has_mpl(): from aws_cryptographic_materialproviders.mpl.client import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig from aws_cryptographic_materialproviders.mpl.models import CreateDefaultCryptographicMaterialsManagerInput from aws_cryptographic_materialproviders.mpl.references import IKeyring - - HAS_MPL = True -except ImportError: - HAS_MPL = False + _HAS_MPL = True +else: + _HAS_MPL = False _LOGGER = logging.getLogger(__name__) @@ -146,7 +146,7 @@ class _ClientConfig(object): # pylint: disable=too-many-instance-attributes key_provider = attr.ib( hash=True, default=None, validator=attr.validators.optional(attr.validators.instance_of(MasterKeyProvider)) ) - if HAS_MPL: + if _HAS_MPL: keyring = attr.ib( hash=True, default=None, validator=attr.validators.optional(attr.validators.instance_of(IKeyring)) ) @@ -194,9 +194,9 @@ def _no_mpl_attrs_post_init(self): def __attrs_post_init__(self): """Normalize inputs to crypto material manager.""" - if HAS_MPL: + if _HAS_MPL: self._has_mpl_attrs_post_init() - elif not HAS_MPL: + else: self._no_mpl_attrs_post_init() diff --git a/test/unit/mpl/README.md b/test/unit/mpl/README.md new file mode 100644 index 000000000..839feb7a2 --- /dev/null +++ b/test/unit/mpl/README.md @@ -0,0 +1 @@ +Tests in this file REQUIRE the aws-cryptographic-material-providers module to be installed in order to run. \ No newline at end of file diff --git a/test/unit/mpl/test_cmm_handler.py b/test/unit/mpl/test_cmm_handler.py new file mode 100644 index 000000000..45b49ed91 --- /dev/null +++ b/test/unit/mpl/test_cmm_handler.py @@ -0,0 +1,111 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Test suite to verify the mpl_import_handler module handles importing the MPL correctly.""" +import pytest +from mock import MagicMock, patch + +from aws_encryption_sdk.mpl.cmm_handler import CMMHandler + +from aws_encryption_sdk.mpl.materials_handlers import DecryptionMaterialsHandler, EncryptionMaterialsHandler +from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest +from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager +from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager + +from aws_encryption_sdk.materials_managers import ( + DecryptionMaterials as Native_DecryptionMaterials, + EncryptionMaterials as Native_EncryptionMaterials, +) + +from aws_cryptographic_materialproviders.mpl.models import ( + AlgorithmSuiteIdESDK, + CommitmentPolicyESDK, + DecryptMaterialsInput, + DecryptMaterialsOutput, + EncryptedDataKey as MPL_EncryptedDataKey, + GetEncryptionMaterialsInput, + GetEncryptionMaterialsOutput, + ) + +from aws_cryptographic_materialproviders.mpl.models import ( + DecryptionMaterials as MPL_DecryptionMaterials, + EncryptedDataKey as MPL_EncryptedDataKey, + EncryptionMaterials as MPL_EncryptionMaterials, + ) + +mock_native_cmm = MagicMock(__class__=CryptoMaterialsManager) +mock_mpl_cmm = MagicMock(__class__=ICryptographicMaterialsManager) +mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) +mock_encryption_materials_handler = MagicMock(__class__=EncryptionMaterialsHandler) +mock_native_encryption_materials = MagicMock(__class__=Native_EncryptionMaterials) +mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) + +pytestmark = [pytest.mark.unit, pytest.mark.local] + + +def test_GIVEN_native_CMM_WHEN_create_CMMHandler_THEN_is_using_native_cmm_returns_True(): + cmm_handler = CMMHandler(cmm=mock_native_cmm) + assert cmm_handler._is_using_native_cmm() + + +def test_GIVEN_mpl_CMM_WHEN_create_CMMHandler_THEN_is_using_native_cmm_returns_False(): + cmm_handler = CMMHandler(cmm=mock_mpl_cmm) + assert not cmm_handler._is_using_native_cmm() + + +def test_GIVEN_unknown_CMM_WHEN_create_CMMHandler_THEN_raise_ValueError(): + with pytest.raises(ValueError): + CMMHandler(cmm="not a CMM") + + +@patch.object(mock_native_cmm, "get_encryption_materials") +def test_GIVEN_native_CMM_WHEN_get_encryption_materials_THEN_return_native_encryption_materials(mock_get_encryption_materials): + # Mock: native_cmm.get_encryption_materials returns mock native encryption materials + mock_get_encryption_materials.return_value = mock_native_encryption_materials + + cmm_handler = CMMHandler(cmm=mock_native_cmm) + test = cmm_handler.get_encryption_materials(mock_encryption_materials_request) + + # Verify cmm_handler returns EncryptionMaterialsHandler + assert isinstance(test, EncryptionMaterialsHandler) + # Verify returned EncryptionMaterialsHandler uses the output of `get_encryption_materials` + assert test.native_materials == mock_native_encryption_materials + # Verify we actually called `get_encryption_materials` + mock_native_cmm.get_encryption_materials.assert_called_once_with(mock_encryption_materials_request) + + +@patch.object(mock_mpl_cmm, "get_encryption_materials") +@patch("aws_encryption_sdk.mpl.cmm_handler.CMMHandler._native_to_mpl_get_encryption_materials") +def test_GIVEN_mpl_CMM_WHEN_get_encryption_materials_THEN_return_mpl_encryption_materials( + mock_native_to_mpl_get_encryption_materials, + mock_get_encryption_materials, + +): + # Mock: mpl_cmm.get_encryption_materials returns mock MPL encryption materials + mock_get_encryption_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) + mock_get_encryption_materials_output.encryption_materials = mock_mpl_encryption_materials + mock_get_encryption_materials.return_value = mock_get_encryption_materials_output + + # Mock: CMMHandler._native_to_mpl_get_encryption_materials creates a GetEncryptionMaterialsInput + mock_get_encryption_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) + mock_native_to_mpl_get_encryption_materials.return_value = mock_get_encryption_materials_input + + cmm_handler = CMMHandler(cmm=mock_mpl_cmm) + test = cmm_handler.get_encryption_materials(mock_encryption_materials_request) + + # Verify cmm_handler returns EncryptionMaterialsHandler + assert isinstance(test, EncryptionMaterialsHandler) + # Verify returned EncryptionMaterialsHandler uses the output of `get_encryption_materials` + assert test.mpl_materials == mock_mpl_encryption_materials + # Verify we actually called `get_encryption_materials` + mock_mpl_cmm.get_encryption_materials.assert_called_once_with(mock_get_encryption_materials_input) + diff --git a/test/unit/test_mpl_import_handler.py b/test/unit/test_mpl_import_handler.py new file mode 100644 index 000000000..c17c358b4 --- /dev/null +++ b/test/unit/test_mpl_import_handler.py @@ -0,0 +1,34 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Test suite to verify the mpl_import_handler module handles importing the MPL correctly.""" +import pytest +from mock import patch + +from aws_encryption_sdk.mpl import mpl_import_handler + +pytestmark = [pytest.mark.unit, pytest.mark.local] + +@patch("aws_encryption_sdk.mpl.mpl_import_handler._import_mpl") +def test_GIVEN_import_mpl_succeeds_WHEN_call_has_mpl_THEN_return_True(import_mock): + # Mock a successful import of `aws_cryptographic_material_providers` + import_mock.return_value = None # No exception means successful import + + assert mpl_import_handler.has_mpl() is True + +@patch("aws_encryption_sdk.mpl.mpl_import_handler._import_mpl") +def test_GIVEN_import_mpl_fails_WHEN_call_has_mpl_THEN_return_False(import_mock): + # Mock not having a `aws_cryptographic_material_providers` module, + # even if it is installed in the Python environment + import_mock.side_effect = ImportError() + + assert not mpl_import_handler.has_mpl() \ No newline at end of file diff --git a/test/unit/test_streaming_client_configs.py b/test/unit/test_streaming_client_configs.py index 426f8f85f..80b7fdb28 100644 --- a/test/unit/test_streaming_client_configs.py +++ b/test/unit/test_streaming_client_configs.py @@ -154,3 +154,6 @@ def test_client_config_converts(kwargs, stream_type): assert isinstance(test.source, stream_type) if test.key_provider is not None: assert isinstance(test.materials_manager, DefaultCryptoMaterialsManager) + +def test_GIVEN_has_mpl_WHEN_import_THEN_imports_mpl_modules(): + \ No newline at end of file diff --git a/test/unit/test_streaming_client_mpl_import.py b/test/unit/test_streaming_client_mpl_import.py new file mode 100644 index 000000000..8ce016caf --- /dev/null +++ b/test/unit/test_streaming_client_mpl_import.py @@ -0,0 +1,42 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Unit test suite to validate aws_encryption_sdk.streaming_client MPL import logic.""" +import io + +import pytest +from mock import patch +from importlib import reload + +import aws_encryption_sdk.streaming_client + +pytestmark = [pytest.mark.unit, pytest.mark.local] + +@patch.object(aws_encryption_sdk.streaming_client.mpl_import_handler, "has_mpl") +def test_GIVEN_has_mpl_returns_True_WHEN_import_streaming_client_THEN_imports_mpl_modules(has_mpl_mock): + has_mpl_mock.return_value = True + + # Reload module given the mock + reload(aws_encryption_sdk.streaming_client) + + assert hasattr(aws_encryption_sdk.streaming_client, "_HAS_MPL") + assert aws_encryption_sdk.streaming_client._HAS_MPL is True + +@patch.object(aws_encryption_sdk.streaming_client.mpl_import_handler, "has_mpl") +def test_GIVEN_has_mpl_returns_False_WHEN_import_streaming_client_THEN_does_not_import_mpl_modules(has_mpl_mock): + has_mpl_mock.return_value = False + + # Reload module given the mock + reload(aws_encryption_sdk.streaming_client) + + assert hasattr(aws_encryption_sdk.streaming_client, "_HAS_MPL") + assert aws_encryption_sdk.streaming_client._HAS_MPL is False \ No newline at end of file diff --git a/tox.ini b/tox.ini index 61d65b11d..3cc7017e1 100644 --- a/tox.ini +++ b/tox.ini @@ -6,7 +6,8 @@ envlist = # The `-mpl` suffix tells tox to install the MPL py{311,312}-{local,integ,accept,examples}{,-mpl}, # >=3.11: run ONLY the MPL-specific tests (requires a special IAM role) - py{311,312}-{mplexamples}-mpl + # the extra `-mpl` suffix tells tox to install the MPL + py{311,312}-mpl{local,examples}-mpl nocmk, bandit, doc8, readme, docs, {flake8,pylint}{,-tests,-examples}, @@ -74,7 +75,8 @@ deps = # install the MPL if in environment mpl: -rrequirements_mpl.txt commands = - local: {[testenv:base-command]commands} test/ -m local + local: {[testenv:base-command]commands} test/ -m local --ignore test/mpl/ + local: {[testenv:base-command]commands} test/mpl/ -m local integ: {[testenv:base-command]commands} test/ -m integ accept: {[testenv:base-command]commands} test/ -m accept examples: {[testenv:base-command]commands} examples/test/ -m examples --ignore examples/test/keyrings/ From a7416b18f9afa367e5053f0bd936735da7ec6e01 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 13 Feb 2024 09:26:04 -0800 Subject: [PATCH 057/422] add mpl coverage --- buildspec.yml | 2 ++ codebuild/coverage/coverage_mpl.yml | 14 ++++++++++++++ tox.ini | 6 +++++- 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 codebuild/coverage/coverage_mpl.yml diff --git a/buildspec.yml b/buildspec.yml index c718c3df5..3d70c144d 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -106,6 +106,8 @@ batch: - identifier: code_coverage buildspec: codebuild/coverage/coverage.yml + - identifier: code_coverage_mpl + buildspec: codebuild/coverage/coverage_mpl.yml - identifier: compliance buildspec: codebuild/compliance/compliance.yml diff --git a/codebuild/coverage/coverage_mpl.yml b/codebuild/coverage/coverage_mpl.yml new file mode 100644 index 000000000..5dcc65382 --- /dev/null +++ b/codebuild/coverage/coverage_mpl.yml @@ -0,0 +1,14 @@ +version: 0.2 + +env: + variables: + TOXENV: "mplcoverage-mpl" + +phases: + install: + runtime-versions: + python: latest + build: + commands: + - pip install "tox < 4.0" + - tox diff --git a/tox.ini b/tox.ini index 3cc7017e1..7aacb047a 100644 --- a/tox.ini +++ b/tox.ini @@ -6,6 +6,8 @@ envlist = # The `-mpl` suffix tells tox to install the MPL py{311,312}-{local,integ,accept,examples}{,-mpl}, # >=3.11: run ONLY the MPL-specific tests (requires a special IAM role) + # the `mpl` prefix runs only MPL-specific tests + # (non-MPL-specific tests are run from the line above) # the extra `-mpl` suffix tells tox to install the MPL py{311,312}-mpl{local,examples}-mpl nocmk, @@ -87,7 +89,9 @@ commands = # Run code coverage on the unit tests [testenv:coverage] -commands = {[testenv:base-command]commands} --cov aws_encryption_sdk test/ -m local +commands = {[testenv:base-command]commands} --cov aws_encryption_sdk test/ -m local --ignore test/unit/mpl/ +[testenv:mplcoverage] +commands = {[testenv:base-command]commands} --cov aws_encryption_sdk test/unit/mpl/ -m local # Verify that local tests work without environment variables present [testenv:nocmk] From 7b3dc5fc303afad5b0d3519b12de833135e3d326 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 13 Feb 2024 09:30:19 -0800 Subject: [PATCH 058/422] . --- tox.ini | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index 7aacb047a..194b4d412 100644 --- a/tox.ini +++ b/tox.ini @@ -77,14 +77,14 @@ deps = # install the MPL if in environment mpl: -rrequirements_mpl.txt commands = - local: {[testenv:base-command]commands} test/ -m local --ignore test/mpl/ - local: {[testenv:base-command]commands} test/mpl/ -m local - integ: {[testenv:base-command]commands} test/ -m integ - accept: {[testenv:base-command]commands} test/ -m accept + local: {[testenv:base-command]commands} test/ -m local --ignore test/unit/mpl/ + mpllocal: {[testenv:base-command]commands} test/unit/mpl/ -m local + integ: {[testenv:base-command]commands} test/ -m integ --ignore test/unit/mpl/ + accept: {[testenv:base-command]commands} test/ -m accept --ignore test/unit/mpl/ examples: {[testenv:base-command]commands} examples/test/ -m examples --ignore examples/test/keyrings/ # MPL keyring examples require a special IAM role; run these separately under a separate set of permissions mplexamples: {[testenv:base-command]commands} examples/test/keyrings -m examples - all: {[testenv:base-command]commands} test/ examples/test/ + all: {[testenv:base-command]commands} test/ examples/test/ --ignore test/unit/mpl/ manual: {[testenv:base-command]commands} # Run code coverage on the unit tests From 7a5e4eb9e6055576759d45950bd66ace921d65aa Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 13 Feb 2024 09:33:38 -0800 Subject: [PATCH 059/422] . --- test/unit/test_streaming_client_configs.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/unit/test_streaming_client_configs.py b/test/unit/test_streaming_client_configs.py index 80b7fdb28..426f8f85f 100644 --- a/test/unit/test_streaming_client_configs.py +++ b/test/unit/test_streaming_client_configs.py @@ -154,6 +154,3 @@ def test_client_config_converts(kwargs, stream_type): assert isinstance(test.source, stream_type) if test.key_provider is not None: assert isinstance(test.materials_manager, DefaultCryptoMaterialsManager) - -def test_GIVEN_has_mpl_WHEN_import_THEN_imports_mpl_modules(): - \ No newline at end of file From 0649995f59e6b3de7ad68d5ff7497a44dc021c31 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 13 Feb 2024 15:46:47 -0800 Subject: [PATCH 060/422] mock imports --- src/aws_encryption_sdk/streaming_client.py | 2 +- test/unit/test_streaming_client_mpl_import.py | 11 +++++++++++ test/unit/test_streaming_client_stream_decryptor.py | 2 +- tox.ini | 2 +- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 106121377..044626c7f 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -1095,7 +1095,7 @@ def close(self): """Closes out the stream.""" _LOGGER.debug("Closing stream") if not hasattr(self, "footer"): - raise SerializationError("Footer not read, message may be corrupted or data key may be incorrect") + raise SerializationError("Footer not read") super(StreamDecryptor, self).close() diff --git a/test/unit/test_streaming_client_mpl_import.py b/test/unit/test_streaming_client_mpl_import.py index 8ce016caf..f71f337b7 100644 --- a/test/unit/test_streaming_client_mpl_import.py +++ b/test/unit/test_streaming_client_mpl_import.py @@ -17,6 +17,8 @@ from mock import patch from importlib import reload +from mock import Mock + import aws_encryption_sdk.streaming_client pytestmark = [pytest.mark.unit, pytest.mark.local] @@ -25,6 +27,15 @@ def test_GIVEN_has_mpl_returns_True_WHEN_import_streaming_client_THEN_imports_mpl_modules(has_mpl_mock): has_mpl_mock.return_value = True + # Mock any imports used in the try/catch block + # If more imports are added there, then this needs to be expanded + # This unit test should pass even if the MPL is not installed + import sys + sys.modules['aws_cryptographic_materialproviders.mpl.client'] = Mock() + sys.modules['aws_cryptographic_materialproviders.mpl.config'] = Mock() + sys.modules['aws_cryptographic_materialproviders.mpl.models'] = Mock() + sys.modules['aws_cryptographic_materialproviders.mpl.references'] = Mock() + # Reload module given the mock reload(aws_encryption_sdk.streaming_client) diff --git a/test/unit/test_streaming_client_stream_decryptor.py b/test/unit/test_streaming_client_stream_decryptor.py index 94b22b092..157755094 100644 --- a/test/unit/test_streaming_client_stream_decryptor.py +++ b/test/unit/test_streaming_client_stream_decryptor.py @@ -767,4 +767,4 @@ def test_close_no_footer(self, mock_close): ) with pytest.raises(SerializationError) as excinfo: test_decryptor.close() - excinfo.match("Footer not read, message may be corrupted or data key may be incorrect") + excinfo.match("Footer not read") diff --git a/tox.ini b/tox.ini index 194b4d412..9e2d95477 100644 --- a/tox.ini +++ b/tox.ini @@ -90,7 +90,7 @@ commands = # Run code coverage on the unit tests [testenv:coverage] commands = {[testenv:base-command]commands} --cov aws_encryption_sdk test/ -m local --ignore test/unit/mpl/ -[testenv:mplcoverage] +[testenv:mplcoverage-mpl] commands = {[testenv:base-command]commands} --cov aws_encryption_sdk test/unit/mpl/ -m local # Verify that local tests work without environment variables present From 6691fa2d81c211e7a97be9450f028ca89b47bcb6 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 15:19:17 -0800 Subject: [PATCH 061/422] refactor, fix --- .github/workflows/ci_static-analysis.yaml | 14 +++++- examples/src/keyrings/hierarchical_keyring.py | 50 ++++++++++--------- .../{ => internal}/mpl/__init__.py | 0 .../{ => internal}/mpl/cmm_handler.py | 2 +- .../{ => internal}/mpl/materials_handlers.py | 4 +- .../internal/mpl/mpl_import_handler.py | 21 ++++++++ .../mpl/mpl_import_handler.py | 14 ------ src/aws_encryption_sdk/streaming_client.py | 6 +-- test/unit/mpl/test_cmm_handler.py | 44 ++++++---------- test/unit/test_mpl_import_handler.py | 10 ++-- test/unit/test_streaming_client_mpl_import.py | 12 ++--- tox.ini | 22 +++++--- 12 files changed, 108 insertions(+), 91 deletions(-) rename src/aws_encryption_sdk/{ => internal}/mpl/__init__.py (100%) rename src/aws_encryption_sdk/{ => internal}/mpl/cmm_handler.py (98%) rename src/aws_encryption_sdk/{ => internal}/mpl/materials_handlers.py (98%) create mode 100644 src/aws_encryption_sdk/internal/mpl/mpl_import_handler.py delete mode 100644 src/aws_encryption_sdk/mpl/mpl_import_handler.py diff --git a/.github/workflows/ci_static-analysis.yaml b/.github/workflows/ci_static-analysis.yaml index 0093ae9a9..85d7f4a62 100644 --- a/.github/workflows/ci_static-analysis.yaml +++ b/.github/workflows/ci_static-analysis.yaml @@ -13,6 +13,9 @@ jobs: strategy: fail-fast: false matrix: + python: + - 3.8 + - 3.11 category: - bandit - doc8 @@ -26,15 +29,22 @@ jobs: - pylint-examples - black-check - isort-check + optional_mpl_dependency: + - "" + - -mpl + exclude: + # MPL is not supported on <3.11 + - python: 3.8 + optional_mpl_dependency: -mpl steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: ${{ matrix.python }} - run: | python -m pip install --upgrade pip pip install --upgrade -r dev_requirements/ci-requirements.txt - name: run test env: - TOXENV: ${{ matrix.category }} + TOXENV: ${{ matrix.category }}${{ matrix.optional_mpl_dependency }} run: tox -- -vv diff --git a/examples/src/keyrings/hierarchical_keyring.py b/examples/src/keyrings/hierarchical_keyring.py index a99728b6e..21108d9a0 100644 --- a/examples/src/keyrings/hierarchical_keyring.py +++ b/examples/src/keyrings/hierarchical_keyring.py @@ -4,23 +4,28 @@ import sys import boto3 -from aws_cryptographic_materialproviders.keystore.client import KeyStore -from aws_cryptographic_materialproviders.keystore.config import KeyStoreConfig -from aws_cryptographic_materialproviders.keystore.models import CreateKeyInput, KMSConfigurationKmsKeyArn -from aws_cryptographic_materialproviders.mpl.client import AwsCryptographicMaterialProviders -from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig -from aws_cryptographic_materialproviders.mpl.models import ( - CacheTypeDefault, - CreateAwsKmsHierarchicalKeyringInput, - DefaultCache, - GetBranchKeyIdInput, - GetBranchKeyIdOutput, -) -from aws_cryptographic_materialproviders.mpl.references import IBranchKeyIdSupplier, IKeyring +from typing import Dict import aws_encryption_sdk from aws_encryption_sdk import CommitmentPolicy from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError +from aws_encryption_sdk.internal.mpl import mpl_import_handler + +if mpl_import_handler.has_mpl(): + # noqa pylint: disable=import-error + from aws_cryptographic_materialproviders.keystore.client import KeyStore + from aws_cryptographic_materialproviders.keystore.config import KeyStoreConfig + from aws_cryptographic_materialproviders.keystore.models import CreateKeyInput, KMSConfigurationKmsKeyArn + from aws_cryptographic_materialproviders.mpl.client import AwsCryptographicMaterialProviders + from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig + from aws_cryptographic_materialproviders.mpl.models import ( + CacheTypeDefault, + CreateAwsKmsHierarchicalKeyringInput, + DefaultCache, + GetBranchKeyIdInput, + GetBranchKeyIdOutput, + ) + from aws_cryptographic_materialproviders.mpl.references import IBranchKeyIdSupplier, IKeyring module_root_dir = '/'.join(__file__.split("/")[:-1]) @@ -71,6 +76,7 @@ def encrypt_and_decrypt_with_keyring( branch_key_id_B: str = keystore.create_key(input=CreateKeyInput()).branch_key_identifier class ExampleBranchKeyIdSupplier(IBranchKeyIdSupplier): + """Example implementation of a branch key ID supplier.""" branch_key_id_for_tenant_A: str branch_key_id_for_tenant_B: str @@ -80,9 +86,11 @@ def __init__(self, tenant_1_id, tenant_2_id): def get_branch_key_id( self, - input: GetBranchKeyIdInput + # Change this to `native_input` + input: GetBranchKeyIdInput # noqa pylint: disable=redefined-builtin ) -> GetBranchKeyIdOutput: - encryption_context: dict[str, str] = input.encryption_context + """Returns branch key ID from the tenant ID in input's encryption context.""" + encryption_context: Dict[str, str] = input.encryption_context if b"tenant" not in encryption_context: raise ValueError("EncryptionContext invalid, does not contain expected tenant key value pair.") @@ -128,7 +136,7 @@ def get_branch_key_id( # The Branch Key Id supplier uses the encryption context to determine which branch key id will # be used to encrypt data. # Create encryption context for TenantA - encryption_context_A: dict[str, str] = { + encryption_context_A: Dict[str, str] = { "tenant": "TenantA", "encryption": "context", "is not": "secret", @@ -138,7 +146,7 @@ def get_branch_key_id( } # Create encryption context for TenantB - encryption_context_B: dict[str, str] = { + encryption_context_B: Dict[str, str] = { "tenant": "TenantB", "encryption": "context", "is not": "secret", @@ -191,8 +199,6 @@ def get_branch_key_id( input=keyring_input_B ) - # TODO: Run the decrypt, get expected exception type - # This should fail try: client.decrypt( source=ciphertext_A, @@ -201,7 +207,7 @@ def get_branch_key_id( except AWSEncryptionSDKClientError: pass - # # This should fail + # This should fail try: client.decrypt( source=ciphertext_B, @@ -220,6 +226,4 @@ def get_branch_key_id( source=ciphertext_B, keyring=hierarchical_keyring_B ) - assert plaintext_bytes_B == EXAMPLE_DATA - -# Also, a thread-safe example ig + assert plaintext_bytes_B == EXAMPLE_DATA \ No newline at end of file diff --git a/src/aws_encryption_sdk/mpl/__init__.py b/src/aws_encryption_sdk/internal/mpl/__init__.py similarity index 100% rename from src/aws_encryption_sdk/mpl/__init__.py rename to src/aws_encryption_sdk/internal/mpl/__init__.py diff --git a/src/aws_encryption_sdk/mpl/cmm_handler.py b/src/aws_encryption_sdk/internal/mpl/cmm_handler.py similarity index 98% rename from src/aws_encryption_sdk/mpl/cmm_handler.py rename to src/aws_encryption_sdk/internal/mpl/cmm_handler.py index 1f6c9ff41..c285afa04 100644 --- a/src/aws_encryption_sdk/mpl/cmm_handler.py +++ b/src/aws_encryption_sdk/internal/mpl/cmm_handler.py @@ -23,7 +23,7 @@ from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError from aws_encryption_sdk.identifiers import CommitmentPolicy -from aws_encryption_sdk.mpl.materials_handlers import DecryptionMaterialsHandler, EncryptionMaterialsHandler +from aws_encryption_sdk.internal.mpl.materials_handlers import DecryptionMaterialsHandler, EncryptionMaterialsHandler from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager from aws_encryption_sdk.structures import EncryptedDataKey as Native_EncryptedDataKey diff --git a/src/aws_encryption_sdk/mpl/materials_handlers.py b/src/aws_encryption_sdk/internal/mpl/materials_handlers.py similarity index 98% rename from src/aws_encryption_sdk/mpl/materials_handlers.py rename to src/aws_encryption_sdk/internal/mpl/materials_handlers.py index df5b57d53..bf32c2718 100644 --- a/src/aws_encryption_sdk/mpl/materials_handlers.py +++ b/src/aws_encryption_sdk/internal/mpl/materials_handlers.py @@ -48,8 +48,8 @@ def __init__( elif isinstance(materials, MPL_EncryptionMaterials): self.mpl_materials = materials else: - raise ValueError("Invalid EncryptionMaterials passed to EncryptionMaterialsHandler. " \ - f"materials: {materials}") + raise ValueError("Invalid EncryptionMaterials passed to EncryptionMaterialsHandler. " + f"materials: {materials}") @property def algorithm(self) -> Algorithm: diff --git a/src/aws_encryption_sdk/internal/mpl/mpl_import_handler.py b/src/aws_encryption_sdk/internal/mpl/mpl_import_handler.py new file mode 100644 index 000000000..55319bc43 --- /dev/null +++ b/src/aws_encryption_sdk/internal/mpl/mpl_import_handler.py @@ -0,0 +1,21 @@ +"""Detects whether the MPL is installed for use by internal ESDK code. +External customers should not need to interact with this. +""" + + +def has_mpl(): + """Returns True if the aws-cryptographic-material-providers library is installed, False otherwise.""" + try: + _import_mpl() + return True + except ImportError: + return False + + +def _import_mpl(): + """Private wrapper for import. + This only exists to help with unit test coverage. + This is not directly tested. + """ + # noqa pylint:disable=unused-import,import-outside-toplevel,import-error + import aws_cryptographic_materialproviders diff --git a/src/aws_encryption_sdk/mpl/mpl_import_handler.py b/src/aws_encryption_sdk/mpl/mpl_import_handler.py deleted file mode 100644 index 40669da1e..000000000 --- a/src/aws_encryption_sdk/mpl/mpl_import_handler.py +++ /dev/null @@ -1,14 +0,0 @@ -def has_mpl(): - """Returns True if the aws_cryptographic_materialproviders library is installed, False otherwise.""" - try: - _import_mpl() - return True - except ImportError: - return False - -def _import_mpl(): - """Private wrapper for import to help with unit test coverage. - - This is not directly tested. - """ - import aws_cryptographic_materialproviders \ No newline at end of file diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 044626c7f..c4c15559b 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -22,11 +22,9 @@ import attr import six - from cryptography.hazmat.primitives import serialization import aws_encryption_sdk.internal.utils -from aws_encryption_sdk.mpl.cmm_handler import CMMHandler from aws_encryption_sdk.exceptions import ( ActionNotAllowedError, AWSEncryptionSDKClientError, @@ -60,6 +58,8 @@ serialize_non_framed_close, serialize_non_framed_open, ) +from aws_encryption_sdk.internal.mpl import mpl_import_handler +from aws_encryption_sdk.internal.mpl.cmm_handler import CMMHandler from aws_encryption_sdk.internal.utils.commitment import ( validate_commitment_policy_on_decrypt, validate_commitment_policy_on_encrypt, @@ -71,8 +71,8 @@ from aws_encryption_sdk.materials_managers.default import DefaultCryptoMaterialsManager from aws_encryption_sdk.structures import MessageHeader -from aws_encryption_sdk.mpl import mpl_import_handler if mpl_import_handler.has_mpl(): + # noqa pylint: disable=import-error from aws_cryptographic_materialproviders.mpl.client import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig from aws_cryptographic_materialproviders.mpl.models import CreateDefaultCryptographicMaterialsManagerInput diff --git a/test/unit/mpl/test_cmm_handler.py b/test/unit/mpl/test_cmm_handler.py index 45b49ed91..343ac514b 100644 --- a/test/unit/mpl/test_cmm_handler.py +++ b/test/unit/mpl/test_cmm_handler.py @@ -12,35 +12,21 @@ # language governing permissions and limitations under the License. """Test suite to verify the mpl_import_handler module handles importing the MPL correctly.""" import pytest -from mock import MagicMock, patch - -from aws_encryption_sdk.mpl.cmm_handler import CMMHandler - -from aws_encryption_sdk.mpl.materials_handlers import DecryptionMaterialsHandler, EncryptionMaterialsHandler -from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest -from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager +from aws_cryptographic_materialproviders.mpl.models import ( + EncryptionMaterials as MPL_EncryptionMaterials, + GetEncryptionMaterialsInput, + GetEncryptionMaterialsOutput, +) from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager +from mock import MagicMock, patch +from aws_encryption_sdk.internal.mpl.cmm_handler import CMMHandler +from aws_encryption_sdk.internal.mpl.materials_handlers import EncryptionMaterialsHandler from aws_encryption_sdk.materials_managers import ( - DecryptionMaterials as Native_DecryptionMaterials, EncryptionMaterials as Native_EncryptionMaterials, + EncryptionMaterialsRequest, ) - -from aws_cryptographic_materialproviders.mpl.models import ( - AlgorithmSuiteIdESDK, - CommitmentPolicyESDK, - DecryptMaterialsInput, - DecryptMaterialsOutput, - EncryptedDataKey as MPL_EncryptedDataKey, - GetEncryptionMaterialsInput, - GetEncryptionMaterialsOutput, - ) - -from aws_cryptographic_materialproviders.mpl.models import ( - DecryptionMaterials as MPL_DecryptionMaterials, - EncryptedDataKey as MPL_EncryptedDataKey, - EncryptionMaterials as MPL_EncryptionMaterials, - ) +from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager mock_native_cmm = MagicMock(__class__=CryptoMaterialsManager) mock_mpl_cmm = MagicMock(__class__=ICryptographicMaterialsManager) @@ -68,7 +54,9 @@ def test_GIVEN_unknown_CMM_WHEN_create_CMMHandler_THEN_raise_ValueError(): @patch.object(mock_native_cmm, "get_encryption_materials") -def test_GIVEN_native_CMM_WHEN_get_encryption_materials_THEN_return_native_encryption_materials(mock_get_encryption_materials): +def test_GIVEN_native_CMM_WHEN_get_encryption_materials_THEN_return_native_encryption_materials( + mock_get_encryption_materials +): # Mock: native_cmm.get_encryption_materials returns mock native encryption materials mock_get_encryption_materials.return_value = mock_native_encryption_materials @@ -84,17 +72,16 @@ def test_GIVEN_native_CMM_WHEN_get_encryption_materials_THEN_return_native_encry @patch.object(mock_mpl_cmm, "get_encryption_materials") -@patch("aws_encryption_sdk.mpl.cmm_handler.CMMHandler._native_to_mpl_get_encryption_materials") +@patch("aws_encryption_sdk.internal.mpl.cmm_handler.CMMHandler._native_to_mpl_get_encryption_materials") def test_GIVEN_mpl_CMM_WHEN_get_encryption_materials_THEN_return_mpl_encryption_materials( mock_native_to_mpl_get_encryption_materials, mock_get_encryption_materials, - ): # Mock: mpl_cmm.get_encryption_materials returns mock MPL encryption materials mock_get_encryption_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) mock_get_encryption_materials_output.encryption_materials = mock_mpl_encryption_materials mock_get_encryption_materials.return_value = mock_get_encryption_materials_output - + # Mock: CMMHandler._native_to_mpl_get_encryption_materials creates a GetEncryptionMaterialsInput mock_get_encryption_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) mock_native_to_mpl_get_encryption_materials.return_value = mock_get_encryption_materials_input @@ -108,4 +95,3 @@ def test_GIVEN_mpl_CMM_WHEN_get_encryption_materials_THEN_return_mpl_encryption_ assert test.mpl_materials == mock_mpl_encryption_materials # Verify we actually called `get_encryption_materials` mock_mpl_cmm.get_encryption_materials.assert_called_once_with(mock_get_encryption_materials_input) - diff --git a/test/unit/test_mpl_import_handler.py b/test/unit/test_mpl_import_handler.py index c17c358b4..b82c3092b 100644 --- a/test/unit/test_mpl_import_handler.py +++ b/test/unit/test_mpl_import_handler.py @@ -14,21 +14,23 @@ import pytest from mock import patch -from aws_encryption_sdk.mpl import mpl_import_handler +from aws_encryption_sdk.internal.mpl import mpl_import_handler pytestmark = [pytest.mark.unit, pytest.mark.local] -@patch("aws_encryption_sdk.mpl.mpl_import_handler._import_mpl") + +@patch("aws_encryption_sdk.internal.mpl.mpl_import_handler._import_mpl") def test_GIVEN_import_mpl_succeeds_WHEN_call_has_mpl_THEN_return_True(import_mock): # Mock a successful import of `aws_cryptographic_material_providers` import_mock.return_value = None # No exception means successful import assert mpl_import_handler.has_mpl() is True -@patch("aws_encryption_sdk.mpl.mpl_import_handler._import_mpl") + +@patch("aws_encryption_sdk.internal.mpl.mpl_import_handler._import_mpl") def test_GIVEN_import_mpl_fails_WHEN_call_has_mpl_THEN_return_False(import_mock): # Mock not having a `aws_cryptographic_material_providers` module, # even if it is installed in the Python environment import_mock.side_effect = ImportError() - assert not mpl_import_handler.has_mpl() \ No newline at end of file + assert not mpl_import_handler.has_mpl() diff --git a/test/unit/test_streaming_client_mpl_import.py b/test/unit/test_streaming_client_mpl_import.py index f71f337b7..594ef3478 100644 --- a/test/unit/test_streaming_client_mpl_import.py +++ b/test/unit/test_streaming_client_mpl_import.py @@ -11,18 +11,18 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. """Unit test suite to validate aws_encryption_sdk.streaming_client MPL import logic.""" -import io -import pytest -from mock import patch +import sys from importlib import reload -from mock import Mock +import pytest +from mock import Mock, patch import aws_encryption_sdk.streaming_client pytestmark = [pytest.mark.unit, pytest.mark.local] + @patch.object(aws_encryption_sdk.streaming_client.mpl_import_handler, "has_mpl") def test_GIVEN_has_mpl_returns_True_WHEN_import_streaming_client_THEN_imports_mpl_modules(has_mpl_mock): has_mpl_mock.return_value = True @@ -30,7 +30,6 @@ def test_GIVEN_has_mpl_returns_True_WHEN_import_streaming_client_THEN_imports_mp # Mock any imports used in the try/catch block # If more imports are added there, then this needs to be expanded # This unit test should pass even if the MPL is not installed - import sys sys.modules['aws_cryptographic_materialproviders.mpl.client'] = Mock() sys.modules['aws_cryptographic_materialproviders.mpl.config'] = Mock() sys.modules['aws_cryptographic_materialproviders.mpl.models'] = Mock() @@ -42,6 +41,7 @@ def test_GIVEN_has_mpl_returns_True_WHEN_import_streaming_client_THEN_imports_mp assert hasattr(aws_encryption_sdk.streaming_client, "_HAS_MPL") assert aws_encryption_sdk.streaming_client._HAS_MPL is True + @patch.object(aws_encryption_sdk.streaming_client.mpl_import_handler, "has_mpl") def test_GIVEN_has_mpl_returns_False_WHEN_import_streaming_client_THEN_does_not_import_mpl_modules(has_mpl_mock): has_mpl_mock.return_value = False @@ -50,4 +50,4 @@ def test_GIVEN_has_mpl_returns_False_WHEN_import_streaming_client_THEN_does_not_ reload(aws_encryption_sdk.streaming_client) assert hasattr(aws_encryption_sdk.streaming_client, "_HAS_MPL") - assert aws_encryption_sdk.streaming_client._HAS_MPL is False \ No newline at end of file + assert aws_encryption_sdk.streaming_client._HAS_MPL is False diff --git a/tox.ini b/tox.ini index 9e2d95477..20b4d9426 100644 --- a/tox.ini +++ b/tox.ini @@ -3,12 +3,18 @@ envlist = # <3.11: run all non-MPL tests py{37,38,39,310}-{local,integ,accept,examples}, # >=3.11: run all tests with MPL installed and without MPL installed - # The `-mpl` suffix tells tox to install the MPL + # The `-mpl` suffix tells tox to install the MPL. + # In the case where the suffix IS NOT appended, + # this runs tests for the target version WITHOUT the MPL installed. + # In the case where the suffix IS appended, + # this runs tests for the target version WITH the MPL installed. + # This does not run any MPL-specific tests; it only runs non-MPL-specific + # tests in a test environment that also has the MPL. py{311,312}-{local,integ,accept,examples}{,-mpl}, - # >=3.11: run ONLY the MPL-specific tests (requires a special IAM role) - # the `mpl` prefix runs only MPL-specific tests - # (non-MPL-specific tests are run from the line above) - # the extra `-mpl` suffix tells tox to install the MPL + # >=3.11: Run ONLY the MPL-specific tests. + # These must be separate from the above target. + # These require the `-mpl` suffix so tox installs the MPL. + # The `mpl` prefix runs only MPL-specific tests py{311,312}-mpl{local,examples}-mpl nocmk, bandit, doc8, readme, docs, @@ -78,6 +84,7 @@ deps = mpl: -rrequirements_mpl.txt commands = local: {[testenv:base-command]commands} test/ -m local --ignore test/unit/mpl/ + # MPL unit tests require the MPL to be installed mpllocal: {[testenv:base-command]commands} test/unit/mpl/ -m local integ: {[testenv:base-command]commands} test/ -m integ --ignore test/unit/mpl/ accept: {[testenv:base-command]commands} test/ -m accept --ignore test/unit/mpl/ @@ -194,13 +201,14 @@ commands = --max-module-lines=1500 \ src/aws_encryption_sdk/ \ setup.py + --ignore-paths=src/aws_encryption_sdk/internal/mpl/ [testenv:pylint-examples] basepython = {[testenv:pylint]basepython} deps = {[testenv:pylint]deps} commands = - pylint --rcfile=examples/src/pylintrc examples/src/ - pylint --rcfile=examples/test/pylintrc --disable R0801 examples/test/ + pylint --rcfile=examples/src/pylintrc examples/src/ --ignore-paths=examples/src/keyrings + pylint --rcfile=examples/test/pylintrc --disable R0801 examples/test/ --ignore-paths=examples/test/keyrings [testenv:pylint-tests] basepython = {[testenv:pylint]basepython} From 3ae1e069df8b2ea75762ce61ba4ea5f3798c24a0 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 15:22:21 -0800 Subject: [PATCH 062/422] refactor, fix --- .github/workflows/ci_static-analysis.yaml | 8 -------- examples/src/keyrings/hierarchical_keyring.py | 3 ++- src/aws_encryption_sdk/internal/mpl/cmm_handler.py | 2 -- 3 files changed, 2 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci_static-analysis.yaml b/.github/workflows/ci_static-analysis.yaml index 85d7f4a62..802bad2bc 100644 --- a/.github/workflows/ci_static-analysis.yaml +++ b/.github/workflows/ci_static-analysis.yaml @@ -15,7 +15,6 @@ jobs: matrix: python: - 3.8 - - 3.11 category: - bandit - doc8 @@ -29,13 +28,6 @@ jobs: - pylint-examples - black-check - isort-check - optional_mpl_dependency: - - "" - - -mpl - exclude: - # MPL is not supported on <3.11 - - python: 3.8 - optional_mpl_dependency: -mpl steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 diff --git a/examples/src/keyrings/hierarchical_keyring.py b/examples/src/keyrings/hierarchical_keyring.py index 21108d9a0..56af60115 100644 --- a/examples/src/keyrings/hierarchical_keyring.py +++ b/examples/src/keyrings/hierarchical_keyring.py @@ -77,6 +77,7 @@ def encrypt_and_decrypt_with_keyring( class ExampleBranchKeyIdSupplier(IBranchKeyIdSupplier): """Example implementation of a branch key ID supplier.""" + branch_key_id_for_tenant_A: str branch_key_id_for_tenant_B: str @@ -226,4 +227,4 @@ def get_branch_key_id( source=ciphertext_B, keyring=hierarchical_keyring_B ) - assert plaintext_bytes_B == EXAMPLE_DATA \ No newline at end of file + assert plaintext_bytes_B == EXAMPLE_DATA diff --git a/src/aws_encryption_sdk/internal/mpl/cmm_handler.py b/src/aws_encryption_sdk/internal/mpl/cmm_handler.py index c285afa04..9789651e5 100644 --- a/src/aws_encryption_sdk/internal/mpl/cmm_handler.py +++ b/src/aws_encryption_sdk/internal/mpl/cmm_handler.py @@ -76,9 +76,7 @@ def get_encryption_materials( mpl_input: GetEncryptionMaterialsInput = CMMHandler._native_to_mpl_get_encryption_materials( request ) - print(f"mpl_input: {mpl_input}") mpl_output: GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) - print(f"mpl_output: {mpl_output}") return EncryptionMaterialsHandler(mpl_output.encryption_materials) except AwsCryptographicMaterialProvidersException as mpl_exception: # Wrap MPL error into the ESDK error type From 2b5fc7281f1ec263529f4b9c0c9635d62a7a4524 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 15:27:26 -0800 Subject: [PATCH 063/422] refactor, fix --- src/aws_encryption_sdk/internal/mpl/mpl_import_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aws_encryption_sdk/internal/mpl/mpl_import_handler.py b/src/aws_encryption_sdk/internal/mpl/mpl_import_handler.py index 55319bc43..5dd0a7b3e 100644 --- a/src/aws_encryption_sdk/internal/mpl/mpl_import_handler.py +++ b/src/aws_encryption_sdk/internal/mpl/mpl_import_handler.py @@ -17,5 +17,5 @@ def _import_mpl(): This only exists to help with unit test coverage. This is not directly tested. """ - # noqa pylint:disable=unused-import,import-outside-toplevel,import-error - import aws_cryptographic_materialproviders + # pylint:disable=unused-import,import-outside-toplevel,import-error + import aws_cryptographic_materialproviders # noqa F401 From a940dc57a7c9a11296d5200446dde48f0def7d77 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 15:33:45 -0800 Subject: [PATCH 064/422] refactor, fix --- .github/workflows/ci_static-analysis.yaml | 2 +- src/aws_encryption_sdk/internal/crypto/authentication.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_static-analysis.yaml b/.github/workflows/ci_static-analysis.yaml index 802bad2bc..03fa62165 100644 --- a/.github/workflows/ci_static-analysis.yaml +++ b/.github/workflows/ci_static-analysis.yaml @@ -38,5 +38,5 @@ jobs: pip install --upgrade -r dev_requirements/ci-requirements.txt - name: run test env: - TOXENV: ${{ matrix.category }}${{ matrix.optional_mpl_dependency }} + TOXENV: ${{ matrix.category }} run: tox -- -vv diff --git a/src/aws_encryption_sdk/internal/crypto/authentication.py b/src/aws_encryption_sdk/internal/crypto/authentication.py index a6446981e..d7ff35278 100644 --- a/src/aws_encryption_sdk/internal/crypto/authentication.py +++ b/src/aws_encryption_sdk/internal/crypto/authentication.py @@ -78,7 +78,7 @@ def from_key_bytes(cls, algorithm, key_bytes, encoding=serialization.Encoding.DE """ if encoding == serialization.Encoding.DER: key = serialization.load_der_private_key(data=key_bytes, password=None, backend=default_backend()) - elif serialization.Encoding.PEM: + elif encoding == serialization.Encoding.PEM: key = serialization.load_pem_private_key(data=key_bytes, password=None, backend=default_backend()) else: raise ValueError(f"Unsupported encoding for Signer: {encoding}") From 708ab5e26f1227cc5f72b58abd5d92bcf27cc89a Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 15:55:36 -0800 Subject: [PATCH 065/422] it works locally but fails on gha --- test/unit/test_crypto_authentication_signer.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/unit/test_crypto_authentication_signer.py b/test/unit/test_crypto_authentication_signer.py index 11271abfb..9584cf441 100644 --- a/test/unit/test_crypto_authentication_signer.py +++ b/test/unit/test_crypto_authentication_signer.py @@ -81,7 +81,13 @@ def test_signer_from_key_bytes(patch_default_backend, patch_serialization, patch mock_algorithm_info = MagicMock(return_value=sentinel.algorithm_info, spec=patch_ec.EllipticCurve) _algorithm = MagicMock(signing_algorithm_info=mock_algorithm_info) - signer = Signer.from_key_bytes(algorithm=_algorithm, key_bytes=sentinel.key_bytes) + # signer = Signer.from_key_bytes(algorithm=_algorithm, key_bytes=sentinel.key_bytes) + + signer = Signer.from_key_bytes( + algorithm=_algorithm, + key_bytes=sentinel.key_bytes, + encoding=patch_serialization.encoding.DER + ) patch_serialization.load_der_private_key.assert_called_once_with( data=sentinel.key_bytes, password=None, backend=patch_default_backend.return_value From ffd295c10253a7c0f1437309396eef28f076118f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 15:57:57 -0800 Subject: [PATCH 066/422] it works locally but fails on gha --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 20b4d9426..1c133adaa 100644 --- a/tox.ini +++ b/tox.ini @@ -110,7 +110,7 @@ passenv = setenv = ######################################################### deps = -rdev_requirements/test-requirements.txt -commands = {[testenv:base-command]commands} test/ -m local +commands = {[testenv:base-command]commands} test/ -m local --ignore test/unit/mpl/ # Collect requirements for use in upstream tests [testenv:freeze-upstream-requirements-base] @@ -142,7 +142,7 @@ commands = {[testenv:freeze-upstream-requirements-base]commands} test/upstream-r [testenv:test-upstream-requirements-base] sitepackages = False recreate = True -commands = {[testenv:base-command]commands} test/ -m local +commands = {[testenv:base-command]commands} test/ -m local --ignore test/unit/mpl/ # Test frozen upstream requirements for Python 3.7 [testenv:test-upstream-requirements-py37] From 1ba175c5391b7cf673b5c8e56be661951a072730 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 16:01:21 -0800 Subject: [PATCH 067/422] it works locally but fails on gha --- test/unit/test_crypto_authentication_signer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/test_crypto_authentication_signer.py b/test/unit/test_crypto_authentication_signer.py index 9584cf441..2e5f5a4fd 100644 --- a/test/unit/test_crypto_authentication_signer.py +++ b/test/unit/test_crypto_authentication_signer.py @@ -86,7 +86,7 @@ def test_signer_from_key_bytes(patch_default_backend, patch_serialization, patch signer = Signer.from_key_bytes( algorithm=_algorithm, key_bytes=sentinel.key_bytes, - encoding=patch_serialization.encoding.DER + encoding=patch_serialization.Encoding.DER ) patch_serialization.load_der_private_key.assert_called_once_with( From fa175ba6ed28758222b39e08569b310bfa08e363 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 16:09:12 -0800 Subject: [PATCH 068/422] it works locally but fails on gha --- .github/workflows/ci_tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 603f54371..d46c19c48 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -24,8 +24,8 @@ jobs: fail-fast: true matrix: os: - - ubuntu-latest - - windows-latest + # - ubuntu-latest + # - windows-latest - macos-latest python: - 3.7 From 2f90a970683bb9cf17469c88d51a28b8d5bd8baf Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 16:15:22 -0800 Subject: [PATCH 069/422] it works locally but fails on gha --- src/aws_encryption_sdk/streaming_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index c4c15559b..4e9d5d07c 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -247,7 +247,7 @@ def __new__(cls, **kwargs): ): raise TypeError("Can't instantiate abstract class {}".format(cls.__name__)) - instance = super(_EncryptionStream, cls).__new__(cls) + instance = super().__new__(cls) config = kwargs.pop("config", None) if not isinstance(config, instance._config_class): # pylint: disable=protected-access From df9215f6e72695a69fc042e2d7d326c341c006ff Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 16:19:20 -0800 Subject: [PATCH 070/422] it works locally but fails on gha --- src/aws_encryption_sdk/streaming_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 4e9d5d07c..c4c15559b 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -247,7 +247,7 @@ def __new__(cls, **kwargs): ): raise TypeError("Can't instantiate abstract class {}".format(cls.__name__)) - instance = super().__new__(cls) + instance = super(_EncryptionStream, cls).__new__(cls) config = kwargs.pop("config", None) if not isinstance(config, instance._config_class): # pylint: disable=protected-access From b57e4a397cfa4ca28f0877df1f0dbc608fbf0cfe Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 16:22:47 -0800 Subject: [PATCH 071/422] it works locally but fails on gha --- src/aws_encryption_sdk/streaming_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index c4c15559b..4e9d5d07c 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -247,7 +247,7 @@ def __new__(cls, **kwargs): ): raise TypeError("Can't instantiate abstract class {}".format(cls.__name__)) - instance = super(_EncryptionStream, cls).__new__(cls) + instance = super().__new__(cls) config = kwargs.pop("config", None) if not isinstance(config, instance._config_class): # pylint: disable=protected-access From 9d7ec6d3a2baca380c2e7bdb466b554c36e4a5f2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 16:25:37 -0800 Subject: [PATCH 072/422] it works locally but fails on gha --- src/aws_encryption_sdk/streaming_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 4e9d5d07c..c4c15559b 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -247,7 +247,7 @@ def __new__(cls, **kwargs): ): raise TypeError("Can't instantiate abstract class {}".format(cls.__name__)) - instance = super().__new__(cls) + instance = super(_EncryptionStream, cls).__new__(cls) config = kwargs.pop("config", None) if not isinstance(config, instance._config_class): # pylint: disable=protected-access From 2cbc8451756cedc0efde5a1c73141a8d8741748c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 16:31:22 -0800 Subject: [PATCH 073/422] it works locally but fails on gha --- src/aws_encryption_sdk/streaming_client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index c4c15559b..3bda700e7 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -247,7 +247,10 @@ def __new__(cls, **kwargs): ): raise TypeError("Can't instantiate abstract class {}".format(cls.__name__)) - instance = super(_EncryptionStream, cls).__new__(cls) + if issubclass(StreamEncryptor, _EncryptionStream): + instance = super(_EncryptionStream, cls).__new__(cls) + else: + raise ValueError(f"issubclass {issubclass(StreamEncryptor, _EncryptionStream)}") config = kwargs.pop("config", None) if not isinstance(config, instance._config_class): # pylint: disable=protected-access From def946d841461aa19a5dc73aeaf1471de7f7e351 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 16:37:51 -0800 Subject: [PATCH 074/422] it works locally but fails on gha --- src/aws_encryption_sdk/streaming_client.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 3bda700e7..4e9d5d07c 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -247,10 +247,7 @@ def __new__(cls, **kwargs): ): raise TypeError("Can't instantiate abstract class {}".format(cls.__name__)) - if issubclass(StreamEncryptor, _EncryptionStream): - instance = super(_EncryptionStream, cls).__new__(cls) - else: - raise ValueError(f"issubclass {issubclass(StreamEncryptor, _EncryptionStream)}") + instance = super().__new__(cls) config = kwargs.pop("config", None) if not isinstance(config, instance._config_class): # pylint: disable=protected-access From dff6ac0134e29ed4b98d046701ee75b11cd26450 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 17:12:17 -0800 Subject: [PATCH 075/422] it works locally but fails on gha --- src/aws_encryption_sdk/streaming_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 4e9d5d07c..4b849b818 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -71,14 +71,14 @@ from aws_encryption_sdk.materials_managers.default import DefaultCryptoMaterialsManager from aws_encryption_sdk.structures import MessageHeader -if mpl_import_handler.has_mpl(): +try: # noqa pylint: disable=import-error from aws_cryptographic_materialproviders.mpl.client import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig from aws_cryptographic_materialproviders.mpl.models import CreateDefaultCryptographicMaterialsManagerInput from aws_cryptographic_materialproviders.mpl.references import IKeyring _HAS_MPL = True -else: +except ImportError: _HAS_MPL = False _LOGGER = logging.getLogger(__name__) From 78f0b0faed85977346c2bb6652899a8316c0e10e Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 17:22:16 -0800 Subject: [PATCH 076/422] it works locally but fails on gha --- src/aws_encryption_sdk/streaming_client.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 4b849b818..1186faf2b 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -158,6 +158,9 @@ class _ClientConfig(object): # pylint: disable=too-many-instance-attributes ) # DEPRECATED: Value is no longer configurable here. Parameter left here to avoid breaking consumers. def _has_mpl_attrs_post_init(self): + if not hasattr(self, "keyring"): + self._no_mpl_attrs_post_init() + return if not _exactly_one_arg_is_not_none(self.materials_manager, self.key_provider, self.keyring): raise TypeError("Exactly one of keyring, materials_manager, or key_provider must be provided") if self.materials_manager is None: From 20a469e3b2e7fb8357096af9462856fa6b43604c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 20 Feb 2024 17:24:24 -0800 Subject: [PATCH 077/422] it works locally but fails on gha --- src/aws_encryption_sdk/streaming_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 1186faf2b..f9b550e6d 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -250,7 +250,7 @@ def __new__(cls, **kwargs): ): raise TypeError("Can't instantiate abstract class {}".format(cls.__name__)) - instance = super().__new__(cls) + instance = super(_EncryptionStream, cls).__new__(cls) config = kwargs.pop("config", None) if not isinstance(config, instance._config_class): # pylint: disable=protected-access From 66859a7b5316013200b42a6b00c2ebb1a51fcc18 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 10:10:57 -0800 Subject: [PATCH 078/422] fix tests --- examples/src/keyrings/hierarchical_keyring.py | 33 +++++++++-------- .../internal/mpl/mpl_import_handler.py | 21 ----------- src/aws_encryption_sdk/streaming_client.py | 4 +-- test/unit/mpl/test_cmm_handler.py | 2 +- test/unit/test_mpl_import_handler.py | 36 ------------------- test/unit/test_streaming_client_mpl_import.py | 32 ++++++++--------- 6 files changed, 33 insertions(+), 95 deletions(-) delete mode 100644 src/aws_encryption_sdk/internal/mpl/mpl_import_handler.py delete mode 100644 test/unit/test_mpl_import_handler.py diff --git a/examples/src/keyrings/hierarchical_keyring.py b/examples/src/keyrings/hierarchical_keyring.py index 56af60115..3e56d1e56 100644 --- a/examples/src/keyrings/hierarchical_keyring.py +++ b/examples/src/keyrings/hierarchical_keyring.py @@ -9,23 +9,22 @@ import aws_encryption_sdk from aws_encryption_sdk import CommitmentPolicy from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError -from aws_encryption_sdk.internal.mpl import mpl_import_handler - -if mpl_import_handler.has_mpl(): - # noqa pylint: disable=import-error - from aws_cryptographic_materialproviders.keystore.client import KeyStore - from aws_cryptographic_materialproviders.keystore.config import KeyStoreConfig - from aws_cryptographic_materialproviders.keystore.models import CreateKeyInput, KMSConfigurationKmsKeyArn - from aws_cryptographic_materialproviders.mpl.client import AwsCryptographicMaterialProviders - from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig - from aws_cryptographic_materialproviders.mpl.models import ( - CacheTypeDefault, - CreateAwsKmsHierarchicalKeyringInput, - DefaultCache, - GetBranchKeyIdInput, - GetBranchKeyIdOutput, - ) - from aws_cryptographic_materialproviders.mpl.references import IBranchKeyIdSupplier, IKeyring + +# ignore missing MPL for pylint, but the MPL is required for this example +# noqa pylint: disable=import-error +from aws_cryptographic_materialproviders.keystore.client import KeyStore +from aws_cryptographic_materialproviders.keystore.config import KeyStoreConfig +from aws_cryptographic_materialproviders.keystore.models import CreateKeyInput, KMSConfigurationKmsKeyArn +from aws_cryptographic_materialproviders.mpl.client import AwsCryptographicMaterialProviders +from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig +from aws_cryptographic_materialproviders.mpl.models import ( + CacheTypeDefault, + CreateAwsKmsHierarchicalKeyringInput, + DefaultCache, + GetBranchKeyIdInput, + GetBranchKeyIdOutput, +) +from aws_cryptographic_materialproviders.mpl.references import IBranchKeyIdSupplier, IKeyring module_root_dir = '/'.join(__file__.split("/")[:-1]) diff --git a/src/aws_encryption_sdk/internal/mpl/mpl_import_handler.py b/src/aws_encryption_sdk/internal/mpl/mpl_import_handler.py deleted file mode 100644 index 5dd0a7b3e..000000000 --- a/src/aws_encryption_sdk/internal/mpl/mpl_import_handler.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Detects whether the MPL is installed for use by internal ESDK code. -External customers should not need to interact with this. -""" - - -def has_mpl(): - """Returns True if the aws-cryptographic-material-providers library is installed, False otherwise.""" - try: - _import_mpl() - return True - except ImportError: - return False - - -def _import_mpl(): - """Private wrapper for import. - This only exists to help with unit test coverage. - This is not directly tested. - """ - # pylint:disable=unused-import,import-outside-toplevel,import-error - import aws_cryptographic_materialproviders # noqa F401 diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index f9b550e6d..8e004e84e 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -58,7 +58,6 @@ serialize_non_framed_close, serialize_non_framed_open, ) -from aws_encryption_sdk.internal.mpl import mpl_import_handler from aws_encryption_sdk.internal.mpl.cmm_handler import CMMHandler from aws_encryption_sdk.internal.utils.commitment import ( validate_commitment_policy_on_decrypt, @@ -72,13 +71,14 @@ from aws_encryption_sdk.structures import MessageHeader try: + # pylint should pass even if the MPL isn't installed # noqa pylint: disable=import-error from aws_cryptographic_materialproviders.mpl.client import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig from aws_cryptographic_materialproviders.mpl.models import CreateDefaultCryptographicMaterialsManagerInput from aws_cryptographic_materialproviders.mpl.references import IKeyring _HAS_MPL = True -except ImportError: +except ImportError as e: _HAS_MPL = False _LOGGER = logging.getLogger(__name__) diff --git a/test/unit/mpl/test_cmm_handler.py b/test/unit/mpl/test_cmm_handler.py index 343ac514b..d16374899 100644 --- a/test/unit/mpl/test_cmm_handler.py +++ b/test/unit/mpl/test_cmm_handler.py @@ -10,7 +10,7 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Test suite to verify the mpl_import_handler module handles importing the MPL correctly.""" +"""Test suite to verify the cmm_handler module delegates correctly.""" import pytest from aws_cryptographic_materialproviders.mpl.models import ( EncryptionMaterials as MPL_EncryptionMaterials, diff --git a/test/unit/test_mpl_import_handler.py b/test/unit/test_mpl_import_handler.py deleted file mode 100644 index b82c3092b..000000000 --- a/test/unit/test_mpl_import_handler.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. -"""Test suite to verify the mpl_import_handler module handles importing the MPL correctly.""" -import pytest -from mock import patch - -from aws_encryption_sdk.internal.mpl import mpl_import_handler - -pytestmark = [pytest.mark.unit, pytest.mark.local] - - -@patch("aws_encryption_sdk.internal.mpl.mpl_import_handler._import_mpl") -def test_GIVEN_import_mpl_succeeds_WHEN_call_has_mpl_THEN_return_True(import_mock): - # Mock a successful import of `aws_cryptographic_material_providers` - import_mock.return_value = None # No exception means successful import - - assert mpl_import_handler.has_mpl() is True - - -@patch("aws_encryption_sdk.internal.mpl.mpl_import_handler._import_mpl") -def test_GIVEN_import_mpl_fails_WHEN_call_has_mpl_THEN_return_False(import_mock): - # Mock not having a `aws_cryptographic_material_providers` module, - # even if it is installed in the Python environment - import_mock.side_effect = ImportError() - - assert not mpl_import_handler.has_mpl() diff --git a/test/unit/test_streaming_client_mpl_import.py b/test/unit/test_streaming_client_mpl_import.py index 594ef3478..ebafd199f 100644 --- a/test/unit/test_streaming_client_mpl_import.py +++ b/test/unit/test_streaming_client_mpl_import.py @@ -23,31 +23,27 @@ pytestmark = [pytest.mark.unit, pytest.mark.local] -@patch.object(aws_encryption_sdk.streaming_client.mpl_import_handler, "has_mpl") -def test_GIVEN_has_mpl_returns_True_WHEN_import_streaming_client_THEN_imports_mpl_modules(has_mpl_mock): - has_mpl_mock.return_value = True +# Check if MPL is installed, and skip tests based on whether it is +try: + import aws_cryptographic_materialproviders + HAS_MPL = True +except ImportError as e: + HAS_MPL = False - # Mock any imports used in the try/catch block - # If more imports are added there, then this needs to be expanded - # This unit test should pass even if the MPL is not installed - sys.modules['aws_cryptographic_materialproviders.mpl.client'] = Mock() - sys.modules['aws_cryptographic_materialproviders.mpl.config'] = Mock() - sys.modules['aws_cryptographic_materialproviders.mpl.models'] = Mock() - sys.modules['aws_cryptographic_materialproviders.mpl.references'] = Mock() - # Reload module given the mock - reload(aws_encryption_sdk.streaming_client) +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_test_has_mpl_is_True_THEN_streaming_client_has_mpl_is_TRUE(): + """If the MPL IS installed in the runtime environment, + assert the streaming client has _HAS_MPL set to True""" assert hasattr(aws_encryption_sdk.streaming_client, "_HAS_MPL") assert aws_encryption_sdk.streaming_client._HAS_MPL is True -@patch.object(aws_encryption_sdk.streaming_client.mpl_import_handler, "has_mpl") -def test_GIVEN_has_mpl_returns_False_WHEN_import_streaming_client_THEN_does_not_import_mpl_modules(has_mpl_mock): - has_mpl_mock.return_value = False - - # Reload module given the mock - reload(aws_encryption_sdk.streaming_client) +@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") +def test_GIVEN_test_has_mpl_is_False_THEN_streaming_client_has_mpl_is_TRUE(): + """If the MPL IS NOT installed in the runtime environment, + assert the streaming client has _HAS_MPL set to False""" assert hasattr(aws_encryption_sdk.streaming_client, "_HAS_MPL") assert aws_encryption_sdk.streaming_client._HAS_MPL is False From bf8f67c1fd4a8523c4a0a76eb5f5e0245e5b25b5 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 10:18:23 -0800 Subject: [PATCH 079/422] cleanup --- examples/src/keyrings/hierarchical_keyring.py | 11 +++++------ src/aws_encryption_sdk/streaming_client.py | 2 +- test/unit/test_streaming_client_mpl_import.py | 12 ++++-------- 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/examples/src/keyrings/hierarchical_keyring.py b/examples/src/keyrings/hierarchical_keyring.py index 3e56d1e56..50f620456 100644 --- a/examples/src/keyrings/hierarchical_keyring.py +++ b/examples/src/keyrings/hierarchical_keyring.py @@ -4,12 +4,6 @@ import sys import boto3 -from typing import Dict - -import aws_encryption_sdk -from aws_encryption_sdk import CommitmentPolicy -from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError - # ignore missing MPL for pylint, but the MPL is required for this example # noqa pylint: disable=import-error from aws_cryptographic_materialproviders.keystore.client import KeyStore @@ -25,6 +19,11 @@ GetBranchKeyIdOutput, ) from aws_cryptographic_materialproviders.mpl.references import IBranchKeyIdSupplier, IKeyring +from typing import Dict + +import aws_encryption_sdk +from aws_encryption_sdk import CommitmentPolicy +from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError module_root_dir = '/'.join(__file__.split("/")[:-1]) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 8e004e84e..ad998088c 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -78,7 +78,7 @@ from aws_cryptographic_materialproviders.mpl.models import CreateDefaultCryptographicMaterialsManagerInput from aws_cryptographic_materialproviders.mpl.references import IKeyring _HAS_MPL = True -except ImportError as e: +except ImportError: _HAS_MPL = False _LOGGER = logging.getLogger(__name__) diff --git a/test/unit/test_streaming_client_mpl_import.py b/test/unit/test_streaming_client_mpl_import.py index ebafd199f..3eda0ad63 100644 --- a/test/unit/test_streaming_client_mpl_import.py +++ b/test/unit/test_streaming_client_mpl_import.py @@ -12,11 +12,7 @@ # language governing permissions and limitations under the License. """Unit test suite to validate aws_encryption_sdk.streaming_client MPL import logic.""" -import sys -from importlib import reload - import pytest -from mock import Mock, patch import aws_encryption_sdk.streaming_client @@ -25,14 +21,14 @@ # Check if MPL is installed, and skip tests based on whether it is try: - import aws_cryptographic_materialproviders + import aws_cryptographic_materialproviders # noqa pylint: disable=unused-import HAS_MPL = True -except ImportError as e: +except ImportError: HAS_MPL = False @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_test_has_mpl_is_True_THEN_streaming_client_has_mpl_is_TRUE(): +def test_GIVEN_test_has_mpl_is_True_THEN_streaming_client_has_mpl_is_True(): """If the MPL IS installed in the runtime environment, assert the streaming client has _HAS_MPL set to True""" @@ -41,7 +37,7 @@ def test_GIVEN_test_has_mpl_is_True_THEN_streaming_client_has_mpl_is_TRUE(): @pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") -def test_GIVEN_test_has_mpl_is_False_THEN_streaming_client_has_mpl_is_TRUE(): +def test_GIVEN_test_has_mpl_is_False_THEN_streaming_client_has_mpl_is_False(): """If the MPL IS NOT installed in the runtime environment, assert the streaming client has _HAS_MPL set to False""" From b24be113a9a179d9cc22602b06a55222fc557ef3 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 10:23:07 -0800 Subject: [PATCH 080/422] re-enable test --- .github/workflows/ci_tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index d46c19c48..603f54371 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -24,8 +24,8 @@ jobs: fail-fast: true matrix: os: - # - ubuntu-latest - # - windows-latest + - ubuntu-latest + - windows-latest - macos-latest python: - 3.7 From acba1b0143ee6b40312b5a02ba6d95ea736b9d0a Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 10:25:52 -0800 Subject: [PATCH 081/422] re-enable test --- test/unit/test_streaming_client_mpl_import.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/unit/test_streaming_client_mpl_import.py b/test/unit/test_streaming_client_mpl_import.py index 3eda0ad63..a4ca87e2a 100644 --- a/test/unit/test_streaming_client_mpl_import.py +++ b/test/unit/test_streaming_client_mpl_import.py @@ -19,7 +19,9 @@ pytestmark = [pytest.mark.unit, pytest.mark.local] -# Check if MPL is installed, and skip tests based on whether it is +# Check if MPL is installed, and skip tests based on its installation status +# Ideally, this logic would be based on mocking imports and testing logic, +# but doing that introduces errors that cause other tests to fail. try: import aws_cryptographic_materialproviders # noqa pylint: disable=unused-import HAS_MPL = True From 42b7b745dec470b06bf1fecaae6d0578b17df2e1 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 10:29:57 -0800 Subject: [PATCH 082/422] longpaths --- .github/workflows/ci_tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 603f54371..a8bad7bfb 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -70,6 +70,7 @@ jobs: python-version: ${{ matrix.python }} architecture: ${{ matrix.architecture }} - run: | + git config --system core.longpaths true python -m pip install --upgrade pip pip install --upgrade -r dev_requirements/ci-requirements.txt - name: run test From f226e7e8bd5b3df06fda12f63be601939a0a8711 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 10:34:14 -0800 Subject: [PATCH 083/422] longpaths --- .github/workflows/ci_tests.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index a8bad7bfb..cab32dcc7 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -64,13 +64,16 @@ jobs: - python: 3.10 optional_mpl_dependency: -mpl steps: + - name: Support longpaths + run: | + git config --global core.longpaths true + - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} architecture: ${{ matrix.architecture }} - run: | - git config --system core.longpaths true python -m pip install --upgrade pip pip install --upgrade -r dev_requirements/ci-requirements.txt - name: run test From aa2f80a2aaff191082ed0844ded4ebe812eafb2c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 10:39:24 -0800 Subject: [PATCH 084/422] debug windows fail --- .github/workflows/ci_tests.yaml | 4 ++-- src/aws_encryption_sdk/streaming_client.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index cab32dcc7..132c65bd0 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -24,9 +24,9 @@ jobs: fail-fast: true matrix: os: - - ubuntu-latest + # - ubuntu-latest - windows-latest - - macos-latest + # - macos-latest python: - 3.7 - 3.8 diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index ad998088c..bc37d7de5 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -78,7 +78,8 @@ from aws_cryptographic_materialproviders.mpl.models import CreateDefaultCryptographicMaterialsManagerInput from aws_cryptographic_materialproviders.mpl.references import IKeyring _HAS_MPL = True -except ImportError: +except ImportError as e: + print(e) _HAS_MPL = False _LOGGER = logging.getLogger(__name__) From bc002b682dd7113734cf8494e17e48a1462e18c6 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 10:42:48 -0800 Subject: [PATCH 085/422] debug windows fail --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 1c133adaa..fa3e8530f 100644 --- a/tox.ini +++ b/tox.ini @@ -56,7 +56,7 @@ envlist = # coverage :: Runs code coverage, failing the build if coverage is below the configured threshold [testenv:base-command] -commands = pytest --basetemp={envtmpdir} -l {posargs} +commands = pytest --basetemp={envtmpdir} -l {posargs} -s -v [testenv] ; passenv = AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2,AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_SESSION_TOKEN,AWS_CONTAINER_CREDENTIALS_RELATIVE_URI,AWS_PROFILE,PIP_CONFIG_FILE From 8dd0303615d72712dc7c190f6e0724656c26bfb2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 10:46:34 -0800 Subject: [PATCH 086/422] debug windows fail --- src/aws_encryption_sdk/streaming_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index bc37d7de5..4a00b99f4 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -79,7 +79,7 @@ from aws_cryptographic_materialproviders.mpl.references import IKeyring _HAS_MPL = True except ImportError as e: - print(e) + print("IMPORT ERROR" + str(e)) _HAS_MPL = False _LOGGER = logging.getLogger(__name__) From 1e9db3b3b0d32da9c55e4e918c9320318026a611 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 10:50:22 -0800 Subject: [PATCH 087/422] debug windows fail --- .github/workflows/ci_tests.yaml | 9 ++++++--- src/aws_encryption_sdk/streaming_client.py | 1 - 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 132c65bd0..d64d3ce79 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -24,9 +24,12 @@ jobs: fail-fast: true matrix: os: - # - ubuntu-latest - - windows-latest - # - macos-latest + - ubuntu-latest + # Windows fails due to "No module named 'Wrappers'" + # This SHOULD be fixed once Dafny generates fully-qualified import statements + # Disable for now + # - windows-latest + - macos-latest python: - 3.7 - 3.8 diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 4a00b99f4..8e004e84e 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -79,7 +79,6 @@ from aws_cryptographic_materialproviders.mpl.references import IKeyring _HAS_MPL = True except ImportError as e: - print("IMPORT ERROR" + str(e)) _HAS_MPL = False _LOGGER = logging.getLogger(__name__) From 74d4e667d1828dd6437f1aeb42436c0b44c2d7c0 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 10:53:26 -0800 Subject: [PATCH 088/422] disable windows until pythonpath --- src/aws_encryption_sdk/streaming_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 8e004e84e..ad998088c 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -78,7 +78,7 @@ from aws_cryptographic_materialproviders.mpl.models import CreateDefaultCryptographicMaterialsManagerInput from aws_cryptographic_materialproviders.mpl.references import IKeyring _HAS_MPL = True -except ImportError as e: +except ImportError: _HAS_MPL = False _LOGGER = logging.getLogger(__name__) From 1bb23e862a73d96a376e207aeb82b3b2d3e19ec4 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 11:07:51 -0800 Subject: [PATCH 089/422] expand testing --- .github/workflows/ci_static-analysis.yaml | 4 +--- .github/workflows/ci_tests.yaml | 10 ++++++++++ codebuild/py311/examples_mpl.yml | 7 ++++--- codebuild/py312/awses_local_mpl.yml | 2 ++ codebuild/py312/examples_mpl.yml | 10 +++++++--- codebuild/py312/integ_mpl.yml | 2 ++ tox.ini | 11 ++++++++--- 7 files changed, 34 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci_static-analysis.yaml b/.github/workflows/ci_static-analysis.yaml index 03fa62165..0093ae9a9 100644 --- a/.github/workflows/ci_static-analysis.yaml +++ b/.github/workflows/ci_static-analysis.yaml @@ -13,8 +13,6 @@ jobs: strategy: fail-fast: false matrix: - python: - - 3.8 category: - bandit - doc8 @@ -32,7 +30,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: ${{ matrix.python }} + python-version: 3.8 - run: | python -m pip install --upgrade pip pip install --upgrade -r dev_requirements/ci-requirements.txt diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index d64d3ce79..58f9b1b11 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -44,10 +44,14 @@ jobs: category: - local - accept + - mpllocal + - mplaccept # These require credentials. # Enable them once we sort how to provide them. # - integ # - examples + # Append '-mpl' to some test environments. + # This suffix signals to tox to install the MPL in the test environment. optional_mpl_dependency: - "" - -mpl @@ -66,7 +70,13 @@ jobs: optional_mpl_dependency: -mpl - python: 3.10 optional_mpl_dependency: -mpl + # mpllocal and mplaccept require the MPL to be installed + - python: mpllocal + optional_mpl_dependency: + - python: mplaccept + optional_mpl_dependency: steps: + # Support long Dafny filenames (used in MPL and DBESDK repos) - name: Support longpaths run: | git config --global core.longpaths true diff --git a/codebuild/py311/examples_mpl.yml b/codebuild/py311/examples_mpl.yml index f8f2a6a01..19a5dec05 100644 --- a/codebuild/py311/examples_mpl.yml +++ b/codebuild/py311/examples_mpl.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - # No TOXENV; examples using the MPL switch envs + # No TOXENV. This runs multiple environments. REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f @@ -20,14 +20,15 @@ phases: build: commands: - pip install "tox < 4.0" + # Run non-MPL-specific tests with the MPL installed - tox -e py311-examples-mpl - # Assume special role + # Assume special role to access keystore - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-Py311ExamplesMpl") - export TMP_ROLE - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') - aws sts get-caller-identity - # Run special role-specific examples + # Run MPL-specific tests with special role - tox -e py311-mplexamples-mpl diff --git a/codebuild/py312/awses_local_mpl.yml b/codebuild/py312/awses_local_mpl.yml index db25f4f57..1d0f80319 100644 --- a/codebuild/py312/awses_local_mpl.yml +++ b/codebuild/py312/awses_local_mpl.yml @@ -1,3 +1,5 @@ +# Runs the same tests as awses_local in an environment with the MPL installed. +# This asserts existing tests continue to pass with the MPL installed. version: 0.2 env: diff --git a/codebuild/py312/examples_mpl.yml b/codebuild/py312/examples_mpl.yml index ba0660024..366222441 100644 --- a/codebuild/py312/examples_mpl.yml +++ b/codebuild/py312/examples_mpl.yml @@ -1,8 +1,11 @@ +# Runs the same tests as examples in an environment with the MPL installed +# to assert existing tests continue to pass with the MPL installed. +# Then, run MPL-specific tests. version: 0.2 env: variables: - # No TOXENV; examples using the MPL switch envs + # No TOXENV. This runs multiple environments. REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f @@ -25,13 +28,14 @@ phases: - pip install --upgrade pip - pip install setuptools - pip install "tox < 4.0" + # Run non-MPL-specific tests with the MPL installed - tox -e py312-examples-mpl - # Assume special role + # Assume special role to access keystore - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-Py311ExamplesMpl") - export TMP_ROLE - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') - aws sts get-caller-identity - # Run special role-specific examples + # Run MPL-specific tests with special role - tox -e py312-mplexamples-mpl diff --git a/codebuild/py312/integ_mpl.yml b/codebuild/py312/integ_mpl.yml index 553f41e8a..e292acc57 100644 --- a/codebuild/py312/integ_mpl.yml +++ b/codebuild/py312/integ_mpl.yml @@ -1,3 +1,5 @@ +# Runs the same tests as integ in an environment with the MPL installed. +# This asserts existing tests continue to pass with the MPL installed. version: 0.2 env: diff --git a/tox.ini b/tox.ini index fa3e8530f..e5a585ca5 100644 --- a/tox.ini +++ b/tox.ini @@ -15,7 +15,7 @@ envlist = # These must be separate from the above target. # These require the `-mpl` suffix so tox installs the MPL. # The `mpl` prefix runs only MPL-specific tests - py{311,312}-mpl{local,examples}-mpl + py{311,312}-mpl{local,integ,accept,examples}-mpl nocmk, bandit, doc8, readme, docs, {flake8,pylint}{,-tests,-examples}, @@ -56,7 +56,7 @@ envlist = # coverage :: Runs code coverage, failing the build if coverage is below the configured threshold [testenv:base-command] -commands = pytest --basetemp={envtmpdir} -l {posargs} -s -v +commands = pytest --basetemp={envtmpdir} -l {posargs} [testenv] ; passenv = AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2,AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_SESSION_TOKEN,AWS_CONTAINER_CREDENTIALS_RELATIVE_URI,AWS_PROFILE,PIP_CONFIG_FILE @@ -87,11 +87,16 @@ commands = # MPL unit tests require the MPL to be installed mpllocal: {[testenv:base-command]commands} test/unit/mpl/ -m local integ: {[testenv:base-command]commands} test/ -m integ --ignore test/unit/mpl/ + # MPL integ tests require the MPL to be installed + mplinteg: {[testenv:base-command]commands} test/unit/mpl -m integ accept: {[testenv:base-command]commands} test/ -m accept --ignore test/unit/mpl/ + # MPL accept tests require the MPL to be installed + mplaccept: {[testenv:base-command]commands} test/unit/mpl -m accept examples: {[testenv:base-command]commands} examples/test/ -m examples --ignore examples/test/keyrings/ # MPL keyring examples require a special IAM role; run these separately under a separate set of permissions mplexamples: {[testenv:base-command]commands} examples/test/keyrings -m examples - all: {[testenv:base-command]commands} test/ examples/test/ --ignore test/unit/mpl/ + all: {[testenv:base-command]commands} test/ examples/test/ --ignore test/unit/mpl/ --ignore examples/test/keyrings/ + mplall: {[testenv:base-command]commands} test/unit/mpl/ examples/test/keyrings/ manual: {[testenv:base-command]commands} # Run code coverage on the unit tests From 1ee69cefbb3f0d3486c2d0e873a1ac6c0a4bf8ff Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 11:11:05 -0800 Subject: [PATCH 090/422] expand testing --- .github/workflows/ci_tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 58f9b1b11..daa108060 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -72,9 +72,9 @@ jobs: optional_mpl_dependency: -mpl # mpllocal and mplaccept require the MPL to be installed - python: mpllocal - optional_mpl_dependency: + optional_mpl_dependency: "" - python: mplaccept - optional_mpl_dependency: + optional_mpl_dependency: "" steps: # Support long Dafny filenames (used in MPL and DBESDK repos) - name: Support longpaths From b33f2f706fb1dbb9f98d64b06a0c7ad6e97d08b2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 11:12:01 -0800 Subject: [PATCH 091/422] expand testing --- .github/workflows/ci_tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index daa108060..495192a81 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -71,9 +71,9 @@ jobs: - python: 3.10 optional_mpl_dependency: -mpl # mpllocal and mplaccept require the MPL to be installed - - python: mpllocal + - category: mpllocal optional_mpl_dependency: "" - - python: mplaccept + - category: mplaccept optional_mpl_dependency: "" steps: # Support long Dafny filenames (used in MPL and DBESDK repos) From c582888a3f064ab1ab258f9c8bcb8fac80df6a36 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 11:16:22 -0800 Subject: [PATCH 092/422] expand testing --- tox.ini | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index e5a585ca5..cee1e35a4 100644 --- a/tox.ini +++ b/tox.ini @@ -15,7 +15,7 @@ envlist = # These must be separate from the above target. # These require the `-mpl` suffix so tox installs the MPL. # The `mpl` prefix runs only MPL-specific tests - py{311,312}-mpl{local,integ,accept,examples}-mpl + py{311,312}-mpl{local,examples}-mpl nocmk, bandit, doc8, readme, docs, {flake8,pylint}{,-tests,-examples}, @@ -87,11 +87,9 @@ commands = # MPL unit tests require the MPL to be installed mpllocal: {[testenv:base-command]commands} test/unit/mpl/ -m local integ: {[testenv:base-command]commands} test/ -m integ --ignore test/unit/mpl/ - # MPL integ tests require the MPL to be installed - mplinteg: {[testenv:base-command]commands} test/unit/mpl -m integ + # No MPL-specific integ tests accept: {[testenv:base-command]commands} test/ -m accept --ignore test/unit/mpl/ - # MPL accept tests require the MPL to be installed - mplaccept: {[testenv:base-command]commands} test/unit/mpl -m accept + # No MPL-specific accept tests examples: {[testenv:base-command]commands} examples/test/ -m examples --ignore examples/test/keyrings/ # MPL keyring examples require a special IAM role; run these separately under a separate set of permissions mplexamples: {[testenv:base-command]commands} examples/test/keyrings -m examples From 5ae44f5a077a240c1a9cd49bd49b08d165a4ad6c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 11:19:22 -0800 Subject: [PATCH 093/422] expand testing --- .github/workflows/ci_tests.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 495192a81..803d4741e 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -45,7 +45,6 @@ jobs: - local - accept - mpllocal - - mplaccept # These require credentials. # Enable them once we sort how to provide them. # - integ @@ -70,11 +69,9 @@ jobs: optional_mpl_dependency: -mpl - python: 3.10 optional_mpl_dependency: -mpl - # mpllocal and mplaccept require the MPL to be installed + # mpllocal requires the MPL to be installed - category: mpllocal optional_mpl_dependency: "" - - category: mplaccept - optional_mpl_dependency: "" steps: # Support long Dafny filenames (used in MPL and DBESDK repos) - name: Support longpaths From cb7e3d1c8c8a58548ef03cdbc22f663d676c179f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 11:45:49 -0800 Subject: [PATCH 094/422] cleanup --- codebuild/coverage/coverage_mpl.yml | 2 +- examples/src/keyrings/hierarchical_keyring.py | 60 ++++++------------- examples/src/keyrings/module_.py | 2 +- examples/src/module_.py | 2 +- .../unit/test_crypto_authentication_signer.py | 6 +- test_vector_handlers/tox.ini | 4 +- tox.ini | 3 +- 7 files changed, 27 insertions(+), 52 deletions(-) diff --git a/codebuild/coverage/coverage_mpl.yml b/codebuild/coverage/coverage_mpl.yml index 5dcc65382..922705569 100644 --- a/codebuild/coverage/coverage_mpl.yml +++ b/codebuild/coverage/coverage_mpl.yml @@ -7,7 +7,7 @@ env: phases: install: runtime-versions: - python: latest + python: 3.11 build: commands: - pip install "tox < 4.0" diff --git a/examples/src/keyrings/hierarchical_keyring.py b/examples/src/keyrings/hierarchical_keyring.py index 50f620456..8f8707013 100644 --- a/examples/src/keyrings/hierarchical_keyring.py +++ b/examples/src/keyrings/hierarchical_keyring.py @@ -15,8 +15,6 @@ CacheTypeDefault, CreateAwsKmsHierarchicalKeyringInput, DefaultCache, - GetBranchKeyIdInput, - GetBranchKeyIdOutput, ) from aws_cryptographic_materialproviders.mpl.references import IBranchKeyIdSupplier, IKeyring from typing import Dict @@ -25,6 +23,8 @@ from aws_encryption_sdk import CommitmentPolicy from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError +from .example_branch_key_id_supplier import ExampleBranchKeyIdSupplier + module_root_dir = '/'.join(__file__.split("/")[:-1]) sys.path.append(module_root_dir) @@ -73,39 +73,6 @@ def encrypt_and_decrypt_with_keyring( branch_key_id_A: str = keystore.create_key(input=CreateKeyInput()).branch_key_identifier branch_key_id_B: str = keystore.create_key(input=CreateKeyInput()).branch_key_identifier - class ExampleBranchKeyIdSupplier(IBranchKeyIdSupplier): - """Example implementation of a branch key ID supplier.""" - - branch_key_id_for_tenant_A: str - branch_key_id_for_tenant_B: str - - def __init__(self, tenant_1_id, tenant_2_id): - self.branch_key_id_for_tenant_A = tenant_1_id - self.branch_key_id_for_tenant_B = tenant_2_id - - def get_branch_key_id( - self, - # Change this to `native_input` - input: GetBranchKeyIdInput # noqa pylint: disable=redefined-builtin - ) -> GetBranchKeyIdOutput: - """Returns branch key ID from the tenant ID in input's encryption context.""" - encryption_context: Dict[str, str] = input.encryption_context - - if b"tenant" not in encryption_context: - raise ValueError("EncryptionContext invalid, does not contain expected tenant key value pair.") - - tenant_key_id: str = encryption_context.get(b"tenant") - branch_key_id: str - - if tenant_key_id == b"TenantA": - branch_key_id = self.branch_key_id_for_tenant_A - elif tenant_key_id == b"TenantB": - branch_key_id = self.branch_key_id_for_tenant_B - else: - raise ValueError(f"Item does not contain valid tenant ID: {tenant_key_id=}") - - return GetBranchKeyIdOutput(branch_key_id=branch_key_id) - # 5. Create a branch key supplier that maps the branch key id to a more readable format branch_key_id_supplier: IBranchKeyIdSupplier = ExampleBranchKeyIdSupplier( tenant_1_id=branch_key_id_A, @@ -132,8 +99,10 @@ def get_branch_key_id( input=keyring_input ) - # The Branch Key Id supplier uses the encryption context to determine which branch key id will - # be used to encrypt data. + # 7. Create encryption context for both tenants. + # The Branch Key Id supplier uses the encryption context to determine which branch key id will + # be used to encrypt data. + # Create encryption context for TenantA encryption_context_A: Dict[str, str] = { "tenant": "TenantA", @@ -154,7 +123,7 @@ def get_branch_key_id( "the data you are handling": "is what you think it is", } - # Encrypt the data for encryptionContextA & encryptionContextB + # 8. Encrypt the data for encryptionContextA & encryptionContextB ciphertext_A, _ = client.encrypt( source=EXAMPLE_DATA, keyring=hierarchical_keyring, @@ -166,8 +135,8 @@ def get_branch_key_id( encryption_context=encryption_context_B ) - # To attest that TenantKeyB cannot decrypt a message written by TenantKeyA - # let's construct more restrictive hierarchical keyrings. + # 9. To attest that TenantKeyB cannot decrypt a message written by TenantKeyA, + # let's construct more restrictive hierarchical keyrings. keyring_input_A: CreateAwsKmsHierarchicalKeyringInput = CreateAwsKmsHierarchicalKeyringInput( key_store=keystore, branch_key_id=branch_key_id_A, @@ -198,6 +167,11 @@ def get_branch_key_id( input=keyring_input_B ) + # 10. Demonstrate that data encrypted by one tenant's key + # cannot be decrypted with by a keyring specific to another tenant. + + # Keyring with tenant B's branch key cannot decrypt data encrypted with tenant A's branch key + # This will fail and raise a AWSEncryptionSDKClientError, which we swallow ONLY for demonstration purposes. try: client.decrypt( source=ciphertext_A, @@ -206,7 +180,8 @@ def get_branch_key_id( except AWSEncryptionSDKClientError: pass - # This should fail + # Keyring with tenant A's branch key cannot decrypt data encrypted with tenant B's branch key. + # This will fail and raise a AWSEncryptionSDKClientError, which we swallow ONLY for demonstration purposes. try: client.decrypt( source=ciphertext_B, @@ -215,7 +190,8 @@ def get_branch_key_id( except AWSEncryptionSDKClientError: pass - # These should succeed + # 10. Demonstrate that data encrypted by one tenant's branch key can be decrypted by that tenant, + # and that the decrypted data matches the input data. plaintext_bytes_A, _ = client.decrypt( source=ciphertext_A, keyring=hierarchical_keyring_A diff --git a/examples/src/keyrings/module_.py b/examples/src/keyrings/module_.py index d9a8c058f..3e8d3062a 100644 --- a/examples/src/keyrings/module_.py +++ b/examples/src/keyrings/module_.py @@ -1 +1 @@ -"""Should remove this.""" +"""Should remove this once PYTHONPATH issues are resolved by adding doo files.""" diff --git a/examples/src/module_.py b/examples/src/module_.py index d9a8c058f..3e8d3062a 100644 --- a/examples/src/module_.py +++ b/examples/src/module_.py @@ -1 +1 @@ -"""Should remove this.""" +"""Should remove this once PYTHONPATH issues are resolved by adding doo files.""" diff --git a/test/unit/test_crypto_authentication_signer.py b/test/unit/test_crypto_authentication_signer.py index 2e5f5a4fd..bd7227fd3 100644 --- a/test/unit/test_crypto_authentication_signer.py +++ b/test/unit/test_crypto_authentication_signer.py @@ -81,8 +81,10 @@ def test_signer_from_key_bytes(patch_default_backend, patch_serialization, patch mock_algorithm_info = MagicMock(return_value=sentinel.algorithm_info, spec=patch_ec.EllipticCurve) _algorithm = MagicMock(signing_algorithm_info=mock_algorithm_info) - # signer = Signer.from_key_bytes(algorithm=_algorithm, key_bytes=sentinel.key_bytes) - + # Explicitly pass in patched serialization module. + # Patching the module introduces namespace issues + # which causes the method's `isinstance` checks to fail + # by changing the namespace from `serialization.Encoding.DER` to `Encoding.DER`. signer = Signer.from_key_bytes( algorithm=_algorithm, key_bytes=sentinel.key_bytes, diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index 7004080e3..df2707f6a 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -2,7 +2,7 @@ envlist = # The test vectors depend on new features now, # so until release we can only effectively test the local version of the ESDK. - py{37,38,39,310}-awses_local{,-mpl}, + py{37,38,39,310}-awses_local # 1.2.0 and 1.2.max are being difficult because of attrs bandit, doc8, readme, {flake8,pylint}{,-tests}, @@ -48,8 +48,6 @@ passenv = sitepackages = False deps = -rtest/requirements.txt - # install the MPL if in environment - mpl: -r../requirements_mpl.txt .. commands = {[testenv:base-command]commands} diff --git a/tox.ini b/tox.ini index cee1e35a4..ae30f3122 100644 --- a/tox.ini +++ b/tox.ini @@ -59,7 +59,6 @@ envlist = commands = pytest --basetemp={envtmpdir} -l {posargs} [testenv] -; passenv = AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1,AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2,AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_SESSION_TOKEN,AWS_CONTAINER_CREDENTIALS_RELATIVE_URI,AWS_PROFILE,PIP_CONFIG_FILE passenv = # Identifies AWS KMS key id to use in integration tests AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID \ @@ -80,7 +79,7 @@ passenv = sitepackages = False deps = -rdev_requirements/test-requirements.txt - # install the MPL if in environment + # install the MPL requirements if the `-mpl` suffix is present mpl: -rrequirements_mpl.txt commands = local: {[testenv:base-command]commands} test/ -m local --ignore test/unit/mpl/ From b026b532b59b177cfb63be0019c508d829b41aea Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 11:53:41 -0800 Subject: [PATCH 095/422] cleanup --- .../internal/mpl/cmm_handler.py | 1 - .../internal/mpl/materials_handlers.py | 6 --- .../internal/utils/__init__.py | 18 +++++++++ src/aws_encryption_sdk/streaming_client.py | 37 +++++++------------ 4 files changed, 31 insertions(+), 31 deletions(-) diff --git a/src/aws_encryption_sdk/internal/mpl/cmm_handler.py b/src/aws_encryption_sdk/internal/mpl/cmm_handler.py index 9789651e5..1575e0187 100644 --- a/src/aws_encryption_sdk/internal/mpl/cmm_handler.py +++ b/src/aws_encryption_sdk/internal/mpl/cmm_handler.py @@ -29,7 +29,6 @@ from aws_encryption_sdk.structures import EncryptedDataKey as Native_EncryptedDataKey -# TODO-MPL Should this implement interface...? seems like yes since it implements all of interface methods class CMMHandler(CryptoMaterialsManager): """ In instances where encryption materials may be provided by either diff --git a/src/aws_encryption_sdk/internal/mpl/materials_handlers.py b/src/aws_encryption_sdk/internal/mpl/materials_handlers.py index bf32c2718..79312f863 100644 --- a/src/aws_encryption_sdk/internal/mpl/materials_handlers.py +++ b/src/aws_encryption_sdk/internal/mpl/materials_handlers.py @@ -93,9 +93,6 @@ def data_encryption_key(self) -> DataKey: if hasattr(self, "native_materials"): return self.native_materials.data_encryption_key else: - # TODO-MPL This impl is probably wrong, but works for for now - # If this works for all features, great! Remove this comment before launch. - # Otherwise, fix the implementation. mpl_dek = self.mpl_materials.plaintext_data_key return DataKey( # key_provider is unused, but the return type is DataKey @@ -149,9 +146,6 @@ def data_key(self) -> DataKey: if hasattr(self, "native_materials"): return self.native_materials.data_key else: - # TODO-MPL This impl is probably wrong, but works for for now - # If this works for all features, great! Remove this comment before launch. - # Otherwise, fix the implementation. return DataKey( key_provider=MasterKeyInfo( provider_id="", diff --git a/src/aws_encryption_sdk/internal/utils/__init__.py b/src/aws_encryption_sdk/internal/utils/__init__.py index dac38ac73..b65f6df0f 100644 --- a/src/aws_encryption_sdk/internal/utils/__init__.py +++ b/src/aws_encryption_sdk/internal/utils/__init__.py @@ -163,3 +163,21 @@ def source_data_key_length_check(source_data_key, algorithm): actual=len(source_data_key.data_key), required=algorithm.kdf_input_len ) ) + +def exactly_one_arg_is_not_none(*args): + """ + Helper function for internal ESDK logic. + Returns `True` if exactly one item in the list is not `None`. + Returns `False` otherwise. + """ + # Have not found any `not None` + found_one = False + for arg in args: + if arg is not None: + if found_one is False: + # Have not already found a `not None`, found a `not None` => only one `not None` (so far) + found_one = True + else: + # Already found a `not None`, found another `not None` => not exactly one `not None` + return False + return found_one \ No newline at end of file diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index ad998088c..72f18c117 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -59,6 +59,7 @@ serialize_non_framed_open, ) from aws_encryption_sdk.internal.mpl.cmm_handler import CMMHandler +from aws_encryption_sdk.internal.utils import exactly_one_arg_is_not_none from aws_encryption_sdk.internal.utils.commitment import ( validate_commitment_policy_on_decrypt, validate_commitment_policy_on_encrypt, @@ -84,25 +85,6 @@ _LOGGER = logging.getLogger(__name__) -def _exactly_one_arg_is_not_none(*args): - """ - Private helper function. - Returns `True` if exactly one item in the list is not `None`. - Returns `False` otherwise. - """ - # Have not found any `not None` - found_one = False - for arg in args: - if arg is not None: - if found_one is False: - # Have not already found a `not None`, found a `not None` => only one `not None` (so far) - found_one = True - else: - # Already found a `not None`, found another `not None` => not exactly one `not None` - return False - return found_one - - @attr.s(hash=True) # pylint: disable=too-many-instance-attributes @six.add_metaclass(abc.ABCMeta) class _ClientConfig(object): # pylint: disable=too-many-instance-attributes @@ -147,6 +129,7 @@ class _ClientConfig(object): # pylint: disable=too-many-instance-attributes hash=True, default=None, validator=attr.validators.optional(attr.validators.instance_of(MasterKeyProvider)) ) if _HAS_MPL: + # Keyrings are only available if the MPL is installed in the runtime keyring = attr.ib( hash=True, default=None, validator=attr.validators.optional(attr.validators.instance_of(IKeyring)) ) @@ -158,10 +141,13 @@ class _ClientConfig(object): # pylint: disable=too-many-instance-attributes ) # DEPRECATED: Value is no longer configurable here. Parameter left here to avoid breaking consumers. def _has_mpl_attrs_post_init(self): + """If the MPL is present in the runtime, perform MPL-specific post-init logic + to validate the new object has a valid state. + """ if not hasattr(self, "keyring"): self._no_mpl_attrs_post_init() return - if not _exactly_one_arg_is_not_none(self.materials_manager, self.key_provider, self.keyring): + if not exactly_one_arg_is_not_none(self.materials_manager, self.key_provider, self.keyring): raise TypeError("Exactly one of keyring, materials_manager, or key_provider must be provided") if self.materials_manager is None: if self.key_provider is not None: @@ -187,6 +173,9 @@ def _has_mpl_attrs_post_init(self): self.materials_manager = cmm_handler def _no_mpl_attrs_post_init(self): + """If the MPL is NOT present in the runtime, perform post-init logic + to validate the new object has a valid state. + """ both_cmm_and_mkp_defined = self.materials_manager is not None and self.key_provider is not None neither_cmm_nor_mkp_defined = self.materials_manager is None and self.key_provider is None @@ -560,8 +549,8 @@ def _prep_message(self): if self._encryption_materials.signing_key is None: self.signer = None else: - # MPL verification key is NOT key bytes, it is bytes of the compressed point - # TODO-MPL: clean this up, least-privilege violation. + # MPL verification key is PEM bytes, not DER bytes. + # If the underlying CMM is from the MPL, load PEM bytes. if (isinstance(self.config.materials_manager, CMMHandler) and hasattr(self.config.materials_manager, "mpl_cmm")): self.signer = Signer.from_key_bytes( @@ -928,8 +917,8 @@ def _read_header(self): if decryption_materials.verification_key is None: self.verifier = None else: - # MPL verification key is NOT key bytes, it is bytes of the compressed point - # TODO-MPL: clean this up, least-privilege violation. + # MPL verification key is NOT key bytes; it is bytes of the compressed point. + # If the underlying CMM is from the MPL, load PEM bytes. if (isinstance(self.config.materials_manager, CMMHandler) and hasattr(self.config.materials_manager, "mpl_cmm")): self.verifier = Verifier.from_encoded_point( From 50afa3ade9d8c5f2712392b925fe3b621e97ba4d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 11:54:25 -0800 Subject: [PATCH 096/422] cleanup --- src/aws_encryption_sdk/streaming_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 72f18c117..032ed7d15 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -918,7 +918,7 @@ def _read_header(self): self.verifier = None else: # MPL verification key is NOT key bytes; it is bytes of the compressed point. - # If the underlying CMM is from the MPL, load PEM bytes. + # If the underlying CMM is from the MPL, load bytes from encoded point. if (isinstance(self.config.materials_manager, CMMHandler) and hasattr(self.config.materials_manager, "mpl_cmm")): self.verifier = Verifier.from_encoded_point( From 1c612a0b9e9cc85bae1f71a7d1027d2901c5de82 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 12:01:11 -0800 Subject: [PATCH 097/422] cleanup --- examples/src/keyrings/hierarchical_keyring.py | 2 +- src/aws_encryption_sdk/internal/utils/__init__.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/examples/src/keyrings/hierarchical_keyring.py b/examples/src/keyrings/hierarchical_keyring.py index 8f8707013..c71719346 100644 --- a/examples/src/keyrings/hierarchical_keyring.py +++ b/examples/src/keyrings/hierarchical_keyring.py @@ -169,7 +169,7 @@ def encrypt_and_decrypt_with_keyring( # 10. Demonstrate that data encrypted by one tenant's key # cannot be decrypted with by a keyring specific to another tenant. - + # Keyring with tenant B's branch key cannot decrypt data encrypted with tenant A's branch key # This will fail and raise a AWSEncryptionSDKClientError, which we swallow ONLY for demonstration purposes. try: diff --git a/src/aws_encryption_sdk/internal/utils/__init__.py b/src/aws_encryption_sdk/internal/utils/__init__.py index b65f6df0f..b08121281 100644 --- a/src/aws_encryption_sdk/internal/utils/__init__.py +++ b/src/aws_encryption_sdk/internal/utils/__init__.py @@ -164,11 +164,15 @@ def source_data_key_length_check(source_data_key, algorithm): ) ) + def exactly_one_arg_is_not_none(*args): """ Helper function for internal ESDK logic. - Returns `True` if exactly one item in the list is not `None`. + Returns `True` if exactly one item in the provided arguments is not `None`. Returns `False` otherwise. + + :param args: Input arguments to check + :returns: `True` if exactly one item in the provided arguments is not `None`; `False` otherwise """ # Have not found any `not None` found_one = False @@ -180,4 +184,4 @@ def exactly_one_arg_is_not_none(*args): else: # Already found a `not None`, found another `not None` => not exactly one `not None` return False - return found_one \ No newline at end of file + return found_one From bcdb4ba37f189d8fe6407c66f3ee3ccb1dbc7ebe Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 12:19:22 -0800 Subject: [PATCH 098/422] add missing file --- .../example_branch_key_id_supplier.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 examples/src/keyrings/example_branch_key_id_supplier.py diff --git a/examples/src/keyrings/example_branch_key_id_supplier.py b/examples/src/keyrings/example_branch_key_id_supplier.py new file mode 100644 index 000000000..a3ef0df6f --- /dev/null +++ b/examples/src/keyrings/example_branch_key_id_supplier.py @@ -0,0 +1,37 @@ +from aws_cryptographic_materialproviders.mpl.models import GetBranchKeyIdInput, GetBranchKeyIdOutput +from aws_cryptographic_materialproviders.mpl.references import IBranchKeyIdSupplier +from typing import Dict + + +class ExampleBranchKeyIdSupplier(IBranchKeyIdSupplier): + """Example implementation of a branch key ID supplier.""" + + branch_key_id_for_tenant_A: str + branch_key_id_for_tenant_B: str + + def __init__(self, tenant_1_id, tenant_2_id): + self.branch_key_id_for_tenant_A = tenant_1_id + self.branch_key_id_for_tenant_B = tenant_2_id + + def get_branch_key_id( + self, + # Change this to `native_input` + input: GetBranchKeyIdInput # noqa pylint: disable=redefined-builtin + ) -> GetBranchKeyIdOutput: + """Returns branch key ID from the tenant ID in input's encryption context.""" + encryption_context: Dict[str, str] = input.encryption_context + + if b"tenant" not in encryption_context: + raise ValueError("EncryptionContext invalid, does not contain expected tenant key value pair.") + + tenant_key_id: str = encryption_context.get(b"tenant") + branch_key_id: str + + if tenant_key_id == b"TenantA": + branch_key_id = self.branch_key_id_for_tenant_A + elif tenant_key_id == b"TenantB": + branch_key_id = self.branch_key_id_for_tenant_B + else: + raise ValueError(f"Item does not contain valid tenant ID: {tenant_key_id=}") + + return GetBranchKeyIdOutput(branch_key_id=branch_key_id) \ No newline at end of file From 41fe2f9facf04bbbdf0ccf0168c20aa9e27e059c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 12:26:47 -0800 Subject: [PATCH 099/422] add missing file --- test_vector_handlers/tox.ini | 3 +++ tox.ini | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index df2707f6a..e5e467d8a 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -3,6 +3,7 @@ envlist = # The test vectors depend on new features now, # so until release we can only effectively test the local version of the ESDK. py{37,38,39,310}-awses_local + py{311,312}-awses_local{,-mpl} # 1.2.0 and 1.2.max are being difficult because of attrs bandit, doc8, readme, {flake8,pylint}{,-tests}, @@ -48,6 +49,8 @@ passenv = sitepackages = False deps = -rtest/requirements.txt + # Install the MPL requirements if the `-mpl` suffix is present + mpl: -rrequirements_mpl.txt .. commands = {[testenv:base-command]commands} diff --git a/tox.ini b/tox.ini index ae30f3122..72e8ec9fa 100644 --- a/tox.ini +++ b/tox.ini @@ -79,7 +79,7 @@ passenv = sitepackages = False deps = -rdev_requirements/test-requirements.txt - # install the MPL requirements if the `-mpl` suffix is present + # Install the MPL requirements if the `-mpl` suffix is present mpl: -rrequirements_mpl.txt commands = local: {[testenv:base-command]commands} test/ -m local --ignore test/unit/mpl/ From 1ba857e74a1e7e117a8208ac21e235d1c5d2e18a Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 12:31:48 -0800 Subject: [PATCH 100/422] add missing file --- test_vector_handlers/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index e5e467d8a..580b641e0 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -50,7 +50,7 @@ sitepackages = False deps = -rtest/requirements.txt # Install the MPL requirements if the `-mpl` suffix is present - mpl: -rrequirements_mpl.txt + mpl: -r../requirements_mpl.txt .. commands = {[testenv:base-command]commands} From 74bfe127d6ea86de4637b1dfea3621f703bff0cd Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 21 Feb 2024 12:38:32 -0800 Subject: [PATCH 101/422] cleanup --- examples/src/keyrings/example_branch_key_id_supplier.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/src/keyrings/example_branch_key_id_supplier.py b/examples/src/keyrings/example_branch_key_id_supplier.py index a3ef0df6f..a06280fa1 100644 --- a/examples/src/keyrings/example_branch_key_id_supplier.py +++ b/examples/src/keyrings/example_branch_key_id_supplier.py @@ -1,3 +1,5 @@ +"""Example implementation of a branch key ID supplier.""" + from aws_cryptographic_materialproviders.mpl.models import GetBranchKeyIdInput, GetBranchKeyIdOutput from aws_cryptographic_materialproviders.mpl.references import IBranchKeyIdSupplier from typing import Dict @@ -10,12 +12,13 @@ class ExampleBranchKeyIdSupplier(IBranchKeyIdSupplier): branch_key_id_for_tenant_B: str def __init__(self, tenant_1_id, tenant_2_id): + """Example constructor for a branch key ID supplier.""" self.branch_key_id_for_tenant_A = tenant_1_id self.branch_key_id_for_tenant_B = tenant_2_id def get_branch_key_id( self, - # Change this to `native_input` + # TODO-MPL: Change this to `native_input` in Smithy-Dafny input: GetBranchKeyIdInput # noqa pylint: disable=redefined-builtin ) -> GetBranchKeyIdOutput: """Returns branch key ID from the tenant ID in input's encryption context.""" @@ -34,4 +37,4 @@ def get_branch_key_id( else: raise ValueError(f"Item does not contain valid tenant ID: {tenant_key_id=}") - return GetBranchKeyIdOutput(branch_key_id=branch_key_id) \ No newline at end of file + return GetBranchKeyIdOutput(branch_key_id=branch_key_id) From b3b9a0ffd82d962ef4ccea15813b1ba09d6aac3d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 22 Feb 2024 12:07:24 -0800 Subject: [PATCH 102/422] refactor --- .github/workflows/ci_tests.yaml | 1 + .../internal/mpl/cmm_handler.py | 157 ----------------- .../internal/mpl/materials_handlers.py | 164 ------------------ src/aws_encryption_sdk/streaming_client.py | 10 +- test/unit/mpl/README.md | 1 - test/unit/mpl/test_cmm_handler.py | 97 ----------- 6 files changed, 5 insertions(+), 425 deletions(-) delete mode 100644 src/aws_encryption_sdk/internal/mpl/cmm_handler.py delete mode 100644 src/aws_encryption_sdk/internal/mpl/materials_handlers.py delete mode 100644 test/unit/mpl/README.md delete mode 100644 test/unit/mpl/test_cmm_handler.py diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 803d4741e..3a6b16d45 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -27,6 +27,7 @@ jobs: - ubuntu-latest # Windows fails due to "No module named 'Wrappers'" # This SHOULD be fixed once Dafny generates fully-qualified import statements + # (i.e. doo files, per-package module names) # Disable for now # - windows-latest - macos-latest diff --git a/src/aws_encryption_sdk/internal/mpl/cmm_handler.py b/src/aws_encryption_sdk/internal/mpl/cmm_handler.py deleted file mode 100644 index 1575e0187..000000000 --- a/src/aws_encryption_sdk/internal/mpl/cmm_handler.py +++ /dev/null @@ -1,157 +0,0 @@ -"""Retrieves encryption/decryption materials from an underlying materials provider.""" - -# These dependencies are only loaded if you install the MPL. -try: - # pylint seems to struggle with this conditional import - # pylint: disable=unused-import - from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException - from aws_cryptographic_materialproviders.mpl.models import ( - AlgorithmSuiteIdESDK, - CommitmentPolicyESDK, - DecryptMaterialsInput, - DecryptMaterialsOutput, - EncryptedDataKey as MPL_EncryptedDataKey, - GetEncryptionMaterialsInput, - GetEncryptionMaterialsOutput, - ) - from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager - -except ImportError: - pass - -from typing import List - -from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError -from aws_encryption_sdk.identifiers import CommitmentPolicy -from aws_encryption_sdk.internal.mpl.materials_handlers import DecryptionMaterialsHandler, EncryptionMaterialsHandler -from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest -from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager -from aws_encryption_sdk.structures import EncryptedDataKey as Native_EncryptedDataKey - - -class CMMHandler(CryptoMaterialsManager): - """ - In instances where encryption materials may be provided by either - an implementation of the native - `aws_encryption_sdk.materials_managers.base.CryptoMaterialsManager` - or an implementation of the MPL's - `aws_cryptographic_materialproviders.mpl.references.ICryptographicMaterialsManager`, - this provides the correct materials based on the underlying materials manager. - """ - - native_cmm: CryptoMaterialsManager - mpl_cmm: 'ICryptographicMaterialsManager' - - def _is_using_native_cmm(self): - return hasattr(self, "native_cmm") and not hasattr(self, "mpl_cmm") - - def __init__( - self, - cmm: 'CryptoMaterialsManager | ICryptographicMaterialsManager' - ): - """ - Create DecryptionMaterialsHandler. - :param cmm: Underlying cryptographic materials manager - """ - if isinstance(cmm, CryptoMaterialsManager): - self.native_cmm = cmm - elif isinstance(cmm, ICryptographicMaterialsManager): - self.mpl_cmm = cmm - else: - raise ValueError(f"Invalid CMM passed to CMMHandler. cmm: {cmm}") - - def get_encryption_materials( - self, - request: EncryptionMaterialsRequest - ) -> EncryptionMaterialsHandler: - """ - Returns an EncryptionMaterialsHandler for the configured CMM. - :param request: Request for encryption materials - """ - if self._is_using_native_cmm(): - return EncryptionMaterialsHandler(self.native_cmm.get_encryption_materials(request)) - else: - try: - mpl_input: GetEncryptionMaterialsInput = CMMHandler._native_to_mpl_get_encryption_materials( - request - ) - mpl_output: GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) - return EncryptionMaterialsHandler(mpl_output.encryption_materials) - except AwsCryptographicMaterialProvidersException as mpl_exception: - # Wrap MPL error into the ESDK error type - # so customers only have to catch ESDK error types. - raise AWSEncryptionSDKClientError(mpl_exception) - - @staticmethod - def _native_to_mpl_get_encryption_materials( - request: EncryptionMaterialsRequest - ) -> 'GetEncryptionMaterialsInput': - output: GetEncryptionMaterialsInput = GetEncryptionMaterialsInput( - encryption_context=request.encryption_context, - commitment_policy=CMMHandler._native_to_mpl_commmitment_policy( - request.commitment_policy - ), - max_plaintext_length=request.plaintext_length, - ) - return output - - @staticmethod - def _native_to_mpl_commmitment_policy( - native_commitment_policy: CommitmentPolicy - ) -> 'CommitmentPolicyESDK': - if native_commitment_policy == CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT: - return CommitmentPolicyESDK(value="FORBID_ENCRYPT_ALLOW_DECRYPT") - elif native_commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT: - return CommitmentPolicyESDK(value="REQUIRE_ENCRYPT_ALLOW_DECRYPT") - elif native_commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT: - return CommitmentPolicyESDK(value="REQUIRE_ENCRYPT_REQUIRE_DECRYPT") - else: - raise ValueError(f"Invalid native_commitment_policy: {native_commitment_policy}") - - def decrypt_materials( - self, - request: DecryptionMaterialsRequest - ) -> DecryptionMaterialsHandler: - """ - Returns a DecryptionMaterialsHandler for the configured CMM. - :param request: Request for decryption materials - """ - if self._is_using_native_cmm(): - return DecryptionMaterialsHandler(self.native_cmm.decrypt_materials(request)) - else: - try: - mpl_input: 'DecryptMaterialsInput' = \ - CMMHandler._create_mpl_decrypt_materials_input_from_request(request) - mpl_output: 'DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(mpl_input) - return DecryptionMaterialsHandler(mpl_output.decryption_materials) - except AwsCryptographicMaterialProvidersException as mpl_exception: - # Wrap MPL error into the ESDK error type - # so customers only have to catch ESDK error types. - raise AWSEncryptionSDKClientError(mpl_exception) - - @staticmethod - def _native_algorithm_id_to_mpl_algorithm_id(native_algorithm_id: str) -> 'AlgorithmSuiteIdESDK': - # MPL algorithm suite ID = hexstr(native_algorithm_id) padded to 4 digits post-`x`. - return AlgorithmSuiteIdESDK(f"{native_algorithm_id:#0{6}x}") - - @staticmethod - def _create_mpl_decrypt_materials_input_from_request( - request: DecryptionMaterialsRequest - ) -> 'DecryptMaterialsInput': - key_blob_list: List[Native_EncryptedDataKey] = request.encrypted_data_keys - list_edks = [MPL_EncryptedDataKey( - key_provider_id=key_blob.key_provider.provider_id, - key_provider_info=key_blob.key_provider.key_info, - ciphertext=key_blob.encrypted_data_key, - ) for key_blob in key_blob_list] - output: DecryptMaterialsInput = DecryptMaterialsInput( - algorithm_suite_id=CMMHandler._native_algorithm_id_to_mpl_algorithm_id( - request.algorithm.algorithm_id - ), - commitment_policy=CMMHandler._native_to_mpl_commmitment_policy( - request.commitment_policy - ), - encrypted_data_keys=list_edks, - encryption_context=request.encryption_context, - ) - return output diff --git a/src/aws_encryption_sdk/internal/mpl/materials_handlers.py b/src/aws_encryption_sdk/internal/mpl/materials_handlers.py deleted file mode 100644 index 79312f863..000000000 --- a/src/aws_encryption_sdk/internal/mpl/materials_handlers.py +++ /dev/null @@ -1,164 +0,0 @@ -"""Provides encryption/decryption materials from an underlying materials provider.""" -# These dependencies are only loaded if you install the MPL. -try: - from aws_cryptographic_materialproviders.mpl.models import ( - DecryptionMaterials as MPL_DecryptionMaterials, - EncryptedDataKey as MPL_EncryptedDataKey, - EncryptionMaterials as MPL_EncryptionMaterials, - ) -except ImportError: - pass - -from typing import Dict, List, Set - -from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite -from aws_encryption_sdk.materials_managers import ( - DecryptionMaterials as Native_DecryptionMaterials, - EncryptionMaterials as Native_EncryptionMaterials, -) -from aws_encryption_sdk.structures import DataKey, EncryptedDataKey as Native_EncryptedDataKey, MasterKeyInfo - - -def _mpl_algorithm_id_to_native_algorithm_id(mpl_algorithm_id: str): - # MPL algorithm suite ID == hex(native algorithm suite ID) - return int(mpl_algorithm_id, 16) - - -class EncryptionMaterialsHandler: - """ - In instances where encryption materials may be provided by either - the native `aws_encryption_sdk.materials_managers.EncryptionMaterials` - or the MPL's `aws_cryptographic_materialproviders.mpl.models.EncryptionMaterials`, - this provides the correct materials based on the configured materials provider. - """ - - native_materials: Native_EncryptionMaterials - mpl_materials: 'MPL_EncryptionMaterials' - - def __init__( - self, - materials: 'Native_EncryptionMaterials | MPL_EncryptionMaterials' - ): - """ - Create EncryptionMaterialsHandler. - :param materials: Underlying encryption materials - """ - if isinstance(materials, Native_EncryptionMaterials): - self.native_materials = materials - elif isinstance(materials, MPL_EncryptionMaterials): - self.mpl_materials = materials - else: - raise ValueError("Invalid EncryptionMaterials passed to EncryptionMaterialsHandler. " - f"materials: {materials}") - - @property - def algorithm(self) -> Algorithm: - """Materials' native Algorithm.""" - if hasattr(self, "native_materials"): - return self.native_materials.algorithm - else: - return AlgorithmSuite.get_by_id( - _mpl_algorithm_id_to_native_algorithm_id( - self.mpl_materials.algorithm_suite.id.value - ) - ) - - @property - def encryption_context(self) -> Dict[str, str]: - """Materials' encryption context.""" - if hasattr(self, "native_materials"): - return self.native_materials.encryption_context - else: - return self.mpl_materials.encryption_context - - @property - def encrypted_data_keys(self) -> List[Native_EncryptedDataKey]: - """Materials' encrypted data keys.""" - if hasattr(self, "native_materials"): - return self.native_materials.encrypted_data_keys - else: - mpl_edk_list: List[MPL_EncryptedDataKey] = self.mpl_materials.encrypted_data_keys - key_blob_list: Set[Native_EncryptedDataKey] = {Native_EncryptedDataKey( - key_provider=MasterKeyInfo( - provider_id=mpl_edk.key_provider_id, - key_info=mpl_edk.key_provider_info, - ), - encrypted_data_key=mpl_edk.ciphertext, - ) for mpl_edk in mpl_edk_list} - return key_blob_list - - @property - def data_encryption_key(self) -> DataKey: - """Materials' data encryption key.""" - if hasattr(self, "native_materials"): - return self.native_materials.data_encryption_key - else: - mpl_dek = self.mpl_materials.plaintext_data_key - return DataKey( - # key_provider is unused, but the return type is DataKey - key_provider=MasterKeyInfo( - provider_id="", - key_info=b'' - ), - data_key=mpl_dek, - encrypted_data_key=b'', # No encrypted DEK - ) - - @property - def signing_key(self) -> bytes: - """Materials' signing key.""" - if hasattr(self, "native_materials"): - return self.native_materials.signing_key - else: - return self.mpl_materials.signing_key - - -class DecryptionMaterialsHandler: - """ - In instances where decryption materials may be provided by either - the native `aws_encryption_sdk.materials_managers.DecryptionMaterials` - or the MPL's `aws_cryptographic_materialproviders.mpl.models.DecryptionMaterials`, - this provides the correct materials based on the configured materials provider. - """ - - native_materials: Native_DecryptionMaterials - mpl_materials: 'MPL_DecryptionMaterials' - - def __init__( - self, - materials: 'Native_DecryptionMaterials | MPL_DecryptionMaterials' - ): - """ - Create DecryptionMaterialsHandler. - :param materials: Underlying decryption materials - """ - if isinstance(materials, Native_DecryptionMaterials): - self.native_materials = materials - elif isinstance(materials, MPL_DecryptionMaterials): - self.mpl_materials = materials - else: - raise ValueError(f"Invalid DecryptionMaterials passed to DecryptionMaterialsHandler.\ - materials: {materials}") - - @property - def data_key(self) -> DataKey: - """Materials' data key.""" - if hasattr(self, "native_materials"): - return self.native_materials.data_key - else: - return DataKey( - key_provider=MasterKeyInfo( - provider_id="", - key_info=b'' - ), - data_key=self.mpl_materials.plaintext_data_key, - encrypted_data_key=b'', - ) - - @property - def verification_key(self) -> bytes: - """Materials' verification key.""" - if hasattr(self, "native_materials"): - return self.native_materials.verification_key - else: - return self.mpl_materials.verification_key diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 032ed7d15..61f2f88c6 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -58,7 +58,7 @@ serialize_non_framed_close, serialize_non_framed_open, ) -from aws_encryption_sdk.internal.mpl.cmm_handler import CMMHandler +from aws_encryption_sdk.materials_managers.mpl.cmm import MPLCMMHandler from aws_encryption_sdk.internal.utils import exactly_one_arg_is_not_none from aws_encryption_sdk.internal.utils.commitment import ( validate_commitment_policy_on_decrypt, @@ -169,7 +169,7 @@ def _has_mpl_attrs_post_init(self): keyring=self.keyring ) ) - cmm_handler: CryptoMaterialsManager = CMMHandler(cmm) + cmm_handler: CryptoMaterialsManager = MPLCMMHandler(cmm) self.materials_manager = cmm_handler def _no_mpl_attrs_post_init(self): @@ -551,8 +551,7 @@ def _prep_message(self): else: # MPL verification key is PEM bytes, not DER bytes. # If the underlying CMM is from the MPL, load PEM bytes. - if (isinstance(self.config.materials_manager, CMMHandler) - and hasattr(self.config.materials_manager, "mpl_cmm")): + if (isinstance(self.config.materials_manager, MPLCMMHandler)): self.signer = Signer.from_key_bytes( algorithm=self._encryption_materials.algorithm, key_bytes=self._encryption_materials.signing_key, encoding=serialization.Encoding.PEM, @@ -919,8 +918,7 @@ def _read_header(self): else: # MPL verification key is NOT key bytes; it is bytes of the compressed point. # If the underlying CMM is from the MPL, load bytes from encoded point. - if (isinstance(self.config.materials_manager, CMMHandler) - and hasattr(self.config.materials_manager, "mpl_cmm")): + if (isinstance(self.config.materials_manager, MPLCMMHandler)): self.verifier = Verifier.from_encoded_point( algorithm=header.algorithm, encoded_point=base64.b64encode(decryption_materials.verification_key) diff --git a/test/unit/mpl/README.md b/test/unit/mpl/README.md deleted file mode 100644 index 839feb7a2..000000000 --- a/test/unit/mpl/README.md +++ /dev/null @@ -1 +0,0 @@ -Tests in this file REQUIRE the aws-cryptographic-material-providers module to be installed in order to run. \ No newline at end of file diff --git a/test/unit/mpl/test_cmm_handler.py b/test/unit/mpl/test_cmm_handler.py deleted file mode 100644 index d16374899..000000000 --- a/test/unit/mpl/test_cmm_handler.py +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. -"""Test suite to verify the cmm_handler module delegates correctly.""" -import pytest -from aws_cryptographic_materialproviders.mpl.models import ( - EncryptionMaterials as MPL_EncryptionMaterials, - GetEncryptionMaterialsInput, - GetEncryptionMaterialsOutput, -) -from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager -from mock import MagicMock, patch - -from aws_encryption_sdk.internal.mpl.cmm_handler import CMMHandler -from aws_encryption_sdk.internal.mpl.materials_handlers import EncryptionMaterialsHandler -from aws_encryption_sdk.materials_managers import ( - EncryptionMaterials as Native_EncryptionMaterials, - EncryptionMaterialsRequest, -) -from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager - -mock_native_cmm = MagicMock(__class__=CryptoMaterialsManager) -mock_mpl_cmm = MagicMock(__class__=ICryptographicMaterialsManager) -mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) -mock_encryption_materials_handler = MagicMock(__class__=EncryptionMaterialsHandler) -mock_native_encryption_materials = MagicMock(__class__=Native_EncryptionMaterials) -mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) - -pytestmark = [pytest.mark.unit, pytest.mark.local] - - -def test_GIVEN_native_CMM_WHEN_create_CMMHandler_THEN_is_using_native_cmm_returns_True(): - cmm_handler = CMMHandler(cmm=mock_native_cmm) - assert cmm_handler._is_using_native_cmm() - - -def test_GIVEN_mpl_CMM_WHEN_create_CMMHandler_THEN_is_using_native_cmm_returns_False(): - cmm_handler = CMMHandler(cmm=mock_mpl_cmm) - assert not cmm_handler._is_using_native_cmm() - - -def test_GIVEN_unknown_CMM_WHEN_create_CMMHandler_THEN_raise_ValueError(): - with pytest.raises(ValueError): - CMMHandler(cmm="not a CMM") - - -@patch.object(mock_native_cmm, "get_encryption_materials") -def test_GIVEN_native_CMM_WHEN_get_encryption_materials_THEN_return_native_encryption_materials( - mock_get_encryption_materials -): - # Mock: native_cmm.get_encryption_materials returns mock native encryption materials - mock_get_encryption_materials.return_value = mock_native_encryption_materials - - cmm_handler = CMMHandler(cmm=mock_native_cmm) - test = cmm_handler.get_encryption_materials(mock_encryption_materials_request) - - # Verify cmm_handler returns EncryptionMaterialsHandler - assert isinstance(test, EncryptionMaterialsHandler) - # Verify returned EncryptionMaterialsHandler uses the output of `get_encryption_materials` - assert test.native_materials == mock_native_encryption_materials - # Verify we actually called `get_encryption_materials` - mock_native_cmm.get_encryption_materials.assert_called_once_with(mock_encryption_materials_request) - - -@patch.object(mock_mpl_cmm, "get_encryption_materials") -@patch("aws_encryption_sdk.internal.mpl.cmm_handler.CMMHandler._native_to_mpl_get_encryption_materials") -def test_GIVEN_mpl_CMM_WHEN_get_encryption_materials_THEN_return_mpl_encryption_materials( - mock_native_to_mpl_get_encryption_materials, - mock_get_encryption_materials, -): - # Mock: mpl_cmm.get_encryption_materials returns mock MPL encryption materials - mock_get_encryption_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) - mock_get_encryption_materials_output.encryption_materials = mock_mpl_encryption_materials - mock_get_encryption_materials.return_value = mock_get_encryption_materials_output - - # Mock: CMMHandler._native_to_mpl_get_encryption_materials creates a GetEncryptionMaterialsInput - mock_get_encryption_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) - mock_native_to_mpl_get_encryption_materials.return_value = mock_get_encryption_materials_input - - cmm_handler = CMMHandler(cmm=mock_mpl_cmm) - test = cmm_handler.get_encryption_materials(mock_encryption_materials_request) - - # Verify cmm_handler returns EncryptionMaterialsHandler - assert isinstance(test, EncryptionMaterialsHandler) - # Verify returned EncryptionMaterialsHandler uses the output of `get_encryption_materials` - assert test.mpl_materials == mock_mpl_encryption_materials - # Verify we actually called `get_encryption_materials` - mock_mpl_cmm.get_encryption_materials.assert_called_once_with(mock_get_encryption_materials_input) From a594125a635c0741e121d87bf367940b22f7610e Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 22 Feb 2024 12:07:33 -0800 Subject: [PATCH 103/422] refactor --- .../materials_managers/mpl/__init__.py | 13 ++ .../materials_managers/mpl/cmm.py | 142 ++++++++++++++++++ .../materials_managers/mpl/materials.py | 135 +++++++++++++++++ 3 files changed, 290 insertions(+) create mode 100644 src/aws_encryption_sdk/materials_managers/mpl/__init__.py create mode 100644 src/aws_encryption_sdk/materials_managers/mpl/cmm.py create mode 100644 src/aws_encryption_sdk/materials_managers/mpl/materials.py diff --git a/src/aws_encryption_sdk/materials_managers/mpl/__init__.py b/src/aws_encryption_sdk/materials_managers/mpl/__init__.py new file mode 100644 index 000000000..295400d76 --- /dev/null +++ b/src/aws_encryption_sdk/materials_managers/mpl/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Modules related to the MPL's materials managers interfaces.""" diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py new file mode 100644 index 000000000..e16b49d51 --- /dev/null +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -0,0 +1,142 @@ +"""Retrieves encryption/decryption materials from the MPL.""" + +# These dependencies are only loaded if you install the MPL. +try: + # pylint seems to struggle with this conditional import + # pylint: disable=unused-import + from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException + from aws_cryptographic_materialproviders.mpl.models import ( + AlgorithmSuiteIdESDK, + CommitmentPolicyESDK, + DecryptMaterialsInput, + DecryptMaterialsOutput, + EncryptedDataKey as MPL_EncryptedDataKey, + GetEncryptionMaterialsInput, + GetEncryptionMaterialsOutput, + ) + from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager + +except ImportError: + pass + +from typing import List + +from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError +from aws_encryption_sdk.identifiers import CommitmentPolicy +from aws_encryption_sdk.materials_managers.mpl.materials import MPLEncryptionMaterials, MPLDecryptionMaterials +from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest +from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager +from aws_encryption_sdk.structures import EncryptedDataKey as Native_EncryptedDataKey + + +class MPLCMMHandler(CryptoMaterialsManager): + """ + In instances where encryption materials are provided by an implementation of the MPL's + `aws_cryptographic_materialproviders.mpl.references.ICryptographicMaterialsManager`, + this maps the ESDK CMM interfaces to the MPL CMM. + """ + + mpl_cmm: 'ICryptographicMaterialsManager' + + def __init__( + self, + mpl_cmm: 'ICryptographicMaterialsManager' + ): + """ + Create DecryptionMaterialsHandler. + :param cmm: Underlying cryptographic materials manager + """ + if isinstance(mpl_cmm, ICryptographicMaterialsManager): + self.mpl_cmm = mpl_cmm + else: + raise ValueError(f"Invalid CMM passed to MPLCMMHandler. cmm: {mpl_cmm}") + + def get_encryption_materials( + self, + request: EncryptionMaterialsRequest + ) -> MPLEncryptionMaterials: + """ + Returns an EncryptionMaterialsHandler for the configured CMM. + :param request: Request for encryption materials + """ + try: + mpl_input: GetEncryptionMaterialsInput = MPLCMMHandler._native_to_mpl_get_encryption_materials( + request + ) + mpl_output: GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) + return MPLEncryptionMaterials(mpl_output.encryption_materials) + except AwsCryptographicMaterialProvidersException as mpl_exception: + # Wrap MPL error into the ESDK error type + # so customers only have to catch ESDK error types. + raise AWSEncryptionSDKClientError(mpl_exception) + + @staticmethod + def _native_to_mpl_get_encryption_materials( + request: EncryptionMaterialsRequest + ) -> 'GetEncryptionMaterialsInput': + output: GetEncryptionMaterialsInput = GetEncryptionMaterialsInput( + encryption_context=request.encryption_context, + commitment_policy=MPLCMMHandler._native_to_mpl_commmitment_policy( + request.commitment_policy + ), + max_plaintext_length=request.plaintext_length, + ) + return output + + @staticmethod + def _native_to_mpl_commmitment_policy( + native_commitment_policy: CommitmentPolicy + ) -> 'CommitmentPolicyESDK': + if native_commitment_policy == CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT: + return CommitmentPolicyESDK(value="FORBID_ENCRYPT_ALLOW_DECRYPT") + elif native_commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT: + return CommitmentPolicyESDK(value="REQUIRE_ENCRYPT_ALLOW_DECRYPT") + elif native_commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT: + return CommitmentPolicyESDK(value="REQUIRE_ENCRYPT_REQUIRE_DECRYPT") + else: + raise ValueError(f"Invalid native_commitment_policy: {native_commitment_policy}") + + def decrypt_materials( + self, + request: DecryptionMaterialsRequest + ) -> MPLDecryptionMaterials: + """ + Returns a DecryptionMaterialsHandler for the configured CMM. + :param request: Request for decryption materials + """ + try: + mpl_input: 'DecryptMaterialsInput' = \ + MPLCMMHandler._create_mpl_decrypt_materials_input_from_request(request) + mpl_output: 'DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(mpl_input) + return MPLDecryptionMaterials(mpl_output.decryption_materials) + except AwsCryptographicMaterialProvidersException as mpl_exception: + # Wrap MPL error into the ESDK error type + # so customers only have to catch ESDK error types. + raise AWSEncryptionSDKClientError(mpl_exception) + + @staticmethod + def _native_algorithm_id_to_mpl_algorithm_id(native_algorithm_id: str) -> 'AlgorithmSuiteIdESDK': + # MPL algorithm suite ID = hexstr(native_algorithm_id) padded to 4 digits post-`x`. + return AlgorithmSuiteIdESDK(f"{native_algorithm_id:#0{6}x}") + + @staticmethod + def _create_mpl_decrypt_materials_input_from_request( + request: DecryptionMaterialsRequest + ) -> 'DecryptMaterialsInput': + key_blob_list: List[Native_EncryptedDataKey] = request.encrypted_data_keys + list_edks = [MPL_EncryptedDataKey( + key_provider_id=key_blob.key_provider.provider_id, + key_provider_info=key_blob.key_provider.key_info, + ciphertext=key_blob.encrypted_data_key, + ) for key_blob in key_blob_list] + output: DecryptMaterialsInput = DecryptMaterialsInput( + algorithm_suite_id=MPLCMMHandler._native_algorithm_id_to_mpl_algorithm_id( + request.algorithm.algorithm_id + ), + commitment_policy=MPLCMMHandler._native_to_mpl_commmitment_policy( + request.commitment_policy + ), + encrypted_data_keys=list_edks, + encryption_context=request.encryption_context, + ) + return output diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py new file mode 100644 index 000000000..fdcf2ec06 --- /dev/null +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -0,0 +1,135 @@ +"""Provides encryption/decryption materials from an underlying materials provider.""" +# These dependencies are only loaded if you install the MPL. +try: + from aws_cryptographic_materialproviders.mpl.models import ( + DecryptionMaterials as MPL_DecryptionMaterials, + EncryptedDataKey as MPL_EncryptedDataKey, + EncryptionMaterials as MPL_EncryptionMaterials, + ) +except ImportError: + pass + +from typing import Dict, List, Set + +from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite +from aws_encryption_sdk.materials_managers import ( + DecryptionMaterials as Native_DecryptionMaterials, + EncryptionMaterials as Native_EncryptionMaterials, +) +from aws_encryption_sdk.structures import DataKey, EncryptedDataKey as Native_EncryptedDataKey, MasterKeyInfo + + +def _mpl_algorithm_id_to_native_algorithm_id(mpl_algorithm_id: str) -> int: + # MPL algorithm suite ID == hex(native algorithm suite ID) + return int(mpl_algorithm_id, 16) + + +class MPLEncryptionMaterials(Native_EncryptionMaterials): + """ + In instances where encryption materials are be provided by + the MPL's `aws_cryptographic_materialproviders.mpl.models.EncryptionMaterials`, + this maps the ESDK interfaces to the underlying MPL materials. + """ + + mpl_materials: 'MPL_EncryptionMaterials' + + def __init__( + self, + materials: 'MPL_EncryptionMaterials' + ): + """ + Create MPLEncryptionMaterialsHandler. + :param materials: Underlying encryption materials + """ + if isinstance(materials, MPL_EncryptionMaterials): + self.mpl_materials = materials + else: + raise ValueError("Invalid EncryptionMaterials passed to EncryptionMaterialsHandler. " + f"materials: {materials}") + + @property + def algorithm(self) -> Algorithm: + """Materials' native Algorithm.""" + return AlgorithmSuite.get_by_id( + _mpl_algorithm_id_to_native_algorithm_id( + self.mpl_materials.algorithm_suite.id.value + ) + ) + + @property + def encryption_context(self) -> Dict[str, str]: + """Materials' encryption context.""" + return self.mpl_materials.encryption_context + + @property + def encrypted_data_keys(self) -> List[Native_EncryptedDataKey]: + """Materials' encrypted data keys.""" + mpl_edk_list: List[MPL_EncryptedDataKey] = self.mpl_materials.encrypted_data_keys + key_blob_list: Set[Native_EncryptedDataKey] = {Native_EncryptedDataKey( + key_provider=MasterKeyInfo( + provider_id=mpl_edk.key_provider_id, + key_info=mpl_edk.key_provider_info, + ), + encrypted_data_key=mpl_edk.ciphertext, + ) for mpl_edk in mpl_edk_list} + return key_blob_list + + @property + def data_encryption_key(self) -> DataKey: + """Materials' data encryption key.""" + mpl_dek = self.mpl_materials.plaintext_data_key + return DataKey( + # key_provider is unused, but the return type is DataKey + key_provider=MasterKeyInfo( + provider_id="", + key_info=b'' + ), + data_key=mpl_dek, + encrypted_data_key=b'', # No encrypted DEK + ) + + @property + def signing_key(self) -> bytes: + """Materials' signing key.""" + return self.mpl_materials.signing_key + + +class MPLDecryptionMaterials(Native_DecryptionMaterials): + """ + In instances where decryption materials are be provided by + the MPL's `aws_cryptographic_materialproviders.mpl.models.DecryptionMaterials`, + this maps the ESDK interfaces to the underlying MPL materials. + """ + + mpl_materials: 'MPL_DecryptionMaterials' + + def __init__( + self, + materials: 'MPL_DecryptionMaterials' + ): + """ + Create DecryptionMaterialsHandler. + :param materials: Underlying decryption materials + """ + if isinstance(materials, MPL_DecryptionMaterials): + self.mpl_materials = materials + else: + raise ValueError(f"Invalid DecryptionMaterials passed to DecryptionMaterialsHandler.\ + materials: {materials}") + + @property + def data_key(self) -> DataKey: + """Materials' data key.""" + return DataKey( + key_provider=MasterKeyInfo( + provider_id="", + key_info=b'' + ), + data_key=self.mpl_materials.plaintext_data_key, + encrypted_data_key=b'', + ) + + @property + def verification_key(self) -> bytes: + """Materials' verification key.""" + return self.mpl_materials.verification_key From fdd2eda60f42aeea832b7851db1487dfb2784882 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 10:03:34 -0800 Subject: [PATCH 104/422] unit tests --- .../internal/mpl/__init__.py | 13 -------- .../materials_managers/mpl/cmm.py | 20 ++++++----- .../materials_managers/mpl/materials.py | 33 +++++++++++-------- 3 files changed, 32 insertions(+), 34 deletions(-) delete mode 100644 src/aws_encryption_sdk/internal/mpl/__init__.py diff --git a/src/aws_encryption_sdk/internal/mpl/__init__.py b/src/aws_encryption_sdk/internal/mpl/__init__.py deleted file mode 100644 index 41497cc20..000000000 --- a/src/aws_encryption_sdk/internal/mpl/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. -"""Modules related to the MPL.""" diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index e16b49d51..cd789b994 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -15,9 +15,9 @@ GetEncryptionMaterialsOutput, ) from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager - + _HAS_MPL = True except ImportError: - pass + _HAS_MPL = False from typing import List @@ -43,9 +43,12 @@ def __init__( mpl_cmm: 'ICryptographicMaterialsManager' ): """ - Create DecryptionMaterialsHandler. - :param cmm: Underlying cryptographic materials manager + Create MPLCMMHandler. + :param mpl_cmm: Underlying MPL cryptographic materials manager """ + if not _HAS_MPL: + raise ImportError("You MUST install the aws-cryptographic-material-providers " + f"library to create an instance of {MPLCMMHandler}") if isinstance(mpl_cmm, ICryptographicMaterialsManager): self.mpl_cmm = mpl_cmm else: @@ -74,11 +77,12 @@ def get_encryption_materials( def _native_to_mpl_get_encryption_materials( request: EncryptionMaterialsRequest ) -> 'GetEncryptionMaterialsInput': + commitment_policy = MPLCMMHandler._native_to_mpl_commmitment_policy( + request.commitment_policy + ) output: GetEncryptionMaterialsInput = GetEncryptionMaterialsInput( encryption_context=request.encryption_context, - commitment_policy=MPLCMMHandler._native_to_mpl_commmitment_policy( - request.commitment_policy - ), + commitment_policy=commitment_policy, max_plaintext_length=request.plaintext_length, ) return output @@ -101,7 +105,7 @@ def decrypt_materials( request: DecryptionMaterialsRequest ) -> MPLDecryptionMaterials: """ - Returns a DecryptionMaterialsHandler for the configured CMM. + Returns a MPLDecryptionMaterials for the configured CMM. :param request: Request for decryption materials """ try: diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index fdcf2ec06..bd4b5f729 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -6,8 +6,9 @@ EncryptedDataKey as MPL_EncryptedDataKey, EncryptionMaterials as MPL_EncryptionMaterials, ) + _HAS_MPL = True except ImportError: - pass + _HAS_MPL = False from typing import Dict, List, Set @@ -35,17 +36,20 @@ class MPLEncryptionMaterials(Native_EncryptionMaterials): def __init__( self, - materials: 'MPL_EncryptionMaterials' + mpl_materials: 'MPL_EncryptionMaterials' ): """ - Create MPLEncryptionMaterialsHandler. + Create MPLEncryptionMaterials. :param materials: Underlying encryption materials """ - if isinstance(materials, MPL_EncryptionMaterials): - self.mpl_materials = materials + if not _HAS_MPL: + raise ImportError("You MUST install the aws-cryptographic-material-providers " + f"library to create an instance of {MPLEncryptionMaterials}") + if isinstance(mpl_materials, MPL_EncryptionMaterials): + self.mpl_materials = mpl_materials else: - raise ValueError("Invalid EncryptionMaterials passed to EncryptionMaterialsHandler. " - f"materials: {materials}") + raise ValueError("Invalid EncryptionMaterials passed to MPLEncryptionMaterials. " + f"materials: {mpl_materials}") @property def algorithm(self) -> Algorithm: @@ -105,17 +109,20 @@ class MPLDecryptionMaterials(Native_DecryptionMaterials): def __init__( self, - materials: 'MPL_DecryptionMaterials' + mpl_materials: 'MPL_DecryptionMaterials' ): """ - Create DecryptionMaterialsHandler. + Create MPLDecryptionMaterials. :param materials: Underlying decryption materials """ - if isinstance(materials, MPL_DecryptionMaterials): - self.mpl_materials = materials + if not _HAS_MPL: + raise ImportError("You MUST install the aws-cryptographic-material-providers " + f"library to create an instance of {MPLDecryptionMaterials}") + if isinstance(mpl_materials, MPL_DecryptionMaterials): + self.mpl_materials = mpl_materials else: - raise ValueError(f"Invalid DecryptionMaterials passed to DecryptionMaterialsHandler.\ - materials: {materials}") + raise ValueError(f"Invalid DecryptionMaterials passed to MPLDecryptionMaterials.\ + materials: {mpl_materials}") @property def data_key(self) -> DataKey: From 0138f226a73bff13fe1e24d865b09ec7e2ff42b2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 10:03:47 -0800 Subject: [PATCH 105/422] unit tests --- test/unit/test_material_managers_mpl_cmm.py | 278 ++++++++++++++++++ .../test_material_managers_mpl_materials.py | 221 ++++++++++++++ 2 files changed, 499 insertions(+) create mode 100644 test/unit/test_material_managers_mpl_cmm.py create mode 100644 test/unit/test_material_managers_mpl_materials.py diff --git a/test/unit/test_material_managers_mpl_cmm.py b/test/unit/test_material_managers_mpl_cmm.py new file mode 100644 index 000000000..77bf5502d --- /dev/null +++ b/test/unit/test_material_managers_mpl_cmm.py @@ -0,0 +1,278 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" + +import pytest +from mock import MagicMock, patch + + +from aws_encryption_sdk.identifiers import CommitmentPolicy +import aws_encryption_sdk.materials_managers.mpl.cmm +from aws_encryption_sdk.materials_managers.mpl.cmm import MPLCMMHandler +from aws_encryption_sdk.materials_managers.mpl.materials import ( + MPLEncryptionMaterials, + MPLDecryptionMaterials, +) + +pytestmark = [pytest.mark.unit, pytest.mark.local] + + +# Check if MPL is installed, and skip tests based on its installation status +# Ideally, this logic would be based on mocking imports and testing logic, +# but doing that introduces errors that cause other tests to fail. +try: + from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException + from aws_cryptographic_materialproviders.mpl.models import ( + AlgorithmSuiteIdESDK, + CommitmentPolicyESDK, + DecryptMaterialsInput, + DecryptionMaterials as MPL_DecryptionMaterials, + EncryptionMaterials as MPL_EncryptionMaterials, + GetEncryptionMaterialsInput, + GetEncryptionMaterialsOutput, + ) + from aws_cryptographic_materialproviders.mpl.references import ( + ICryptographicMaterialsManager + ) + HAS_MPL = True + + mock_mpl_cmm = MagicMock(__class__=ICryptographicMaterialsManager) + mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) + mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) + +except ImportError: + HAS_MPL = False + + # Ensure references to these mocks exist, even if they aren't used in a non-MPL context + mock_mpl_cmm = None + mock_mpl_encryption_materials = None + mock_mpl_decrypt_materials = None + +from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError +from aws_encryption_sdk.materials_managers import ( + EncryptionMaterialsRequest, + DecryptionMaterialsRequest, +) + + +mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) +mock_encryption_materials_handler = MagicMock(__class__=MPLEncryptionMaterials) +mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) + +@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") +def test_GIVEN_test_has_mpl_is_False_THEN_cmm_has_mpl_is_False(): + """If the MPL IS NOT installed in the runtime environment, + assert the cmm has _HAS_MPL set to False""" + + assert hasattr(aws_encryption_sdk.materials_managers.mpl.cmm, "_HAS_MPL") + assert aws_encryption_sdk.materials_managers.mpl.cmm._HAS_MPL is False + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_test_has_mpl_is_True_THEN_cmm_has_mpl_is_True(): + """If the MPL IS installed in the runtime environment, + assert the cmm has _HAS_MPL set to True""" + + assert hasattr(aws_encryption_sdk.materials_managers.mpl.cmm, "_HAS_MPL") + assert aws_encryption_sdk.materials_managers.mpl.cmm._HAS_MPL is True + + +@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_THEN_raise_ImportError(): + with pytest.raises(ImportError): + MPLCMMHandler(mpl_cmm="doesn't matter") + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_valid_mpl_cmm_THEN_return_new_MPLCMMHandler(): + mpl_cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) + + assert mpl_cmm_handler.mpl_cmm == mock_mpl_cmm + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_invalid_mpl_cmm_THEN_raise_ValueError(): + with pytest.raises(ValueError): + MPLCMMHandler(mpl_cmm="not a valid mpl_cmm") + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +@patch.object(mock_mpl_cmm, "get_encryption_materials") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_get_encryption_materials") +def test_GIVEN_valid_request_WHEN_call_get_encryption_materials_THEN_return_MPLEncryptionMaterials( + mock_native_to_mpl_get_encryption_materials, + mock_get_encryption_materials, +): + + # Mock: mpl_cmm.get_encryption_materials returns mock MPL encryption materials + mock_get_encryption_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) + mock_get_encryption_materials_output.encryption_materials = mock_mpl_encryption_materials + mock_get_encryption_materials.return_value = mock_get_encryption_materials_output + + # Mock: CMMHandler._native_to_mpl_get_encryption_materials creates a GetEncryptionMaterialsInput + mock_get_encryption_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) + mock_native_to_mpl_get_encryption_materials.return_value = mock_get_encryption_materials_input + + cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) + test = cmm_handler.get_encryption_materials(mock_encryption_materials_request) + + # Verify cmm_handler returns MPLEncryptionMaterials + assert isinstance(test, MPLEncryptionMaterials) + # Verify returned EncryptionMaterialsHandler uses the output of `get_encryption_materials` + assert test.mpl_materials == mock_mpl_encryption_materials + # Verify we actually called `get_encryption_materials` + mock_mpl_cmm.get_encryption_materials.assert_called_once_with(mock_get_encryption_materials_input) + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") +def test_GIVEN_get_encryption_materials_raises_MPL_Exception_WHEN_call_get_encryption_materials_THEN_raise_ESDK_Exception( + _ +): + with pytest.raises(AWSEncryptionSDKClientError): + with patch.object(mock_mpl_cmm, "get_encryption_materials", + side_effect=AwsCryptographicMaterialProvidersException("any")): + + cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) + cmm_handler.get_encryption_materials(mock_encryption_materials_request) + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") +def test_GIVEN_native_to_mpl_commmitment_policy_returns_valid_policy_WHEN_call_native_to_mpl_get_encryption_materials_THEN_returns_GetEncryptionMaterialsInput( + mock_mpl_commitment_policy +): + mock_commitment_policy = MagicMock(__class__=CommitmentPolicyESDK) + mock_mpl_commitment_policy.return_value = mock_commitment_policy + + output = MPLCMMHandler._native_to_mpl_get_encryption_materials(mock_encryption_materials_request) + + # verify correctness of returned value + assert isinstance(output, GetEncryptionMaterialsInput) + assert output.encryption_context == mock_encryption_materials_request.encryption_context + assert output.commitment_policy == mock_commitment_policy + assert output.max_plaintext_length == mock_encryption_materials_request.plaintext_length + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_FORBID_ENCRYPT_ALLOW_DECRYPT(): + native_commitment_policy = CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT + + output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) + + assert isinstance(output, CommitmentPolicyESDK) + assert output.value == "FORBID_ENCRYPT_ALLOW_DECRYPT" + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_ALLOW_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_REQUIRE_ENCRYPT_ALLOW_DECRYPT(): + native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT + + output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) + + assert isinstance(output, CommitmentPolicyESDK) + assert output.value == "REQUIRE_ENCRYPT_ALLOW_DECRYPT" + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_REQUIRE_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_REQUIRE_ENCRYPT_REQUIRE_DECRYPT(): + native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT + + output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) + + assert isinstance(output, CommitmentPolicyESDK) + assert output.value == "REQUIRE_ENCRYPT_REQUIRE_DECRYPT" + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_CommitmentPolicy_unrecognized_WHEN_call_native_to_mpl_commmitment_policyTHEN_raise_ValueError(): + native_commitment_policy = "not a commitment policy" + + with pytest.raises(ValueError): + MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +@patch.object(mock_mpl_cmm, "decrypt_materials") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._create_mpl_decrypt_materials_input_from_request") +def test_GIVEN_valid_request_WHEN_call_decrypt_materials_THEN_return_MPLDecryptionMaterials( + mock_native_to_mpl_decrypt_materials, + mock_get_encryption_materials, +): + + # Mock: mpl_cmm.get_decryption_materials returns mock MPL decryption materials + mock_decrypt_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) + mock_decrypt_materials_output.decryption_materials = mock_mpl_decrypt_materials + mock_get_encryption_materials.return_value = mock_decrypt_materials_output + + # Mock: CMMHandler._create_mpl_decrypt_materials_input_from_request creates a DecryptMaterialsInput + mock_decrypt_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) + mock_native_to_mpl_decrypt_materials.return_value = mock_decrypt_materials_input + + cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) + output = cmm_handler.decrypt_materials(mock_decryption_materials_request) + + # Verify cmm_handler returns MPLDecryptionMaterials + assert isinstance(output, MPLDecryptionMaterials) + # Verify returned MPLDecryptionMaterials uses the output of `decrypt_materials` + assert output.mpl_materials == mock_mpl_decrypt_materials + # Verify we actually called `decrypt_materials` + mock_mpl_cmm.decrypt_materials.assert_called_once_with(mock_decrypt_materials_input) + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._create_mpl_decrypt_materials_input_from_request") +def test_GIVEN_decrypt_materials_raises_MPL_Exception_WHEN_call_decrypt_materials_THEN_raise_ESDK_Exception( + _ +): + with pytest.raises(AWSEncryptionSDKClientError): + with patch.object(mock_mpl_cmm, "decrypt_materials", + side_effect=AwsCryptographicMaterialProvidersException("any")): + + cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) + cmm_handler.decrypt_materials(mock_decryption_materials_request) + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_WHEN_call_native_algorithm_id_to_mpl_algorithm_id_THEN_returns_valid_AlgorithmSuiteIdESDK(): + some_native_algorithm_id = 0x0000 # Not a real algorithm ID, but fits the format + + mpl_output = MPLCMMHandler._native_algorithm_id_to_mpl_algorithm_id( + some_native_algorithm_id + ) + + assert isinstance(mpl_output, AlgorithmSuiteIdESDK) + assert mpl_output.value == "0x0000" + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_algorithm_id_to_mpl_algorithm_id") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") +def test__create_mpl_decrypt_materials_input_from_request( + mock_mpl_commitment_policy, + mock_mpl_algorithm_id, +): + mock_algorithm_id = "0x1234" # Some fake algorithm ID that fits the format + mock_mpl_algorithm_id.return_value = mock_algorithm_id + mock_commitment_policy = MagicMock(__class__=CommitmentPolicyESDK) + mock_mpl_commitment_policy.return_value = mock_commitment_policy + + # mock_decryption_materials_request.algorithm = + + output = MPLCMMHandler._create_mpl_decrypt_materials_input_from_request(mock_decryption_materials_request) + + assert isinstance(output, DecryptMaterialsInput) + assert output.algorithm_suite_id == mock_algorithm_id + assert output.commitment_policy == mock_commitment_policy + assert output.encryption_context == mock_decryption_materials_request.encryption_context + + assert len(output.encrypted_data_keys) == len(mock_decryption_materials_request.encrypted_data_keys) + for i in range(len(output.encrypted_data_keys)): + # Assume input[i] == output[i], seems to work + output_edk = output.encrypted_data_keys[i] + input_edk = mock_decryption_materials_request[i] + assert output_edk.key_provider_id == input_edk.key_provider.provider_id + assert output_edk.key_provider_info == input_edk.key_provider.key_info + assert output_edk.ciphertext == input_edk.encrypted_data_key diff --git a/test/unit/test_material_managers_mpl_materials.py b/test/unit/test_material_managers_mpl_materials.py new file mode 100644 index 000000000..250efeb7e --- /dev/null +++ b/test/unit/test_material_managers_mpl_materials.py @@ -0,0 +1,221 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" + +import pytest +from mock import MagicMock, patch, PropertyMock +from typing import Dict, List + +from aws_encryption_sdk.identifiers import CommitmentPolicy +import aws_encryption_sdk.materials_managers.mpl.materials +from aws_encryption_sdk.materials_managers.mpl.materials import ( + MPLEncryptionMaterials, + MPLDecryptionMaterials, +) +from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite + +pytestmark = [pytest.mark.unit, pytest.mark.local] + + +# Check if MPL is installed, and skip tests based on its installation status +# Ideally, this logic would be based on mocking imports and testing logic, +# but doing that introduces errors that cause other tests to fail. +try: + from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException + from aws_cryptographic_materialproviders.mpl.models import ( + AlgorithmSuiteIdESDK, + CommitmentPolicyESDK, + DecryptMaterialsInput, + DecryptionMaterials as MPL_DecryptionMaterials, + EncryptedDataKey as MPL_EncryptedDataKey, + EncryptionMaterials as MPL_EncryptionMaterials, + GetEncryptionMaterialsInput, + GetEncryptionMaterialsOutput, + ) + from aws_cryptographic_materialproviders.mpl.references import ( + ICryptographicMaterialsManager + ) + HAS_MPL = True + + mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) + mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) + +except ImportError: + HAS_MPL = False + + # Ensure references to these mocks exist, even if they aren't used in a non-MPL context + mock_mpl_cmm = None + mock_mpl_encryption_materials = None + mock_mpl_decrypt_materials = None + +from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError +from aws_encryption_sdk.materials_managers import ( + EncryptionMaterialsRequest, + DecryptionMaterialsRequest, +) + + +mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) +mock_encryption_materials_handler = MagicMock(__class__=MPLEncryptionMaterials) +mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) + +@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") +def test_GIVEN_test_has_mpl_is_False_THEN_cmm_has_mpl_is_False(): + """If the MPL IS NOT installed in the runtime environment, + assert the cmm has _HAS_MPL set to False""" + + assert hasattr(aws_encryption_sdk.materials_managers.mpl.materials, "_HAS_MPL") + assert aws_encryption_sdk.materials_managers.mpl.materials._HAS_MPL is False + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_test_has_mpl_is_True_THEN_cmm_has_mpl_is_True(): + """If the MPL IS installed in the runtime environment, + assert the cmm has _HAS_MPL set to True""" + + assert hasattr(aws_encryption_sdk.materials_managers.mpl.materials, "_HAS_MPL") + assert aws_encryption_sdk.materials_managers.mpl.materials._HAS_MPL is True + + +@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_THEN_raise_ImportError(): + with pytest.raises(ImportError): + MPLEncryptionMaterials(mpl_materials="doesn't matter") + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_valid_mpl_cmm_THEN_return_new_MPLCMMHandler(): + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + + assert mpl_encryption_materials.mpl_materials == mock_mpl_encryption_materials + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_invalid_mpl_cmm_THEN_raise_ValueError(): + with pytest.raises(ValueError): + MPLEncryptionMaterials(mpl_materials="not a valid mpl_materials") + +def test_mpl_to_native(): + some_mpl_algorithm_id = "0x1234" # Not a real algorithm ID, but fits the format + + native_output = aws_encryption_sdk.materials_managers.mpl.materials._mpl_algorithm_id_to_native_algorithm_id( + some_mpl_algorithm_id + ) + + assert native_output == 0x1234 + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +@patch("aws_encryption_sdk.materials_managers.mpl.materials._mpl_algorithm_id_to_native_algorithm_id") +@patch("aws_encryption_sdk.materials_managers.mpl.materials.AlgorithmSuite.get_by_id") +def test_GIVEN_valid_mpl_algorithm_id_WHEN_get_algorithm_THEN_valid_native_algorithm_id( + mock_algorithm, + mock_native_algorithm_id, +): + # Mock valid conversion from MPL to native algorithm ID + mock_native_algorithm_id.return_value = 0x1234 + + # Mock valid lookup in native AlgorithmSuite lookup + mock_algorithm.return_value = MagicMock(__class__=AlgorithmSuite) + + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + output = mpl_encryption_materials.algorithm + assert output == mock_algorithm() # property calls automatically, we need to call the mock + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GecTHEN_valid_native_algorithm_id(): + mock_encryption_context = MagicMock(__class__=Dict[str, str]) + mock_mpl_encryption_materials.encryption_context = mock_encryption_context + + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + output = mpl_encryption_materials.encryption_context + + assert output == mock_encryption_context + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GecTHEN_valid_nativefadsf_algorithm_id(): + mock_edk = MagicMock(__class__=MPL_EncryptedDataKey) + mock_mpl_key_provider_id = MagicMock(__class__=str) + mock_edk.key_provider_id = mock_mpl_key_provider_id + mock_mpl_key_provider_info = MagicMock(__class__=bytes) + mock_edk.key_provider_info = mock_mpl_key_provider_info + mock_mpl_ciphertext = MagicMock(__class__=bytes) + mock_edk.ciphertext = mock_mpl_ciphertext + + mock_edks = [ mock_edk ] + mock_mpl_encryption_materials.encrypted_data_keys = mock_edks + + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + output = mpl_encryption_materials.encrypted_data_keys + output_as_list = list(output) + + assert len(output_as_list) == len(mock_edks) + for i in range(len(output_as_list)): + # assume output[i] corresponds to input[i] + native_edk = output_as_list[i] + mpl_edk = mock_edks[i] + + assert native_edk.encrypted_data_key == mpl_edk.ciphertext + assert native_edk.key_provider.provider_id == mpl_edk.key_provider_id + assert native_edk.key_provider.key_info == mpl_edk.key_provider_info + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GecTHEN_valid_nativefadsffadsfa_algorithm_id(): + mock_data_key = MagicMock(__class__=bytes) + mock_mpl_encryption_materials.plaintext_data_key = mock_data_key + + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + output = mpl_encryption_materials.data_encryption_key + + assert output.key_provider.provider_id == "" + assert output.key_provider.key_info == b"" + assert output.data_key == mock_data_key + assert output.encrypted_data_key == b"" + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GecTHEN_valid_nativefasdfasdffadsf_algorithm_id(): + mock_signing_key = MagicMock(__class__=bytes) + mock_mpl_encryption_materials.signing_key = mock_signing_key + + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + output = mpl_encryption_materials.signing_key + + assert output == mock_signing_key + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GecTHEN_valid_nativeffasdfasdadsffadsfa_algorithm_id(): + mock_data_key = MagicMock(__class__=bytes) + mock_mpl_decrypt_materials.plaintext_data_key = mock_data_key + + mpl_decryption_materials = MPLDecryptionMaterials(mpl_materials=mock_mpl_decrypt_materials) + output = mpl_decryption_materials.data_key + + assert output.key_provider.provider_id == "" + assert output.key_provider.key_info == b"" + assert output.data_key == mock_data_key + assert output.encrypted_data_key == b"" + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GecTHEN_validadsfasdf_nativefasdfasdffadsf_algorithm_id(): + mock_verification_key = MagicMock(__class__=bytes) + mock_mpl_decrypt_materials.verification_key = mock_verification_key + + mpl_decryption_materials = MPLDecryptionMaterials(mpl_materials=mock_mpl_decrypt_materials) + output = mpl_decryption_materials.verification_key + + assert output == mock_verification_key From f213e1912c4c87ead95eb92734c959d5ea91a388 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 10:14:13 -0800 Subject: [PATCH 106/422] upgrade image --- buildspec.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/buildspec.yml b/buildspec.yml index 3d70c144d..5dbd3f2b8 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -108,6 +108,8 @@ batch: buildspec: codebuild/coverage/coverage.yml - identifier: code_coverage_mpl buildspec: codebuild/coverage/coverage_mpl.yml + env: + image: aws/codebuild/standard:7.0 - identifier: compliance buildspec: codebuild/compliance/compliance.yml From d55f2963b82270f1a3377ff524a55ae663b5675a Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 10:23:56 -0800 Subject: [PATCH 107/422] refactor tests --- test/unit/mpl/__init__.py | 12 + .../mpl/test_material_managers_mpl_cmm.py | 278 ++++++++++++++++++ .../test_material_managers_mpl_materials.py | 221 ++++++++++++++ tox.ini | 12 +- 4 files changed, 516 insertions(+), 7 deletions(-) create mode 100644 test/unit/mpl/__init__.py create mode 100644 test/unit/mpl/test_material_managers_mpl_cmm.py create mode 100644 test/unit/mpl/test_material_managers_mpl_materials.py diff --git a/test/unit/mpl/__init__.py b/test/unit/mpl/__init__.py new file mode 100644 index 000000000..53a960891 --- /dev/null +++ b/test/unit/mpl/__init__.py @@ -0,0 +1,12 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. diff --git a/test/unit/mpl/test_material_managers_mpl_cmm.py b/test/unit/mpl/test_material_managers_mpl_cmm.py new file mode 100644 index 000000000..77bf5502d --- /dev/null +++ b/test/unit/mpl/test_material_managers_mpl_cmm.py @@ -0,0 +1,278 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" + +import pytest +from mock import MagicMock, patch + + +from aws_encryption_sdk.identifiers import CommitmentPolicy +import aws_encryption_sdk.materials_managers.mpl.cmm +from aws_encryption_sdk.materials_managers.mpl.cmm import MPLCMMHandler +from aws_encryption_sdk.materials_managers.mpl.materials import ( + MPLEncryptionMaterials, + MPLDecryptionMaterials, +) + +pytestmark = [pytest.mark.unit, pytest.mark.local] + + +# Check if MPL is installed, and skip tests based on its installation status +# Ideally, this logic would be based on mocking imports and testing logic, +# but doing that introduces errors that cause other tests to fail. +try: + from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException + from aws_cryptographic_materialproviders.mpl.models import ( + AlgorithmSuiteIdESDK, + CommitmentPolicyESDK, + DecryptMaterialsInput, + DecryptionMaterials as MPL_DecryptionMaterials, + EncryptionMaterials as MPL_EncryptionMaterials, + GetEncryptionMaterialsInput, + GetEncryptionMaterialsOutput, + ) + from aws_cryptographic_materialproviders.mpl.references import ( + ICryptographicMaterialsManager + ) + HAS_MPL = True + + mock_mpl_cmm = MagicMock(__class__=ICryptographicMaterialsManager) + mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) + mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) + +except ImportError: + HAS_MPL = False + + # Ensure references to these mocks exist, even if they aren't used in a non-MPL context + mock_mpl_cmm = None + mock_mpl_encryption_materials = None + mock_mpl_decrypt_materials = None + +from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError +from aws_encryption_sdk.materials_managers import ( + EncryptionMaterialsRequest, + DecryptionMaterialsRequest, +) + + +mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) +mock_encryption_materials_handler = MagicMock(__class__=MPLEncryptionMaterials) +mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) + +@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") +def test_GIVEN_test_has_mpl_is_False_THEN_cmm_has_mpl_is_False(): + """If the MPL IS NOT installed in the runtime environment, + assert the cmm has _HAS_MPL set to False""" + + assert hasattr(aws_encryption_sdk.materials_managers.mpl.cmm, "_HAS_MPL") + assert aws_encryption_sdk.materials_managers.mpl.cmm._HAS_MPL is False + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_test_has_mpl_is_True_THEN_cmm_has_mpl_is_True(): + """If the MPL IS installed in the runtime environment, + assert the cmm has _HAS_MPL set to True""" + + assert hasattr(aws_encryption_sdk.materials_managers.mpl.cmm, "_HAS_MPL") + assert aws_encryption_sdk.materials_managers.mpl.cmm._HAS_MPL is True + + +@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_THEN_raise_ImportError(): + with pytest.raises(ImportError): + MPLCMMHandler(mpl_cmm="doesn't matter") + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_valid_mpl_cmm_THEN_return_new_MPLCMMHandler(): + mpl_cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) + + assert mpl_cmm_handler.mpl_cmm == mock_mpl_cmm + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_invalid_mpl_cmm_THEN_raise_ValueError(): + with pytest.raises(ValueError): + MPLCMMHandler(mpl_cmm="not a valid mpl_cmm") + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +@patch.object(mock_mpl_cmm, "get_encryption_materials") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_get_encryption_materials") +def test_GIVEN_valid_request_WHEN_call_get_encryption_materials_THEN_return_MPLEncryptionMaterials( + mock_native_to_mpl_get_encryption_materials, + mock_get_encryption_materials, +): + + # Mock: mpl_cmm.get_encryption_materials returns mock MPL encryption materials + mock_get_encryption_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) + mock_get_encryption_materials_output.encryption_materials = mock_mpl_encryption_materials + mock_get_encryption_materials.return_value = mock_get_encryption_materials_output + + # Mock: CMMHandler._native_to_mpl_get_encryption_materials creates a GetEncryptionMaterialsInput + mock_get_encryption_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) + mock_native_to_mpl_get_encryption_materials.return_value = mock_get_encryption_materials_input + + cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) + test = cmm_handler.get_encryption_materials(mock_encryption_materials_request) + + # Verify cmm_handler returns MPLEncryptionMaterials + assert isinstance(test, MPLEncryptionMaterials) + # Verify returned EncryptionMaterialsHandler uses the output of `get_encryption_materials` + assert test.mpl_materials == mock_mpl_encryption_materials + # Verify we actually called `get_encryption_materials` + mock_mpl_cmm.get_encryption_materials.assert_called_once_with(mock_get_encryption_materials_input) + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") +def test_GIVEN_get_encryption_materials_raises_MPL_Exception_WHEN_call_get_encryption_materials_THEN_raise_ESDK_Exception( + _ +): + with pytest.raises(AWSEncryptionSDKClientError): + with patch.object(mock_mpl_cmm, "get_encryption_materials", + side_effect=AwsCryptographicMaterialProvidersException("any")): + + cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) + cmm_handler.get_encryption_materials(mock_encryption_materials_request) + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") +def test_GIVEN_native_to_mpl_commmitment_policy_returns_valid_policy_WHEN_call_native_to_mpl_get_encryption_materials_THEN_returns_GetEncryptionMaterialsInput( + mock_mpl_commitment_policy +): + mock_commitment_policy = MagicMock(__class__=CommitmentPolicyESDK) + mock_mpl_commitment_policy.return_value = mock_commitment_policy + + output = MPLCMMHandler._native_to_mpl_get_encryption_materials(mock_encryption_materials_request) + + # verify correctness of returned value + assert isinstance(output, GetEncryptionMaterialsInput) + assert output.encryption_context == mock_encryption_materials_request.encryption_context + assert output.commitment_policy == mock_commitment_policy + assert output.max_plaintext_length == mock_encryption_materials_request.plaintext_length + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_FORBID_ENCRYPT_ALLOW_DECRYPT(): + native_commitment_policy = CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT + + output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) + + assert isinstance(output, CommitmentPolicyESDK) + assert output.value == "FORBID_ENCRYPT_ALLOW_DECRYPT" + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_ALLOW_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_REQUIRE_ENCRYPT_ALLOW_DECRYPT(): + native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT + + output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) + + assert isinstance(output, CommitmentPolicyESDK) + assert output.value == "REQUIRE_ENCRYPT_ALLOW_DECRYPT" + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_REQUIRE_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_REQUIRE_ENCRYPT_REQUIRE_DECRYPT(): + native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT + + output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) + + assert isinstance(output, CommitmentPolicyESDK) + assert output.value == "REQUIRE_ENCRYPT_REQUIRE_DECRYPT" + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_CommitmentPolicy_unrecognized_WHEN_call_native_to_mpl_commmitment_policyTHEN_raise_ValueError(): + native_commitment_policy = "not a commitment policy" + + with pytest.raises(ValueError): + MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +@patch.object(mock_mpl_cmm, "decrypt_materials") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._create_mpl_decrypt_materials_input_from_request") +def test_GIVEN_valid_request_WHEN_call_decrypt_materials_THEN_return_MPLDecryptionMaterials( + mock_native_to_mpl_decrypt_materials, + mock_get_encryption_materials, +): + + # Mock: mpl_cmm.get_decryption_materials returns mock MPL decryption materials + mock_decrypt_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) + mock_decrypt_materials_output.decryption_materials = mock_mpl_decrypt_materials + mock_get_encryption_materials.return_value = mock_decrypt_materials_output + + # Mock: CMMHandler._create_mpl_decrypt_materials_input_from_request creates a DecryptMaterialsInput + mock_decrypt_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) + mock_native_to_mpl_decrypt_materials.return_value = mock_decrypt_materials_input + + cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) + output = cmm_handler.decrypt_materials(mock_decryption_materials_request) + + # Verify cmm_handler returns MPLDecryptionMaterials + assert isinstance(output, MPLDecryptionMaterials) + # Verify returned MPLDecryptionMaterials uses the output of `decrypt_materials` + assert output.mpl_materials == mock_mpl_decrypt_materials + # Verify we actually called `decrypt_materials` + mock_mpl_cmm.decrypt_materials.assert_called_once_with(mock_decrypt_materials_input) + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._create_mpl_decrypt_materials_input_from_request") +def test_GIVEN_decrypt_materials_raises_MPL_Exception_WHEN_call_decrypt_materials_THEN_raise_ESDK_Exception( + _ +): + with pytest.raises(AWSEncryptionSDKClientError): + with patch.object(mock_mpl_cmm, "decrypt_materials", + side_effect=AwsCryptographicMaterialProvidersException("any")): + + cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) + cmm_handler.decrypt_materials(mock_decryption_materials_request) + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_WHEN_call_native_algorithm_id_to_mpl_algorithm_id_THEN_returns_valid_AlgorithmSuiteIdESDK(): + some_native_algorithm_id = 0x0000 # Not a real algorithm ID, but fits the format + + mpl_output = MPLCMMHandler._native_algorithm_id_to_mpl_algorithm_id( + some_native_algorithm_id + ) + + assert isinstance(mpl_output, AlgorithmSuiteIdESDK) + assert mpl_output.value == "0x0000" + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_algorithm_id_to_mpl_algorithm_id") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") +def test__create_mpl_decrypt_materials_input_from_request( + mock_mpl_commitment_policy, + mock_mpl_algorithm_id, +): + mock_algorithm_id = "0x1234" # Some fake algorithm ID that fits the format + mock_mpl_algorithm_id.return_value = mock_algorithm_id + mock_commitment_policy = MagicMock(__class__=CommitmentPolicyESDK) + mock_mpl_commitment_policy.return_value = mock_commitment_policy + + # mock_decryption_materials_request.algorithm = + + output = MPLCMMHandler._create_mpl_decrypt_materials_input_from_request(mock_decryption_materials_request) + + assert isinstance(output, DecryptMaterialsInput) + assert output.algorithm_suite_id == mock_algorithm_id + assert output.commitment_policy == mock_commitment_policy + assert output.encryption_context == mock_decryption_materials_request.encryption_context + + assert len(output.encrypted_data_keys) == len(mock_decryption_materials_request.encrypted_data_keys) + for i in range(len(output.encrypted_data_keys)): + # Assume input[i] == output[i], seems to work + output_edk = output.encrypted_data_keys[i] + input_edk = mock_decryption_materials_request[i] + assert output_edk.key_provider_id == input_edk.key_provider.provider_id + assert output_edk.key_provider_info == input_edk.key_provider.key_info + assert output_edk.ciphertext == input_edk.encrypted_data_key diff --git a/test/unit/mpl/test_material_managers_mpl_materials.py b/test/unit/mpl/test_material_managers_mpl_materials.py new file mode 100644 index 000000000..250efeb7e --- /dev/null +++ b/test/unit/mpl/test_material_managers_mpl_materials.py @@ -0,0 +1,221 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" + +import pytest +from mock import MagicMock, patch, PropertyMock +from typing import Dict, List + +from aws_encryption_sdk.identifiers import CommitmentPolicy +import aws_encryption_sdk.materials_managers.mpl.materials +from aws_encryption_sdk.materials_managers.mpl.materials import ( + MPLEncryptionMaterials, + MPLDecryptionMaterials, +) +from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite + +pytestmark = [pytest.mark.unit, pytest.mark.local] + + +# Check if MPL is installed, and skip tests based on its installation status +# Ideally, this logic would be based on mocking imports and testing logic, +# but doing that introduces errors that cause other tests to fail. +try: + from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException + from aws_cryptographic_materialproviders.mpl.models import ( + AlgorithmSuiteIdESDK, + CommitmentPolicyESDK, + DecryptMaterialsInput, + DecryptionMaterials as MPL_DecryptionMaterials, + EncryptedDataKey as MPL_EncryptedDataKey, + EncryptionMaterials as MPL_EncryptionMaterials, + GetEncryptionMaterialsInput, + GetEncryptionMaterialsOutput, + ) + from aws_cryptographic_materialproviders.mpl.references import ( + ICryptographicMaterialsManager + ) + HAS_MPL = True + + mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) + mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) + +except ImportError: + HAS_MPL = False + + # Ensure references to these mocks exist, even if they aren't used in a non-MPL context + mock_mpl_cmm = None + mock_mpl_encryption_materials = None + mock_mpl_decrypt_materials = None + +from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError +from aws_encryption_sdk.materials_managers import ( + EncryptionMaterialsRequest, + DecryptionMaterialsRequest, +) + + +mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) +mock_encryption_materials_handler = MagicMock(__class__=MPLEncryptionMaterials) +mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) + +@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") +def test_GIVEN_test_has_mpl_is_False_THEN_cmm_has_mpl_is_False(): + """If the MPL IS NOT installed in the runtime environment, + assert the cmm has _HAS_MPL set to False""" + + assert hasattr(aws_encryption_sdk.materials_managers.mpl.materials, "_HAS_MPL") + assert aws_encryption_sdk.materials_managers.mpl.materials._HAS_MPL is False + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_test_has_mpl_is_True_THEN_cmm_has_mpl_is_True(): + """If the MPL IS installed in the runtime environment, + assert the cmm has _HAS_MPL set to True""" + + assert hasattr(aws_encryption_sdk.materials_managers.mpl.materials, "_HAS_MPL") + assert aws_encryption_sdk.materials_managers.mpl.materials._HAS_MPL is True + + +@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_THEN_raise_ImportError(): + with pytest.raises(ImportError): + MPLEncryptionMaterials(mpl_materials="doesn't matter") + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_valid_mpl_cmm_THEN_return_new_MPLCMMHandler(): + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + + assert mpl_encryption_materials.mpl_materials == mock_mpl_encryption_materials + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_invalid_mpl_cmm_THEN_raise_ValueError(): + with pytest.raises(ValueError): + MPLEncryptionMaterials(mpl_materials="not a valid mpl_materials") + +def test_mpl_to_native(): + some_mpl_algorithm_id = "0x1234" # Not a real algorithm ID, but fits the format + + native_output = aws_encryption_sdk.materials_managers.mpl.materials._mpl_algorithm_id_to_native_algorithm_id( + some_mpl_algorithm_id + ) + + assert native_output == 0x1234 + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +@patch("aws_encryption_sdk.materials_managers.mpl.materials._mpl_algorithm_id_to_native_algorithm_id") +@patch("aws_encryption_sdk.materials_managers.mpl.materials.AlgorithmSuite.get_by_id") +def test_GIVEN_valid_mpl_algorithm_id_WHEN_get_algorithm_THEN_valid_native_algorithm_id( + mock_algorithm, + mock_native_algorithm_id, +): + # Mock valid conversion from MPL to native algorithm ID + mock_native_algorithm_id.return_value = 0x1234 + + # Mock valid lookup in native AlgorithmSuite lookup + mock_algorithm.return_value = MagicMock(__class__=AlgorithmSuite) + + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + output = mpl_encryption_materials.algorithm + assert output == mock_algorithm() # property calls automatically, we need to call the mock + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GecTHEN_valid_native_algorithm_id(): + mock_encryption_context = MagicMock(__class__=Dict[str, str]) + mock_mpl_encryption_materials.encryption_context = mock_encryption_context + + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + output = mpl_encryption_materials.encryption_context + + assert output == mock_encryption_context + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GecTHEN_valid_nativefadsf_algorithm_id(): + mock_edk = MagicMock(__class__=MPL_EncryptedDataKey) + mock_mpl_key_provider_id = MagicMock(__class__=str) + mock_edk.key_provider_id = mock_mpl_key_provider_id + mock_mpl_key_provider_info = MagicMock(__class__=bytes) + mock_edk.key_provider_info = mock_mpl_key_provider_info + mock_mpl_ciphertext = MagicMock(__class__=bytes) + mock_edk.ciphertext = mock_mpl_ciphertext + + mock_edks = [ mock_edk ] + mock_mpl_encryption_materials.encrypted_data_keys = mock_edks + + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + output = mpl_encryption_materials.encrypted_data_keys + output_as_list = list(output) + + assert len(output_as_list) == len(mock_edks) + for i in range(len(output_as_list)): + # assume output[i] corresponds to input[i] + native_edk = output_as_list[i] + mpl_edk = mock_edks[i] + + assert native_edk.encrypted_data_key == mpl_edk.ciphertext + assert native_edk.key_provider.provider_id == mpl_edk.key_provider_id + assert native_edk.key_provider.key_info == mpl_edk.key_provider_info + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GecTHEN_valid_nativefadsffadsfa_algorithm_id(): + mock_data_key = MagicMock(__class__=bytes) + mock_mpl_encryption_materials.plaintext_data_key = mock_data_key + + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + output = mpl_encryption_materials.data_encryption_key + + assert output.key_provider.provider_id == "" + assert output.key_provider.key_info == b"" + assert output.data_key == mock_data_key + assert output.encrypted_data_key == b"" + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GecTHEN_valid_nativefasdfasdffadsf_algorithm_id(): + mock_signing_key = MagicMock(__class__=bytes) + mock_mpl_encryption_materials.signing_key = mock_signing_key + + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + output = mpl_encryption_materials.signing_key + + assert output == mock_signing_key + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GecTHEN_valid_nativeffasdfasdadsffadsfa_algorithm_id(): + mock_data_key = MagicMock(__class__=bytes) + mock_mpl_decrypt_materials.plaintext_data_key = mock_data_key + + mpl_decryption_materials = MPLDecryptionMaterials(mpl_materials=mock_mpl_decrypt_materials) + output = mpl_decryption_materials.data_key + + assert output.key_provider.provider_id == "" + assert output.key_provider.key_info == b"" + assert output.data_key == mock_data_key + assert output.encrypted_data_key == b"" + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_GecTHEN_validadsfasdf_nativefasdfasdffadsf_algorithm_id(): + mock_verification_key = MagicMock(__class__=bytes) + mock_mpl_decrypt_materials.verification_key = mock_verification_key + + mpl_decryption_materials = MPLDecryptionMaterials(mpl_materials=mock_mpl_decrypt_materials) + output = mpl_decryption_materials.verification_key + + assert output == mock_verification_key diff --git a/tox.ini b/tox.ini index 72e8ec9fa..346e4fae0 100644 --- a/tox.ini +++ b/tox.ini @@ -84,23 +84,21 @@ deps = commands = local: {[testenv:base-command]commands} test/ -m local --ignore test/unit/mpl/ # MPL unit tests require the MPL to be installed - mpllocal: {[testenv:base-command]commands} test/unit/mpl/ -m local - integ: {[testenv:base-command]commands} test/ -m integ --ignore test/unit/mpl/ - # No MPL-specific integ tests - accept: {[testenv:base-command]commands} test/ -m accept --ignore test/unit/mpl/ - # No MPL-specific accept tests + mpllocal: {[testenv:base-command]commands} test/ -m local + integ: {[testenv:base-command]commands} test/ -m integ + accept: {[testenv:base-command]commands} test/ -m accept examples: {[testenv:base-command]commands} examples/test/ -m examples --ignore examples/test/keyrings/ # MPL keyring examples require a special IAM role; run these separately under a separate set of permissions mplexamples: {[testenv:base-command]commands} examples/test/keyrings -m examples all: {[testenv:base-command]commands} test/ examples/test/ --ignore test/unit/mpl/ --ignore examples/test/keyrings/ - mplall: {[testenv:base-command]commands} test/unit/mpl/ examples/test/keyrings/ + mplall: {[testenv:base-command]commands} test/ examples/test/ manual: {[testenv:base-command]commands} # Run code coverage on the unit tests [testenv:coverage] commands = {[testenv:base-command]commands} --cov aws_encryption_sdk test/ -m local --ignore test/unit/mpl/ [testenv:mplcoverage-mpl] -commands = {[testenv:base-command]commands} --cov aws_encryption_sdk test/unit/mpl/ -m local +commands = {[testenv:base-command]commands} --cov aws_encryption_sdk test/ -m local # Verify that local tests work without environment variables present [testenv:nocmk] From 5ec46687b47ab907cc4a53ca9cc18a4f677c65e6 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 10:33:22 -0800 Subject: [PATCH 108/422] refactor tests --- .../materials_managers/mpl/cmm.py | 32 +- .../materials_managers/mpl/materials.py | 22 +- test/unit/mpl/__init__.py | 12 - .../mpl/test_material_managers_mpl_cmm.py | 278 ------------------ .../test_material_managers_mpl_materials.py | 221 -------------- test/unit/test_material_managers_mpl_cmm.py | 278 ------------------ .../test_material_managers_mpl_materials.py | 221 -------------- tox.ini | 6 +- 8 files changed, 20 insertions(+), 1050 deletions(-) delete mode 100644 test/unit/mpl/__init__.py delete mode 100644 test/unit/mpl/test_material_managers_mpl_cmm.py delete mode 100644 test/unit/mpl/test_material_managers_mpl_materials.py delete mode 100644 test/unit/test_material_managers_mpl_cmm.py delete mode 100644 test/unit/test_material_managers_mpl_materials.py diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index cd789b994..1e3e3fb34 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -1,23 +1,16 @@ """Retrieves encryption/decryption materials from the MPL.""" -# These dependencies are only loaded if you install the MPL. -try: - # pylint seems to struggle with this conditional import - # pylint: disable=unused-import - from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException - from aws_cryptographic_materialproviders.mpl.models import ( - AlgorithmSuiteIdESDK, - CommitmentPolicyESDK, - DecryptMaterialsInput, - DecryptMaterialsOutput, - EncryptedDataKey as MPL_EncryptedDataKey, - GetEncryptionMaterialsInput, - GetEncryptionMaterialsOutput, - ) - from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager - _HAS_MPL = True -except ImportError: - _HAS_MPL = False +from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException +from aws_cryptographic_materialproviders.mpl.models import ( + AlgorithmSuiteIdESDK, + CommitmentPolicyESDK, + DecryptMaterialsInput, + DecryptMaterialsOutput, + EncryptedDataKey as MPL_EncryptedDataKey, + GetEncryptionMaterialsInput, + GetEncryptionMaterialsOutput, +) +from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager from typing import List @@ -46,9 +39,6 @@ def __init__( Create MPLCMMHandler. :param mpl_cmm: Underlying MPL cryptographic materials manager """ - if not _HAS_MPL: - raise ImportError("You MUST install the aws-cryptographic-material-providers " - f"library to create an instance of {MPLCMMHandler}") if isinstance(mpl_cmm, ICryptographicMaterialsManager): self.mpl_cmm = mpl_cmm else: diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index bd4b5f729..1ea2a199d 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -1,14 +1,10 @@ """Provides encryption/decryption materials from an underlying materials provider.""" -# These dependencies are only loaded if you install the MPL. -try: - from aws_cryptographic_materialproviders.mpl.models import ( - DecryptionMaterials as MPL_DecryptionMaterials, - EncryptedDataKey as MPL_EncryptedDataKey, - EncryptionMaterials as MPL_EncryptionMaterials, - ) - _HAS_MPL = True -except ImportError: - _HAS_MPL = False + +from aws_cryptographic_materialproviders.mpl.models import ( + DecryptionMaterials as MPL_DecryptionMaterials, + EncryptedDataKey as MPL_EncryptedDataKey, + EncryptionMaterials as MPL_EncryptionMaterials, +) from typing import Dict, List, Set @@ -42,9 +38,6 @@ def __init__( Create MPLEncryptionMaterials. :param materials: Underlying encryption materials """ - if not _HAS_MPL: - raise ImportError("You MUST install the aws-cryptographic-material-providers " - f"library to create an instance of {MPLEncryptionMaterials}") if isinstance(mpl_materials, MPL_EncryptionMaterials): self.mpl_materials = mpl_materials else: @@ -115,9 +108,6 @@ def __init__( Create MPLDecryptionMaterials. :param materials: Underlying decryption materials """ - if not _HAS_MPL: - raise ImportError("You MUST install the aws-cryptographic-material-providers " - f"library to create an instance of {MPLDecryptionMaterials}") if isinstance(mpl_materials, MPL_DecryptionMaterials): self.mpl_materials = mpl_materials else: diff --git a/test/unit/mpl/__init__.py b/test/unit/mpl/__init__.py deleted file mode 100644 index 53a960891..000000000 --- a/test/unit/mpl/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. diff --git a/test/unit/mpl/test_material_managers_mpl_cmm.py b/test/unit/mpl/test_material_managers_mpl_cmm.py deleted file mode 100644 index 77bf5502d..000000000 --- a/test/unit/mpl/test_material_managers_mpl_cmm.py +++ /dev/null @@ -1,278 +0,0 @@ -# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. -"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" - -import pytest -from mock import MagicMock, patch - - -from aws_encryption_sdk.identifiers import CommitmentPolicy -import aws_encryption_sdk.materials_managers.mpl.cmm -from aws_encryption_sdk.materials_managers.mpl.cmm import MPLCMMHandler -from aws_encryption_sdk.materials_managers.mpl.materials import ( - MPLEncryptionMaterials, - MPLDecryptionMaterials, -) - -pytestmark = [pytest.mark.unit, pytest.mark.local] - - -# Check if MPL is installed, and skip tests based on its installation status -# Ideally, this logic would be based on mocking imports and testing logic, -# but doing that introduces errors that cause other tests to fail. -try: - from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException - from aws_cryptographic_materialproviders.mpl.models import ( - AlgorithmSuiteIdESDK, - CommitmentPolicyESDK, - DecryptMaterialsInput, - DecryptionMaterials as MPL_DecryptionMaterials, - EncryptionMaterials as MPL_EncryptionMaterials, - GetEncryptionMaterialsInput, - GetEncryptionMaterialsOutput, - ) - from aws_cryptographic_materialproviders.mpl.references import ( - ICryptographicMaterialsManager - ) - HAS_MPL = True - - mock_mpl_cmm = MagicMock(__class__=ICryptographicMaterialsManager) - mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) - mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) - -except ImportError: - HAS_MPL = False - - # Ensure references to these mocks exist, even if they aren't used in a non-MPL context - mock_mpl_cmm = None - mock_mpl_encryption_materials = None - mock_mpl_decrypt_materials = None - -from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError -from aws_encryption_sdk.materials_managers import ( - EncryptionMaterialsRequest, - DecryptionMaterialsRequest, -) - - -mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) -mock_encryption_materials_handler = MagicMock(__class__=MPLEncryptionMaterials) -mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) - -@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") -def test_GIVEN_test_has_mpl_is_False_THEN_cmm_has_mpl_is_False(): - """If the MPL IS NOT installed in the runtime environment, - assert the cmm has _HAS_MPL set to False""" - - assert hasattr(aws_encryption_sdk.materials_managers.mpl.cmm, "_HAS_MPL") - assert aws_encryption_sdk.materials_managers.mpl.cmm._HAS_MPL is False - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_test_has_mpl_is_True_THEN_cmm_has_mpl_is_True(): - """If the MPL IS installed in the runtime environment, - assert the cmm has _HAS_MPL set to True""" - - assert hasattr(aws_encryption_sdk.materials_managers.mpl.cmm, "_HAS_MPL") - assert aws_encryption_sdk.materials_managers.mpl.cmm._HAS_MPL is True - - -@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_THEN_raise_ImportError(): - with pytest.raises(ImportError): - MPLCMMHandler(mpl_cmm="doesn't matter") - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_valid_mpl_cmm_THEN_return_new_MPLCMMHandler(): - mpl_cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) - - assert mpl_cmm_handler.mpl_cmm == mock_mpl_cmm - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_invalid_mpl_cmm_THEN_raise_ValueError(): - with pytest.raises(ValueError): - MPLCMMHandler(mpl_cmm="not a valid mpl_cmm") - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -@patch.object(mock_mpl_cmm, "get_encryption_materials") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_get_encryption_materials") -def test_GIVEN_valid_request_WHEN_call_get_encryption_materials_THEN_return_MPLEncryptionMaterials( - mock_native_to_mpl_get_encryption_materials, - mock_get_encryption_materials, -): - - # Mock: mpl_cmm.get_encryption_materials returns mock MPL encryption materials - mock_get_encryption_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) - mock_get_encryption_materials_output.encryption_materials = mock_mpl_encryption_materials - mock_get_encryption_materials.return_value = mock_get_encryption_materials_output - - # Mock: CMMHandler._native_to_mpl_get_encryption_materials creates a GetEncryptionMaterialsInput - mock_get_encryption_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) - mock_native_to_mpl_get_encryption_materials.return_value = mock_get_encryption_materials_input - - cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) - test = cmm_handler.get_encryption_materials(mock_encryption_materials_request) - - # Verify cmm_handler returns MPLEncryptionMaterials - assert isinstance(test, MPLEncryptionMaterials) - # Verify returned EncryptionMaterialsHandler uses the output of `get_encryption_materials` - assert test.mpl_materials == mock_mpl_encryption_materials - # Verify we actually called `get_encryption_materials` - mock_mpl_cmm.get_encryption_materials.assert_called_once_with(mock_get_encryption_materials_input) - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") -def test_GIVEN_get_encryption_materials_raises_MPL_Exception_WHEN_call_get_encryption_materials_THEN_raise_ESDK_Exception( - _ -): - with pytest.raises(AWSEncryptionSDKClientError): - with patch.object(mock_mpl_cmm, "get_encryption_materials", - side_effect=AwsCryptographicMaterialProvidersException("any")): - - cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) - cmm_handler.get_encryption_materials(mock_encryption_materials_request) - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") -def test_GIVEN_native_to_mpl_commmitment_policy_returns_valid_policy_WHEN_call_native_to_mpl_get_encryption_materials_THEN_returns_GetEncryptionMaterialsInput( - mock_mpl_commitment_policy -): - mock_commitment_policy = MagicMock(__class__=CommitmentPolicyESDK) - mock_mpl_commitment_policy.return_value = mock_commitment_policy - - output = MPLCMMHandler._native_to_mpl_get_encryption_materials(mock_encryption_materials_request) - - # verify correctness of returned value - assert isinstance(output, GetEncryptionMaterialsInput) - assert output.encryption_context == mock_encryption_materials_request.encryption_context - assert output.commitment_policy == mock_commitment_policy - assert output.max_plaintext_length == mock_encryption_materials_request.plaintext_length - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_FORBID_ENCRYPT_ALLOW_DECRYPT(): - native_commitment_policy = CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT - - output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) - - assert isinstance(output, CommitmentPolicyESDK) - assert output.value == "FORBID_ENCRYPT_ALLOW_DECRYPT" - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_ALLOW_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_REQUIRE_ENCRYPT_ALLOW_DECRYPT(): - native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT - - output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) - - assert isinstance(output, CommitmentPolicyESDK) - assert output.value == "REQUIRE_ENCRYPT_ALLOW_DECRYPT" - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_REQUIRE_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_REQUIRE_ENCRYPT_REQUIRE_DECRYPT(): - native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT - - output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) - - assert isinstance(output, CommitmentPolicyESDK) - assert output.value == "REQUIRE_ENCRYPT_REQUIRE_DECRYPT" - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_CommitmentPolicy_unrecognized_WHEN_call_native_to_mpl_commmitment_policyTHEN_raise_ValueError(): - native_commitment_policy = "not a commitment policy" - - with pytest.raises(ValueError): - MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -@patch.object(mock_mpl_cmm, "decrypt_materials") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._create_mpl_decrypt_materials_input_from_request") -def test_GIVEN_valid_request_WHEN_call_decrypt_materials_THEN_return_MPLDecryptionMaterials( - mock_native_to_mpl_decrypt_materials, - mock_get_encryption_materials, -): - - # Mock: mpl_cmm.get_decryption_materials returns mock MPL decryption materials - mock_decrypt_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) - mock_decrypt_materials_output.decryption_materials = mock_mpl_decrypt_materials - mock_get_encryption_materials.return_value = mock_decrypt_materials_output - - # Mock: CMMHandler._create_mpl_decrypt_materials_input_from_request creates a DecryptMaterialsInput - mock_decrypt_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) - mock_native_to_mpl_decrypt_materials.return_value = mock_decrypt_materials_input - - cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) - output = cmm_handler.decrypt_materials(mock_decryption_materials_request) - - # Verify cmm_handler returns MPLDecryptionMaterials - assert isinstance(output, MPLDecryptionMaterials) - # Verify returned MPLDecryptionMaterials uses the output of `decrypt_materials` - assert output.mpl_materials == mock_mpl_decrypt_materials - # Verify we actually called `decrypt_materials` - mock_mpl_cmm.decrypt_materials.assert_called_once_with(mock_decrypt_materials_input) - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._create_mpl_decrypt_materials_input_from_request") -def test_GIVEN_decrypt_materials_raises_MPL_Exception_WHEN_call_decrypt_materials_THEN_raise_ESDK_Exception( - _ -): - with pytest.raises(AWSEncryptionSDKClientError): - with patch.object(mock_mpl_cmm, "decrypt_materials", - side_effect=AwsCryptographicMaterialProvidersException("any")): - - cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) - cmm_handler.decrypt_materials(mock_decryption_materials_request) - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_WHEN_call_native_algorithm_id_to_mpl_algorithm_id_THEN_returns_valid_AlgorithmSuiteIdESDK(): - some_native_algorithm_id = 0x0000 # Not a real algorithm ID, but fits the format - - mpl_output = MPLCMMHandler._native_algorithm_id_to_mpl_algorithm_id( - some_native_algorithm_id - ) - - assert isinstance(mpl_output, AlgorithmSuiteIdESDK) - assert mpl_output.value == "0x0000" - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_algorithm_id_to_mpl_algorithm_id") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") -def test__create_mpl_decrypt_materials_input_from_request( - mock_mpl_commitment_policy, - mock_mpl_algorithm_id, -): - mock_algorithm_id = "0x1234" # Some fake algorithm ID that fits the format - mock_mpl_algorithm_id.return_value = mock_algorithm_id - mock_commitment_policy = MagicMock(__class__=CommitmentPolicyESDK) - mock_mpl_commitment_policy.return_value = mock_commitment_policy - - # mock_decryption_materials_request.algorithm = - - output = MPLCMMHandler._create_mpl_decrypt_materials_input_from_request(mock_decryption_materials_request) - - assert isinstance(output, DecryptMaterialsInput) - assert output.algorithm_suite_id == mock_algorithm_id - assert output.commitment_policy == mock_commitment_policy - assert output.encryption_context == mock_decryption_materials_request.encryption_context - - assert len(output.encrypted_data_keys) == len(mock_decryption_materials_request.encrypted_data_keys) - for i in range(len(output.encrypted_data_keys)): - # Assume input[i] == output[i], seems to work - output_edk = output.encrypted_data_keys[i] - input_edk = mock_decryption_materials_request[i] - assert output_edk.key_provider_id == input_edk.key_provider.provider_id - assert output_edk.key_provider_info == input_edk.key_provider.key_info - assert output_edk.ciphertext == input_edk.encrypted_data_key diff --git a/test/unit/mpl/test_material_managers_mpl_materials.py b/test/unit/mpl/test_material_managers_mpl_materials.py deleted file mode 100644 index 250efeb7e..000000000 --- a/test/unit/mpl/test_material_managers_mpl_materials.py +++ /dev/null @@ -1,221 +0,0 @@ -# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. -"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" - -import pytest -from mock import MagicMock, patch, PropertyMock -from typing import Dict, List - -from aws_encryption_sdk.identifiers import CommitmentPolicy -import aws_encryption_sdk.materials_managers.mpl.materials -from aws_encryption_sdk.materials_managers.mpl.materials import ( - MPLEncryptionMaterials, - MPLDecryptionMaterials, -) -from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite - -pytestmark = [pytest.mark.unit, pytest.mark.local] - - -# Check if MPL is installed, and skip tests based on its installation status -# Ideally, this logic would be based on mocking imports and testing logic, -# but doing that introduces errors that cause other tests to fail. -try: - from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException - from aws_cryptographic_materialproviders.mpl.models import ( - AlgorithmSuiteIdESDK, - CommitmentPolicyESDK, - DecryptMaterialsInput, - DecryptionMaterials as MPL_DecryptionMaterials, - EncryptedDataKey as MPL_EncryptedDataKey, - EncryptionMaterials as MPL_EncryptionMaterials, - GetEncryptionMaterialsInput, - GetEncryptionMaterialsOutput, - ) - from aws_cryptographic_materialproviders.mpl.references import ( - ICryptographicMaterialsManager - ) - HAS_MPL = True - - mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) - mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) - -except ImportError: - HAS_MPL = False - - # Ensure references to these mocks exist, even if they aren't used in a non-MPL context - mock_mpl_cmm = None - mock_mpl_encryption_materials = None - mock_mpl_decrypt_materials = None - -from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError -from aws_encryption_sdk.materials_managers import ( - EncryptionMaterialsRequest, - DecryptionMaterialsRequest, -) - - -mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) -mock_encryption_materials_handler = MagicMock(__class__=MPLEncryptionMaterials) -mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) - -@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") -def test_GIVEN_test_has_mpl_is_False_THEN_cmm_has_mpl_is_False(): - """If the MPL IS NOT installed in the runtime environment, - assert the cmm has _HAS_MPL set to False""" - - assert hasattr(aws_encryption_sdk.materials_managers.mpl.materials, "_HAS_MPL") - assert aws_encryption_sdk.materials_managers.mpl.materials._HAS_MPL is False - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_test_has_mpl_is_True_THEN_cmm_has_mpl_is_True(): - """If the MPL IS installed in the runtime environment, - assert the cmm has _HAS_MPL set to True""" - - assert hasattr(aws_encryption_sdk.materials_managers.mpl.materials, "_HAS_MPL") - assert aws_encryption_sdk.materials_managers.mpl.materials._HAS_MPL is True - - -@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_THEN_raise_ImportError(): - with pytest.raises(ImportError): - MPLEncryptionMaterials(mpl_materials="doesn't matter") - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_valid_mpl_cmm_THEN_return_new_MPLCMMHandler(): - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) - - assert mpl_encryption_materials.mpl_materials == mock_mpl_encryption_materials - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_invalid_mpl_cmm_THEN_raise_ValueError(): - with pytest.raises(ValueError): - MPLEncryptionMaterials(mpl_materials="not a valid mpl_materials") - -def test_mpl_to_native(): - some_mpl_algorithm_id = "0x1234" # Not a real algorithm ID, but fits the format - - native_output = aws_encryption_sdk.materials_managers.mpl.materials._mpl_algorithm_id_to_native_algorithm_id( - some_mpl_algorithm_id - ) - - assert native_output == 0x1234 - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -@patch("aws_encryption_sdk.materials_managers.mpl.materials._mpl_algorithm_id_to_native_algorithm_id") -@patch("aws_encryption_sdk.materials_managers.mpl.materials.AlgorithmSuite.get_by_id") -def test_GIVEN_valid_mpl_algorithm_id_WHEN_get_algorithm_THEN_valid_native_algorithm_id( - mock_algorithm, - mock_native_algorithm_id, -): - # Mock valid conversion from MPL to native algorithm ID - mock_native_algorithm_id.return_value = 0x1234 - - # Mock valid lookup in native AlgorithmSuite lookup - mock_algorithm.return_value = MagicMock(__class__=AlgorithmSuite) - - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) - output = mpl_encryption_materials.algorithm - assert output == mock_algorithm() # property calls automatically, we need to call the mock - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GecTHEN_valid_native_algorithm_id(): - mock_encryption_context = MagicMock(__class__=Dict[str, str]) - mock_mpl_encryption_materials.encryption_context = mock_encryption_context - - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) - output = mpl_encryption_materials.encryption_context - - assert output == mock_encryption_context - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GecTHEN_valid_nativefadsf_algorithm_id(): - mock_edk = MagicMock(__class__=MPL_EncryptedDataKey) - mock_mpl_key_provider_id = MagicMock(__class__=str) - mock_edk.key_provider_id = mock_mpl_key_provider_id - mock_mpl_key_provider_info = MagicMock(__class__=bytes) - mock_edk.key_provider_info = mock_mpl_key_provider_info - mock_mpl_ciphertext = MagicMock(__class__=bytes) - mock_edk.ciphertext = mock_mpl_ciphertext - - mock_edks = [ mock_edk ] - mock_mpl_encryption_materials.encrypted_data_keys = mock_edks - - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) - output = mpl_encryption_materials.encrypted_data_keys - output_as_list = list(output) - - assert len(output_as_list) == len(mock_edks) - for i in range(len(output_as_list)): - # assume output[i] corresponds to input[i] - native_edk = output_as_list[i] - mpl_edk = mock_edks[i] - - assert native_edk.encrypted_data_key == mpl_edk.ciphertext - assert native_edk.key_provider.provider_id == mpl_edk.key_provider_id - assert native_edk.key_provider.key_info == mpl_edk.key_provider_info - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GecTHEN_valid_nativefadsffadsfa_algorithm_id(): - mock_data_key = MagicMock(__class__=bytes) - mock_mpl_encryption_materials.plaintext_data_key = mock_data_key - - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) - output = mpl_encryption_materials.data_encryption_key - - assert output.key_provider.provider_id == "" - assert output.key_provider.key_info == b"" - assert output.data_key == mock_data_key - assert output.encrypted_data_key == b"" - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GecTHEN_valid_nativefasdfasdffadsf_algorithm_id(): - mock_signing_key = MagicMock(__class__=bytes) - mock_mpl_encryption_materials.signing_key = mock_signing_key - - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) - output = mpl_encryption_materials.signing_key - - assert output == mock_signing_key - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GecTHEN_valid_nativeffasdfasdadsffadsfa_algorithm_id(): - mock_data_key = MagicMock(__class__=bytes) - mock_mpl_decrypt_materials.plaintext_data_key = mock_data_key - - mpl_decryption_materials = MPLDecryptionMaterials(mpl_materials=mock_mpl_decrypt_materials) - output = mpl_decryption_materials.data_key - - assert output.key_provider.provider_id == "" - assert output.key_provider.key_info == b"" - assert output.data_key == mock_data_key - assert output.encrypted_data_key == b"" - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GecTHEN_validadsfasdf_nativefasdfasdffadsf_algorithm_id(): - mock_verification_key = MagicMock(__class__=bytes) - mock_mpl_decrypt_materials.verification_key = mock_verification_key - - mpl_decryption_materials = MPLDecryptionMaterials(mpl_materials=mock_mpl_decrypt_materials) - output = mpl_decryption_materials.verification_key - - assert output == mock_verification_key diff --git a/test/unit/test_material_managers_mpl_cmm.py b/test/unit/test_material_managers_mpl_cmm.py deleted file mode 100644 index 77bf5502d..000000000 --- a/test/unit/test_material_managers_mpl_cmm.py +++ /dev/null @@ -1,278 +0,0 @@ -# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. -"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" - -import pytest -from mock import MagicMock, patch - - -from aws_encryption_sdk.identifiers import CommitmentPolicy -import aws_encryption_sdk.materials_managers.mpl.cmm -from aws_encryption_sdk.materials_managers.mpl.cmm import MPLCMMHandler -from aws_encryption_sdk.materials_managers.mpl.materials import ( - MPLEncryptionMaterials, - MPLDecryptionMaterials, -) - -pytestmark = [pytest.mark.unit, pytest.mark.local] - - -# Check if MPL is installed, and skip tests based on its installation status -# Ideally, this logic would be based on mocking imports and testing logic, -# but doing that introduces errors that cause other tests to fail. -try: - from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException - from aws_cryptographic_materialproviders.mpl.models import ( - AlgorithmSuiteIdESDK, - CommitmentPolicyESDK, - DecryptMaterialsInput, - DecryptionMaterials as MPL_DecryptionMaterials, - EncryptionMaterials as MPL_EncryptionMaterials, - GetEncryptionMaterialsInput, - GetEncryptionMaterialsOutput, - ) - from aws_cryptographic_materialproviders.mpl.references import ( - ICryptographicMaterialsManager - ) - HAS_MPL = True - - mock_mpl_cmm = MagicMock(__class__=ICryptographicMaterialsManager) - mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) - mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) - -except ImportError: - HAS_MPL = False - - # Ensure references to these mocks exist, even if they aren't used in a non-MPL context - mock_mpl_cmm = None - mock_mpl_encryption_materials = None - mock_mpl_decrypt_materials = None - -from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError -from aws_encryption_sdk.materials_managers import ( - EncryptionMaterialsRequest, - DecryptionMaterialsRequest, -) - - -mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) -mock_encryption_materials_handler = MagicMock(__class__=MPLEncryptionMaterials) -mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) - -@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") -def test_GIVEN_test_has_mpl_is_False_THEN_cmm_has_mpl_is_False(): - """If the MPL IS NOT installed in the runtime environment, - assert the cmm has _HAS_MPL set to False""" - - assert hasattr(aws_encryption_sdk.materials_managers.mpl.cmm, "_HAS_MPL") - assert aws_encryption_sdk.materials_managers.mpl.cmm._HAS_MPL is False - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_test_has_mpl_is_True_THEN_cmm_has_mpl_is_True(): - """If the MPL IS installed in the runtime environment, - assert the cmm has _HAS_MPL set to True""" - - assert hasattr(aws_encryption_sdk.materials_managers.mpl.cmm, "_HAS_MPL") - assert aws_encryption_sdk.materials_managers.mpl.cmm._HAS_MPL is True - - -@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_THEN_raise_ImportError(): - with pytest.raises(ImportError): - MPLCMMHandler(mpl_cmm="doesn't matter") - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_valid_mpl_cmm_THEN_return_new_MPLCMMHandler(): - mpl_cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) - - assert mpl_cmm_handler.mpl_cmm == mock_mpl_cmm - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_invalid_mpl_cmm_THEN_raise_ValueError(): - with pytest.raises(ValueError): - MPLCMMHandler(mpl_cmm="not a valid mpl_cmm") - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -@patch.object(mock_mpl_cmm, "get_encryption_materials") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_get_encryption_materials") -def test_GIVEN_valid_request_WHEN_call_get_encryption_materials_THEN_return_MPLEncryptionMaterials( - mock_native_to_mpl_get_encryption_materials, - mock_get_encryption_materials, -): - - # Mock: mpl_cmm.get_encryption_materials returns mock MPL encryption materials - mock_get_encryption_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) - mock_get_encryption_materials_output.encryption_materials = mock_mpl_encryption_materials - mock_get_encryption_materials.return_value = mock_get_encryption_materials_output - - # Mock: CMMHandler._native_to_mpl_get_encryption_materials creates a GetEncryptionMaterialsInput - mock_get_encryption_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) - mock_native_to_mpl_get_encryption_materials.return_value = mock_get_encryption_materials_input - - cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) - test = cmm_handler.get_encryption_materials(mock_encryption_materials_request) - - # Verify cmm_handler returns MPLEncryptionMaterials - assert isinstance(test, MPLEncryptionMaterials) - # Verify returned EncryptionMaterialsHandler uses the output of `get_encryption_materials` - assert test.mpl_materials == mock_mpl_encryption_materials - # Verify we actually called `get_encryption_materials` - mock_mpl_cmm.get_encryption_materials.assert_called_once_with(mock_get_encryption_materials_input) - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") -def test_GIVEN_get_encryption_materials_raises_MPL_Exception_WHEN_call_get_encryption_materials_THEN_raise_ESDK_Exception( - _ -): - with pytest.raises(AWSEncryptionSDKClientError): - with patch.object(mock_mpl_cmm, "get_encryption_materials", - side_effect=AwsCryptographicMaterialProvidersException("any")): - - cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) - cmm_handler.get_encryption_materials(mock_encryption_materials_request) - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") -def test_GIVEN_native_to_mpl_commmitment_policy_returns_valid_policy_WHEN_call_native_to_mpl_get_encryption_materials_THEN_returns_GetEncryptionMaterialsInput( - mock_mpl_commitment_policy -): - mock_commitment_policy = MagicMock(__class__=CommitmentPolicyESDK) - mock_mpl_commitment_policy.return_value = mock_commitment_policy - - output = MPLCMMHandler._native_to_mpl_get_encryption_materials(mock_encryption_materials_request) - - # verify correctness of returned value - assert isinstance(output, GetEncryptionMaterialsInput) - assert output.encryption_context == mock_encryption_materials_request.encryption_context - assert output.commitment_policy == mock_commitment_policy - assert output.max_plaintext_length == mock_encryption_materials_request.plaintext_length - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_FORBID_ENCRYPT_ALLOW_DECRYPT(): - native_commitment_policy = CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT - - output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) - - assert isinstance(output, CommitmentPolicyESDK) - assert output.value == "FORBID_ENCRYPT_ALLOW_DECRYPT" - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_ALLOW_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_REQUIRE_ENCRYPT_ALLOW_DECRYPT(): - native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT - - output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) - - assert isinstance(output, CommitmentPolicyESDK) - assert output.value == "REQUIRE_ENCRYPT_ALLOW_DECRYPT" - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_REQUIRE_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_REQUIRE_ENCRYPT_REQUIRE_DECRYPT(): - native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT - - output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) - - assert isinstance(output, CommitmentPolicyESDK) - assert output.value == "REQUIRE_ENCRYPT_REQUIRE_DECRYPT" - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_CommitmentPolicy_unrecognized_WHEN_call_native_to_mpl_commmitment_policyTHEN_raise_ValueError(): - native_commitment_policy = "not a commitment policy" - - with pytest.raises(ValueError): - MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -@patch.object(mock_mpl_cmm, "decrypt_materials") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._create_mpl_decrypt_materials_input_from_request") -def test_GIVEN_valid_request_WHEN_call_decrypt_materials_THEN_return_MPLDecryptionMaterials( - mock_native_to_mpl_decrypt_materials, - mock_get_encryption_materials, -): - - # Mock: mpl_cmm.get_decryption_materials returns mock MPL decryption materials - mock_decrypt_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) - mock_decrypt_materials_output.decryption_materials = mock_mpl_decrypt_materials - mock_get_encryption_materials.return_value = mock_decrypt_materials_output - - # Mock: CMMHandler._create_mpl_decrypt_materials_input_from_request creates a DecryptMaterialsInput - mock_decrypt_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) - mock_native_to_mpl_decrypt_materials.return_value = mock_decrypt_materials_input - - cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) - output = cmm_handler.decrypt_materials(mock_decryption_materials_request) - - # Verify cmm_handler returns MPLDecryptionMaterials - assert isinstance(output, MPLDecryptionMaterials) - # Verify returned MPLDecryptionMaterials uses the output of `decrypt_materials` - assert output.mpl_materials == mock_mpl_decrypt_materials - # Verify we actually called `decrypt_materials` - mock_mpl_cmm.decrypt_materials.assert_called_once_with(mock_decrypt_materials_input) - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._create_mpl_decrypt_materials_input_from_request") -def test_GIVEN_decrypt_materials_raises_MPL_Exception_WHEN_call_decrypt_materials_THEN_raise_ESDK_Exception( - _ -): - with pytest.raises(AWSEncryptionSDKClientError): - with patch.object(mock_mpl_cmm, "decrypt_materials", - side_effect=AwsCryptographicMaterialProvidersException("any")): - - cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) - cmm_handler.decrypt_materials(mock_decryption_materials_request) - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_WHEN_call_native_algorithm_id_to_mpl_algorithm_id_THEN_returns_valid_AlgorithmSuiteIdESDK(): - some_native_algorithm_id = 0x0000 # Not a real algorithm ID, but fits the format - - mpl_output = MPLCMMHandler._native_algorithm_id_to_mpl_algorithm_id( - some_native_algorithm_id - ) - - assert isinstance(mpl_output, AlgorithmSuiteIdESDK) - assert mpl_output.value == "0x0000" - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_algorithm_id_to_mpl_algorithm_id") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") -def test__create_mpl_decrypt_materials_input_from_request( - mock_mpl_commitment_policy, - mock_mpl_algorithm_id, -): - mock_algorithm_id = "0x1234" # Some fake algorithm ID that fits the format - mock_mpl_algorithm_id.return_value = mock_algorithm_id - mock_commitment_policy = MagicMock(__class__=CommitmentPolicyESDK) - mock_mpl_commitment_policy.return_value = mock_commitment_policy - - # mock_decryption_materials_request.algorithm = - - output = MPLCMMHandler._create_mpl_decrypt_materials_input_from_request(mock_decryption_materials_request) - - assert isinstance(output, DecryptMaterialsInput) - assert output.algorithm_suite_id == mock_algorithm_id - assert output.commitment_policy == mock_commitment_policy - assert output.encryption_context == mock_decryption_materials_request.encryption_context - - assert len(output.encrypted_data_keys) == len(mock_decryption_materials_request.encrypted_data_keys) - for i in range(len(output.encrypted_data_keys)): - # Assume input[i] == output[i], seems to work - output_edk = output.encrypted_data_keys[i] - input_edk = mock_decryption_materials_request[i] - assert output_edk.key_provider_id == input_edk.key_provider.provider_id - assert output_edk.key_provider_info == input_edk.key_provider.key_info - assert output_edk.ciphertext == input_edk.encrypted_data_key diff --git a/test/unit/test_material_managers_mpl_materials.py b/test/unit/test_material_managers_mpl_materials.py deleted file mode 100644 index 250efeb7e..000000000 --- a/test/unit/test_material_managers_mpl_materials.py +++ /dev/null @@ -1,221 +0,0 @@ -# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. -"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" - -import pytest -from mock import MagicMock, patch, PropertyMock -from typing import Dict, List - -from aws_encryption_sdk.identifiers import CommitmentPolicy -import aws_encryption_sdk.materials_managers.mpl.materials -from aws_encryption_sdk.materials_managers.mpl.materials import ( - MPLEncryptionMaterials, - MPLDecryptionMaterials, -) -from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite - -pytestmark = [pytest.mark.unit, pytest.mark.local] - - -# Check if MPL is installed, and skip tests based on its installation status -# Ideally, this logic would be based on mocking imports and testing logic, -# but doing that introduces errors that cause other tests to fail. -try: - from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException - from aws_cryptographic_materialproviders.mpl.models import ( - AlgorithmSuiteIdESDK, - CommitmentPolicyESDK, - DecryptMaterialsInput, - DecryptionMaterials as MPL_DecryptionMaterials, - EncryptedDataKey as MPL_EncryptedDataKey, - EncryptionMaterials as MPL_EncryptionMaterials, - GetEncryptionMaterialsInput, - GetEncryptionMaterialsOutput, - ) - from aws_cryptographic_materialproviders.mpl.references import ( - ICryptographicMaterialsManager - ) - HAS_MPL = True - - mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) - mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) - -except ImportError: - HAS_MPL = False - - # Ensure references to these mocks exist, even if they aren't used in a non-MPL context - mock_mpl_cmm = None - mock_mpl_encryption_materials = None - mock_mpl_decrypt_materials = None - -from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError -from aws_encryption_sdk.materials_managers import ( - EncryptionMaterialsRequest, - DecryptionMaterialsRequest, -) - - -mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) -mock_encryption_materials_handler = MagicMock(__class__=MPLEncryptionMaterials) -mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) - -@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") -def test_GIVEN_test_has_mpl_is_False_THEN_cmm_has_mpl_is_False(): - """If the MPL IS NOT installed in the runtime environment, - assert the cmm has _HAS_MPL set to False""" - - assert hasattr(aws_encryption_sdk.materials_managers.mpl.materials, "_HAS_MPL") - assert aws_encryption_sdk.materials_managers.mpl.materials._HAS_MPL is False - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_test_has_mpl_is_True_THEN_cmm_has_mpl_is_True(): - """If the MPL IS installed in the runtime environment, - assert the cmm has _HAS_MPL set to True""" - - assert hasattr(aws_encryption_sdk.materials_managers.mpl.materials, "_HAS_MPL") - assert aws_encryption_sdk.materials_managers.mpl.materials._HAS_MPL is True - - -@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_THEN_raise_ImportError(): - with pytest.raises(ImportError): - MPLEncryptionMaterials(mpl_materials="doesn't matter") - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_valid_mpl_cmm_THEN_return_new_MPLCMMHandler(): - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) - - assert mpl_encryption_materials.mpl_materials == mock_mpl_encryption_materials - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_invalid_mpl_cmm_THEN_raise_ValueError(): - with pytest.raises(ValueError): - MPLEncryptionMaterials(mpl_materials="not a valid mpl_materials") - -def test_mpl_to_native(): - some_mpl_algorithm_id = "0x1234" # Not a real algorithm ID, but fits the format - - native_output = aws_encryption_sdk.materials_managers.mpl.materials._mpl_algorithm_id_to_native_algorithm_id( - some_mpl_algorithm_id - ) - - assert native_output == 0x1234 - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -@patch("aws_encryption_sdk.materials_managers.mpl.materials._mpl_algorithm_id_to_native_algorithm_id") -@patch("aws_encryption_sdk.materials_managers.mpl.materials.AlgorithmSuite.get_by_id") -def test_GIVEN_valid_mpl_algorithm_id_WHEN_get_algorithm_THEN_valid_native_algorithm_id( - mock_algorithm, - mock_native_algorithm_id, -): - # Mock valid conversion from MPL to native algorithm ID - mock_native_algorithm_id.return_value = 0x1234 - - # Mock valid lookup in native AlgorithmSuite lookup - mock_algorithm.return_value = MagicMock(__class__=AlgorithmSuite) - - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) - output = mpl_encryption_materials.algorithm - assert output == mock_algorithm() # property calls automatically, we need to call the mock - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GecTHEN_valid_native_algorithm_id(): - mock_encryption_context = MagicMock(__class__=Dict[str, str]) - mock_mpl_encryption_materials.encryption_context = mock_encryption_context - - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) - output = mpl_encryption_materials.encryption_context - - assert output == mock_encryption_context - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GecTHEN_valid_nativefadsf_algorithm_id(): - mock_edk = MagicMock(__class__=MPL_EncryptedDataKey) - mock_mpl_key_provider_id = MagicMock(__class__=str) - mock_edk.key_provider_id = mock_mpl_key_provider_id - mock_mpl_key_provider_info = MagicMock(__class__=bytes) - mock_edk.key_provider_info = mock_mpl_key_provider_info - mock_mpl_ciphertext = MagicMock(__class__=bytes) - mock_edk.ciphertext = mock_mpl_ciphertext - - mock_edks = [ mock_edk ] - mock_mpl_encryption_materials.encrypted_data_keys = mock_edks - - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) - output = mpl_encryption_materials.encrypted_data_keys - output_as_list = list(output) - - assert len(output_as_list) == len(mock_edks) - for i in range(len(output_as_list)): - # assume output[i] corresponds to input[i] - native_edk = output_as_list[i] - mpl_edk = mock_edks[i] - - assert native_edk.encrypted_data_key == mpl_edk.ciphertext - assert native_edk.key_provider.provider_id == mpl_edk.key_provider_id - assert native_edk.key_provider.key_info == mpl_edk.key_provider_info - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GecTHEN_valid_nativefadsffadsfa_algorithm_id(): - mock_data_key = MagicMock(__class__=bytes) - mock_mpl_encryption_materials.plaintext_data_key = mock_data_key - - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) - output = mpl_encryption_materials.data_encryption_key - - assert output.key_provider.provider_id == "" - assert output.key_provider.key_info == b"" - assert output.data_key == mock_data_key - assert output.encrypted_data_key == b"" - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GecTHEN_valid_nativefasdfasdffadsf_algorithm_id(): - mock_signing_key = MagicMock(__class__=bytes) - mock_mpl_encryption_materials.signing_key = mock_signing_key - - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) - output = mpl_encryption_materials.signing_key - - assert output == mock_signing_key - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GecTHEN_valid_nativeffasdfasdadsffadsfa_algorithm_id(): - mock_data_key = MagicMock(__class__=bytes) - mock_mpl_decrypt_materials.plaintext_data_key = mock_data_key - - mpl_decryption_materials = MPLDecryptionMaterials(mpl_materials=mock_mpl_decrypt_materials) - output = mpl_decryption_materials.data_key - - assert output.key_provider.provider_id == "" - assert output.key_provider.key_info == b"" - assert output.data_key == mock_data_key - assert output.encrypted_data_key == b"" - - -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") -def test_GecTHEN_validadsfasdf_nativefasdfasdffadsf_algorithm_id(): - mock_verification_key = MagicMock(__class__=bytes) - mock_mpl_decrypt_materials.verification_key = mock_verification_key - - mpl_decryption_materials = MPLDecryptionMaterials(mpl_materials=mock_mpl_decrypt_materials) - output = mpl_decryption_materials.verification_key - - assert output == mock_verification_key diff --git a/tox.ini b/tox.ini index 346e4fae0..e48f9f3b5 100644 --- a/tox.ini +++ b/tox.ini @@ -82,7 +82,7 @@ deps = # Install the MPL requirements if the `-mpl` suffix is present mpl: -rrequirements_mpl.txt commands = - local: {[testenv:base-command]commands} test/ -m local --ignore test/unit/mpl/ + local: {[testenv:base-command]commands} test/ -m local --ignore test/mpl/ # MPL unit tests require the MPL to be installed mpllocal: {[testenv:base-command]commands} test/ -m local integ: {[testenv:base-command]commands} test/ -m integ @@ -90,13 +90,13 @@ commands = examples: {[testenv:base-command]commands} examples/test/ -m examples --ignore examples/test/keyrings/ # MPL keyring examples require a special IAM role; run these separately under a separate set of permissions mplexamples: {[testenv:base-command]commands} examples/test/keyrings -m examples - all: {[testenv:base-command]commands} test/ examples/test/ --ignore test/unit/mpl/ --ignore examples/test/keyrings/ + all: {[testenv:base-command]commands} test/ examples/test/ --ignore test/mpl/ --ignore examples/test/keyrings/ mplall: {[testenv:base-command]commands} test/ examples/test/ manual: {[testenv:base-command]commands} # Run code coverage on the unit tests [testenv:coverage] -commands = {[testenv:base-command]commands} --cov aws_encryption_sdk test/ -m local --ignore test/unit/mpl/ +commands = {[testenv:base-command]commands} --cov aws_encryption_sdk test/ -m local --ignore test/mpl/ [testenv:mplcoverage-mpl] commands = {[testenv:base-command]commands} --cov aws_encryption_sdk test/ -m local From 61ba4dec6bdc404a13bb245fa5bbb2078d014edc Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 10:36:39 -0800 Subject: [PATCH 109/422] refactor tests --- src/aws_encryption_sdk/streaming_client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 61f2f88c6..01d4ca5ac 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -58,7 +58,6 @@ serialize_non_framed_close, serialize_non_framed_open, ) -from aws_encryption_sdk.materials_managers.mpl.cmm import MPLCMMHandler from aws_encryption_sdk.internal.utils import exactly_one_arg_is_not_none from aws_encryption_sdk.internal.utils.commitment import ( validate_commitment_policy_on_decrypt, @@ -79,6 +78,10 @@ from aws_cryptographic_materialproviders.mpl.models import CreateDefaultCryptographicMaterialsManagerInput from aws_cryptographic_materialproviders.mpl.references import IKeyring _HAS_MPL = True + + # Import internal ESDK modules that depend on the MPL + from aws_encryption_sdk.materials_managers.mpl.cmm import MPLCMMHandler + except ImportError: _HAS_MPL = False From 95c5be6bfb7fea0ef31ebcc8e5ddec0ac07df9fd Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 10:37:17 -0800 Subject: [PATCH 110/422] refactor tests --- src/aws_encryption_sdk/streaming_client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 01d4ca5ac..a3c05bbb7 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -554,7 +554,8 @@ def _prep_message(self): else: # MPL verification key is PEM bytes, not DER bytes. # If the underlying CMM is from the MPL, load PEM bytes. - if (isinstance(self.config.materials_manager, MPLCMMHandler)): + if (_HAS_MPL + and isinstance(self.config.materials_manager, MPLCMMHandler)): self.signer = Signer.from_key_bytes( algorithm=self._encryption_materials.algorithm, key_bytes=self._encryption_materials.signing_key, encoding=serialization.Encoding.PEM, @@ -921,7 +922,8 @@ def _read_header(self): else: # MPL verification key is NOT key bytes; it is bytes of the compressed point. # If the underlying CMM is from the MPL, load bytes from encoded point. - if (isinstance(self.config.materials_manager, MPLCMMHandler)): + if (_HAS_MPL + and isinstance(self.config.materials_manager, MPLCMMHandler)): self.verifier = Verifier.from_encoded_point( algorithm=header.algorithm, encoded_point=base64.b64encode(decryption_materials.verification_key) From 9566873a946acb70622962bdbcc8b2086e88e16d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 10:51:45 -0800 Subject: [PATCH 111/422] refactor tests --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index e48f9f3b5..9e3fe2b98 100644 --- a/tox.ini +++ b/tox.ini @@ -96,7 +96,7 @@ commands = # Run code coverage on the unit tests [testenv:coverage] -commands = {[testenv:base-command]commands} --cov aws_encryption_sdk test/ -m local --ignore test/mpl/ +commands = {[testenv:base-command]commands} --cov aws_encryption_sdk --cov-config=.coveragerc test/ -m local --ignore test/mpl/ [testenv:mplcoverage-mpl] commands = {[testenv:base-command]commands} --cov aws_encryption_sdk test/ -m local From 66420832da4a4a8806254f0b8487ee03452a17d1 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 11:16:25 -0800 Subject: [PATCH 112/422] fix cov --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 9e3fe2b98..70ed4281f 100644 --- a/tox.ini +++ b/tox.ini @@ -96,9 +96,9 @@ commands = # Run code coverage on the unit tests [testenv:coverage] -commands = {[testenv:base-command]commands} --cov aws_encryption_sdk --cov-config=.coveragerc test/ -m local --ignore test/mpl/ +commands = {[testenv:base-command]commands} --cov aws_encryption_sdk test/ -m local --ignore test/mpl/ [testenv:mplcoverage-mpl] -commands = {[testenv:base-command]commands} --cov aws_encryption_sdk test/ -m local +commands = {[testenv:base-command]commands} --cov --cov-config=.coveragercmpl aws_encryption_sdk test/ -m local # Verify that local tests work without environment variables present [testenv:nocmk] From 51d2804343797d98848f7eea2b8f333f9f7f8a46 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 11:22:26 -0800 Subject: [PATCH 113/422] fix cov --- .coveragerc | 7 + .coveragercmpl | 1 + .gitignore | 5 +- test/mpl/README.md | 1 + test/mpl/__init__.py | 13 + .../unit/test_material_managers_mpl_cmm.py | 250 ++++++++++++++++++ .../test_material_managers_mpl_materials.py | 197 ++++++++++++++ 7 files changed, 472 insertions(+), 2 deletions(-) create mode 100644 .coveragerc create mode 100644 .coveragercmpl create mode 100644 test/mpl/README.md create mode 100644 test/mpl/__init__.py create mode 100644 test/mpl/unit/test_material_managers_mpl_cmm.py create mode 100644 test/mpl/unit/test_material_managers_mpl_materials.py diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..8957349aa --- /dev/null +++ b/.coveragerc @@ -0,0 +1,7 @@ +# .coveragerc file when running coverage WITHOUT coverage for the MPL +# This prevents the ESDK without the MPL from considering the MPL-specific modules as "missed" coverage +[run] +omit = */aws_encryption_sdk/materials_managers/mpl/* + +[report] +omit = */aws_encryption_sdk/materials_managers/mpl/* \ No newline at end of file diff --git a/.coveragercmpl b/.coveragercmpl new file mode 100644 index 000000000..31a7b4407 --- /dev/null +++ b/.coveragercmpl @@ -0,0 +1 @@ +# .coveragerc file when running coverage WITH coverage for the MPL diff --git a/.gitignore b/.gitignore index 63097dcba..fc224adc4 100644 --- a/.gitignore +++ b/.gitignore @@ -19,8 +19,9 @@ docs/build __pycache__ *.egg-info -# Coverage.py -.coverage* +# Coverage.py, NOT .coveragerc nor .coveragercmpl +.coverage +.coverage.py # MyPy .mypy_cache diff --git a/test/mpl/README.md b/test/mpl/README.md new file mode 100644 index 000000000..7ae7134d0 --- /dev/null +++ b/test/mpl/README.md @@ -0,0 +1 @@ +Tests in this directory REQUIRE the [aws-cryptographic-material-providers](https://github.com/aws/aws-cryptographic-material-providers-library) library to execute. \ No newline at end of file diff --git a/test/mpl/__init__.py b/test/mpl/__init__.py new file mode 100644 index 000000000..b976c1308 --- /dev/null +++ b/test/mpl/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Module containing tests that REQUIRE the aws-cryptographic-material-providers library to run.""" \ No newline at end of file diff --git a/test/mpl/unit/test_material_managers_mpl_cmm.py b/test/mpl/unit/test_material_managers_mpl_cmm.py new file mode 100644 index 000000000..b1589b1cf --- /dev/null +++ b/test/mpl/unit/test_material_managers_mpl_cmm.py @@ -0,0 +1,250 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" + +import pytest +from mock import MagicMock, patch + + +from aws_encryption_sdk.identifiers import CommitmentPolicy +import aws_encryption_sdk.materials_managers.mpl.cmm +from aws_encryption_sdk.materials_managers.mpl.cmm import MPLCMMHandler +from aws_encryption_sdk.materials_managers.mpl.materials import ( + MPLEncryptionMaterials, + MPLDecryptionMaterials, +) + +pytestmark = [pytest.mark.unit, pytest.mark.local] + + +from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException +from aws_cryptographic_materialproviders.mpl.models import ( + AlgorithmSuiteIdESDK, + CommitmentPolicyESDK, + DecryptMaterialsInput, + DecryptionMaterials as MPL_DecryptionMaterials, + EncryptionMaterials as MPL_EncryptionMaterials, + GetEncryptionMaterialsInput, + GetEncryptionMaterialsOutput, +) +from aws_cryptographic_materialproviders.mpl.references import ( + ICryptographicMaterialsManager +) + +mock_mpl_cmm = MagicMock(__class__=ICryptographicMaterialsManager) +mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) +mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) + + +from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError +from aws_encryption_sdk.materials_managers import ( + EncryptionMaterialsRequest, + DecryptionMaterialsRequest, +) + + +mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) +mock_encryption_materials_handler = MagicMock(__class__=MPLEncryptionMaterials) +mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) + +def test_GIVEN_test_has_mpl_is_False_THEN_cmm_has_mpl_is_False(): + """If the MPL IS NOT installed in the runtime environment, + assert the cmm has _HAS_MPL set to False""" + + assert hasattr(aws_encryption_sdk.materials_managers.mpl.cmm, "_HAS_MPL") + assert aws_encryption_sdk.materials_managers.mpl.cmm._HAS_MPL is False + + +def test_GIVEN_test_has_mpl_is_True_THEN_cmm_has_mpl_is_True(): + """If the MPL IS installed in the runtime environment, + assert the cmm has _HAS_MPL set to True""" + + assert hasattr(aws_encryption_sdk.materials_managers.mpl.cmm, "_HAS_MPL") + assert aws_encryption_sdk.materials_managers.mpl.cmm._HAS_MPL is True + + +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_THEN_raise_ImportError(): + with pytest.raises(ImportError): + MPLCMMHandler(mpl_cmm="doesn't matter") + + +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_valid_mpl_cmm_THEN_return_new_MPLCMMHandler(): + mpl_cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) + + assert mpl_cmm_handler.mpl_cmm == mock_mpl_cmm + + +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_invalid_mpl_cmm_THEN_raise_ValueError(): + with pytest.raises(ValueError): + MPLCMMHandler(mpl_cmm="not a valid mpl_cmm") + + +@patch.object(mock_mpl_cmm, "get_encryption_materials") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_get_encryption_materials") +def test_GIVEN_valid_request_WHEN_call_get_encryption_materials_THEN_return_MPLEncryptionMaterials( + mock_native_to_mpl_get_encryption_materials, + mock_get_encryption_materials, +): + + # Mock: mpl_cmm.get_encryption_materials returns mock MPL encryption materials + mock_get_encryption_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) + mock_get_encryption_materials_output.encryption_materials = mock_mpl_encryption_materials + mock_get_encryption_materials.return_value = mock_get_encryption_materials_output + + # Mock: CMMHandler._native_to_mpl_get_encryption_materials creates a GetEncryptionMaterialsInput + mock_get_encryption_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) + mock_native_to_mpl_get_encryption_materials.return_value = mock_get_encryption_materials_input + + cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) + test = cmm_handler.get_encryption_materials(mock_encryption_materials_request) + + # Verify cmm_handler returns MPLEncryptionMaterials + assert isinstance(test, MPLEncryptionMaterials) + # Verify returned EncryptionMaterialsHandler uses the output of `get_encryption_materials` + assert test.mpl_materials == mock_mpl_encryption_materials + # Verify we actually called `get_encryption_materials` + mock_mpl_cmm.get_encryption_materials.assert_called_once_with(mock_get_encryption_materials_input) + + +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") +def test_GIVEN_get_encryption_materials_raises_MPL_Exception_WHEN_call_get_encryption_materials_THEN_raise_ESDK_Exception( + _ +): + with pytest.raises(AWSEncryptionSDKClientError): + with patch.object(mock_mpl_cmm, "get_encryption_materials", + side_effect=AwsCryptographicMaterialProvidersException("any")): + + cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) + cmm_handler.get_encryption_materials(mock_encryption_materials_request) + +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") +def test_GIVEN_native_to_mpl_commmitment_policy_returns_valid_policy_WHEN_call_native_to_mpl_get_encryption_materials_THEN_returns_GetEncryptionMaterialsInput( + mock_mpl_commitment_policy +): + mock_commitment_policy = MagicMock(__class__=CommitmentPolicyESDK) + mock_mpl_commitment_policy.return_value = mock_commitment_policy + + output = MPLCMMHandler._native_to_mpl_get_encryption_materials(mock_encryption_materials_request) + + # verify correctness of returned value + assert isinstance(output, GetEncryptionMaterialsInput) + assert output.encryption_context == mock_encryption_materials_request.encryption_context + assert output.commitment_policy == mock_commitment_policy + assert output.max_plaintext_length == mock_encryption_materials_request.plaintext_length + + +def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_FORBID_ENCRYPT_ALLOW_DECRYPT(): + native_commitment_policy = CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT + + output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) + + assert isinstance(output, CommitmentPolicyESDK) + assert output.value == "FORBID_ENCRYPT_ALLOW_DECRYPT" + +def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_ALLOW_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_REQUIRE_ENCRYPT_ALLOW_DECRYPT(): + native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT + + output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) + + assert isinstance(output, CommitmentPolicyESDK) + assert output.value == "REQUIRE_ENCRYPT_ALLOW_DECRYPT" + +def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_REQUIRE_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_REQUIRE_ENCRYPT_REQUIRE_DECRYPT(): + native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT + + output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) + + assert isinstance(output, CommitmentPolicyESDK) + assert output.value == "REQUIRE_ENCRYPT_REQUIRE_DECRYPT" + +def test_GIVEN_CommitmentPolicy_unrecognized_WHEN_call_native_to_mpl_commmitment_policyTHEN_raise_ValueError(): + native_commitment_policy = "not a commitment policy" + + with pytest.raises(ValueError): + MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) + +@patch.object(mock_mpl_cmm, "decrypt_materials") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._create_mpl_decrypt_materials_input_from_request") +def test_GIVEN_valid_request_WHEN_call_decrypt_materials_THEN_return_MPLDecryptionMaterials( + mock_native_to_mpl_decrypt_materials, + mock_get_encryption_materials, +): + + # Mock: mpl_cmm.get_decryption_materials returns mock MPL decryption materials + mock_decrypt_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) + mock_decrypt_materials_output.decryption_materials = mock_mpl_decrypt_materials + mock_get_encryption_materials.return_value = mock_decrypt_materials_output + + # Mock: CMMHandler._create_mpl_decrypt_materials_input_from_request creates a DecryptMaterialsInput + mock_decrypt_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) + mock_native_to_mpl_decrypt_materials.return_value = mock_decrypt_materials_input + + cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) + output = cmm_handler.decrypt_materials(mock_decryption_materials_request) + + # Verify cmm_handler returns MPLDecryptionMaterials + assert isinstance(output, MPLDecryptionMaterials) + # Verify returned MPLDecryptionMaterials uses the output of `decrypt_materials` + assert output.mpl_materials == mock_mpl_decrypt_materials + # Verify we actually called `decrypt_materials` + mock_mpl_cmm.decrypt_materials.assert_called_once_with(mock_decrypt_materials_input) + +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._create_mpl_decrypt_materials_input_from_request") +def test_GIVEN_decrypt_materials_raises_MPL_Exception_WHEN_call_decrypt_materials_THEN_raise_ESDK_Exception( + _ +): + with pytest.raises(AWSEncryptionSDKClientError): + with patch.object(mock_mpl_cmm, "decrypt_materials", + side_effect=AwsCryptographicMaterialProvidersException("any")): + + cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) + cmm_handler.decrypt_materials(mock_decryption_materials_request) + +def test_WHEN_call_native_algorithm_id_to_mpl_algorithm_id_THEN_returns_valid_AlgorithmSuiteIdESDK(): + some_native_algorithm_id = 0x0000 # Not a real algorithm ID, but fits the format + + mpl_output = MPLCMMHandler._native_algorithm_id_to_mpl_algorithm_id( + some_native_algorithm_id + ) + + assert isinstance(mpl_output, AlgorithmSuiteIdESDK) + assert mpl_output.value == "0x0000" + +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_algorithm_id_to_mpl_algorithm_id") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") +def test__create_mpl_decrypt_materials_input_from_request( + mock_mpl_commitment_policy, + mock_mpl_algorithm_id, +): + mock_algorithm_id = "0x1234" # Some fake algorithm ID that fits the format + mock_mpl_algorithm_id.return_value = mock_algorithm_id + mock_commitment_policy = MagicMock(__class__=CommitmentPolicyESDK) + mock_mpl_commitment_policy.return_value = mock_commitment_policy + + # mock_decryption_materials_request.algorithm = + + output = MPLCMMHandler._create_mpl_decrypt_materials_input_from_request(mock_decryption_materials_request) + + assert isinstance(output, DecryptMaterialsInput) + assert output.algorithm_suite_id == mock_algorithm_id + assert output.commitment_policy == mock_commitment_policy + assert output.encryption_context == mock_decryption_materials_request.encryption_context + + assert len(output.encrypted_data_keys) == len(mock_decryption_materials_request.encrypted_data_keys) + for i in range(len(output.encrypted_data_keys)): + # Assume input[i] == output[i], seems to work + output_edk = output.encrypted_data_keys[i] + input_edk = mock_decryption_materials_request[i] + assert output_edk.key_provider_id == input_edk.key_provider.provider_id + assert output_edk.key_provider_info == input_edk.key_provider.key_info + assert output_edk.ciphertext == input_edk.encrypted_data_key diff --git a/test/mpl/unit/test_material_managers_mpl_materials.py b/test/mpl/unit/test_material_managers_mpl_materials.py new file mode 100644 index 000000000..dfd6b2769 --- /dev/null +++ b/test/mpl/unit/test_material_managers_mpl_materials.py @@ -0,0 +1,197 @@ +# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" + +import pytest +from mock import MagicMock, patch, PropertyMock +from typing import Dict, List + +from aws_encryption_sdk.identifiers import CommitmentPolicy +import aws_encryption_sdk.materials_managers.mpl.materials +from aws_encryption_sdk.materials_managers.mpl.materials import ( + MPLEncryptionMaterials, + MPLDecryptionMaterials, +) +from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite + +pytestmark = [pytest.mark.unit, pytest.mark.local] + + +from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException +from aws_cryptographic_materialproviders.mpl.models import ( + AlgorithmSuiteIdESDK, + CommitmentPolicyESDK, + DecryptMaterialsInput, + DecryptionMaterials as MPL_DecryptionMaterials, + EncryptedDataKey as MPL_EncryptedDataKey, + EncryptionMaterials as MPL_EncryptionMaterials, + GetEncryptionMaterialsInput, + GetEncryptionMaterialsOutput, +) +from aws_cryptographic_materialproviders.mpl.references import ( + ICryptographicMaterialsManager +) + +mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) +mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) + + +from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError +from aws_encryption_sdk.materials_managers import ( + EncryptionMaterialsRequest, + DecryptionMaterialsRequest, +) + + +mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) +mock_encryption_materials_handler = MagicMock(__class__=MPLEncryptionMaterials) +mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) + +def test_GIVEN_test_has_mpl_is_False_THEN_cmm_has_mpl_is_False(): + """If the MPL IS NOT installed in the runtime environment, + assert the cmm has _HAS_MPL set to False""" + + assert hasattr(aws_encryption_sdk.materials_managers.mpl.materials, "_HAS_MPL") + assert aws_encryption_sdk.materials_managers.mpl.materials._HAS_MPL is False + + +def test_GIVEN_test_has_mpl_is_True_THEN_cmm_has_mpl_is_True(): + """If the MPL IS installed in the runtime environment, + assert the cmm has _HAS_MPL set to True""" + + assert hasattr(aws_encryption_sdk.materials_managers.mpl.materials, "_HAS_MPL") + assert aws_encryption_sdk.materials_managers.mpl.materials._HAS_MPL is True + + +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_THEN_raise_ImportError(): + with pytest.raises(ImportError): + MPLEncryptionMaterials(mpl_materials="doesn't matter") + + +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_valid_mpl_cmm_THEN_return_new_MPLCMMHandler(): + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + + assert mpl_encryption_materials.mpl_materials == mock_mpl_encryption_materials + + +def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_invalid_mpl_cmm_THEN_raise_ValueError(): + with pytest.raises(ValueError): + MPLEncryptionMaterials(mpl_materials="not a valid mpl_materials") + +def test_mpl_to_native(): + some_mpl_algorithm_id = "0x1234" # Not a real algorithm ID, but fits the format + + native_output = aws_encryption_sdk.materials_managers.mpl.materials._mpl_algorithm_id_to_native_algorithm_id( + some_mpl_algorithm_id + ) + + assert native_output == 0x1234 + + +@patch("aws_encryption_sdk.materials_managers.mpl.materials._mpl_algorithm_id_to_native_algorithm_id") +@patch("aws_encryption_sdk.materials_managers.mpl.materials.AlgorithmSuite.get_by_id") +def test_GIVEN_valid_mpl_algorithm_id_WHEN_get_algorithm_THEN_valid_native_algorithm_id( + mock_algorithm, + mock_native_algorithm_id, +): + # Mock valid conversion from MPL to native algorithm ID + mock_native_algorithm_id.return_value = 0x1234 + + # Mock valid lookup in native AlgorithmSuite lookup + mock_algorithm.return_value = MagicMock(__class__=AlgorithmSuite) + + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + output = mpl_encryption_materials.algorithm + assert output == mock_algorithm() # property calls automatically, we need to call the mock + + +def test_GecTHEN_valid_native_algorithm_id(): + mock_encryption_context = MagicMock(__class__=Dict[str, str]) + mock_mpl_encryption_materials.encryption_context = mock_encryption_context + + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + output = mpl_encryption_materials.encryption_context + + assert output == mock_encryption_context + + +def test_GecTHEN_valid_nativefadsf_algorithm_id(): + mock_edk = MagicMock(__class__=MPL_EncryptedDataKey) + mock_mpl_key_provider_id = MagicMock(__class__=str) + mock_edk.key_provider_id = mock_mpl_key_provider_id + mock_mpl_key_provider_info = MagicMock(__class__=bytes) + mock_edk.key_provider_info = mock_mpl_key_provider_info + mock_mpl_ciphertext = MagicMock(__class__=bytes) + mock_edk.ciphertext = mock_mpl_ciphertext + + mock_edks = [ mock_edk ] + mock_mpl_encryption_materials.encrypted_data_keys = mock_edks + + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + output = mpl_encryption_materials.encrypted_data_keys + output_as_list = list(output) + + assert len(output_as_list) == len(mock_edks) + for i in range(len(output_as_list)): + # assume output[i] corresponds to input[i] + native_edk = output_as_list[i] + mpl_edk = mock_edks[i] + + assert native_edk.encrypted_data_key == mpl_edk.ciphertext + assert native_edk.key_provider.provider_id == mpl_edk.key_provider_id + assert native_edk.key_provider.key_info == mpl_edk.key_provider_info + +def test_GecTHEN_valid_nativefadsffadsfa_algorithm_id(): + mock_data_key = MagicMock(__class__=bytes) + mock_mpl_encryption_materials.plaintext_data_key = mock_data_key + + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + output = mpl_encryption_materials.data_encryption_key + + assert output.key_provider.provider_id == "" + assert output.key_provider.key_info == b"" + assert output.data_key == mock_data_key + assert output.encrypted_data_key == b"" + + +def test_GecTHEN_valid_nativefasdfasdffadsf_algorithm_id(): + mock_signing_key = MagicMock(__class__=bytes) + mock_mpl_encryption_materials.signing_key = mock_signing_key + + mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + output = mpl_encryption_materials.signing_key + + assert output == mock_signing_key + + +def test_GecTHEN_valid_nativeffasdfasdadsffadsfa_algorithm_id(): + mock_data_key = MagicMock(__class__=bytes) + mock_mpl_decrypt_materials.plaintext_data_key = mock_data_key + + mpl_decryption_materials = MPLDecryptionMaterials(mpl_materials=mock_mpl_decrypt_materials) + output = mpl_decryption_materials.data_key + + assert output.key_provider.provider_id == "" + assert output.key_provider.key_info == b"" + assert output.data_key == mock_data_key + assert output.encrypted_data_key == b"" + + +def test_GecTHEN_validadsfasdf_nativefasdfasdffadsf_algorithm_id(): + mock_verification_key = MagicMock(__class__=bytes) + mock_mpl_decrypt_materials.verification_key = mock_verification_key + + mpl_decryption_materials = MPLDecryptionMaterials(mpl_materials=mock_mpl_decrypt_materials) + output = mpl_decryption_materials.verification_key + + assert output == mock_verification_key From 51e5db501c41d5c8e2351daa7b4331a21132f2b7 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 11:26:15 -0800 Subject: [PATCH 114/422] fix cov --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 70ed4281f..1cc78c45e 100644 --- a/tox.ini +++ b/tox.ini @@ -98,7 +98,7 @@ commands = [testenv:coverage] commands = {[testenv:base-command]commands} --cov aws_encryption_sdk test/ -m local --ignore test/mpl/ [testenv:mplcoverage-mpl] -commands = {[testenv:base-command]commands} --cov --cov-config=.coveragercmpl aws_encryption_sdk test/ -m local +commands = {[testenv:base-command]commands} --cov-config=.coveragercmpl --cov aws_encryption_sdk test/ -m local # Verify that local tests work without environment variables present [testenv:nocmk] From e2354613a6d9c14c8cd4116a399e27010010d6f4 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 11:29:58 -0800 Subject: [PATCH 115/422] fix cov --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 1cc78c45e..419188b54 100644 --- a/tox.ini +++ b/tox.ini @@ -85,8 +85,8 @@ commands = local: {[testenv:base-command]commands} test/ -m local --ignore test/mpl/ # MPL unit tests require the MPL to be installed mpllocal: {[testenv:base-command]commands} test/ -m local - integ: {[testenv:base-command]commands} test/ -m integ - accept: {[testenv:base-command]commands} test/ -m accept + integ: {[testenv:base-command]commands} test/ -m integ --ignore test/mpl/ + accept: {[testenv:base-command]commands} test/ -m accept --ignore test/mpl/ examples: {[testenv:base-command]commands} examples/test/ -m examples --ignore examples/test/keyrings/ # MPL keyring examples require a special IAM role; run these separately under a separate set of permissions mplexamples: {[testenv:base-command]commands} examples/test/keyrings -m examples From e7c745fbdb8e33d342a9027196519c53833fcba8 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 11:34:58 -0800 Subject: [PATCH 116/422] fix tests --- .../unit/test_material_managers_mpl_cmm.py | 20 ------------------- .../test_material_managers_mpl_materials.py | 20 ------------------- tox.ini | 8 +++++--- 3 files changed, 5 insertions(+), 43 deletions(-) diff --git a/test/mpl/unit/test_material_managers_mpl_cmm.py b/test/mpl/unit/test_material_managers_mpl_cmm.py index b1589b1cf..cae334722 100644 --- a/test/mpl/unit/test_material_managers_mpl_cmm.py +++ b/test/mpl/unit/test_material_managers_mpl_cmm.py @@ -57,26 +57,6 @@ mock_encryption_materials_handler = MagicMock(__class__=MPLEncryptionMaterials) mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) -def test_GIVEN_test_has_mpl_is_False_THEN_cmm_has_mpl_is_False(): - """If the MPL IS NOT installed in the runtime environment, - assert the cmm has _HAS_MPL set to False""" - - assert hasattr(aws_encryption_sdk.materials_managers.mpl.cmm, "_HAS_MPL") - assert aws_encryption_sdk.materials_managers.mpl.cmm._HAS_MPL is False - - -def test_GIVEN_test_has_mpl_is_True_THEN_cmm_has_mpl_is_True(): - """If the MPL IS installed in the runtime environment, - assert the cmm has _HAS_MPL set to True""" - - assert hasattr(aws_encryption_sdk.materials_managers.mpl.cmm, "_HAS_MPL") - assert aws_encryption_sdk.materials_managers.mpl.cmm._HAS_MPL is True - - -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_THEN_raise_ImportError(): - with pytest.raises(ImportError): - MPLCMMHandler(mpl_cmm="doesn't matter") - def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_valid_mpl_cmm_THEN_return_new_MPLCMMHandler(): mpl_cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) diff --git a/test/mpl/unit/test_material_managers_mpl_materials.py b/test/mpl/unit/test_material_managers_mpl_materials.py index dfd6b2769..bb83b89fd 100644 --- a/test/mpl/unit/test_material_managers_mpl_materials.py +++ b/test/mpl/unit/test_material_managers_mpl_materials.py @@ -57,26 +57,6 @@ mock_encryption_materials_handler = MagicMock(__class__=MPLEncryptionMaterials) mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) -def test_GIVEN_test_has_mpl_is_False_THEN_cmm_has_mpl_is_False(): - """If the MPL IS NOT installed in the runtime environment, - assert the cmm has _HAS_MPL set to False""" - - assert hasattr(aws_encryption_sdk.materials_managers.mpl.materials, "_HAS_MPL") - assert aws_encryption_sdk.materials_managers.mpl.materials._HAS_MPL is False - - -def test_GIVEN_test_has_mpl_is_True_THEN_cmm_has_mpl_is_True(): - """If the MPL IS installed in the runtime environment, - assert the cmm has _HAS_MPL set to True""" - - assert hasattr(aws_encryption_sdk.materials_managers.mpl.materials, "_HAS_MPL") - assert aws_encryption_sdk.materials_managers.mpl.materials._HAS_MPL is True - - -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_THEN_raise_ImportError(): - with pytest.raises(ImportError): - MPLEncryptionMaterials(mpl_materials="doesn't matter") - def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_valid_mpl_cmm_THEN_return_new_MPLCMMHandler(): mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) diff --git a/tox.ini b/tox.ini index 419188b54..3daa40e47 100644 --- a/tox.ini +++ b/tox.ini @@ -12,9 +12,11 @@ envlist = # tests in a test environment that also has the MPL. py{311,312}-{local,integ,accept,examples}{,-mpl}, # >=3.11: Run ONLY the MPL-specific tests. - # These must be separate from the above target. - # These require the `-mpl` suffix so tox installs the MPL. - # The `mpl` prefix runs only MPL-specific tests + # These must be separate from the above target, since + # these require the `-mpl` suffix. + # The `mpl` prefix specifies a separate target, + # i.e. `mpllocal` instead of `local`. + # `mplXXX` contains tests using MPL components. py{311,312}-mpl{local,examples}-mpl nocmk, bandit, doc8, readme, docs, From fee4f36f44c94bb357d10272190359fdf5d82e62 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 16:57:03 -0800 Subject: [PATCH 117/422] test cleanup --- .../example_branch_key_id_supplier.py | 5 +- .../materials_managers/mpl/cmm.py | 78 +++---- .../materials_managers/mpl/materials.py | 12 +- src/aws_encryption_sdk/streaming_client.py | 8 +- .../unit/test_material_managers_mpl_cmm.py | 207 +++++++++++------- .../test_material_managers_mpl_materials.py | 139 +++++++----- 6 files changed, 271 insertions(+), 178 deletions(-) diff --git a/examples/src/keyrings/example_branch_key_id_supplier.py b/examples/src/keyrings/example_branch_key_id_supplier.py index a06280fa1..ba9ae060c 100644 --- a/examples/src/keyrings/example_branch_key_id_supplier.py +++ b/examples/src/keyrings/example_branch_key_id_supplier.py @@ -18,11 +18,10 @@ def __init__(self, tenant_1_id, tenant_2_id): def get_branch_key_id( self, - # TODO-MPL: Change this to `native_input` in Smithy-Dafny - input: GetBranchKeyIdInput # noqa pylint: disable=redefined-builtin + param: GetBranchKeyIdInput ) -> GetBranchKeyIdOutput: """Returns branch key ID from the tenant ID in input's encryption context.""" - encryption_context: Dict[str, str] = input.encryption_context + encryption_context: Dict[str, str] = param.encryption_context if b"tenant" not in encryption_context: raise ValueError("EncryptionContext invalid, does not contain expected tenant key value pair.") diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index 1e3e3fb34..c97c070f0 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -2,62 +2,64 @@ from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException from aws_cryptographic_materialproviders.mpl.models import ( - AlgorithmSuiteIdESDK, - CommitmentPolicyESDK, - DecryptMaterialsInput, - DecryptMaterialsOutput, + AlgorithmSuiteIdESDK as MPL_AlgorithmSuiteIdESDK, + CommitmentPolicyESDK as MPL_CommitmentPolicyESDK, + DecryptMaterialsInput as MPL_DecryptMaterialsInput, + DecryptMaterialsOutput as MPL_DecryptMaterialsOutput, EncryptedDataKey as MPL_EncryptedDataKey, - GetEncryptionMaterialsInput, - GetEncryptionMaterialsOutput, + GetEncryptionMaterialsInput as MPL_GetEncryptionMaterialsInput, + GetEncryptionMaterialsOutput as MPL_GetEncryptionMaterialsOutput, +) +from aws_cryptographic_materialproviders.mpl.references import ( + ICryptographicMaterialsManager as MPL_ICryptographicMaterialsManager ) -from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager from typing import List from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError from aws_encryption_sdk.identifiers import CommitmentPolicy -from aws_encryption_sdk.materials_managers.mpl.materials import MPLEncryptionMaterials, MPLDecryptionMaterials +from aws_encryption_sdk.materials_managers.mpl.materials import EncryptionMaterialsFromMPL, DecryptionMaterialsFromMPL from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager from aws_encryption_sdk.structures import EncryptedDataKey as Native_EncryptedDataKey -class MPLCMMHandler(CryptoMaterialsManager): +class CryptoMaterialsManagerFromMPL(CryptoMaterialsManager): """ In instances where encryption materials are provided by an implementation of the MPL's - `aws_cryptographic_materialproviders.mpl.references.ICryptographicMaterialsManager`, + `aws_cryptographic_materialproviders.mpl.references.MPL_ICryptographicMaterialsManager`, this maps the ESDK CMM interfaces to the MPL CMM. """ - mpl_cmm: 'ICryptographicMaterialsManager' + mpl_cmm: 'MPL_ICryptographicMaterialsManager' def __init__( self, - mpl_cmm: 'ICryptographicMaterialsManager' + mpl_cmm: 'MPL_ICryptographicMaterialsManager' ): """ - Create MPLCMMHandler. + Create CryptoMaterialsManagerFromMPL. :param mpl_cmm: Underlying MPL cryptographic materials manager """ - if isinstance(mpl_cmm, ICryptographicMaterialsManager): + if isinstance(mpl_cmm, MPL_ICryptographicMaterialsManager): self.mpl_cmm = mpl_cmm else: - raise ValueError(f"Invalid CMM passed to MPLCMMHandler. cmm: {mpl_cmm}") + raise ValueError(f"Invalid CMM passed to CryptoMaterialsManagerFromMPL. cmm: {mpl_cmm}") def get_encryption_materials( self, request: EncryptionMaterialsRequest - ) -> MPLEncryptionMaterials: + ) -> EncryptionMaterialsFromMPL: """ Returns an EncryptionMaterialsHandler for the configured CMM. :param request: Request for encryption materials """ try: - mpl_input: GetEncryptionMaterialsInput = MPLCMMHandler._native_to_mpl_get_encryption_materials( + mpl_input: MPL_GetEncryptionMaterialsInput = CryptoMaterialsManagerFromMPL._native_to_mpl_get_encryption_materials( request ) - mpl_output: GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) - return MPLEncryptionMaterials(mpl_output.encryption_materials) + mpl_output: MPL_GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) + return EncryptionMaterialsFromMPL(mpl_output.encryption_materials) except AwsCryptographicMaterialProvidersException as mpl_exception: # Wrap MPL error into the ESDK error type # so customers only have to catch ESDK error types. @@ -66,11 +68,11 @@ def get_encryption_materials( @staticmethod def _native_to_mpl_get_encryption_materials( request: EncryptionMaterialsRequest - ) -> 'GetEncryptionMaterialsInput': - commitment_policy = MPLCMMHandler._native_to_mpl_commmitment_policy( + ) -> 'MPL_GetEncryptionMaterialsInput': + commitment_policy = CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy( request.commitment_policy ) - output: GetEncryptionMaterialsInput = GetEncryptionMaterialsInput( + output: MPL_GetEncryptionMaterialsInput = MPL_GetEncryptionMaterialsInput( encryption_context=request.encryption_context, commitment_policy=commitment_policy, max_plaintext_length=request.plaintext_length, @@ -80,54 +82,54 @@ def _native_to_mpl_get_encryption_materials( @staticmethod def _native_to_mpl_commmitment_policy( native_commitment_policy: CommitmentPolicy - ) -> 'CommitmentPolicyESDK': + ) -> 'MPL_CommitmentPolicyESDK': if native_commitment_policy == CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT: - return CommitmentPolicyESDK(value="FORBID_ENCRYPT_ALLOW_DECRYPT") + return MPL_CommitmentPolicyESDK(value="FORBID_ENCRYPT_ALLOW_DECRYPT") elif native_commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT: - return CommitmentPolicyESDK(value="REQUIRE_ENCRYPT_ALLOW_DECRYPT") + return MPL_CommitmentPolicyESDK(value="REQUIRE_ENCRYPT_ALLOW_DECRYPT") elif native_commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT: - return CommitmentPolicyESDK(value="REQUIRE_ENCRYPT_REQUIRE_DECRYPT") + return MPL_CommitmentPolicyESDK(value="REQUIRE_ENCRYPT_REQUIRE_DECRYPT") else: raise ValueError(f"Invalid native_commitment_policy: {native_commitment_policy}") def decrypt_materials( self, request: DecryptionMaterialsRequest - ) -> MPLDecryptionMaterials: + ) -> DecryptionMaterialsFromMPL: """ - Returns a MPLDecryptionMaterials for the configured CMM. + Returns a DecryptionMaterialsFromMPL for the configured CMM. :param request: Request for decryption materials """ try: - mpl_input: 'DecryptMaterialsInput' = \ - MPLCMMHandler._create_mpl_decrypt_materials_input_from_request(request) - mpl_output: 'DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(mpl_input) - return MPLDecryptionMaterials(mpl_output.decryption_materials) + mpl_input: 'MPL_DecryptMaterialsInput' = \ + CryptoMaterialsManagerFromMPL._create_mpl_decrypt_materials_input_from_request(request) + mpl_output: 'MPL_DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(mpl_input) + return DecryptionMaterialsFromMPL(mpl_output.decryption_materials) except AwsCryptographicMaterialProvidersException as mpl_exception: # Wrap MPL error into the ESDK error type # so customers only have to catch ESDK error types. raise AWSEncryptionSDKClientError(mpl_exception) @staticmethod - def _native_algorithm_id_to_mpl_algorithm_id(native_algorithm_id: str) -> 'AlgorithmSuiteIdESDK': + def _native_algorithm_id_to_mpl_algorithm_id(native_algorithm_id: str) -> 'MPL_AlgorithmSuiteIdESDK': # MPL algorithm suite ID = hexstr(native_algorithm_id) padded to 4 digits post-`x`. - return AlgorithmSuiteIdESDK(f"{native_algorithm_id:#0{6}x}") + return MPL_AlgorithmSuiteIdESDK(f"{native_algorithm_id:#0{6}x}") @staticmethod def _create_mpl_decrypt_materials_input_from_request( request: DecryptionMaterialsRequest - ) -> 'DecryptMaterialsInput': + ) -> 'MPL_DecryptMaterialsInput': key_blob_list: List[Native_EncryptedDataKey] = request.encrypted_data_keys list_edks = [MPL_EncryptedDataKey( key_provider_id=key_blob.key_provider.provider_id, key_provider_info=key_blob.key_provider.key_info, ciphertext=key_blob.encrypted_data_key, ) for key_blob in key_blob_list] - output: DecryptMaterialsInput = DecryptMaterialsInput( - algorithm_suite_id=MPLCMMHandler._native_algorithm_id_to_mpl_algorithm_id( + output: MPL_DecryptMaterialsInput = MPL_DecryptMaterialsInput( + algorithm_suite_id=CryptoMaterialsManagerFromMPL._native_algorithm_id_to_mpl_algorithm_id( request.algorithm.algorithm_id ), - commitment_policy=MPLCMMHandler._native_to_mpl_commmitment_policy( + commitment_policy=CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy( request.commitment_policy ), encrypted_data_keys=list_edks, diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index 1ea2a199d..2bdf3f810 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -21,7 +21,7 @@ def _mpl_algorithm_id_to_native_algorithm_id(mpl_algorithm_id: str) -> int: return int(mpl_algorithm_id, 16) -class MPLEncryptionMaterials(Native_EncryptionMaterials): +class EncryptionMaterialsFromMPL(Native_EncryptionMaterials): """ In instances where encryption materials are be provided by the MPL's `aws_cryptographic_materialproviders.mpl.models.EncryptionMaterials`, @@ -35,13 +35,13 @@ def __init__( mpl_materials: 'MPL_EncryptionMaterials' ): """ - Create MPLEncryptionMaterials. + Create EncryptionMaterialsFromMPL. :param materials: Underlying encryption materials """ if isinstance(mpl_materials, MPL_EncryptionMaterials): self.mpl_materials = mpl_materials else: - raise ValueError("Invalid EncryptionMaterials passed to MPLEncryptionMaterials. " + raise ValueError("Invalid EncryptionMaterials passed to EncryptionMaterialsFromMPL. " f"materials: {mpl_materials}") @property @@ -91,7 +91,7 @@ def signing_key(self) -> bytes: return self.mpl_materials.signing_key -class MPLDecryptionMaterials(Native_DecryptionMaterials): +class DecryptionMaterialsFromMPL(Native_DecryptionMaterials): """ In instances where decryption materials are be provided by the MPL's `aws_cryptographic_materialproviders.mpl.models.DecryptionMaterials`, @@ -105,13 +105,13 @@ def __init__( mpl_materials: 'MPL_DecryptionMaterials' ): """ - Create MPLDecryptionMaterials. + Create DecryptionMaterialsFromMPL. :param materials: Underlying decryption materials """ if isinstance(mpl_materials, MPL_DecryptionMaterials): self.mpl_materials = mpl_materials else: - raise ValueError(f"Invalid DecryptionMaterials passed to MPLDecryptionMaterials.\ + raise ValueError(f"Invalid DecryptionMaterials passed to DecryptionMaterialsFromMPL.\ materials: {mpl_materials}") @property diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index a3c05bbb7..72ed4efb7 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -80,7 +80,7 @@ _HAS_MPL = True # Import internal ESDK modules that depend on the MPL - from aws_encryption_sdk.materials_managers.mpl.cmm import MPLCMMHandler + from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL except ImportError: _HAS_MPL = False @@ -172,7 +172,7 @@ def _has_mpl_attrs_post_init(self): keyring=self.keyring ) ) - cmm_handler: CryptoMaterialsManager = MPLCMMHandler(cmm) + cmm_handler: CryptoMaterialsManager = CryptoMaterialsManagerFromMPL(cmm) self.materials_manager = cmm_handler def _no_mpl_attrs_post_init(self): @@ -555,7 +555,7 @@ def _prep_message(self): # MPL verification key is PEM bytes, not DER bytes. # If the underlying CMM is from the MPL, load PEM bytes. if (_HAS_MPL - and isinstance(self.config.materials_manager, MPLCMMHandler)): + and isinstance(self.config.materials_manager, CryptoMaterialsManagerFromMPL)): self.signer = Signer.from_key_bytes( algorithm=self._encryption_materials.algorithm, key_bytes=self._encryption_materials.signing_key, encoding=serialization.Encoding.PEM, @@ -923,7 +923,7 @@ def _read_header(self): # MPL verification key is NOT key bytes; it is bytes of the compressed point. # If the underlying CMM is from the MPL, load bytes from encoded point. if (_HAS_MPL - and isinstance(self.config.materials_manager, MPLCMMHandler)): + and isinstance(self.config.materials_manager, CryptoMaterialsManagerFromMPL)): self.verifier = Verifier.from_encoded_point( algorithm=header.algorithm, encoded_point=base64.b64encode(decryption_materials.verification_key) diff --git a/test/mpl/unit/test_material_managers_mpl_cmm.py b/test/mpl/unit/test_material_managers_mpl_cmm.py index cae334722..22f8bf63e 100644 --- a/test/mpl/unit/test_material_managers_mpl_cmm.py +++ b/test/mpl/unit/test_material_managers_mpl_cmm.py @@ -17,11 +17,10 @@ from aws_encryption_sdk.identifiers import CommitmentPolicy -import aws_encryption_sdk.materials_managers.mpl.cmm -from aws_encryption_sdk.materials_managers.mpl.cmm import MPLCMMHandler +from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL from aws_encryption_sdk.materials_managers.mpl.materials import ( - MPLEncryptionMaterials, - MPLDecryptionMaterials, + EncryptionMaterialsFromMPL, + DecryptionMaterialsFromMPL, ) pytestmark = [pytest.mark.unit, pytest.mark.local] @@ -51,180 +50,234 @@ EncryptionMaterialsRequest, DecryptionMaterialsRequest, ) +from aws_encryption_sdk.structures import EncryptedDataKey as Native_EncryptedDataKey mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) -mock_encryption_materials_handler = MagicMock(__class__=MPLEncryptionMaterials) +mock_encryption_materials_handler = MagicMock(__class__=EncryptionMaterialsFromMPL) mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) +mock_edk = MagicMock(__class__=Native_EncryptedDataKey) +mock_mpl_key_provider_id = MagicMock(__class__=str) +mock_edk.key_provider.provider_id = mock_mpl_key_provider_id +mock_mpl_key_provider_info = MagicMock(__class__=bytes) +mock_edk.key_provider.key_info = mock_mpl_key_provider_info +mock_mpl_encrypted_data_key = MagicMock(__class__=bytes) +mock_edk.encrypted_data_key = mock_mpl_encrypted_data_key -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_valid_mpl_cmm_THEN_return_new_MPLCMMHandler(): - mpl_cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) - - assert mpl_cmm_handler.mpl_cmm == mock_mpl_cmm + +def test_GIVEN_valid_mpl_cmm_WHEN_create_CryptoMaterialsManagerFromMPL_THEN_return_new_CryptoMaterialsManagerFromMPL(): + # Given: valid mpl_cmm + # When: create new CryptoMaterialsManagerFromMPL + mpl_cmm = CryptoMaterialsManagerFromMPL(mpl_cmm=mock_mpl_cmm) + # Then: CryptoMaterialsManagerFromMPL is valid + assert mpl_cmm.mpl_cmm == mock_mpl_cmm -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_invalid_mpl_cmm_THEN_raise_ValueError(): +def test_GIVEN_invalid_mpl_cmm_WHEN_create_CryptoMaterialsManagerFromMPL_THEN_raise_ValueError(): + # Then: raises ValueError with pytest.raises(ValueError): - MPLCMMHandler(mpl_cmm="not a valid mpl_cmm") + # Given: invalid mpl_cmm + # When: create new CryptoMaterialsManagerFromMPL + CryptoMaterialsManagerFromMPL(mpl_cmm="not a valid mpl_cmm") @patch.object(mock_mpl_cmm, "get_encryption_materials") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_get_encryption_materials") -def test_GIVEN_valid_request_WHEN_call_get_encryption_materials_THEN_return_MPLEncryptionMaterials( +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._native_to_mpl_get_encryption_materials") +def test_GIVEN_valid_request_WHEN_get_encryption_materials_THEN_return_EncryptionMaterialsFromMPL( mock_native_to_mpl_get_encryption_materials, mock_get_encryption_materials, ): - # Mock: mpl_cmm.get_encryption_materials returns mock MPL encryption materials + # Given: _native_to_mpl_get_encryption_materials creates a GetEncryptionMaterialsInput + mock_get_encryption_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) + mock_native_to_mpl_get_encryption_materials.return_value = mock_get_encryption_materials_input + + # Given: mpl_cmm.get_encryption_materials returns mock MPL encryption materials mock_get_encryption_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) mock_get_encryption_materials_output.encryption_materials = mock_mpl_encryption_materials mock_get_encryption_materials.return_value = mock_get_encryption_materials_output - # Mock: CMMHandler._native_to_mpl_get_encryption_materials creates a GetEncryptionMaterialsInput - mock_get_encryption_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) - mock_native_to_mpl_get_encryption_materials.return_value = mock_get_encryption_materials_input - - cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) - test = cmm_handler.get_encryption_materials(mock_encryption_materials_request) + # When: get_encryption_materials + cmm = CryptoMaterialsManagerFromMPL(mpl_cmm=mock_mpl_cmm) + output = cmm.get_encryption_materials(mock_encryption_materials_request) - # Verify cmm_handler returns MPLEncryptionMaterials - assert isinstance(test, MPLEncryptionMaterials) + # Then: + # Verify cmm returns EncryptionMaterialsFromMPL + assert isinstance(output, EncryptionMaterialsFromMPL) # Verify returned EncryptionMaterialsHandler uses the output of `get_encryption_materials` - assert test.mpl_materials == mock_mpl_encryption_materials + assert output.mpl_materials == mock_mpl_encryption_materials # Verify we actually called `get_encryption_materials` mock_mpl_cmm.get_encryption_materials.assert_called_once_with(mock_get_encryption_materials_input) -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") -def test_GIVEN_get_encryption_materials_raises_MPL_Exception_WHEN_call_get_encryption_materials_THEN_raise_ESDK_Exception( +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy") +def test_GIVEN_mpl_cmm_raises_MPLException_WHEN_get_encryption_materials_THEN_raise_ESDKException( _ ): + # Then: Raises AWSEncryptionSDKClientError with pytest.raises(AWSEncryptionSDKClientError): + # Given: mpl_cmm.get_encryption_materials raises MPL exception with patch.object(mock_mpl_cmm, "get_encryption_materials", side_effect=AwsCryptographicMaterialProvidersException("any")): - - cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) - cmm_handler.get_encryption_materials(mock_encryption_materials_request) + # When: get_encryption_materials + cmm = CryptoMaterialsManagerFromMPL(mpl_cmm=mock_mpl_cmm) + cmm.get_encryption_materials(mock_encryption_materials_request) -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") -def test_GIVEN_native_to_mpl_commmitment_policy_returns_valid_policy_WHEN_call_native_to_mpl_get_encryption_materials_THEN_returns_GetEncryptionMaterialsInput( +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy") +def test_GIVEN_valid_mpl_commitment_policy_WHEN_native_to_mpl_get_encryption_materials_THEN_returns_GetEncryptionMaterialsInput( mock_mpl_commitment_policy ): + # Given: commitment policy is some MPL ESDK commitment policy mock_commitment_policy = MagicMock(__class__=CommitmentPolicyESDK) mock_mpl_commitment_policy.return_value = mock_commitment_policy - output = MPLCMMHandler._native_to_mpl_get_encryption_materials(mock_encryption_materials_request) + # When: _native_to_mpl_get_encryption_materials + output = CryptoMaterialsManagerFromMPL._native_to_mpl_get_encryption_materials(mock_encryption_materials_request) - # verify correctness of returned value + # Then: returned GetEncryptionMaterialsInput is correct assert isinstance(output, GetEncryptionMaterialsInput) assert output.encryption_context == mock_encryption_materials_request.encryption_context assert output.commitment_policy == mock_commitment_policy assert output.max_plaintext_length == mock_encryption_materials_request.plaintext_length -def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_FORBID_ENCRYPT_ALLOW_DECRYPT(): +def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_CommitmentPolicyESDK_FORBID_ENCRYPT_ALLOW_DECRYPT(): + # Given: native FORBID_ENCRYPT_ALLOW_DECRYPT native_commitment_policy = CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT - output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) + # When: _native_to_mpl_commmitment_policy + output = CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy(native_commitment_policy) + # Then: Returns MPL FORBID_ENCRYPT_ALLOW_DECRYPT assert isinstance(output, CommitmentPolicyESDK) assert output.value == "FORBID_ENCRYPT_ALLOW_DECRYPT" -def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_ALLOW_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_REQUIRE_ENCRYPT_ALLOW_DECRYPT(): +def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_CommitmentPolicyESDK_REQUIRE_ENCRYPT_ALLOW_DECRYPT(): + # Given: native REQUIRE_ENCRYPT_ALLOW_DECRYPT native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT - output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) + # When: _native_to_mpl_commmitment_policy + output = CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy(native_commitment_policy) + # Then: Returns MPL REQUIRE_ENCRYPT_ALLOW_DECRYPT assert isinstance(output, CommitmentPolicyESDK) assert output.value == "REQUIRE_ENCRYPT_ALLOW_DECRYPT" -def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_REQUIRE_DECRYPT_WHEN_call_native_to_mpl_commmitment_policyTHEN_returns_CommitmentPolicyESDK_REQUIRE_ENCRYPT_REQUIRE_DECRYPT(): +def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_REQUIRE_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_CommitmentPolicyESDK_REQUIRE_ENCRYPT_REQUIRE_DECRYPT(): + # Given: native REQUIRE_ENCRYPT_REQUIRE_DECRYPT native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT - output = MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) + # When: _native_to_mpl_commmitment_policy + output = CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy(native_commitment_policy) + # Then: Returns MPL REQUIRE_ENCRYPT_REQUIRE_DECRYPT assert isinstance(output, CommitmentPolicyESDK) assert output.value == "REQUIRE_ENCRYPT_REQUIRE_DECRYPT" -def test_GIVEN_CommitmentPolicy_unrecognized_WHEN_call_native_to_mpl_commmitment_policyTHEN_raise_ValueError(): +def test_GIVEN_CommitmentPolicy_unrecognized_WHEN_native_to_mpl_commmitment_policy_THEN_raise_ValueError(): + # Given: invalid native commitment policy native_commitment_policy = "not a commitment policy" + # Then: Raises ValueError with pytest.raises(ValueError): - MPLCMMHandler._native_to_mpl_commmitment_policy(native_commitment_policy) + # When: _native_to_mpl_commmitment_policy + CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy(native_commitment_policy) @patch.object(mock_mpl_cmm, "decrypt_materials") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._create_mpl_decrypt_materials_input_from_request") -def test_GIVEN_valid_request_WHEN_call_decrypt_materials_THEN_return_MPLDecryptionMaterials( +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._create_mpl_decrypt_materials_input_from_request") +def test_GIVEN_valid_request_WHEN_decrypt_materials_THEN_return_DecryptionMaterialsFromMPL( mock_native_to_mpl_decrypt_materials, mock_get_encryption_materials, ): - # Mock: mpl_cmm.get_decryption_materials returns mock MPL decryption materials + # Given: mpl_cmm.get_decryption_materials returns mock MPL decryption materials mock_decrypt_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) mock_decrypt_materials_output.decryption_materials = mock_mpl_decrypt_materials mock_get_encryption_materials.return_value = mock_decrypt_materials_output - # Mock: CMMHandler._create_mpl_decrypt_materials_input_from_request creates a DecryptMaterialsInput + # Given: CMMHandler._create_mpl_decrypt_materials_input_from_request creates a DecryptMaterialsInput mock_decrypt_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) mock_native_to_mpl_decrypt_materials.return_value = mock_decrypt_materials_input - cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) - output = cmm_handler.decrypt_materials(mock_decryption_materials_request) + # When: decrypt_materials + cmm = CryptoMaterialsManagerFromMPL(mpl_cmm=mock_mpl_cmm) + output = cmm.decrypt_materials(mock_decryption_materials_request) - # Verify cmm_handler returns MPLDecryptionMaterials - assert isinstance(output, MPLDecryptionMaterials) - # Verify returned MPLDecryptionMaterials uses the output of `decrypt_materials` + # Then: + # Verify cmm returns DecryptionMaterialsFromMPL + assert isinstance(output, DecryptionMaterialsFromMPL) + # Verify returned DecryptionMaterialsFromMPL uses the output of `decrypt_materials` assert output.mpl_materials == mock_mpl_decrypt_materials # Verify we actually called `decrypt_materials` mock_mpl_cmm.decrypt_materials.assert_called_once_with(mock_decrypt_materials_input) -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._create_mpl_decrypt_materials_input_from_request") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._create_mpl_decrypt_materials_input_from_request") def test_GIVEN_decrypt_materials_raises_MPL_Exception_WHEN_call_decrypt_materials_THEN_raise_ESDK_Exception( _ ): + # Then: Raises AWSEncryptionSDKClientError with pytest.raises(AWSEncryptionSDKClientError): + # Given: mpl_cmm.decrypt_materials raises MPL exception with patch.object(mock_mpl_cmm, "decrypt_materials", side_effect=AwsCryptographicMaterialProvidersException("any")): - - cmm_handler = MPLCMMHandler(mpl_cmm=mock_mpl_cmm) - cmm_handler.decrypt_materials(mock_decryption_materials_request) + # When: decrypt_materials + cmm = CryptoMaterialsManagerFromMPL(mpl_cmm=mock_mpl_cmm) + cmm.decrypt_materials(mock_decryption_materials_request) -def test_WHEN_call_native_algorithm_id_to_mpl_algorithm_id_THEN_returns_valid_AlgorithmSuiteIdESDK(): - some_native_algorithm_id = 0x0000 # Not a real algorithm ID, but fits the format +def test_GIVEN_valid_native_algorithm_id_WHEN_native_algorithm_id_to_mpl_algorithm_id_THEN_returns_valid_AlgorithmSuiteIdESDK(): + # Given: any native algorithm ID + some_native_algorithm_id = 0x1234 # Not a real algorithm ID, but fits the format - mpl_output = MPLCMMHandler._native_algorithm_id_to_mpl_algorithm_id( + # When: _native_algorithm_id_to_mpl_algorithm_id + mpl_output = CryptoMaterialsManagerFromMPL._native_algorithm_id_to_mpl_algorithm_id( some_native_algorithm_id ) + # Then: returns valid MPL algorithm ID assert isinstance(mpl_output, AlgorithmSuiteIdESDK) - assert mpl_output.value == "0x0000" + assert mpl_output.value == "0x1234" -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_algorithm_id_to_mpl_algorithm_id") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.MPLCMMHandler._native_to_mpl_commmitment_policy") -def test__create_mpl_decrypt_materials_input_from_request( +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._native_algorithm_id_to_mpl_algorithm_id") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy") +def test_GIVEN_valid_request_WHEN_create_mpl_decrypt_materials_input_from_request_THEN_returns_MPL_DecryptMaterialsInput( mock_mpl_commitment_policy, mock_mpl_algorithm_id, ): + # Given: _native_algorithm_id_to_mpl_algorithm_id returns a valid MPL algorithm ID mock_algorithm_id = "0x1234" # Some fake algorithm ID that fits the format mock_mpl_algorithm_id.return_value = mock_algorithm_id + + # Given: _native_to_mpl_commmitment_policy returns some MPL commitment policy mock_commitment_policy = MagicMock(__class__=CommitmentPolicyESDK) mock_mpl_commitment_policy.return_value = mock_commitment_policy - # mock_decryption_materials_request.algorithm = - - output = MPLCMMHandler._create_mpl_decrypt_materials_input_from_request(mock_decryption_materials_request) - - assert isinstance(output, DecryptMaterialsInput) - assert output.algorithm_suite_id == mock_algorithm_id - assert output.commitment_policy == mock_commitment_policy - assert output.encryption_context == mock_decryption_materials_request.encryption_context - - assert len(output.encrypted_data_keys) == len(mock_decryption_materials_request.encrypted_data_keys) - for i in range(len(output.encrypted_data_keys)): - # Assume input[i] == output[i], seems to work - output_edk = output.encrypted_data_keys[i] - input_edk = mock_decryption_materials_request[i] - assert output_edk.key_provider_id == input_edk.key_provider.provider_id - assert output_edk.key_provider_info == input_edk.key_provider.key_info - assert output_edk.ciphertext == input_edk.encrypted_data_key + no_mock_edks = [ mock_edk ] + one_mock_edk = [ mock_edk ] + two_mock_edks = [ mock_edk, mock_edk ] + + # Given: ESK lists of various lengths + for mock_edks in [ no_mock_edks, one_mock_edk, two_mock_edks ]: + + mock_decryption_materials_request.encrypted_data_keys = mock_edks + + # When: _create_mpl_decrypt_materials_input_from_request + output = CryptoMaterialsManagerFromMPL._create_mpl_decrypt_materials_input_from_request(mock_decryption_materials_request) + + # Then: + # Verify general correctness of output structure + assert isinstance(output, DecryptMaterialsInput) + assert output.algorithm_suite_id == mock_algorithm_id + assert output.commitment_policy == mock_commitment_policy + assert output.encryption_context == mock_decryption_materials_request.encryption_context + + assert len(output.encrypted_data_keys) == len(mock_edks) + for i in range(len(output.encrypted_data_keys)): + # Assume input[i] == output[i] to make validation easier + # This is how the src is implemented but is not a requirement. + # If this assumption breaks, we should enhance this test. + output_edk = output.encrypted_data_keys[i] + input_edk = mock_edks[i] + assert output_edk.key_provider_id == input_edk.key_provider.provider_id + assert output_edk.key_provider_info == input_edk.key_provider.key_info + assert output_edk.ciphertext == input_edk.encrypted_data_key \ No newline at end of file diff --git a/test/mpl/unit/test_material_managers_mpl_materials.py b/test/mpl/unit/test_material_managers_mpl_materials.py index bb83b89fd..b39e9bc8d 100644 --- a/test/mpl/unit/test_material_managers_mpl_materials.py +++ b/test/mpl/unit/test_material_managers_mpl_materials.py @@ -14,13 +14,12 @@ import pytest from mock import MagicMock, patch, PropertyMock -from typing import Dict, List +from typing import Dict, List, Set -from aws_encryption_sdk.identifiers import CommitmentPolicy import aws_encryption_sdk.materials_managers.mpl.materials from aws_encryption_sdk.materials_managers.mpl.materials import ( - MPLEncryptionMaterials, - MPLDecryptionMaterials, + EncryptionMaterialsFromMPL, + DecryptionMaterialsFromMPL, ) from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite @@ -54,124 +53,164 @@ mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) -mock_encryption_materials_handler = MagicMock(__class__=MPLEncryptionMaterials) +mock_encryption_materials_handler = MagicMock(__class__=EncryptionMaterialsFromMPL) mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) +mock_edk = MagicMock(__class__=MPL_EncryptedDataKey) +mock_mpl_key_provider_id = MagicMock(__class__=str) +mock_edk.key_provider_id = mock_mpl_key_provider_id +mock_mpl_key_provider_info = MagicMock(__class__=bytes) +mock_edk.key_provider_info = mock_mpl_key_provider_info +mock_mpl_ciphertext = MagicMock(__class__=bytes) +mock_edk.ciphertext = mock_mpl_ciphertext -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_valid_mpl_cmm_THEN_return_new_MPLCMMHandler(): - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + +def test_GIVEN_valid_mpl_materials_WHEN_create_EncryptionMaterialsFromMPL_THEN_return_new_CryptoMaterialsManagerFromMPL(): + # Given: valid mpl_materials + # When: create EncryptionMaterialsFromMPL + mpl_encryption_materials = EncryptionMaterialsFromMPL(mpl_materials=mock_mpl_encryption_materials) + # Then: EncryptionMaterialsFromMPL is valid assert mpl_encryption_materials.mpl_materials == mock_mpl_encryption_materials -def test_GIVEN_test_has_mpl_is_False_WHEN_create_MPLCMMHandler_with_invalid_mpl_cmm_THEN_raise_ValueError(): +def test_GIVEN_invalid_mpl_materials_WHEN_create_EncryptionMaterialsFromMPL_THEN_raise_ValueError(): + # Then: Raise ValueError with pytest.raises(ValueError): - MPLEncryptionMaterials(mpl_materials="not a valid mpl_materials") + # Given: invalid mpl_materials + # When: create EncryptionMaterialsFromMPL + EncryptionMaterialsFromMPL(mpl_materials="not a valid mpl_materials") + -def test_mpl_to_native(): +def test_GIVEN_valid_mpl_algorithm_id_WHEN_mpl_algorithm_id_to_native_algorithm_id_THEN_valid_native_output(): + # Given: any valid MPL algorithm ID some_mpl_algorithm_id = "0x1234" # Not a real algorithm ID, but fits the format + # When: _mpl_algorithm_id_to_native_algorithm_id native_output = aws_encryption_sdk.materials_managers.mpl.materials._mpl_algorithm_id_to_native_algorithm_id( some_mpl_algorithm_id ) + # Then: valid native algorithm ID assert native_output == 0x1234 @patch("aws_encryption_sdk.materials_managers.mpl.materials._mpl_algorithm_id_to_native_algorithm_id") @patch("aws_encryption_sdk.materials_managers.mpl.materials.AlgorithmSuite.get_by_id") -def test_GIVEN_valid_mpl_algorithm_id_WHEN_get_algorithm_THEN_valid_native_algorithm_id( +def test_GIVEN_valid_mpl_algorithm_id_WHEN_EncryptionMaterials_get_algorithm_THEN_valid_native_algorithm_id( mock_algorithm, mock_native_algorithm_id, ): - # Mock valid conversion from MPL to native algorithm ID + # Given: _mpl_algorithm_id_to_native_algorithm_id returns a valid native algorithm ID mock_native_algorithm_id.return_value = 0x1234 - # Mock valid lookup in native AlgorithmSuite lookup + # Given: get_by_id returns a valid native AlgorithmSuite by looking up an ID mock_algorithm.return_value = MagicMock(__class__=AlgorithmSuite) - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + # When: Get algorithm + mpl_encryption_materials = EncryptionMaterialsFromMPL(mpl_materials=mock_mpl_encryption_materials) output = mpl_encryption_materials.algorithm + + # Then: output is valid assert output == mock_algorithm() # property calls automatically, we need to call the mock -def test_GecTHEN_valid_native_algorithm_id(): +def test_GIVEN_valid_encryption_context_WHEN_EncryptionMaterials_get_encryption_context_THEN_valid_encryption_context(): + # Given: valid encryption context mock_encryption_context = MagicMock(__class__=Dict[str, str]) mock_mpl_encryption_materials.encryption_context = mock_encryption_context - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + # When: get encryption context + mpl_encryption_materials = EncryptionMaterialsFromMPL(mpl_materials=mock_mpl_encryption_materials) output = mpl_encryption_materials.encryption_context + # Then: returns valid encryption context assert output == mock_encryption_context -def test_GecTHEN_valid_nativefadsf_algorithm_id(): - mock_edk = MagicMock(__class__=MPL_EncryptedDataKey) - mock_mpl_key_provider_id = MagicMock(__class__=str) - mock_edk.key_provider_id = mock_mpl_key_provider_id - mock_mpl_key_provider_info = MagicMock(__class__=bytes) - mock_edk.key_provider_info = mock_mpl_key_provider_info - mock_mpl_ciphertext = MagicMock(__class__=bytes) - mock_edk.ciphertext = mock_mpl_ciphertext - - mock_edks = [ mock_edk ] - mock_mpl_encryption_materials.encrypted_data_keys = mock_edks - - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) - output = mpl_encryption_materials.encrypted_data_keys - output_as_list = list(output) - - assert len(output_as_list) == len(mock_edks) - for i in range(len(output_as_list)): - # assume output[i] corresponds to input[i] - native_edk = output_as_list[i] - mpl_edk = mock_edks[i] - - assert native_edk.encrypted_data_key == mpl_edk.ciphertext - assert native_edk.key_provider.provider_id == mpl_edk.key_provider_id - assert native_edk.key_provider.key_info == mpl_edk.key_provider_info - -def test_GecTHEN_valid_nativefadsffadsfa_algorithm_id(): +def test_GIVEN_valid_edks_WHEN_EncryptionMaterials_get_edks_THEN_returns_edks(): + + # Given: lists of mocked EDKs of various lengths + no_mock_edks = [] + one_mock_edk = [ mock_edk ] + two_mocked_edks = [ mock_edk, mock_edk ] + for mock_edks in [ no_mock_edks, one_mock_edk, two_mocked_edks ]: + mock_mpl_encryption_materials.encrypted_data_keys = mock_edks + + # When: get EDKs + mpl_encryption_materials = EncryptionMaterialsFromMPL(mpl_materials=mock_mpl_encryption_materials) + output = mpl_encryption_materials.encrypted_data_keys + + # Then: returns EDKs + output_as_list = list(output) + # Native ESDK Python types the EDKs as a set; + # Ensure the MPL's list is collapsed into a set correctly + assert len(output_as_list) == len(set(mock_edks)) + for i in range(len(output_as_list)): + # Assume input[i] == output[i] to make validation easier + # This is how the src is implemented but is not a requirement. + # If this assumption breaks, we should enhance this test. + native_edk = output_as_list[i] + mpl_edk = mock_edks[i] + + assert native_edk.encrypted_data_key == mpl_edk.ciphertext + assert native_edk.key_provider.provider_id == mpl_edk.key_provider_id + assert native_edk.key_provider.key_info == mpl_edk.key_provider_info + + +def test_GIVEN_valid_data_key_WHEN_EncryptionMaterials_get_data_key_THEN_returns_data_key(): + # Given: Valid MPL data key mock_data_key = MagicMock(__class__=bytes) mock_mpl_encryption_materials.plaintext_data_key = mock_data_key - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + # When: get data key + mpl_encryption_materials = EncryptionMaterialsFromMPL(mpl_materials=mock_mpl_encryption_materials) output = mpl_encryption_materials.data_encryption_key + # Then: Returns native data key assert output.key_provider.provider_id == "" assert output.key_provider.key_info == b"" assert output.data_key == mock_data_key assert output.encrypted_data_key == b"" -def test_GecTHEN_valid_nativefasdfasdffadsf_algorithm_id(): +def test_GIVEN_valid_signing_key_WHEN_EncryptionMaterials_get_signing_key_THEN_returns_signing_key(): + # Given: valid signing key mock_signing_key = MagicMock(__class__=bytes) mock_mpl_encryption_materials.signing_key = mock_signing_key - mpl_encryption_materials = MPLEncryptionMaterials(mpl_materials=mock_mpl_encryption_materials) + # When: get signing key + mpl_encryption_materials = EncryptionMaterialsFromMPL(mpl_materials=mock_mpl_encryption_materials) output = mpl_encryption_materials.signing_key + # Then: returns signing key assert output == mock_signing_key -def test_GecTHEN_valid_nativeffasdfasdadsffadsfa_algorithm_id(): +def test_GIVEN_valid_data_key_WHEN_DecryptionMaterials_get_data_key_THEN_returns_data_key(): + # Given: valid MPL data key mock_data_key = MagicMock(__class__=bytes) mock_mpl_decrypt_materials.plaintext_data_key = mock_data_key - mpl_decryption_materials = MPLDecryptionMaterials(mpl_materials=mock_mpl_decrypt_materials) + # When: get data key + mpl_decryption_materials = DecryptionMaterialsFromMPL(mpl_materials=mock_mpl_decrypt_materials) output = mpl_decryption_materials.data_key + # Then: returns valid native data key assert output.key_provider.provider_id == "" assert output.key_provider.key_info == b"" assert output.data_key == mock_data_key assert output.encrypted_data_key == b"" -def test_GecTHEN_validadsfasdf_nativefasdfasdffadsf_algorithm_id(): +def test_GIVEN_valid_verification_key_WHEN_DecryptionMaterials_get_verification_key_THEN_returns_verification_key(): + # Given: valid verification key mock_verification_key = MagicMock(__class__=bytes) mock_mpl_decrypt_materials.verification_key = mock_verification_key - mpl_decryption_materials = MPLDecryptionMaterials(mpl_materials=mock_mpl_decrypt_materials) + # When: get verification key + mpl_decryption_materials = DecryptionMaterialsFromMPL(mpl_materials=mock_mpl_decrypt_materials) output = mpl_decryption_materials.verification_key + # Then: returns verification key assert output == mock_verification_key From ac6471a921407df409bceb9dc1dafbb5e697544c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 17:00:31 -0800 Subject: [PATCH 118/422] test cleanup --- examples/src/keyrings/hierarchical_keyring.py | 6 +- .../unit/test_material_managers_mpl_cmm.py | 56 +++++++++---------- .../test_material_managers_mpl_materials.py | 11 ---- 3 files changed, 31 insertions(+), 42 deletions(-) diff --git a/examples/src/keyrings/hierarchical_keyring.py b/examples/src/keyrings/hierarchical_keyring.py index c71719346..aa87485f9 100644 --- a/examples/src/keyrings/hierarchical_keyring.py +++ b/examples/src/keyrings/hierarchical_keyring.py @@ -4,12 +4,12 @@ import sys import boto3 -# ignore missing MPL for pylint, but the MPL is required for this example +# Ignore missing MPL for pylint, but the MPL is required for this example # noqa pylint: disable=import-error -from aws_cryptographic_materialproviders.keystore.client import KeyStore +from aws_cryptographic_materialproviders.keystore import KeyStore from aws_cryptographic_materialproviders.keystore.config import KeyStoreConfig from aws_cryptographic_materialproviders.keystore.models import CreateKeyInput, KMSConfigurationKmsKeyArn -from aws_cryptographic_materialproviders.mpl.client import AwsCryptographicMaterialProviders +from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig from aws_cryptographic_materialproviders.mpl.models import ( CacheTypeDefault, diff --git a/test/mpl/unit/test_material_managers_mpl_cmm.py b/test/mpl/unit/test_material_managers_mpl_cmm.py index 22f8bf63e..52a4b333c 100644 --- a/test/mpl/unit/test_material_managers_mpl_cmm.py +++ b/test/mpl/unit/test_material_managers_mpl_cmm.py @@ -28,19 +28,19 @@ from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException from aws_cryptographic_materialproviders.mpl.models import ( - AlgorithmSuiteIdESDK, - CommitmentPolicyESDK, - DecryptMaterialsInput, + AlgorithmSuiteIdESDK as MPL_AlgorithmSuiteIdESDK, + CommitmentPolicyESDK as MPL_CommitmentPolicyESDK, + DecryptMaterialsInput as MPL_DecryptMaterialsInput, DecryptionMaterials as MPL_DecryptionMaterials, EncryptionMaterials as MPL_EncryptionMaterials, - GetEncryptionMaterialsInput, - GetEncryptionMaterialsOutput, + GetEncryptionMaterialsInput as MPL_GetEncryptionMaterialsInput, + GetEncryptionMaterialsOutput as MPL_GetEncryptionMaterialsOutput, ) from aws_cryptographic_materialproviders.mpl.references import ( - ICryptographicMaterialsManager + ICryptographicMaterialsManager as MPL_ICryptographicMaterialsManager, ) -mock_mpl_cmm = MagicMock(__class__=ICryptographicMaterialsManager) +mock_mpl_cmm = MagicMock(__class__=MPL_ICryptographicMaterialsManager) mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) @@ -89,12 +89,12 @@ def test_GIVEN_valid_request_WHEN_get_encryption_materials_THEN_return_Encryptio mock_get_encryption_materials, ): - # Given: _native_to_mpl_get_encryption_materials creates a GetEncryptionMaterialsInput - mock_get_encryption_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) + # Given: _native_to_mpl_get_encryption_materials creates a MPL_GetEncryptionMaterialsInput + mock_get_encryption_materials_input = MagicMock(__class__=MPL_GetEncryptionMaterialsInput) mock_native_to_mpl_get_encryption_materials.return_value = mock_get_encryption_materials_input # Given: mpl_cmm.get_encryption_materials returns mock MPL encryption materials - mock_get_encryption_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) + mock_get_encryption_materials_output = MagicMock(__class__=MPL_GetEncryptionMaterialsOutput) mock_get_encryption_materials_output.encryption_materials = mock_mpl_encryption_materials mock_get_encryption_materials.return_value = mock_get_encryption_materials_output @@ -125,24 +125,24 @@ def test_GIVEN_mpl_cmm_raises_MPLException_WHEN_get_encryption_materials_THEN_ra cmm.get_encryption_materials(mock_encryption_materials_request) @patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy") -def test_GIVEN_valid_mpl_commitment_policy_WHEN_native_to_mpl_get_encryption_materials_THEN_returns_GetEncryptionMaterialsInput( +def test_GIVEN_valid_mpl_commitment_policy_WHEN_native_to_mpl_get_encryption_materials_THEN_returns_MPL_GetEncryptionMaterialsInput( mock_mpl_commitment_policy ): # Given: commitment policy is some MPL ESDK commitment policy - mock_commitment_policy = MagicMock(__class__=CommitmentPolicyESDK) + mock_commitment_policy = MagicMock(__class__=MPL_CommitmentPolicyESDK) mock_mpl_commitment_policy.return_value = mock_commitment_policy # When: _native_to_mpl_get_encryption_materials output = CryptoMaterialsManagerFromMPL._native_to_mpl_get_encryption_materials(mock_encryption_materials_request) - # Then: returned GetEncryptionMaterialsInput is correct - assert isinstance(output, GetEncryptionMaterialsInput) + # Then: returned MPL_GetEncryptionMaterialsInput is correct + assert isinstance(output, MPL_GetEncryptionMaterialsInput) assert output.encryption_context == mock_encryption_materials_request.encryption_context assert output.commitment_policy == mock_commitment_policy assert output.max_plaintext_length == mock_encryption_materials_request.plaintext_length -def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_CommitmentPolicyESDK_FORBID_ENCRYPT_ALLOW_DECRYPT(): +def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_MPL_CommitmentPolicyESDK_FORBID_ENCRYPT_ALLOW_DECRYPT(): # Given: native FORBID_ENCRYPT_ALLOW_DECRYPT native_commitment_policy = CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT @@ -150,10 +150,10 @@ def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_ output = CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy(native_commitment_policy) # Then: Returns MPL FORBID_ENCRYPT_ALLOW_DECRYPT - assert isinstance(output, CommitmentPolicyESDK) + assert isinstance(output, MPL_CommitmentPolicyESDK) assert output.value == "FORBID_ENCRYPT_ALLOW_DECRYPT" -def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_CommitmentPolicyESDK_REQUIRE_ENCRYPT_ALLOW_DECRYPT(): +def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_MPL_CommitmentPolicyESDK_REQUIRE_ENCRYPT_ALLOW_DECRYPT(): # Given: native REQUIRE_ENCRYPT_ALLOW_DECRYPT native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT @@ -161,10 +161,10 @@ def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl output = CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy(native_commitment_policy) # Then: Returns MPL REQUIRE_ENCRYPT_ALLOW_DECRYPT - assert isinstance(output, CommitmentPolicyESDK) + assert isinstance(output, MPL_CommitmentPolicyESDK) assert output.value == "REQUIRE_ENCRYPT_ALLOW_DECRYPT" -def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_REQUIRE_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_CommitmentPolicyESDK_REQUIRE_ENCRYPT_REQUIRE_DECRYPT(): +def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_REQUIRE_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_MPL_CommitmentPolicyESDK_REQUIRE_ENCRYPT_REQUIRE_DECRYPT(): # Given: native REQUIRE_ENCRYPT_REQUIRE_DECRYPT native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT @@ -172,7 +172,7 @@ def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_REQUIRE_DECRYPT_WHEN_native_to_m output = CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy(native_commitment_policy) # Then: Returns MPL REQUIRE_ENCRYPT_REQUIRE_DECRYPT - assert isinstance(output, CommitmentPolicyESDK) + assert isinstance(output, MPL_CommitmentPolicyESDK) assert output.value == "REQUIRE_ENCRYPT_REQUIRE_DECRYPT" def test_GIVEN_CommitmentPolicy_unrecognized_WHEN_native_to_mpl_commmitment_policy_THEN_raise_ValueError(): @@ -192,12 +192,12 @@ def test_GIVEN_valid_request_WHEN_decrypt_materials_THEN_return_DecryptionMateri ): # Given: mpl_cmm.get_decryption_materials returns mock MPL decryption materials - mock_decrypt_materials_output = MagicMock(__class__=GetEncryptionMaterialsOutput) + mock_decrypt_materials_output = MagicMock(__class__=MPL_GetEncryptionMaterialsOutput) mock_decrypt_materials_output.decryption_materials = mock_mpl_decrypt_materials mock_get_encryption_materials.return_value = mock_decrypt_materials_output - # Given: CMMHandler._create_mpl_decrypt_materials_input_from_request creates a DecryptMaterialsInput - mock_decrypt_materials_input = MagicMock(__class__=GetEncryptionMaterialsInput) + # Given: CMMHandler._create_mpl_decrypt_materials_input_from_request creates a MPL_DecryptMaterialsInput + mock_decrypt_materials_input = MagicMock(__class__=MPL_GetEncryptionMaterialsInput) mock_native_to_mpl_decrypt_materials.return_value = mock_decrypt_materials_input # When: decrypt_materials @@ -225,7 +225,7 @@ def test_GIVEN_decrypt_materials_raises_MPL_Exception_WHEN_call_decrypt_material cmm = CryptoMaterialsManagerFromMPL(mpl_cmm=mock_mpl_cmm) cmm.decrypt_materials(mock_decryption_materials_request) -def test_GIVEN_valid_native_algorithm_id_WHEN_native_algorithm_id_to_mpl_algorithm_id_THEN_returns_valid_AlgorithmSuiteIdESDK(): +def test_GIVEN_valid_native_algorithm_id_WHEN_native_algorithm_id_to_mpl_algorithm_id_THEN_returns_valid_MPL_AlgorithmSuiteIdESDK(): # Given: any native algorithm ID some_native_algorithm_id = 0x1234 # Not a real algorithm ID, but fits the format @@ -235,12 +235,12 @@ def test_GIVEN_valid_native_algorithm_id_WHEN_native_algorithm_id_to_mpl_algorit ) # Then: returns valid MPL algorithm ID - assert isinstance(mpl_output, AlgorithmSuiteIdESDK) + assert isinstance(mpl_output, MPL_AlgorithmSuiteIdESDK) assert mpl_output.value == "0x1234" @patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._native_algorithm_id_to_mpl_algorithm_id") @patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy") -def test_GIVEN_valid_request_WHEN_create_mpl_decrypt_materials_input_from_request_THEN_returns_MPL_DecryptMaterialsInput( +def test_GIVEN_valid_request_WHEN_create_mpl_decrypt_materials_input_from_request_THEN_returns_MPL_MPL_DecryptMaterialsInput( mock_mpl_commitment_policy, mock_mpl_algorithm_id, ): @@ -249,7 +249,7 @@ def test_GIVEN_valid_request_WHEN_create_mpl_decrypt_materials_input_from_reques mock_mpl_algorithm_id.return_value = mock_algorithm_id # Given: _native_to_mpl_commmitment_policy returns some MPL commitment policy - mock_commitment_policy = MagicMock(__class__=CommitmentPolicyESDK) + mock_commitment_policy = MagicMock(__class__=MPL_CommitmentPolicyESDK) mock_mpl_commitment_policy.return_value = mock_commitment_policy no_mock_edks = [ mock_edk ] @@ -266,7 +266,7 @@ def test_GIVEN_valid_request_WHEN_create_mpl_decrypt_materials_input_from_reques # Then: # Verify general correctness of output structure - assert isinstance(output, DecryptMaterialsInput) + assert isinstance(output, MPL_DecryptMaterialsInput) assert output.algorithm_suite_id == mock_algorithm_id assert output.commitment_policy == mock_commitment_policy assert output.encryption_context == mock_decryption_materials_request.encryption_context diff --git a/test/mpl/unit/test_material_managers_mpl_materials.py b/test/mpl/unit/test_material_managers_mpl_materials.py index b39e9bc8d..92a8c95df 100644 --- a/test/mpl/unit/test_material_managers_mpl_materials.py +++ b/test/mpl/unit/test_material_managers_mpl_materials.py @@ -26,32 +26,21 @@ pytestmark = [pytest.mark.unit, pytest.mark.local] -from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException from aws_cryptographic_materialproviders.mpl.models import ( - AlgorithmSuiteIdESDK, - CommitmentPolicyESDK, - DecryptMaterialsInput, DecryptionMaterials as MPL_DecryptionMaterials, EncryptedDataKey as MPL_EncryptedDataKey, EncryptionMaterials as MPL_EncryptionMaterials, - GetEncryptionMaterialsInput, - GetEncryptionMaterialsOutput, -) -from aws_cryptographic_materialproviders.mpl.references import ( - ICryptographicMaterialsManager ) mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) -from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError from aws_encryption_sdk.materials_managers import ( EncryptionMaterialsRequest, DecryptionMaterialsRequest, ) - mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) mock_encryption_materials_handler = MagicMock(__class__=EncryptionMaterialsFromMPL) mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) From a5ebc19c479f4b6de3874dc4735ca0e27ffbbc38 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 17:18:50 -0800 Subject: [PATCH 119/422] isort --- .../materials_managers/mpl/cmm.py | 5 ++--- .../materials_managers/mpl/materials.py | 1 - test/mpl/unit/test_material_managers_mpl_cmm.py | 14 +++----------- .../unit/test_material_managers_mpl_materials.py | 12 +++--------- 4 files changed, 8 insertions(+), 24 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index c97c070f0..1bbd7c89a 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -11,16 +11,15 @@ GetEncryptionMaterialsOutput as MPL_GetEncryptionMaterialsOutput, ) from aws_cryptographic_materialproviders.mpl.references import ( - ICryptographicMaterialsManager as MPL_ICryptographicMaterialsManager + ICryptographicMaterialsManager as MPL_ICryptographicMaterialsManager, ) - from typing import List from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError from aws_encryption_sdk.identifiers import CommitmentPolicy -from aws_encryption_sdk.materials_managers.mpl.materials import EncryptionMaterialsFromMPL, DecryptionMaterialsFromMPL from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager +from aws_encryption_sdk.materials_managers.mpl.materials import DecryptionMaterialsFromMPL, EncryptionMaterialsFromMPL from aws_encryption_sdk.structures import EncryptedDataKey as Native_EncryptedDataKey diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index 2bdf3f810..31f7d2a65 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -5,7 +5,6 @@ EncryptedDataKey as MPL_EncryptedDataKey, EncryptionMaterials as MPL_EncryptionMaterials, ) - from typing import Dict, List, Set from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite diff --git a/test/mpl/unit/test_material_managers_mpl_cmm.py b/test/mpl/unit/test_material_managers_mpl_cmm.py index 52a4b333c..eb795b7c2 100644 --- a/test/mpl/unit/test_material_managers_mpl_cmm.py +++ b/test/mpl/unit/test_material_managers_mpl_cmm.py @@ -15,13 +15,9 @@ import pytest from mock import MagicMock, patch - from aws_encryption_sdk.identifiers import CommitmentPolicy from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL -from aws_encryption_sdk.materials_managers.mpl.materials import ( - EncryptionMaterialsFromMPL, - DecryptionMaterialsFromMPL, -) +from aws_encryption_sdk.materials_managers.mpl.materials import DecryptionMaterialsFromMPL, EncryptionMaterialsFromMPL pytestmark = [pytest.mark.unit, pytest.mark.local] @@ -30,8 +26,8 @@ from aws_cryptographic_materialproviders.mpl.models import ( AlgorithmSuiteIdESDK as MPL_AlgorithmSuiteIdESDK, CommitmentPolicyESDK as MPL_CommitmentPolicyESDK, - DecryptMaterialsInput as MPL_DecryptMaterialsInput, DecryptionMaterials as MPL_DecryptionMaterials, + DecryptMaterialsInput as MPL_DecryptMaterialsInput, EncryptionMaterials as MPL_EncryptionMaterials, GetEncryptionMaterialsInput as MPL_GetEncryptionMaterialsInput, GetEncryptionMaterialsOutput as MPL_GetEncryptionMaterialsOutput, @@ -46,13 +42,9 @@ from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError -from aws_encryption_sdk.materials_managers import ( - EncryptionMaterialsRequest, - DecryptionMaterialsRequest, -) +from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest from aws_encryption_sdk.structures import EncryptedDataKey as Native_EncryptedDataKey - mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) mock_encryption_materials_handler = MagicMock(__class__=EncryptionMaterialsFromMPL) mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) diff --git a/test/mpl/unit/test_material_managers_mpl_materials.py b/test/mpl/unit/test_material_managers_mpl_materials.py index 92a8c95df..96237998a 100644 --- a/test/mpl/unit/test_material_managers_mpl_materials.py +++ b/test/mpl/unit/test_material_managers_mpl_materials.py @@ -13,15 +13,12 @@ """Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" import pytest -from mock import MagicMock, patch, PropertyMock +from mock import MagicMock, PropertyMock, patch from typing import Dict, List, Set import aws_encryption_sdk.materials_managers.mpl.materials -from aws_encryption_sdk.materials_managers.mpl.materials import ( - EncryptionMaterialsFromMPL, - DecryptionMaterialsFromMPL, -) from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite +from aws_encryption_sdk.materials_managers.mpl.materials import DecryptionMaterialsFromMPL, EncryptionMaterialsFromMPL pytestmark = [pytest.mark.unit, pytest.mark.local] @@ -36,10 +33,7 @@ mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) -from aws_encryption_sdk.materials_managers import ( - EncryptionMaterialsRequest, - DecryptionMaterialsRequest, -) +from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) mock_encryption_materials_handler = MagicMock(__class__=EncryptionMaterialsFromMPL) From 21f361462ec2542056ffcd25ae08bffdb21a7a8d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 17:21:52 -0800 Subject: [PATCH 120/422] fixes --- test/mpl/__init__.py | 2 +- .../unit/test_material_managers_mpl_cmm.py | 28 +++++++++---------- .../test_material_managers_mpl_materials.py | 15 ++++------ 3 files changed, 20 insertions(+), 25 deletions(-) diff --git a/test/mpl/__init__.py b/test/mpl/__init__.py index b976c1308..2a6c71715 100644 --- a/test/mpl/__init__.py +++ b/test/mpl/__init__.py @@ -10,4 +10,4 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Module containing tests that REQUIRE the aws-cryptographic-material-providers library to run.""" \ No newline at end of file +"""Module containing tests that REQUIRE the aws-cryptographic-material-providers library to run.""" diff --git a/test/mpl/unit/test_material_managers_mpl_cmm.py b/test/mpl/unit/test_material_managers_mpl_cmm.py index eb795b7c2..a67c3e5c5 100644 --- a/test/mpl/unit/test_material_managers_mpl_cmm.py +++ b/test/mpl/unit/test_material_managers_mpl_cmm.py @@ -13,15 +13,6 @@ """Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" import pytest -from mock import MagicMock, patch - -from aws_encryption_sdk.identifiers import CommitmentPolicy -from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL -from aws_encryption_sdk.materials_managers.mpl.materials import DecryptionMaterialsFromMPL, EncryptionMaterialsFromMPL - -pytestmark = [pytest.mark.unit, pytest.mark.local] - - from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException from aws_cryptographic_materialproviders.mpl.models import ( AlgorithmSuiteIdESDK as MPL_AlgorithmSuiteIdESDK, @@ -35,20 +26,27 @@ from aws_cryptographic_materialproviders.mpl.references import ( ICryptographicMaterialsManager as MPL_ICryptographicMaterialsManager, ) - -mock_mpl_cmm = MagicMock(__class__=MPL_ICryptographicMaterialsManager) -mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) -mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) - +from mock import MagicMock, patch from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError +from aws_encryption_sdk.identifiers import CommitmentPolicy from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest +from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL +from aws_encryption_sdk.materials_managers.mpl.materials import DecryptionMaterialsFromMPL, EncryptionMaterialsFromMPL from aws_encryption_sdk.structures import EncryptedDataKey as Native_EncryptedDataKey +pytestmark = [pytest.mark.unit, pytest.mark.local] + + mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) -mock_encryption_materials_handler = MagicMock(__class__=EncryptionMaterialsFromMPL) mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) + +mock_mpl_cmm = MagicMock(__class__=MPL_ICryptographicMaterialsManager) +mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) +mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) + + mock_edk = MagicMock(__class__=Native_EncryptedDataKey) mock_mpl_key_provider_id = MagicMock(__class__=str) mock_edk.key_provider.provider_id = mock_mpl_key_provider_id diff --git a/test/mpl/unit/test_material_managers_mpl_materials.py b/test/mpl/unit/test_material_managers_mpl_materials.py index 96237998a..cb3ca7397 100644 --- a/test/mpl/unit/test_material_managers_mpl_materials.py +++ b/test/mpl/unit/test_material_managers_mpl_materials.py @@ -13,28 +13,25 @@ """Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" import pytest +from aws_cryptographic_materialproviders.mpl.models import ( + DecryptionMaterials as MPL_DecryptionMaterials, + EncryptedDataKey as MPL_EncryptedDataKey, + EncryptionMaterials as MPL_EncryptionMaterials, +) from mock import MagicMock, PropertyMock, patch from typing import Dict, List, Set import aws_encryption_sdk.materials_managers.mpl.materials from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite +from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest from aws_encryption_sdk.materials_managers.mpl.materials import DecryptionMaterialsFromMPL, EncryptionMaterialsFromMPL pytestmark = [pytest.mark.unit, pytest.mark.local] -from aws_cryptographic_materialproviders.mpl.models import ( - DecryptionMaterials as MPL_DecryptionMaterials, - EncryptedDataKey as MPL_EncryptedDataKey, - EncryptionMaterials as MPL_EncryptionMaterials, -) - mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) - -from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest - mock_encryption_materials_request = MagicMock(__class__=EncryptionMaterialsRequest) mock_encryption_materials_handler = MagicMock(__class__=EncryptionMaterialsFromMPL) mock_decryption_materials_request = MagicMock(__class__=DecryptionMaterialsRequest) From 22eabb64dec94e8eaf6ccfb7a3fb14a29fcd09eb Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 17:27:38 -0800 Subject: [PATCH 121/422] fix --- decrypt_oracle/.chalice/pipeline.py | 2 +- .../src/aws_encryption_sdk_decrypt_oracle/app.py | 3 ++- .../test/integration/integration_test_utils.py | 3 ++- decrypt_oracle/test/test_n_generate_test_vectors.py | 7 ++++--- examples/test/examples_test_utils.py | 2 +- examples/test/test_i_basic_encryption.py | 1 - ...basic_file_encryption_with_multiple_providers.py | 4 +--- ...i_basic_file_encryption_with_raw_key_provider.py | 1 - examples/test/test_i_data_key_caching_basic.py | 1 - examples/test/test_i_discovery_kms_provider.py | 4 +--- examples/test/test_i_mrk_aware_kms_provider.py | 4 +--- examples/test/test_i_multiple_kms_cmk.py | 4 +--- examples/test/test_i_one_kms_cmk.py | 4 +--- examples/test/test_i_one_kms_cmk_streaming_data.py | 1 - examples/test/test_i_one_kms_cmk_unsigned.py | 4 +--- examples/test/test_i_set_commitment.py | 4 +--- .../materials_managers/mpl/__init__.py | 5 ++++- .../materials_managers/mpl/cmm.py | 12 ++++++++---- .../materials_managers/mpl/materials.py | 5 ++++- src/aws_encryption_sdk/streaming_client.py | 4 ++-- test/mpl/__init__.py | 5 ++++- test/mpl/unit/test_material_managers_mpl_cmm.py | 5 ++++- .../unit/test_material_managers_mpl_materials.py | 7 +++++-- .../src/awses_test_vectors/internal/aws_kms.py | 5 +++-- .../src/awses_test_vectors/internal/util.py | 3 +-- .../manifests/full_message/decrypt.py | 9 ++++----- .../manifests/full_message/decrypt_generation.py | 13 ++++++------- .../manifests/full_message/encrypt.py | 7 +++---- .../src/awses_test_vectors/manifests/keys.py | 4 +--- .../src/awses_test_vectors/manifests/master_key.py | 11 +++++------ .../commands/test_i_full_message_encrypt.py | 1 - 31 files changed, 71 insertions(+), 74 deletions(-) diff --git a/decrypt_oracle/.chalice/pipeline.py b/decrypt_oracle/.chalice/pipeline.py index 9d5573646..c05df6739 100644 --- a/decrypt_oracle/.chalice/pipeline.py +++ b/decrypt_oracle/.chalice/pipeline.py @@ -2,7 +2,6 @@ import argparse import getpass import logging -from typing import Iterable import boto3 import troposphere @@ -20,6 +19,7 @@ ) from botocore.exceptions import ClientError from troposphere import GetAtt, Ref, Sub, Template, codebuild, codepipeline, iam, s3 +from typing import Iterable APPLICATION_NAME = "AwsEncryptionSdkDecryptOraclePython" PIPELINE_STACK_NAME = "{}DeployPipeline".format(APPLICATION_NAME) diff --git a/decrypt_oracle/src/aws_encryption_sdk_decrypt_oracle/app.py b/decrypt_oracle/src/aws_encryption_sdk_decrypt_oracle/app.py index 820b9e015..e250bb3c8 100644 --- a/decrypt_oracle/src/aws_encryption_sdk_decrypt_oracle/app.py +++ b/decrypt_oracle/src/aws_encryption_sdk_decrypt_oracle/app.py @@ -15,10 +15,11 @@ import logging import os +from chalice import Chalice, Response + import aws_encryption_sdk from aws_encryption_sdk.identifiers import CommitmentPolicy from aws_encryption_sdk.key_providers.kms import DiscoveryAwsKmsMasterKeyProvider -from chalice import Chalice, Response from .key_providers.counting import CountingMasterKey from .key_providers.null import NullMasterKey diff --git a/decrypt_oracle/test/integration/integration_test_utils.py b/decrypt_oracle/test/integration/integration_test_utils.py index c03b7f440..9849f1ecc 100644 --- a/decrypt_oracle/test/integration/integration_test_utils.py +++ b/decrypt_oracle/test/integration/integration_test_utils.py @@ -15,10 +15,11 @@ import json import os from collections import namedtuple + +import pytest from typing import Any, Callable, Iterable, Optional, Text import aws_encryption_sdk -import pytest from aws_encryption_sdk.identifiers import CommitmentPolicy from aws_encryption_sdk.key_providers.kms import StrictAwsKmsMasterKeyProvider diff --git a/decrypt_oracle/test/test_n_generate_test_vectors.py b/decrypt_oracle/test/test_n_generate_test_vectors.py index deb3f7c4d..ae9bb7d7d 100644 --- a/decrypt_oracle/test/test_n_generate_test_vectors.py +++ b/decrypt_oracle/test/test_n_generate_test_vectors.py @@ -15,14 +15,15 @@ import binascii import json import os + +import pytest +from aws_encryption_sdk_decrypt_oracle.key_providers.counting import CountingMasterKey +from aws_encryption_sdk_decrypt_oracle.key_providers.null import NullMasterKey from typing import Dict, Iterable, Text import aws_encryption_sdk -import pytest from aws_encryption_sdk.key_providers.base import MasterKeyProvider from aws_encryption_sdk.key_providers.kms import KMSMasterKey -from aws_encryption_sdk_decrypt_oracle.key_providers.counting import CountingMasterKey -from aws_encryption_sdk_decrypt_oracle.key_providers.null import NullMasterKey from .integration.integration_test_utils import test_vectors_filename diff --git a/examples/test/examples_test_utils.py b/examples/test/examples_test_utils.py index 8a51f21c8..08e8cf2f5 100644 --- a/examples/test/examples_test_utils.py +++ b/examples/test/examples_test_utils.py @@ -49,7 +49,7 @@ from integration_test_utils import ( # noqa pylint: disable=unused-import,import-error get_cmk_arn, - get_second_cmk_arn, get_mrk_arn, + get_second_cmk_arn, get_second_mrk_arn, ) diff --git a/examples/test/test_i_basic_encryption.py b/examples/test/test_i_basic_encryption.py index f2a4fab51..aa32d61fa 100644 --- a/examples/test/test_i_basic_encryption.py +++ b/examples/test/test_i_basic_encryption.py @@ -17,7 +17,6 @@ from ..src.basic_encryption import cycle_string from .examples_test_utils import get_cmk_arn, static_plaintext - pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_basic_file_encryption_with_multiple_providers.py b/examples/test/test_i_basic_file_encryption_with_multiple_providers.py index 282a272ab..0792f4958 100644 --- a/examples/test/test_i_basic_file_encryption_with_multiple_providers.py +++ b/examples/test/test_i_basic_file_encryption_with_multiple_providers.py @@ -18,9 +18,7 @@ import pytest from ..src.basic_file_encryption_with_multiple_providers import cycle_file -from .examples_test_utils import get_cmk_arn -from .examples_test_utils import static_plaintext - +from .examples_test_utils import get_cmk_arn, static_plaintext pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_basic_file_encryption_with_raw_key_provider.py b/examples/test/test_i_basic_file_encryption_with_raw_key_provider.py index 710c0ccac..046b7f964 100644 --- a/examples/test/test_i_basic_file_encryption_with_raw_key_provider.py +++ b/examples/test/test_i_basic_file_encryption_with_raw_key_provider.py @@ -19,7 +19,6 @@ from ..src.basic_file_encryption_with_raw_key_provider import cycle_file from .examples_test_utils import static_plaintext - pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_data_key_caching_basic.py b/examples/test/test_i_data_key_caching_basic.py index 734c35692..7a30f4e53 100644 --- a/examples/test/test_i_data_key_caching_basic.py +++ b/examples/test/test_i_data_key_caching_basic.py @@ -16,7 +16,6 @@ from ..src.data_key_caching_basic import encrypt_with_caching from .examples_test_utils import get_cmk_arn - pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_discovery_kms_provider.py b/examples/test/test_i_discovery_kms_provider.py index e9a1c6e71..0f64cbf59 100644 --- a/examples/test/test_i_discovery_kms_provider.py +++ b/examples/test/test_i_discovery_kms_provider.py @@ -16,9 +16,7 @@ import pytest from ..src.discovery_kms_provider import encrypt_decrypt -from .examples_test_utils import get_cmk_arn -from .examples_test_utils import static_plaintext - +from .examples_test_utils import get_cmk_arn, static_plaintext pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_mrk_aware_kms_provider.py b/examples/test/test_i_mrk_aware_kms_provider.py index 8e7a003f8..a90101fa8 100644 --- a/examples/test/test_i_mrk_aware_kms_provider.py +++ b/examples/test/test_i_mrk_aware_kms_provider.py @@ -15,9 +15,7 @@ import pytest from ..src.mrk_aware_kms_provider import encrypt_decrypt -from .examples_test_utils import get_mrk_arn, get_second_mrk_arn -from .examples_test_utils import static_plaintext - +from .examples_test_utils import get_mrk_arn, get_second_mrk_arn, static_plaintext pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_multiple_kms_cmk.py b/examples/test/test_i_multiple_kms_cmk.py index 39369cbc6..2915a0fd7 100644 --- a/examples/test/test_i_multiple_kms_cmk.py +++ b/examples/test/test_i_multiple_kms_cmk.py @@ -16,9 +16,7 @@ import pytest from ..src.multiple_kms_cmk import encrypt_decrypt -from .examples_test_utils import get_cmk_arn, get_second_cmk_arn -from .examples_test_utils import static_plaintext - +from .examples_test_utils import get_cmk_arn, get_second_cmk_arn, static_plaintext pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_one_kms_cmk.py b/examples/test/test_i_one_kms_cmk.py index 71ce74d3d..96dd48dae 100644 --- a/examples/test/test_i_one_kms_cmk.py +++ b/examples/test/test_i_one_kms_cmk.py @@ -16,9 +16,7 @@ import pytest from ..src.one_kms_cmk import encrypt_decrypt -from .examples_test_utils import get_cmk_arn -from .examples_test_utils import static_plaintext - +from .examples_test_utils import get_cmk_arn, static_plaintext pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_one_kms_cmk_streaming_data.py b/examples/test/test_i_one_kms_cmk_streaming_data.py index b22fa4232..f0a3094d0 100644 --- a/examples/test/test_i_one_kms_cmk_streaming_data.py +++ b/examples/test/test_i_one_kms_cmk_streaming_data.py @@ -20,7 +20,6 @@ from ..src.one_kms_cmk_streaming_data import encrypt_decrypt_stream from .examples_test_utils import get_cmk_arn, static_plaintext - pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_one_kms_cmk_unsigned.py b/examples/test/test_i_one_kms_cmk_unsigned.py index 8a2758c96..41f16473d 100644 --- a/examples/test/test_i_one_kms_cmk_unsigned.py +++ b/examples/test/test_i_one_kms_cmk_unsigned.py @@ -16,9 +16,7 @@ import pytest from ..src.one_kms_cmk_unsigned import encrypt_decrypt -from .examples_test_utils import get_cmk_arn -from .examples_test_utils import static_plaintext - +from .examples_test_utils import get_cmk_arn, static_plaintext pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_set_commitment.py b/examples/test/test_i_set_commitment.py index 96247334b..c14a379bf 100644 --- a/examples/test/test_i_set_commitment.py +++ b/examples/test/test_i_set_commitment.py @@ -16,9 +16,7 @@ import pytest from ..src.set_commitment import encrypt_decrypt -from .examples_test_utils import get_cmk_arn -from .examples_test_utils import static_plaintext - +from .examples_test_utils import get_cmk_arn, static_plaintext pytestmark = [pytest.mark.examples] diff --git a/src/aws_encryption_sdk/materials_managers/mpl/__init__.py b/src/aws_encryption_sdk/materials_managers/mpl/__init__.py index 295400d76..7593a3300 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/__init__.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/__init__.py @@ -10,4 +10,7 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Modules related to the MPL's materials managers interfaces.""" +"""Modules related to the MPL's materials managers interfaces. + +The aws-cryptographic-materials-library MUST be installed to use these modules. +""" diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index 1bbd7c89a..24a10139f 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -1,4 +1,7 @@ -"""Retrieves encryption/decryption materials from the MPL.""" +"""Retrieves encryption/decryption materials from the MPL and interfaces them to EDK components. + +The aws-cryptographic-materials-library MUST be installed to use this module. +""" from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException from aws_cryptographic_materialproviders.mpl.models import ( @@ -54,9 +57,10 @@ def get_encryption_materials( :param request: Request for encryption materials """ try: - mpl_input: MPL_GetEncryptionMaterialsInput = CryptoMaterialsManagerFromMPL._native_to_mpl_get_encryption_materials( - request - ) + mpl_input: MPL_GetEncryptionMaterialsInput = \ + CryptoMaterialsManagerFromMPL._native_to_mpl_get_encryption_materials( + request + ) mpl_output: MPL_GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) return EncryptionMaterialsFromMPL(mpl_output.encryption_materials) except AwsCryptographicMaterialProvidersException as mpl_exception: diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index 31f7d2a65..c23e2b038 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -1,4 +1,7 @@ -"""Provides encryption/decryption materials from an underlying materials provider.""" +"""Provides encryption/decryption materials from an underlying materials provider from the MPL. + +The aws-cryptographic-materials-library MUST be installed to use this module. +""" from aws_cryptographic_materialproviders.mpl.models import ( DecryptionMaterials as MPL_DecryptionMaterials, diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 72ed4efb7..959b5ff0b 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -555,7 +555,7 @@ def _prep_message(self): # MPL verification key is PEM bytes, not DER bytes. # If the underlying CMM is from the MPL, load PEM bytes. if (_HAS_MPL - and isinstance(self.config.materials_manager, CryptoMaterialsManagerFromMPL)): + and isinstance(self.config.materials_manager, CryptoMaterialsManagerFromMPL)): self.signer = Signer.from_key_bytes( algorithm=self._encryption_materials.algorithm, key_bytes=self._encryption_materials.signing_key, encoding=serialization.Encoding.PEM, @@ -923,7 +923,7 @@ def _read_header(self): # MPL verification key is NOT key bytes; it is bytes of the compressed point. # If the underlying CMM is from the MPL, load bytes from encoded point. if (_HAS_MPL - and isinstance(self.config.materials_manager, CryptoMaterialsManagerFromMPL)): + and isinstance(self.config.materials_manager, CryptoMaterialsManagerFromMPL)): self.verifier = Verifier.from_encoded_point( algorithm=header.algorithm, encoded_point=base64.b64encode(decryption_materials.verification_key) diff --git a/test/mpl/__init__.py b/test/mpl/__init__.py index 2a6c71715..37f482e0b 100644 --- a/test/mpl/__init__.py +++ b/test/mpl/__init__.py @@ -10,4 +10,7 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Module containing tests that REQUIRE the aws-cryptographic-material-providers library to run.""" +"""Module testing components that use the MPL. + +The aws-cryptographic-materials-library MUST be installed to run tests in this module. +""" diff --git a/test/mpl/unit/test_material_managers_mpl_cmm.py b/test/mpl/unit/test_material_managers_mpl_cmm.py index a67c3e5c5..fa8f76410 100644 --- a/test/mpl/unit/test_material_managers_mpl_cmm.py +++ b/test/mpl/unit/test_material_managers_mpl_cmm.py @@ -10,7 +10,10 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" +"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic. + +The aws-cryptographic-materials-library MUST be installed to run tests in this module. +""" import pytest from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException diff --git a/test/mpl/unit/test_material_managers_mpl_materials.py b/test/mpl/unit/test_material_managers_mpl_materials.py index cb3ca7397..60e12c634 100644 --- a/test/mpl/unit/test_material_managers_mpl_materials.py +++ b/test/mpl/unit/test_material_managers_mpl_materials.py @@ -10,7 +10,10 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" +"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic. + +The aws-cryptographic-materials-library MUST be installed to run tests in this module. +""" import pytest from aws_cryptographic_materialproviders.mpl.models import ( @@ -18,7 +21,7 @@ EncryptedDataKey as MPL_EncryptedDataKey, EncryptionMaterials as MPL_EncryptionMaterials, ) -from mock import MagicMock, PropertyMock, patch +from mock import MagicMock, patch from typing import Dict, List, Set import aws_encryption_sdk.materials_managers.mpl.materials diff --git a/test_vector_handlers/src/awses_test_vectors/internal/aws_kms.py b/test_vector_handlers/src/awses_test_vectors/internal/aws_kms.py index 14c109e7d..3d2088a73 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/aws_kms.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/aws_kms.py @@ -15,14 +15,15 @@ from aws_encryption_sdk.identifiers import AlgorithmSuite except ImportError: from aws_encryption_sdk.identifiers import Algorithm as AlgorithmSuite + +from awses_test_vectors.internal.defaults import ENCODING + from aws_encryption_sdk.key_providers.kms import ( DiscoveryAwsKmsMasterKeyProvider, MRKAwareDiscoveryAwsKmsMasterKeyProvider, StrictAwsKmsMasterKeyProvider, ) -from awses_test_vectors.internal.defaults import ENCODING - # This lets us easily use a single boto3 client per region for all KMS master keys. KMS_MASTER_KEY_PROVIDER = DiscoveryAwsKmsMasterKeyProvider() KMS_MRK_AWARE_MASTER_KEY_PROVIDER = MRKAwareDiscoveryAwsKmsMasterKeyProvider(discovery_region="us-west-2") diff --git a/test_vector_handlers/src/awses_test_vectors/internal/util.py b/test_vector_handlers/src/awses_test_vectors/internal/util.py index da5552f13..67d4ec67f 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/util.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/util.py @@ -24,12 +24,11 @@ from aws_encryption_sdk.identifiers import Algorithm as AlgorithmSuite try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Any, Callable, Dict, Iterable, Type # noqa pylint: disable=unused-import - from awses_test_vectors.internal.mypy_types import ( # noqa pylint: disable=unused-import ISINSTANCE, MANIFEST_VERSION, ) + from typing import Any, Callable, Dict, Iterable, Type # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index c94fd1452..a53f6cc5d 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -20,11 +20,8 @@ from enum import Enum import attr -import aws_encryption_sdk import pytest import six -from aws_encryption_sdk.identifiers import CommitmentPolicy - from awses_test_vectors.internal.defaults import ENCODING from awses_test_vectors.internal.util import ( dictionary_validator, @@ -35,14 +32,16 @@ from awses_test_vectors.manifests.keys import KeysManifest from awses_test_vectors.manifests.master_key import MasterKeySpec, master_key_provider_from_master_key_specs -try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import IO, Callable, Dict, Iterable, Optional # noqa pylint: disable=unused-import +import aws_encryption_sdk +from aws_encryption_sdk.identifiers import CommitmentPolicy +try: # Python 3.5.0 and 3.5.1 have incompatible typing modules from awses_test_vectors.internal.mypy_types import ( # noqa pylint: disable=unused-import DECRYPT_SCENARIO_SPEC, FULL_MESSAGE_DECRYPT_MANIFEST, MASTER_KEY_SPEC, ) + from typing import IO, Callable, Dict, Iterable, Optional # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index e407a1b65..48fc1a6b3 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -22,11 +22,6 @@ import attr import six -from aws_encryption_sdk.caches.local import LocalCryptoMaterialsCache -from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager -from aws_encryption_sdk.materials_managers.caching import CachingCryptoMaterialsManager -from aws_encryption_sdk.materials_managers.default import DefaultCryptoMaterialsManager - from awses_test_vectors.internal.defaults import ENCODING from awses_test_vectors.internal.util import ( dictionary_validator, @@ -45,6 +40,11 @@ from awses_test_vectors.manifests.full_message.encrypt import MessageEncryptionTestScenario from awses_test_vectors.manifests.keys import KeysManifest +from aws_encryption_sdk.caches.local import LocalCryptoMaterialsCache +from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager +from aws_encryption_sdk.materials_managers.caching import CachingCryptoMaterialsManager +from aws_encryption_sdk.materials_managers.default import DefaultCryptoMaterialsManager + try: from aws_encryption_sdk.identifiers import AlgorithmSuite except ImportError: @@ -53,12 +53,11 @@ from awses_test_vectors.manifests.master_key import MasterKeySpec, master_key_provider_from_master_key_specs try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import IO, Callable, Dict, Iterable, Optional # noqa pylint: disable=unused-import - from awses_test_vectors.internal.mypy_types import ( # noqa pylint: disable=unused-import ENCRYPT_SCENARIO_SPEC, PLAINTEXTS_SPEC, ) + from typing import IO, Callable, Dict, Iterable, Optional # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index c77fed1ce..2e88c8a52 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -19,9 +19,7 @@ import os import attr -import aws_encryption_sdk import six - from awses_test_vectors.internal.defaults import ENCODING from awses_test_vectors.internal.util import ( algorithm_suite_from_string_id, @@ -34,6 +32,8 @@ from awses_test_vectors.manifests.keys import KeysManifest from awses_test_vectors.manifests.master_key import MasterKeySpec, master_key_provider_from_master_key_specs +import aws_encryption_sdk + try: from aws_encryption_sdk.identifiers import AlgorithmSuite, CommitmentPolicy except ImportError: @@ -41,12 +41,11 @@ try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import IO, Callable, Dict, Iterable, Optional # noqa pylint: disable=unused-import - from awses_test_vectors.internal.mypy_types import ( # noqa pylint: disable=unused-import ENCRYPT_SCENARIO_SPEC, PLAINTEXTS_SPEC, ) + from typing import IO, Callable, Dict, Iterable, Optional # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/keys.py b/test_vector_handlers/src/awses_test_vectors/manifests/keys.py index cba6b7e25..546dbb489 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/keys.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/keys.py @@ -19,14 +19,11 @@ import attr import six - from awses_test_vectors.internal.aws_kms import arn_from_key_id from awses_test_vectors.internal.defaults import ENCODING from awses_test_vectors.internal.util import dictionary_validator, membership_validator, validate_manifest_type try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Dict, Iterable, Optional, cast # noqa pylint: disable=unused-import - from awses_test_vectors.internal.mypy_types import ( # noqa pylint: disable=unused-import AWS_KMS_KEY_SPEC, KEY_SPEC, @@ -34,6 +31,7 @@ MANIFEST_VERSION, MANUAL_KEY_SPEC, ) + from typing import Dict, Iterable, Optional, cast # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/master_key.py b/test_vector_handlers/src/awses_test_vectors/manifests/master_key.py index a1a7ae4af..8b00870f1 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/master_key.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/master_key.py @@ -17,6 +17,10 @@ """ import attr import six +from awses_test_vectors.internal.aws_kms import KMS_MASTER_KEY_PROVIDER, KMS_MRK_AWARE_MASTER_KEY_PROVIDER +from awses_test_vectors.internal.util import membership_validator +from awses_test_vectors.manifests.keys import KeysManifest, KeySpec # noqa pylint: disable=unused-import + from aws_encryption_sdk.identifiers import EncryptionKeyType, WrappingAlgorithm from aws_encryption_sdk.key_providers.base import MasterKeyProvider # noqa pylint: disable=unused-import from aws_encryption_sdk.key_providers.kms import ( # noqa pylint: disable=unused-import @@ -26,10 +30,6 @@ ) from aws_encryption_sdk.key_providers.raw import RawMasterKey -from awses_test_vectors.internal.aws_kms import KMS_MASTER_KEY_PROVIDER, KMS_MRK_AWARE_MASTER_KEY_PROVIDER -from awses_test_vectors.internal.util import membership_validator -from awses_test_vectors.manifests.keys import KeysManifest, KeySpec # noqa pylint: disable=unused-import - try: from aws_encryption_sdk.internal.crypto.wrapping_keys import WrappingKey except ImportError: @@ -37,9 +37,8 @@ try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from typing import Iterable # noqa pylint: disable=unused-import - from awses_test_vectors.internal.mypy_types import MASTER_KEY_SPEC # noqa pylint: disable=unused-import + from typing import Iterable # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass diff --git a/test_vector_handlers/test/integration/commands/test_i_full_message_encrypt.py b/test_vector_handlers/test/integration/commands/test_i_full_message_encrypt.py index 6305a15da..6928caeba 100644 --- a/test_vector_handlers/test/integration/commands/test_i_full_message_encrypt.py +++ b/test_vector_handlers/test/integration/commands/test_i_full_message_encrypt.py @@ -14,7 +14,6 @@ Integration tests for ``awses_test_vectors.commands``. """ import pytest - from awses_test_vectors.commands import full_message_decrypt, full_message_decrypt_generate, full_message_encrypt from ..integration_test_utils import ( # noqa pylint: disable=unused-import From ac0ceb3e60d9b4b8ff3c3ae44b7ff2b9b0a50af2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 17:29:19 -0800 Subject: [PATCH 122/422] fix --- src/aws_encryption_sdk/materials_managers/mpl/cmm.py | 4 +++- src/aws_encryption_sdk/materials_managers/mpl/materials.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index 24a10139f..1913505c4 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -2,7 +2,8 @@ The aws-cryptographic-materials-library MUST be installed to use this module. """ - +# pylint should pass even if the MPL isn't installed +# noqa pylint: disable=import-error from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException from aws_cryptographic_materialproviders.mpl.models import ( AlgorithmSuiteIdESDK as MPL_AlgorithmSuiteIdESDK, @@ -16,6 +17,7 @@ from aws_cryptographic_materialproviders.mpl.references import ( ICryptographicMaterialsManager as MPL_ICryptographicMaterialsManager, ) +# noqa pylint: enable=import-error from typing import List from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index c23e2b038..faa47cb46 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -2,12 +2,14 @@ The aws-cryptographic-materials-library MUST be installed to use this module. """ - +# pylint should pass even if the MPL isn't installed +# noqa pylint: disable=import-error from aws_cryptographic_materialproviders.mpl.models import ( DecryptionMaterials as MPL_DecryptionMaterials, EncryptedDataKey as MPL_EncryptedDataKey, EncryptionMaterials as MPL_EncryptionMaterials, ) +# noqa pylint: enable=import-error from typing import Dict, List, Set from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite From 2fd88584be77ed1be902e56ff473208e674b2d1d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 17:32:01 -0800 Subject: [PATCH 123/422] oops --- src/aws_encryption_sdk/materials_managers/mpl/cmm.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index 1913505c4..760808bfe 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -3,7 +3,8 @@ The aws-cryptographic-materials-library MUST be installed to use this module. """ # pylint should pass even if the MPL isn't installed -# noqa pylint: disable=import-error +# Also thinks these imports aren't used if it can't import them +# noqa pylint: disable=import-error,unused-import from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException from aws_cryptographic_materialproviders.mpl.models import ( AlgorithmSuiteIdESDK as MPL_AlgorithmSuiteIdESDK, @@ -17,7 +18,7 @@ from aws_cryptographic_materialproviders.mpl.references import ( ICryptographicMaterialsManager as MPL_ICryptographicMaterialsManager, ) -# noqa pylint: enable=import-error +# noqa pylint: enable=import-error,unused-import from typing import List from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError From 51c6a9caad271e1288ae06e07355c8b2cc6c0b85 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 17:33:06 -0800 Subject: [PATCH 124/422] revert --- decrypt_oracle/.chalice/pipeline.py | 2 +- .../src/aws_encryption_sdk_decrypt_oracle/app.py | 3 +-- .../test/integration/integration_test_utils.py | 3 +-- decrypt_oracle/test/test_n_generate_test_vectors.py | 7 +++---- examples/test/examples_test_utils.py | 2 +- examples/test/test_i_basic_encryption.py | 1 + ...basic_file_encryption_with_multiple_providers.py | 4 +++- ...i_basic_file_encryption_with_raw_key_provider.py | 1 + examples/test/test_i_data_key_caching_basic.py | 1 + examples/test/test_i_discovery_kms_provider.py | 4 +++- examples/test/test_i_mrk_aware_kms_provider.py | 4 +++- examples/test/test_i_multiple_kms_cmk.py | 4 +++- examples/test/test_i_one_kms_cmk.py | 4 +++- examples/test/test_i_one_kms_cmk_streaming_data.py | 1 + examples/test/test_i_one_kms_cmk_unsigned.py | 4 +++- examples/test/test_i_set_commitment.py | 4 +++- .../materials_managers/mpl/__init__.py | 5 +---- .../materials_managers/mpl/cmm.py | 7 +++---- .../materials_managers/mpl/materials.py | 5 +++++ src/aws_encryption_sdk/streaming_client.py | 4 ++-- test/mpl/__init__.py | 5 +---- test/mpl/unit/test_material_managers_mpl_cmm.py | 5 +---- .../unit/test_material_managers_mpl_materials.py | 7 ++----- .../src/awses_test_vectors/internal/aws_kms.py | 5 ++--- .../src/awses_test_vectors/internal/util.py | 3 ++- .../manifests/full_message/decrypt.py | 9 +++++---- .../manifests/full_message/decrypt_generation.py | 13 +++++++------ .../manifests/full_message/encrypt.py | 7 ++++--- .../src/awses_test_vectors/manifests/keys.py | 4 +++- .../src/awses_test_vectors/manifests/master_key.py | 11 ++++++----- .../commands/test_i_full_message_encrypt.py | 1 + 31 files changed, 77 insertions(+), 63 deletions(-) diff --git a/decrypt_oracle/.chalice/pipeline.py b/decrypt_oracle/.chalice/pipeline.py index c05df6739..9d5573646 100644 --- a/decrypt_oracle/.chalice/pipeline.py +++ b/decrypt_oracle/.chalice/pipeline.py @@ -2,6 +2,7 @@ import argparse import getpass import logging +from typing import Iterable import boto3 import troposphere @@ -19,7 +20,6 @@ ) from botocore.exceptions import ClientError from troposphere import GetAtt, Ref, Sub, Template, codebuild, codepipeline, iam, s3 -from typing import Iterable APPLICATION_NAME = "AwsEncryptionSdkDecryptOraclePython" PIPELINE_STACK_NAME = "{}DeployPipeline".format(APPLICATION_NAME) diff --git a/decrypt_oracle/src/aws_encryption_sdk_decrypt_oracle/app.py b/decrypt_oracle/src/aws_encryption_sdk_decrypt_oracle/app.py index e250bb3c8..820b9e015 100644 --- a/decrypt_oracle/src/aws_encryption_sdk_decrypt_oracle/app.py +++ b/decrypt_oracle/src/aws_encryption_sdk_decrypt_oracle/app.py @@ -15,11 +15,10 @@ import logging import os -from chalice import Chalice, Response - import aws_encryption_sdk from aws_encryption_sdk.identifiers import CommitmentPolicy from aws_encryption_sdk.key_providers.kms import DiscoveryAwsKmsMasterKeyProvider +from chalice import Chalice, Response from .key_providers.counting import CountingMasterKey from .key_providers.null import NullMasterKey diff --git a/decrypt_oracle/test/integration/integration_test_utils.py b/decrypt_oracle/test/integration/integration_test_utils.py index 9849f1ecc..c03b7f440 100644 --- a/decrypt_oracle/test/integration/integration_test_utils.py +++ b/decrypt_oracle/test/integration/integration_test_utils.py @@ -15,11 +15,10 @@ import json import os from collections import namedtuple - -import pytest from typing import Any, Callable, Iterable, Optional, Text import aws_encryption_sdk +import pytest from aws_encryption_sdk.identifiers import CommitmentPolicy from aws_encryption_sdk.key_providers.kms import StrictAwsKmsMasterKeyProvider diff --git a/decrypt_oracle/test/test_n_generate_test_vectors.py b/decrypt_oracle/test/test_n_generate_test_vectors.py index ae9bb7d7d..deb3f7c4d 100644 --- a/decrypt_oracle/test/test_n_generate_test_vectors.py +++ b/decrypt_oracle/test/test_n_generate_test_vectors.py @@ -15,15 +15,14 @@ import binascii import json import os - -import pytest -from aws_encryption_sdk_decrypt_oracle.key_providers.counting import CountingMasterKey -from aws_encryption_sdk_decrypt_oracle.key_providers.null import NullMasterKey from typing import Dict, Iterable, Text import aws_encryption_sdk +import pytest from aws_encryption_sdk.key_providers.base import MasterKeyProvider from aws_encryption_sdk.key_providers.kms import KMSMasterKey +from aws_encryption_sdk_decrypt_oracle.key_providers.counting import CountingMasterKey +from aws_encryption_sdk_decrypt_oracle.key_providers.null import NullMasterKey from .integration.integration_test_utils import test_vectors_filename diff --git a/examples/test/examples_test_utils.py b/examples/test/examples_test_utils.py index 08e8cf2f5..8a51f21c8 100644 --- a/examples/test/examples_test_utils.py +++ b/examples/test/examples_test_utils.py @@ -49,7 +49,7 @@ from integration_test_utils import ( # noqa pylint: disable=unused-import,import-error get_cmk_arn, - get_mrk_arn, get_second_cmk_arn, + get_mrk_arn, get_second_mrk_arn, ) diff --git a/examples/test/test_i_basic_encryption.py b/examples/test/test_i_basic_encryption.py index aa32d61fa..f2a4fab51 100644 --- a/examples/test/test_i_basic_encryption.py +++ b/examples/test/test_i_basic_encryption.py @@ -17,6 +17,7 @@ from ..src.basic_encryption import cycle_string from .examples_test_utils import get_cmk_arn, static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_basic_file_encryption_with_multiple_providers.py b/examples/test/test_i_basic_file_encryption_with_multiple_providers.py index 0792f4958..282a272ab 100644 --- a/examples/test/test_i_basic_file_encryption_with_multiple_providers.py +++ b/examples/test/test_i_basic_file_encryption_with_multiple_providers.py @@ -18,7 +18,9 @@ import pytest from ..src.basic_file_encryption_with_multiple_providers import cycle_file -from .examples_test_utils import get_cmk_arn, static_plaintext +from .examples_test_utils import get_cmk_arn +from .examples_test_utils import static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_basic_file_encryption_with_raw_key_provider.py b/examples/test/test_i_basic_file_encryption_with_raw_key_provider.py index 046b7f964..710c0ccac 100644 --- a/examples/test/test_i_basic_file_encryption_with_raw_key_provider.py +++ b/examples/test/test_i_basic_file_encryption_with_raw_key_provider.py @@ -19,6 +19,7 @@ from ..src.basic_file_encryption_with_raw_key_provider import cycle_file from .examples_test_utils import static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_data_key_caching_basic.py b/examples/test/test_i_data_key_caching_basic.py index 7a30f4e53..734c35692 100644 --- a/examples/test/test_i_data_key_caching_basic.py +++ b/examples/test/test_i_data_key_caching_basic.py @@ -16,6 +16,7 @@ from ..src.data_key_caching_basic import encrypt_with_caching from .examples_test_utils import get_cmk_arn + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_discovery_kms_provider.py b/examples/test/test_i_discovery_kms_provider.py index 0f64cbf59..e9a1c6e71 100644 --- a/examples/test/test_i_discovery_kms_provider.py +++ b/examples/test/test_i_discovery_kms_provider.py @@ -16,7 +16,9 @@ import pytest from ..src.discovery_kms_provider import encrypt_decrypt -from .examples_test_utils import get_cmk_arn, static_plaintext +from .examples_test_utils import get_cmk_arn +from .examples_test_utils import static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_mrk_aware_kms_provider.py b/examples/test/test_i_mrk_aware_kms_provider.py index a90101fa8..8e7a003f8 100644 --- a/examples/test/test_i_mrk_aware_kms_provider.py +++ b/examples/test/test_i_mrk_aware_kms_provider.py @@ -15,7 +15,9 @@ import pytest from ..src.mrk_aware_kms_provider import encrypt_decrypt -from .examples_test_utils import get_mrk_arn, get_second_mrk_arn, static_plaintext +from .examples_test_utils import get_mrk_arn, get_second_mrk_arn +from .examples_test_utils import static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_multiple_kms_cmk.py b/examples/test/test_i_multiple_kms_cmk.py index 2915a0fd7..39369cbc6 100644 --- a/examples/test/test_i_multiple_kms_cmk.py +++ b/examples/test/test_i_multiple_kms_cmk.py @@ -16,7 +16,9 @@ import pytest from ..src.multiple_kms_cmk import encrypt_decrypt -from .examples_test_utils import get_cmk_arn, get_second_cmk_arn, static_plaintext +from .examples_test_utils import get_cmk_arn, get_second_cmk_arn +from .examples_test_utils import static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_one_kms_cmk.py b/examples/test/test_i_one_kms_cmk.py index 96dd48dae..71ce74d3d 100644 --- a/examples/test/test_i_one_kms_cmk.py +++ b/examples/test/test_i_one_kms_cmk.py @@ -16,7 +16,9 @@ import pytest from ..src.one_kms_cmk import encrypt_decrypt -from .examples_test_utils import get_cmk_arn, static_plaintext +from .examples_test_utils import get_cmk_arn +from .examples_test_utils import static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_one_kms_cmk_streaming_data.py b/examples/test/test_i_one_kms_cmk_streaming_data.py index f0a3094d0..b22fa4232 100644 --- a/examples/test/test_i_one_kms_cmk_streaming_data.py +++ b/examples/test/test_i_one_kms_cmk_streaming_data.py @@ -20,6 +20,7 @@ from ..src.one_kms_cmk_streaming_data import encrypt_decrypt_stream from .examples_test_utils import get_cmk_arn, static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_one_kms_cmk_unsigned.py b/examples/test/test_i_one_kms_cmk_unsigned.py index 41f16473d..8a2758c96 100644 --- a/examples/test/test_i_one_kms_cmk_unsigned.py +++ b/examples/test/test_i_one_kms_cmk_unsigned.py @@ -16,7 +16,9 @@ import pytest from ..src.one_kms_cmk_unsigned import encrypt_decrypt -from .examples_test_utils import get_cmk_arn, static_plaintext +from .examples_test_utils import get_cmk_arn +from .examples_test_utils import static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_set_commitment.py b/examples/test/test_i_set_commitment.py index c14a379bf..96247334b 100644 --- a/examples/test/test_i_set_commitment.py +++ b/examples/test/test_i_set_commitment.py @@ -16,7 +16,9 @@ import pytest from ..src.set_commitment import encrypt_decrypt -from .examples_test_utils import get_cmk_arn, static_plaintext +from .examples_test_utils import get_cmk_arn +from .examples_test_utils import static_plaintext + pytestmark = [pytest.mark.examples] diff --git a/src/aws_encryption_sdk/materials_managers/mpl/__init__.py b/src/aws_encryption_sdk/materials_managers/mpl/__init__.py index 7593a3300..295400d76 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/__init__.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/__init__.py @@ -10,7 +10,4 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Modules related to the MPL's materials managers interfaces. - -The aws-cryptographic-materials-library MUST be installed to use these modules. -""" +"""Modules related to the MPL's materials managers interfaces.""" diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index 760808bfe..ead7c48f1 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -60,10 +60,9 @@ def get_encryption_materials( :param request: Request for encryption materials """ try: - mpl_input: MPL_GetEncryptionMaterialsInput = \ - CryptoMaterialsManagerFromMPL._native_to_mpl_get_encryption_materials( - request - ) + mpl_input: MPL_GetEncryptionMaterialsInput = CryptoMaterialsManagerFromMPL._native_to_mpl_get_encryption_materials( + request + ) mpl_output: MPL_GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) return EncryptionMaterialsFromMPL(mpl_output.encryption_materials) except AwsCryptographicMaterialProvidersException as mpl_exception: diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index faa47cb46..a82a3c372 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -1,9 +1,14 @@ +<<<<<<< HEAD """Provides encryption/decryption materials from an underlying materials provider from the MPL. The aws-cryptographic-materials-library MUST be installed to use this module. """ # pylint should pass even if the MPL isn't installed # noqa pylint: disable=import-error +======= +"""Provides encryption/decryption materials from an underlying materials provider.""" + +>>>>>>> parent of 22eabb6 (fix) from aws_cryptographic_materialproviders.mpl.models import ( DecryptionMaterials as MPL_DecryptionMaterials, EncryptedDataKey as MPL_EncryptedDataKey, diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 959b5ff0b..72ed4efb7 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -555,7 +555,7 @@ def _prep_message(self): # MPL verification key is PEM bytes, not DER bytes. # If the underlying CMM is from the MPL, load PEM bytes. if (_HAS_MPL - and isinstance(self.config.materials_manager, CryptoMaterialsManagerFromMPL)): + and isinstance(self.config.materials_manager, CryptoMaterialsManagerFromMPL)): self.signer = Signer.from_key_bytes( algorithm=self._encryption_materials.algorithm, key_bytes=self._encryption_materials.signing_key, encoding=serialization.Encoding.PEM, @@ -923,7 +923,7 @@ def _read_header(self): # MPL verification key is NOT key bytes; it is bytes of the compressed point. # If the underlying CMM is from the MPL, load bytes from encoded point. if (_HAS_MPL - and isinstance(self.config.materials_manager, CryptoMaterialsManagerFromMPL)): + and isinstance(self.config.materials_manager, CryptoMaterialsManagerFromMPL)): self.verifier = Verifier.from_encoded_point( algorithm=header.algorithm, encoded_point=base64.b64encode(decryption_materials.verification_key) diff --git a/test/mpl/__init__.py b/test/mpl/__init__.py index 37f482e0b..2a6c71715 100644 --- a/test/mpl/__init__.py +++ b/test/mpl/__init__.py @@ -10,7 +10,4 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Module testing components that use the MPL. - -The aws-cryptographic-materials-library MUST be installed to run tests in this module. -""" +"""Module containing tests that REQUIRE the aws-cryptographic-material-providers library to run.""" diff --git a/test/mpl/unit/test_material_managers_mpl_cmm.py b/test/mpl/unit/test_material_managers_mpl_cmm.py index fa8f76410..a67c3e5c5 100644 --- a/test/mpl/unit/test_material_managers_mpl_cmm.py +++ b/test/mpl/unit/test_material_managers_mpl_cmm.py @@ -10,10 +10,7 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic. - -The aws-cryptographic-materials-library MUST be installed to run tests in this module. -""" +"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" import pytest from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException diff --git a/test/mpl/unit/test_material_managers_mpl_materials.py b/test/mpl/unit/test_material_managers_mpl_materials.py index 60e12c634..cb3ca7397 100644 --- a/test/mpl/unit/test_material_managers_mpl_materials.py +++ b/test/mpl/unit/test_material_managers_mpl_materials.py @@ -10,10 +10,7 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic. - -The aws-cryptographic-materials-library MUST be installed to run tests in this module. -""" +"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" import pytest from aws_cryptographic_materialproviders.mpl.models import ( @@ -21,7 +18,7 @@ EncryptedDataKey as MPL_EncryptedDataKey, EncryptionMaterials as MPL_EncryptionMaterials, ) -from mock import MagicMock, patch +from mock import MagicMock, PropertyMock, patch from typing import Dict, List, Set import aws_encryption_sdk.materials_managers.mpl.materials diff --git a/test_vector_handlers/src/awses_test_vectors/internal/aws_kms.py b/test_vector_handlers/src/awses_test_vectors/internal/aws_kms.py index 3d2088a73..14c109e7d 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/aws_kms.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/aws_kms.py @@ -15,15 +15,14 @@ from aws_encryption_sdk.identifiers import AlgorithmSuite except ImportError: from aws_encryption_sdk.identifiers import Algorithm as AlgorithmSuite - -from awses_test_vectors.internal.defaults import ENCODING - from aws_encryption_sdk.key_providers.kms import ( DiscoveryAwsKmsMasterKeyProvider, MRKAwareDiscoveryAwsKmsMasterKeyProvider, StrictAwsKmsMasterKeyProvider, ) +from awses_test_vectors.internal.defaults import ENCODING + # This lets us easily use a single boto3 client per region for all KMS master keys. KMS_MASTER_KEY_PROVIDER = DiscoveryAwsKmsMasterKeyProvider() KMS_MRK_AWARE_MASTER_KEY_PROVIDER = MRKAwareDiscoveryAwsKmsMasterKeyProvider(discovery_region="us-west-2") diff --git a/test_vector_handlers/src/awses_test_vectors/internal/util.py b/test_vector_handlers/src/awses_test_vectors/internal/util.py index 67d4ec67f..da5552f13 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/util.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/util.py @@ -24,11 +24,12 @@ from aws_encryption_sdk.identifiers import Algorithm as AlgorithmSuite try: # Python 3.5.0 and 3.5.1 have incompatible typing modules + from typing import Any, Callable, Dict, Iterable, Type # noqa pylint: disable=unused-import + from awses_test_vectors.internal.mypy_types import ( # noqa pylint: disable=unused-import ISINSTANCE, MANIFEST_VERSION, ) - from typing import Any, Callable, Dict, Iterable, Type # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index a53f6cc5d..c94fd1452 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -20,8 +20,11 @@ from enum import Enum import attr +import aws_encryption_sdk import pytest import six +from aws_encryption_sdk.identifiers import CommitmentPolicy + from awses_test_vectors.internal.defaults import ENCODING from awses_test_vectors.internal.util import ( dictionary_validator, @@ -32,16 +35,14 @@ from awses_test_vectors.manifests.keys import KeysManifest from awses_test_vectors.manifests.master_key import MasterKeySpec, master_key_provider_from_master_key_specs -import aws_encryption_sdk -from aws_encryption_sdk.identifiers import CommitmentPolicy - try: # Python 3.5.0 and 3.5.1 have incompatible typing modules + from typing import IO, Callable, Dict, Iterable, Optional # noqa pylint: disable=unused-import + from awses_test_vectors.internal.mypy_types import ( # noqa pylint: disable=unused-import DECRYPT_SCENARIO_SPEC, FULL_MESSAGE_DECRYPT_MANIFEST, MASTER_KEY_SPEC, ) - from typing import IO, Callable, Dict, Iterable, Optional # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 48fc1a6b3..e407a1b65 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -22,6 +22,11 @@ import attr import six +from aws_encryption_sdk.caches.local import LocalCryptoMaterialsCache +from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager +from aws_encryption_sdk.materials_managers.caching import CachingCryptoMaterialsManager +from aws_encryption_sdk.materials_managers.default import DefaultCryptoMaterialsManager + from awses_test_vectors.internal.defaults import ENCODING from awses_test_vectors.internal.util import ( dictionary_validator, @@ -40,11 +45,6 @@ from awses_test_vectors.manifests.full_message.encrypt import MessageEncryptionTestScenario from awses_test_vectors.manifests.keys import KeysManifest -from aws_encryption_sdk.caches.local import LocalCryptoMaterialsCache -from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager -from aws_encryption_sdk.materials_managers.caching import CachingCryptoMaterialsManager -from aws_encryption_sdk.materials_managers.default import DefaultCryptoMaterialsManager - try: from aws_encryption_sdk.identifiers import AlgorithmSuite except ImportError: @@ -53,11 +53,12 @@ from awses_test_vectors.manifests.master_key import MasterKeySpec, master_key_provider_from_master_key_specs try: # Python 3.5.0 and 3.5.1 have incompatible typing modules + from typing import IO, Callable, Dict, Iterable, Optional # noqa pylint: disable=unused-import + from awses_test_vectors.internal.mypy_types import ( # noqa pylint: disable=unused-import ENCRYPT_SCENARIO_SPEC, PLAINTEXTS_SPEC, ) - from typing import IO, Callable, Dict, Iterable, Optional # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 2e88c8a52..c77fed1ce 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -19,7 +19,9 @@ import os import attr +import aws_encryption_sdk import six + from awses_test_vectors.internal.defaults import ENCODING from awses_test_vectors.internal.util import ( algorithm_suite_from_string_id, @@ -32,8 +34,6 @@ from awses_test_vectors.manifests.keys import KeysManifest from awses_test_vectors.manifests.master_key import MasterKeySpec, master_key_provider_from_master_key_specs -import aws_encryption_sdk - try: from aws_encryption_sdk.identifiers import AlgorithmSuite, CommitmentPolicy except ImportError: @@ -41,11 +41,12 @@ try: # Python 3.5.0 and 3.5.1 have incompatible typing modules + from typing import IO, Callable, Dict, Iterable, Optional # noqa pylint: disable=unused-import + from awses_test_vectors.internal.mypy_types import ( # noqa pylint: disable=unused-import ENCRYPT_SCENARIO_SPEC, PLAINTEXTS_SPEC, ) - from typing import IO, Callable, Dict, Iterable, Optional # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/keys.py b/test_vector_handlers/src/awses_test_vectors/manifests/keys.py index 546dbb489..cba6b7e25 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/keys.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/keys.py @@ -19,11 +19,14 @@ import attr import six + from awses_test_vectors.internal.aws_kms import arn_from_key_id from awses_test_vectors.internal.defaults import ENCODING from awses_test_vectors.internal.util import dictionary_validator, membership_validator, validate_manifest_type try: # Python 3.5.0 and 3.5.1 have incompatible typing modules + from typing import Dict, Iterable, Optional, cast # noqa pylint: disable=unused-import + from awses_test_vectors.internal.mypy_types import ( # noqa pylint: disable=unused-import AWS_KMS_KEY_SPEC, KEY_SPEC, @@ -31,7 +34,6 @@ MANIFEST_VERSION, MANUAL_KEY_SPEC, ) - from typing import Dict, Iterable, Optional, cast # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/master_key.py b/test_vector_handlers/src/awses_test_vectors/manifests/master_key.py index 8b00870f1..a1a7ae4af 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/master_key.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/master_key.py @@ -17,10 +17,6 @@ """ import attr import six -from awses_test_vectors.internal.aws_kms import KMS_MASTER_KEY_PROVIDER, KMS_MRK_AWARE_MASTER_KEY_PROVIDER -from awses_test_vectors.internal.util import membership_validator -from awses_test_vectors.manifests.keys import KeysManifest, KeySpec # noqa pylint: disable=unused-import - from aws_encryption_sdk.identifiers import EncryptionKeyType, WrappingAlgorithm from aws_encryption_sdk.key_providers.base import MasterKeyProvider # noqa pylint: disable=unused-import from aws_encryption_sdk.key_providers.kms import ( # noqa pylint: disable=unused-import @@ -30,6 +26,10 @@ ) from aws_encryption_sdk.key_providers.raw import RawMasterKey +from awses_test_vectors.internal.aws_kms import KMS_MASTER_KEY_PROVIDER, KMS_MRK_AWARE_MASTER_KEY_PROVIDER +from awses_test_vectors.internal.util import membership_validator +from awses_test_vectors.manifests.keys import KeysManifest, KeySpec # noqa pylint: disable=unused-import + try: from aws_encryption_sdk.internal.crypto.wrapping_keys import WrappingKey except ImportError: @@ -37,8 +37,9 @@ try: # Python 3.5.0 and 3.5.1 have incompatible typing modules - from awses_test_vectors.internal.mypy_types import MASTER_KEY_SPEC # noqa pylint: disable=unused-import from typing import Iterable # noqa pylint: disable=unused-import + + from awses_test_vectors.internal.mypy_types import MASTER_KEY_SPEC # noqa pylint: disable=unused-import except ImportError: # pragma: no cover # We only actually need these imports when running the mypy checks pass diff --git a/test_vector_handlers/test/integration/commands/test_i_full_message_encrypt.py b/test_vector_handlers/test/integration/commands/test_i_full_message_encrypt.py index 6928caeba..6305a15da 100644 --- a/test_vector_handlers/test/integration/commands/test_i_full_message_encrypt.py +++ b/test_vector_handlers/test/integration/commands/test_i_full_message_encrypt.py @@ -14,6 +14,7 @@ Integration tests for ``awses_test_vectors.commands``. """ import pytest + from awses_test_vectors.commands import full_message_decrypt, full_message_decrypt_generate, full_message_encrypt from ..integration_test_utils import ( # noqa pylint: disable=unused-import From 800f9de0de601bb647ba2ab57e7df3d4eedfa795 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 17:33:18 -0800 Subject: [PATCH 125/422] revert --- src/aws_encryption_sdk/materials_managers/mpl/materials.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index a82a3c372..faa47cb46 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -1,14 +1,9 @@ -<<<<<<< HEAD """Provides encryption/decryption materials from an underlying materials provider from the MPL. The aws-cryptographic-materials-library MUST be installed to use this module. """ # pylint should pass even if the MPL isn't installed # noqa pylint: disable=import-error -======= -"""Provides encryption/decryption materials from an underlying materials provider.""" - ->>>>>>> parent of 22eabb6 (fix) from aws_cryptographic_materialproviders.mpl.models import ( DecryptionMaterials as MPL_DecryptionMaterials, EncryptedDataKey as MPL_EncryptedDataKey, From ebcb7590c472f82affa86ecc793a5e8a3e494a8b Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 17:35:14 -0800 Subject: [PATCH 126/422] fix --- src/aws_encryption_sdk/materials_managers/mpl/__init__.py | 5 ++++- src/aws_encryption_sdk/materials_managers/mpl/cmm.py | 7 ++++--- src/aws_encryption_sdk/streaming_client.py | 4 ++-- test/mpl/__init__.py | 5 ++++- test/mpl/unit/test_material_managers_mpl_cmm.py | 5 ++++- test/mpl/unit/test_material_managers_mpl_materials.py | 7 +++++-- 6 files changed, 23 insertions(+), 10 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/__init__.py b/src/aws_encryption_sdk/materials_managers/mpl/__init__.py index 295400d76..7593a3300 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/__init__.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/__init__.py @@ -10,4 +10,7 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Modules related to the MPL's materials managers interfaces.""" +"""Modules related to the MPL's materials managers interfaces. + +The aws-cryptographic-materials-library MUST be installed to use these modules. +""" diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index ead7c48f1..760808bfe 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -60,9 +60,10 @@ def get_encryption_materials( :param request: Request for encryption materials """ try: - mpl_input: MPL_GetEncryptionMaterialsInput = CryptoMaterialsManagerFromMPL._native_to_mpl_get_encryption_materials( - request - ) + mpl_input: MPL_GetEncryptionMaterialsInput = \ + CryptoMaterialsManagerFromMPL._native_to_mpl_get_encryption_materials( + request + ) mpl_output: MPL_GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) return EncryptionMaterialsFromMPL(mpl_output.encryption_materials) except AwsCryptographicMaterialProvidersException as mpl_exception: diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 72ed4efb7..959b5ff0b 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -555,7 +555,7 @@ def _prep_message(self): # MPL verification key is PEM bytes, not DER bytes. # If the underlying CMM is from the MPL, load PEM bytes. if (_HAS_MPL - and isinstance(self.config.materials_manager, CryptoMaterialsManagerFromMPL)): + and isinstance(self.config.materials_manager, CryptoMaterialsManagerFromMPL)): self.signer = Signer.from_key_bytes( algorithm=self._encryption_materials.algorithm, key_bytes=self._encryption_materials.signing_key, encoding=serialization.Encoding.PEM, @@ -923,7 +923,7 @@ def _read_header(self): # MPL verification key is NOT key bytes; it is bytes of the compressed point. # If the underlying CMM is from the MPL, load bytes from encoded point. if (_HAS_MPL - and isinstance(self.config.materials_manager, CryptoMaterialsManagerFromMPL)): + and isinstance(self.config.materials_manager, CryptoMaterialsManagerFromMPL)): self.verifier = Verifier.from_encoded_point( algorithm=header.algorithm, encoded_point=base64.b64encode(decryption_materials.verification_key) diff --git a/test/mpl/__init__.py b/test/mpl/__init__.py index 2a6c71715..d3f78d0bf 100644 --- a/test/mpl/__init__.py +++ b/test/mpl/__init__.py @@ -10,4 +10,7 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Module containing tests that REQUIRE the aws-cryptographic-material-providers library to run.""" +"""Module testing components that use the MPL. + +The aws-cryptographic-materials-library MUST be installed to run tests in this module. +""" \ No newline at end of file diff --git a/test/mpl/unit/test_material_managers_mpl_cmm.py b/test/mpl/unit/test_material_managers_mpl_cmm.py index a67c3e5c5..fa8f76410 100644 --- a/test/mpl/unit/test_material_managers_mpl_cmm.py +++ b/test/mpl/unit/test_material_managers_mpl_cmm.py @@ -10,7 +10,10 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" +"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic. + +The aws-cryptographic-materials-library MUST be installed to run tests in this module. +""" import pytest from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException diff --git a/test/mpl/unit/test_material_managers_mpl_materials.py b/test/mpl/unit/test_material_managers_mpl_materials.py index cb3ca7397..6c992ff24 100644 --- a/test/mpl/unit/test_material_managers_mpl_materials.py +++ b/test/mpl/unit/test_material_managers_mpl_materials.py @@ -10,7 +10,10 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic.""" +"""Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.materials logic. + +The aws-cryptographic-materials-library MUST be installed to run tests in this module. +""" import pytest from aws_cryptographic_materialproviders.mpl.models import ( @@ -18,7 +21,7 @@ EncryptedDataKey as MPL_EncryptedDataKey, EncryptionMaterials as MPL_EncryptionMaterials, ) -from mock import MagicMock, PropertyMock, patch +from mock import MagicMock, patch from typing import Dict, List, Set import aws_encryption_sdk.materials_managers.mpl.materials From cf26ca3fdb0b2cccbe011d8c2071453fe06bbb09 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 17:45:15 -0800 Subject: [PATCH 127/422] fix --- .../materials_managers/mpl/cmm.py | 2 +- .../unit/test_material_managers_mpl_cmm.py | 68 ++++++++++++------- .../test_material_managers_mpl_materials.py | 18 ++--- 3 files changed, 53 insertions(+), 35 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index 760808bfe..e0879f3fb 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -2,6 +2,7 @@ The aws-cryptographic-materials-library MUST be installed to use this module. """ +from typing import List # pylint should pass even if the MPL isn't installed # Also thinks these imports aren't used if it can't import them # noqa pylint: disable=import-error,unused-import @@ -19,7 +20,6 @@ ICryptographicMaterialsManager as MPL_ICryptographicMaterialsManager, ) # noqa pylint: enable=import-error,unused-import -from typing import List from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError from aws_encryption_sdk.identifiers import CommitmentPolicy diff --git a/test/mpl/unit/test_material_managers_mpl_cmm.py b/test/mpl/unit/test_material_managers_mpl_cmm.py index fa8f76410..060f19f95 100644 --- a/test/mpl/unit/test_material_managers_mpl_cmm.py +++ b/test/mpl/unit/test_material_managers_mpl_cmm.py @@ -76,16 +76,17 @@ def test_GIVEN_invalid_mpl_cmm_WHEN_create_CryptoMaterialsManagerFromMPL_THEN_ra @patch.object(mock_mpl_cmm, "get_encryption_materials") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._native_to_mpl_get_encryption_materials") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL" + "._native_to_mpl_get_encryption_materials") def test_GIVEN_valid_request_WHEN_get_encryption_materials_THEN_return_EncryptionMaterialsFromMPL( mock_native_to_mpl_get_encryption_materials, mock_get_encryption_materials, ): - + # Given: _native_to_mpl_get_encryption_materials creates a MPL_GetEncryptionMaterialsInput mock_get_encryption_materials_input = MagicMock(__class__=MPL_GetEncryptionMaterialsInput) mock_native_to_mpl_get_encryption_materials.return_value = mock_get_encryption_materials_input - + # Given: mpl_cmm.get_encryption_materials returns mock MPL encryption materials mock_get_encryption_materials_output = MagicMock(__class__=MPL_GetEncryptionMaterialsOutput) mock_get_encryption_materials_output.encryption_materials = mock_mpl_encryption_materials @@ -104,7 +105,8 @@ def test_GIVEN_valid_request_WHEN_get_encryption_materials_THEN_return_Encryptio mock_mpl_cmm.get_encryption_materials.assert_called_once_with(mock_get_encryption_materials_input) -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL" + "._native_to_mpl_commmitment_policy") def test_GIVEN_mpl_cmm_raises_MPLException_WHEN_get_encryption_materials_THEN_raise_ESDKException( _ ): @@ -112,13 +114,15 @@ def test_GIVEN_mpl_cmm_raises_MPLException_WHEN_get_encryption_materials_THEN_ra with pytest.raises(AWSEncryptionSDKClientError): # Given: mpl_cmm.get_encryption_materials raises MPL exception with patch.object(mock_mpl_cmm, "get_encryption_materials", - side_effect=AwsCryptographicMaterialProvidersException("any")): + side_effect=AwsCryptographicMaterialProvidersException("any")): # When: get_encryption_materials cmm = CryptoMaterialsManagerFromMPL(mpl_cmm=mock_mpl_cmm) cmm.get_encryption_materials(mock_encryption_materials_request) -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy") -def test_GIVEN_valid_mpl_commitment_policy_WHEN_native_to_mpl_get_encryption_materials_THEN_returns_MPL_GetEncryptionMaterialsInput( + +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL" + "._native_to_mpl_commmitment_policy") +def test_GIVEN_valid_mpl_commitment_policy_WHEN_native_to_mpl_get_encryption_materials_THEN_returns_MPL_GetEncryptionMaterialsInput( # noqa: E501 mock_mpl_commitment_policy ): # Given: commitment policy is some MPL ESDK commitment policy @@ -126,7 +130,9 @@ def test_GIVEN_valid_mpl_commitment_policy_WHEN_native_to_mpl_get_encryption_mat mock_mpl_commitment_policy.return_value = mock_commitment_policy # When: _native_to_mpl_get_encryption_materials - output = CryptoMaterialsManagerFromMPL._native_to_mpl_get_encryption_materials(mock_encryption_materials_request) + output = CryptoMaterialsManagerFromMPL._native_to_mpl_get_encryption_materials( + mock_encryption_materials_request + ) # Then: returned MPL_GetEncryptionMaterialsInput is correct assert isinstance(output, MPL_GetEncryptionMaterialsInput) @@ -135,7 +141,7 @@ def test_GIVEN_valid_mpl_commitment_policy_WHEN_native_to_mpl_get_encryption_mat assert output.max_plaintext_length == mock_encryption_materials_request.plaintext_length -def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_MPL_CommitmentPolicyESDK_FORBID_ENCRYPT_ALLOW_DECRYPT(): +def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_MPL_CommitmentPolicyESDK_FORBID_ENCRYPT_ALLOW_DECRYPT(): # noqa: E501 # Given: native FORBID_ENCRYPT_ALLOW_DECRYPT native_commitment_policy = CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT @@ -146,7 +152,8 @@ def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_ assert isinstance(output, MPL_CommitmentPolicyESDK) assert output.value == "FORBID_ENCRYPT_ALLOW_DECRYPT" -def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_MPL_CommitmentPolicyESDK_REQUIRE_ENCRYPT_ALLOW_DECRYPT(): + +def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_MPL_CommitmentPolicyESDK_REQUIRE_ENCRYPT_ALLOW_DECRYPT(): # noqa: E501 # Given: native REQUIRE_ENCRYPT_ALLOW_DECRYPT native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT @@ -157,7 +164,8 @@ def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl assert isinstance(output, MPL_CommitmentPolicyESDK) assert output.value == "REQUIRE_ENCRYPT_ALLOW_DECRYPT" -def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_REQUIRE_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_MPL_CommitmentPolicyESDK_REQUIRE_ENCRYPT_REQUIRE_DECRYPT(): + +def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_REQUIRE_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_MPL_CommitmentPolicyESDK_REQUIRE_ENCRYPT_REQUIRE_DECRYPT(): # noqa: E501 # Given: native REQUIRE_ENCRYPT_REQUIRE_DECRYPT native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT @@ -168,6 +176,7 @@ def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_REQUIRE_DECRYPT_WHEN_native_to_m assert isinstance(output, MPL_CommitmentPolicyESDK) assert output.value == "REQUIRE_ENCRYPT_REQUIRE_DECRYPT" + def test_GIVEN_CommitmentPolicy_unrecognized_WHEN_native_to_mpl_commmitment_policy_THEN_raise_ValueError(): # Given: invalid native commitment policy native_commitment_policy = "not a commitment policy" @@ -177,13 +186,14 @@ def test_GIVEN_CommitmentPolicy_unrecognized_WHEN_native_to_mpl_commmitment_poli # When: _native_to_mpl_commmitment_policy CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy(native_commitment_policy) + @patch.object(mock_mpl_cmm, "decrypt_materials") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._create_mpl_decrypt_materials_input_from_request") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL" + "._create_mpl_decrypt_materials_input_from_request") def test_GIVEN_valid_request_WHEN_decrypt_materials_THEN_return_DecryptionMaterialsFromMPL( mock_native_to_mpl_decrypt_materials, mock_get_encryption_materials, ): - # Given: mpl_cmm.get_decryption_materials returns mock MPL decryption materials mock_decrypt_materials_output = MagicMock(__class__=MPL_GetEncryptionMaterialsOutput) mock_decrypt_materials_output.decryption_materials = mock_mpl_decrypt_materials @@ -205,7 +215,9 @@ def test_GIVEN_valid_request_WHEN_decrypt_materials_THEN_return_DecryptionMateri # Verify we actually called `decrypt_materials` mock_mpl_cmm.decrypt_materials.assert_called_once_with(mock_decrypt_materials_input) -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._create_mpl_decrypt_materials_input_from_request") + +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL" + "._create_mpl_decrypt_materials_input_from_request") def test_GIVEN_decrypt_materials_raises_MPL_Exception_WHEN_call_decrypt_materials_THEN_raise_ESDK_Exception( _ ): @@ -213,12 +225,13 @@ def test_GIVEN_decrypt_materials_raises_MPL_Exception_WHEN_call_decrypt_material with pytest.raises(AWSEncryptionSDKClientError): # Given: mpl_cmm.decrypt_materials raises MPL exception with patch.object(mock_mpl_cmm, "decrypt_materials", - side_effect=AwsCryptographicMaterialProvidersException("any")): + side_effect=AwsCryptographicMaterialProvidersException("any")): # When: decrypt_materials cmm = CryptoMaterialsManagerFromMPL(mpl_cmm=mock_mpl_cmm) cmm.decrypt_materials(mock_decryption_materials_request) -def test_GIVEN_valid_native_algorithm_id_WHEN_native_algorithm_id_to_mpl_algorithm_id_THEN_returns_valid_MPL_AlgorithmSuiteIdESDK(): + +def test_GIVEN_valid_native_algorithm_id_WHEN_native_algorithm_id_to_mpl_algorithm_id_THEN_returns_valid_MPL_AlgorithmSuiteIdESDK(): # noqa: E501 # Given: any native algorithm ID some_native_algorithm_id = 0x1234 # Not a real algorithm ID, but fits the format @@ -231,9 +244,12 @@ def test_GIVEN_valid_native_algorithm_id_WHEN_native_algorithm_id_to_mpl_algorit assert isinstance(mpl_output, MPL_AlgorithmSuiteIdESDK) assert mpl_output.value == "0x1234" -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._native_algorithm_id_to_mpl_algorithm_id") -@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy") -def test_GIVEN_valid_request_WHEN_create_mpl_decrypt_materials_input_from_request_THEN_returns_MPL_MPL_DecryptMaterialsInput( + +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL" + "._native_algorithm_id_to_mpl_algorithm_id") +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL" + "._native_to_mpl_commmitment_policy") +def test_GIVEN_valid_request_WHEN_create_mpl_decrypt_materials_input_from_request_THEN_returns_MPL_MPL_DecryptMaterialsInput( # noqa: E501 mock_mpl_commitment_policy, mock_mpl_algorithm_id, ): @@ -245,17 +261,19 @@ def test_GIVEN_valid_request_WHEN_create_mpl_decrypt_materials_input_from_reques mock_commitment_policy = MagicMock(__class__=MPL_CommitmentPolicyESDK) mock_mpl_commitment_policy.return_value = mock_commitment_policy - no_mock_edks = [ mock_edk ] - one_mock_edk = [ mock_edk ] - two_mock_edks = [ mock_edk, mock_edk ] + no_mock_edks = [mock_edk] + one_mock_edk = [mock_edk] + two_mock_edks = [mock_edk, mock_edk] # Given: ESK lists of various lengths - for mock_edks in [ no_mock_edks, one_mock_edk, two_mock_edks ]: + for mock_edks in [no_mock_edks, one_mock_edk, two_mock_edks]: mock_decryption_materials_request.encrypted_data_keys = mock_edks # When: _create_mpl_decrypt_materials_input_from_request - output = CryptoMaterialsManagerFromMPL._create_mpl_decrypt_materials_input_from_request(mock_decryption_materials_request) + output = CryptoMaterialsManagerFromMPL._create_mpl_decrypt_materials_input_from_request( + mock_decryption_materials_request + ) # Then: # Verify general correctness of output structure @@ -273,4 +291,4 @@ def test_GIVEN_valid_request_WHEN_create_mpl_decrypt_materials_input_from_reques input_edk = mock_edks[i] assert output_edk.key_provider_id == input_edk.key_provider.provider_id assert output_edk.key_provider_info == input_edk.key_provider.key_info - assert output_edk.ciphertext == input_edk.encrypted_data_key \ No newline at end of file + assert output_edk.ciphertext == input_edk.encrypted_data_key diff --git a/test/mpl/unit/test_material_managers_mpl_materials.py b/test/mpl/unit/test_material_managers_mpl_materials.py index 6c992ff24..a2333f267 100644 --- a/test/mpl/unit/test_material_managers_mpl_materials.py +++ b/test/mpl/unit/test_material_managers_mpl_materials.py @@ -22,10 +22,10 @@ EncryptionMaterials as MPL_EncryptionMaterials, ) from mock import MagicMock, patch -from typing import Dict, List, Set +from typing import Dict import aws_encryption_sdk.materials_managers.mpl.materials -from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite +from aws_encryption_sdk.identifiers import AlgorithmSuite from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest from aws_encryption_sdk.materials_managers.mpl.materials import DecryptionMaterialsFromMPL, EncryptionMaterialsFromMPL @@ -48,11 +48,11 @@ mock_edk.ciphertext = mock_mpl_ciphertext -def test_GIVEN_valid_mpl_materials_WHEN_create_EncryptionMaterialsFromMPL_THEN_return_new_CryptoMaterialsManagerFromMPL(): +def test_GIVEN_mpl_materials_WHEN_create_EncryptionMaterialsFromMPL_THEN_return_new_CryptoMaterialsManagerFromMPL(): # Given: valid mpl_materials # When: create EncryptionMaterialsFromMPL mpl_encryption_materials = EncryptionMaterialsFromMPL(mpl_materials=mock_mpl_encryption_materials) - + # Then: EncryptionMaterialsFromMPL is valid assert mpl_encryption_materials.mpl_materials == mock_mpl_encryption_materials @@ -93,7 +93,7 @@ def test_GIVEN_valid_mpl_algorithm_id_WHEN_EncryptionMaterials_get_algorithm_THE # When: Get algorithm mpl_encryption_materials = EncryptionMaterialsFromMPL(mpl_materials=mock_mpl_encryption_materials) output = mpl_encryption_materials.algorithm - + # Then: output is valid assert output == mock_algorithm() # property calls automatically, we need to call the mock @@ -112,12 +112,12 @@ def test_GIVEN_valid_encryption_context_WHEN_EncryptionMaterials_get_encryption_ def test_GIVEN_valid_edks_WHEN_EncryptionMaterials_get_edks_THEN_returns_edks(): - + # Given: lists of mocked EDKs of various lengths no_mock_edks = [] - one_mock_edk = [ mock_edk ] - two_mocked_edks = [ mock_edk, mock_edk ] - for mock_edks in [ no_mock_edks, one_mock_edk, two_mocked_edks ]: + one_mock_edk = [mock_edk] + two_mocked_edks = [mock_edk, mock_edk] + for mock_edks in [no_mock_edks, one_mock_edk, two_mocked_edks]: mock_mpl_encryption_materials.encrypted_data_keys = mock_edks # When: get EDKs From 7f27ebdb1cbe0a7b7039d8912ffb8deed032a5e2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 17:46:38 -0800 Subject: [PATCH 128/422] fix --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 3daa40e47..952d3c24f 100644 --- a/tox.ini +++ b/tox.ini @@ -112,7 +112,7 @@ passenv = setenv = ######################################################### deps = -rdev_requirements/test-requirements.txt -commands = {[testenv:base-command]commands} test/ -m local --ignore test/unit/mpl/ +commands = {[testenv:base-command]commands} test/ -m local --ignore test/mpl/ # Collect requirements for use in upstream tests [testenv:freeze-upstream-requirements-base] @@ -144,7 +144,7 @@ commands = {[testenv:freeze-upstream-requirements-base]commands} test/upstream-r [testenv:test-upstream-requirements-base] sitepackages = False recreate = True -commands = {[testenv:base-command]commands} test/ -m local --ignore test/unit/mpl/ +commands = {[testenv:base-command]commands} test/ -m local --ignore --ignore test/mpl/ # Test frozen upstream requirements for Python 3.7 [testenv:test-upstream-requirements-py37] From 00f4721542d8529eb2ca6b0ac621b440940055be Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 17:48:24 -0800 Subject: [PATCH 129/422] fix --- src/aws_encryption_sdk/materials_managers/mpl/materials.py | 2 +- test/mpl/__init__.py | 2 +- tox.ini | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index faa47cb46..39aff2c3c 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -2,6 +2,7 @@ The aws-cryptographic-materials-library MUST be installed to use this module. """ +from typing import Dict, List, Set # pylint should pass even if the MPL isn't installed # noqa pylint: disable=import-error from aws_cryptographic_materialproviders.mpl.models import ( @@ -10,7 +11,6 @@ EncryptionMaterials as MPL_EncryptionMaterials, ) # noqa pylint: enable=import-error -from typing import Dict, List, Set from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite from aws_encryption_sdk.materials_managers import ( diff --git a/test/mpl/__init__.py b/test/mpl/__init__.py index d3f78d0bf..37f482e0b 100644 --- a/test/mpl/__init__.py +++ b/test/mpl/__init__.py @@ -13,4 +13,4 @@ """Module testing components that use the MPL. The aws-cryptographic-materials-library MUST be installed to run tests in this module. -""" \ No newline at end of file +""" diff --git a/tox.ini b/tox.ini index 952d3c24f..3644c973a 100644 --- a/tox.ini +++ b/tox.ini @@ -144,7 +144,7 @@ commands = {[testenv:freeze-upstream-requirements-base]commands} test/upstream-r [testenv:test-upstream-requirements-base] sitepackages = False recreate = True -commands = {[testenv:base-command]commands} test/ -m local --ignore --ignore test/mpl/ +commands = {[testenv:base-command]commands} test/ -m local --ignore test/mpl/ # Test frozen upstream requirements for Python 3.7 [testenv:test-upstream-requirements-py37] From 018b93f3ed2b5eae5bf6c2e4dc5d7837e38a34a3 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 17:55:46 -0800 Subject: [PATCH 130/422] fix --- src/aws_encryption_sdk/materials_managers/mpl/cmm.py | 7 ++++++- src/aws_encryption_sdk/materials_managers/mpl/materials.py | 4 +++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index e0879f3fb..c262cf7ce 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -2,7 +2,6 @@ The aws-cryptographic-materials-library MUST be installed to use this module. """ -from typing import List # pylint should pass even if the MPL isn't installed # Also thinks these imports aren't used if it can't import them # noqa pylint: disable=import-error,unused-import @@ -21,6 +20,9 @@ ) # noqa pylint: enable=import-error,unused-import +# pylint and isort disagree on where this should go. Choose isort and disable pylint for this. +from typing import List # noqa pylint: disable=wrong-import-order + from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError from aws_encryption_sdk.identifiers import CommitmentPolicy from aws_encryption_sdk.materials_managers import DecryptionMaterialsRequest, EncryptionMaterialsRequest @@ -28,6 +30,9 @@ from aws_encryption_sdk.materials_managers.mpl.materials import DecryptionMaterialsFromMPL, EncryptionMaterialsFromMPL from aws_encryption_sdk.structures import EncryptedDataKey as Native_EncryptedDataKey +# noqa pylint: enable=import-error,unused-import + + class CryptoMaterialsManagerFromMPL(CryptoMaterialsManager): """ diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index 39aff2c3c..43579fac6 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -2,7 +2,6 @@ The aws-cryptographic-materials-library MUST be installed to use this module. """ -from typing import Dict, List, Set # pylint should pass even if the MPL isn't installed # noqa pylint: disable=import-error from aws_cryptographic_materialproviders.mpl.models import ( @@ -12,6 +11,9 @@ ) # noqa pylint: enable=import-error +# pylint and isort disagree on where this should go. Choose isort and disable pylint for this. +from typing import Dict, List, Set # noqa pylint: disable=wrong-import-order + from aws_encryption_sdk.identifiers import Algorithm, AlgorithmSuite from aws_encryption_sdk.materials_managers import ( DecryptionMaterials as Native_DecryptionMaterials, From d413b65024d398510f1811c8638109ae7c886336 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 18:00:04 -0800 Subject: [PATCH 131/422] fix --- src/aws_encryption_sdk/materials_managers/mpl/cmm.py | 4 ---- src/aws_encryption_sdk/materials_managers/mpl/materials.py | 2 -- 2 files changed, 6 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index c262cf7ce..a0119a588 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -19,7 +19,6 @@ ICryptographicMaterialsManager as MPL_ICryptographicMaterialsManager, ) # noqa pylint: enable=import-error,unused-import - # pylint and isort disagree on where this should go. Choose isort and disable pylint for this. from typing import List # noqa pylint: disable=wrong-import-order @@ -30,9 +29,6 @@ from aws_encryption_sdk.materials_managers.mpl.materials import DecryptionMaterialsFromMPL, EncryptionMaterialsFromMPL from aws_encryption_sdk.structures import EncryptedDataKey as Native_EncryptedDataKey -# noqa pylint: enable=import-error,unused-import - - class CryptoMaterialsManagerFromMPL(CryptoMaterialsManager): """ diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index 43579fac6..4508d5545 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -9,8 +9,6 @@ EncryptedDataKey as MPL_EncryptedDataKey, EncryptionMaterials as MPL_EncryptionMaterials, ) -# noqa pylint: enable=import-error - # pylint and isort disagree on where this should go. Choose isort and disable pylint for this. from typing import Dict, List, Set # noqa pylint: disable=wrong-import-order From c4ca658d08efdac51f85c04b9aa9cd5cf37a3a60 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 23 Feb 2024 18:09:29 -0800 Subject: [PATCH 132/422] copyright --- examples/src/keyrings/__init__.py | 14 ++------------ .../src/keyrings/example_branch_key_id_supplier.py | 2 ++ examples/test/keyrings/__init__.py | 14 ++------------ .../test/keyrings/test_i_hierarchical_keyring.py | 2 ++ .../materials_managers/mpl/__init__.py | 14 ++------------ .../materials_managers/mpl/cmm.py | 2 ++ .../materials_managers/mpl/materials.py | 2 ++ test/mpl/__init__.py | 14 ++------------ test/mpl/unit/test_material_managers_mpl_cmm.py | 14 ++------------ .../unit/test_material_managers_mpl_materials.py | 14 ++------------ test/unit/test_streaming_client_mpl_import.py | 14 ++------------ 11 files changed, 22 insertions(+), 84 deletions(-) diff --git a/examples/src/keyrings/__init__.py b/examples/src/keyrings/__init__.py index e8fd618b1..120179eda 100644 --- a/examples/src/keyrings/__init__.py +++ b/examples/src/keyrings/__init__.py @@ -1,13 +1,3 @@ -# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 """Stub module indicator to make linter configuration simpler.""" diff --git a/examples/src/keyrings/example_branch_key_id_supplier.py b/examples/src/keyrings/example_branch_key_id_supplier.py index ba9ae060c..7b390cdda 100644 --- a/examples/src/keyrings/example_branch_key_id_supplier.py +++ b/examples/src/keyrings/example_branch_key_id_supplier.py @@ -1,3 +1,5 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 """Example implementation of a branch key ID supplier.""" from aws_cryptographic_materialproviders.mpl.models import GetBranchKeyIdInput, GetBranchKeyIdOutput diff --git a/examples/test/keyrings/__init__.py b/examples/test/keyrings/__init__.py index e8fd618b1..120179eda 100644 --- a/examples/test/keyrings/__init__.py +++ b/examples/test/keyrings/__init__.py @@ -1,13 +1,3 @@ -# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 """Stub module indicator to make linter configuration simpler.""" diff --git a/examples/test/keyrings/test_i_hierarchical_keyring.py b/examples/test/keyrings/test_i_hierarchical_keyring.py index d80bb565d..4cae478d7 100644 --- a/examples/test/keyrings/test_i_hierarchical_keyring.py +++ b/examples/test/keyrings/test_i_hierarchical_keyring.py @@ -1,3 +1,5 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 """Unit test suite for the hierarchical keyring example.""" import pytest diff --git a/src/aws_encryption_sdk/materials_managers/mpl/__init__.py b/src/aws_encryption_sdk/materials_managers/mpl/__init__.py index 7593a3300..be75f3566 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/__init__.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/__init__.py @@ -1,15 +1,5 @@ -# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 """Modules related to the MPL's materials managers interfaces. The aws-cryptographic-materials-library MUST be installed to use these modules. diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index a0119a588..53a4b3505 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -1,3 +1,5 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 """Retrieves encryption/decryption materials from the MPL and interfaces them to EDK components. The aws-cryptographic-materials-library MUST be installed to use this module. diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index 4508d5545..dfd1bd6fc 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -1,3 +1,5 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 """Provides encryption/decryption materials from an underlying materials provider from the MPL. The aws-cryptographic-materials-library MUST be installed to use this module. diff --git a/test/mpl/__init__.py b/test/mpl/__init__.py index 37f482e0b..79522d342 100644 --- a/test/mpl/__init__.py +++ b/test/mpl/__init__.py @@ -1,15 +1,5 @@ -# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 """Module testing components that use the MPL. The aws-cryptographic-materials-library MUST be installed to run tests in this module. diff --git a/test/mpl/unit/test_material_managers_mpl_cmm.py b/test/mpl/unit/test_material_managers_mpl_cmm.py index 060f19f95..80d6f00ee 100644 --- a/test/mpl/unit/test_material_managers_mpl_cmm.py +++ b/test/mpl/unit/test_material_managers_mpl_cmm.py @@ -1,15 +1,5 @@ -# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 """Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.cmm logic. The aws-cryptographic-materials-library MUST be installed to run tests in this module. diff --git a/test/mpl/unit/test_material_managers_mpl_materials.py b/test/mpl/unit/test_material_managers_mpl_materials.py index a2333f267..9e76556a2 100644 --- a/test/mpl/unit/test_material_managers_mpl_materials.py +++ b/test/mpl/unit/test_material_managers_mpl_materials.py @@ -1,15 +1,5 @@ -# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 """Unit test suite to validate aws_encryption_sdk.materials_managers.mpl.materials logic. The aws-cryptographic-materials-library MUST be installed to run tests in this module. diff --git a/test/unit/test_streaming_client_mpl_import.py b/test/unit/test_streaming_client_mpl_import.py index a4ca87e2a..638b04fd6 100644 --- a/test/unit/test_streaming_client_mpl_import.py +++ b/test/unit/test_streaming_client_mpl_import.py @@ -1,15 +1,5 @@ -# Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 """Unit test suite to validate aws_encryption_sdk.streaming_client MPL import logic.""" import pytest From d99b6667bb1a7d65a36af598889edeab2beecfc6 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 26 Feb 2024 12:40:24 -0800 Subject: [PATCH 133/422] more unit tests --- src/aws_encryption_sdk/streaming_client.py | 34 +++-- .../unit/test_crypto_authentication_signer.py | 63 +++++++-- test/unit/test_streaming_client_configs.py | 96 +++++++++++++ .../test_streaming_client_stream_decryptor.py | 132 +++++++++++++++++- .../test_streaming_client_stream_encryptor.py | 79 +++++++++++ test/unit/test_utils.py | 25 ++++ 6 files changed, 401 insertions(+), 28 deletions(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 959b5ff0b..2cfcc9a02 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -73,8 +73,9 @@ try: # pylint should pass even if the MPL isn't installed # noqa pylint: disable=import-error - from aws_cryptographic_materialproviders.mpl.client import AwsCryptographicMaterialProviders + from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig + from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException from aws_cryptographic_materialproviders.mpl.models import CreateDefaultCryptographicMaterialsManagerInput from aws_cryptographic_materialproviders.mpl.references import IKeyring _HAS_MPL = True @@ -147,9 +148,6 @@ def _has_mpl_attrs_post_init(self): """If the MPL is present in the runtime, perform MPL-specific post-init logic to validate the new object has a valid state. """ - if not hasattr(self, "keyring"): - self._no_mpl_attrs_post_init() - return if not exactly_one_arg_is_not_none(self.materials_manager, self.key_provider, self.keyring): raise TypeError("Exactly one of keyring, materials_manager, or key_provider must be provided") if self.materials_manager is None: @@ -159,21 +157,21 @@ def _has_mpl_attrs_post_init(self): master_key_provider=self.key_provider ) elif self.keyring is not None: - # No CMM, provided MPL keyring => create MPL's DefaultCryptographicMaterialsManager - if not isinstance(self.keyring, IKeyring): - raise ValueError(f"Argument provided to keyring MUST be a {IKeyring}. \ - Found {self.keyring.__class__.__name__}") - - mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( - config=MaterialProvidersConfig() - ) - cmm = mat_prov.create_default_cryptographic_materials_manager( - CreateDefaultCryptographicMaterialsManagerInput( - keyring=self.keyring + try: + mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + config=MaterialProvidersConfig() ) - ) - cmm_handler: CryptoMaterialsManager = CryptoMaterialsManagerFromMPL(cmm) - self.materials_manager = cmm_handler + cmm = mat_prov.create_default_cryptographic_materials_manager( + CreateDefaultCryptographicMaterialsManagerInput( + keyring=self.keyring + ) + ) + cmm_handler: CryptoMaterialsManager = CryptoMaterialsManagerFromMPL(cmm) + self.materials_manager = cmm_handler + except AwsCryptographicMaterialProvidersException as mpl_exception: + # Wrap MPL error into the ESDK error type + # so customers only have to catch ESDK error types. + raise AWSEncryptionSDKClientError(mpl_exception) def _no_mpl_attrs_post_init(self): """If the MPL is NOT present in the runtime, perform post-init logic diff --git a/test/unit/test_crypto_authentication_signer.py b/test/unit/test_crypto_authentication_signer.py index bd7227fd3..c37c97bde 100644 --- a/test/unit/test_crypto_authentication_signer.py +++ b/test/unit/test_crypto_authentication_signer.py @@ -12,7 +12,8 @@ # language governing permissions and limitations under the License. """Unit test suite for ``aws_encryption_sdk.internal.crypto.authentication.Signer``.""" import pytest -from mock import MagicMock, sentinel +from mock import MagicMock, sentinel, patch +import cryptography.hazmat.primitives.serialization from pytest_mock import mocker # noqa pylint: disable=unused-import import aws_encryption_sdk.internal.crypto.authentication @@ -75,28 +76,72 @@ def test_f_signer_from_key_bytes(): def test_f_signer_key_bytes(): test = Signer(algorithm=ALGORITHM, key=VALUES["ecc_private_key_prime"]) assert test.key_bytes() == VALUES["ecc_private_key_prime_private_bytes"] + +def test_GIVEN_no_encoding_WHEN_signer_from_key_bytes_THEN_load_der_private_key( + patch_default_backend, + patch_build_hasher, + patch_ec +): + mock_algorithm_info = MagicMock(return_value=sentinel.algorithm_info, spec=patch_ec.EllipticCurve) + _algorithm = MagicMock(signing_algorithm_info=mock_algorithm_info) -def test_signer_from_key_bytes(patch_default_backend, patch_serialization, patch_build_hasher, patch_ec): + # Make a new patched serialization module for this test. + # The default patch introduces serialization as `serialization.Encoding.DER` + # from within the src, but is `Encoding.DER` in the test. + # This namespace change causes the src's `isinstance` checks to fail. + # Mock the `serialization.Encoding.DER` + with patch.object(cryptography.hazmat.primitives, "serialization"): + # Mock the `serialization.load_der_private_key` + with patch.object(aws_encryption_sdk.internal.crypto.authentication.serialization, "load_der_private_key") as mock_der: + Signer.from_key_bytes( + algorithm=_algorithm, + key_bytes=sentinel.key_bytes, + ) + + mock_der.assert_called_once_with( + data=sentinel.key_bytes, password=None, backend=patch_default_backend.return_value + ) + + +def test_GIVEN_PEM_encoding_WHEN_signer_from_key_bytes_THEN_load_pem_private_key( + patch_default_backend, + patch_serialization, + patch_build_hasher, + patch_ec +): mock_algorithm_info = MagicMock(return_value=sentinel.algorithm_info, spec=patch_ec.EllipticCurve) _algorithm = MagicMock(signing_algorithm_info=mock_algorithm_info) - # Explicitly pass in patched serialization module. - # Patching the module introduces namespace issues - # which causes the method's `isinstance` checks to fail - # by changing the namespace from `serialization.Encoding.DER` to `Encoding.DER`. signer = Signer.from_key_bytes( algorithm=_algorithm, key_bytes=sentinel.key_bytes, - encoding=patch_serialization.Encoding.DER + encoding=patch_serialization.Encoding.PEM ) - patch_serialization.load_der_private_key.assert_called_once_with( + patch_serialization.load_pem_private_key.assert_called_once_with( data=sentinel.key_bytes, password=None, backend=patch_default_backend.return_value ) assert isinstance(signer, Signer) assert signer.algorithm is _algorithm - assert signer.key is patch_serialization.load_der_private_key.return_value + assert signer.key is patch_serialization.load_pem_private_key.return_value + + +def test_GIVEN_unrecognized_encoding_WHEN_signer_from_key_bytes_THEN_raise_ValueError( + patch_default_backend, + patch_serialization, + patch_build_hasher, + patch_ec +): + mock_algorithm_info = MagicMock(return_value=sentinel.algorithm_info, spec=patch_ec.EllipticCurve) + _algorithm = MagicMock(signing_algorithm_info=mock_algorithm_info) + + with pytest.raises(ValueError): + signer = Signer.from_key_bytes( + algorithm=_algorithm, + key_bytes=sentinel.key_bytes, + encoding="not an encoding" + ) def test_signer_key_bytes(patch_default_backend, patch_serialization, patch_build_hasher, patch_ec): diff --git a/test/unit/test_streaming_client_configs.py b/test/unit/test_streaming_client_configs.py index 426f8f85f..c76a64ea7 100644 --- a/test/unit/test_streaming_client_configs.py +++ b/test/unit/test_streaming_client_configs.py @@ -15,6 +15,7 @@ import pytest import six +from mock import patch from aws_encryption_sdk import CommitmentPolicy from aws_encryption_sdk.internal.defaults import ALGORITHM, FRAME_LENGTH, LINE_LENGTH @@ -28,6 +29,22 @@ pytestmark = [pytest.mark.unit, pytest.mark.local] +# Check if MPL is installed, and skip tests based on its installation status +# Ideally, this logic would be based on mocking imports and testing logic, +# but doing that introduces errors that cause other tests to fail. +try: + from aws_cryptographic_materialproviders.mpl.references import ( + IKeyring, + ) + HAS_MPL = True + + from aws_encryption_sdk.materials_managers.mpl.cmm import ( + CryptoMaterialsManagerFromMPL, + ) +except ImportError: + HAS_MPL = False + + class FakeCryptoMaterialsManager(CryptoMaterialsManager): def get_encryption_materials(self, request): return @@ -42,6 +59,14 @@ class FakeMasterKeyProvider(MasterKeyProvider): def _new_master_key(self, key_id): return + +if HAS_MPL: + class FakeKeyring(IKeyring): + def on_encrypt(self, param): + return + + def on_decrypt(self, param): + return BASE_KWARGS = dict( @@ -126,6 +151,18 @@ def test_client_config_defaults(): assert test.max_encrypted_data_keys is None +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_client_config_with_mpl_attr(): + test = _ClientConfig(**BASE_KWARGS) + assert hasattr(test, "keyring") + + +@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") +def test_client_config_no_mpl(): + test = _ClientConfig(**BASE_KWARGS) + assert not hasattr(test, "keyring") + + def test_encryptor_config_defaults(): test = EncryptorConfig(**BASE_KWARGS) assert test.encryption_context == {} @@ -154,3 +191,62 @@ def test_client_config_converts(kwargs, stream_type): assert isinstance(test.source, stream_type) if test.key_provider is not None: assert isinstance(test.materials_manager, DefaultCryptoMaterialsManager) + + +@pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") +@patch.object(_ClientConfig, "_no_mpl_attrs_post_init") +def test_GIVEN_no_mpl_WHEN_attrs_post_init_THEN_calls_no_mpl_method( + mock_no_mpl_attrs_post_init, +): + _ClientConfig(**BASE_KWARGS) + mock_no_mpl_attrs_post_init.assert_called_once_with() + + +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +@patch.object(_ClientConfig, "_has_mpl_attrs_post_init") +def test_GIVEN_has_mpl_WHEN_attrs_post_init_THEN_calls_no_mpl_method( + _has_mpl_attrs_post_init, +): + _ClientConfig(**BASE_KWARGS) + _has_mpl_attrs_post_init.assert_called_once_with() + + +@pytest.mark.parametrize( + "kwargs, stream_type", + ( + (dict(source=b"", materials_manager=FakeCryptoMaterialsManager()), io.BytesIO), + (dict(source=b"", key_provider=FakeMasterKeyProvider()), io.BytesIO), + (dict(source="", materials_manager=FakeCryptoMaterialsManager()), io.BytesIO), + (dict(source=io.BytesIO(), materials_manager=FakeCryptoMaterialsManager()), io.BytesIO), + (dict(source=six.StringIO(), materials_manager=FakeCryptoMaterialsManager()), six.StringIO), + (dict(source=b"", keyring=FakeKeyring()), io.BytesIO), + ), +) +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_client_configs_with_mpl( + kwargs, + stream_type +): + kwargs["commitment_policy"] = CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT + + test = _ClientConfig(**kwargs) + + # In all cases, config should have a materials manager + assert test.materials_manager is not None + + # If materials manager was provided, it should be directly used + if hasattr(kwargs, "materials_manager"): + assert kwargs["materials_manager"] == test.materials_manager + + # If MPL keyring was provided, it should be wrapped in MPL materials manager + if hasattr(kwargs, "keyring"): + assert test.keyring is not None + assert test.keyring == kwargs["keyring"] + assert isinstance(test.keyring, IKeyring) + assert isinstance(test.materials_manager, CryptoMaterialsManagerFromMPL) + + # If native key_provider was provided, it should be wrapped in native materials manager + if hasattr(kwargs, "key_provider"): + assert test.key_provider is not None + assert test.key_provider == kwargs["key_provider"] + assert isinstance(test.materials_manager, DefaultCryptoMaterialsManager) diff --git a/test/unit/test_streaming_client_stream_decryptor.py b/test/unit/test_streaming_client_stream_decryptor.py index 157755094..c8a17e650 100644 --- a/test/unit/test_streaming_client_stream_decryptor.py +++ b/test/unit/test_streaming_client_stream_decryptor.py @@ -33,14 +33,36 @@ pytestmark = [pytest.mark.unit, pytest.mark.local] +# Check if MPL is installed, and skip tests based on its installation status +# Ideally, this logic would be based on mocking imports and testing logic, +# but doing that introduces errors that cause other tests to fail. +try: + from aws_cryptographic_materialproviders.mpl.references import ( + IKeyring, + ) + HAS_MPL = True + + from aws_encryption_sdk.materials_managers.mpl.cmm import ( + CryptoMaterialsManagerFromMPL, + ) +except ImportError: + HAS_MPL = False + + class TestStreamDecryptor(object): @pytest.fixture(autouse=True) def apply_fixtures(self): self.mock_key_provider = MagicMock(__class__=MasterKeyProvider) self.mock_materials_manager = MagicMock(__class__=CryptoMaterialsManager) - self.mock_materials_manager.decrypt_materials.return_value = MagicMock( + self.mock_decrypt_materials = MagicMock( data_key=VALUES["data_key_obj"], verification_key=sentinel.verification_key ) + self.mock_materials_manager.decrypt_materials.return_value = self.mock_decrypt_materials + + if HAS_MPL: + self.mock_mpl_materials_manager = MagicMock(__class__=CryptoMaterialsManagerFromMPL) + self.mock_mpl_materials_manager.decrypt_materials.return_value = self.mock_decrypt_materials + self.mock_header = MagicMock() self.mock_header.version = SerializationVersion.V1 self.mock_header.algorithm = MagicMock( @@ -213,6 +235,114 @@ def test_read_header(self, mock_derive_datakey, mock_decrypt_materials_request, assert test_header is self.mock_header assert test_header_auth is sentinel.header_auth + @patch("aws_encryption_sdk.streaming_client.DecryptionMaterialsRequest") + @patch("aws_encryption_sdk.streaming_client.derive_data_encryption_key") + @patch("aws_encryption_sdk.streaming_client.Verifier") + @pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") + def test_GIVEN_verification_key_AND_no_mpl_WHEN_read_header_THEN_calls_from_key_bytes( + self, + mock_verifier, + *_, + ): + mock_verifier_instance = MagicMock() + mock_verifier.from_key_bytes.return_value = mock_verifier_instance + ct_stream = io.BytesIO(VALUES["data_128"]) + mock_commitment_policy = MagicMock(__class__=CommitmentPolicy) + test_decryptor = StreamDecryptor( + materials_manager=self.mock_materials_manager, + source=ct_stream, + commitment_policy=mock_commitment_policy, + ) + test_decryptor.source_stream = ct_stream + test_decryptor._stream_length = len(VALUES["data_128"]) + + test_decryptor._read_header() + + mock_verifier.from_key_bytes.assert_called_once_with( + algorithm=self.mock_header.algorithm, key_bytes=sentinel.verification_key + ) + + @patch("aws_encryption_sdk.streaming_client.DecryptionMaterialsRequest") + @patch("aws_encryption_sdk.streaming_client.derive_data_encryption_key") + @patch("aws_encryption_sdk.streaming_client.Verifier") + @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") + def test_GIVEN_verification_key_AND_has_mpl_AND_not_MPLCMM_WHEN_read_header_THEN_calls_from_key_bytes( + self, + mock_verifier, + *_, + ): + mock_verifier_instance = MagicMock() + mock_verifier.from_key_bytes.return_value = mock_verifier_instance + ct_stream = io.BytesIO(VALUES["data_128"]) + mock_commitment_policy = MagicMock(__class__=CommitmentPolicy) + test_decryptor = StreamDecryptor( + materials_manager=self.mock_materials_manager, + source=ct_stream, + commitment_policy=mock_commitment_policy, + ) + test_decryptor.source_stream = ct_stream + test_decryptor._stream_length = len(VALUES["data_128"]) + + test_decryptor._read_header() + + mock_verifier.from_key_bytes.assert_called_once_with( + algorithm=self.mock_header.algorithm, key_bytes=sentinel.verification_key + ) + + @patch("aws_encryption_sdk.streaming_client.DecryptionMaterialsRequest") + @patch("aws_encryption_sdk.streaming_client.derive_data_encryption_key") + @patch("aws_encryption_sdk.streaming_client.Verifier") + @patch("base64.b64encode") + @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") + def test_GIVEN_verification_key_AND_has_mpl_AND_has_MPLCMM_WHEN_read_header_THEN_calls_from_encoded_point( + self, + mock_b64encoding, + mock_verifier, + *_, + ): + mock_verifier_instance = MagicMock() + mock_verifier.from_key_bytes.return_value = mock_verifier_instance + ct_stream = io.BytesIO(VALUES["data_128"]) + mock_commitment_policy = MagicMock(__class__=CommitmentPolicy) + test_decryptor = StreamDecryptor( + materials_manager=self.mock_mpl_materials_manager, + source=ct_stream, + commitment_policy=mock_commitment_policy, + ) + test_decryptor.source_stream = ct_stream + test_decryptor._stream_length = len(VALUES["data_128"]) + + test_decryptor._read_header() + + mock_verifier.from_encoded_point.assert_called_once_with( + algorithm=self.mock_header.algorithm, encoded_point=mock_b64encoding() + ) + + # @patch("aws_encryption_sdk.streaming_client.Verifier") + # @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") + # def test_GIVEN_verification_key_AND_has_mpl_AND_not_MPLCMM_WHEN_read_header_THEN_calls_from_key_bytes( + # self, + # mock_verifier, + # ): + # mock_verifier_instance = MagicMock() + # mock_verifier.from_key_bytes.return_value = mock_verifier_instance + # ct_stream = io.BytesIO(VALUES["data_128"]) + # mock_commitment_policy = MagicMock(__class__=CommitmentPolicy) + # test_decryptor = StreamDecryptor( + # materials_manager=self.mock_materials_manager, + # source=ct_stream, + # commitment_policy=mock_commitment_policy, + # ) + # test_decryptor.source_stream = ct_stream + # test_decryptor._stream_length = len(VALUES["data_128"]) + + # test_decryptor._read_header() + + # mock_verifier.from_key_bytes.assert_called_once_with( + # algorithm=self.mock_header.algorithm, key_bytes=sentinel.verification_key + # ) + + @patch("aws_encryption_sdk.streaming_client.derive_data_encryption_key") def test_read_header_frame_too_large(self, mock_derive_datakey): self.mock_header.content_type = ContentType.FRAMED_DATA diff --git a/test/unit/test_streaming_client_stream_encryptor.py b/test/unit/test_streaming_client_stream_encryptor.py index 5bfd0c903..11664411a 100644 --- a/test/unit/test_streaming_client_stream_encryptor.py +++ b/test/unit/test_streaming_client_stream_encryptor.py @@ -13,6 +13,7 @@ """Unit test suite for aws_encryption_sdk.streaming_client.StreamEncryptor""" import io +from cryptography.hazmat.primitives import serialization import pytest import six from mock import MagicMock, call, patch, sentinel @@ -37,6 +38,22 @@ pytestmark = [pytest.mark.unit, pytest.mark.local] +# Check if MPL is installed, and skip tests based on its installation status +# Ideally, this logic would be based on mocking imports and testing logic, +# but doing that introduces errors that cause other tests to fail. +try: + from aws_cryptographic_materialproviders.mpl.references import ( + IKeyring, + ) + HAS_MPL = True + + from aws_encryption_sdk.materials_managers.mpl.cmm import ( + CryptoMaterialsManagerFromMPL, + ) +except ImportError: + HAS_MPL = False + + class TestStreamEncryptor(object): @pytest.fixture(autouse=True) def apply_fixtures(self): @@ -60,6 +77,10 @@ def apply_fixtures(self): self.mock_master_keys_set, ) + if HAS_MPL: + self.mock_mpl_materials_manager = MagicMock(__class__=CryptoMaterialsManagerFromMPL) + self.mock_mpl_materials_manager.get_encryption_materials.return_value = self.mock_encryption_materials + self.mock_master_key = MagicMock(__class__=MasterKey) self.mock_frame_length = MagicMock(__class__=int) @@ -366,6 +387,64 @@ def test_prep_message_non_framed_message(self, mock_write_header, mock_prep_non_ test_encryptor._prep_message() mock_prep_non_framed.assert_called_once_with() + @pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") + def test_GIVEN_no_mpl_AND_uses_signer_WHEN_prep_message_THEN_signer_uses_default_encoding(self): + self.mock_encryption_materials.algorithm = Algorithm.AES_128_GCM_IV12_TAG16 + test_encryptor = StreamEncryptor( + source=VALUES["data_128"], + materials_manager=self.mock_materials_manager, + frame_length=self.mock_frame_length, + algorithm=Algorithm.AES_128_GCM_IV12_TAG16, + commitment_policy=self.mock_commitment_policy, + signature_policy=self.mock_signature_policy, + ) + test_encryptor.content_type = ContentType.FRAMED_DATA + with patch.object(self.mock_signer, "from_key_bytes"): + test_encryptor._prep_message() + self.mock_signer.from_key_bytes.assert_called_once_with( + algorithm=self.mock_encryption_materials.algorithm, + key_bytes=self.mock_encryption_materials.signing_key + ) + + @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") + def test_GIVEN_has_mpl_AND_not_MPLCMM_AND_uses_signer_WHEN_prep_message_THEN_signer_uses_default_encoding(self): + self.mock_encryption_materials.algorithm = Algorithm.AES_128_GCM_IV12_TAG16 + test_encryptor = StreamEncryptor( + source=VALUES["data_128"], + materials_manager=self.mock_materials_manager, + frame_length=self.mock_frame_length, + algorithm=Algorithm.AES_128_GCM_IV12_TAG16, + commitment_policy=self.mock_commitment_policy, + signature_policy=self.mock_signature_policy, + ) + test_encryptor.content_type = ContentType.FRAMED_DATA + with patch.object(self.mock_signer, "from_key_bytes"): + test_encryptor._prep_message() + self.mock_signer.from_key_bytes.assert_called_once_with( + algorithm=self.mock_encryption_materials.algorithm, + key_bytes=self.mock_encryption_materials.signing_key + ) + + @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") + def test_GIVEN_has_mpl_AND_has_MPLCMM_AND_uses_signer_WHEN_prep_message_THEN_signer_uses_default_encoding(self): + self.mock_encryption_materials.algorithm = Algorithm.AES_128_GCM_IV12_TAG16 + test_encryptor = StreamEncryptor( + source=VALUES["data_128"], + materials_manager=self.mock_mpl_materials_manager, + frame_length=self.mock_frame_length, + algorithm=Algorithm.AES_128_GCM_IV12_TAG16, + commitment_policy=self.mock_commitment_policy, + signature_policy=self.mock_signature_policy, + ) + test_encryptor.content_type = ContentType.FRAMED_DATA + with patch.object(self.mock_signer, "from_key_bytes"): + test_encryptor._prep_message() + self.mock_signer.from_key_bytes.assert_called_once_with( + algorithm=self.mock_encryption_materials.algorithm, + key_bytes=self.mock_encryption_materials.signing_key, + encoding=serialization.Encoding.PEM + ) + def test_prep_message_no_signer(self): self.mock_encryption_materials.algorithm = Algorithm.AES_128_GCM_IV12_TAG16 test_encryptor = StreamEncryptor( diff --git a/test/unit/test_utils.py b/test/unit/test_utils.py index c6d565108..d717b51c7 100644 --- a/test/unit/test_utils.py +++ b/test/unit/test_utils.py @@ -265,3 +265,28 @@ def test_source_data_key_length_check_invalid(self): source_data_key=mock_data_key, algorithm=mock_algorithm ) excinfo.match("Invalid Source Data Key length 4 for algorithm required: 5") + + def test_exactly_one_arg_is_not_none(self): + # No args => no args are not None + assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none() is False + assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none( + None + ) is False + assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none( + "not None" + ) is True + assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none( + "not None", "also not None" + ) is False + assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none( + "not None", None + ) is True + assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none( + "not None", "also not None" + ) is False + assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none( + None, "not None" + ) is True + assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none( + None, None + ) is False \ No newline at end of file From 49cb7c8d8b57f125c22fcddd206f18a31347e7fc Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 26 Feb 2024 12:45:57 -0800 Subject: [PATCH 134/422] more unit tests --- test/unit/test_streaming_client_configs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/test_streaming_client_configs.py b/test/unit/test_streaming_client_configs.py index c76a64ea7..3e49e6747 100644 --- a/test/unit/test_streaming_client_configs.py +++ b/test/unit/test_streaming_client_configs.py @@ -211,6 +211,7 @@ def test_GIVEN_has_mpl_WHEN_attrs_post_init_THEN_calls_no_mpl_method( _has_mpl_attrs_post_init.assert_called_once_with() +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") @pytest.mark.parametrize( "kwargs, stream_type", ( @@ -222,7 +223,6 @@ def test_GIVEN_has_mpl_WHEN_attrs_post_init_THEN_calls_no_mpl_method( (dict(source=b"", keyring=FakeKeyring()), io.BytesIO), ), ) -@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") def test_client_configs_with_mpl( kwargs, stream_type From 705113a3ce2fcb2bda264c3f453125bd20db6a96 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 26 Feb 2024 12:54:24 -0800 Subject: [PATCH 135/422] more unit tests --- test/unit/test_streaming_client_configs.py | 38 ++++++++++++++++++---- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/test/unit/test_streaming_client_configs.py b/test/unit/test_streaming_client_configs.py index 3e49e6747..120931cb8 100644 --- a/test/unit/test_streaming_client_configs.py +++ b/test/unit/test_streaming_client_configs.py @@ -215,17 +215,15 @@ def test_GIVEN_has_mpl_WHEN_attrs_post_init_THEN_calls_no_mpl_method( @pytest.mark.parametrize( "kwargs, stream_type", ( - (dict(source=b"", materials_manager=FakeCryptoMaterialsManager()), io.BytesIO), - (dict(source=b"", key_provider=FakeMasterKeyProvider()), io.BytesIO), - (dict(source="", materials_manager=FakeCryptoMaterialsManager()), io.BytesIO), - (dict(source=io.BytesIO(), materials_manager=FakeCryptoMaterialsManager()), io.BytesIO), - (dict(source=six.StringIO(), materials_manager=FakeCryptoMaterialsManager()), six.StringIO), - (dict(source=b"", keyring=FakeKeyring()), io.BytesIO), + (dict(source=b"", materials_manager=FakeCryptoMaterialsManager())), + (dict(source=b"", key_provider=FakeMasterKeyProvider())), + (dict(source="", materials_manager=FakeCryptoMaterialsManager())), + (dict(source=io.BytesIO(), materials_manager=FakeCryptoMaterialsManager())), + (dict(source=six.StringIO(), materials_manager=FakeCryptoMaterialsManager())), ), ) def test_client_configs_with_mpl( kwargs, - stream_type ): kwargs["commitment_policy"] = CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT @@ -250,3 +248,29 @@ def test_client_configs_with_mpl( assert test.key_provider is not None assert test.key_provider == kwargs["key_provider"] assert isinstance(test.materials_manager, DefaultCryptoMaterialsManager) + + +# This needs its own test; pytest parametrize cannot use a conditionally-loaded type +@pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") +def test_keyring_client_config_with_mpl( +): + kwargs = { + "source": b"", + "keyring": FakeKeyring() + } + + test = _ClientConfig(**kwargs) + + # In all cases, config should have a materials manager + assert test.materials_manager is not None + + # If materials manager was provided, it should be directly used + if hasattr(kwargs, "materials_manager"): + assert kwargs["materials_manager"] == test.materials_manager + + # If MPL keyring was provided, it should be wrapped in MPL materials manager + if hasattr(kwargs, "keyring"): + assert test.keyring is not None + assert test.keyring == kwargs["keyring"] + assert isinstance(test.keyring, IKeyring) + assert isinstance(test.materials_manager, CryptoMaterialsManagerFromMPL) From f76d7f9f76c1eeaaf14aafbeade594680066515b Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 26 Feb 2024 12:56:49 -0800 Subject: [PATCH 136/422] more unit tests --- test/unit/test_streaming_client_configs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/test_streaming_client_configs.py b/test/unit/test_streaming_client_configs.py index 120931cb8..1a3fc89bc 100644 --- a/test/unit/test_streaming_client_configs.py +++ b/test/unit/test_streaming_client_configs.py @@ -213,7 +213,7 @@ def test_GIVEN_has_mpl_WHEN_attrs_post_init_THEN_calls_no_mpl_method( @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") @pytest.mark.parametrize( - "kwargs, stream_type", + "kwargs", ( (dict(source=b"", materials_manager=FakeCryptoMaterialsManager())), (dict(source=b"", key_provider=FakeMasterKeyProvider())), From 0da2a4f2c7dee0c0bd333bb42872b83f322af6e9 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 26 Feb 2024 13:10:59 -0800 Subject: [PATCH 137/422] more unit tests --- .../unit/test_crypto_authentication_signer.py | 9 ++++ test/unit/test_streaming_client_configs.py | 10 +++- .../test_streaming_client_stream_decryptor.py | 46 +++++++------------ .../test_streaming_client_stream_encryptor.py | 20 +++++--- 4 files changed, 47 insertions(+), 38 deletions(-) diff --git a/test/unit/test_crypto_authentication_signer.py b/test/unit/test_crypto_authentication_signer.py index c37c97bde..58cad2a7e 100644 --- a/test/unit/test_crypto_authentication_signer.py +++ b/test/unit/test_crypto_authentication_signer.py @@ -94,11 +94,14 @@ def test_GIVEN_no_encoding_WHEN_signer_from_key_bytes_THEN_load_der_private_key( with patch.object(cryptography.hazmat.primitives, "serialization"): # Mock the `serialization.load_der_private_key` with patch.object(aws_encryption_sdk.internal.crypto.authentication.serialization, "load_der_private_key") as mock_der: + # When: from_key_bytes Signer.from_key_bytes( algorithm=_algorithm, key_bytes=sentinel.key_bytes, + # Given: No encoding provided => default arg ) + # Then: calls load_der_private_key mock_der.assert_called_once_with( data=sentinel.key_bytes, password=None, backend=patch_default_backend.return_value ) @@ -113,12 +116,15 @@ def test_GIVEN_PEM_encoding_WHEN_signer_from_key_bytes_THEN_load_pem_private_key mock_algorithm_info = MagicMock(return_value=sentinel.algorithm_info, spec=patch_ec.EllipticCurve) _algorithm = MagicMock(signing_algorithm_info=mock_algorithm_info) + # When: from_key_bytes signer = Signer.from_key_bytes( algorithm=_algorithm, key_bytes=sentinel.key_bytes, + # Given: PEM encoding encoding=patch_serialization.Encoding.PEM ) + # Then: calls load_pem_private_key patch_serialization.load_pem_private_key.assert_called_once_with( data=sentinel.key_bytes, password=None, backend=patch_default_backend.return_value ) @@ -136,10 +142,13 @@ def test_GIVEN_unrecognized_encoding_WHEN_signer_from_key_bytes_THEN_raise_Value mock_algorithm_info = MagicMock(return_value=sentinel.algorithm_info, spec=patch_ec.EllipticCurve) _algorithm = MagicMock(signing_algorithm_info=mock_algorithm_info) + # Then: Raises ValueError with pytest.raises(ValueError): + # When: from_key_bytes signer = Signer.from_key_bytes( algorithm=_algorithm, key_bytes=sentinel.key_bytes, + # Given: Invalid encoding encoding="not an encoding" ) diff --git a/test/unit/test_streaming_client_configs.py b/test/unit/test_streaming_client_configs.py index 1a3fc89bc..38f6de930 100644 --- a/test/unit/test_streaming_client_configs.py +++ b/test/unit/test_streaming_client_configs.py @@ -193,22 +193,28 @@ def test_client_config_converts(kwargs, stream_type): assert isinstance(test.materials_manager, DefaultCryptoMaterialsManager) +# Given: no MPL @pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") @patch.object(_ClientConfig, "_no_mpl_attrs_post_init") def test_GIVEN_no_mpl_WHEN_attrs_post_init_THEN_calls_no_mpl_method( mock_no_mpl_attrs_post_init, ): + # When: attrs_post_init _ClientConfig(**BASE_KWARGS) + # Then: calls _no_mpl_attrs_post_init mock_no_mpl_attrs_post_init.assert_called_once_with() +# Given: has MPL @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") @patch.object(_ClientConfig, "_has_mpl_attrs_post_init") def test_GIVEN_has_mpl_WHEN_attrs_post_init_THEN_calls_no_mpl_method( - _has_mpl_attrs_post_init, + mock_has_mpl_attrs_post_init, ): + # When: attrs_post_init _ClientConfig(**BASE_KWARGS) - _has_mpl_attrs_post_init.assert_called_once_with() + # Then: calls _has_mpl_attrs_post_init + mock_has_mpl_attrs_post_init.assert_called_once_with() @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") diff --git a/test/unit/test_streaming_client_stream_decryptor.py b/test/unit/test_streaming_client_stream_decryptor.py index c8a17e650..fc45cc393 100644 --- a/test/unit/test_streaming_client_stream_decryptor.py +++ b/test/unit/test_streaming_client_stream_decryptor.py @@ -37,14 +37,11 @@ # Ideally, this logic would be based on mocking imports and testing logic, # but doing that introduces errors that cause other tests to fail. try: - from aws_cryptographic_materialproviders.mpl.references import ( - IKeyring, - ) - HAS_MPL = True - from aws_encryption_sdk.materials_managers.mpl.cmm import ( CryptoMaterialsManagerFromMPL, ) + HAS_MPL = True + except ImportError: HAS_MPL = False @@ -238,12 +235,14 @@ def test_read_header(self, mock_derive_datakey, mock_decrypt_materials_request, @patch("aws_encryption_sdk.streaming_client.DecryptionMaterialsRequest") @patch("aws_encryption_sdk.streaming_client.derive_data_encryption_key") @patch("aws_encryption_sdk.streaming_client.Verifier") + # Given: no MPL @pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") def test_GIVEN_verification_key_AND_no_mpl_WHEN_read_header_THEN_calls_from_key_bytes( self, mock_verifier, *_, ): + # Given: verification key mock_verifier_instance = MagicMock() mock_verifier.from_key_bytes.return_value = mock_verifier_instance ct_stream = io.BytesIO(VALUES["data_128"]) @@ -256,8 +255,10 @@ def test_GIVEN_verification_key_AND_no_mpl_WHEN_read_header_THEN_calls_from_key_ test_decryptor.source_stream = ct_stream test_decryptor._stream_length = len(VALUES["data_128"]) + # When: read header test_decryptor._read_header() + # Then: calls from_key_bytes mock_verifier.from_key_bytes.assert_called_once_with( algorithm=self.mock_header.algorithm, key_bytes=sentinel.verification_key ) @@ -265,17 +266,20 @@ def test_GIVEN_verification_key_AND_no_mpl_WHEN_read_header_THEN_calls_from_key_ @patch("aws_encryption_sdk.streaming_client.DecryptionMaterialsRequest") @patch("aws_encryption_sdk.streaming_client.derive_data_encryption_key") @patch("aws_encryption_sdk.streaming_client.Verifier") + # Given: has MPL @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") def test_GIVEN_verification_key_AND_has_mpl_AND_not_MPLCMM_WHEN_read_header_THEN_calls_from_key_bytes( self, mock_verifier, *_, ): + # Given: verification key mock_verifier_instance = MagicMock() mock_verifier.from_key_bytes.return_value = mock_verifier_instance ct_stream = io.BytesIO(VALUES["data_128"]) mock_commitment_policy = MagicMock(__class__=CommitmentPolicy) test_decryptor = StreamDecryptor( + # Given: native CMM materials_manager=self.mock_materials_manager, source=ct_stream, commitment_policy=mock_commitment_policy, @@ -283,8 +287,10 @@ def test_GIVEN_verification_key_AND_has_mpl_AND_not_MPLCMM_WHEN_read_header_THEN test_decryptor.source_stream = ct_stream test_decryptor._stream_length = len(VALUES["data_128"]) + # When: read_header test_decryptor._read_header() + # Then: calls from_key_bytess mock_verifier.from_key_bytes.assert_called_once_with( algorithm=self.mock_header.algorithm, key_bytes=sentinel.verification_key ) @@ -293,6 +299,7 @@ def test_GIVEN_verification_key_AND_has_mpl_AND_not_MPLCMM_WHEN_read_header_THEN @patch("aws_encryption_sdk.streaming_client.derive_data_encryption_key") @patch("aws_encryption_sdk.streaming_client.Verifier") @patch("base64.b64encode") + # Given: has MPL @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") def test_GIVEN_verification_key_AND_has_mpl_AND_has_MPLCMM_WHEN_read_header_THEN_calls_from_encoded_point( self, @@ -300,11 +307,13 @@ def test_GIVEN_verification_key_AND_has_mpl_AND_has_MPLCMM_WHEN_read_header_THEN mock_verifier, *_, ): + # Given: Verification key mock_verifier_instance = MagicMock() mock_verifier.from_key_bytes.return_value = mock_verifier_instance ct_stream = io.BytesIO(VALUES["data_128"]) mock_commitment_policy = MagicMock(__class__=CommitmentPolicy) test_decryptor = StreamDecryptor( + # Given: MPL CMM materials_manager=self.mock_mpl_materials_manager, source=ct_stream, commitment_policy=mock_commitment_policy, @@ -312,37 +321,14 @@ def test_GIVEN_verification_key_AND_has_mpl_AND_has_MPLCMM_WHEN_read_header_THEN test_decryptor.source_stream = ct_stream test_decryptor._stream_length = len(VALUES["data_128"]) + # When: read header test_decryptor._read_header() + # Then: calls from_encoded_point mock_verifier.from_encoded_point.assert_called_once_with( algorithm=self.mock_header.algorithm, encoded_point=mock_b64encoding() ) - # @patch("aws_encryption_sdk.streaming_client.Verifier") - # @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") - # def test_GIVEN_verification_key_AND_has_mpl_AND_not_MPLCMM_WHEN_read_header_THEN_calls_from_key_bytes( - # self, - # mock_verifier, - # ): - # mock_verifier_instance = MagicMock() - # mock_verifier.from_key_bytes.return_value = mock_verifier_instance - # ct_stream = io.BytesIO(VALUES["data_128"]) - # mock_commitment_policy = MagicMock(__class__=CommitmentPolicy) - # test_decryptor = StreamDecryptor( - # materials_manager=self.mock_materials_manager, - # source=ct_stream, - # commitment_policy=mock_commitment_policy, - # ) - # test_decryptor.source_stream = ct_stream - # test_decryptor._stream_length = len(VALUES["data_128"]) - - # test_decryptor._read_header() - - # mock_verifier.from_key_bytes.assert_called_once_with( - # algorithm=self.mock_header.algorithm, key_bytes=sentinel.verification_key - # ) - - @patch("aws_encryption_sdk.streaming_client.derive_data_encryption_key") def test_read_header_frame_too_large(self, mock_derive_datakey): self.mock_header.content_type = ContentType.FRAMED_DATA diff --git a/test/unit/test_streaming_client_stream_encryptor.py b/test/unit/test_streaming_client_stream_encryptor.py index 11664411a..bb4ba1c5e 100644 --- a/test/unit/test_streaming_client_stream_encryptor.py +++ b/test/unit/test_streaming_client_stream_encryptor.py @@ -42,14 +42,11 @@ # Ideally, this logic would be based on mocking imports and testing logic, # but doing that introduces errors that cause other tests to fail. try: - from aws_cryptographic_materialproviders.mpl.references import ( - IKeyring, - ) - HAS_MPL = True - from aws_encryption_sdk.materials_managers.mpl.cmm import ( CryptoMaterialsManagerFromMPL, ) + HAS_MPL = True + except ImportError: HAS_MPL = False @@ -387,6 +384,7 @@ def test_prep_message_non_framed_message(self, mock_write_header, mock_prep_non_ test_encryptor._prep_message() mock_prep_non_framed.assert_called_once_with() + # Given: no MPL @pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") def test_GIVEN_no_mpl_AND_uses_signer_WHEN_prep_message_THEN_signer_uses_default_encoding(self): self.mock_encryption_materials.algorithm = Algorithm.AES_128_GCM_IV12_TAG16 @@ -400,17 +398,21 @@ def test_GIVEN_no_mpl_AND_uses_signer_WHEN_prep_message_THEN_signer_uses_default ) test_encryptor.content_type = ContentType.FRAMED_DATA with patch.object(self.mock_signer, "from_key_bytes"): + # When: prep message test_encryptor._prep_message() + # Then: calls from_key_bytes with default encoding self.mock_signer.from_key_bytes.assert_called_once_with( algorithm=self.mock_encryption_materials.algorithm, key_bytes=self.mock_encryption_materials.signing_key ) + # Given: has MPL @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") def test_GIVEN_has_mpl_AND_not_MPLCMM_AND_uses_signer_WHEN_prep_message_THEN_signer_uses_default_encoding(self): self.mock_encryption_materials.algorithm = Algorithm.AES_128_GCM_IV12_TAG16 test_encryptor = StreamEncryptor( source=VALUES["data_128"], + # Given: native CMM materials_manager=self.mock_materials_manager, frame_length=self.mock_frame_length, algorithm=Algorithm.AES_128_GCM_IV12_TAG16, @@ -419,17 +421,21 @@ def test_GIVEN_has_mpl_AND_not_MPLCMM_AND_uses_signer_WHEN_prep_message_THEN_sig ) test_encryptor.content_type = ContentType.FRAMED_DATA with patch.object(self.mock_signer, "from_key_bytes"): + # When: prep_message test_encryptor._prep_message() + # Then: calls from_key_bytes with default encoding self.mock_signer.from_key_bytes.assert_called_once_with( algorithm=self.mock_encryption_materials.algorithm, key_bytes=self.mock_encryption_materials.signing_key ) + # Given: has MPL @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") - def test_GIVEN_has_mpl_AND_has_MPLCMM_AND_uses_signer_WHEN_prep_message_THEN_signer_uses_default_encoding(self): + def test_GIVEN_has_mpl_AND_has_MPLCMM_AND_uses_signer_WHEN_prep_message_THEN_signer_uses_PEM_encoding(self): self.mock_encryption_materials.algorithm = Algorithm.AES_128_GCM_IV12_TAG16 test_encryptor = StreamEncryptor( source=VALUES["data_128"], + # Given: MPL CMM materials_manager=self.mock_mpl_materials_manager, frame_length=self.mock_frame_length, algorithm=Algorithm.AES_128_GCM_IV12_TAG16, @@ -438,10 +444,12 @@ def test_GIVEN_has_mpl_AND_has_MPLCMM_AND_uses_signer_WHEN_prep_message_THEN_sig ) test_encryptor.content_type = ContentType.FRAMED_DATA with patch.object(self.mock_signer, "from_key_bytes"): + # When: prep_message test_encryptor._prep_message() self.mock_signer.from_key_bytes.assert_called_once_with( algorithm=self.mock_encryption_materials.algorithm, key_bytes=self.mock_encryption_materials.signing_key, + # Then: calls from_key_bytes with PEM encoding encoding=serialization.Encoding.PEM ) From 0040b2c67af302cf5e624838f2f54222d7aa85f3 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 26 Feb 2024 13:16:37 -0800 Subject: [PATCH 138/422] cleanup --- src/aws_encryption_sdk/streaming_client.py | 2 +- .../unit/test_crypto_authentication_signer.py | 15 ++++++----- test/unit/test_streaming_client_configs.py | 25 ++++++++----------- .../test_streaming_client_stream_decryptor.py | 8 +++--- .../test_streaming_client_stream_encryptor.py | 6 ++--- test/unit/test_utils.py | 16 ++++++------ 6 files changed, 34 insertions(+), 38 deletions(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 2cfcc9a02..cc9a6bb0f 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -171,7 +171,7 @@ def _has_mpl_attrs_post_init(self): except AwsCryptographicMaterialProvidersException as mpl_exception: # Wrap MPL error into the ESDK error type # so customers only have to catch ESDK error types. - raise AWSEncryptionSDKClientError(mpl_exception) + raise AWSEncryptionSDKClientError(mpl_exception) def _no_mpl_attrs_post_init(self): """If the MPL is NOT present in the runtime, perform post-init logic diff --git a/test/unit/test_crypto_authentication_signer.py b/test/unit/test_crypto_authentication_signer.py index 58cad2a7e..425f672ed 100644 --- a/test/unit/test_crypto_authentication_signer.py +++ b/test/unit/test_crypto_authentication_signer.py @@ -11,9 +11,9 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. """Unit test suite for ``aws_encryption_sdk.internal.crypto.authentication.Signer``.""" -import pytest -from mock import MagicMock, sentinel, patch import cryptography.hazmat.primitives.serialization +import pytest +from mock import MagicMock, patch, sentinel from pytest_mock import mocker # noqa pylint: disable=unused-import import aws_encryption_sdk.internal.crypto.authentication @@ -76,7 +76,7 @@ def test_f_signer_from_key_bytes(): def test_f_signer_key_bytes(): test = Signer(algorithm=ALGORITHM, key=VALUES["ecc_private_key_prime"]) assert test.key_bytes() == VALUES["ecc_private_key_prime_private_bytes"] - + def test_GIVEN_no_encoding_WHEN_signer_from_key_bytes_THEN_load_der_private_key( patch_default_backend, @@ -93,7 +93,10 @@ def test_GIVEN_no_encoding_WHEN_signer_from_key_bytes_THEN_load_der_private_key( # Mock the `serialization.Encoding.DER` with patch.object(cryptography.hazmat.primitives, "serialization"): # Mock the `serialization.load_der_private_key` - with patch.object(aws_encryption_sdk.internal.crypto.authentication.serialization, "load_der_private_key") as mock_der: + with patch.object( + aws_encryption_sdk.internal.crypto.authentication.serialization, + "load_der_private_key" + ) as mock_der: # When: from_key_bytes Signer.from_key_bytes( algorithm=_algorithm, @@ -106,7 +109,7 @@ def test_GIVEN_no_encoding_WHEN_signer_from_key_bytes_THEN_load_der_private_key( data=sentinel.key_bytes, password=None, backend=patch_default_backend.return_value ) - + def test_GIVEN_PEM_encoding_WHEN_signer_from_key_bytes_THEN_load_pem_private_key( patch_default_backend, patch_serialization, @@ -145,7 +148,7 @@ def test_GIVEN_unrecognized_encoding_WHEN_signer_from_key_bytes_THEN_raise_Value # Then: Raises ValueError with pytest.raises(ValueError): # When: from_key_bytes - signer = Signer.from_key_bytes( + Signer.from_key_bytes( algorithm=_algorithm, key_bytes=sentinel.key_bytes, # Given: Invalid encoding diff --git a/test/unit/test_streaming_client_configs.py b/test/unit/test_streaming_client_configs.py index 38f6de930..26ef86be8 100644 --- a/test/unit/test_streaming_client_configs.py +++ b/test/unit/test_streaming_client_configs.py @@ -33,14 +33,10 @@ # Ideally, this logic would be based on mocking imports and testing logic, # but doing that introduces errors that cause other tests to fail. try: - from aws_cryptographic_materialproviders.mpl.references import ( - IKeyring, - ) + from aws_cryptographic_materialproviders.mpl.references import IKeyring HAS_MPL = True - from aws_encryption_sdk.materials_managers.mpl.cmm import ( - CryptoMaterialsManagerFromMPL, - ) + from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL except ImportError: HAS_MPL = False @@ -59,14 +55,15 @@ class FakeMasterKeyProvider(MasterKeyProvider): def _new_master_key(self, key_id): return - + + if HAS_MPL: class FakeKeyring(IKeyring): def on_encrypt(self, param): - return - + return + def on_decrypt(self, param): - return + return BASE_KWARGS = dict( @@ -234,10 +231,10 @@ def test_client_configs_with_mpl( kwargs["commitment_policy"] = CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT test = _ClientConfig(**kwargs) - + # In all cases, config should have a materials manager assert test.materials_manager is not None - + # If materials manager was provided, it should be directly used if hasattr(kwargs, "materials_manager"): assert kwargs["materials_manager"] == test.materials_manager @@ -266,10 +263,10 @@ def test_keyring_client_config_with_mpl( } test = _ClientConfig(**kwargs) - + # In all cases, config should have a materials manager assert test.materials_manager is not None - + # If materials manager was provided, it should be directly used if hasattr(kwargs, "materials_manager"): assert kwargs["materials_manager"] == test.materials_manager diff --git a/test/unit/test_streaming_client_stream_decryptor.py b/test/unit/test_streaming_client_stream_decryptor.py index fc45cc393..e06cad308 100644 --- a/test/unit/test_streaming_client_stream_decryptor.py +++ b/test/unit/test_streaming_client_stream_decryptor.py @@ -37,9 +37,7 @@ # Ideally, this logic would be based on mocking imports and testing logic, # but doing that introduces errors that cause other tests to fail. try: - from aws_encryption_sdk.materials_managers.mpl.cmm import ( - CryptoMaterialsManagerFromMPL, - ) + from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL HAS_MPL = True except ImportError: @@ -55,7 +53,7 @@ def apply_fixtures(self): data_key=VALUES["data_key_obj"], verification_key=sentinel.verification_key ) self.mock_materials_manager.decrypt_materials.return_value = self.mock_decrypt_materials - + if HAS_MPL: self.mock_mpl_materials_manager = MagicMock(__class__=CryptoMaterialsManagerFromMPL) self.mock_mpl_materials_manager.decrypt_materials.return_value = self.mock_decrypt_materials @@ -258,7 +256,7 @@ def test_GIVEN_verification_key_AND_no_mpl_WHEN_read_header_THEN_calls_from_key_ # When: read header test_decryptor._read_header() - # Then: calls from_key_bytes + # Then: calls from_key_bytes mock_verifier.from_key_bytes.assert_called_once_with( algorithm=self.mock_header.algorithm, key_bytes=sentinel.verification_key ) diff --git a/test/unit/test_streaming_client_stream_encryptor.py b/test/unit/test_streaming_client_stream_encryptor.py index bb4ba1c5e..e43752689 100644 --- a/test/unit/test_streaming_client_stream_encryptor.py +++ b/test/unit/test_streaming_client_stream_encryptor.py @@ -13,9 +13,9 @@ """Unit test suite for aws_encryption_sdk.streaming_client.StreamEncryptor""" import io -from cryptography.hazmat.primitives import serialization import pytest import six +from cryptography.hazmat.primitives import serialization from mock import MagicMock, call, patch, sentinel import aws_encryption_sdk.internal.defaults @@ -42,9 +42,7 @@ # Ideally, this logic would be based on mocking imports and testing logic, # but doing that introduces errors that cause other tests to fail. try: - from aws_encryption_sdk.materials_managers.mpl.cmm import ( - CryptoMaterialsManagerFromMPL, - ) + from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL HAS_MPL = True except ImportError: diff --git a/test/unit/test_utils.py b/test/unit/test_utils.py index d717b51c7..69f9f060d 100644 --- a/test/unit/test_utils.py +++ b/test/unit/test_utils.py @@ -268,25 +268,25 @@ def test_source_data_key_length_check_invalid(self): def test_exactly_one_arg_is_not_none(self): # No args => no args are not None - assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none() is False + assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none() is False assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none( None - ) is False + ) is False assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none( "not None" - ) is True + ) is True assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none( "not None", "also not None" - ) is False + ) is False assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none( "not None", None - ) is True + ) is True assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none( "not None", "also not None" - ) is False + ) is False assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none( None, "not None" - ) is True + ) is True assert aws_encryption_sdk.internal.utils.exactly_one_arg_is_not_none( None, None - ) is False \ No newline at end of file + ) is False From 9131433f84f2f796a7e29f221e774020008aeeca Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 26 Feb 2024 13:20:17 -0800 Subject: [PATCH 139/422] cleanup --- src/aws_encryption_sdk/streaming_client.py | 2 +- test/unit/test_streaming_client_configs.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index cc9a6bb0f..5bf953244 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -171,7 +171,7 @@ def _has_mpl_attrs_post_init(self): except AwsCryptographicMaterialProvidersException as mpl_exception: # Wrap MPL error into the ESDK error type # so customers only have to catch ESDK error types. - raise AWSEncryptionSDKClientError(mpl_exception) + raise AWSEncryptionSDKClientError(mpl_exception) def _no_mpl_attrs_post_init(self): """If the MPL is NOT present in the runtime, perform post-init logic diff --git a/test/unit/test_streaming_client_configs.py b/test/unit/test_streaming_client_configs.py index 26ef86be8..18886f65b 100644 --- a/test/unit/test_streaming_client_configs.py +++ b/test/unit/test_streaming_client_configs.py @@ -259,7 +259,8 @@ def test_keyring_client_config_with_mpl( ): kwargs = { "source": b"", - "keyring": FakeKeyring() + "keyring": FakeKeyring(), + "commitment_policy": CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT } test = _ClientConfig(**kwargs) From e6826eb3fdc5773dc00f5bfb1113a7d8f0c67fd3 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 28 Feb 2024 12:20:56 -0800 Subject: [PATCH 140/422] poc impl --- src/aws_encryption_sdk/__init__.py | 1 + .../internal/formatting/serialize.py | 35 +++++-- .../materials_managers/mpl/cmm.py | 1 + .../materials_managers/mpl/materials.py | 11 +++ src/aws_encryption_sdk/streaming_client.py | 93 +++++++++++++++---- 5 files changed, 115 insertions(+), 26 deletions(-) diff --git a/src/aws_encryption_sdk/__init__.py b/src/aws_encryption_sdk/__init__.py index 661d41ee6..4b35e6744 100644 --- a/src/aws_encryption_sdk/__init__.py +++ b/src/aws_encryption_sdk/__init__.py @@ -185,6 +185,7 @@ def decrypt(self, **kwargs): If source_length is not provided and read() is called, will attempt to seek() to the end of the stream and tell() to find the length of source data. + :param dict encryption_context: Dictionary defining encryption context :param int max_body_length: Maximum frame size (or content length for non-framed messages) in bytes to read from ciphertext message. :returns: Tuple containing the decrypted plaintext and the message header object diff --git a/src/aws_encryption_sdk/internal/formatting/serialize.py b/src/aws_encryption_sdk/internal/formatting/serialize.py index b4d866099..718d4ad7d 100644 --- a/src/aws_encryption_sdk/internal/formatting/serialize.py +++ b/src/aws_encryption_sdk/internal/formatting/serialize.py @@ -218,7 +218,13 @@ def _serialize_header_auth_v1(algorithm, header, data_encryption_key, signer=Non return output -def _serialize_header_auth_v2(algorithm, header, data_encryption_key, signer=None): +def _serialize_header_auth_v2( + algorithm, + header, + data_encryption_key, + signer=None, + required_encryption_context_bytes=None + ): """Creates serialized header authentication data for messages in serialization version V2. :param algorithm: Algorithm to use for encryption @@ -230,13 +236,22 @@ def _serialize_header_auth_v2(algorithm, header, data_encryption_key, signer=Non :returns: Serialized header authentication data :rtype: bytes """ - header_auth = encrypt( - algorithm=algorithm, - key=data_encryption_key, - plaintext=b"", - associated_data=header, - iv=header_auth_iv(algorithm), - ) + if required_encryption_context_bytes is None: + header_auth = encrypt( + algorithm=algorithm, + key=data_encryption_key, + plaintext=b"", + associated_data=header, + iv=header_auth_iv(algorithm), + ) + else: + header_auth = encrypt( + algorithm=algorithm, + key=data_encryption_key, + plaintext=b"", + associated_data=header + required_encryption_context_bytes, + iv=header_auth_iv(algorithm), + ) output = struct.pack( ">{tag_len}s".format(tag_len=algorithm.tag_len), header_auth.tag, @@ -246,7 +261,7 @@ def _serialize_header_auth_v2(algorithm, header, data_encryption_key, signer=Non return output -def serialize_header_auth(version, algorithm, header, data_encryption_key, signer=None): +def serialize_header_auth(version, algorithm, header, data_encryption_key, signer=None, required_encryption_context_bytes=None): """Creates serialized header authentication data. :param version: The serialization version of the message @@ -263,7 +278,7 @@ def serialize_header_auth(version, algorithm, header, data_encryption_key, signe if version == SerializationVersion.V1: return _serialize_header_auth_v1(algorithm, header, data_encryption_key, signer) elif version == SerializationVersion.V2: - return _serialize_header_auth_v2(algorithm, header, data_encryption_key, signer) + return _serialize_header_auth_v2(algorithm, header, data_encryption_key, signer, required_encryption_context_bytes) else: raise SerializationError("Unrecognized message format version: {}".format(version)) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index 53a4b3505..8df42bf48 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -143,5 +143,6 @@ def _create_mpl_decrypt_materials_input_from_request( ), encrypted_data_keys=list_edks, encryption_context=request.encryption_context, + reproduced_encryption_context=request.reproduced_encryption_context, ) return output diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index dfd1bd6fc..d2abf182c 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -95,6 +95,12 @@ def data_encryption_key(self) -> DataKey: def signing_key(self) -> bytes: """Materials' signing key.""" return self.mpl_materials.signing_key + + + @property + def required_encryption_context_keys(self) -> bytes: + """Materials' required encryption context keys.""" + return self.mpl_materials.required_encryption_context_keys class DecryptionMaterialsFromMPL(Native_DecryptionMaterials): @@ -136,3 +142,8 @@ def data_key(self) -> DataKey: def verification_key(self) -> bytes: """Materials' verification key.""" return self.mpl_materials.verification_key + + @property + def required_encryption_context_keys(self) -> bytes: + """Materials' required encryption context keys.""" + return self.mpl_materials.required_encryption_context_keys diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 5bf953244..6d779c79e 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -593,11 +593,23 @@ def generate_header(self, message_id): if self._encryption_materials.algorithm.message_format_version == 0x02: version = SerializationVersion.V2 + if hasattr(self._encryption_materials, "required_encryption_context_keys"): + self._required_encryption_context = {} + self._stored_encryption_context = {} + for (k, v) in self._encryption_materials.encryption_context: + if k in self._encryption_materials.required_encryption_context_keys: + self._required_encryption_context[k] = v + else: + self._stored_encryption_context[k] = v + else: + self._stored_encryption_context = self._encryption_materials.encryption_context, + self._required_encryption_context = None + kwargs = dict( version=version, algorithm=self._encryption_materials.algorithm, message_id=message_id, - encryption_context=self._encryption_materials.encryption_context, + encryption_context=self._stored_encryption_context, encrypted_data_keys=self._encryption_materials.encrypted_data_keys, content_type=self.content_type, frame_length=self.config.frame_length, @@ -621,13 +633,27 @@ def generate_header(self, message_id): def _write_header(self): """Builds the message header and writes it to the output stream.""" self.output_buffer += serialize_header(header=self._header, signer=self.signer) - self.output_buffer += serialize_header_auth( - version=self._header.version, - algorithm=self._encryption_materials.algorithm, - header=self.output_buffer, - data_encryption_key=self._derived_data_key, - signer=self.signer, - ) + + if self._required_encryption_context is not None: + required_ec_serialized = aws_encryption_sdk.internal.formatting.encryption_context.serialize_encryption_context( + self._required_encryption_context + ) + self.output_buffer += serialize_header_auth( + version=self._header.version, + algorithm=self._encryption_materials.algorithm, + header=self.output_buffer, + data_encryption_key=self._derived_data_key, + signer=self.signer, + required_encryption_context_bytes=required_ec_serialized, + ) + else: + self.output_buffer += serialize_header_auth( + version=self._header.version, + algorithm=self._encryption_materials.algorithm, + header=self.output_buffer, + data_encryption_key=self._derived_data_key, + signer=self.signer, + ) def _prep_non_framed(self): """Prepare the opening data for a non-framed message.""" @@ -907,14 +933,32 @@ def _read_header(self): found=header.frame_length, custom=self.config.max_body_length ) ) - - decrypt_materials_request = DecryptionMaterialsRequest( - encrypted_data_keys=header.encrypted_data_keys, - algorithm=header.algorithm, - encryption_context=header.encryption_context, - commitment_policy=self.config.commitment_policy, - ) + + if hasattr(self, "encryption_context"): + decrypt_materials_request = DecryptionMaterialsRequest( + encrypted_data_keys=header.encrypted_data_keys, + algorithm=header.algorithm, + encryption_context=header.encryption_context, + commitment_policy=self.config.commitment_policy, + reproduced_encryption_context=self.encryption_context + ) + else: + decrypt_materials_request = DecryptionMaterialsRequest( + encrypted_data_keys=header.encrypted_data_keys, + algorithm=header.algorithm, + encryption_context=header.encryption_context, + commitment_policy=self.config.commitment_policy, + ) decryption_materials = self.config.materials_manager.decrypt_materials(request=decrypt_materials_request) + + if hasattr(decryption_materials, "required_encryption_context_keys"): + self._required_encryption_context = {} + for (k, v) in self._encryption_materials.encryption_context: + if k in self._encryption_materials.required_encryption_context_keys: + self._required_encryption_context[k] = v + else: + self._required_encryption_context = None + if decryption_materials.verification_key is None: self.verifier = None else: @@ -953,7 +997,24 @@ def _read_header(self): "message. Halting processing of this message." ) - validate_header(header=header, header_auth=header_auth, raw_header=raw_header, data_key=self._derived_data_key) + if required_ec_serialized is not None: + required_ec_serialized = aws_encryption_sdk.internal.formatting.encryption_context.serialize_encryption_context( + self._required_encryption_context + ) + + validate_header( + header=header, + header_auth=header_auth, + raw_header=raw_header + required_ec_serialized, + data_key=self._derived_data_key + ) + else: + validate_header( + header=header, + header_auth=header_auth, + raw_header=raw_header, + data_key=self._derived_data_key + ) return header, header_auth From a9fa1a5579dde63c6b5556991452075738329608 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 28 Feb 2024 13:34:13 -0800 Subject: [PATCH 141/422] passing --- .../materials_managers/__init__.py | 5 ++ .../materials_managers/mpl/materials.py | 5 ++ src/aws_encryption_sdk/streaming_client.py | 63 +++++++++++++++---- 3 files changed, 62 insertions(+), 11 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/__init__.py b/src/aws_encryption_sdk/materials_managers/__init__.py index 9db1dafae..f1eb30023 100644 --- a/src/aws_encryption_sdk/materials_managers/__init__.py +++ b/src/aws_encryption_sdk/materials_managers/__init__.py @@ -89,11 +89,16 @@ class DecryptionMaterialsRequest(object): :param encrypted_data_keys: Set of encrypted data keys :type encrypted_data_keys: set of `aws_encryption_sdk.structures.EncryptedDataKey` :param dict encryption_context: Encryption context to provide to master keys for underlying decrypt requests + :param dict reproduced_encryption_context: TODO """ algorithm = attr.ib(validator=attr.validators.instance_of(Algorithm)) encrypted_data_keys = attr.ib(validator=attr.validators.instance_of(set)) encryption_context = attr.ib(validator=attr.validators.instance_of(dict)) + reproduced_encryption_context = attr.ib( + default=None, + validator=attr.validators.optional(attr.validators.instance_of(dict)) + ) commitment_policy = attr.ib( default=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, validator=attr.validators.optional(attr.validators.instance_of(CommitmentPolicy)), diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index d2abf182c..5b066c7c7 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -143,6 +143,11 @@ def verification_key(self) -> bytes: """Materials' verification key.""" return self.mpl_materials.verification_key + @property + def encryption_context(self) -> Dict[str, str]: + """Materials' encryption context.""" + return self.mpl_materials.encryption_context + @property def required_encryption_context_keys(self) -> bytes: """Materials' required encryption context keys.""" diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 6d779c79e..f678c4b77 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -76,8 +76,13 @@ from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException - from aws_cryptographic_materialproviders.mpl.models import CreateDefaultCryptographicMaterialsManagerInput - from aws_cryptographic_materialproviders.mpl.references import IKeyring + from aws_cryptographic_materialproviders.mpl.models import ( + CreateDefaultCryptographicMaterialsManagerInput, + ) + from aws_cryptographic_materialproviders.mpl.references import ( + ICryptographicMaterialsManager, + IKeyring, + ) _HAS_MPL = True # Import internal ESDK modules that depend on the MPL @@ -126,9 +131,30 @@ class _ClientConfig(object): # pylint: disable=too-many-instance-attributes max_encrypted_data_keys = attr.ib( hash=True, default=None, validator=attr.validators.optional(attr.validators.instance_of(int)) ) - materials_manager = attr.ib( - hash=True, default=None, validator=attr.validators.optional(attr.validators.instance_of(CryptoMaterialsManager)) - ) + if _HAS_MPL: + # With the MPL, the provided materials_manager can be an instance of + # either the native interface or an MPL interface. + # If it implements the MPL interface, this constructor will + # internally wrap it in a native interface. + materials_manager = attr.ib( + hash=True, + default=None, + validator=attr.validators.optional( + attr.validators.instance_of( + (CryptoMaterialsManager, ICryptographicMaterialsManager) + ) + ) + ) + else: + materials_manager = attr.ib( + hash=True, + default=None, + validator=attr.validators.optional( + attr.validators.instance_of( + CryptoMaterialsManager + ) + ) + ) key_provider = attr.ib( hash=True, default=None, validator=attr.validators.optional(attr.validators.instance_of(MasterKeyProvider)) ) @@ -172,6 +198,12 @@ def _has_mpl_attrs_post_init(self): # Wrap MPL error into the ESDK error type # so customers only have to catch ESDK error types. raise AWSEncryptionSDKClientError(mpl_exception) + # TODO-MPL: MUST wrap MPL with native + elif (self.materials_manager is not None + and isinstance(self.materials_manager, ICryptographicMaterialsManager)): + # If the provided materials manager implements an MPL interface, + # wrap it in a native interface. + self.materials_manager = CryptoMaterialsManagerFromMPL(self.materials_manager) def _no_mpl_attrs_post_init(self): """If the MPL is NOT present in the runtime, perform post-init logic @@ -596,7 +628,8 @@ def generate_header(self, message_id): if hasattr(self._encryption_materials, "required_encryption_context_keys"): self._required_encryption_context = {} self._stored_encryption_context = {} - for (k, v) in self._encryption_materials.encryption_context: + print(f"{self._encryption_materials.encryption_context=}") + for (k, v) in self._encryption_materials.encryption_context.items(): if k in self._encryption_materials.required_encryption_context_keys: self._required_encryption_context[k] = v else: @@ -856,6 +889,11 @@ class DecryptorConfig(_ClientConfig): max_body_length = attr.ib( hash=True, default=None, validator=attr.validators.optional(attr.validators.instance_of(six.integer_types)) ) + encryption_context = attr.ib( + hash=False, # dictionaries are not hashable + default=attr.Factory(dict), + validator=attr.validators.instance_of(dict), + ) class StreamDecryptor(_EncryptionStream): # pylint: disable=too-many-instance-attributes @@ -934,13 +972,15 @@ def _read_header(self): ) ) - if hasattr(self, "encryption_context"): + print(f"{self.config.encryption_context=}") + + if hasattr(self.config, "encryption_context"): decrypt_materials_request = DecryptionMaterialsRequest( encrypted_data_keys=header.encrypted_data_keys, algorithm=header.algorithm, encryption_context=header.encryption_context, commitment_policy=self.config.commitment_policy, - reproduced_encryption_context=self.encryption_context + reproduced_encryption_context=self.config.encryption_context ) else: decrypt_materials_request = DecryptionMaterialsRequest( @@ -949,12 +989,13 @@ def _read_header(self): encryption_context=header.encryption_context, commitment_policy=self.config.commitment_policy, ) + print(f"{decrypt_materials_request=}") decryption_materials = self.config.materials_manager.decrypt_materials(request=decrypt_materials_request) if hasattr(decryption_materials, "required_encryption_context_keys"): self._required_encryption_context = {} - for (k, v) in self._encryption_materials.encryption_context: - if k in self._encryption_materials.required_encryption_context_keys: + for (k, v) in decryption_materials.encryption_context.items(): + if k in decryption_materials.required_encryption_context_keys: self._required_encryption_context[k] = v else: self._required_encryption_context = None @@ -997,7 +1038,7 @@ def _read_header(self): "message. Halting processing of this message." ) - if required_ec_serialized is not None: + if self._required_encryption_context is not None: required_ec_serialized = aws_encryption_sdk.internal.formatting.encryption_context.serialize_encryption_context( self._required_encryption_context ) From 4eeb85889b498004eb865762e6503a6991bbee84 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 28 Feb 2024 14:03:10 -0800 Subject: [PATCH 142/422] cleanup --- src/aws_encryption_sdk/__init__.py | 4 +- .../internal/formatting/serialize.py | 37 +++++++++++++++---- .../materials_managers/__init__.py | 4 +- src/aws_encryption_sdk/streaming_client.py | 36 ++++++++++++++---- 4 files changed, 64 insertions(+), 17 deletions(-) diff --git a/src/aws_encryption_sdk/__init__.py b/src/aws_encryption_sdk/__init__.py index 4b35e6744..96898d446 100644 --- a/src/aws_encryption_sdk/__init__.py +++ b/src/aws_encryption_sdk/__init__.py @@ -185,7 +185,9 @@ def decrypt(self, **kwargs): If source_length is not provided and read() is called, will attempt to seek() to the end of the stream and tell() to find the length of source data. - :param dict encryption_context: Dictionary defining encryption context + :param dict encryption_context: Dictionary defining encryption context to validate + on decrypt. This is ONLY validated on decrypt if using the required encryption + context CMM from the aws-cryptographic-materialproviders library. :param int max_body_length: Maximum frame size (or content length for non-framed messages) in bytes to read from ciphertext message. :returns: Tuple containing the decrypted plaintext and the message header object diff --git a/src/aws_encryption_sdk/internal/formatting/serialize.py b/src/aws_encryption_sdk/internal/formatting/serialize.py index 718d4ad7d..344c94703 100644 --- a/src/aws_encryption_sdk/internal/formatting/serialize.py +++ b/src/aws_encryption_sdk/internal/formatting/serialize.py @@ -219,12 +219,12 @@ def _serialize_header_auth_v1(algorithm, header, data_encryption_key, signer=Non def _serialize_header_auth_v2( - algorithm, - header, - data_encryption_key, - signer=None, - required_encryption_context_bytes=None - ): + algorithm, + header, + data_encryption_key, + signer=None, + required_encryption_context_bytes=None +): """Creates serialized header authentication data for messages in serialization version V2. :param algorithm: Algorithm to use for encryption @@ -233,6 +233,11 @@ def _serialize_header_auth_v2( :param bytes data_encryption_key: Data key with which to encrypt message :param signer: Cryptographic signer object (optional) :type signer: aws_encryption_sdk.Signer + :param required_encryption_context_bytes: Serialized encryption context items + for all items whose keys are in the required_encryption_context list. + This is ONLY processed if using the aws-cryptographic-materialproviders library + AND its required encryption context CMM. (optional) + :type required_encryption_context_bytes: bytes :returns: Serialized header authentication data :rtype: bytes """ @@ -249,6 +254,11 @@ def _serialize_header_auth_v2( algorithm=algorithm, key=data_encryption_key, plaintext=b"", + # The AAD MUST be the concatenation of the serialized message header body and the serialization + # of encryption context to only authenticate. The encryption context to only authenticate MUST + # be the encryption context in the encryption materials filtered to only contain key value + # pairs listed in the encryption material's required encryption context keys serialized + # according to the encryption context serialization specification. associated_data=header + required_encryption_context_bytes, iv=header_auth_iv(algorithm), ) @@ -261,7 +271,14 @@ def _serialize_header_auth_v2( return output -def serialize_header_auth(version, algorithm, header, data_encryption_key, signer=None, required_encryption_context_bytes=None): +def serialize_header_auth( + version, + algorithm, + header, + data_encryption_key, + signer=None, + required_encryption_context_bytes=None +): """Creates serialized header authentication data. :param version: The serialization version of the message @@ -272,6 +289,12 @@ def serialize_header_auth(version, algorithm, header, data_encryption_key, signe :param bytes data_encryption_key: Data key with which to encrypt message :param signer: Cryptographic signer object (optional) :type signer: aws_encryption_sdk.Signer + :param required_encryption_context_bytes: Serialized encryption context items + for all items whose keys are in the required_encryption_context list. + This is ONLY processed if using the aws-cryptographic-materialproviders library + AND its required encryption context CMM + AND if using the v2 message format. (optional) + :type required_encryption_context_bytes: bytes :returns: Serialized header authentication data :rtype: bytes """ diff --git a/src/aws_encryption_sdk/materials_managers/__init__.py b/src/aws_encryption_sdk/materials_managers/__init__.py index f1eb30023..cc8cdcf6f 100644 --- a/src/aws_encryption_sdk/materials_managers/__init__.py +++ b/src/aws_encryption_sdk/materials_managers/__init__.py @@ -89,7 +89,9 @@ class DecryptionMaterialsRequest(object): :param encrypted_data_keys: Set of encrypted data keys :type encrypted_data_keys: set of `aws_encryption_sdk.structures.EncryptedDataKey` :param dict encryption_context: Encryption context to provide to master keys for underlying decrypt requests - :param dict reproduced_encryption_context: TODO + :param dict reproduced_encryption_context: Encryption context to provide on decrypt. + This is ONLY processed if using the required encryption context CMM from the + aws-cryptographic-materialproviders library. """ algorithm = attr.ib(validator=attr.validators.instance_of(Algorithm)) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index f678c4b77..568543863 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -198,11 +198,11 @@ def _has_mpl_attrs_post_init(self): # Wrap MPL error into the ESDK error type # so customers only have to catch ESDK error types. raise AWSEncryptionSDKClientError(mpl_exception) - # TODO-MPL: MUST wrap MPL with native + + # If the provided materials_manager is directly from the MPL, wrap it in a native interface + # for internal use. elif (self.materials_manager is not None and isinstance(self.materials_manager, ICryptographicMaterialsManager)): - # If the provided materials manager implements an MPL interface, - # wrap it in a native interface. self.materials_manager = CryptoMaterialsManagerFromMPL(self.materials_manager) def _no_mpl_attrs_post_init(self): @@ -625,15 +625,18 @@ def generate_header(self, message_id): if self._encryption_materials.algorithm.message_format_version == 0x02: version = SerializationVersion.V2 + # If the underlying materials_provider provided required_encryption_context_keys + # (ex. if the materials_provider is a required encryption context CMM), + # then partition the encryption context based on those keys. if hasattr(self._encryption_materials, "required_encryption_context_keys"): self._required_encryption_context = {} self._stored_encryption_context = {} - print(f"{self._encryption_materials.encryption_context=}") for (k, v) in self._encryption_materials.encryption_context.items(): if k in self._encryption_materials.required_encryption_context_keys: self._required_encryption_context[k] = v else: self._stored_encryption_context[k] = v + # Otherwise, store all encryption context with the message. else: self._stored_encryption_context = self._encryption_materials.encryption_context, self._required_encryption_context = None @@ -667,6 +670,8 @@ def _write_header(self): """Builds the message header and writes it to the output stream.""" self.output_buffer += serialize_header(header=self._header, signer=self.signer) + # If there is _required_encryption_context, + # serialize it, then authenticate it if self._required_encryption_context is not None: required_ec_serialized = aws_encryption_sdk.internal.formatting.encryption_context.serialize_encryption_context( self._required_encryption_context @@ -679,6 +684,7 @@ def _write_header(self): signer=self.signer, required_encryption_context_bytes=required_ec_serialized, ) + # Otherwise, do not pass in any required encryption context else: self.output_buffer += serialize_header_auth( version=self._header.version, @@ -884,6 +890,9 @@ class DecryptorConfig(_ClientConfig): :param int max_body_length: Maximum frame size (or content length for non-framed messages) in bytes to read from ciphertext message. + :param dict encryption_context: Dictionary defining encryption context to validate + on decrypt. This is ONLY validated on decrypt if using the required encryption + context CMM from the aws-cryptographic-materialproviders library. """ max_body_length = attr.ib( @@ -971,9 +980,9 @@ def _read_header(self): found=header.frame_length, custom=self.config.max_body_length ) ) - - print(f"{self.config.encryption_context=}") - + + # If encryption_context is provided on decrypt, + # pass it to the DecryptionMaterialsRequest if hasattr(self.config, "encryption_context"): decrypt_materials_request = DecryptionMaterialsRequest( encrypted_data_keys=header.encrypted_data_keys, @@ -989,9 +998,12 @@ def _read_header(self): encryption_context=header.encryption_context, commitment_policy=self.config.commitment_policy, ) - print(f"{decrypt_materials_request=}") + decryption_materials = self.config.materials_manager.decrypt_materials(request=decrypt_materials_request) + # If the materials_manager passed required_encryption_context_keys, + # get the items out of the encryption_context with the keys. + # The items are used in header validation. if hasattr(decryption_materials, "required_encryption_context_keys"): self._required_encryption_context = {} for (k, v) in decryption_materials.encryption_context.items(): @@ -1038,7 +1050,12 @@ def _read_header(self): "message. Halting processing of this message." ) + # If _required_encryption_context is present, + # serialize it and pass it to validate_header. if self._required_encryption_context is not None: + # The authenticated only encryption context is all encryption context key-value pairs where the + # key exists in Required Encryption Context Keys. It is then serialized according to the + # message header Key Value Pairs. required_ec_serialized = aws_encryption_sdk.internal.formatting.encryption_context.serialize_encryption_context( self._required_encryption_context ) @@ -1046,6 +1063,9 @@ def _read_header(self): validate_header( header=header, header_auth=header_auth, + # When verifying the header, the AAD input to the authenticated encryption algorithm + # specified by the algorithm suite is the message header body and the serialized + # authenticated only encryption context. raw_header=raw_header + required_ec_serialized, data_key=self._derived_data_key ) From 21a8c938eb44f352a76fd358e1c865f453e3f75a Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 28 Feb 2024 16:08:02 -0800 Subject: [PATCH 143/422] protect --- examples/src/basic_encryption.py | 2 +- examples/test/test_i_basic_encryption.py | 2 +- src/aws_encryption_sdk/streaming_client.py | 15 ++++++++++++++- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/examples/src/basic_encryption.py b/examples/src/basic_encryption.py index cfe8ac791..68be5c594 100644 --- a/examples/src/basic_encryption.py +++ b/examples/src/basic_encryption.py @@ -38,7 +38,7 @@ def cycle_string(key_arn, source_plaintext, botocore_session=None): ciphertext, encryptor_header = client.encrypt(source=source_plaintext, key_provider=master_key_provider) # Decrypt the ciphertext - cycled_plaintext, decrypted_header = client.decrypt(source=ciphertext, key_provider=master_key_provider) + cycled_plaintext, decrypted_header = client.decrypt(source=ciphertext, key_provider=master_key_provider, encryption_context={"a": "v"}) # Verify that the "cycled" (encrypted, then decrypted) plaintext is identical to the source plaintext assert cycled_plaintext == source_plaintext diff --git a/examples/test/test_i_basic_encryption.py b/examples/test/test_i_basic_encryption.py index f2a4fab51..5f509800e 100644 --- a/examples/test/test_i_basic_encryption.py +++ b/examples/test/test_i_basic_encryption.py @@ -23,5 +23,5 @@ def test_cycle_string(): plaintext = static_plaintext - cmk_arn = get_cmk_arn() + cmk_arn = "arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f" cycle_string(key_arn=cmk_arn, source_plaintext=plaintext, botocore_session=botocore.session.Session()) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 568543863..e3513de97 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -638,7 +638,7 @@ def generate_header(self, message_id): self._stored_encryption_context[k] = v # Otherwise, store all encryption context with the message. else: - self._stored_encryption_context = self._encryption_materials.encryption_context, + self._stored_encryption_context = self._encryption_materials.encryption_context self._required_encryption_context = None kwargs = dict( @@ -1001,6 +1001,19 @@ def _read_header(self): decryption_materials = self.config.materials_manager.decrypt_materials(request=decrypt_materials_request) + # Guard against possible misunderstanding of "encryption context on decrypt". + # The `encryption_context` parameter on the client's `decrypt` method + # is ONLY meant to be used in conjunction with a `materials_manager` + # that validates the encryption context provided to the decrypt method + if hasattr(self.config, "encryption_context"): + try: + assert hasattr(decryption_materials, "required_encryption_context_keys") + except AssertionError as e: + raise ValueError("encryption_context on decrypt is not supported with the configured CMM: " + f"{self.config.materials_manager}. " + "You MUST pass a CMM that supports required encryption context keys to " + "validate encryption context on decrypt.") + # If the materials_manager passed required_encryption_context_keys, # get the items out of the encryption_context with the keys. # The items are used in header validation. From de870b8495dcb34bb795edfa2bd72e24bf8f3c61 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 28 Feb 2024 16:08:23 -0800 Subject: [PATCH 144/422] ex --- .../required_encryption_context_cmm.py | 130 ++++++++++++++++++ .../test_i_required_encryption_context_cmm.py | 13 ++ src/aws_encryption_sdk/streaming_client.py | 6 +- 3 files changed, 147 insertions(+), 2 deletions(-) create mode 100644 examples/src/keyrings/required_encryption_context_cmm.py create mode 100644 examples/test/keyrings/test_i_required_encryption_context_cmm.py diff --git a/examples/src/keyrings/required_encryption_context_cmm.py b/examples/src/keyrings/required_encryption_context_cmm.py new file mode 100644 index 000000000..c36a4b2bd --- /dev/null +++ b/examples/src/keyrings/required_encryption_context_cmm.py @@ -0,0 +1,130 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Example showing basic encryption and decryption of a value already in memory.""" +import sys + +import boto3 +# Ignore missing MPL for pylint, but the MPL is required for this example +# noqa pylint: disable=import-error +from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders +from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig +from aws_cryptographic_materialproviders.mpl.models import ( + CacheTypeDefault, + CreateAwsKmsKeyringInput, + CreateDefaultCryptographicMaterialsManagerInput, + CreateRequiredEncryptionContextCMMInput, + DefaultCache, +) +from aws_cryptographic_materialproviders.mpl.references import ( + IKeyring, + ICryptographicMaterialsManager, +) +from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL +from typing import Dict + +import aws_encryption_sdk +from aws_encryption_sdk import CommitmentPolicy +from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError + +from .example_branch_key_id_supplier import ExampleBranchKeyIdSupplier + +module_root_dir = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(module_root_dir) + +EXAMPLE_DATA: bytes = b"Hello World" + + +def encrypt_and_decrypt_with_keyring( + kms_key_id: str +): + """Creates a hierarchical keyring using the provided resources, then encrypts and decrypts a string with it.""" + # 1. Instantiate the encryption SDK client. + # This builds the client with the REQUIRE_ENCRYPT_REQUIRE_DECRYPT commitment policy, + # which enforces that this client only encrypts using committing algorithm suites and enforces + # that this client will only decrypt encrypted messages that were created with a committing + # algorithm suite. + # This is the default commitment policy if you were to build the client as + # `client = aws_encryption_sdk.EncryptionSDKClient()`. + + client = aws_encryption_sdk.EncryptionSDKClient( + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT + ) + + # 7. Create an encryption context. + #// Most encrypted data should have an associated encryption context + #// to protect integrity. This sample uses placeholder values. + #// For more information see: + #// blogs.aws.amazon.com/security/post/Tx2LZ6WBJJANTNW/How-to-Protect-the-Integrity-of-Your-Encrypted-Data-by-Using-AWS-Key-Management + encryption_context: Dict[str, str] = { + "key1": "value1", + "key2": "value2", + "requiredKey1": "requiredValue1", + "requiredKey2": "requiredValue2", + } + + #// 3. Create list of required encryption context keys. + #// This is a list of keys that must be present in the encryption context. + required_encryption_context_keys: List[str] = ["requiredKey1", "requiredKey2"] + + #// 4. Create the AWS KMS keyring. + mpl: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + config=MaterialProvidersConfig() + ) + keyring_input: CreateAwsKmsKeyringInput = CreateAwsKmsKeyringInput( + kms_key_id=kms_key_id, + kms_client=boto3.client('kms', region_name="us-west-2") + ) + kms_keyring: IKeyring = mpl.create_aws_kms_keyring(keyring_input) + + #// 5. Create the required encryption context CMM. + underlying_cmm: ICryptographicMaterialsManager = \ + mpl.create_default_cryptographic_materials_manager( + CreateDefaultCryptographicMaterialsManagerInput( + keyring=kms_keyring + ) + ) + + required_ec_cmm: ICryptographicMaterialsManager = \ + mpl.create_required_encryption_context_cmm( + CreateRequiredEncryptionContextCMMInput( + required_encryption_context_keys=required_encryption_context_keys, + underlying_cmm=underlying_cmm, + ) + ) + + # 6. Encrypt the data + ciphertext, _ = client.encrypt( + source=EXAMPLE_DATA, + materials_manager=required_ec_cmm, + encryption_context=encryption_context + ) + + # // 7. Reproduce the encryption context. + # // The reproduced encryption context MUST contain a value for + # // every key in the configured required encryption context keys during encryption with + # // Required Encryption Context CMM. + reproduced_encryption_context: Dict[str, str] = { + "requiredKey1": "requiredValue1", + "requiredKey2": "requiredValue2", + } + + # 8. Decrypt the data + plaintext_bytes_A, _ = client.decrypt( + source=ciphertext, + materials_manager=required_ec_cmm, + encryption_context=reproduced_encryption_context + ) + assert plaintext_bytes_A == EXAMPLE_DATA + + + # 9. If we don't provide the required encryption context, this should fail + try: + plaintext_bytes_A, _ = client.decrypt( + source=ciphertext, + materials_manager=required_ec_cmm, + # no encryption context while using required encryption context CMM makes decryption fail + ) + assert plaintext_bytes_A == EXAMPLE_DATA + except AWSEncryptionSDKClientError: + pass \ No newline at end of file diff --git a/examples/test/keyrings/test_i_required_encryption_context_cmm.py b/examples/test/keyrings/test_i_required_encryption_context_cmm.py new file mode 100644 index 000000000..9512a06ee --- /dev/null +++ b/examples/test/keyrings/test_i_required_encryption_context_cmm.py @@ -0,0 +1,13 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Unit test suite for the hierarchical keyring example.""" +import pytest + +from ...src.keyrings.required_encryption_context_cmm import encrypt_and_decrypt_with_keyring + +pytestmark = [pytest.mark.examples] + + +def test_encrypt_and_decrypt_with_keyring(): + key_arn = "arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f" + encrypt_and_decrypt_with_keyring(key_arn) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index e3513de97..58cd52051 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -1003,8 +1003,10 @@ def _read_header(self): # Guard against possible misunderstanding of "encryption context on decrypt". # The `encryption_context` parameter on the client's `decrypt` method - # is ONLY meant to be used in conjunction with a `materials_manager` - # that validates the encryption context provided to the decrypt method + # is ONLY meant to be used in conjunction with a `materials_manager` + # that validates the encryption context provided to the decrypt method. + # This guards against accidentially passing encryption context on decrypt + # and not realizing nothing is being validated. if hasattr(self.config, "encryption_context"): try: assert hasattr(decryption_materials, "required_encryption_context_keys") From eedf1a3c269473117dc284c52b0d34f63943915b Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 28 Feb 2024 16:24:03 -0800 Subject: [PATCH 145/422] changes --- examples/src/basic_encryption.py | 2 +- .../keyrings/required_encryption_context_cmm.py | 10 +++++----- src/aws_encryption_sdk/streaming_client.py | 17 +---------------- 3 files changed, 7 insertions(+), 22 deletions(-) diff --git a/examples/src/basic_encryption.py b/examples/src/basic_encryption.py index 68be5c594..cfe8ac791 100644 --- a/examples/src/basic_encryption.py +++ b/examples/src/basic_encryption.py @@ -38,7 +38,7 @@ def cycle_string(key_arn, source_plaintext, botocore_session=None): ciphertext, encryptor_header = client.encrypt(source=source_plaintext, key_provider=master_key_provider) # Decrypt the ciphertext - cycled_plaintext, decrypted_header = client.decrypt(source=ciphertext, key_provider=master_key_provider, encryption_context={"a": "v"}) + cycled_plaintext, decrypted_header = client.decrypt(source=ciphertext, key_provider=master_key_provider) # Verify that the "cycled" (encrypted, then decrypted) plaintext is identical to the source plaintext assert cycled_plaintext == source_plaintext diff --git a/examples/src/keyrings/required_encryption_context_cmm.py b/examples/src/keyrings/required_encryption_context_cmm.py index c36a4b2bd..51455fd4f 100644 --- a/examples/src/keyrings/required_encryption_context_cmm.py +++ b/examples/src/keyrings/required_encryption_context_cmm.py @@ -109,7 +109,7 @@ def encrypt_and_decrypt_with_keyring( "requiredKey2": "requiredValue2", } - # 8. Decrypt the data + # # 8. Decrypt the data plaintext_bytes_A, _ = client.decrypt( source=ciphertext, materials_manager=required_ec_cmm, @@ -117,14 +117,14 @@ def encrypt_and_decrypt_with_keyring( ) assert plaintext_bytes_A == EXAMPLE_DATA - - # 9. If we don't provide the required encryption context, this should fail + # 9. If we don't provide the required encryption context, + # decryption will fail. try: plaintext_bytes_A, _ = client.decrypt( source=ciphertext, materials_manager=required_ec_cmm, # no encryption context while using required encryption context CMM makes decryption fail ) - assert plaintext_bytes_A == EXAMPLE_DATA + raise Exception("If this exception is raised, decryption somehow succeeded!") except AWSEncryptionSDKClientError: - pass \ No newline at end of file + pass diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 58cd52051..7e6ede8cd 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -1000,22 +1000,7 @@ def _read_header(self): ) decryption_materials = self.config.materials_manager.decrypt_materials(request=decrypt_materials_request) - - # Guard against possible misunderstanding of "encryption context on decrypt". - # The `encryption_context` parameter on the client's `decrypt` method - # is ONLY meant to be used in conjunction with a `materials_manager` - # that validates the encryption context provided to the decrypt method. - # This guards against accidentially passing encryption context on decrypt - # and not realizing nothing is being validated. - if hasattr(self.config, "encryption_context"): - try: - assert hasattr(decryption_materials, "required_encryption_context_keys") - except AssertionError as e: - raise ValueError("encryption_context on decrypt is not supported with the configured CMM: " - f"{self.config.materials_manager}. " - "You MUST pass a CMM that supports required encryption context keys to " - "validate encryption context on decrypt.") - + # If the materials_manager passed required_encryption_context_keys, # get the items out of the encryption_context with the keys. # The items are used in header validation. From 1db73ebee0d6df0e2ca76b4f16998f018a042d0d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 28 Feb 2024 16:58:26 -0800 Subject: [PATCH 146/422] changes --- examples/src/keyrings/hierarchical_keyring.py | 33 ++++++++++++++- .../required_encryption_context_cmm.py | 40 +++++++++++-------- .../keyrings/test_i_hierarchical_keyring.py | 2 +- .../test_i_required_encryption_context_cmm.py | 2 +- examples/test/test_i_basic_encryption.py | 2 +- src/aws_encryption_sdk/streaming_client.py | 28 ++++++------- 6 files changed, 72 insertions(+), 35 deletions(-) diff --git a/examples/src/keyrings/hierarchical_keyring.py b/examples/src/keyrings/hierarchical_keyring.py index aa87485f9..b75421359 100644 --- a/examples/src/keyrings/hierarchical_keyring.py +++ b/examples/src/keyrings/hierarchical_keyring.py @@ -1,6 +1,36 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""Example showing basic encryption and decryption of a value already in memory.""" +""" +This example sets up the Hierarchical Keyring, which establishes a key hierarchy where "branch" +keys are persisted in DynamoDb. These branch keys are used to protect your data keys, and these +branch keys are themselves protected by a KMS Key. + +Establishing a key hierarchy like this has two benefits: +First, by caching the branch key material, and only calling KMS to re-establish authentication +regularly according to your configured TTL, you limit how often you need to call KMS to protect +your data. This is a performance security tradeoff, where your authentication, audit, and logging +from KMS is no longer one-to-one with every encrypt or decrypt call. Additionally, KMS Cloudtrail +cannot be used to distinguish Encrypt and Decrypt calls, and you cannot restrict who has +Encryption rights from who has Decryption rights since they both ONLY need KMS:Decrypt. However, +the benefit is that you no longer have to make a network call to KMS for every encrypt or +decrypt. + +Second, this key hierarchy facilitates cryptographic isolation of a tenant's data in a +multi-tenant data store. Each tenant can have a unique Branch Key, that is only used to protect +the tenant's data. You can either statically configure a single branch key to ensure you are +restricting access to a single tenant, or you can implement an interface that selects the Branch +Key based on the Encryption Context. + +This example demonstrates configuring a Hierarchical Keyring with a Branch Key ID Supplier to +encrypt and decrypt data for two separate tenants. + +This example requires access to the DDB Table where you are storing the Branch Keys. This +table must be configured with the following primary key configuration: - Partition key is named +"partition_key" with type (S) - Sort key is named "sort_key" with type (S) + +This example also requires using a KMS Key. You need the following access on this key: - +GenerateDataKeyWithoutPlaintext - Decrypt +""" import sys import boto3 @@ -25,6 +55,7 @@ from .example_branch_key_id_supplier import ExampleBranchKeyIdSupplier +# TODO-MPL: Remove this as part of removing PYTHONPATH hacks module_root_dir = '/'.join(__file__.split("/")[:-1]) sys.path.append(module_root_dir) diff --git a/examples/src/keyrings/required_encryption_context_cmm.py b/examples/src/keyrings/required_encryption_context_cmm.py index 51455fd4f..f3d58c922 100644 --- a/examples/src/keyrings/required_encryption_context_cmm.py +++ b/examples/src/keyrings/required_encryption_context_cmm.py @@ -1,6 +1,11 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""Example showing basic encryption and decryption of a value already in memory.""" +""" +Demonstrate an encrypt/decrypt cycle using a Required Encryption Context CMM. +A required encryption context CMM asks for required keys in the encryption context field +on encrypt such that they will not be stored on the message, but WILL be included in the header signature. +On decrypt, the client MUST supply the key/value pair(s) that were not stored to successfully decrypt the message. +""" import sys import boto3 @@ -28,6 +33,7 @@ from .example_branch_key_id_supplier import ExampleBranchKeyIdSupplier +# TODO-MPL: Remove this as part of removing PYTHONPATH hacks module_root_dir = '/'.join(__file__.split("/")[:-1]) sys.path.append(module_root_dir) @@ -51,11 +57,11 @@ def encrypt_and_decrypt_with_keyring( commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT ) - # 7. Create an encryption context. - #// Most encrypted data should have an associated encryption context - #// to protect integrity. This sample uses placeholder values. - #// For more information see: - #// blogs.aws.amazon.com/security/post/Tx2LZ6WBJJANTNW/How-to-Protect-the-Integrity-of-Your-Encrypted-Data-by-Using-AWS-Key-Management + # 2. Create an encryption context. + # Most encrypted data should have an associated encryption context + # to protect integrity. This sample uses placeholder values. + # For more information see: + # blogs.aws.amazon.com/security/post/Tx2LZ6WBJJANTNW/How-to-Protect-the-Integrity-of-Your-Encrypted-Data-by-Using-AWS-Key-Management # noqa: E501 encryption_context: Dict[str, str] = { "key1": "value1", "key2": "value2", @@ -63,11 +69,11 @@ def encrypt_and_decrypt_with_keyring( "requiredKey2": "requiredValue2", } - #// 3. Create list of required encryption context keys. - #// This is a list of keys that must be present in the encryption context. + # 3. Create list of required encryption context keys. + # This is a list of keys that must be present in the encryption context. required_encryption_context_keys: List[str] = ["requiredKey1", "requiredKey2"] - #// 4. Create the AWS KMS keyring. + # 4. Create the AWS KMS keyring. mpl: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( config=MaterialProvidersConfig() ) @@ -77,7 +83,7 @@ def encrypt_and_decrypt_with_keyring( ) kms_keyring: IKeyring = mpl.create_aws_kms_keyring(keyring_input) - #// 5. Create the required encryption context CMM. + # 5. Create the required encryption context CMM. underlying_cmm: ICryptographicMaterialsManager = \ mpl.create_default_cryptographic_materials_manager( CreateDefaultCryptographicMaterialsManagerInput( @@ -100,16 +106,16 @@ def encrypt_and_decrypt_with_keyring( encryption_context=encryption_context ) - # // 7. Reproduce the encryption context. - # // The reproduced encryption context MUST contain a value for - # // every key in the configured required encryption context keys during encryption with - # // Required Encryption Context CMM. + # 7. Reproduce the encryption context. + # The reproduced encryption context MUST contain a value for + # every key in the configured required encryption context keys during encryption with + # Required Encryption Context CMM. reproduced_encryption_context: Dict[str, str] = { "requiredKey1": "requiredValue1", "requiredKey2": "requiredValue2", } - # # 8. Decrypt the data + # 8. Decrypt the data plaintext_bytes_A, _ = client.decrypt( source=ciphertext, materials_manager=required_ec_cmm, @@ -117,13 +123,13 @@ def encrypt_and_decrypt_with_keyring( ) assert plaintext_bytes_A == EXAMPLE_DATA - # 9. If we don't provide the required encryption context, + # 9. Extra: Demonstrate that if we don't provide the required encryption context, # decryption will fail. try: plaintext_bytes_A, _ = client.decrypt( source=ciphertext, materials_manager=required_ec_cmm, - # no encryption context while using required encryption context CMM makes decryption fail + # No encryption context while using required encryption context CMM makes decryption fail. ) raise Exception("If this exception is raised, decryption somehow succeeded!") except AWSEncryptionSDKClientError: diff --git a/examples/test/keyrings/test_i_hierarchical_keyring.py b/examples/test/keyrings/test_i_hierarchical_keyring.py index 4cae478d7..c4583534a 100644 --- a/examples/test/keyrings/test_i_hierarchical_keyring.py +++ b/examples/test/keyrings/test_i_hierarchical_keyring.py @@ -1,6 +1,6 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""Unit test suite for the hierarchical keyring example.""" +"""Test suite for the hierarchical keyring example.""" import pytest from ...src.keyrings.hierarchical_keyring import encrypt_and_decrypt_with_keyring diff --git a/examples/test/keyrings/test_i_required_encryption_context_cmm.py b/examples/test/keyrings/test_i_required_encryption_context_cmm.py index 9512a06ee..724705faa 100644 --- a/examples/test/keyrings/test_i_required_encryption_context_cmm.py +++ b/examples/test/keyrings/test_i_required_encryption_context_cmm.py @@ -1,6 +1,6 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""Unit test suite for the hierarchical keyring example.""" +"""Test suite for the required encryption context CMM example.""" import pytest from ...src.keyrings.required_encryption_context_cmm import encrypt_and_decrypt_with_keyring diff --git a/examples/test/test_i_basic_encryption.py b/examples/test/test_i_basic_encryption.py index 5f509800e..f2a4fab51 100644 --- a/examples/test/test_i_basic_encryption.py +++ b/examples/test/test_i_basic_encryption.py @@ -23,5 +23,5 @@ def test_cycle_string(): plaintext = static_plaintext - cmk_arn = "arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f" + cmk_arn = get_cmk_arn() cycle_string(key_arn=cmk_arn, source_plaintext=plaintext, botocore_session=botocore.session.Session()) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 7e6ede8cd..34ba01c59 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -80,8 +80,8 @@ CreateDefaultCryptographicMaterialsManagerInput, ) from aws_cryptographic_materialproviders.mpl.references import ( - ICryptographicMaterialsManager, - IKeyring, + ICryptographicMaterialsManager as MPL_ICryptographicMaterialsManager, + IKeyring as MPL_IKeyring, ) _HAS_MPL = True @@ -141,7 +141,7 @@ class _ClientConfig(object): # pylint: disable=too-many-instance-attributes default=None, validator=attr.validators.optional( attr.validators.instance_of( - (CryptoMaterialsManager, ICryptographicMaterialsManager) + (CryptoMaterialsManager, MPL_ICryptographicMaterialsManager) ) ) ) @@ -161,7 +161,7 @@ class _ClientConfig(object): # pylint: disable=too-many-instance-attributes if _HAS_MPL: # Keyrings are only available if the MPL is installed in the runtime keyring = attr.ib( - hash=True, default=None, validator=attr.validators.optional(attr.validators.instance_of(IKeyring)) + hash=True, default=None, validator=attr.validators.optional(attr.validators.instance_of(MPL_IKeyring)) ) source_length = attr.ib( hash=True, default=None, validator=attr.validators.optional(attr.validators.instance_of(six.integer_types)) @@ -202,7 +202,7 @@ def _has_mpl_attrs_post_init(self): # If the provided materials_manager is directly from the MPL, wrap it in a native interface # for internal use. elif (self.materials_manager is not None - and isinstance(self.materials_manager, ICryptographicMaterialsManager)): + and isinstance(self.materials_manager, MPL_ICryptographicMaterialsManager)): self.materials_manager = CryptoMaterialsManagerFromMPL(self.materials_manager) def _no_mpl_attrs_post_init(self): @@ -437,10 +437,10 @@ class EncryptorConfig(_ClientConfig): :param key_provider: `MasterKeyProvider` from which to obtain data keys for encryption (either `materials_manager` or `key_provider` required) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider - :param keyring: `IKeyring` from the aws_cryptographic_materialproviders library + :param keyring: `MPL_IKeyring` from the aws_cryptographic_materialproviders library which handles encryption and decryption :type keyring: - aws_cryptographic_materialproviders.mpl.references.IKeyring + aws_cryptographic_materialproviders.mpl.references.MPL_IKeyring :param int source_length: Length of source data (optional) .. note:: @@ -492,10 +492,10 @@ class StreamEncryptor(_EncryptionStream): # pylint: disable=too-many-instance-a :param key_provider: `MasterKeyProvider` from which to obtain data keys for encryption (either `materials_manager` or `key_provider` required) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider - :param keyring: `IKeyring` from the aws_cryptographic_materialproviders library + :param keyring: `MPL_IKeyring` from the aws_cryptographic_materialproviders library which handles encryption and decryption :type keyring: - aws_cryptographic_materialproviders.mpl.references.IKeyring + aws_cryptographic_materialproviders.mpl.references.MPL_IKeyring :param int source_length: Length of source data (optional) .. note:: @@ -878,10 +878,10 @@ class DecryptorConfig(_ClientConfig): :param key_provider: `MasterKeyProvider` from which to obtain data keys for decryption (either `keyring`, `materials_manager` or `key_provider` required) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider - :param keyring: `IKeyring` from the aws_cryptographic_materialproviders library + :param keyring: `MPL_IKeyring` from the aws_cryptographic_materialproviders library which handles encryption and decryption :type keyring: - aws_cryptographic_materialproviders.mpl.references.IKeyring + aws_cryptographic_materialproviders.mpl.references.MPL_IKeyring :param int source_length: Length of source data (optional) .. note:: @@ -926,10 +926,10 @@ class StreamDecryptor(_EncryptionStream): # pylint: disable=too-many-instance-a :param key_provider: `MasterKeyProvider` from which to obtain data keys for decryption (either `materials_manager` or `key_provider` required) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider - :param keyring: `IKeyring` from the aws_cryptographic_materialproviders library + :param keyring: `MPL_IKeyring` from the aws_cryptographic_materialproviders library which handles encryption and decryption :type keyring: - aws_cryptographic_materialproviders.mpl.references.IKeyring + aws_cryptographic_materialproviders.mpl.references.MPL_IKeyring :param int source_length: Length of source data (optional) .. note:: @@ -1000,7 +1000,7 @@ def _read_header(self): ) decryption_materials = self.config.materials_manager.decrypt_materials(request=decrypt_materials_request) - + # If the materials_manager passed required_encryption_context_keys, # get the items out of the encryption_context with the keys. # The items are used in header validation. From 8415c2cbb2eb138e330816374dccc0f15cce6a38 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 28 Feb 2024 17:00:17 -0800 Subject: [PATCH 147/422] cleanup --- src/aws_encryption_sdk/streaming_client.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 34ba01c59..953a82a66 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -437,10 +437,10 @@ class EncryptorConfig(_ClientConfig): :param key_provider: `MasterKeyProvider` from which to obtain data keys for encryption (either `materials_manager` or `key_provider` required) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider - :param keyring: `MPL_IKeyring` from the aws_cryptographic_materialproviders library + :param keyring: `IKeyring` from the aws_cryptographic_materialproviders library which handles encryption and decryption :type keyring: - aws_cryptographic_materialproviders.mpl.references.MPL_IKeyring + aws_cryptographic_materialproviders.mpl.references.IKeyring :param int source_length: Length of source data (optional) .. note:: @@ -492,10 +492,10 @@ class StreamEncryptor(_EncryptionStream): # pylint: disable=too-many-instance-a :param key_provider: `MasterKeyProvider` from which to obtain data keys for encryption (either `materials_manager` or `key_provider` required) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider - :param keyring: `MPL_IKeyring` from the aws_cryptographic_materialproviders library + :param keyring: `IKeyring` from the aws_cryptographic_materialproviders library which handles encryption and decryption :type keyring: - aws_cryptographic_materialproviders.mpl.references.MPL_IKeyring + aws_cryptographic_materialproviders.mpl.references.IKeyring :param int source_length: Length of source data (optional) .. note:: @@ -878,10 +878,10 @@ class DecryptorConfig(_ClientConfig): :param key_provider: `MasterKeyProvider` from which to obtain data keys for decryption (either `keyring`, `materials_manager` or `key_provider` required) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider - :param keyring: `MPL_IKeyring` from the aws_cryptographic_materialproviders library + :param keyring: `IKeyring` from the aws_cryptographic_materialproviders library which handles encryption and decryption :type keyring: - aws_cryptographic_materialproviders.mpl.references.MPL_IKeyring + aws_cryptographic_materialproviders.mpl.references.IKeyring :param int source_length: Length of source data (optional) .. note:: @@ -926,10 +926,10 @@ class StreamDecryptor(_EncryptionStream): # pylint: disable=too-many-instance-a :param key_provider: `MasterKeyProvider` from which to obtain data keys for decryption (either `materials_manager` or `key_provider` required) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider - :param keyring: `MPL_IKeyring` from the aws_cryptographic_materialproviders library + :param keyring: `IKeyring` from the aws_cryptographic_materialproviders library which handles encryption and decryption :type keyring: - aws_cryptographic_materialproviders.mpl.references.MPL_IKeyring + aws_cryptographic_materialproviders.mpl.references.IKeyring :param int source_length: Length of source data (optional) .. note:: From 20bdaffca7b83444348fdfb0e6abab7c3af5c8c9 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 28 Feb 2024 18:21:42 -0800 Subject: [PATCH 148/422] cleanup --- src/aws_encryption_sdk/streaming_client.py | 2 +- .../test_streaming_client_stream_decryptor.py | 42 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 953a82a66..4a742b91f 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -672,7 +672,7 @@ def _write_header(self): # If there is _required_encryption_context, # serialize it, then authenticate it - if self._required_encryption_context is not None: + if hasattr(self, "_required_encryption_context"): required_ec_serialized = aws_encryption_sdk.internal.formatting.encryption_context.serialize_encryption_context( self._required_encryption_context ) diff --git a/test/unit/test_streaming_client_stream_decryptor.py b/test/unit/test_streaming_client_stream_decryptor.py index e06cad308..4929646b5 100644 --- a/test/unit/test_streaming_client_stream_decryptor.py +++ b/test/unit/test_streaming_client_stream_decryptor.py @@ -193,6 +193,9 @@ def test_read_header(self, mock_derive_datakey, mock_decrypt_materials_request, test_decryptor.source_stream = ct_stream test_decryptor._stream_length = len(VALUES["data_128"]) + # Mock: hasattr(self.config, "encryption_context") returns False + del test_decryptor.config.encryption_context + test_header, test_header_auth = test_decryptor._read_header() self.mock_deserialize_header.assert_called_once_with(ct_stream, None) @@ -230,6 +233,45 @@ def test_read_header(self, mock_derive_datakey, mock_decrypt_materials_request, assert test_header is self.mock_header assert test_header_auth is sentinel.header_auth + @patch("aws_encryption_sdk.streaming_client.derive_data_encryption_key") + @patch("aws_encryption_sdk.streaming_client.DecryptionMaterialsRequest") + @patch("aws_encryption_sdk.streaming_client.Verifier") + # Given: no MPL + @pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") + def test_GIVEN_verification_key_AND_no_mpl_WHEN_read_header_THEN_calls_from_key_bytes( + self, + mock_verifier, + mock_decrypt_materials_request, + *_, + ): + + mock_verifier_instance = MagicMock() + mock_verifier.from_key_bytes.return_value = mock_verifier_instance + ct_stream = io.BytesIO(VALUES["data_128"]) + mock_commitment_policy = MagicMock(__class__=CommitmentPolicy) + test_decryptor = StreamDecryptor( + materials_manager=self.mock_materials_manager, + source=ct_stream, + commitment_policy=mock_commitment_policy, + ) + test_decryptor.source_stream = ct_stream + test_decryptor._stream_length = len(VALUES["data_128"]) + # Given: self.config has "encryption_context" + any_reproduced_ec = {"some": "ec"} + test_decryptor.config.encryption_context = any_reproduced_ec + + # When: read header + test_decryptor._read_header() + + # Then: calls decrypt_materials with reproduced_encryption_context + mock_decrypt_materials_request.assert_called_once_with( + encrypted_data_keys=self.mock_header.encrypted_data_keys, + algorithm=self.mock_header.algorithm, + encryption_context=sentinel.encryption_context, + commitment_policy=mock_commitment_policy, + reproduced_encryption_context=any_reproduced_ec, + ) + @patch("aws_encryption_sdk.streaming_client.DecryptionMaterialsRequest") @patch("aws_encryption_sdk.streaming_client.derive_data_encryption_key") @patch("aws_encryption_sdk.streaming_client.Verifier") From 6bf6094eff239f36c7dffb010e2b59cb7e948ab4 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 28 Feb 2024 18:29:50 -0800 Subject: [PATCH 149/422] cleanup --- .../internal/formatting/serialize.py | 8 +++++--- .../materials_managers/mpl/materials.py | 1 - src/aws_encryption_sdk/streaming_client.py | 18 ++++++++++-------- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/aws_encryption_sdk/internal/formatting/serialize.py b/src/aws_encryption_sdk/internal/formatting/serialize.py index 344c94703..5a054989c 100644 --- a/src/aws_encryption_sdk/internal/formatting/serialize.py +++ b/src/aws_encryption_sdk/internal/formatting/serialize.py @@ -237,7 +237,7 @@ def _serialize_header_auth_v2( for all items whose keys are in the required_encryption_context list. This is ONLY processed if using the aws-cryptographic-materialproviders library AND its required encryption context CMM. (optional) - :type required_encryption_context_bytes: bytes + :type required_encryption_context_bytes: bytes :returns: Serialized header authentication data :rtype: bytes """ @@ -294,14 +294,16 @@ def serialize_header_auth( This is ONLY processed if using the aws-cryptographic-materialproviders library AND its required encryption context CMM AND if using the v2 message format. (optional) - :type required_encryption_context_bytes: bytes + :type required_encryption_context_bytes: bytes :returns: Serialized header authentication data :rtype: bytes """ if version == SerializationVersion.V1: return _serialize_header_auth_v1(algorithm, header, data_encryption_key, signer) elif version == SerializationVersion.V2: - return _serialize_header_auth_v2(algorithm, header, data_encryption_key, signer, required_encryption_context_bytes) + return _serialize_header_auth_v2( + algorithm, header, data_encryption_key, signer, required_encryption_context_bytes + ) else: raise SerializationError("Unrecognized message format version: {}".format(version)) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index 5b066c7c7..5e4a66318 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -95,7 +95,6 @@ def data_encryption_key(self) -> DataKey: def signing_key(self) -> bytes: """Materials' signing key.""" return self.mpl_materials.signing_key - @property def required_encryption_context_keys(self) -> bytes: diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 4a742b91f..048a6caa3 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -198,7 +198,7 @@ def _has_mpl_attrs_post_init(self): # Wrap MPL error into the ESDK error type # so customers only have to catch ESDK error types. raise AWSEncryptionSDKClientError(mpl_exception) - + # If the provided materials_manager is directly from the MPL, wrap it in a native interface # for internal use. elif (self.materials_manager is not None @@ -673,9 +673,10 @@ def _write_header(self): # If there is _required_encryption_context, # serialize it, then authenticate it if hasattr(self, "_required_encryption_context"): - required_ec_serialized = aws_encryption_sdk.internal.formatting.encryption_context.serialize_encryption_context( - self._required_encryption_context - ) + required_ec_serialized = \ + aws_encryption_sdk.internal.formatting.encryption_context.serialize_encryption_context( + self._required_encryption_context + ) self.output_buffer += serialize_header_auth( version=self._header.version, algorithm=self._encryption_materials.algorithm, @@ -955,7 +956,7 @@ def _prep_message(self): self._prep_non_framed() self._message_prepped = True - def _read_header(self): + def _read_header(self): # noqa: C901 """Reads the message header from the input stream. :returns: tuple containing deserialized header and header_auth objects @@ -1056,9 +1057,10 @@ def _read_header(self): # The authenticated only encryption context is all encryption context key-value pairs where the # key exists in Required Encryption Context Keys. It is then serialized according to the # message header Key Value Pairs. - required_ec_serialized = aws_encryption_sdk.internal.formatting.encryption_context.serialize_encryption_context( - self._required_encryption_context - ) + required_ec_serialized = \ + aws_encryption_sdk.internal.formatting.encryption_context.serialize_encryption_context( + self._required_encryption_context + ) validate_header( header=header, From febe6dba05160087e6ac905f2f1582c79c56a1e8 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 28 Feb 2024 18:38:46 -0800 Subject: [PATCH 150/422] cleanup --- .../keyrings/required_encryption_context_cmm.py | 14 +++----------- .../internal/formatting/serialize.py | 10 +++++----- .../materials_managers/mpl/materials.py | 4 ++++ src/aws_encryption_sdk/streaming_client.py | 17 +++++++++-------- .../test_streaming_client_stream_decryptor.py | 2 +- 5 files changed, 22 insertions(+), 25 deletions(-) diff --git a/examples/src/keyrings/required_encryption_context_cmm.py b/examples/src/keyrings/required_encryption_context_cmm.py index f3d58c922..9f8de9976 100644 --- a/examples/src/keyrings/required_encryption_context_cmm.py +++ b/examples/src/keyrings/required_encryption_context_cmm.py @@ -14,25 +14,17 @@ from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig from aws_cryptographic_materialproviders.mpl.models import ( - CacheTypeDefault, CreateAwsKmsKeyringInput, CreateDefaultCryptographicMaterialsManagerInput, CreateRequiredEncryptionContextCMMInput, - DefaultCache, ) -from aws_cryptographic_materialproviders.mpl.references import ( - IKeyring, - ICryptographicMaterialsManager, -) -from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL -from typing import Dict +from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager, IKeyring +from typing import Dict, List import aws_encryption_sdk from aws_encryption_sdk import CommitmentPolicy from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError -from .example_branch_key_id_supplier import ExampleBranchKeyIdSupplier - # TODO-MPL: Remove this as part of removing PYTHONPATH hacks module_root_dir = '/'.join(__file__.split("/")[:-1]) @@ -98,7 +90,7 @@ def encrypt_and_decrypt_with_keyring( underlying_cmm=underlying_cmm, ) ) - + # 6. Encrypt the data ciphertext, _ = client.encrypt( source=EXAMPLE_DATA, diff --git a/src/aws_encryption_sdk/internal/formatting/serialize.py b/src/aws_encryption_sdk/internal/formatting/serialize.py index 5a054989c..66f4800de 100644 --- a/src/aws_encryption_sdk/internal/formatting/serialize.py +++ b/src/aws_encryption_sdk/internal/formatting/serialize.py @@ -223,7 +223,7 @@ def _serialize_header_auth_v2( header, data_encryption_key, signer=None, - required_encryption_context_bytes=None + required_ec_bytes=None ): """Creates serialized header authentication data for messages in serialization version V2. @@ -241,7 +241,7 @@ def _serialize_header_auth_v2( :returns: Serialized header authentication data :rtype: bytes """ - if required_encryption_context_bytes is None: + if required_ec_bytes is None: header_auth = encrypt( algorithm=algorithm, key=data_encryption_key, @@ -259,7 +259,7 @@ def _serialize_header_auth_v2( # be the encryption context in the encryption materials filtered to only contain key value # pairs listed in the encryption material's required encryption context keys serialized # according to the encryption context serialization specification. - associated_data=header + required_encryption_context_bytes, + associated_data=header + required_ec_bytes, iv=header_auth_iv(algorithm), ) output = struct.pack( @@ -277,7 +277,7 @@ def serialize_header_auth( header, data_encryption_key, signer=None, - required_encryption_context_bytes=None + required_ec_bytes=None ): """Creates serialized header authentication data. @@ -302,7 +302,7 @@ def serialize_header_auth( return _serialize_header_auth_v1(algorithm, header, data_encryption_key, signer) elif version == SerializationVersion.V2: return _serialize_header_auth_v2( - algorithm, header, data_encryption_key, signer, required_encryption_context_bytes + algorithm, header, data_encryption_key, signer, required_ec_bytes ) else: raise SerializationError("Unrecognized message format version: {}".format(version)) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index 5e4a66318..54ea21b39 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -97,6 +97,8 @@ def signing_key(self) -> bytes: return self.mpl_materials.signing_key @property + # Pylint thinks this name is too long, but it's the best descriptor for this... + # pylint: disable=invalid-name def required_encryption_context_keys(self) -> bytes: """Materials' required encryption context keys.""" return self.mpl_materials.required_encryption_context_keys @@ -148,6 +150,8 @@ def encryption_context(self) -> Dict[str, str]: return self.mpl_materials.encryption_context @property + # Pylint thinks this name is too long, but it's the best descriptor for this... + # pylint: disable=invalid-name def required_encryption_context_keys(self) -> bytes: """Materials' required encryption context keys.""" return self.mpl_materials.required_encryption_context_keys diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 048a6caa3..bbae73bef 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -76,9 +76,7 @@ from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException - from aws_cryptographic_materialproviders.mpl.models import ( - CreateDefaultCryptographicMaterialsManagerInput, - ) + from aws_cryptographic_materialproviders.mpl.models import CreateDefaultCryptographicMaterialsManagerInput from aws_cryptographic_materialproviders.mpl.references import ( ICryptographicMaterialsManager as MPL_ICryptographicMaterialsManager, IKeyring as MPL_IKeyring, @@ -631,11 +629,11 @@ def generate_header(self, message_id): if hasattr(self._encryption_materials, "required_encryption_context_keys"): self._required_encryption_context = {} self._stored_encryption_context = {} - for (k, v) in self._encryption_materials.encryption_context.items(): - if k in self._encryption_materials.required_encryption_context_keys: - self._required_encryption_context[k] = v + for (key, value) in self._encryption_materials.encryption_context.items(): + if key in self._encryption_materials.required_encryption_context_keys: + self._required_encryption_context[key] = value else: - self._stored_encryption_context[k] = v + self._stored_encryption_context[key] = value # Otherwise, store all encryption context with the message. else: self._stored_encryption_context = self._encryption_materials.encryption_context @@ -956,7 +954,10 @@ def _prep_message(self): self._prep_non_framed() self._message_prepped = True - def _read_header(self): # noqa: C901 + # TODO-MPL: Refactor this function, remove these linter disablers + # noqa: C901 + # pylint: disable=too-many-branches + def _read_header(self): """Reads the message header from the input stream. :returns: tuple containing deserialized header and header_auth objects diff --git a/test/unit/test_streaming_client_stream_decryptor.py b/test/unit/test_streaming_client_stream_decryptor.py index 4929646b5..2066dcbdb 100644 --- a/test/unit/test_streaming_client_stream_decryptor.py +++ b/test/unit/test_streaming_client_stream_decryptor.py @@ -238,7 +238,7 @@ def test_read_header(self, mock_derive_datakey, mock_decrypt_materials_request, @patch("aws_encryption_sdk.streaming_client.Verifier") # Given: no MPL @pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") - def test_GIVEN_verification_key_AND_no_mpl_WHEN_read_header_THEN_calls_from_key_bytes( + def test_GIVEN_decrypt_config_has_ec_WHEN_read_header_THEN_calls_decrypt_materials_with_reproduced_ec( self, mock_verifier, mock_decrypt_materials_request, From dc8abca6925ce2cd139ef53c7f3b2c8f9b3e09ce Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 28 Feb 2024 18:40:26 -0800 Subject: [PATCH 151/422] cleanup --- src/aws_encryption_sdk/streaming_client.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index bbae73bef..9488c9f08 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -954,10 +954,8 @@ def _prep_message(self): self._prep_non_framed() self._message_prepped = True - # TODO-MPL: Refactor this function, remove these linter disablers - # noqa: C901 - # pylint: disable=too-many-branches - def _read_header(self): + # TODO-MPL: Refactor this function, remove linter disablers + def _read_header(self): # noqa pylint: disable=too-many-branches """Reads the message header from the input stream. :returns: tuple containing deserialized header and header_auth objects From 8ff46f4520e510767d89365634c0e2da6a139f58 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 28 Feb 2024 18:41:15 -0800 Subject: [PATCH 152/422] cleanup --- src/aws_encryption_sdk/streaming_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 9488c9f08..ffef9cd3a 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -1006,9 +1006,9 @@ def _read_header(self): # noqa pylint: disable=too-many-branches # The items are used in header validation. if hasattr(decryption_materials, "required_encryption_context_keys"): self._required_encryption_context = {} - for (k, v) in decryption_materials.encryption_context.items(): - if k in decryption_materials.required_encryption_context_keys: - self._required_encryption_context[k] = v + for (key, value) in decryption_materials.encryption_context.items(): + if key in decryption_materials.required_encryption_context_keys: + self._required_encryption_context[key] = value else: self._required_encryption_context = None From aba7cccad05ba9ba60b49fb14a9ee3153354b66b Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 28 Feb 2024 18:42:53 -0800 Subject: [PATCH 153/422] cleanup --- src/aws_encryption_sdk/streaming_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index ffef9cd3a..fb0935ff2 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -681,7 +681,7 @@ def _write_header(self): header=self.output_buffer, data_encryption_key=self._derived_data_key, signer=self.signer, - required_encryption_context_bytes=required_ec_serialized, + required_ec_bytes=required_ec_serialized, ) # Otherwise, do not pass in any required encryption context else: From 40fecc05de1e42eca0d860ac2b04bee5670d504c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 29 Feb 2024 10:54:22 -0800 Subject: [PATCH 154/422] all message format versions --- .../internal/formatting/serialize.py | 44 ++++++++++++++----- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/src/aws_encryption_sdk/internal/formatting/serialize.py b/src/aws_encryption_sdk/internal/formatting/serialize.py index 66f4800de..310cf1436 100644 --- a/src/aws_encryption_sdk/internal/formatting/serialize.py +++ b/src/aws_encryption_sdk/internal/formatting/serialize.py @@ -189,7 +189,13 @@ def serialize_header(header, signer=None): raise SerializationError("Unrecognized message format version: {}".format(header.version)) -def _serialize_header_auth_v1(algorithm, header, data_encryption_key, signer=None): +def _serialize_header_auth_v1( + algorithm, + header, + data_encryption_key, + signer=None, + required_ec_bytes=None +): """Creates serialized header authentication data for messages in serialization version V1. :param algorithm: Algorithm to use for encryption @@ -198,16 +204,35 @@ def _serialize_header_auth_v1(algorithm, header, data_encryption_key, signer=Non :param bytes data_encryption_key: Data key with which to encrypt message :param signer: Cryptographic signer object (optional) :type signer: aws_encryption_sdk.Signer + :param required_encryption_context_bytes: Serialized encryption context items + for all items whose keys are in the required_encryption_context list. + This is ONLY processed if using the aws-cryptographic-materialproviders library + AND its required encryption context CMM. (optional) + :type required_encryption_context_bytes: bytes :returns: Serialized header authentication data :rtype: bytes """ - header_auth = encrypt( - algorithm=algorithm, - key=data_encryption_key, - plaintext=b"", - associated_data=header, - iv=header_auth_iv(algorithm), - ) + if required_ec_bytes is None: + header_auth = encrypt( + algorithm=algorithm, + key=data_encryption_key, + plaintext=b"", + associated_data=header, + iv=header_auth_iv(algorithm), + ) + else: + header_auth = encrypt( + algorithm=algorithm, + key=data_encryption_key, + plaintext=b"", + # The AAD MUST be the concatenation of the serialized message header body and the serialization + # of encryption context to only authenticate. The encryption context to only authenticate MUST + # be the encryption context in the encryption materials filtered to only contain key value + # pairs listed in the encryption material's required encryption context keys serialized + # according to the encryption context serialization specification. + associated_data=header + required_ec_bytes, + iv=header_auth_iv(algorithm), + ) output = struct.pack( ">{iv_len}s{tag_len}s".format(iv_len=algorithm.iv_len, tag_len=algorithm.tag_len), header_auth.iv, @@ -292,8 +317,7 @@ def serialize_header_auth( :param required_encryption_context_bytes: Serialized encryption context items for all items whose keys are in the required_encryption_context list. This is ONLY processed if using the aws-cryptographic-materialproviders library - AND its required encryption context CMM - AND if using the v2 message format. (optional) + AND its required encryption context CMM. (optional) :type required_encryption_context_bytes: bytes :returns: Serialized header authentication data :rtype: bytes From 52043b9fd3c5cb3ce40b222f7cfe2e3e0427a771 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 29 Feb 2024 10:55:07 -0800 Subject: [PATCH 155/422] sync upstream --- src/aws_encryption_sdk/internal/formatting/serialize.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/internal/formatting/serialize.py b/src/aws_encryption_sdk/internal/formatting/serialize.py index 310cf1436..9f1325f98 100644 --- a/src/aws_encryption_sdk/internal/formatting/serialize.py +++ b/src/aws_encryption_sdk/internal/formatting/serialize.py @@ -323,7 +323,9 @@ def serialize_header_auth( :rtype: bytes """ if version == SerializationVersion.V1: - return _serialize_header_auth_v1(algorithm, header, data_encryption_key, signer) + return _serialize_header_auth_v1( + algorithm, header, data_encryption_key, signer, required_ec_bytes + ) elif version == SerializationVersion.V2: return _serialize_header_auth_v2( algorithm, header, data_encryption_key, signer, required_ec_bytes From d256bf53fd6be9e9b8ddba61b638602f3cd60b2b Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 6 Mar 2024 11:11:36 -0800 Subject: [PATCH 156/422] working --- .../materials_managers/mpl/cmm.py | 5 + .../commands/full_message_decrypt.py | 9 +- .../commands/full_message_decrypt_generate.py | 9 +- .../manifests/full_message/decrypt.py | 72 ++++++++-- .../full_message/decrypt_generation.py | 79 +++++++++-- .../manifests/full_message/encrypt.py | 133 ++++++++++++++++++ 6 files changed, 283 insertions(+), 24 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index 8df42bf48..56c936012 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -85,6 +85,9 @@ def _native_to_mpl_get_encryption_materials( encryption_context=request.encryption_context, commitment_policy=commitment_policy, max_plaintext_length=request.plaintext_length, + algorithm_suite_id=CryptoMaterialsManagerFromMPL._native_algorithm_id_to_mpl_algorithm_id( + request.algorithm.algorithm_id + ) ) return output @@ -112,6 +115,8 @@ def decrypt_materials( try: mpl_input: 'MPL_DecryptMaterialsInput' = \ CryptoMaterialsManagerFromMPL._create_mpl_decrypt_materials_input_from_request(request) + print(f"{mpl_input=}") + print(f"{self.mpl_cmm._impl.__dict__=}") mpl_output: 'MPL_DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(mpl_input) return DecryptionMaterialsFromMPL(mpl_output.decryption_materials) except AwsCryptographicMaterialProvidersException as mpl_exception: diff --git a/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt.py b/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt.py index baf1d1f03..ad457fbd3 100644 --- a/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt.py @@ -29,9 +29,16 @@ def cli(args=None): parser.add_argument( "--input", required=True, type=argparse.FileType("r"), help="Existing full message decrypt manifest" ) + parser.add_argument( + "--keyrings", + action="store_true", + required=False, + default=False, + help="Use keyring interfaces to encrypt", + ) parsed = parser.parse_args(args) - decrypt_manifest = MessageDecryptionManifest.from_file(parsed.input) + decrypt_manifest = MessageDecryptionManifest.from_file(parsed.input, parsed.keyrings) decrypt_manifest.run() diff --git a/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt_generate.py b/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt_generate.py index 5d8b94893..9d5c9e1fa 100644 --- a/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt_generate.py +++ b/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt_generate.py @@ -39,9 +39,16 @@ def cli(args=None): dest="json_indent", help="Output human-readable JSON", ) + parser.add_argument( + "--keyrings", + action="store_true", + required=False, + default=False, + help="Use keyring interfaces to encrypt", + ) parsed = parser.parse_args(args) - encrypt_manifest = MessageDecryptionGenerationManifest.from_file(parsed.input) + encrypt_manifest = MessageDecryptionGenerationManifest.from_file(parsed.input, parsed.keyrings) encrypt_manifest.run_and_write_to_dir(target_directory=parsed.output, json_indent=parsed.json_indent) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index c94fd1452..dd09dc440 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -35,6 +35,9 @@ from awses_test_vectors.manifests.keys import KeysManifest from awses_test_vectors.manifests.master_key import MasterKeySpec, master_key_provider_from_master_key_specs +from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_provider_from_master_key_specs + + try: # Python 3.5.0 and 3.5.1 have incompatible typing modules from typing import IO, Callable, Dict, Iterable, Optional # noqa pylint: disable=unused-import @@ -202,6 +205,7 @@ class MessageDecryptionTestScenario(object): master_key_specs = attr.ib(validator=iterable_validator(list, MasterKeySpec)) master_key_provider_fn = attr.ib(validator=attr.validators.is_callable()) result = attr.ib(validator=attr.validators.instance_of(MessageDecryptionTestResult)) + keyrings = attr.ib(validator=attr.validators.instance_of(bool)) decryption_method = attr.ib( default=None, validator=attr.validators.optional(attr.validators.instance_of(DecryptionMethod)) ) @@ -216,6 +220,7 @@ def __init__( result, # type: MessageDecryptionTestResult master_key_specs, # type: Iterable[MasterKeySpec] master_key_provider_fn, # type: Callable + keyrings, # type: bool decryption_method=None, # type: Optional[DecryptionMethod] description=None, # type: Optional[str] ): # noqa=D107 @@ -231,6 +236,7 @@ def __init__( self.master_key_provider_fn = master_key_provider_fn self.decryption_method = decryption_method self.description = description + self.keyrings = keyrings attr.validate(self) @classmethod @@ -240,6 +246,8 @@ def from_scenario( plaintext_reader, # type: Callable[[str], bytes] ciphertext_reader, # type: Callable[[str], bytes] keys, # type: KeysManifest + keyrings, # type: bool + keys_uri, # type: str ): # type: (...) -> MessageDecryptionTestScenario """Load from a scenario specification. @@ -252,10 +260,18 @@ def from_scenario( :rtype: MessageDecryptionTestScenario """ raw_master_key_specs = scenario["master-keys"] # type: Iterable[MASTER_KEY_SPEC] - master_key_specs = [MasterKeySpec.from_scenario(spec) for spec in raw_master_key_specs] + if keyrings: + master_key_specs = [KeyringSpec.from_scenario(spec) for spec in raw_master_key_specs] + else: + master_key_specs = [MasterKeySpec.from_scenario(spec) for spec in raw_master_key_specs] + + print(f"{master_key_specs=}") def master_key_provider_fn(): - return master_key_provider_from_master_key_specs(keys, master_key_specs) + if keyrings: + return keyring_provider_from_master_key_specs(keys_uri, master_key_specs) + else: + return master_key_provider_from_master_key_specs(keys, master_key_specs) decryption_method_spec = scenario.get("decryption-method") decryption_method = DecryptionMethod(decryption_method_spec) if decryption_method_spec else None @@ -268,6 +284,7 @@ def master_key_provider_fn(): master_key_specs=master_key_specs, master_key_provider_fn=master_key_provider_fn, result=result, + keyrings=keyrings, decryption_method=decryption_method, description=scenario.get("description"), ) @@ -292,16 +309,27 @@ def scenario_spec(self): return spec def _one_shot_decrypt(self): + keyring = self.master_key_provider_fn() + print(f"{keyring=}") client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - return client.decrypt(source=self.ciphertext, key_provider=self.master_key_provider_fn()) + if self.keyrings: + return client.decrypt(source=self.ciphertext, keyring=keyring) + else: + return client.decrypt(source=self.ciphertext, key_provider=self.master_key_provider_fn()) def _streaming_decrypt(self): result = bytearray() client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - with client.stream(source=self.ciphertext, mode="d", key_provider=self.master_key_provider_fn()) as decryptor: - for chunk in decryptor: - result.extend(chunk) - return result, decryptor.header + if self.keyrings: + with client.stream(source=self.ciphertext, mode="d", keyring=self.master_key_provider_fn()) as decryptor: + for chunk in decryptor: + result.extend(chunk) + return result, decryptor.header + else: + with client.stream(source=self.ciphertext, mode="d", key_provider=self.master_key_provider_fn()) as decryptor: + for chunk in decryptor: + result.extend(chunk) + return result, decryptor.header def _streaming_decrypt_unsigned(self): result = bytearray() @@ -388,11 +416,12 @@ def manifest_spec(self): return {"manifest": manifest_spec, "client": client_spec, "keys": self.keys_uri, "tests": test_specs} @classmethod - def from_file(cls, input_file): + def from_file(cls, input_file, keyrings): # type: (IO) -> MessageDecryptionManifest """Load from a file containing a full message decrypt manifest. :param file input_file: File object for file containing JSON manifest + :param bool keyrings: True if should encrypt with keyring interfaces; False otherwise :return: Loaded manifest :rtype: MessageDecryptionManifest """ @@ -407,6 +436,10 @@ def from_file(cls, input_file): version = raw_manifest["manifest"]["version"] # type: int keys_uri = raw_manifest["keys"] # type: str + keys_uri = raw_manifest["keys"] + keys_filename = keys_uri.replace("file://", "") + joined = os.path.join(parent_dir, keys_filename) + raw_keys_manifest = json.loads(root_reader(keys_uri).decode(ENCODING)) keys = KeysManifest.from_manifest_spec(raw_keys_manifest) @@ -415,10 +448,31 @@ def from_file(cls, input_file): raw_scenarios = raw_manifest["tests"] # type: Dict[str, DECRYPT_SCENARIO_SPEC] test_scenarios = { name: MessageDecryptionTestScenario.from_scenario( - scenario=scenario, plaintext_reader=root_reader, ciphertext_reader=root_reader, keys=keys + scenario=scenario, + plaintext_reader=root_reader, + ciphertext_reader=root_reader, + keys=keys, + keyrings=False, + keys_uri=joined, ) for name, scenario in raw_scenarios.items() } + # If optional keyrings argument is true, + # also add scenarios to decrypt with keyrings. + if keyrings: + keyrings_test_scenarios = { + name + "-keyring": MessageDecryptionTestScenario.from_scenario( + scenario=scenario, + plaintext_reader=root_reader, + ciphertext_reader=root_reader, + keys=keys, + keyrings=True, + keys_uri=joined, + ) + for name, scenario in raw_scenarios.items() + } + # Merge into test_scenarios + test_scenarios = {**keyrings_test_scenarios, **test_scenarios} return cls( keys_uri=keys_uri, diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index e407a1b65..040ec07d8 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -23,10 +23,25 @@ import attr import six from aws_encryption_sdk.caches.local import LocalCryptoMaterialsCache +from aws_encryption_sdk.key_providers.base import MasterKeyProvider from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager from aws_encryption_sdk.materials_managers.caching import CachingCryptoMaterialsManager from aws_encryption_sdk.materials_managers.default import DefaultCryptoMaterialsManager +from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders +from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig +from aws_cryptographic_materialproviders.mpl.references import ( + IKeyring, + CryptographicMaterialsManager, +) +from aws_cryptographic_materialproviders.mpl.models import ( + CreateDefaultCryptographicMaterialsManagerInput, +) +from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL + +from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_provider_from_master_key_specs + + from awses_test_vectors.internal.defaults import ENCODING from awses_test_vectors.internal.util import ( dictionary_validator, @@ -92,9 +107,23 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, pl return: a list of (ciphertext, result) pairs """ - materials_manager = DefaultCryptoMaterialsManager( - generation_scenario.encryption_scenario.master_key_provider_fn() - ) + tmp = generation_scenario.encryption_scenario.master_key_provider_fn() + if isinstance(tmp, MasterKeyProvider): + materials_manager = DefaultCryptoMaterialsManager( + tmp + ) + elif isinstance(tmp, IKeyring): + mpl = AwsCryptographicMaterialProviders(MaterialProvidersConfig()) + mpl_cmm = mpl.create_default_cryptographic_materials_manager( + CreateDefaultCryptographicMaterialsManagerInput( + keyring=tmp + ) + ) + materials_manager = CryptoMaterialsManagerFromMPL( + mpl_cmm=mpl_cmm + ) + else: + raise ValueError(f"Unrecognized master_key_provider_fn return type: {str(tmp)}") ciphertext_to_decrypt = generation_scenario.encryption_scenario.run(materials_manager) if generation_scenario.result: expected_result = generation_scenario.result @@ -334,30 +363,40 @@ class MessageDecryptionTestScenarioGenerator(object): decryption_master_key_specs = attr.ib(validator=iterable_validator(list, MasterKeySpec)) decryption_master_key_provider_fn = attr.ib(validator=attr.validators.is_callable()) result = attr.ib(validator=attr.validators.optional(attr.validators.instance_of(MessageDecryptionTestResult))) + keyrings = attr.ib(validator=attr.validators.instance_of(bool)) @classmethod - def from_scenario(cls, scenario, keys, plaintexts): + def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): """Load from a scenario specification. :param dict scenario: Scenario specification JSON :param KeysManifest keys: Loaded keys :param dict plaintexts: Mapping of plaintext names to plaintext values + :param bool keyrings: True if should encrypt with keyring interfaces; False otherwise :return: Loaded test scenario :rtype: MessageDecryptionTestScenarioGenerator """ encryption_scenario_spec = scenario["encryption-scenario"] - encryption_scenario = MessageEncryptionTestScenario.from_scenario(encryption_scenario_spec, keys, plaintexts) + encryption_scenario = MessageEncryptionTestScenario.from_scenario(encryption_scenario_spec, keys, plaintexts, keyrings, keys_uri) tampering = scenario.get("tampering") tampering_method = TamperingMethod.from_tampering_spec(tampering) decryption_method_spec = scenario.get("decryption-method") decryption_method = DecryptionMethod(decryption_method_spec) if decryption_method_spec else None if "decryption-master-keys" in scenario: - decryption_master_key_specs = [ - MasterKeySpec.from_scenario(spec) for spec in scenario["decryption-master-keys"] - ] + if keyrings: + decryption_master_key_specs = [ + KeyringSpec.from_scenario(spec) for spec in scenario["decryption-master-keys"] + ] + else: + decryption_master_key_specs = [ + MasterKeySpec.from_scenario(spec) for spec in scenario["decryption-master-keys"] + ] def decryption_master_key_provider_fn(): - return master_key_provider_from_master_key_specs(keys, decryption_master_key_specs) + if keyrings: + return keyring_provider_from_master_key_specs(keys_uri, decryption_master_key_specs) + else: + return master_key_provider_from_master_key_specs(keys, decryption_master_key_specs) else: decryption_master_key_specs = encryption_scenario.master_key_specs @@ -372,6 +411,7 @@ def decryption_master_key_provider_fn(): decryption_master_key_specs=decryption_master_key_specs, decryption_master_key_provider_fn=decryption_master_key_provider_fn, result=result, + keyrings=keyrings, ) def run(self, ciphertext_writer, plaintext_uri): @@ -400,6 +440,7 @@ def decryption_test_scenario_pair(self, ciphertext_writer, ciphertext_to_decrypt master_key_provider_fn=self.decryption_master_key_provider_fn, decryption_method=self.decryption_method, result=expected_result, + keyrings=self.keyrings, ), ) @@ -414,12 +455,14 @@ class MessageDecryptionGenerationManifest(object): :param KeysManifest keys: Loaded keys :param dict plaintexts: Mapping of plaintext names to plaintext values :param dict tests: Mapping of test scenario names to :class:`MessageDecryptionGenerationManifest`s + :param bool keyrings: True if should encrypt with keyring interfaces; False otherwise """ version = attr.ib(validator=membership_validator(SUPPORTED_VERSIONS)) keys = attr.ib(validator=attr.validators.instance_of(KeysManifest)) plaintexts = attr.ib(validator=dictionary_validator(six.string_types, six.binary_type)) tests = attr.ib(validator=dictionary_validator(six.string_types, MessageDecryptionTestScenarioGenerator)) + keyrings = attr.ib(validator=attr.validators.instance_of(bool)) type_name = "awses-decrypt-generate" @staticmethod @@ -434,11 +477,12 @@ def _generate_plaintexts(plaintexts_specs): return {name: os.urandom(size) for name, size in plaintexts_specs.items()} @classmethod - def from_file(cls, input_file): + def from_file(cls, input_file, keyrings): # type: (IO) -> MessageDecryptionGenerationManifest """Load from a file containing a full message encrypt manifest. :param file input_file: File object for file containing JSON manifest + :param bool keyrings: True if should encrypt with keyring interfaces; False otherwise :return: Loaded manifest :rtype: MessageEncryptionManifest """ @@ -449,18 +493,27 @@ def from_file(cls, input_file): parent_dir = os.path.abspath(os.path.dirname(input_file.name)) reader = file_reader(parent_dir) - raw_keys_manifest = json.loads(reader(raw_manifest["keys"]).decode(ENCODING)) + keys_uri = raw_manifest["keys"] + keys_filename = keys_uri.replace("file://", "") + print(f"{parent_dir=}") + print(f"{input_file=}") + print(f"{keys_uri=}") + print(f"{parent_dir+keys_uri=}") + print(f"{os.path.join(parent_dir, keys_uri)=}") + joined = os.path.join(parent_dir, keys_filename) + raw_keys_manifest = json.loads(reader(keys_uri).decode(ENCODING)) keys = KeysManifest.from_manifest_spec(raw_keys_manifest) plaintexts = cls._generate_plaintexts(raw_manifest["plaintexts"]) tests = {} + # For some bizarre reason, the for name, scenario in raw_manifest["tests"].items(): try: tests[name] = MessageDecryptionTestScenarioGenerator.from_scenario( - scenario=scenario, keys=keys, plaintexts=plaintexts + scenario=scenario, keys=keys, plaintexts=plaintexts, keyrings=keyrings, keys_uri=joined, ) except NotImplementedError: continue - return cls(version=raw_manifest["manifest"]["version"], keys=keys, plaintexts=plaintexts, tests=tests) + return cls(version=raw_manifest["manifest"]["version"], keys=keys, plaintexts=plaintexts, tests=tests, keyrings=keyrings) def run_and_write_to_dir(self, target_directory, json_indent=None): # type: (str, Optional[int]) -> None diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index c77fed1ce..b22071b95 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -39,6 +39,13 @@ except ImportError: from aws_encryption_sdk.identifiers import Algorithm as AlgorithmSuite +try: + import aws_cryptographic_materialproviders +except ImportError as e: + print("IMPORT OOPS") + print(e) + +from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_provider_from_master_key_specs try: # Python 3.5.0 and 3.5.1 have incompatible typing modules from typing import IO, Callable, Dict, Iterable, Optional # noqa pylint: disable=unused-import @@ -76,6 +83,54 @@ class MessageEncryptionTestScenario(object): algorithm = attr.ib(validator=attr.validators.instance_of(AlgorithmSuite)) frame_size = attr.ib(validator=attr.validators.instance_of(int)) encryption_context = attr.ib(validator=dictionary_validator(six.string_types, six.string_types)) + master_key = attr.ib(validator=attr.validators.instance_of(bool)) + + @classmethod + def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): + # type: (ENCRYPT_SCENARIO_SPEC, KeysManifest, Dict[str, bytes], bool) -> MessageEncryptionTestScenario + """Load from a scenario specification. + + :param dict scenario: Scenario specification JSON + :param KeysManifest keys: Loaded keys + :param dict plaintexts: Mapping of plaintext names to plaintext values + :param bool keyrings: True if should encrypt with master key interfaces; False otherwise + :return: Loaded test scenario + :rtype: MessageEncryptionTestScenario + """ + + if keyrings: + print("KEYRINGS") + return MessageEncryptionWithKeyringsTestScenario.from_scenario( + scenario, keys_uri, plaintexts + ) + else: + return MessageEncryptionWithMasterKeysTestScenario.from_scenario( + scenario, keys, plaintexts + ) + + def run(self, materials_manager=None): + """Run this scenario, writing the resulting ciphertext with ``ciphertext_writer`` and returning + a :class:`MessageDecryptionTestScenario` that describes the matching decrypt scenario. + + :param callable ciphertext_writer: Callable that will write the requested named ciphertext and + return a URI locating the written data + :param str plaintext_uri: URI locating the written plaintext data for this scenario + :return: Decrypt test scenario that describes the generated scenario + :rtype: MessageDecryptionTestScenario + """ + raise NotImplementedError("MUST specify keyrings bool") + + +@attr.s +class MessageEncryptionWithMasterKeysTestScenario(MessageEncryptionTestScenario): + # pylint: disable=too-many-instance-attributes + """Data class for a single full message decrypt test scenario that uses master keys. + + :param master_key_specs: Iterable of loaded master key specifications + :type master_key_specs: iterable of :class:`MasterKeySpec` + :param Callable master_key_provider_fn: + """ + master_key_specs = attr.ib(validator=iterable_validator(list, MasterKeySpec)) master_key_provider_fn = attr.ib(validator=attr.validators.is_callable()) @@ -102,6 +157,7 @@ def master_key_provider_fn(): algorithm=algorithm, frame_size=scenario["frame-size"], encryption_context=scenario["encryption-context"], + master_key=True, master_key_specs=master_key_specs, master_key_provider_fn=master_key_provider_fn, ) @@ -134,6 +190,83 @@ def run(self, materials_manager=None): ciphertext, _header = client.encrypt(**encrypt_kwargs) return ciphertext +@attr.s +class MessageEncryptionWithKeyringsTestScenario(MessageEncryptionTestScenario): + # pylint: disable=too-many-instance-attributes + """Data class for a single full message decrypt test scenario that uses keyrings. + + :param master_key_specs: Iterable of loaded master key specifications + :type master_key_specs: iterable of :class:`MasterKeySpec` + :param Callable master_key_provider_fn: + """ + + master_key_specs = attr.ib(validator=iterable_validator(list, MasterKeySpec)) + master_key_provider_fn = attr.ib(validator=attr.validators.is_callable()) + + @classmethod + def from_scenario(cls, scenario, keys_uri, plaintexts): + print("FROM_SCENARIO") + print(f"{len(scenario['master-keys'])=}") + # type: (ENCRYPT_SCENARIO_SPEC, KeysManifest, Dict[str, bytes]) -> MessageEncryptionTestScenario + """Load from a scenario specification. + + :param dict scenario: Scenario specification JSON + :param KeysManifest keys: Loaded keys + :param dict plaintexts: Mapping of plaintext names to plaintext values + :return: Loaded test scenario + :rtype: MessageEncryptionTestScenario + """ + print("1") + algorithm = algorithm_suite_from_string_id(scenario["algorithm"]) + print("2") + # manifest still keys these as `master-keys` even though these are keyrings + try: + master_key_specs = [KeyringSpec.from_scenario(spec) for spec in scenario["master-keys"]] + except Exception as e: + print(e) + + def keyring_provider_fn(): + return keyring_provider_from_master_key_specs(keys_uri, master_key_specs) + + return cls( + plaintext_name=scenario["plaintext"], + plaintext=plaintexts[scenario["plaintext"]], + algorithm=algorithm, + frame_size=scenario["frame-size"], + encryption_context=scenario["encryption-context"], + master_key=True, + master_key_specs=master_key_specs, + master_key_provider_fn=keyring_provider_fn, + ) + + def run(self, materials_manager=None): + """Run this scenario, writing the resulting ciphertext with ``ciphertext_writer`` and returning + a :class:`MessageDecryptionTestScenario` that describes the matching decrypt scenario. + + :param callable ciphertext_writer: Callable that will write the requested named ciphertext and + return a URI locating the written data + :param str plaintext_uri: URI locating the written plaintext data for this scenario + :return: Decrypt test scenario that describes the generated scenario + :rtype: MessageDecryptionTestScenario + """ + commitment_policy = CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT + if self.algorithm.is_committing(): + commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT + + client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=commitment_policy) + print(f"{self.algorithm=}") + encrypt_kwargs = dict( + source=self.plaintext, + algorithm=self.algorithm, + frame_length=self.frame_size, + encryption_context=self.encryption_context, + ) + if materials_manager: + encrypt_kwargs["materials_manager"] = materials_manager + else: + encrypt_kwargs["keyring"] = self.keyring_provider_fn() + ciphertext, _header = client.encrypt(**encrypt_kwargs) + return ciphertext @attr.s class MessageEncryptionManifest(object): From 488bdda1180c25cafa674fe5888be1bda32a7d2b Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 6 Mar 2024 15:57:26 -0800 Subject: [PATCH 157/422] cleanup --- .../materials_managers/mpl/cmm.py | 2 - .../commands/full_message_decrypt.py | 10 +++ .../commands/full_message_decrypt_generate.py | 9 ++ .../commands/full_message_encrypt.py | 10 +++ .../manifests/full_message/decrypt.py | 57 +++++++------ .../full_message/decrypt_generation.py | 84 +++++++++++-------- .../manifests/full_message/encrypt.py | 11 +-- 7 files changed, 117 insertions(+), 66 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index 56c936012..3749dde97 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -115,8 +115,6 @@ def decrypt_materials( try: mpl_input: 'MPL_DecryptMaterialsInput' = \ CryptoMaterialsManagerFromMPL._create_mpl_decrypt_materials_input_from_request(request) - print(f"{mpl_input=}") - print(f"{self.mpl_cmm._impl.__dict__=}") mpl_output: 'MPL_DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(mpl_input) return DecryptionMaterialsFromMPL(mpl_output.decryption_materials) except AwsCryptographicMaterialProvidersException as mpl_exception: diff --git a/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt.py b/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt.py index ad457fbd3..f28354f31 100644 --- a/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt.py @@ -15,6 +15,13 @@ from awses_test_vectors.manifests.full_message.decrypt import MessageDecryptionManifest +try: + import aws_cryptographic_materialproviders # noqa pylint: disable=unused-import + _HAS_MPL = True +except Exception as e: + _HAS_MPL = False + + try: # Python 3.5.0 and 3.5.1 have incompatible typing modules from typing import Iterable, Optional # noqa pylint: disable=unused-import except ImportError: # pragma: no cover @@ -39,6 +46,9 @@ def cli(args=None): parsed = parser.parse_args(args) + if parsed.keyrings and not _HAS_MPL: + raise ImportError("The --keyrings flag requires the aws-cryptographic-material-providers library.") + decrypt_manifest = MessageDecryptionManifest.from_file(parsed.input, parsed.keyrings) decrypt_manifest.run() diff --git a/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt_generate.py b/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt_generate.py index 9d5c9e1fa..ae6afa538 100644 --- a/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt_generate.py +++ b/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt_generate.py @@ -15,6 +15,12 @@ from awses_test_vectors.manifests.full_message.decrypt_generation import MessageDecryptionGenerationManifest +try: + import aws_cryptographic_materialproviders # noqa pylint: disable=unused-import + _HAS_MPL = True +except Exception as e: + _HAS_MPL = False + try: # Python 3.5.0 and 3.5.1 have incompatible typing modules from typing import Iterable, Optional # noqa pylint: disable=unused-import except ImportError: # pragma: no cover @@ -49,6 +55,9 @@ def cli(args=None): parsed = parser.parse_args(args) + if parsed.keyrings and not _HAS_MPL: + raise ImportError("The --keyrings flag requires the aws-cryptographic-material-providers library.") + encrypt_manifest = MessageDecryptionGenerationManifest.from_file(parsed.input, parsed.keyrings) encrypt_manifest.run_and_write_to_dir(target_directory=parsed.output, json_indent=parsed.json_indent) diff --git a/test_vector_handlers/src/awses_test_vectors/commands/full_message_encrypt.py b/test_vector_handlers/src/awses_test_vectors/commands/full_message_encrypt.py index 2b8b92f3c..6bea002dc 100644 --- a/test_vector_handlers/src/awses_test_vectors/commands/full_message_encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/commands/full_message_encrypt.py @@ -15,6 +15,13 @@ from awses_test_vectors.manifests.full_message.encrypt import MessageEncryptionManifest +try: + import aws_cryptographic_materialproviders # noqa pylint: disable=unused-import + _HAS_MPL = True +except Exception as e: + _HAS_MPL = False + + try: # Python 3.5.0 and 3.5.1 have incompatible typing modules from typing import Iterable, Optional # noqa pylint: disable=unused-import except ImportError: # pragma: no cover @@ -32,6 +39,9 @@ def cli(args=None): parsed = parser.parse_args(args) + if parsed.keyrings and not _HAS_MPL: + raise ImportError("The --keyrings flag requires the aws-cryptographic-material-providers library.") + encrypt_manifest = MessageEncryptionManifest.from_file(parsed.input) encrypt_manifest.run() diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index dd09dc440..797eadf67 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -35,7 +35,13 @@ from awses_test_vectors.manifests.keys import KeysManifest from awses_test_vectors.manifests.master_key import MasterKeySpec, master_key_provider_from_master_key_specs -from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_provider_from_master_key_specs +try: + from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_from_master_key_specs + + _HAS_MPL = True + +except ImportError as e: + _HAS_MPL = False try: # Python 3.5.0 and 3.5.1 have incompatible typing modules @@ -195,6 +201,7 @@ class MessageDecryptionTestScenario(object): :param master_key_specs: Iterable of master key specifications :type master_key_specs: iterable of :class:`MasterKeySpec` :param Callable master_key_provider_fn: + :param bool keyrings: True if should decrypt with keyring interfaces; False otherwise :param str description: Description of test scenario (optional) """ @@ -260,16 +267,15 @@ def from_scenario( :rtype: MessageDecryptionTestScenario """ raw_master_key_specs = scenario["master-keys"] # type: Iterable[MASTER_KEY_SPEC] - if keyrings: - master_key_specs = [KeyringSpec.from_scenario(spec) for spec in raw_master_key_specs] - else: - master_key_specs = [MasterKeySpec.from_scenario(spec) for spec in raw_master_key_specs] - - print(f"{master_key_specs=}") + master_key_specs = [MasterKeySpec.from_scenario(spec) for spec in raw_master_key_specs] + # if keyrings: + # master_key_specs = [KeyringSpec.from_scenario(spec) for spec in raw_master_key_specs] + # else: + # master_key_specs = [MasterKeySpec.from_scenario(spec) for spec in raw_master_key_specs] def master_key_provider_fn(): if keyrings: - return keyring_provider_from_master_key_specs(keys_uri, master_key_specs) + return keyring_from_master_key_specs(keys_uri, master_key_specs) else: return master_key_provider_from_master_key_specs(keys, master_key_specs) @@ -309,28 +315,30 @@ def scenario_spec(self): return spec def _one_shot_decrypt(self): - keyring = self.master_key_provider_fn() - print(f"{keyring=}") client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) if self.keyrings: - return client.decrypt(source=self.ciphertext, keyring=keyring) + return client.decrypt(source=self.ciphertext, keyring=self.master_key_provider_fn()) else: return client.decrypt(source=self.ciphertext, key_provider=self.master_key_provider_fn()) def _streaming_decrypt(self): result = bytearray() client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + + kwargs = { + "source": self.ciphertext, + "mode": "d" + } if self.keyrings: - with client.stream(source=self.ciphertext, mode="d", keyring=self.master_key_provider_fn()) as decryptor: - for chunk in decryptor: - result.extend(chunk) - return result, decryptor.header + kwargs["keyring"] = self.master_key_provider_fn() else: - with client.stream(source=self.ciphertext, mode="d", key_provider=self.master_key_provider_fn()) as decryptor: - for chunk in decryptor: - result.extend(chunk) - return result, decryptor.header + kwargs["key_provider"] = self.master_key_provider_fn() + with client.stream(**kwargs) as decryptor: + for chunk in decryptor: + result.extend(chunk) + return result, decryptor.header + def _streaming_decrypt_unsigned(self): result = bytearray() client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) @@ -421,7 +429,7 @@ def from_file(cls, input_file, keyrings): """Load from a file containing a full message decrypt manifest. :param file input_file: File object for file containing JSON manifest - :param bool keyrings: True if should encrypt with keyring interfaces; False otherwise + :param bool keyrings: True if should decrypt with keyring interfaces; False otherwise :return: Loaded manifest :rtype: MessageDecryptionManifest """ @@ -436,9 +444,10 @@ def from_file(cls, input_file, keyrings): version = raw_manifest["manifest"]["version"] # type: int keys_uri = raw_manifest["keys"] # type: str + # MPL TestVector keyring needs to know the path to the keys file keys_uri = raw_manifest["keys"] keys_filename = keys_uri.replace("file://", "") - joined = os.path.join(parent_dir, keys_filename) + keys_abs_path = os.path.join(parent_dir, keys_filename) raw_keys_manifest = json.loads(root_reader(keys_uri).decode(ENCODING)) keys = KeysManifest.from_manifest_spec(raw_keys_manifest) @@ -453,7 +462,7 @@ def from_file(cls, input_file, keyrings): ciphertext_reader=root_reader, keys=keys, keyrings=False, - keys_uri=joined, + keys_uri=keys_abs_path, ) for name, scenario in raw_scenarios.items() } @@ -467,11 +476,11 @@ def from_file(cls, input_file, keyrings): ciphertext_reader=root_reader, keys=keys, keyrings=True, - keys_uri=joined, + keys_uri=keys_abs_path, ) for name, scenario in raw_scenarios.items() } - # Merge into test_scenarios + # Merge keyring scenarios into test_scenarios test_scenarios = {**keyrings_test_scenarios, **test_scenarios} return cls( diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 040ec07d8..61a62dd22 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -28,18 +28,23 @@ from aws_encryption_sdk.materials_managers.caching import CachingCryptoMaterialsManager from aws_encryption_sdk.materials_managers.default import DefaultCryptoMaterialsManager -from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders -from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig -from aws_cryptographic_materialproviders.mpl.references import ( - IKeyring, - CryptographicMaterialsManager, -) -from aws_cryptographic_materialproviders.mpl.models import ( - CreateDefaultCryptographicMaterialsManagerInput, -) -from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL +try: + from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders + from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig + from aws_cryptographic_materialproviders.mpl.references import ( + IKeyring, + CryptographicMaterialsManager, + ) + from aws_cryptographic_materialproviders.mpl.models import ( + CreateDefaultCryptographicMaterialsManagerInput, + ) + from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL -from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_provider_from_master_key_specs + from awses_test_vectors.manifests.mpl_keyring import keyring_from_master_key_specs + + _HAS_MPL = True +except ImportError as e: + _HAS_MPL = False from awses_test_vectors.internal.defaults import ENCODING @@ -107,23 +112,23 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, pl return: a list of (ciphertext, result) pairs """ - tmp = generation_scenario.encryption_scenario.master_key_provider_fn() - if isinstance(tmp, MasterKeyProvider): + key_provider = generation_scenario.encryption_scenario.master_key_provider_fn() + if isinstance(key_provider, MasterKeyProvider): materials_manager = DefaultCryptoMaterialsManager( - tmp + key_provider ) - elif isinstance(tmp, IKeyring): + elif isinstance(key_provider, IKeyring): mpl = AwsCryptographicMaterialProviders(MaterialProvidersConfig()) mpl_cmm = mpl.create_default_cryptographic_materials_manager( CreateDefaultCryptographicMaterialsManagerInput( - keyring=tmp + keyring=key_provider ) ) materials_manager = CryptoMaterialsManagerFromMPL( mpl_cmm=mpl_cmm ) else: - raise ValueError(f"Unrecognized master_key_provider_fn return type: {str(tmp)}") + raise ValueError(f"Unrecognized master_key_provider_fn return type: {str(key_provider)}") ciphertext_to_decrypt = generation_scenario.encryption_scenario.run(materials_manager) if generation_scenario.result: expected_result = generation_scenario.result @@ -355,6 +360,7 @@ class MessageDecryptionTestScenarioGenerator(object): :type decryption_master_key_specs: iterable of :class:`MasterKeySpec` :param Callable decryption_master_key_provider_fn: :param result: + :param bool keyrings: True if should encrypt with keyring interfaces; False otherwise """ encryption_scenario = attr.ib(validator=attr.validators.instance_of(MessageEncryptionTestScenario)) @@ -373,28 +379,39 @@ def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): :param KeysManifest keys: Loaded keys :param dict plaintexts: Mapping of plaintext names to plaintext values :param bool keyrings: True if should encrypt with keyring interfaces; False otherwise + :param string keys_uri: Filepath to keys manifest. Used by MPL TestVector keyring constructor. :return: Loaded test scenario :rtype: MessageDecryptionTestScenarioGenerator """ encryption_scenario_spec = scenario["encryption-scenario"] - encryption_scenario = MessageEncryptionTestScenario.from_scenario(encryption_scenario_spec, keys, plaintexts, keyrings, keys_uri) + encryption_scenario = MessageEncryptionTestScenario.from_scenario( + encryption_scenario_spec, + keys, + plaintexts, + keyrings, + keys_uri, + ) tampering = scenario.get("tampering") tampering_method = TamperingMethod.from_tampering_spec(tampering) decryption_method_spec = scenario.get("decryption-method") decryption_method = DecryptionMethod(decryption_method_spec) if decryption_method_spec else None if "decryption-master-keys" in scenario: - if keyrings: - decryption_master_key_specs = [ - KeyringSpec.from_scenario(spec) for spec in scenario["decryption-master-keys"] - ] - else: - decryption_master_key_specs = [ - MasterKeySpec.from_scenario(spec) for spec in scenario["decryption-master-keys"] - ] + decryption_master_key_specs = [ + MasterKeySpec.from_scenario(spec) for spec in scenario["decryption-master-keys"] + ] + + # if keyrings: + # decryption_master_key_specs = [ + # KeyringSpec.from_scenario(spec) for spec in scenario["decryption-master-keys"] + # ] + # else: + # decryption_master_key_specs = [ + # MasterKeySpec.from_scenario(spec) for spec in scenario["decryption-master-keys"] + # ] def decryption_master_key_provider_fn(): if keyrings: - return keyring_provider_from_master_key_specs(keys_uri, decryption_master_key_specs) + return keyring_from_master_key_specs(keys_uri, decryption_master_key_specs) else: return master_key_provider_from_master_key_specs(keys, decryption_master_key_specs) @@ -493,23 +510,20 @@ def from_file(cls, input_file, keyrings): parent_dir = os.path.abspath(os.path.dirname(input_file.name)) reader = file_reader(parent_dir) + + # MPL TestVector keyring needs to know the path to the keys file keys_uri = raw_manifest["keys"] keys_filename = keys_uri.replace("file://", "") - print(f"{parent_dir=}") - print(f"{input_file=}") - print(f"{keys_uri=}") - print(f"{parent_dir+keys_uri=}") - print(f"{os.path.join(parent_dir, keys_uri)=}") - joined = os.path.join(parent_dir, keys_filename) + keys_abs_path = os.path.join(parent_dir, keys_filename) + raw_keys_manifest = json.loads(reader(keys_uri).decode(ENCODING)) keys = KeysManifest.from_manifest_spec(raw_keys_manifest) plaintexts = cls._generate_plaintexts(raw_manifest["plaintexts"]) tests = {} - # For some bizarre reason, the for name, scenario in raw_manifest["tests"].items(): try: tests[name] = MessageDecryptionTestScenarioGenerator.from_scenario( - scenario=scenario, keys=keys, plaintexts=plaintexts, keyrings=keyrings, keys_uri=joined, + scenario=scenario, keys=keys, plaintexts=plaintexts, keyrings=keyrings, keys_uri=keys_abs_path, ) except NotImplementedError: continue diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index b22071b95..4e0edf0ca 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -40,12 +40,13 @@ from aws_encryption_sdk.identifiers import Algorithm as AlgorithmSuite try: - import aws_cryptographic_materialproviders + from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_from_master_key_specs + + _HAS_MPL = True + except ImportError as e: - print("IMPORT OOPS") - print(e) + _HAS_MPL = False -from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_provider_from_master_key_specs try: # Python 3.5.0 and 3.5.1 have incompatible typing modules from typing import IO, Callable, Dict, Iterable, Optional # noqa pylint: disable=unused-import @@ -226,7 +227,7 @@ def from_scenario(cls, scenario, keys_uri, plaintexts): print(e) def keyring_provider_fn(): - return keyring_provider_from_master_key_specs(keys_uri, master_key_specs) + return keyring_from_master_key_specs(keys_uri, master_key_specs) return cls( plaintext_name=scenario["plaintext"], From cea9dab5cd32875aa21e8cd0e4a5bf0f34e7de34 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 6 Mar 2024 16:16:07 -0800 Subject: [PATCH 158/422] cleanup: --- .../manifests/full_message/decrypt.py | 4 ++-- .../manifests/full_message/encrypt.py | 11 +---------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index 797eadf67..6336400ce 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -36,7 +36,7 @@ from awses_test_vectors.manifests.master_key import MasterKeySpec, master_key_provider_from_master_key_specs try: - from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_from_master_key_specs + from awses_test_vectors.manifests.mpl_keyring import keyring_from_master_key_specs _HAS_MPL = True @@ -338,7 +338,7 @@ def _streaming_decrypt(self): for chunk in decryptor: result.extend(chunk) return result, decryptor.header - + def _streaming_decrypt_unsigned(self): result = bytearray() client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 4e0edf0ca..66c729056 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -100,7 +100,6 @@ def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): """ if keyrings: - print("KEYRINGS") return MessageEncryptionWithKeyringsTestScenario.from_scenario( scenario, keys_uri, plaintexts ) @@ -206,8 +205,6 @@ class MessageEncryptionWithKeyringsTestScenario(MessageEncryptionTestScenario): @classmethod def from_scenario(cls, scenario, keys_uri, plaintexts): - print("FROM_SCENARIO") - print(f"{len(scenario['master-keys'])=}") # type: (ENCRYPT_SCENARIO_SPEC, KeysManifest, Dict[str, bytes]) -> MessageEncryptionTestScenario """Load from a scenario specification. @@ -217,14 +214,9 @@ def from_scenario(cls, scenario, keys_uri, plaintexts): :return: Loaded test scenario :rtype: MessageEncryptionTestScenario """ - print("1") algorithm = algorithm_suite_from_string_id(scenario["algorithm"]) - print("2") # manifest still keys these as `master-keys` even though these are keyrings - try: - master_key_specs = [KeyringSpec.from_scenario(spec) for spec in scenario["master-keys"]] - except Exception as e: - print(e) + master_key_specs = [KeyringSpec.from_scenario(spec) for spec in scenario["master-keys"]] def keyring_provider_fn(): return keyring_from_master_key_specs(keys_uri, master_key_specs) @@ -255,7 +247,6 @@ def run(self, materials_manager=None): commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=commitment_policy) - print(f"{self.algorithm=}") encrypt_kwargs = dict( source=self.plaintext, algorithm=self.algorithm, From 98fd0af3479678bde4457ea6a4d713c52cc4097d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 09:06:06 -0800 Subject: [PATCH 159/422] fix testvector run --- .../awses_test_vectors/commands/full_message_encrypt.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test_vector_handlers/src/awses_test_vectors/commands/full_message_encrypt.py b/test_vector_handlers/src/awses_test_vectors/commands/full_message_encrypt.py index 6bea002dc..1e0484276 100644 --- a/test_vector_handlers/src/awses_test_vectors/commands/full_message_encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/commands/full_message_encrypt.py @@ -36,6 +36,13 @@ def cli(args=None): parser.add_argument( "--input", required=True, type=argparse.FileType("r"), help="Existing full message encrypt manifest" ) + parser.add_argument( + "--keyrings", + action="store_true", + required=False, + default=False, + help="Use keyring interfaces to encrypt", + ) parsed = parser.parse_args(args) From b4ba23c5b7912a7da74d40da1887ba668f4168a0 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 09:12:14 -0800 Subject: [PATCH 160/422] rm module_ --- examples/src/keyrings/module_.py | 1 - examples/src/module_.py | 1 - 2 files changed, 2 deletions(-) delete mode 100644 examples/src/keyrings/module_.py delete mode 100644 examples/src/module_.py diff --git a/examples/src/keyrings/module_.py b/examples/src/keyrings/module_.py deleted file mode 100644 index 3e8d3062a..000000000 --- a/examples/src/keyrings/module_.py +++ /dev/null @@ -1 +0,0 @@ -"""Should remove this once PYTHONPATH issues are resolved by adding doo files.""" diff --git a/examples/src/module_.py b/examples/src/module_.py deleted file mode 100644 index 3e8d3062a..000000000 --- a/examples/src/module_.py +++ /dev/null @@ -1 +0,0 @@ -"""Should remove this once PYTHONPATH issues are resolved by adding doo files.""" From 3e6dfa1b8f578b9b4fb2e73e73d2b1880dab2a0c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 09:28:47 -0800 Subject: [PATCH 161/422] kwargify input --- .../materials_managers/mpl/cmm.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index 3749dde97..c454a114e 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -81,13 +81,19 @@ def _native_to_mpl_get_encryption_materials( commitment_policy = CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy( request.commitment_policy ) + mpl_input_kwargs = { + "encryption_context": request.encryption_context, + "commitment_policy": commitment_policy, + "max_plaintext_length": request.plaintext_length, + } + if request.algorithm is not None: + mpl_input_kwargs["algorithm_suite_id"] = \ + CryptoMaterialsManagerFromMPL._native_algorithm_id_to_mpl_algorithm_id( + request.algorithm.algorithm_id + ) + output: MPL_GetEncryptionMaterialsInput = MPL_GetEncryptionMaterialsInput( - encryption_context=request.encryption_context, - commitment_policy=commitment_policy, - max_plaintext_length=request.plaintext_length, - algorithm_suite_id=CryptoMaterialsManagerFromMPL._native_algorithm_id_to_mpl_algorithm_id( - request.algorithm.algorithm_id - ) + **mpl_input_kwargs ) return output From bcc689ce8723158e41de963d71c9e4eb1772b425 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 09:40:44 -0800 Subject: [PATCH 162/422] fix testvector --- .../commands/full_message_encrypt.py | 2 +- .../manifests/full_message/encrypt.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/commands/full_message_encrypt.py b/test_vector_handlers/src/awses_test_vectors/commands/full_message_encrypt.py index 1e0484276..5294e1791 100644 --- a/test_vector_handlers/src/awses_test_vectors/commands/full_message_encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/commands/full_message_encrypt.py @@ -49,6 +49,6 @@ def cli(args=None): if parsed.keyrings and not _HAS_MPL: raise ImportError("The --keyrings flag requires the aws-cryptographic-material-providers library.") - encrypt_manifest = MessageEncryptionManifest.from_file(parsed.input) + encrypt_manifest = MessageEncryptionManifest.from_file(parsed.input, parsed.keyrings) encrypt_manifest.run() diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 66c729056..eabbe7343 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -290,11 +290,12 @@ def _generate_plaintexts(plaintexts_specs): return {name: os.urandom(size) for name, size in plaintexts_specs.items()} @classmethod - def from_file(cls, input_file): + def from_file(cls, input_file, keyrings): # type: (IO) -> MessageEncryptionManifest """Load frome a file containing a full message encrypt manifest. :param file input_file: File object for file containing JSON manifest + :param bool keyrings: True if should encrypt with keyring interfaces; False otherwise :return: Loaded manifest :rtype: MessageEncryptionManifest """ @@ -305,14 +306,20 @@ def from_file(cls, input_file): parent_dir = os.path.abspath(os.path.dirname(input_file.name)) reader = file_reader(parent_dir) - raw_keys_manifest = json.loads(reader(raw_manifest["keys"]).decode(ENCODING)) + + # MPL TestVector keyring needs to know the path to the keys file + keys_uri = raw_manifest["keys"] + keys_filename = keys_uri.replace("file://", "") + keys_abs_path = os.path.join(parent_dir, keys_filename) + + raw_keys_manifest = json.loads(reader(keys_uri).decode(ENCODING)) keys = KeysManifest.from_manifest_spec(raw_keys_manifest) plaintexts = cls._generate_plaintexts(raw_manifest["plaintexts"]) tests = {} for name, scenario in raw_manifest["tests"].items(): try: tests[name] = MessageEncryptionTestScenario.from_scenario( - scenario=scenario, keys=keys, plaintexts=plaintexts + scenario=scenario, keys=keys, plaintexts=plaintexts, keyrings=keyrings, keys_uri=keys_abs_path ) except NotImplementedError: continue From ca3e1654cdc0f926c4715a030a827b341ade072c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 10:06:04 -0800 Subject: [PATCH 163/422] cleanup --- .../materials_managers/mpl/cmm.py | 2 +- .../commands/full_message_decrypt.py | 4 +- .../commands/full_message_decrypt_generate.py | 4 +- .../commands/full_message_encrypt.py | 4 +- .../manifests/full_message/encrypt.py | 264 ++++++++++-------- 5 files changed, 154 insertions(+), 124 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index c454a114e..ebef5f7ac 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -90,7 +90,7 @@ def _native_to_mpl_get_encryption_materials( mpl_input_kwargs["algorithm_suite_id"] = \ CryptoMaterialsManagerFromMPL._native_algorithm_id_to_mpl_algorithm_id( request.algorithm.algorithm_id - ) + ) output: MPL_GetEncryptionMaterialsInput = MPL_GetEncryptionMaterialsInput( **mpl_input_kwargs diff --git a/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt.py b/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt.py index f28354f31..2a44cd597 100644 --- a/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt.py @@ -16,9 +16,9 @@ from awses_test_vectors.manifests.full_message.decrypt import MessageDecryptionManifest try: - import aws_cryptographic_materialproviders # noqa pylint: disable=unused-import + import aws_cryptographic_materialproviders # noqa pylint: disable=unused-import,import-error _HAS_MPL = True -except Exception as e: +except ImportError: _HAS_MPL = False diff --git a/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt_generate.py b/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt_generate.py index ae6afa538..69fe44d78 100644 --- a/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt_generate.py +++ b/test_vector_handlers/src/awses_test_vectors/commands/full_message_decrypt_generate.py @@ -16,9 +16,9 @@ from awses_test_vectors.manifests.full_message.decrypt_generation import MessageDecryptionGenerationManifest try: - import aws_cryptographic_materialproviders # noqa pylint: disable=unused-import + import aws_cryptographic_materialproviders # noqa pylint: disable=unused-import,import-error _HAS_MPL = True -except Exception as e: +except ImportError: _HAS_MPL = False try: # Python 3.5.0 and 3.5.1 have incompatible typing modules diff --git a/test_vector_handlers/src/awses_test_vectors/commands/full_message_encrypt.py b/test_vector_handlers/src/awses_test_vectors/commands/full_message_encrypt.py index 5294e1791..268d7ca99 100644 --- a/test_vector_handlers/src/awses_test_vectors/commands/full_message_encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/commands/full_message_encrypt.py @@ -16,9 +16,9 @@ from awses_test_vectors.manifests.full_message.encrypt import MessageEncryptionManifest try: - import aws_cryptographic_materialproviders # noqa pylint: disable=unused-import + import aws_cryptographic_materialproviders # noqa pylint: disable=unused-import,import-error _HAS_MPL = True -except Exception as e: +except ImportError: _HAS_MPL = False diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index eabbe7343..1323fce88 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -88,60 +88,14 @@ class MessageEncryptionTestScenario(object): @classmethod def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): - # type: (ENCRYPT_SCENARIO_SPEC, KeysManifest, Dict[str, bytes], bool) -> MessageEncryptionTestScenario - """Load from a scenario specification. - - :param dict scenario: Scenario specification JSON - :param KeysManifest keys: Loaded keys - :param dict plaintexts: Mapping of plaintext names to plaintext values - :param bool keyrings: True if should encrypt with master key interfaces; False otherwise - :return: Loaded test scenario - :rtype: MessageEncryptionTestScenario - """ - - if keyrings: - return MessageEncryptionWithKeyringsTestScenario.from_scenario( - scenario, keys_uri, plaintexts - ) - else: - return MessageEncryptionWithMasterKeysTestScenario.from_scenario( - scenario, keys, plaintexts - ) - - def run(self, materials_manager=None): - """Run this scenario, writing the resulting ciphertext with ``ciphertext_writer`` and returning - a :class:`MessageDecryptionTestScenario` that describes the matching decrypt scenario. - - :param callable ciphertext_writer: Callable that will write the requested named ciphertext and - return a URI locating the written data - :param str plaintext_uri: URI locating the written plaintext data for this scenario - :return: Decrypt test scenario that describes the generated scenario - :rtype: MessageDecryptionTestScenario - """ - raise NotImplementedError("MUST specify keyrings bool") - - -@attr.s -class MessageEncryptionWithMasterKeysTestScenario(MessageEncryptionTestScenario): - # pylint: disable=too-many-instance-attributes - """Data class for a single full message decrypt test scenario that uses master keys. - - :param master_key_specs: Iterable of loaded master key specifications - :type master_key_specs: iterable of :class:`MasterKeySpec` - :param Callable master_key_provider_fn: - """ - - master_key_specs = attr.ib(validator=iterable_validator(list, MasterKeySpec)) - master_key_provider_fn = attr.ib(validator=attr.validators.is_callable()) - - @classmethod - def from_scenario(cls, scenario, keys, plaintexts): - # type: (ENCRYPT_SCENARIO_SPEC, KeysManifest, Dict[str, bytes]) -> MessageEncryptionTestScenario + # type: (ENCRYPT_SCENARIO_SPEC, KeysManifest, Dict[str, bytes], bool, str) -> MessageEncryptionTestScenario """Load from a scenario specification. :param dict scenario: Scenario specification JSON :param KeysManifest keys: Loaded keys :param dict plaintexts: Mapping of plaintext names to plaintext values + :param bool keyrings: True if should encrypt with keyring interfaces; False otherwise + :param str keys_uri: Path to the keys manifest :return: Loaded test scenario :rtype: MessageEncryptionTestScenario """ @@ -149,6 +103,8 @@ def from_scenario(cls, scenario, keys, plaintexts): master_key_specs = [MasterKeySpec.from_scenario(spec) for spec in scenario["master-keys"]] def master_key_provider_fn(): + if keyrings: + return keyring_from_master_key_specs(keys_uri, master_key_specs) return master_key_provider_from_master_key_specs(keys, master_key_specs) return cls( @@ -160,6 +116,8 @@ def master_key_provider_fn(): master_key=True, master_key_specs=master_key_specs, master_key_provider_fn=master_key_provider_fn, + keyrings=keyrings, + keys_uri=keys_uri, ) def run(self, materials_manager=None): @@ -185,80 +143,152 @@ def run(self, materials_manager=None): ) if materials_manager: encrypt_kwargs["materials_manager"] = materials_manager + elif self.keyrings: + encrypt_kwargs["keyring"] = self.master_key_provider_fn() else: encrypt_kwargs["key_provider"] = self.master_key_provider_fn() ciphertext, _header = client.encrypt(**encrypt_kwargs) return ciphertext -@attr.s -class MessageEncryptionWithKeyringsTestScenario(MessageEncryptionTestScenario): - # pylint: disable=too-many-instance-attributes - """Data class for a single full message decrypt test scenario that uses keyrings. - - :param master_key_specs: Iterable of loaded master key specifications - :type master_key_specs: iterable of :class:`MasterKeySpec` - :param Callable master_key_provider_fn: - """ - - master_key_specs = attr.ib(validator=iterable_validator(list, MasterKeySpec)) - master_key_provider_fn = attr.ib(validator=attr.validators.is_callable()) - - @classmethod - def from_scenario(cls, scenario, keys_uri, plaintexts): - # type: (ENCRYPT_SCENARIO_SPEC, KeysManifest, Dict[str, bytes]) -> MessageEncryptionTestScenario - """Load from a scenario specification. - - :param dict scenario: Scenario specification JSON - :param KeysManifest keys: Loaded keys - :param dict plaintexts: Mapping of plaintext names to plaintext values - :return: Loaded test scenario - :rtype: MessageEncryptionTestScenario - """ - algorithm = algorithm_suite_from_string_id(scenario["algorithm"]) - # manifest still keys these as `master-keys` even though these are keyrings - master_key_specs = [KeyringSpec.from_scenario(spec) for spec in scenario["master-keys"]] - - def keyring_provider_fn(): - return keyring_from_master_key_specs(keys_uri, master_key_specs) - - return cls( - plaintext_name=scenario["plaintext"], - plaintext=plaintexts[scenario["plaintext"]], - algorithm=algorithm, - frame_size=scenario["frame-size"], - encryption_context=scenario["encryption-context"], - master_key=True, - master_key_specs=master_key_specs, - master_key_provider_fn=keyring_provider_fn, - ) - - def run(self, materials_manager=None): - """Run this scenario, writing the resulting ciphertext with ``ciphertext_writer`` and returning - a :class:`MessageDecryptionTestScenario` that describes the matching decrypt scenario. - :param callable ciphertext_writer: Callable that will write the requested named ciphertext and - return a URI locating the written data - :param str plaintext_uri: URI locating the written plaintext data for this scenario - :return: Decrypt test scenario that describes the generated scenario - :rtype: MessageDecryptionTestScenario - """ - commitment_policy = CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT - if self.algorithm.is_committing(): - commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT - - client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=commitment_policy) - encrypt_kwargs = dict( - source=self.plaintext, - algorithm=self.algorithm, - frame_length=self.frame_size, - encryption_context=self.encryption_context, - ) - if materials_manager: - encrypt_kwargs["materials_manager"] = materials_manager - else: - encrypt_kwargs["keyring"] = self.keyring_provider_fn() - ciphertext, _header = client.encrypt(**encrypt_kwargs) - return ciphertext +# @attr.s +# class MessageEncryptionWithMasterKeysTestScenario(MessageEncryptionTestScenario): +# # pylint: disable=too-many-instance-attributes +# """Data class for a single full message decrypt test scenario that uses master keys. + +# :param master_key_specs: Iterable of loaded master key specifications +# :type master_key_specs: iterable of :class:`MasterKeySpec` +# :param Callable master_key_provider_fn: +# """ + +# master_key_specs = attr.ib(validator=iterable_validator(list, MasterKeySpec)) +# master_key_provider_fn = attr.ib(validator=attr.validators.is_callable()) + +# @classmethod +# def from_scenario(cls, scenario, keys, plaintexts): +# # type: (ENCRYPT_SCENARIO_SPEC, KeysManifest, Dict[str, bytes]) -> MessageEncryptionTestScenario +# """Load from a scenario specification. + +# :param dict scenario: Scenario specification JSON +# :param KeysManifest keys: Loaded keys +# :param dict plaintexts: Mapping of plaintext names to plaintext values +# :return: Loaded test scenario +# :rtype: MessageEncryptionTestScenario +# """ +# algorithm = algorithm_suite_from_string_id(scenario["algorithm"]) +# master_key_specs = [MasterKeySpec.from_scenario(spec) for spec in scenario["master-keys"]] + +# def master_key_provider_fn(): +# return master_key_provider_from_master_key_specs(keys, master_key_specs) + +# return cls( +# plaintext_name=scenario["plaintext"], +# plaintext=plaintexts[scenario["plaintext"]], +# algorithm=algorithm, +# frame_size=scenario["frame-size"], +# encryption_context=scenario["encryption-context"], +# master_key=True, +# master_key_specs=master_key_specs, +# master_key_provider_fn=master_key_provider_fn, +# ) + +# def run(self, materials_manager=None): +# """Run this scenario, writing the resulting ciphertext with ``ciphertext_writer`` and returning +# a :class:`MessageDecryptionTestScenario` that describes the matching decrypt scenario. + +# :param callable ciphertext_writer: Callable that will write the requested named ciphertext and +# return a URI locating the written data +# :param str plaintext_uri: URI locating the written plaintext data for this scenario +# :return: Decrypt test scenario that describes the generated scenario +# :rtype: MessageDecryptionTestScenario +# """ +# commitment_policy = CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT +# if self.algorithm.is_committing(): +# commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT + +# client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=commitment_policy) +# encrypt_kwargs = dict( +# source=self.plaintext, +# algorithm=self.algorithm, +# frame_length=self.frame_size, +# encryption_context=self.encryption_context, +# ) +# if materials_manager: +# encrypt_kwargs["materials_manager"] = materials_manager +# else: +# encrypt_kwargs["key_provider"] = self.master_key_provider_fn() +# ciphertext, _header = client.encrypt(**encrypt_kwargs) +# return ciphertext + +# @attr.s +# class MessageEncryptionWithKeyringsTestScenario(MessageEncryptionTestScenario): +# # pylint: disable=too-many-instance-attributes +# """Data class for a single full message decrypt test scenario that uses keyrings. + +# :param master_key_specs: Iterable of loaded master key specifications +# :type master_key_specs: iterable of :class:`MasterKeySpec` +# :param Callable master_key_provider_fn: +# """ + +# master_key_specs = attr.ib(validator=iterable_validator(list, MasterKeySpec)) +# master_key_provider_fn = attr.ib(validator=attr.validators.is_callable()) + +# @classmethod +# def from_scenario(cls, scenario, keys_uri, plaintexts): +# # type: (ENCRYPT_SCENARIO_SPEC, KeysManifest, Dict[str, bytes]) -> MessageEncryptionTestScenario +# """Load from a scenario specification. + +# :param dict scenario: Scenario specification JSON +# :param KeysManifest keys: Loaded keys +# :param dict plaintexts: Mapping of plaintext names to plaintext values +# :return: Loaded test scenario +# :rtype: MessageEncryptionTestScenario +# """ +# algorithm = algorithm_suite_from_string_id(scenario["algorithm"]) +# # manifest still keys these as `master-keys` even though these are keyrings +# master_key_specs = [KeyringSpec.from_scenario(spec) for spec in scenario["master-keys"]] + +# def keyring_provider_fn(): +# return keyring_from_master_key_specs(keys_uri, master_key_specs) + +# return cls( +# plaintext_name=scenario["plaintext"], +# plaintext=plaintexts[scenario["plaintext"]], +# algorithm=algorithm, +# frame_size=scenario["frame-size"], +# encryption_context=scenario["encryption-context"], +# master_key=True, +# master_key_specs=master_key_specs, +# master_key_provider_fn=keyring_provider_fn, +# ) + +# def run(self, materials_manager=None): +# """Run this scenario, writing the resulting ciphertext with ``ciphertext_writer`` and returning +# a :class:`MessageDecryptionTestScenario` that describes the matching decrypt scenario. + +# :param callable ciphertext_writer: Callable that will write the requested named ciphertext and +# return a URI locating the written data +# :param str plaintext_uri: URI locating the written plaintext data for this scenario +# :return: Decrypt test scenario that describes the generated scenario +# :rtype: MessageDecryptionTestScenario +# """ +# commitment_policy = CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT +# if self.algorithm.is_committing(): +# commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT + +# client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=commitment_policy) +# encrypt_kwargs = dict( +# source=self.plaintext, +# algorithm=self.algorithm, +# frame_length=self.frame_size, +# encryption_context=self.encryption_context, +# ) +# if materials_manager: +# encrypt_kwargs["materials_manager"] = materials_manager +# else: +# encrypt_kwargs["keyring"] = self.keyring_provider_fn() +# ciphertext, _header = client.encrypt(**encrypt_kwargs) +# return ciphertext @attr.s class MessageEncryptionManifest(object): From d6d1493e552bde19f39d86488e23712799c5c7b9 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 10:15:35 -0800 Subject: [PATCH 164/422] cleanup --- .../awses_test_vectors/manifests/full_message/encrypt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 1323fce88..09c2acf6a 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -84,7 +84,9 @@ class MessageEncryptionTestScenario(object): algorithm = attr.ib(validator=attr.validators.instance_of(AlgorithmSuite)) frame_size = attr.ib(validator=attr.validators.instance_of(int)) encryption_context = attr.ib(validator=dictionary_validator(six.string_types, six.string_types)) - master_key = attr.ib(validator=attr.validators.instance_of(bool)) + master_key_specs = attr.ib(validator=iterable_validator(list, MasterKeySpec)) + master_key_provider_fn = attr.ib(validator=attr.validators.is_callable()) + keyrings = attr.ib(validator=attr.validators.instance_of(bool)) @classmethod def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): @@ -113,11 +115,9 @@ def master_key_provider_fn(): algorithm=algorithm, frame_size=scenario["frame-size"], encryption_context=scenario["encryption-context"], - master_key=True, master_key_specs=master_key_specs, master_key_provider_fn=master_key_provider_fn, keyrings=keyrings, - keys_uri=keys_uri, ) def run(self, materials_manager=None): From 68ce94a1feb16a518e00799be465f9c6bd94d048 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 10:23:20 -0800 Subject: [PATCH 165/422] flake8 --- .../manifests/full_message/decrypt.py | 4 +--- .../manifests/full_message/decrypt_generation.py | 15 +++++++++------ .../manifests/full_message/encrypt.py | 9 +++------ 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index 6336400ce..73c5edd65 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -37,10 +37,8 @@ try: from awses_test_vectors.manifests.mpl_keyring import keyring_from_master_key_specs - _HAS_MPL = True - -except ImportError as e: +except ImportError: _HAS_MPL = False diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 61a62dd22..cf5e26ff4 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -33,7 +33,6 @@ from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig from aws_cryptographic_materialproviders.mpl.references import ( IKeyring, - CryptographicMaterialsManager, ) from aws_cryptographic_materialproviders.mpl.models import ( CreateDefaultCryptographicMaterialsManagerInput, @@ -41,10 +40,8 @@ from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL from awses_test_vectors.manifests.mpl_keyring import keyring_from_master_key_specs - - _HAS_MPL = True -except ImportError as e: - _HAS_MPL = False +except ImportError: + pass from awses_test_vectors.internal.defaults import ENCODING @@ -527,7 +524,13 @@ def from_file(cls, input_file, keyrings): ) except NotImplementedError: continue - return cls(version=raw_manifest["manifest"]["version"], keys=keys, plaintexts=plaintexts, tests=tests, keyrings=keyrings) + return cls( + version=raw_manifest["manifest"]["version"], + keys=keys, + plaintexts=plaintexts, + ests=tests, + keyrings=keyrings, + ) def run_and_write_to_dir(self, target_directory, json_indent=None): # type: (str, Optional[int]) -> None diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 09c2acf6a..42c9e14d2 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -40,12 +40,9 @@ from aws_encryption_sdk.identifiers import Algorithm as AlgorithmSuite try: - from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_from_master_key_specs - - _HAS_MPL = True - -except ImportError as e: - _HAS_MPL = False + from awses_test_vectors.manifests.mpl_keyring import keyring_from_master_key_specs +except ImportError: + pass try: # Python 3.5.0 and 3.5.1 have incompatible typing modules From 9269fc4666eca876f0d1e7280906cba8af5e0c58 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 10:30:02 -0800 Subject: [PATCH 166/422] pylint --- .../manifests/full_message/decrypt.py | 13 ++++--------- .../manifests/full_message/decrypt_generation.py | 13 ++----------- .../manifests/full_message/encrypt.py | 1 + 3 files changed, 7 insertions(+), 20 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index 73c5edd65..cb7df63c9 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -188,7 +188,7 @@ class DecryptionMethod(Enum): @attr.s(init=False) class MessageDecryptionTestScenario(object): - # pylint: disable=too-many-arguments + # pylint: disable=too-many-arguments,too-many-instance-attributes """Data class for a single full message decrypt test scenario. Handles serialization and deserialization to and from manifest specs. @@ -266,16 +266,11 @@ def from_scenario( """ raw_master_key_specs = scenario["master-keys"] # type: Iterable[MASTER_KEY_SPEC] master_key_specs = [MasterKeySpec.from_scenario(spec) for spec in raw_master_key_specs] - # if keyrings: - # master_key_specs = [KeyringSpec.from_scenario(spec) for spec in raw_master_key_specs] - # else: - # master_key_specs = [MasterKeySpec.from_scenario(spec) for spec in raw_master_key_specs] def master_key_provider_fn(): if keyrings: return keyring_from_master_key_specs(keys_uri, master_key_specs) - else: - return master_key_provider_from_master_key_specs(keys, master_key_specs) + return master_key_provider_from_master_key_specs(keys, master_key_specs) decryption_method_spec = scenario.get("decryption-method") decryption_method = DecryptionMethod(decryption_method_spec) if decryption_method_spec else None @@ -316,8 +311,7 @@ def _one_shot_decrypt(self): client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) if self.keyrings: return client.decrypt(source=self.ciphertext, keyring=self.master_key_provider_fn()) - else: - return client.decrypt(source=self.ciphertext, key_provider=self.master_key_provider_fn()) + return client.decrypt(source=self.ciphertext, key_provider=self.master_key_provider_fn()) def _streaming_decrypt(self): result = bytearray() @@ -423,6 +417,7 @@ def manifest_spec(self): @classmethod def from_file(cls, input_file, keyrings): + # noqa pylint disable=too-many-locals # type: (IO) -> MessageDecryptionManifest """Load from a file containing a full message decrypt manifest. diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index cf5e26ff4..41453fdb6 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -370,6 +370,7 @@ class MessageDecryptionTestScenarioGenerator(object): @classmethod def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): + # noqa pylint disable=too-many-arguments,too-many-locals """Load from a scenario specification. :param dict scenario: Scenario specification JSON @@ -397,20 +398,10 @@ def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): MasterKeySpec.from_scenario(spec) for spec in scenario["decryption-master-keys"] ] - # if keyrings: - # decryption_master_key_specs = [ - # KeyringSpec.from_scenario(spec) for spec in scenario["decryption-master-keys"] - # ] - # else: - # decryption_master_key_specs = [ - # MasterKeySpec.from_scenario(spec) for spec in scenario["decryption-master-keys"] - # ] - def decryption_master_key_provider_fn(): if keyrings: return keyring_from_master_key_specs(keys_uri, decryption_master_key_specs) - else: - return master_key_provider_from_master_key_specs(keys, decryption_master_key_specs) + return master_key_provider_from_master_key_specs(keys, decryption_master_key_specs) else: decryption_master_key_specs = encryption_scenario.master_key_specs diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 42c9e14d2..2d3c06249 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -87,6 +87,7 @@ class MessageEncryptionTestScenario(object): @classmethod def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): + # noqa pylint disable=too-many-arguments # type: (ENCRYPT_SCENARIO_SPEC, KeysManifest, Dict[str, bytes], bool, str) -> MessageEncryptionTestScenario """Load from a scenario specification. From 0a972e6ce811fcc1ce018980f434060d61739322 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 10:33:43 -0800 Subject: [PATCH 167/422] pylint --- .../src/awses_test_vectors/manifests/full_message/decrypt.py | 2 +- .../manifests/full_message/decrypt_generation.py | 2 +- .../src/awses_test_vectors/manifests/full_message/encrypt.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index cb7df63c9..f5c93058d 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -417,7 +417,7 @@ def manifest_spec(self): @classmethod def from_file(cls, input_file, keyrings): - # noqa pylint disable=too-many-locals + # pylint: disable=too-many-locals # type: (IO) -> MessageDecryptionManifest """Load from a file containing a full message decrypt manifest. diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 41453fdb6..2e434e780 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -370,7 +370,7 @@ class MessageDecryptionTestScenarioGenerator(object): @classmethod def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): - # noqa pylint disable=too-many-arguments,too-many-locals + # pylint: disable=too-many-arguments,too-many-locals """Load from a scenario specification. :param dict scenario: Scenario specification JSON diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 2d3c06249..084981eaf 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -87,7 +87,7 @@ class MessageEncryptionTestScenario(object): @classmethod def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): - # noqa pylint disable=too-many-arguments + # pylint: disable=too-many-arguments # type: (ENCRYPT_SCENARIO_SPEC, KeysManifest, Dict[str, bytes], bool, str) -> MessageEncryptionTestScenario """Load from a scenario specification. From e185c35a61243b4f3bfabaad8c60f2c75e12f1b7 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 10:54:58 -0800 Subject: [PATCH 168/422] fix --- test/mpl/unit/test_material_managers_mpl_cmm.py | 2 +- .../manifests/full_message/decrypt_generation.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/mpl/unit/test_material_managers_mpl_cmm.py b/test/mpl/unit/test_material_managers_mpl_cmm.py index 80d6f00ee..16323d496 100644 --- a/test/mpl/unit/test_material_managers_mpl_cmm.py +++ b/test/mpl/unit/test_material_managers_mpl_cmm.py @@ -96,7 +96,7 @@ def test_GIVEN_valid_request_WHEN_get_encryption_materials_THEN_return_Encryptio @patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL" - "._native_to_mpl_commmitment_policy") + "._native_to_mpl_get_encryption_materials") def test_GIVEN_mpl_cmm_raises_MPLException_WHEN_get_encryption_materials_THEN_raise_ESDKException( _ ): diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 2e434e780..3b0b50a94 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -519,7 +519,7 @@ def from_file(cls, input_file, keyrings): version=raw_manifest["manifest"]["version"], keys=keys, plaintexts=plaintexts, - ests=tests, + tests=tests, keyrings=keyrings, ) From 4f8633786f79a76d6292276882dcd94efe97c44a Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 10:59:57 -0800 Subject: [PATCH 169/422] fix --- test/mpl/unit/test_material_managers_mpl_cmm.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/mpl/unit/test_material_managers_mpl_cmm.py b/test/mpl/unit/test_material_managers_mpl_cmm.py index 16323d496..8701cde00 100644 --- a/test/mpl/unit/test_material_managers_mpl_cmm.py +++ b/test/mpl/unit/test_material_managers_mpl_cmm.py @@ -110,10 +110,13 @@ def test_GIVEN_mpl_cmm_raises_MPLException_WHEN_get_encryption_materials_THEN_ra cmm.get_encryption_materials(mock_encryption_materials_request) +@patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL" + "._native_algorithm_id_to_mpl_algorithm_id") @patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL" "._native_to_mpl_commmitment_policy") def test_GIVEN_valid_mpl_commitment_policy_WHEN_native_to_mpl_get_encryption_materials_THEN_returns_MPL_GetEncryptionMaterialsInput( # noqa: E501 - mock_mpl_commitment_policy + mock_mpl_commitment_policy, + mock_mpl_algorithm, ): # Given: commitment policy is some MPL ESDK commitment policy mock_commitment_policy = MagicMock(__class__=MPL_CommitmentPolicyESDK) @@ -129,6 +132,7 @@ def test_GIVEN_valid_mpl_commitment_policy_WHEN_native_to_mpl_get_encryption_mat assert output.encryption_context == mock_encryption_materials_request.encryption_context assert output.commitment_policy == mock_commitment_policy assert output.max_plaintext_length == mock_encryption_materials_request.plaintext_length + assert output.algorithm_suite_id == mock_mpl_algorithm() def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_MPL_CommitmentPolicyESDK_FORBID_ENCRYPT_ALLOW_DECRYPT(): # noqa: E501 From dd89e32af52012e64a6d44c60489e888db051fbd Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 13:14:12 -0800 Subject: [PATCH 170/422] add mpl --- buildspec.yml | 8 ++++++++ .../manifests/full_message/decrypt.py | 11 +++++++++-- .../manifests/full_message/decrypt_generation.py | 13 +++++++++---- .../manifests/full_message/encrypt.py | 12 ++++++++++-- test_vector_handlers/tox.ini | 9 ++++++++- 5 files changed, 44 insertions(+), 9 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index 5dbd3f2b8..fff7c68d1 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -78,6 +78,10 @@ batch: buildspec: codebuild/py311/awses_local_mpl.yml env: image: aws/codebuild/standard:7.0 + - identifier: py311_mplawses_latest_mpl + buildspec: codebuild/py311/mplawses_local_mpl.yml + env: + image: aws/codebuild/standard:7.0 - identifier: py312_integ buildspec: codebuild/py312/integ.yml @@ -103,6 +107,10 @@ batch: buildspec: codebuild/py312/awses_local_mpl.yml env: image: aws/codebuild/standard:7.0 + - identifier: py312_mplawses_latest_mpl + buildspec: codebuild/py312/mplawses_local_mpl.yml + env: + image: aws/codebuild/standard:7.0 - identifier: code_coverage buildspec: codebuild/coverage/coverage.yml diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index f5c93058d..6d4a4a75d 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -36,7 +36,7 @@ from awses_test_vectors.manifests.master_key import MasterKeySpec, master_key_provider_from_master_key_specs try: - from awses_test_vectors.manifests.mpl_keyring import keyring_from_master_key_specs + from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_from_master_key_specs _HAS_MPL = True except ImportError: _HAS_MPL = False @@ -265,7 +265,14 @@ def from_scenario( :rtype: MessageDecryptionTestScenario """ raw_master_key_specs = scenario["master-keys"] # type: Iterable[MASTER_KEY_SPEC] - master_key_specs = [MasterKeySpec.from_scenario(spec) for spec in raw_master_key_specs] + if keyrings: + master_key_specs = [ + KeyringSpec.from_scenario(spec) for spec in raw_master_key_specs + ] + else: + master_key_specs = [ + MasterKeySpec.from_scenario(spec) for spec in raw_master_key_specs + ] def master_key_provider_fn(): if keyrings: diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 3b0b50a94..4bbb3df5d 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -39,7 +39,7 @@ ) from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL - from awses_test_vectors.manifests.mpl_keyring import keyring_from_master_key_specs + from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_from_master_key_specs except ImportError: pass @@ -394,9 +394,14 @@ def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): decryption_method_spec = scenario.get("decryption-method") decryption_method = DecryptionMethod(decryption_method_spec) if decryption_method_spec else None if "decryption-master-keys" in scenario: - decryption_master_key_specs = [ - MasterKeySpec.from_scenario(spec) for spec in scenario["decryption-master-keys"] - ] + if keyrings: + decryption_master_key_specs = [ + KeyringSpec.from_scenario(spec) for spec in scenario["decryption-master-keys"] + ] + else: + decryption_master_key_specs = [ + MasterKeySpec.from_scenario(spec) for spec in scenario["decryption-master-keys"] + ] def decryption_master_key_provider_fn(): if keyrings: diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 084981eaf..82a5e379a 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -40,7 +40,7 @@ from aws_encryption_sdk.identifiers import Algorithm as AlgorithmSuite try: - from awses_test_vectors.manifests.mpl_keyring import keyring_from_master_key_specs + from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_from_master_key_specs except ImportError: pass @@ -100,7 +100,15 @@ def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): :rtype: MessageEncryptionTestScenario """ algorithm = algorithm_suite_from_string_id(scenario["algorithm"]) - master_key_specs = [MasterKeySpec.from_scenario(spec) for spec in scenario["master-keys"]] + + if keyrings: + master_key_specs = [ + KeyringSpec.from_scenario(spec) for spec in scenario["master-keys"] + ] + else: + master_key_specs = [ + MasterKeySpec.from_scenario(spec) for spec in scenario["master-keys"] + ] def master_key_provider_fn(): if keyrings: diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index 580b641e0..eeb672833 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -4,6 +4,7 @@ envlist = # so until release we can only effectively test the local version of the ESDK. py{37,38,39,310}-awses_local py{311,312}-awses_local{,-mpl} + py{311,312}-mplawses_local-mpl # 1.2.0 and 1.2.max are being difficult because of attrs bandit, doc8, readme, {flake8,pylint}{,-tests}, @@ -36,7 +37,7 @@ envlist = # release :: Builds dist files and uploads to pypi pypirc profile. [testenv:base-command] -commands = pytest --basetemp={envtmpdir} -l --cov awses_test_vectors test/ {posargs} +commands = pytest --basetemp={envtmpdir} -l --cov awses_test_vectors test/ --ignore test/keyrings/ {posargs} [testenv] passenv = @@ -55,6 +56,12 @@ deps = commands = {[testenv:base-command]commands} +[testenv:mplawses_local] +basepython = python3 +sitepackages = False +deps = .. +commands = pytest --basetemp={envtmpdir} -l test/ {posargs} + [testenv:full-encrypt] basepython = python3 sitepackages = False From 5c5fb4b5c8e4716a6237a11e88ae0bb149325198 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 13:20:46 -0800 Subject: [PATCH 171/422] missing --- codebuild/py311/mplawses_local_mpl.yml | 26 ++++++++++++++++++++ codebuild/py312/mplawses_local_mpl.yml | 33 ++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 codebuild/py311/mplawses_local_mpl.yml create mode 100644 codebuild/py312/mplawses_local_mpl.yml diff --git a/codebuild/py311/mplawses_local_mpl.yml b/codebuild/py311/mplawses_local_mpl.yml new file mode 100644 index 000000000..22cd1dd81 --- /dev/null +++ b/codebuild/py311/mplawses_local_mpl.yml @@ -0,0 +1,26 @@ +version: 0.2 + +env: + variables: + TOXENV: "py311-mplawses_local-mpl" + REGION: "us-west-2" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + python: 3.11 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - tox diff --git a/codebuild/py312/mplawses_local_mpl.yml b/codebuild/py312/mplawses_local_mpl.yml new file mode 100644 index 000000000..c9ab1b618 --- /dev/null +++ b/codebuild/py312/mplawses_local_mpl.yml @@ -0,0 +1,33 @@ +# Runs the same tests as awses_local in an environment with the MPL installed. +# This asserts existing tests continue to pass with the MPL installed. +version: 0.2 + +env: + variables: + TOXENV: "py312-mplawses_local-mpl" + REGION: "us-west-2" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + python: latest + build: + commands: + - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - + - pyenv install --skip-existing 3.12.0 + - pyenv local 3.12.0 + - pip install --upgrade pip + - pip install setuptools + - pip install "tox < 4.0" + - cd test_vector_handlers + - tox From 9cf2191b8400655e316dae92b4da64055a2cb2bd Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 13:29:20 -0800 Subject: [PATCH 172/422] fix --- test_vector_handlers/tox.ini | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index eeb672833..607f854d2 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -4,7 +4,7 @@ envlist = # so until release we can only effectively test the local version of the ESDK. py{37,38,39,310}-awses_local py{311,312}-awses_local{,-mpl} - py{311,312}-mplawses_local-mpl + py{311,312}-mplvectors-mpl # 1.2.0 and 1.2.max are being difficult because of attrs bandit, doc8, readme, {flake8,pylint}{,-tests}, @@ -37,7 +37,7 @@ envlist = # release :: Builds dist files and uploads to pypi pypirc profile. [testenv:base-command] -commands = pytest --basetemp={envtmpdir} -l --cov awses_test_vectors test/ --ignore test/keyrings/ {posargs} +commands = pytest --basetemp={envtmpdir} -l --cov awses_test_vectors test/integration {posargs} [testenv] passenv = @@ -56,11 +56,11 @@ deps = commands = {[testenv:base-command]commands} -[testenv:mplawses_local] +[testenv:mplvectors-mpl] basepython = python3 sitepackages = False deps = .. -commands = pytest --basetemp={envtmpdir} -l test/ {posargs} +commands = pytest --basetemp={envtmpdir} -l test/keyrings {posargs} [testenv:full-encrypt] basepython = python3 From df10d915062ceff9bb7da349e16541c1983a0ad9 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 13:32:11 -0800 Subject: [PATCH 173/422] fix --- codebuild/py311/mplawses_local_mpl.yml | 2 +- codebuild/py312/mplawses_local_mpl.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/codebuild/py311/mplawses_local_mpl.yml b/codebuild/py311/mplawses_local_mpl.yml index 22cd1dd81..92dbdb086 100644 --- a/codebuild/py311/mplawses_local_mpl.yml +++ b/codebuild/py311/mplawses_local_mpl.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - TOXENV: "py311-mplawses_local-mpl" + TOXENV: "py311-mplvectors-mpl" REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f diff --git a/codebuild/py312/mplawses_local_mpl.yml b/codebuild/py312/mplawses_local_mpl.yml index c9ab1b618..e3f06e7f6 100644 --- a/codebuild/py312/mplawses_local_mpl.yml +++ b/codebuild/py312/mplawses_local_mpl.yml @@ -4,7 +4,7 @@ version: 0.2 env: variables: - TOXENV: "py312-mplawses_local-mpl" + TOXENV: "py312-mplvectors-mpl" REGION: "us-west-2" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f From e332d5b80b03727c1bc7790d1dffa1170b451ad4 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 13:43:15 -0800 Subject: [PATCH 174/422] fix --- test_vector_handlers/tox.ini | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index 607f854d2..0b18bf715 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -37,7 +37,7 @@ envlist = # release :: Builds dist files and uploads to pypi pypirc profile. [testenv:base-command] -commands = pytest --basetemp={envtmpdir} -l --cov awses_test_vectors test/integration {posargs} +commands = pytest --basetemp={envtmpdir} -l --cov awses_test_vectors {posargs} [testenv] passenv = @@ -54,13 +54,8 @@ deps = mpl: -r../requirements_mpl.txt .. commands = - {[testenv:base-command]commands} - -[testenv:mplvectors-mpl] -basepython = python3 -sitepackages = False -deps = .. -commands = pytest --basetemp={envtmpdir} -l test/keyrings {posargs} + awses_local: {[testenv:base-command]commands} test/integration + mplvectors: {[testenv:base-command]commands} test/keyrings [testenv:full-encrypt] basepython = python3 From 8dbeeb34591d44592f0f2eece21ceedab88f052e Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 13:46:19 -0800 Subject: [PATCH 175/422] add missing files --- .../manifests/mpl_keyring.py | 119 ++++++++++ test_vector_handlers/test/__init__.py | 0 .../test/keyrings/__init__.py | 0 .../test/keyrings/integration/__init__.py | 0 .../keyrings/integration/commands/__init__.py | 0 .../test_i_full_message_encrypt_keyrings.py | 62 +++++ test_vector_handlers/test/keys.json | 214 ++++++++++++++++++ 7 files changed, 395 insertions(+) create mode 100644 test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py create mode 100644 test_vector_handlers/test/__init__.py create mode 100644 test_vector_handlers/test/keyrings/__init__.py create mode 100644 test_vector_handlers/test/keyrings/integration/__init__.py create mode 100644 test_vector_handlers/test/keyrings/integration/commands/__init__.py create mode 100644 test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py create mode 100644 test_vector_handlers/test/keys.json diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py new file mode 100644 index 000000000..820cd00c2 --- /dev/null +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -0,0 +1,119 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Keyring Manifest handler. + +This REQUIRES the aws-cryptographic-material-providers library. +""" +import attr + +from aws_cryptography_materialproviderstestvectorkeys.smithygenerated.\ + aws_cryptography_materialproviderstestvectorkeys.models import ( + GetKeyDescriptionInput, + GetKeyDescriptionOutput, + TestVectorKeyringInput, + ) +from aws_cryptography_materialproviderstestvectorkeys.smithygenerated.\ + aws_cryptography_materialproviderstestvectorkeys.client import ( + KeyVectors, + ) +from aws_cryptography_materialproviderstestvectorkeys.smithygenerated.\ + aws_cryptography_materialproviderstestvectorkeys.config import ( + KeyVectorsConfig + ) +from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders +from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig +from aws_cryptographic_materialproviders.mpl.references import IKeyring +from aws_cryptographic_materialproviders.mpl.models import CreateMultiKeyringInput + +from awses_test_vectors.manifests.keys import KeysManifest # noqa pylint disable=unused-import + +import json + +from .master_key import MasterKeySpec + + +@attr.s +class KeyringSpec(MasterKeySpec): # pylint: disable=too-many-instance-attributes + """AWS Encryption SDK master key specification utilities. + + Described in AWS Crypto Tools Test Vector Framework features #0003 and #0004. + + :param str type_name: Master key type name + :param str key_name: Name of key in keys spec + :param str provider_id: Master key provider ID + :param str encryption_algorithm: Wrapping key encryption algorithm (required for raw master keys) + :param str padding_algorithm: Wrapping key padding algorithm (required for raw master keys) + :param str padding_hash: Wrapping key padding hash (required for raw master keys) + """ + + def keyring(self, keys_uri): + # type: (KeysManifest) -> IKeyring + """Build a keyring using this specification. + + :param str keys_uri: Path to the keys manifest + """ + + keyvectors = KeyVectors(KeyVectorsConfig(key_manifest_path=keys_uri)) + + # Construct the input to KeyVectorsConfig + input_as_dict = { + "type": self.type_name, + "key": self.key_name, + "provider-id": self.provider_id, + "encryption-algorithm": self.encryption_algorithm, + "padding-algorithm": self.padding_algorithm, + "padding-hash": self.padding_hash + } + # stringify the dict + input_as_string = json.dumps(input_as_dict) + # convert to unicode code point (expected representation) + encoded_json = [ord(c) for c in input_as_string] + + output: GetKeyDescriptionOutput = keyvectors.get_key_description( + GetKeyDescriptionInput(json=encoded_json) + ) + + keyring: IKeyring = keyvectors.create_test_vector_keyring( + TestVectorKeyringInput( + key_description=output.key_description + ) + ) + + return keyring + + +def keyring_from_master_key_specs(keys_uri, master_key_specs): + # type: (str, list[KeyringSpec]) -> IKeyring + """Build and combine all keyrings identified by the provided specs and + using the provided keys. + + :param str keys_uri: Path to the keys manifest + :param master_key_specs: Master key specs from which to load master keys + :type master_key_specs: iterable of MasterKeySpec + :return: Master key provider combining all loaded master keys + :rtype: IKeyring + """ + keyrings = [spec.keyring(keys_uri) for spec in master_key_specs] + primary = keyrings[0] + others = keyrings[1:] + + mpl: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + MaterialProvidersConfig() + ) + multi_keyring: IKeyring = mpl.create_multi_keyring( + CreateMultiKeyringInput( + generator=primary, + child_keyrings=others + ) + ) + return multi_keyring diff --git a/test_vector_handlers/test/__init__.py b/test_vector_handlers/test/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test_vector_handlers/test/keyrings/__init__.py b/test_vector_handlers/test/keyrings/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test_vector_handlers/test/keyrings/integration/__init__.py b/test_vector_handlers/test/keyrings/integration/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test_vector_handlers/test/keyrings/integration/commands/__init__.py b/test_vector_handlers/test/keyrings/integration/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py b/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py new file mode 100644 index 000000000..56bf3112c --- /dev/null +++ b/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py @@ -0,0 +1,62 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +""" +Integration tests for `awses_test_vectors.commands` with keyrings. +""" +import pytest + +from awses_test_vectors.commands import full_message_decrypt, full_message_decrypt_generate, full_message_encrypt + +from ....integration.integration_test_utils import ( # noqa pylint: disable=unused-import + full_message_decrypt_generation_vectors, + full_message_encrypt_vectors, +) + +pytestmark = [pytest.mark.integ] + + +def test_full_message_encrypt_canonical_full(full_message_encrypt_vectors): + full_message_encrypt.cli(["--input", full_message_encrypt_vectors]) + full_message_encrypt.cli(["--input", full_message_encrypt_vectors], "--keyrings") + + +def test_full_message_cycle_canonical_full(tmpdir, full_message_decrypt_generation_vectors): + # Generate vectors using keyring interfaces + keyring_output_dir = tmpdir.join("output-keyrings") + full_message_decrypt_generate.cli([ + "--output", + str(keyring_output_dir), + "--input", + full_message_decrypt_generation_vectors, + "--keyrings" + ]) + # Generate vectors using master key interfaces + master_key_output_dir = tmpdir.join("output-master-key") + full_message_decrypt_generate.cli([ + "--output", + str(master_key_output_dir), + "--input", + full_message_decrypt_generation_vectors + ]) + + # Validate that vectors generated using keyring interfaces + # can be decrypted by BOTH keyring and master key interfaces + keyring_decrypt_manifest_file = keyring_output_dir.join("manifest.json") + full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file), "--keyrings"]) + full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file)]) + + # Validate that vectors generated using master key interfaces + # can be decrypted by BOTH keyring and master key interfaces + master_key_decrypt_manifest_file = keyring_output_dir.join("manifest.json") + full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file), "--keyrings"]) + full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file)]) diff --git a/test_vector_handlers/test/keys.json b/test_vector_handlers/test/keys.json new file mode 100644 index 000000000..304dae5f7 --- /dev/null +++ b/test_vector_handlers/test/keys.json @@ -0,0 +1,214 @@ +{ + "manifest": { + "type": "keys", + "version": 3 + }, + "keys": { + "aes-128": { + "encrypt": true, + "decrypt": true, + "algorithm": "aes", + "type": "symmetric", + "bits": 128, + "encoding": "base64", + "material": "AAECAwQFBgcICRAREhMUFQ==", + "key-id": "aes-128" + }, + "aes-192": { + "encrypt": true, + "decrypt": true, + "algorithm": "aes", + "type": "symmetric", + "bits": 192, + "encoding": "base64", + "material": "AAECAwQFBgcICRAREhMUFRYXGBkgISIj", + "key-id": "aes-192" + }, + "aes-256": { + "encrypt": true, + "decrypt": true, + "algorithm": "aes", + "type": "symmetric", + "bits": 256, + "encoding": "base64", + "material": "AAECAwQFBgcICRAREhMUFRYXGBkgISIjJCUmJygpMDE=", + "key-id": "aes-256" + }, + "rsa-4096-private": { + "encrypt": true, + "decrypt": true, + "algorithm": "rsa", + "type": "private", + "bits": 4096, + "encoding": "pem", + "material": "-----BEGIN PRIVATE KEY-----\nMIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCztGg1gQ8AjCzz\n1VX6StqtW//jBt2ZQBoApaBa7FmLmdr0YlKaeEKSrItGbvA9tBjgsKhrn8gxTGQc\nuxgM92651jRCbQZyjE6W8kodijhGMXsfKJLfgPp2/I7gZ3dqrSZkejFIYLFb/uF/\nTfAQzNyJUldYdeFojSUPqevMgSAusTgv7dXYt4BCO9mxMp35tgyp5k4vazKJVUgB\nTw87AAYZUGugmi94Wb9JSnqUKI3QzaRN7JADZrHdBO1lIBryfCsjtTnZc7NWZ0yJ\nwmzLY+C5b3y17cy44N0rbjI2QciRhqZ4/9SZ/9ImyFQlB3lr9NSndcT4eE5YC6bH\nba0gOUK9lLXVy6TZ+nRZ4dSddoLX03mpYp+8cQpK6DO3L/PeUY/si0WGsXZfWokd\n4ACwvXWSOjotzjwqwTW8q9udbhUvIHfB02JW+ZQ07b209fBpHRDkZuveOTedTN2Q\nQei4dZDjWW5s4cIIE3dXXeaH8yC02ERIeN+aY6eHngSsP2xoDV3sKNN/yDbCqaMS\nq8ZJbo2rvOFxZHa2nWiV+VLugfO6Xj8jeGeR8vopvbEBZZpAq+Dea2xjY4+XMUQ/\nS1HlRwc9+nkJ5LVfODuE3q9EgJbqbiXe7YckWV3ZqQMybW+dLPxEJs9buOntgHFS\nRYmbKky0bti/ZoZlcZtS0zyjVxlqsQIDAQABAoICAEr3m/GWIXgNAkPGX9PGnmtr\n0dgX6SIhh7d1YOwNZV3DlYAV9HfUa5Fcwc1kQny7QRWbHOepBI7sW2dQ9buTDXIh\nVjPP37yxo6d89EZWfxtpUP+yoXL0D4jL257qCvtJuJZ6E00qaVMDhXbiQKABlo8C\n9sVEiABhwXBDZsctpwtTiykTgv6hrrPy2+H8R8MAm0/VcBCAG9kG5r8FCEmIvQKa\ndgvNxrfiWNZuZ6yfLmpJH54SbhG9Kb4WbCKfvh4ihqyi0btRdSM6fMeLgG9o/zrc\ns54B0kHeLOYNVo0j7FQpZBFeSIbmHfln4RKBh7ntrTke/Ejbh3NbiPvxWSP0P067\nSYWPkQpip2q0ION81wSQZ1haP2GewFFu4IEjG3DlqqpKKGLqXrmjMufnildVFpBx\nir+MgvgQfEBoGEx0aElyO7QuRYaEiXeb/BhMZeC5O65YhJrWSuTVizh3xgJWjgfV\naYwYgxN8SBXBhXLIVvnPhadTqsW1C/aevLOk110eSFWcHf+FCK781ykIzcpXoRGX\nOwWcZzC/fmSABS0yH56ow+I0tjdLIEEMhoa4/kkamioHOJ4yyB+W1DO6/DnMyQlx\ng7y2WsAaIEBoWUARy776k70xPPMtYAxzFXI9KhqRVrPfeaRZ+ojeyLyr3GQGyyoo\ncuGRdMUblsmODv4ixmOxAoIBAQDvkznvVYNdP3Eg5vQeLm/qsP6dLejLijBLeq9i\n7DZH2gRpKcflXZxCkRjsKDDE+fgDcBYEp2zYfRIVvgrxlTQZdaSG+GoDcbjbNQn3\ndjCCtOOACioN/vg2zFlX4Bs6Q+NaV7g5qP5SUaxUBjuHLe7Nc+ZkyheMHuNYVLvk\nHL/IoWyANpZYjMUU3xMbL/J29Gz7CPGr8Si28TihAHGfcNgn8S04OQZhTX+bU805\n/+7B4XW47Mthg/u7hlqFl+YIAaSJYvWkEaVP1A9I7Ve0aMDSMWwzTg9cle2uVaL3\n+PTzWY5coBlHKjqAg9ufhYSDhAqBd/JOSlv8RwcA3PDXJ6C/AoIBAQDABmXXYQky\n7phExXBvkLtJt2TBGjjwulf4R8TC6W5F51jJuoqY/mTqYcLcOn2nYGVwoFvPsy/Q\nCTjfODwJBXzbloXtYFR3PWAeL1Y6+7Cm+koMWIPJyVbD5Fzm+gZStM0GwP8FhDt2\nWt8fWEyXmoLdAy6RAwiEmCagEh8o+13oBfwnBllbz7TxaErsUuR+XVgl/iHwztdv\ncdJKyRgaFfWSh9aiO7EMV2rBGWsoX09SRvprPFAGx8Ffm7YcqIk34QXsQyc45Dyn\nCwkvypxHoaB3ot/48FeFm9IubApb/ctv+EgkBfL4S4bdwRXS1rt+0+QihBoFyP2o\nJ91cdm4hEWCPAoIBAQC6l11hFaYZo0bWDGsHcr2B+dZkzxPoKznQH76n+jeQoLIc\nwgjJkK4afm39yJOrZtEOxGaxu0CgIFFMk9ZsL/wC9EhvQt02z4TdXiLkFK5VrtMd\nr0zv16y06VWQhqBOMf/KJlX6uq9RqADi9HO6pkC+zc0cpPXQEWKaMmygju+kMG2U\nMm/IieMZjWCRJTfgBCE5J88qTsqaKagkZXcZakdAXKwOhQN+F2EStiM6UCZB5PrO\nS8dfrO8ML+ki8Zqck8L1qhiNb5zkXtKExy4u+gNr8khGcT6vqqoSxOoH3mPRgOfL\nJnppne8wlwIf7Vq3H8ka6zPSXEHma999gZcmy9t7AoIBAGbQhiLl79j3a0wXMvZp\nVf5IVYgXFDnAbG2hb7a06bhAAIgyexcjzsC4C2+DWdgOgwHkuoPg+062QV8zauGh\nsJKaa6cHlvIpSJeg3NjD/nfJN3CYzCd0yCIm2Z9Ka6xI5iYhm+pGPNhIG4Na8deS\ngVL46yv1pc/o73VxfoGg5UzgN3xlp97Cva0sHEGguHr4W8Qr59xZw3wGQ4SLW35M\nF6qXVNKUh12GSMCPbZK2RXBWVKqqJmca+WzJoJ6DlsT2lQdFhXCus9L007xlDXxF\nC/hCmw1dEl+VaNo2Ou26W/zdwTKYhNlxBwsg4SB8nPNxXIsmlBBY54froFhriNfn\nx/0CggEAUzz+VMtjoEWw2HSHLOXrO4EmwJniNgiiwfX3DfZE4tMNZgqZwLkq67ns\nT0n3b0XfAOOkLgMZrUoOxPHkxFeyLLf7pAEJe7QNB+Qilw8e2zVqtiJrRk6uDIGJ\nSv+yM52zkImZAe2jOdU3KeUZxSMmb5vIoiPBm+tb2WupAg3YdpKn1/jWTpVmV/+G\nUtTLVE6YpAyFp1gMxhutE9vfIS94ek+vt03AoEOlltt6hqZfv3xmY8vGuAjlnj12\nzHaq+fhCRPsbsZkzJ9nIVdXYnNIEGtMGNnxax7tYRej/UXqyazbxHiJ0iPF4PeDn\ndzxtGxpeTBi+KhKlca8SlCdCqYwG6Q==\n-----END PRIVATE KEY-----", + "key-id": "rsa-4096" + }, + "rsa-4096-public": { + "encrypt": true, + "decrypt": false, + "algorithm": "rsa", + "type": "public", + "bits": 4096, + "encoding": "pem", + "material": "-----BEGIN PUBLIC KEY-----\nMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAs7RoNYEPAIws89VV+kra\nrVv/4wbdmUAaAKWgWuxZi5na9GJSmnhCkqyLRm7wPbQY4LCoa5/IMUxkHLsYDPdu\nudY0Qm0GcoxOlvJKHYo4RjF7HyiS34D6dvyO4Gd3aq0mZHoxSGCxW/7hf03wEMzc\niVJXWHXhaI0lD6nrzIEgLrE4L+3V2LeAQjvZsTKd+bYMqeZOL2syiVVIAU8POwAG\nGVBroJoveFm/SUp6lCiN0M2kTeyQA2ax3QTtZSAa8nwrI7U52XOzVmdMicJsy2Pg\nuW98te3MuODdK24yNkHIkYameP/Umf/SJshUJQd5a/TUp3XE+HhOWAumx22tIDlC\nvZS11cuk2fp0WeHUnXaC19N5qWKfvHEKSugzty/z3lGP7ItFhrF2X1qJHeAAsL11\nkjo6Lc48KsE1vKvbnW4VLyB3wdNiVvmUNO29tPXwaR0Q5Gbr3jk3nUzdkEHouHWQ\n41lubOHCCBN3V13mh/MgtNhESHjfmmOnh54ErD9saA1d7CjTf8g2wqmjEqvGSW6N\nq7zhcWR2tp1olflS7oHzul4/I3hnkfL6Kb2xAWWaQKvg3mtsY2OPlzFEP0tR5UcH\nPfp5CeS1Xzg7hN6vRICW6m4l3u2HJFld2akDMm1vnSz8RCbPW7jp7YBxUkWJmypM\ntG7Yv2aGZXGbUtM8o1cZarECAwEAAQ==\n-----END PUBLIC KEY-----", + "key-id": "rsa-4096" + }, + "us-west-2-decryptable": { + "encrypt": true, + "decrypt": true, + "type": "aws-kms", + "key-id": "arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f" + }, + "us-west-2-encrypt-only": { + "encrypt": true, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws:kms:us-west-2:658956600833:key/590fd781-ddde-4036-abec-3e1ab5a5d2ad" + }, + "us-west-2-mrk": { + "encrypt": true, + "decrypt": true, + "type": "aws-kms", + "key-id": "arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "us-east-1-mrk": { + "encrypt": true, + "decrypt": true, + "type": "aws-kms", + "key-id": "arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "aws:kms:us-west-2:658956600833:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "aws:kms:us-west-2:658956600833:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + ":aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": ":aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "arn-not:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn-not:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "arn:kms:us-west-2:658956600833:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:kms:us-west-2:658956600833:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "arn::kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn::kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "arn:aws-not:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws-not:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "arn:aws:us-west-2:658956600833:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws:us-west-2:658956600833:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "arn:aws::us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws::us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "arn:aws:kms-not:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws:kms-not:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "arn:aws:kms:658956600833:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws:kms:658956600833:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "arn:aws:kms::658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws:kms::658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "arn:aws:kms:us-west-2:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws:kms:us-west-2:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "arn:aws:kms:us-west-2::key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws:kms:us-west-2::key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "arn:aws:kms:us-west-2:658956600833-not:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws:kms:us-west-2:658956600833-not:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "arn:aws:kms:us-west-2:658956600833:mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws:kms:us-west-2:658956600833:mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "arn:aws:kms:us-west-2:658956600833:/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws:kms:us-west-2:658956600833:/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "arn:aws:kms:us-west-2:658956600833:key-not/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws:kms:us-west-2:658956600833:key-not/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "arn:aws:kms:us-west-2:658956600833:key": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws:kms:us-west-2:658956600833:key" + }, + "arn:aws:kms:us-west-2:658956600833:key/": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws:kms:us-west-2:658956600833:key/" + }, + "arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7-not": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7-not" + }, + "arn:aws:kms:us-west-2:658956600833:alias/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "arn:aws:kms:us-west-2:658956600833:alias/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + }, + "mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { + "encrypt": false, + "decrypt": false, + "type": "aws-kms", + "key-id": "mrk-80bd8ecdcd4342aebd84b7dc9da498a7" + } + } +} From 15a69761bb737f89824b61fc461acbc7fbf4e1f4 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 14:24:56 -0800 Subject: [PATCH 176/422] install testvectors --- test_vector_handlers/requirements_mpl.txt | 1 + test_vector_handlers/tox.ini | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 test_vector_handlers/requirements_mpl.txt diff --git a/test_vector_handlers/requirements_mpl.txt b/test_vector_handlers/requirements_mpl.txt new file mode 100644 index 000000000..c7927a851 --- /dev/null +++ b/test_vector_handlers/requirements_mpl.txt @@ -0,0 +1 @@ +amazon-cryptographic-material-providers-test-vectors @ git+https://github.com/aws/aws-cryptographic-material-providers-library.git@lucmcdon/python-mpl#subdirectory=TestVectorsAwsCryptographicMaterialProviders/runtimes/python \ No newline at end of file diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index 0b18bf715..949efc89a 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -51,7 +51,7 @@ sitepackages = False deps = -rtest/requirements.txt # Install the MPL requirements if the `-mpl` suffix is present - mpl: -r../requirements_mpl.txt + mpl: -r../requirements_mpl.txt -rtest/requirements_mpl.txt .. commands = awses_local: {[testenv:base-command]commands} test/integration From 46ac8a0dbf77b24f710a221879701e02014576ba Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 14:27:37 -0800 Subject: [PATCH 177/422] fix --- test_vector_handlers/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index 949efc89a..c107d3527 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -51,7 +51,7 @@ sitepackages = False deps = -rtest/requirements.txt # Install the MPL requirements if the `-mpl` suffix is present - mpl: -r../requirements_mpl.txt -rtest/requirements_mpl.txt + mpl: -r../requirements_mpl.txt -rrequirements_mpl.txt .. commands = awses_local: {[testenv:base-command]commands} test/integration From 104ff8a341b4a14f59585027d2d5bef1b6aa8c4f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 14:33:32 -0800 Subject: [PATCH 178/422] fix --- test_vector_handlers/tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index c107d3527..9f86d448f 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -51,7 +51,8 @@ sitepackages = False deps = -rtest/requirements.txt # Install the MPL requirements if the `-mpl` suffix is present - mpl: -r../requirements_mpl.txt -rrequirements_mpl.txt + mpl: -r../requirements_mpl.txt + mpl: -rrequirements_mpl.txt .. commands = awses_local: {[testenv:base-command]commands} test/integration From 65227c2b2380cc78077d74ec2ca1978b1802bfa2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 7 Mar 2024 14:40:20 -0800 Subject: [PATCH 179/422] fix --- test_vector_handlers/tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index 9f86d448f..c2ff913c2 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -51,7 +51,6 @@ sitepackages = False deps = -rtest/requirements.txt # Install the MPL requirements if the `-mpl` suffix is present - mpl: -r../requirements_mpl.txt mpl: -rrequirements_mpl.txt .. commands = From a2484a0271ca4cf1ef0b99c345279b3c73d1da0a Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 11 Mar 2024 16:01:25 -0700 Subject: [PATCH 180/422] working --- .../materials_managers/mpl/cmm.py | 13 ++++ .../manifests/full_message/decrypt.py | 2 +- .../full_message/decrypt_generation.py | 2 +- .../manifests/full_message/encrypt.py | 2 +- .../manifests/mpl_keyring.py | 58 ++++++++++++----- .../integration/integration_test_utils.py | 2 +- .../test_i_full_message_encrypt_keyrings.py | 62 ++++++++++++------- test_vector_handlers/tox.ini | 4 +- 8 files changed, 101 insertions(+), 44 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index ebef5f7ac..9efb8c1f0 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -68,6 +68,11 @@ def get_encryption_materials( request ) mpl_output: MPL_GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) + + print(f"{mpl_output.as_dict()=}") + + mpl_output.encryption_materials.encrypted_data_keys[0].key_provider_info = b"rsa-4096-private" + return EncryptionMaterialsFromMPL(mpl_output.encryption_materials) except AwsCryptographicMaterialProvidersException as mpl_exception: # Wrap MPL error into the ESDK error type @@ -118,15 +123,23 @@ def decrypt_materials( Returns a DecryptionMaterialsFromMPL for the configured CMM. :param request: Request for decryption materials """ + from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.errors import CollectionOfErrors as COE try: mpl_input: 'MPL_DecryptMaterialsInput' = \ CryptoMaterialsManagerFromMPL._create_mpl_decrypt_materials_input_from_request(request) + print(f"{mpl_input.as_dict()=}") + # input() mpl_output: 'MPL_DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(mpl_input) + print(f"{mpl_output.as_dict()=}") + # input() return DecryptionMaterialsFromMPL(mpl_output.decryption_materials) except AwsCryptographicMaterialProvidersException as mpl_exception: # Wrap MPL error into the ESDK error type # so customers only have to catch ESDK error types. raise AWSEncryptionSDKClientError(mpl_exception) + except COE as coe: + print(f"{coe.list=}") + raise AWSEncryptionSDKClientError(coe) @staticmethod def _native_algorithm_id_to_mpl_algorithm_id(native_algorithm_id: str) -> 'MPL_AlgorithmSuiteIdESDK': diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index 6d4a4a75d..0b8dfe3b8 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -276,7 +276,7 @@ def from_scenario( def master_key_provider_fn(): if keyrings: - return keyring_from_master_key_specs(keys_uri, master_key_specs) + return keyring_from_master_key_specs(keys, keys_uri, master_key_specs, "decrypt") return master_key_provider_from_master_key_specs(keys, master_key_specs) decryption_method_spec = scenario.get("decryption-method") diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 4bbb3df5d..3fce71e36 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -405,7 +405,7 @@ def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): def decryption_master_key_provider_fn(): if keyrings: - return keyring_from_master_key_specs(keys_uri, decryption_master_key_specs) + return keyring_from_master_key_specs(keys, keys_uri, decryption_master_key_specs, "decrypt-generation") return master_key_provider_from_master_key_specs(keys, decryption_master_key_specs) else: diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 82a5e379a..a3d351317 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -112,7 +112,7 @@ def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): def master_key_provider_fn(): if keyrings: - return keyring_from_master_key_specs(keys_uri, master_key_specs) + return keyring_from_master_key_specs(keys, keys_uri, master_key_specs, "encrypt") return master_key_provider_from_master_key_specs(keys, master_key_specs) return cls( diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index 820cd00c2..d64c323d0 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -22,19 +22,12 @@ GetKeyDescriptionOutput, TestVectorKeyringInput, ) -from aws_cryptography_materialproviderstestvectorkeys.smithygenerated.\ - aws_cryptography_materialproviderstestvectorkeys.client import ( - KeyVectors, - ) -from aws_cryptography_materialproviderstestvectorkeys.smithygenerated.\ - aws_cryptography_materialproviderstestvectorkeys.config import ( - KeyVectorsConfig - ) from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig from aws_cryptographic_materialproviders.mpl.references import IKeyring from aws_cryptographic_materialproviders.mpl.models import CreateMultiKeyringInput +from awses_test_vectors.internal.keyvectors_provider import KeyVectorsProvider from awses_test_vectors.manifests.keys import KeysManifest # noqa pylint disable=unused-import import json @@ -56,26 +49,55 @@ class KeyringSpec(MasterKeySpec): # pylint: disable=too-many-instance-attribute :param str padding_hash: Wrapping key padding hash (required for raw master keys) """ - def keyring(self, keys_uri): + def keyring(self, keys, keys_uri, mode): # type: (KeysManifest) -> IKeyring """Build a keyring using this specification. :param str keys_uri: Path to the keys manifest """ - keyvectors = KeyVectors(KeyVectorsConfig(key_manifest_path=keys_uri)) + ''' + encryptmaterials keyProviderInfo = rsa-4096-public' + MUST be private. + somehow, it is writing "rsa-4096-public". + + ''' + + print(f"{keys=}") + + keyvectors = KeyVectorsProvider.get_keyvectors(keys_path=keys_uri) # Construct the input to KeyVectorsConfig - input_as_dict = { + input_kwargs = { "type": self.type_name, "key": self.key_name, "provider-id": self.provider_id, "encryption-algorithm": self.encryption_algorithm, - "padding-algorithm": self.padding_algorithm, - "padding-hash": self.padding_hash + } + if self.padding_algorithm is not None and self.padding_algorithm is not "": + input_kwargs["padding-algorithm"] = self.padding_algorithm + if self.padding_hash is not None: + input_kwargs["padding-hash"] = self.padding_hash + + # Normalize input for MPL + if input_kwargs["type"] == "raw" \ + and input_kwargs["encryption-algorithm"] == "rsa": + if input_kwargs["key"] == "rsa-4096-private" \ + and (mode == "decrypt-generate" or mode == "encrypt"): + print(f"changed private to public") + input_kwargs["key"] = "rsa-4096-public" + # if input_kwargs["key"] == "rsa-4096-private" \ + # and (mode == "decrypt"): + # input_kwargs["provider-id"] = "rsa-4096-public" + if "padding-hash" not in input_kwargs: + print("added paddinghash") + input_kwargs["padding-hash"] = "sha1" + + print(f"keyring {input_kwargs=}") + # stringify the dict - input_as_string = json.dumps(input_as_dict) + input_as_string = json.dumps(input_kwargs) # convert to unicode code point (expected representation) encoded_json = [ord(c) for c in input_as_string] @@ -83,6 +105,10 @@ def keyring(self, keys_uri): GetKeyDescriptionInput(json=encoded_json) ) + print(f"{output.key_description.value=}") + + keyvectors + keyring: IKeyring = keyvectors.create_test_vector_keyring( TestVectorKeyringInput( key_description=output.key_description @@ -92,7 +118,7 @@ def keyring(self, keys_uri): return keyring -def keyring_from_master_key_specs(keys_uri, master_key_specs): +def keyring_from_master_key_specs(keys, keys_uri, master_key_specs, mode): # type: (str, list[KeyringSpec]) -> IKeyring """Build and combine all keyrings identified by the provided specs and using the provided keys. @@ -103,7 +129,7 @@ def keyring_from_master_key_specs(keys_uri, master_key_specs): :return: Master key provider combining all loaded master keys :rtype: IKeyring """ - keyrings = [spec.keyring(keys_uri) for spec in master_key_specs] + keyrings = [spec.keyring(keys, keys_uri, mode) for spec in master_key_specs] primary = keyrings[0] others = keyrings[1:] diff --git a/test_vector_handlers/test/integration/integration_test_utils.py b/test_vector_handlers/test/integration/integration_test_utils.py index fbe6cf7b7..25efe6b79 100644 --- a/test_vector_handlers/test/integration/integration_test_utils.py +++ b/test_vector_handlers/test/integration/integration_test_utils.py @@ -33,5 +33,5 @@ def full_message_encrypt_vectors(): @pytest.fixture def full_message_decrypt_generation_vectors(): return os.path.join( - vectors_dir(), "features", "CANONICAL-GENERATED-MANIFESTS", "0006-awses-message-decryption-generation.v2.json" + vectors_dir(), "features", "CANONICAL-GENERATED-MANIFESTS", "decrypt-generate-lite.json" ) diff --git a/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py b/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py index 56bf3112c..de5f10299 100644 --- a/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py +++ b/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py @@ -22,17 +22,20 @@ full_message_encrypt_vectors, ) +import cProfile + pytestmark = [pytest.mark.integ] -def test_full_message_encrypt_canonical_full(full_message_encrypt_vectors): - full_message_encrypt.cli(["--input", full_message_encrypt_vectors]) - full_message_encrypt.cli(["--input", full_message_encrypt_vectors], "--keyrings") +# def test_full_message_encrypt_canonical_full(full_message_encrypt_vectors): +# full_message_encrypt.cli(["--input", full_message_encrypt_vectors]) +# full_message_encrypt.cli(["--input", full_message_encrypt_vectors], "--keyrings") def test_full_message_cycle_canonical_full(tmpdir, full_message_decrypt_generation_vectors): # Generate vectors using keyring interfaces keyring_output_dir = tmpdir.join("output-keyrings") + print("Generating vectors with keyrings... ", end="") full_message_decrypt_generate.cli([ "--output", str(keyring_output_dir), @@ -40,23 +43,38 @@ def test_full_message_cycle_canonical_full(tmpdir, full_message_decrypt_generati full_message_decrypt_generation_vectors, "--keyrings" ]) - # Generate vectors using master key interfaces - master_key_output_dir = tmpdir.join("output-master-key") - full_message_decrypt_generate.cli([ - "--output", - str(master_key_output_dir), - "--input", - full_message_decrypt_generation_vectors - ]) + print("done") + + # print("Generating vectors with master keys... ", end="") + # # Generate vectors using master key interfaces + # master_key_output_dir = tmpdir.join("output-master-key") + # full_message_decrypt_generate.cli([ + # "--output", + # str(master_key_output_dir), + # "--input", + # full_message_decrypt_generation_vectors + # ]) + # print("done") + + # # Validate that vectors generated using keyring interfaces + # # can be decrypted by BOTH keyring and master key interfaces + # keyring_decrypt_manifest_file = keyring_output_dir.join("manifest.json") + # print("Decrypting keyring-encrypted vectors with keyrings... ", end="") + # full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file), "--keyrings"]) + # print("done") + + # print("Decrypting keyring-encrypted vectors with master keys... ", end="") + # full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file)]) + # print("done") + + # # Validate that vectors generated using master key interfaces + # # can be decrypted by BOTH keyring and master key interfaces + # master_key_decrypt_manifest_file = keyring_output_dir.join("manifest.json") + + # print("Decrypting master key-encrypted vectors with keyrings... ", end="") + # full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file), "--keyrings"]) + # print("done") - # Validate that vectors generated using keyring interfaces - # can be decrypted by BOTH keyring and master key interfaces - keyring_decrypt_manifest_file = keyring_output_dir.join("manifest.json") - full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file), "--keyrings"]) - full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file)]) - - # Validate that vectors generated using master key interfaces - # can be decrypted by BOTH keyring and master key interfaces - master_key_decrypt_manifest_file = keyring_output_dir.join("manifest.json") - full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file), "--keyrings"]) - full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file)]) + # print("Decrypting master key-encrypted vectors with master keys... ", end="") + # full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file)]) + # print("done") diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index c2ff913c2..18b3710e5 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -37,7 +37,7 @@ envlist = # release :: Builds dist files and uploads to pypi pypirc profile. [testenv:base-command] -commands = pytest --basetemp={envtmpdir} -l --cov awses_test_vectors {posargs} +commands = python3 -m cProfile -o profile.txt -m pytest --basetemp={envtmpdir} -l --cov awses_test_vectors {posargs} [testenv] passenv = @@ -55,7 +55,7 @@ deps = .. commands = awses_local: {[testenv:base-command]commands} test/integration - mplvectors: {[testenv:base-command]commands} test/keyrings + mplvectors: {[testenv:base-command]commands} test/keyrings -s -v [testenv:full-encrypt] basepython = python3 From faa92a0e1df93a0c749bd8c2d2d1db273cccf264 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 12 Mar 2024 11:56:55 -0700 Subject: [PATCH 181/422] decrypt-lite workign --- .../materials_managers/mpl/cmm.py | 20 +++++++++++++--- .../materials_managers/mpl/materials.py | 2 ++ .../full_message/decrypt_generation.py | 2 +- .../manifests/mpl_keyring.py | 23 +++++++++++++++++++ 4 files changed, 43 insertions(+), 4 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index 9efb8c1f0..1655e0746 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -67,11 +67,25 @@ def get_encryption_materials( CryptoMaterialsManagerFromMPL._native_to_mpl_get_encryption_materials( request ) + mpl_output: MPL_GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) - print(f"{mpl_output.as_dict()=}") + print(f"get {mpl_output=}") + + # ???????????????????????????? + # kpis = set() + # for edk in mpl_output.encryption_materials.encrypted_data_keys: + # kpis.add(edk.key_provider_info) + + # print(kpis) + # input + + # if len(kpis) == 1: + # for edk in mpl_output.encryption_materials.encrypted_data_keys: + # if edk.key_provider_info == b"rsa-4096-public": + # edk.key_provider_info = b"rsa-4096-private" - mpl_output.encryption_materials.encrypted_data_keys[0].key_provider_info = b"rsa-4096-private" + # mpl_output.encryption_materials.encrypted_data_keys[0].key_provider_info = b"rsa-4096-private" return EncryptionMaterialsFromMPL(mpl_output.encryption_materials) except AwsCryptographicMaterialProvidersException as mpl_exception: @@ -139,7 +153,7 @@ def decrypt_materials( raise AWSEncryptionSDKClientError(mpl_exception) except COE as coe: print(f"{coe.list=}") - raise AWSEncryptionSDKClientError(coe) + # raise AWSEncryptionSDKClientError(coe) @staticmethod def _native_algorithm_id_to_mpl_algorithm_id(native_algorithm_id: str) -> 'MPL_AlgorithmSuiteIdESDK': diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index 54ea21b39..9f3d4f0fb 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -75,6 +75,8 @@ def encrypted_data_keys(self) -> List[Native_EncryptedDataKey]: ), encrypted_data_key=mpl_edk.ciphertext, ) for mpl_edk in mpl_edk_list} + print(f"{key_blob_list=}") + # input() return key_blob_list @property diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 3fce71e36..5536fc845 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -484,7 +484,7 @@ def _generate_plaintexts(plaintexts_specs): :return: Mapping of plaintext name to randomly generated bytes :rtype: dict """ - return {name: os.urandom(size) for name, size in plaintexts_specs.items()} + return {name: b"a" * size for name, size in plaintexts_specs.items()} @classmethod def from_file(cls, input_file, keyrings): diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index d64c323d0..c9e834439 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -67,6 +67,8 @@ def keyring(self, keys, keys_uri, mode): keyvectors = KeyVectorsProvider.get_keyvectors(keys_path=keys_uri) + changed = False + # Construct the input to KeyVectorsConfig input_kwargs = { "type": self.type_name, @@ -86,6 +88,7 @@ def keyring(self, keys, keys_uri, mode): if input_kwargs["key"] == "rsa-4096-private" \ and (mode == "decrypt-generate" or mode == "encrypt"): print(f"changed private to public") + changed = True input_kwargs["key"] = "rsa-4096-public" # if input_kwargs["key"] == "rsa-4096-private" \ # and (mode == "decrypt"): @@ -115,6 +118,20 @@ def keyring(self, keys, keys_uri, mode): ) ) + import _dafny + import UTF8 + + if hasattr(keyring, "_impl"): + if hasattr(keyring._impl, "_keyName"): + if keyring._impl._keyName == UTF8.default__.Encode(_dafny.Seq("rsa-4096-public")).value \ + and (mode == "decrypt-generate" or mode == "encrypt"): + if changed: + print("YES") + # input() + print(f"changed public to private") + keyring._impl._keyName = UTF8.default__.Encode(_dafny.Seq("rsa-4096-private")).value + + return keyring @@ -129,7 +146,13 @@ def keyring_from_master_key_specs(keys, keys_uri, master_key_specs, mode): :return: Master key provider combining all loaded master keys :rtype: IKeyring """ + # print(f"{master_key_specs=}") + # input() keyrings = [spec.keyring(keys, keys_uri, mode) for spec in master_key_specs] + # print(f"speckeyrings {keyrings=}") + # input() + # print(f"speckeys {keys=}") + # input() primary = keyrings[0] others = keyrings[1:] From bbc36f9cf9a271d958c5ff69f7b34e4773e9c10c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 12 Mar 2024 12:20:42 -0700 Subject: [PATCH 182/422] ..? --- .../manifests/mpl_keyring.py | 56 ++++++++++++------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index c9e834439..0f63ca143 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -82,23 +82,23 @@ def keyring(self, keys, keys_uri, mode): if self.padding_hash is not None: input_kwargs["padding-hash"] = self.padding_hash - # Normalize input for MPL if input_kwargs["type"] == "raw" \ and input_kwargs["encryption-algorithm"] == "rsa": + # Weird hack #1. + # If generating decrypt vectors (i.e. encrypting) + # and the manifest specified an RSA private key, + # change the input to KeyVectors to a public key. + # KeyVectors requires a public key to encrypt. + # If this is not done, then keyring.OnEncrypt fails with + # "A RawRSAKeyring without a public key cannot provide OnEncrypt" if input_kwargs["key"] == "rsa-4096-private" \ and (mode == "decrypt-generate" or mode == "encrypt"): - print(f"changed private to public") changed = True input_kwargs["key"] = "rsa-4096-public" - # if input_kwargs["key"] == "rsa-4096-private" \ - # and (mode == "decrypt"): - # input_kwargs["provider-id"] = "rsa-4096-public" + # Specify default padding-hash if "padding-hash" not in input_kwargs: - print("added paddinghash") input_kwargs["padding-hash"] = "sha1" - print(f"keyring {input_kwargs=}") - # stringify the dict input_as_string = json.dumps(input_kwargs) # convert to unicode code point (expected representation) @@ -108,16 +108,39 @@ def keyring(self, keys, keys_uri, mode): GetKeyDescriptionInput(json=encoded_json) ) - print(f"{output.key_description.value=}") - - keyvectors - keyring: IKeyring = keyvectors.create_test_vector_keyring( TestVectorKeyringInput( key_description=output.key_description ) ) + # Weird hack #2. + # Generating decrypt vectors for RSA keys. + # The MPL sets the encrypting keyring's keyName to "rsa-4096-private", + # somewhat undoing weird hack #1. + # Weird hack #1 allows the encrypting keyring to be created with a public key. + # However, it also changes the keyName of the encrypting keyring, + # which is changed back with this hack. + # If this is not done, then decryption fails + # (for BOTH native master keys and MPL keyrings) + # with error + # native master keys: "Unable to decrypt any data key" + # MPL: "Raw RSA Key was unable to decrypt any encrypted data key" + # + # digging, they key is unable to decrypt + # because the EDK keyProviderInfo differs from the keyring keyName, + # and this check fails: + # https://github.com/aws/aws-cryptographic-material-providers-library/blob/bd549c88cefc93ba8a2d204bd23134b3b12c69fb/AwsCryptographicMaterialProviders/dafny/AwsCryptographicMaterialProviders/src/Keyrings/RawRSAKeyring.dfy#L382 + # due to the two variables not being equal: + # edk.keyProviderInfo='rsa-4096-public' + # decrypting keyring.keyName='rsa-4096-private' + # + # changing the encrypting keyring's keyName back to 'rsa-4096-private' + # (somewhat undoing weird hack #1) + # sets edk.keyProviderInfo='rsa-4096-private', + # which allows this check to pass on decrypt. + # This "works" because all of the test vectors pass with these two hacks. + # But this seems weird. import _dafny import UTF8 @@ -126,9 +149,6 @@ def keyring(self, keys, keys_uri, mode): if keyring._impl._keyName == UTF8.default__.Encode(_dafny.Seq("rsa-4096-public")).value \ and (mode == "decrypt-generate" or mode == "encrypt"): if changed: - print("YES") - # input() - print(f"changed public to private") keyring._impl._keyName = UTF8.default__.Encode(_dafny.Seq("rsa-4096-private")).value @@ -146,13 +166,7 @@ def keyring_from_master_key_specs(keys, keys_uri, master_key_specs, mode): :return: Master key provider combining all loaded master keys :rtype: IKeyring """ - # print(f"{master_key_specs=}") - # input() keyrings = [spec.keyring(keys, keys_uri, mode) for spec in master_key_specs] - # print(f"speckeyrings {keyrings=}") - # input() - # print(f"speckeys {keys=}") - # input() primary = keyrings[0] others = keyrings[1:] From 3bf820cd0e85aa90da2feb7338d94a5403f2b657 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 12 Mar 2024 13:39:42 -0700 Subject: [PATCH 183/422] missing --- .../internal/keyvectors_provider.py | 25 +++++++++++++++++++ .../integration/integration_test_utils.py | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py diff --git a/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py b/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py new file mode 100644 index 000000000..12dc980e3 --- /dev/null +++ b/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py @@ -0,0 +1,25 @@ +from aws_cryptography_materialproviderstestvectorkeys.smithygenerated.\ + aws_cryptography_materialproviderstestvectorkeys.client import ( + KeyVectors, + ) +from aws_cryptography_materialproviderstestvectorkeys.smithygenerated.\ + aws_cryptography_materialproviderstestvectorkeys.config import ( + KeyVectorsConfig + ) + +keyvectors_instances = {} + +class KeyVectorsProvider: + """Singleton manager for the KeyVectors client. + + This is used because Dafny's JSON deserializer implementation is slow with large files. + It deserializes the file at keys_path and takes >1 minute to do this. + """ + + instance: KeyVectors + + @classmethod + def get_keyvectors(self, keys_path): + if not keys_path in keyvectors_instances: + keyvectors_instances[keys_path] = KeyVectors(KeyVectorsConfig(key_manifest_path=keys_path)) + return keyvectors_instances[keys_path] diff --git a/test_vector_handlers/test/integration/integration_test_utils.py b/test_vector_handlers/test/integration/integration_test_utils.py index 25efe6b79..fbe6cf7b7 100644 --- a/test_vector_handlers/test/integration/integration_test_utils.py +++ b/test_vector_handlers/test/integration/integration_test_utils.py @@ -33,5 +33,5 @@ def full_message_encrypt_vectors(): @pytest.fixture def full_message_decrypt_generation_vectors(): return os.path.join( - vectors_dir(), "features", "CANONICAL-GENERATED-MANIFESTS", "decrypt-generate-lite.json" + vectors_dir(), "features", "CANONICAL-GENERATED-MANIFESTS", "0006-awses-message-decryption-generation.v2.json" ) From eb40abb61fb16f1442e21fc620437f1f9b001c4a Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 12 Mar 2024 14:07:35 -0700 Subject: [PATCH 184/422] cleanup --- .../manifests/mpl_keyring.py | 35 +++++++++++-------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index 0f63ca143..d420a3be7 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -84,7 +84,9 @@ def keyring(self, keys, keys_uri, mode): if input_kwargs["type"] == "raw" \ and input_kwargs["encryption-algorithm"] == "rsa": - # Weird hack #1. + # Weird hack #1: + # Gets public key for encryption instead of private key. + # # If generating decrypt vectors (i.e. encrypting) # and the manifest specified an RSA private key, # change the input to KeyVectors to a public key. @@ -114,33 +116,36 @@ def keyring(self, keys, keys_uri, mode): ) ) - # Weird hack #2. - # Generating decrypt vectors for RSA keys. - # The MPL sets the encrypting keyring's keyName to "rsa-4096-private", - # somewhat undoing weird hack #1. + # Weird hack #2: + # Sets keyProviderInfo to "private" even though the material is "public". + # # Weird hack #1 allows the encrypting keyring to be created with a public key. - # However, it also changes the keyName of the encrypting keyring, - # which is changed back with this hack. + # However, it also changes the keyName of the encrypting keyring. + # This hack changes it back. + # # If this is not done, then decryption fails # (for BOTH native master keys and MPL keyrings) # with error # native master keys: "Unable to decrypt any data key" # MPL: "Raw RSA Key was unable to decrypt any encrypted data key" # - # digging, they key is unable to decrypt + # Digging, the keyring is unable to decrypt in the MPL # because the EDK keyProviderInfo differs from the keyring keyName, # and this check fails: # https://github.com/aws/aws-cryptographic-material-providers-library/blob/bd549c88cefc93ba8a2d204bd23134b3b12c69fb/AwsCryptographicMaterialProviders/dafny/AwsCryptographicMaterialProviders/src/Keyrings/RawRSAKeyring.dfy#L382 # due to the two variables not being equal: # edk.keyProviderInfo='rsa-4096-public' - # decrypting keyring.keyName='rsa-4096-private' + # keyring.keyName='rsa-4096-private' + # + # Changing the encrypting keyring's keyName back to 'rsa-4096-private' + # sets any EDKs this keyring encrypts to now have + # keyName="rsa-4096-private". + # However, keyvectors has still retrieved the public key material to encrypt with. + # So it any EDKs it encrypts will use the public material, but have keyName="rsa-4096-private". # - # changing the encrypting keyring's keyName back to 'rsa-4096-private' - # (somewhat undoing weird hack #1) - # sets edk.keyProviderInfo='rsa-4096-private', - # which allows this check to pass on decrypt. - # This "works" because all of the test vectors pass with these two hacks. - # But this seems weird. + # This configuration seems to be correct, because + # all of the test vectors (master keys and MPL) pass with these two hacks. + # But this seems weird, and we didn't have to do this in Java. import _dafny import UTF8 From bc0d5ff1e78e6af426826abe92e0ec144c597d66 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 12 Mar 2024 18:11:16 -0700 Subject: [PATCH 185/422] wip --- .../manifests/full_message/decrypt.py | 2 +- .../manifests/mpl_keyring.py | 1 - .../integration/integration_test_utils.py | 21 +++++++++++++++---- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index 0b8dfe3b8..752315ee6 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -56,7 +56,7 @@ CLIENT_NAME = "aws/aws-encryption-sdk-python" CURRENT_VERSION = 2 -SUPPORTED_VERSIONS = (2,) +SUPPORTED_VERSIONS = (2,4,) @attr.s(init=False) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index d420a3be7..7fd0f3323 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -156,7 +156,6 @@ def keyring(self, keys, keys_uri, mode): if changed: keyring._impl._keyName = UTF8.default__.Encode(_dafny.Seq("rsa-4096-private")).value - return keyring diff --git a/test_vector_handlers/test/integration/integration_test_utils.py b/test_vector_handlers/test/integration/integration_test_utils.py index fbe6cf7b7..4fee77378 100644 --- a/test_vector_handlers/test/integration/integration_test_utils.py +++ b/test_vector_handlers/test/integration/integration_test_utils.py @@ -18,20 +18,33 @@ import pytest -def vectors_dir(): - here = os.path.abspath(os.path.dirname(__file__)) +here = os.path.abspath(os.path.dirname(__file__)) + + +def legacy_vectors_dir(): return os.path.abspath(os.path.join(here, "..", "aws-crypto-tools-test-vector-framework")) +def mpl_vectors_dir(): + return os.path.abspath(os.path.join(here, "..", "golden-manifest-TODORENAMEANDGETFROMGHA")) + + @pytest.fixture def full_message_encrypt_vectors(): return os.path.join( - vectors_dir(), "features", "CANONICAL-GENERATED-MANIFESTS", "0003-awses-message-encryption.v2.json" + legacy_vectors_dir(), "features", "CANONICAL-GENERATED-MANIFESTS", "0003-awses-message-encryption.v2.json" + ) + + +@pytest.fixture +def full_message_decrypt_generation_vectors(): + return os.path.join( + legacy_vectors_dir(), "features", "CANONICAL-GENERATED-MANIFESTS", "0006-awses-message-decryption-generation.v2.json" ) @pytest.fixture def full_message_decrypt_generation_vectors(): return os.path.join( - vectors_dir(), "features", "CANONICAL-GENERATED-MANIFESTS", "0006-awses-message-decryption-generation.v2.json" + mpl_vectors_dir(), "manifest.json" ) From fb7d10c7cb547c401d2107900ad47ddff9024b22 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 11:11:04 -0700 Subject: [PATCH 186/422] requiredec working --- .../manifests/full_message/decrypt.py | 132 ++++++++++++++++-- 1 file changed, 124 insertions(+), 8 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index 752315ee6..1c04a83a2 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -200,6 +200,9 @@ class MessageDecryptionTestScenario(object): :type master_key_specs: iterable of :class:`MasterKeySpec` :param Callable master_key_provider_fn: :param bool keyrings: True if should decrypt with keyring interfaces; False otherwise + :param str cmm_type: `cmm` from test vector manifest; "Default" if not specified + :param str encryption_context: Any encryption context to validate on decrypt if using + keyrings AND the required encryption context CMM :param str description: Description of test scenario (optional) """ @@ -211,6 +214,7 @@ class MessageDecryptionTestScenario(object): master_key_provider_fn = attr.ib(validator=attr.validators.is_callable()) result = attr.ib(validator=attr.validators.instance_of(MessageDecryptionTestResult)) keyrings = attr.ib(validator=attr.validators.instance_of(bool)) + cmm_type = attr.ib(validator=attr.validators.instance_of(str)) decryption_method = attr.ib( default=None, validator=attr.validators.optional(attr.validators.instance_of(DecryptionMethod)) ) @@ -226,6 +230,8 @@ def __init__( master_key_specs, # type: Iterable[MasterKeySpec] master_key_provider_fn, # type: Callable keyrings, # type: bool + cmm_type, # type: str + encryption_context, # type: Dict[str, str] decryption_method=None, # type: Optional[DecryptionMethod] description=None, # type: Optional[str] ): # noqa=D107 @@ -239,9 +245,11 @@ def __init__( self.result = result self.master_key_specs = master_key_specs self.master_key_provider_fn = master_key_provider_fn + self.keyrings = keyrings + self.cmm_type = cmm_type + self.encryption_context = encryption_context self.decryption_method = decryption_method self.description = description - self.keyrings = keyrings attr.validate(self) @classmethod @@ -284,6 +292,26 @@ def master_key_provider_fn(): result_spec = scenario["result"] result = MessageDecryptionTestResult.from_result_spec(result_spec, plaintext_reader) + encryption_context = scenario["encryption-context"] + + # MPL test vectors add CMM types to the test vectors manifests + if "cmm" in scenario: + if scenario["cmm"] == "Default": + # Master keys and keyrings can handle default CMM + cmm_type = scenario["cmm"] + elif scenario["cmm"] == "RequiredEncryptionContext": + # Skip RequiredEncryptionContext CMM for master keys; + # This is unsupported for master keys + if keyrings: + cmm_type = scenario["cmm"] + else: + return None + else: + raise ValueError("Unrecognized cmm_type: " + cmm_type) + else: + # If unspecified, set "Default" as the default + cmm_type = "Default" + return cls( ciphertext_uri=scenario["ciphertext"], ciphertext=ciphertext_reader(scenario["ciphertext"]), @@ -291,6 +319,8 @@ def master_key_provider_fn(): master_key_provider_fn=master_key_provider_fn, result=result, keyrings=keyrings, + encryption_context=encryption_context, + cmm_type=cmm_type, decryption_method=decryption_method, description=scenario.get("description"), ) @@ -316,9 +346,50 @@ def scenario_spec(self): def _one_shot_decrypt(self): client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - if self.keyrings: - return client.decrypt(source=self.ciphertext, keyring=self.master_key_provider_fn()) - return client.decrypt(source=self.ciphertext, key_provider=self.master_key_provider_fn()) + if self.cmm_type == "Default": + if self.keyrings: + return client.decrypt(source=self.ciphertext, keyring=self.master_key_provider_fn()) + return client.decrypt(source=self.ciphertext, key_provider=self.master_key_provider_fn()) + elif self.cmm_type == "RequiredEncryptionContext": + # We need to make a custom CMM and pass it into the client + assert self.keyrings + + from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders + from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig + from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager + from aws_cryptographic_materialproviders.mpl.models import ( + CreateDefaultCryptographicMaterialsManagerInput, + CreateRequiredEncryptionContextCMMInput, + ) + + + mpl: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + config=MaterialProvidersConfig() + ) + + underlying_cmm: ICryptographicMaterialsManager = \ + mpl.create_default_cryptographic_materials_manager( + CreateDefaultCryptographicMaterialsManagerInput( + keyring=self.master_key_provider_fn() + ) + ) + + required_ec_cmm: ICryptographicMaterialsManager = \ + mpl.create_required_encryption_context_cmm( + CreateRequiredEncryptionContextCMMInput( + # Currently, the test vector manifest requires that + # if using the required encryption context CMM, + # both and only "key1" and "key2" are required. + required_encryption_context_keys=["key1", "key2"], + underlying_cmm=underlying_cmm, + ) + ) + + return client.decrypt( + source=self.ciphertext, + materials_manager=required_ec_cmm, + encryption_context = self.encryption_context, + ) def _streaming_decrypt(self): result = bytearray() @@ -328,10 +399,48 @@ def _streaming_decrypt(self): "source": self.ciphertext, "mode": "d" } - if self.keyrings: - kwargs["keyring"] = self.master_key_provider_fn() - else: - kwargs["key_provider"] = self.master_key_provider_fn() + if self.cmm_type == "Default": + if self.keyrings: + kwargs["keyring"] = self.master_key_provider_fn() + else: + kwargs["key_provider"] = self.master_key_provider_fn() + elif self.cmm_type == "RequiredEncryptionContext": + # We need to make a custom CMM and pass it into the client + assert self.keyrings + + from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders + from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig + from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager + from aws_cryptographic_materialproviders.mpl.models import ( + CreateDefaultCryptographicMaterialsManagerInput, + CreateRequiredEncryptionContextCMMInput, + ) + + + mpl: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + config=MaterialProvidersConfig() + ) + + underlying_cmm: ICryptographicMaterialsManager = \ + mpl.create_default_cryptographic_materials_manager( + CreateDefaultCryptographicMaterialsManagerInput( + keyring=self.master_key_provider_fn() + ) + ) + + required_ec_cmm: ICryptographicMaterialsManager = \ + mpl.create_required_encryption_context_cmm( + CreateRequiredEncryptionContextCMMInput( + # Currently, the test vector manifest requires that + # if using the required encryption context CMM, + # both and only "key1" and "key2" are required. + required_encryption_context_keys=["key1", "key2"], + underlying_cmm=underlying_cmm, + ) + ) + + kwargs["materials_manager"] = required_ec_cmm + kwargs["encryption_context"] = self.encryption_context with client.stream(**kwargs) as decryptor: for chunk in decryptor: @@ -483,6 +592,13 @@ def from_file(cls, input_file, keyrings): # Merge keyring scenarios into test_scenarios test_scenarios = {**keyrings_test_scenarios, **test_scenarios} + # Remove any `None` scenarios from test scenarios. + # `None` scenarios indicate the loader determined the scenario is invalid. + # e.g. cmm_type = "RequiredEncryptionContext" with master keys + for name in list(test_scenarios.keys()): + if test_scenarios[name] is None: + del test_scenarios[name] + return cls( keys_uri=keys_uri, keys=keys, From 344824b1250b73f193527065b93121fb13b31645 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 11:45:08 -0700 Subject: [PATCH 187/422] debug cb --- .../src/awses_test_vectors/manifests/mpl_keyring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index 7fd0f3323..8b3fa98d8 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -77,7 +77,7 @@ def keyring(self, keys, keys_uri, mode): "encryption-algorithm": self.encryption_algorithm, } - if self.padding_algorithm is not None and self.padding_algorithm is not "": + if self.padding_algorithm is not None and self.padding_algorithm != "": input_kwargs["padding-algorithm"] = self.padding_algorithm if self.padding_hash is not None: input_kwargs["padding-hash"] = self.padding_hash From 697f2ffbb74d7dfb18b508f299fc261952b3d7ac Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 12:06:31 -0700 Subject: [PATCH 188/422] fix cb --- .../full_message/decrypt_generation.py | 2 ++ .../integration/integration_test_utils.py | 13 +++++++++++- .../test_i_full_message_encrypt_keyrings.py | 20 +++++++++---------- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 5536fc845..a048dcc32 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -451,6 +451,8 @@ def decryption_test_scenario_pair(self, ciphertext_writer, ciphertext_to_decrypt decryption_method=self.decryption_method, result=expected_result, keyrings=self.keyrings, + cmm_type="Default", + encryption_context={} ), ) diff --git a/test_vector_handlers/test/integration/integration_test_utils.py b/test_vector_handlers/test/integration/integration_test_utils.py index 4fee77378..696dc8648 100644 --- a/test_vector_handlers/test/integration/integration_test_utils.py +++ b/test_vector_handlers/test/integration/integration_test_utils.py @@ -29,6 +29,10 @@ def mpl_vectors_dir(): return os.path.abspath(os.path.join(here, "..", "golden-manifest-TODORENAMEANDGETFROMGHA")) +def required_ec_vectors_dir(): + return os.path.abspath(os.path.join(here, "..", "required-ec-TODORENAMEANDGETFROMGHA")) + + @pytest.fixture def full_message_encrypt_vectors(): return os.path.join( @@ -44,7 +48,14 @@ def full_message_decrypt_generation_vectors(): @pytest.fixture -def full_message_decrypt_generation_vectors(): +def mpl_decrypt_vectors(): return os.path.join( mpl_vectors_dir(), "manifest.json" ) + + +@pytest.fixture +def required_encryption_context_cmm_decrypt_vectors(): + return os.path.join( + required_ec_vectors_dir(), "manifest.json" + ) \ No newline at end of file diff --git a/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py b/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py index de5f10299..37c33e417 100644 --- a/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py +++ b/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py @@ -34,16 +34,16 @@ def test_full_message_cycle_canonical_full(tmpdir, full_message_decrypt_generation_vectors): # Generate vectors using keyring interfaces - keyring_output_dir = tmpdir.join("output-keyrings") - print("Generating vectors with keyrings... ", end="") - full_message_decrypt_generate.cli([ - "--output", - str(keyring_output_dir), - "--input", - full_message_decrypt_generation_vectors, - "--keyrings" - ]) - print("done") + # keyring_output_dir = tmpdir.join("output-keyrings") + # print("Generating vectors with keyrings... ", end="") + # full_message_decrypt_generate.cli([ + # "--output", + # str(keyring_output_dir), + # "--input", + # full_message_decrypt_generation_vectors, + # "--keyrings" + # ]) + # print("done") # print("Generating vectors with master keys... ", end="") # # Generate vectors using master key interfaces From 7dbc00ad796ab4239761542d11c5adce470df5f2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 12:12:20 -0700 Subject: [PATCH 189/422] fix cb --- .../test_i_full_message_encrypt_keyrings.py | 82 +++++++++---------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py b/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py index 37c33e417..c86a23ab9 100644 --- a/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py +++ b/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py @@ -27,54 +27,54 @@ pytestmark = [pytest.mark.integ] -# def test_full_message_encrypt_canonical_full(full_message_encrypt_vectors): -# full_message_encrypt.cli(["--input", full_message_encrypt_vectors]) -# full_message_encrypt.cli(["--input", full_message_encrypt_vectors], "--keyrings") +def test_full_message_encrypt_canonical_full(full_message_encrypt_vectors): + full_message_encrypt.cli(["--input", full_message_encrypt_vectors]) + full_message_encrypt.cli(["--input", full_message_encrypt_vectors], "--keyrings") def test_full_message_cycle_canonical_full(tmpdir, full_message_decrypt_generation_vectors): # Generate vectors using keyring interfaces - # keyring_output_dir = tmpdir.join("output-keyrings") - # print("Generating vectors with keyrings... ", end="") - # full_message_decrypt_generate.cli([ - # "--output", - # str(keyring_output_dir), - # "--input", - # full_message_decrypt_generation_vectors, - # "--keyrings" - # ]) - # print("done") + keyring_output_dir = tmpdir.join("output-keyrings") + print("Generating vectors with keyrings... ", end="") + full_message_decrypt_generate.cli([ + "--output", + str(keyring_output_dir), + "--input", + full_message_decrypt_generation_vectors, + "--keyrings" + ]) + print("done") - # print("Generating vectors with master keys... ", end="") - # # Generate vectors using master key interfaces - # master_key_output_dir = tmpdir.join("output-master-key") - # full_message_decrypt_generate.cli([ - # "--output", - # str(master_key_output_dir), - # "--input", - # full_message_decrypt_generation_vectors - # ]) - # print("done") + print("Generating vectors with master keys... ", end="") + # Generate vectors using master key interfaces + master_key_output_dir = tmpdir.join("output-master-key") + full_message_decrypt_generate.cli([ + "--output", + str(master_key_output_dir), + "--input", + full_message_decrypt_generation_vectors + ]) + print("done") - # # Validate that vectors generated using keyring interfaces - # # can be decrypted by BOTH keyring and master key interfaces - # keyring_decrypt_manifest_file = keyring_output_dir.join("manifest.json") - # print("Decrypting keyring-encrypted vectors with keyrings... ", end="") - # full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file), "--keyrings"]) - # print("done") + # Validate that vectors generated using keyring interfaces + # can be decrypted by BOTH keyring and master key interfaces + keyring_decrypt_manifest_file = keyring_output_dir.join("manifest.json") + print("Decrypting keyring-encrypted vectors with keyrings... ", end="") + full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file), "--keyrings"]) + print("done") - # print("Decrypting keyring-encrypted vectors with master keys... ", end="") - # full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file)]) - # print("done") + print("Decrypting keyring-encrypted vectors with master keys... ", end="") + full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file)]) + print("done") - # # Validate that vectors generated using master key interfaces - # # can be decrypted by BOTH keyring and master key interfaces - # master_key_decrypt_manifest_file = keyring_output_dir.join("manifest.json") + # Validate that vectors generated using master key interfaces + # can be decrypted by BOTH keyring and master key interfaces + master_key_decrypt_manifest_file = keyring_output_dir.join("manifest.json") - # print("Decrypting master key-encrypted vectors with keyrings... ", end="") - # full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file), "--keyrings"]) - # print("done") + print("Decrypting master key-encrypted vectors with keyrings... ", end="") + full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file), "--keyrings"]) + print("done") - # print("Decrypting master key-encrypted vectors with master keys... ", end="") - # full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file)]) - # print("done") + print("Decrypting master key-encrypted vectors with master keys... ", end="") + full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file)]) + print("done") From 7036337ce16b1017bff3d54e4f8bbf40ab92e9d8 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 12:41:00 -0700 Subject: [PATCH 190/422] debug cb --- .../integration/commands/test_i_full_message_encrypt_keyrings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py b/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py index c86a23ab9..6d037a586 100644 --- a/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py +++ b/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py @@ -22,7 +22,6 @@ full_message_encrypt_vectors, ) -import cProfile pytestmark = [pytest.mark.integ] From 7a44191d33f95f2f9b7e6c0fddd89a134c03b228 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 12:46:27 -0700 Subject: [PATCH 191/422] debug gha --- src/aws_encryption_sdk/materials_managers/mpl/cmm.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index 1655e0746..c4a1a1fb6 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -70,8 +70,6 @@ def get_encryption_materials( mpl_output: MPL_GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) - print(f"get {mpl_output=}") - # ???????????????????????????? # kpis = set() # for edk in mpl_output.encryption_materials.encrypted_data_keys: From 298235ab8ce31b0ac8a7e26af56f18b3f424fe7c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 12:50:01 -0700 Subject: [PATCH 192/422] temp rm cov --- test_vector_handlers/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index 18b3710e5..5c96def2b 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -37,7 +37,7 @@ envlist = # release :: Builds dist files and uploads to pypi pypirc profile. [testenv:base-command] -commands = python3 -m cProfile -o profile.txt -m pytest --basetemp={envtmpdir} -l --cov awses_test_vectors {posargs} +commands = python3 -m cProfile -o profile.txt -m pytest --basetemp={envtmpdir} -l {posargs} [testenv] passenv = From 4c1d0a06e4e4e5ccf887f63d66c579cdc2edcde0 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 12:51:10 -0700 Subject: [PATCH 193/422] temp rm cov --- src/aws_encryption_sdk/materials_managers/mpl/cmm.py | 7 ++++--- src/aws_encryption_sdk/materials_managers/mpl/materials.py | 2 +- .../src/awses_test_vectors/manifests/mpl_keyring.py | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index c4a1a1fb6..8edf52151 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -139,10 +139,10 @@ def decrypt_materials( try: mpl_input: 'MPL_DecryptMaterialsInput' = \ CryptoMaterialsManagerFromMPL._create_mpl_decrypt_materials_input_from_request(request) - print(f"{mpl_input.as_dict()=}") + # print(f"{mpl_input.as_dict()=}") # input() mpl_output: 'MPL_DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(mpl_input) - print(f"{mpl_output.as_dict()=}") + # print(f"{mpl_output.as_dict()=}") # input() return DecryptionMaterialsFromMPL(mpl_output.decryption_materials) except AwsCryptographicMaterialProvidersException as mpl_exception: @@ -150,7 +150,8 @@ def decrypt_materials( # so customers only have to catch ESDK error types. raise AWSEncryptionSDKClientError(mpl_exception) except COE as coe: - print(f"{coe.list=}") + # print(f"{coe.list=}") + pass # raise AWSEncryptionSDKClientError(coe) @staticmethod diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index 9f3d4f0fb..b70e48efe 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -75,7 +75,7 @@ def encrypted_data_keys(self) -> List[Native_EncryptedDataKey]: ), encrypted_data_key=mpl_edk.ciphertext, ) for mpl_edk in mpl_edk_list} - print(f"{key_blob_list=}") + # print(f"{key_blob_list=}") # input() return key_blob_list diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index 8b3fa98d8..40421e931 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -63,7 +63,7 @@ def keyring(self, keys, keys_uri, mode): ''' - print(f"{keys=}") + # print(f"{keys=}") keyvectors = KeyVectorsProvider.get_keyvectors(keys_path=keys_uri) From 12e00605b27183772fffcb916963940ff4f31856 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 12:51:51 -0700 Subject: [PATCH 194/422] debug gha --- .../src/awses_test_vectors/manifests/mpl_keyring.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index 40421e931..d914b5501 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -63,7 +63,6 @@ def keyring(self, keys, keys_uri, mode): ''' - # print(f"{keys=}") keyvectors = KeyVectorsProvider.get_keyvectors(keys_path=keys_uri) From 9ca61e24c9b9b2563f04c6d251de0c33c5c5cd7d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 13:00:37 -0700 Subject: [PATCH 195/422] debug gha --- .../materials_managers/mpl/cmm.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index 8edf52151..c398904a9 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -67,7 +67,7 @@ def get_encryption_materials( CryptoMaterialsManagerFromMPL._native_to_mpl_get_encryption_materials( request ) - + mpl_output: MPL_GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) # ???????????????????????????? @@ -77,7 +77,7 @@ def get_encryption_materials( # print(kpis) # input - + # if len(kpis) == 1: # for edk in mpl_output.encryption_materials.encrypted_data_keys: # if edk.key_provider_info == b"rsa-4096-public": @@ -135,24 +135,15 @@ def decrypt_materials( Returns a DecryptionMaterialsFromMPL for the configured CMM. :param request: Request for decryption materials """ - from aws_cryptographic_materialproviders.smithygenerated.aws_cryptography_materialproviders.errors import CollectionOfErrors as COE try: mpl_input: 'MPL_DecryptMaterialsInput' = \ CryptoMaterialsManagerFromMPL._create_mpl_decrypt_materials_input_from_request(request) - # print(f"{mpl_input.as_dict()=}") - # input() mpl_output: 'MPL_DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(mpl_input) - # print(f"{mpl_output.as_dict()=}") - # input() return DecryptionMaterialsFromMPL(mpl_output.decryption_materials) except AwsCryptographicMaterialProvidersException as mpl_exception: # Wrap MPL error into the ESDK error type # so customers only have to catch ESDK error types. raise AWSEncryptionSDKClientError(mpl_exception) - except COE as coe: - # print(f"{coe.list=}") - pass - # raise AWSEncryptionSDKClientError(coe) @staticmethod def _native_algorithm_id_to_mpl_algorithm_id(native_algorithm_id: str) -> 'MPL_AlgorithmSuiteIdESDK': From 5deac12dc81b6988799e32e03560a6b2685349eb Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 13:13:35 -0700 Subject: [PATCH 196/422] debug cb --- .../awses_test_vectors/manifests/full_message/decrypt.py | 6 ++++-- test_vector_handlers/tox.ini | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index 1c04a83a2..430b81157 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -352,7 +352,8 @@ def _one_shot_decrypt(self): return client.decrypt(source=self.ciphertext, key_provider=self.master_key_provider_fn()) elif self.cmm_type == "RequiredEncryptionContext": # We need to make a custom CMM and pass it into the client - assert self.keyrings + if not self.keyrings: + raise ValueError("Must provide keyrings arg to use RequiredEncryptionContext") from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig @@ -406,7 +407,8 @@ def _streaming_decrypt(self): kwargs["key_provider"] = self.master_key_provider_fn() elif self.cmm_type == "RequiredEncryptionContext": # We need to make a custom CMM and pass it into the client - assert self.keyrings + if not self.keyrings: + raise ValueError("Must provide keyrings arg to use RequiredEncryptionContext") from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index 5c96def2b..497baed4b 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -37,7 +37,7 @@ envlist = # release :: Builds dist files and uploads to pypi pypirc profile. [testenv:base-command] -commands = python3 -m cProfile -o profile.txt -m pytest --basetemp={envtmpdir} -l {posargs} +commands = python3 -m cProfile -o profile.txt -m pytest --basetemp={envtmpdir} -l --cov awses_test_vectors {posargs} [testenv] passenv = @@ -55,7 +55,7 @@ deps = .. commands = awses_local: {[testenv:base-command]commands} test/integration - mplvectors: {[testenv:base-command]commands} test/keyrings -s -v + mplvectors: {[testenv:base-command]commands} test/keyrings [testenv:full-encrypt] basepython = python3 From ff99fe7990677d0914ae320acb50b0dd2856e1c9 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 14:37:03 -0700 Subject: [PATCH 197/422] debug cb --- .../manifests/full_message/decrypt.py | 51 +++++++++++++++++++ .../test_i_full_message_encrypt_keyrings.py | 2 +- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index 430b81157..cc4a6eac0 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -452,6 +452,57 @@ def _streaming_decrypt(self): def _streaming_decrypt_unsigned(self): result = bytearray() client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + + stream_kwargs = { + "source": self.ciphertext, + "mode": "decrypt-unsigned", + } + + if self.cmm_type == "Default": + if self.keyrings: + stream_kwargs["keyring"] = self.master_key_provider_fn() + else: + stream_kwargs["key_provider"] = self.master_key_provider_fn() + elif self.cmm_type == "RequiredEncryptionContext": + # We need to make a custom CMM and pass it into the client + if not self.keyrings: + raise ValueError("Must provide keyrings arg to use RequiredEncryptionContext") + + from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders + from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig + from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager + from aws_cryptographic_materialproviders.mpl.models import ( + CreateDefaultCryptographicMaterialsManagerInput, + CreateRequiredEncryptionContextCMMInput, + ) + + + mpl: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + config=MaterialProvidersConfig() + ) + + underlying_cmm: ICryptographicMaterialsManager = \ + mpl.create_default_cryptographic_materials_manager( + CreateDefaultCryptographicMaterialsManagerInput( + keyring=self.master_key_provider_fn() + ) + ) + + required_ec_cmm: ICryptographicMaterialsManager = \ + mpl.create_required_encryption_context_cmm( + CreateRequiredEncryptionContextCMMInput( + # Currently, the test vector manifest requires that + # if using the required encryption context CMM, + # both and only "key1" and "key2" are required. + required_encryption_context_keys=["key1", "key2"], + underlying_cmm=underlying_cmm, + ) + ) + + stream_kwargs["materials_manager"] = required_ec_cmm + stream_kwargs["encryption_context"] = self.encryption_context + + with client.stream( source=self.ciphertext, mode="decrypt-unsigned", key_provider=self.master_key_provider_fn() ) as decryptor: diff --git a/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py b/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py index 6d037a586..7ecbdb69f 100644 --- a/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py +++ b/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py @@ -28,7 +28,7 @@ def test_full_message_encrypt_canonical_full(full_message_encrypt_vectors): full_message_encrypt.cli(["--input", full_message_encrypt_vectors]) - full_message_encrypt.cli(["--input", full_message_encrypt_vectors], "--keyrings") + full_message_encrypt.cli(["--input", full_message_encrypt_vectors, "--keyrings"]) def test_full_message_cycle_canonical_full(tmpdir, full_message_decrypt_generation_vectors): From 6a4b7045c02cbec3afbed7c33e078ed13c10fafb Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 14:37:22 -0700 Subject: [PATCH 198/422] debug cb --- .../src/awses_test_vectors/manifests/full_message/decrypt.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index cc4a6eac0..1f214c89e 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -502,10 +502,7 @@ def _streaming_decrypt_unsigned(self): stream_kwargs["materials_manager"] = required_ec_cmm stream_kwargs["encryption_context"] = self.encryption_context - - with client.stream( - source=self.ciphertext, mode="decrypt-unsigned", key_provider=self.master_key_provider_fn() - ) as decryptor: + with client.stream(**stream_kwargs) as decryptor: for chunk in decryptor: result.extend(chunk) return result, decryptor.header From 0dbd4f636d7ff767cf56e02977a669c5775fcdbc Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 15:38:34 -0700 Subject: [PATCH 199/422] fix cb --- .../manifests/full_message/decrypt.py | 70 +++++++++---------- .../full_message/decrypt_generation.py | 19 ++++- .../manifests/full_message/encrypt.py | 2 +- .../manifests/mpl_keyring.py | 67 +++++++++--------- 4 files changed, 83 insertions(+), 75 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index 1f214c89e..a0f847762 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -37,6 +37,14 @@ try: from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_from_master_key_specs + from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders + from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig + from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager + from aws_cryptographic_materialproviders.mpl.models import ( + CreateDefaultCryptographicMaterialsManagerInput, + CreateRequiredEncryptionContextCMMInput, + ) + _HAS_MPL = True except ImportError: _HAS_MPL = False @@ -262,6 +270,7 @@ def from_scenario( keyrings, # type: bool keys_uri, # type: str ): + # pylint: disable=too-many-locals # type: (...) -> MessageDecryptionTestScenario """Load from a scenario specification. @@ -284,7 +293,7 @@ def from_scenario( def master_key_provider_fn(): if keyrings: - return keyring_from_master_key_specs(keys, keys_uri, master_key_specs, "decrypt") + return keyring_from_master_key_specs(keys_uri, master_key_specs, "decrypt") return master_key_provider_from_master_key_specs(keys, master_key_specs) decryption_method_spec = scenario.get("decryption-method") @@ -293,7 +302,7 @@ def master_key_provider_fn(): result = MessageDecryptionTestResult.from_result_spec(result_spec, plaintext_reader) encryption_context = scenario["encryption-context"] - + # MPL test vectors add CMM types to the test vectors manifests if "cmm" in scenario: if scenario["cmm"] == "Default": @@ -350,19 +359,13 @@ def _one_shot_decrypt(self): if self.keyrings: return client.decrypt(source=self.ciphertext, keyring=self.master_key_provider_fn()) return client.decrypt(source=self.ciphertext, key_provider=self.master_key_provider_fn()) - elif self.cmm_type == "RequiredEncryptionContext": + if self.cmm_type == "RequiredEncryptionContext": # We need to make a custom CMM and pass it into the client if not self.keyrings: raise ValueError("Must provide keyrings arg to use RequiredEncryptionContext") - - from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders - from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig - from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager - from aws_cryptographic_materialproviders.mpl.models import ( - CreateDefaultCryptographicMaterialsManagerInput, - CreateRequiredEncryptionContextCMMInput, - ) - + if not _HAS_MPL: + raise ValueError("Must install the aws-cryptographic-material-providers library" + "to use RequiredEncryptionContext") mpl: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( config=MaterialProvidersConfig() @@ -378,20 +381,23 @@ def _one_shot_decrypt(self): required_ec_cmm: ICryptographicMaterialsManager = \ mpl.create_required_encryption_context_cmm( CreateRequiredEncryptionContextCMMInput( - # Currently, the test vector manifest requires that + # Currently, the test vector manifest requires that # if using the required encryption context CMM, # both and only "key1" and "key2" are required. required_encryption_context_keys=["key1", "key2"], underlying_cmm=underlying_cmm, ) ) - + return client.decrypt( source=self.ciphertext, materials_manager=required_ec_cmm, encryption_context = self.encryption_context, ) + # If the cmm type was not in if/elif above, raise error + raise ValueError(f"Unrecognized cmm_type: {self.cmm_type}") + def _streaming_decrypt(self): result = bytearray() client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) @@ -409,15 +415,9 @@ def _streaming_decrypt(self): # We need to make a custom CMM and pass it into the client if not self.keyrings: raise ValueError("Must provide keyrings arg to use RequiredEncryptionContext") - - from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders - from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig - from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager - from aws_cryptographic_materialproviders.mpl.models import ( - CreateDefaultCryptographicMaterialsManagerInput, - CreateRequiredEncryptionContextCMMInput, - ) - + if not _HAS_MPL: + raise ValueError("Must install the aws-cryptographic-material-providers library" + "to use RequiredEncryptionContext") mpl: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( config=MaterialProvidersConfig() @@ -433,16 +433,18 @@ def _streaming_decrypt(self): required_ec_cmm: ICryptographicMaterialsManager = \ mpl.create_required_encryption_context_cmm( CreateRequiredEncryptionContextCMMInput( - # Currently, the test vector manifest requires that + # Currently, the test vector manifest requires that # if using the required encryption context CMM, # both and only "key1" and "key2" are required. required_encryption_context_keys=["key1", "key2"], underlying_cmm=underlying_cmm, ) ) - + kwargs["materials_manager"] = required_ec_cmm kwargs["encryption_context"] = self.encryption_context + else: + raise ValueError(f"Unrecognized cmm_type: {self.cmm_type}") with client.stream(**kwargs) as decryptor: for chunk in decryptor: @@ -467,15 +469,9 @@ def _streaming_decrypt_unsigned(self): # We need to make a custom CMM and pass it into the client if not self.keyrings: raise ValueError("Must provide keyrings arg to use RequiredEncryptionContext") - - from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders - from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig - from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager - from aws_cryptographic_materialproviders.mpl.models import ( - CreateDefaultCryptographicMaterialsManagerInput, - CreateRequiredEncryptionContextCMMInput, - ) - + if not _HAS_MPL: + raise ValueError("Must install the aws-cryptographic-material-providers library" + "to use RequiredEncryptionContext") mpl: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( config=MaterialProvidersConfig() @@ -491,16 +487,18 @@ def _streaming_decrypt_unsigned(self): required_ec_cmm: ICryptographicMaterialsManager = \ mpl.create_required_encryption_context_cmm( CreateRequiredEncryptionContextCMMInput( - # Currently, the test vector manifest requires that + # Currently, the test vector manifest requires that # if using the required encryption context CMM, # both and only "key1" and "key2" are required. required_encryption_context_keys=["key1", "key2"], underlying_cmm=underlying_cmm, ) ) - + stream_kwargs["materials_manager"] = required_ec_cmm stream_kwargs["encryption_context"] = self.encryption_context + else: + raise ValueError(f"Unrecognized cmm_type: {self.cmm_type}") with client.stream(**stream_kwargs) as decryptor: for chunk in decryptor: diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index a048dcc32..8d80c46e2 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -40,8 +40,10 @@ from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_from_master_key_specs + + _HAS_MPL = True except ImportError: - pass + _HAS_MPL = False from awses_test_vectors.internal.defaults import ENCODING @@ -314,7 +316,18 @@ def __init__(self, master_key_provider): Create a new CMM that wraps a new DefaultCryptoMaterialsManager based on the given master key provider. """ - self.wrapped_default_cmm = DefaultCryptoMaterialsManager(master_key_provider) + if isinstance(master_key_provider, MasterKeyProvider): + self.wrapped_default_cmm = DefaultCryptoMaterialsManager(master_key_provider) + elif _HAS_MPL and isinstance(master_key_provider, IKeyring): + mpl = AwsCryptographicMaterialProviders(MaterialProvidersConfig()) + mpl_cmm = mpl.create_default_cryptographic_materials_manager( + CreateDefaultCryptographicMaterialsManagerInput( + keyring=master_key_provider + ) + ) + self.wrapped_default_cmm = CryptoMaterialsManagerFromMPL(mpl_cmm=mpl_cmm) + else: + raise TypeError(f"Unrecognized master_key_provider type: {master_key_provider}") def get_encryption_materials(self, request): """ @@ -405,7 +418,7 @@ def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): def decryption_master_key_provider_fn(): if keyrings: - return keyring_from_master_key_specs(keys, keys_uri, decryption_master_key_specs, "decrypt-generation") + return keyring_from_master_key_specs(keys_uri, decryption_master_key_specs, "decrypt-generation") return master_key_provider_from_master_key_specs(keys, decryption_master_key_specs) else: diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index a3d351317..0c2580fa8 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -112,7 +112,7 @@ def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): def master_key_provider_fn(): if keyrings: - return keyring_from_master_key_specs(keys, keys_uri, master_key_specs, "encrypt") + return keyring_from_master_key_specs(keys_uri, master_key_specs, "encrypt") return master_key_provider_from_master_key_specs(keys, master_key_specs) return cls( diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index d914b5501..7094ec35c 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -14,8 +14,11 @@ This REQUIRES the aws-cryptographic-material-providers library. """ +import json import attr +# Ignore missing MPL for pylint, but the MPL is required for this example +# noqa pylint: disable=import-error from aws_cryptography_materialproviderstestvectorkeys.smithygenerated.\ aws_cryptography_materialproviderstestvectorkeys.models import ( GetKeyDescriptionInput, @@ -27,10 +30,12 @@ from aws_cryptographic_materialproviders.mpl.references import IKeyring from aws_cryptographic_materialproviders.mpl.models import CreateMultiKeyringInput +import _dafny +import UTF8 + from awses_test_vectors.internal.keyvectors_provider import KeyVectorsProvider -from awses_test_vectors.manifests.keys import KeysManifest # noqa pylint disable=unused-import +from awses_test_vectors.manifests.keys import KeysManifest -import json from .master_key import MasterKeySpec @@ -49,24 +54,18 @@ class KeyringSpec(MasterKeySpec): # pylint: disable=too-many-instance-attribute :param str padding_hash: Wrapping key padding hash (required for raw master keys) """ - def keyring(self, keys, keys_uri, mode): + def keyring(self, keys_uri, mode): # type: (KeysManifest) -> IKeyring """Build a keyring using this specification. :param str keys_uri: Path to the keys manifest """ - ''' - encryptmaterials keyProviderInfo = rsa-4096-public' - MUST be private. - somehow, it is writing "rsa-4096-public". - - ''' - - keyvectors = KeyVectorsProvider.get_keyvectors(keys_path=keys_uri) - changed = False + # Variable to flag whether we changed anything in weird hack #1. + # Signals to weird hack #2 whether it should execute. + changed_key_name_from_private_to_public = False # Construct the input to KeyVectorsConfig input_kwargs = { @@ -74,7 +73,7 @@ def keyring(self, keys, keys_uri, mode): "key": self.key_name, "provider-id": self.provider_id, "encryption-algorithm": self.encryption_algorithm, - + } if self.padding_algorithm is not None and self.padding_algorithm != "": input_kwargs["padding-algorithm"] = self.padding_algorithm @@ -85,7 +84,7 @@ def keyring(self, keys, keys_uri, mode): and input_kwargs["encryption-algorithm"] == "rsa": # Weird hack #1: # Gets public key for encryption instead of private key. - # + # # If generating decrypt vectors (i.e. encrypting) # and the manifest specified an RSA private key, # change the input to KeyVectors to a public key. @@ -93,8 +92,8 @@ def keyring(self, keys, keys_uri, mode): # If this is not done, then keyring.OnEncrypt fails with # "A RawRSAKeyring without a public key cannot provide OnEncrypt" if input_kwargs["key"] == "rsa-4096-private" \ - and (mode == "decrypt-generate" or mode == "encrypt"): - changed = True + and mode in ("decrypt-generate", "encrypt"): + changed_key_name_from_private_to_public = True input_kwargs["key"] = "rsa-4096-public" # Specify default padding-hash if "padding-hash" not in input_kwargs: @@ -117,17 +116,17 @@ def keyring(self, keys, keys_uri, mode): # Weird hack #2: # Sets keyProviderInfo to "private" even though the material is "public". - # + # # Weird hack #1 allows the encrypting keyring to be created with a public key. # However, it also changes the keyName of the encrypting keyring. # This hack changes it back. - # + # # If this is not done, then decryption fails # (for BOTH native master keys and MPL keyrings) - # with error + # with error # native master keys: "Unable to decrypt any data key" # MPL: "Raw RSA Key was unable to decrypt any encrypted data key" - # + # # Digging, the keyring is unable to decrypt in the MPL # because the EDK keyProviderInfo differs from the keyring keyName, # and this check fails: @@ -135,30 +134,28 @@ def keyring(self, keys, keys_uri, mode): # due to the two variables not being equal: # edk.keyProviderInfo='rsa-4096-public' # keyring.keyName='rsa-4096-private' - # - # Changing the encrypting keyring's keyName back to 'rsa-4096-private' - # sets any EDKs this keyring encrypts to now have + # + # Changing the encrypting keyring's keyName back to 'rsa-4096-private' + # sets any EDKs this keyring encrypts to now have # keyName="rsa-4096-private". # However, keyvectors has still retrieved the public key material to encrypt with. # So it any EDKs it encrypts will use the public material, but have keyName="rsa-4096-private". - # - # This configuration seems to be correct, because + # + # This configuration seems to be correct, because # all of the test vectors (master keys and MPL) pass with these two hacks. # But this seems weird, and we didn't have to do this in Java. - import _dafny - import UTF8 - - if hasattr(keyring, "_impl"): - if hasattr(keyring._impl, "_keyName"): + if hasattr(keyring, "_impl"): # pylint: disable=protected-access + if hasattr(keyring._impl, "_keyName"): # pylint: disable=protected-access if keyring._impl._keyName == UTF8.default__.Encode(_dafny.Seq("rsa-4096-public")).value \ - and (mode == "decrypt-generate" or mode == "encrypt"): - if changed: - keyring._impl._keyName = UTF8.default__.Encode(_dafny.Seq("rsa-4096-private")).value + and mode in ("decrypt-generate", "encrypt"): # pylint: disable=protected-access + if changed_key_name_from_private_to_public: + # pylint: disable=protected-access + keyring._impl._keyName = UTF8.default__.Encode(_dafny.Seq("rsa-4096-private")).value return keyring -def keyring_from_master_key_specs(keys, keys_uri, master_key_specs, mode): +def keyring_from_master_key_specs(keys_uri, master_key_specs, mode): # type: (str, list[KeyringSpec]) -> IKeyring """Build and combine all keyrings identified by the provided specs and using the provided keys. @@ -169,7 +166,7 @@ def keyring_from_master_key_specs(keys, keys_uri, master_key_specs, mode): :return: Master key provider combining all loaded master keys :rtype: IKeyring """ - keyrings = [spec.keyring(keys, keys_uri, mode) for spec in master_key_specs] + keyrings = [spec.keyring(keys_uri, mode) for spec in master_key_specs] primary = keyrings[0] others = keyrings[1:] From 357594b766e4d6c8a7c1815c92fdd271f85dcf95 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 16:37:48 -0700 Subject: [PATCH 200/422] debug cb --- .../manifests/full_message/decrypt_generation.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 8d80c46e2..b6806d8ca 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -41,6 +41,11 @@ from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_from_master_key_specs + from aws_encryption_sdk.materials_managers.mpl.materials import ( + EncryptionMaterialsFromMPL + ) + from awses_test_vectors.internal.half_signing_mpl_materials import HalfSigningEncryptionMaterialsFromMPL + _HAS_MPL = True except ImportError: _HAS_MPL = False @@ -297,7 +302,6 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, _p generation_scenario.decryption_test_scenario_pair(ciphertext_writer, ciphertext_to_decrypt, expected_result) ] - class HalfSigningCryptoMaterialsManager(CryptoMaterialsManager): """ Custom CMM that generates materials for an unsigned algorithm suite @@ -340,6 +344,11 @@ def get_encryption_materials(self, request): signing_request.algorithm = AlgorithmSuite.AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384 result = self.wrapped_default_cmm.get_encryption_materials(signing_request) + + if _HAS_MPL: + if isinstance(result, EncryptionMaterialsFromMPL): + result = HalfSigningEncryptionMaterialsFromMPL(result) + result.algorithm = request.algorithm result.signing_key = None From 3760ebe3967909d8b2fc8b439f1a2725f3eefb34 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 16:47:08 -0700 Subject: [PATCH 201/422] debug gha --- .../internal/keyvectors_provider.py | 13 +++++++------ .../test/integration/integration_test_utils.py | 7 +++++-- .../test_i_full_message_encrypt_keyrings.py | 14 -------------- 3 files changed, 12 insertions(+), 22 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py b/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py index 12dc980e3..85b8ef9fa 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py @@ -1,3 +1,6 @@ +"""Singleton provider for the KeyVectors client.""" +# # Ignore missing MPL TestVectors for pylint, but the MPL TestVectors is required for this file +# pylint: disable=import-error from aws_cryptography_materialproviderstestvectorkeys.smithygenerated.\ aws_cryptography_materialproviderstestvectorkeys.client import ( KeyVectors, @@ -9,17 +12,15 @@ keyvectors_instances = {} +# pylint: disable=too-few-public-methods class KeyVectorsProvider: - """Singleton manager for the KeyVectors client. - - This is used because Dafny's JSON deserializer implementation is slow with large files. - It deserializes the file at keys_path and takes >1 minute to do this. - """ + """Singleton manager for the KeyVectors client.""" instance: KeyVectors @classmethod - def get_keyvectors(self, keys_path): + def get_keyvectors(cls, keys_path): + """Returns the singleton KeyVectors client.""" if not keys_path in keyvectors_instances: keyvectors_instances[keys_path] = KeyVectors(KeyVectorsConfig(key_manifest_path=keys_path)) return keyvectors_instances[keys_path] diff --git a/test_vector_handlers/test/integration/integration_test_utils.py b/test_vector_handlers/test/integration/integration_test_utils.py index 696dc8648..b8c8beb56 100644 --- a/test_vector_handlers/test/integration/integration_test_utils.py +++ b/test_vector_handlers/test/integration/integration_test_utils.py @@ -43,7 +43,10 @@ def full_message_encrypt_vectors(): @pytest.fixture def full_message_decrypt_generation_vectors(): return os.path.join( - legacy_vectors_dir(), "features", "CANONICAL-GENERATED-MANIFESTS", "0006-awses-message-decryption-generation.v2.json" + legacy_vectors_dir(), + "features", + "CANONICAL-GENERATED-MANIFESTS", + "0006-awses-message-decryption-generation.v2.json" ) @@ -58,4 +61,4 @@ def mpl_decrypt_vectors(): def required_encryption_context_cmm_decrypt_vectors(): return os.path.join( required_ec_vectors_dir(), "manifest.json" - ) \ No newline at end of file + ) diff --git a/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py b/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py index 7ecbdb69f..6ffd97b60 100644 --- a/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py +++ b/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py @@ -34,7 +34,6 @@ def test_full_message_encrypt_canonical_full(full_message_encrypt_vectors): def test_full_message_cycle_canonical_full(tmpdir, full_message_decrypt_generation_vectors): # Generate vectors using keyring interfaces keyring_output_dir = tmpdir.join("output-keyrings") - print("Generating vectors with keyrings... ", end="") full_message_decrypt_generate.cli([ "--output", str(keyring_output_dir), @@ -42,9 +41,7 @@ def test_full_message_cycle_canonical_full(tmpdir, full_message_decrypt_generati full_message_decrypt_generation_vectors, "--keyrings" ]) - print("done") - print("Generating vectors with master keys... ", end="") # Generate vectors using master key interfaces master_key_output_dir = tmpdir.join("output-master-key") full_message_decrypt_generate.cli([ @@ -53,27 +50,16 @@ def test_full_message_cycle_canonical_full(tmpdir, full_message_decrypt_generati "--input", full_message_decrypt_generation_vectors ]) - print("done") # Validate that vectors generated using keyring interfaces # can be decrypted by BOTH keyring and master key interfaces keyring_decrypt_manifest_file = keyring_output_dir.join("manifest.json") - print("Decrypting keyring-encrypted vectors with keyrings... ", end="") full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file), "--keyrings"]) - print("done") - - print("Decrypting keyring-encrypted vectors with master keys... ", end="") full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file)]) - print("done") # Validate that vectors generated using master key interfaces # can be decrypted by BOTH keyring and master key interfaces master_key_decrypt_manifest_file = keyring_output_dir.join("manifest.json") - print("Decrypting master key-encrypted vectors with keyrings... ", end="") full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file), "--keyrings"]) - print("done") - - print("Decrypting master key-encrypted vectors with master keys... ", end="") full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file)]) - print("done") From 7b984fe85fdb35d6f3b940a298752329774154f6 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 16:53:12 -0700 Subject: [PATCH 202/422] add missing file --- .../internal/half_signing_mpl_materials.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py diff --git a/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py b/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py new file mode 100644 index 000000000..80c96ad7a --- /dev/null +++ b/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py @@ -0,0 +1,27 @@ +from aws_encryption_sdk.materials_managers.mpl.materials import ( + EncryptionMaterialsFromMPL +) + + +class HalfSigningEncryptionMaterialsFromMPL(EncryptionMaterialsFromMPL): + @EncryptionMaterialsFromMPL.algorithm.setter + def algorithm(self, algorithm): + self.set_algorithm = algorithm + + @EncryptionMaterialsFromMPL.algorithm.getter + def algorithm(self): + if hasattr(self, "set_algorithm"): + return self.set_algorithm + else: + return self.algorithm + + @EncryptionMaterialsFromMPL.signing_key.setter + def signing_key(self, signing_key): + self.set_signing_key = signing_key + + @EncryptionMaterialsFromMPL.signing_key.getter + def signing_key(self): + if hasattr(self, "set_signing_key"): + return self.set_signing_key + else: + return self.signing_key From 9b7a58d4c14c228573168b6473cf50766c7cf8f1 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 17:03:20 -0700 Subject: [PATCH 203/422] debug gha --- .../internal/half_signing_mpl_materials.py | 34 ++++++++++++------- .../internal/keyvectors_provider.py | 3 +- .../manifests/full_message/decrypt.py | 4 +-- .../full_message/decrypt_generation.py | 1 + .../manifests/mpl_keyring.py | 6 ++-- 5 files changed, 30 insertions(+), 18 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py b/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py index 80c96ad7a..481167684 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py @@ -1,27 +1,37 @@ +"""Allows overriding the algorithm and signing_key for EncryptionMaterialsFromMPL. +This must ONLY be used in testing and NOT in production.. +This is used in testing malicious message modification (HalfSigningTampering). +""" from aws_encryption_sdk.materials_managers.mpl.materials import ( EncryptionMaterialsFromMPL ) class HalfSigningEncryptionMaterialsFromMPL(EncryptionMaterialsFromMPL): - @EncryptionMaterialsFromMPL.algorithm.setter - def algorithm(self, algorithm): - self.set_algorithm = algorithm - + """Allows overriding the algorithm and signing_key for EncryptionMaterialsFromMPL. + This must ONLY be used in testing and NOT in production.. + This is used in testing malicious message modification (HalfSigningTampering). + """ + # pylint thinks EncryptionMaterialsFromMPL.algorithm is a method + # pylint: disable=invalid-overridden-method @EncryptionMaterialsFromMPL.algorithm.getter def algorithm(self): if hasattr(self, "set_algorithm"): return self.set_algorithm - else: - return self.algorithm - - @EncryptionMaterialsFromMPL.signing_key.setter - def signing_key(self, signing_key): - self.set_signing_key = signing_key + return self.algorithm + @algorithm.setter + def algorithm(self, algorithm): + self.set_algorithm = algorithm + + # pylint thinks EncryptionMaterialsFromMPL.signing_key is a method + # pylint: disable=invalid-overridden-method @EncryptionMaterialsFromMPL.signing_key.getter def signing_key(self): if hasattr(self, "set_signing_key"): return self.set_signing_key - else: - return self.signing_key + return self.signing_key + + @signing_key.setter + def signing_key(self, signing_key): + self.set_signing_key = signing_key diff --git a/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py b/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py index 85b8ef9fa..71e75c025 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py @@ -12,6 +12,7 @@ keyvectors_instances = {} + # pylint: disable=too-few-public-methods class KeyVectorsProvider: """Singleton manager for the KeyVectors client.""" @@ -21,6 +22,6 @@ class KeyVectorsProvider: @classmethod def get_keyvectors(cls, keys_path): """Returns the singleton KeyVectors client.""" - if not keys_path in keyvectors_instances: + if keys_path not in keyvectors_instances: keyvectors_instances[keys_path] = KeyVectors(KeyVectorsConfig(key_manifest_path=keys_path)) return keyvectors_instances[keys_path] diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index a0f847762..91628d1ee 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -64,7 +64,7 @@ CLIENT_NAME = "aws/aws-encryption-sdk-python" CURRENT_VERSION = 2 -SUPPORTED_VERSIONS = (2,4,) +SUPPORTED_VERSIONS = (2, 4,) @attr.s(init=False) @@ -392,7 +392,7 @@ def _one_shot_decrypt(self): return client.decrypt( source=self.ciphertext, materials_manager=required_ec_cmm, - encryption_context = self.encryption_context, + encryption_context=self.encryption_context, ) # If the cmm type was not in if/elif above, raise error diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index b6806d8ca..d48285ef2 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -302,6 +302,7 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, _p generation_scenario.decryption_test_scenario_pair(ciphertext_writer, ciphertext_to_decrypt, expected_result) ] + class HalfSigningCryptoMaterialsManager(CryptoMaterialsManager): """ Custom CMM that generates materials for an unsigned algorithm suite diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index 7094ec35c..02a41f1d1 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -34,7 +34,7 @@ import UTF8 from awses_test_vectors.internal.keyvectors_provider import KeyVectorsProvider -from awses_test_vectors.manifests.keys import KeysManifest +from awses_test_vectors.manifests.keys import KeysManifest # noqa: disable=F401 from .master_key import MasterKeySpec @@ -92,7 +92,7 @@ def keyring(self, keys_uri, mode): # If this is not done, then keyring.OnEncrypt fails with # "A RawRSAKeyring without a public key cannot provide OnEncrypt" if input_kwargs["key"] == "rsa-4096-private" \ - and mode in ("decrypt-generate", "encrypt"): + and mode in ("decrypt-generate", "encrypt"): changed_key_name_from_private_to_public = True input_kwargs["key"] = "rsa-4096-public" # Specify default padding-hash @@ -147,7 +147,7 @@ def keyring(self, keys_uri, mode): if hasattr(keyring, "_impl"): # pylint: disable=protected-access if hasattr(keyring._impl, "_keyName"): # pylint: disable=protected-access if keyring._impl._keyName == UTF8.default__.Encode(_dafny.Seq("rsa-4096-public")).value \ - and mode in ("decrypt-generate", "encrypt"): # pylint: disable=protected-access + and mode in ("decrypt-generate", "encrypt"): # pylint: disable=protected-access if changed_key_name_from_private_to_public: # pylint: disable=protected-access keyring._impl._keyName = UTF8.default__.Encode(_dafny.Seq("rsa-4096-private")).value From 93fee671031ed511fd14c569ab691b028d42dd82 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 17:06:44 -0700 Subject: [PATCH 204/422] debug cb --- .../src/awses_test_vectors/manifests/full_message/decrypt.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index 91628d1ee..4432502c5 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -301,7 +301,10 @@ def master_key_provider_fn(): result_spec = scenario["result"] result = MessageDecryptionTestResult.from_result_spec(result_spec, plaintext_reader) - encryption_context = scenario["encryption-context"] + if "encryption-context" in scenario: + encryption_context = scenario["encryption-context"] + else: + encryption_context = {} # MPL test vectors add CMM types to the test vectors manifests if "cmm" in scenario: From e65fec4878d141ee18286acc74ae4c387f699182 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 17:16:17 -0700 Subject: [PATCH 205/422] debug gha --- .../internal/half_signing_mpl_materials.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py b/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py index 481167684..b70a901d6 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py @@ -2,6 +2,8 @@ This must ONLY be used in testing and NOT in production.. This is used in testing malicious message modification (HalfSigningTampering). """ +# Ignore missing MPL for pylint, but the MPL is required for this class +# pylint: disable=import-error,no-name-in-module from aws_encryption_sdk.materials_managers.mpl.materials import ( EncryptionMaterialsFromMPL ) @@ -16,6 +18,9 @@ class HalfSigningEncryptionMaterialsFromMPL(EncryptionMaterialsFromMPL): # pylint: disable=invalid-overridden-method @EncryptionMaterialsFromMPL.algorithm.getter def algorithm(self): + """Returns any previously-provided overriden algorithm; + if none was provided, returns underlying algorithm from encryption materials. + """ if hasattr(self, "set_algorithm"): return self.set_algorithm return self.algorithm @@ -28,6 +33,9 @@ def algorithm(self, algorithm): # pylint: disable=invalid-overridden-method @EncryptionMaterialsFromMPL.signing_key.getter def signing_key(self): + """Returns any previously-provided overriden signing_key; + if none was provided, returns underlying signing_key from encryption materials. + """ if hasattr(self, "set_signing_key"): return self.set_signing_key return self.signing_key From f4ebbba80a6e71ba816b6cabf504129d52be590d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 17:21:28 -0700 Subject: [PATCH 206/422] imperative mood --- .../internal/half_signing_mpl_materials.py | 5 +++-- .../src/awses_test_vectors/internal/keyvectors_provider.py | 2 +- .../src/awses_test_vectors/manifests/mpl_keyring.py | 1 - 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py b/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py index b70a901d6..b782e029c 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py @@ -14,11 +14,12 @@ class HalfSigningEncryptionMaterialsFromMPL(EncryptionMaterialsFromMPL): This must ONLY be used in testing and NOT in production.. This is used in testing malicious message modification (HalfSigningTampering). """ + # pylint thinks EncryptionMaterialsFromMPL.algorithm is a method # pylint: disable=invalid-overridden-method @EncryptionMaterialsFromMPL.algorithm.getter def algorithm(self): - """Returns any previously-provided overriden algorithm; + """Return any previously-provided overriden algorithm; if none was provided, returns underlying algorithm from encryption materials. """ if hasattr(self, "set_algorithm"): @@ -33,7 +34,7 @@ def algorithm(self, algorithm): # pylint: disable=invalid-overridden-method @EncryptionMaterialsFromMPL.signing_key.getter def signing_key(self): - """Returns any previously-provided overriden signing_key; + """Return any previously-provided overriden signing_key; if none was provided, returns underlying signing_key from encryption materials. """ if hasattr(self, "set_signing_key"): diff --git a/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py b/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py index 71e75c025..305459026 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py @@ -21,7 +21,7 @@ class KeyVectorsProvider: @classmethod def get_keyvectors(cls, keys_path): - """Returns the singleton KeyVectors client.""" + """Return the singleton KeyVectors client.""" if keys_path not in keyvectors_instances: keyvectors_instances[keys_path] = KeyVectors(KeyVectorsConfig(key_manifest_path=keys_path)) return keyvectors_instances[keys_path] diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index 02a41f1d1..ea702e4eb 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -57,7 +57,6 @@ class KeyringSpec(MasterKeySpec): # pylint: disable=too-many-instance-attribute def keyring(self, keys_uri, mode): # type: (KeysManifest) -> IKeyring """Build a keyring using this specification. - :param str keys_uri: Path to the keys manifest """ From 05511a820656d2363c303b791acc1e7dddec3fb2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 17:24:23 -0700 Subject: [PATCH 207/422] fix gha --- .../src/awses_test_vectors/manifests/mpl_keyring.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index ea702e4eb..74e6a5737 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -57,9 +57,9 @@ class KeyringSpec(MasterKeySpec): # pylint: disable=too-many-instance-attribute def keyring(self, keys_uri, mode): # type: (KeysManifest) -> IKeyring """Build a keyring using this specification. + :param str keys_uri: Path to the keys manifest """ - keyvectors = KeyVectorsProvider.get_keyvectors(keys_path=keys_uri) # Variable to flag whether we changed anything in weird hack #1. From c9c58e6ef45aea9f6c94a24dee02d489c41bffa4 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 17:27:24 -0700 Subject: [PATCH 208/422] fix gha --- .../src/awses_test_vectors/manifests/mpl_keyring.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index 74e6a5737..55a9276c9 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -57,7 +57,6 @@ class KeyringSpec(MasterKeySpec): # pylint: disable=too-many-instance-attribute def keyring(self, keys_uri, mode): # type: (KeysManifest) -> IKeyring """Build a keyring using this specification. - :param str keys_uri: Path to the keys manifest """ keyvectors = KeyVectorsProvider.get_keyvectors(keys_path=keys_uri) From a62f1b4d2a1c824da5796409d93e43b6fcb03075 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 18:18:19 -0700 Subject: [PATCH 209/422] debug cb --- .../internal/half_signing_mpl_materials.py | 29 ++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py b/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py index b782e029c..b8f405fde 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py @@ -15,16 +15,21 @@ class HalfSigningEncryptionMaterialsFromMPL(EncryptionMaterialsFromMPL): This is used in testing malicious message modification (HalfSigningTampering). """ + _underlying_materials: EncryptionMaterialsFromMPL + + def __init__(self, underling_materials): + self._underlying_materials = underling_materials + # pylint thinks EncryptionMaterialsFromMPL.algorithm is a method # pylint: disable=invalid-overridden-method - @EncryptionMaterialsFromMPL.algorithm.getter + @property def algorithm(self): """Return any previously-provided overriden algorithm; if none was provided, returns underlying algorithm from encryption materials. """ if hasattr(self, "set_algorithm"): return self.set_algorithm - return self.algorithm + return self._underlying_materials.algorithm @algorithm.setter def algorithm(self, algorithm): @@ -32,15 +37,31 @@ def algorithm(self, algorithm): # pylint thinks EncryptionMaterialsFromMPL.signing_key is a method # pylint: disable=invalid-overridden-method - @EncryptionMaterialsFromMPL.signing_key.getter + @property def signing_key(self): """Return any previously-provided overriden signing_key; if none was provided, returns underlying signing_key from encryption materials. """ if hasattr(self, "set_signing_key"): return self.set_signing_key - return self.signing_key + return self._underlying_materials.algorithm @signing_key.setter def signing_key(self, signing_key): self.set_signing_key = signing_key + + @property + def encryption_context(self): + return self._underlying_materials.encryption_context + + @property + def encrypted_data_keys(self): + return self._underlying_materials.encrypted_data_keys + + @property + def data_encryption_key(self): + return self._underlying_materials.data_encryption_key + + @property + def required_encryption_context_keys(self): + return self._underlying_materials.required_encryption_context_keys From 752c98c4e8fd6bc0a8ee5d19110b3cd0b8b8e9f5 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 18:28:30 -0700 Subject: [PATCH 210/422] no more profile --- test_vector_handlers/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index 497baed4b..c2ff913c2 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -37,7 +37,7 @@ envlist = # release :: Builds dist files and uploads to pypi pypirc profile. [testenv:base-command] -commands = python3 -m cProfile -o profile.txt -m pytest --basetemp={envtmpdir} -l --cov awses_test_vectors {posargs} +commands = pytest --basetemp={envtmpdir} -l --cov awses_test_vectors {posargs} [testenv] passenv = From d2897e5cb0023a7b4624a4ef4ef9325d0dc58a8a Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 19:33:33 -0700 Subject: [PATCH 211/422] debug cb --- .../internal/half_signing_mpl_materials.py | 2 +- .../full_message/decrypt_generation.py | 34 ++++++++++++++----- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py b/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py index b8f405fde..0f16d57f6 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py @@ -1,6 +1,6 @@ """Allows overriding the algorithm and signing_key for EncryptionMaterialsFromMPL. This must ONLY be used in testing and NOT in production.. -This is used in testing malicious message modification (HalfSigningTampering). +This is used in message tampering testing. """ # Ignore missing MPL for pylint, but the MPL is required for this class # pylint: disable=import-error,no-name-in-module diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index d48285ef2..d002c67ad 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -169,16 +169,34 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, _p master_key_provider = generation_scenario.encryption_scenario.master_key_provider_fn() # Use a caching CMM to avoid generating a new data key every time. - cache = LocalCryptoMaterialsCache(10) - caching_cmm = CachingCryptoMaterialsManager( - master_key_provider=master_key_provider, - cache=cache, - max_age=60.0, - max_messages_encrypted=100, - ) + if isinstance(master_key_provider, MasterKeyProvider): + cache = LocalCryptoMaterialsCache(10) + cmm = CachingCryptoMaterialsManager( + master_key_provider=master_key_provider, + cache=cache, + max_age=60.0, + max_messages_encrypted=100, + ) + cmm = caching_cmm + elif _HAS_MPL and isinstance(master_key_provider, IKeyring): + mpl = AwsCryptographicMaterialProviders(MaterialProvidersConfig()) + mpl_caching_cmm = mpl.create_default_cryptographic_materials_manager( + CreateDefaultCryptographicMaterialsManagerInput( + + ) + ) + mpl_cmm = mpl.create_default_cryptographic_materials_manager( + CreateDefaultCryptographicMaterialsManagerInput( + keyring=master_key_provider + ) + ) + cmm = CryptoMaterialsManagerFromMPL(mpl_cmm=mpl_cmm) + else: + raise TypeError(f"Unrecognized master_key_provider type: {master_key_provider}") + return [ self.run_scenario_with_new_provider_info( - ciphertext_writer, generation_scenario, caching_cmm, new_provider_info + ciphertext_writer, generation_scenario, cmm, new_provider_info ) for new_provider_info in self.new_provider_infos ] From dc7887df8ef2bca11700f6a6027c12a7dada6d41 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 20:51:59 -0700 Subject: [PATCH 212/422] debug cb --- .../manifests/full_message/decrypt_generation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index d002c67ad..8dcda0eb6 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -171,7 +171,7 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, _p # Use a caching CMM to avoid generating a new data key every time. if isinstance(master_key_provider, MasterKeyProvider): cache = LocalCryptoMaterialsCache(10) - cmm = CachingCryptoMaterialsManager( + caching_cmm = CachingCryptoMaterialsManager( master_key_provider=master_key_provider, cache=cache, max_age=60.0, From 36a46303b1fe30c8eb811764c29cd3f2ffe4054c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 13 Mar 2024 21:16:30 -0700 Subject: [PATCH 213/422] debug cb --- .../manifests/full_message/decrypt_generation.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 8dcda0eb6..847229b84 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -180,11 +180,6 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, _p cmm = caching_cmm elif _HAS_MPL and isinstance(master_key_provider, IKeyring): mpl = AwsCryptographicMaterialProviders(MaterialProvidersConfig()) - mpl_caching_cmm = mpl.create_default_cryptographic_materials_manager( - CreateDefaultCryptographicMaterialsManagerInput( - - ) - ) mpl_cmm = mpl.create_default_cryptographic_materials_manager( CreateDefaultCryptographicMaterialsManagerInput( keyring=master_key_provider From 736c1f4a30b96ffa3183d7d195c44f1e727eacb9 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 09:15:19 -0700 Subject: [PATCH 214/422] debug cb --- .../awses_test_vectors/manifests/full_message/encrypt.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 0c2580fa8..25697e15c 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -41,8 +41,10 @@ try: from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_from_master_key_specs + + _HAS_MPL = True except ImportError: - pass + _HAS_MPL = False try: # Python 3.5.0 and 3.5.1 have incompatible typing modules @@ -149,9 +151,9 @@ def run(self, materials_manager=None): ) if materials_manager: encrypt_kwargs["materials_manager"] = materials_manager - elif self.keyrings: + elif isinstance(self.master_key_provider_fn(), MasterKeySpec): encrypt_kwargs["keyring"] = self.master_key_provider_fn() - else: + elif _HAS_MPL and isinstance(self.master_key_provider_fn(), KeyringSpec): encrypt_kwargs["key_provider"] = self.master_key_provider_fn() ciphertext, _header = client.encrypt(**encrypt_kwargs) return ciphertext From 1adfb12d56eb00ef9d0c6a9b851fc071e7826d7f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 09:44:50 -0700 Subject: [PATCH 215/422] debug cb --- .../awses_test_vectors/manifests/full_message/encrypt.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 25697e15c..3415bb1f5 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -152,9 +152,11 @@ def run(self, materials_manager=None): if materials_manager: encrypt_kwargs["materials_manager"] = materials_manager elif isinstance(self.master_key_provider_fn(), MasterKeySpec): - encrypt_kwargs["keyring"] = self.master_key_provider_fn() - elif _HAS_MPL and isinstance(self.master_key_provider_fn(), KeyringSpec): encrypt_kwargs["key_provider"] = self.master_key_provider_fn() + elif _HAS_MPL and isinstance(self.master_key_provider_fn(), KeyringSpec): + encrypt_kwargs["keyring"] = self.master_key_provider_fn() + else: + raise TypeError(f"Unrecognized master_key_provider_fn return type: {self.master_key_provider_fn()}") ciphertext, _header = client.encrypt(**encrypt_kwargs) return ciphertext From dd4b495b625a5a42bbbd592c2f893b7246c0a273 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 10:15:37 -0700 Subject: [PATCH 216/422] debug cb --- codebuild/py312/awses_local_mpl.yml | 4 ++-- codebuild/py312/mplawses_local_mpl.yml | 3 +-- .../manifests/full_message/encrypt.py | 10 ++++++++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/codebuild/py312/awses_local_mpl.yml b/codebuild/py312/awses_local_mpl.yml index 1d0f80319..96ca5bc28 100644 --- a/codebuild/py312/awses_local_mpl.yml +++ b/codebuild/py312/awses_local_mpl.yml @@ -1,5 +1,5 @@ -# Runs the same tests as awses_local in an environment with the MPL installed. -# This asserts existing tests continue to pass with the MPL installed. +# Runs test vectors using native constructs in an environment with the MPL installed. +# This asserts that installing the MPL does not change existing behavior. version: 0.2 env: diff --git a/codebuild/py312/mplawses_local_mpl.yml b/codebuild/py312/mplawses_local_mpl.yml index e3f06e7f6..d932f0461 100644 --- a/codebuild/py312/mplawses_local_mpl.yml +++ b/codebuild/py312/mplawses_local_mpl.yml @@ -1,5 +1,4 @@ -# Runs the same tests as awses_local in an environment with the MPL installed. -# This asserts existing tests continue to pass with the MPL installed. +# Runs MPL-specific test vectors. version: 0.2 env: diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 3415bb1f5..6343b7044 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -31,15 +31,21 @@ membership_validator, validate_manifest_type, ) +from aws_encryption_sdk.key_providers.base import MasterKeyProvider from awses_test_vectors.manifests.keys import KeysManifest from awses_test_vectors.manifests.master_key import MasterKeySpec, master_key_provider_from_master_key_specs + try: from aws_encryption_sdk.identifiers import AlgorithmSuite, CommitmentPolicy except ImportError: from aws_encryption_sdk.identifiers import Algorithm as AlgorithmSuite try: + from aws_cryptographic_materialproviders.mpl.references import ( + IKeyring, + ) + from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_from_master_key_specs _HAS_MPL = True @@ -151,9 +157,9 @@ def run(self, materials_manager=None): ) if materials_manager: encrypt_kwargs["materials_manager"] = materials_manager - elif isinstance(self.master_key_provider_fn(), MasterKeySpec): + elif isinstance(self.master_key_provider_fn(), MasterKeyProvider): encrypt_kwargs["key_provider"] = self.master_key_provider_fn() - elif _HAS_MPL and isinstance(self.master_key_provider_fn(), KeyringSpec): + elif _HAS_MPL and isinstance(self.master_key_provider_fn(), IKeyring): encrypt_kwargs["keyring"] = self.master_key_provider_fn() else: raise TypeError(f"Unrecognized master_key_provider_fn return type: {self.master_key_provider_fn()}") From 76b1f29c8ff29e98305c82c5538475d90a7fe01b Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 11:08:09 -0700 Subject: [PATCH 217/422] debug cb --- buildspec.yml | 142 ++++++++++++++++++ codebuild/py312/mplawses_local_mpl.yml | 2 +- .../full_message/decrypt_generation.py | 36 ++++- .../test/keyrings/__init__.py | 0 .../test/keyrings/integration/__init__.py | 0 .../keyrings/integration/commands/__init__.py | 0 .../test_i_full_message_encrypt_keyrings.py | 65 -------- test_vector_handlers/tox.ini | 2 +- 8 files changed, 179 insertions(+), 68 deletions(-) delete mode 100644 test_vector_handlers/test/keyrings/__init__.py delete mode 100644 test_vector_handlers/test/keyrings/integration/__init__.py delete mode 100644 test_vector_handlers/test/keyrings/integration/commands/__init__.py delete mode 100644 test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py diff --git a/buildspec.yml b/buildspec.yml index fff7c68d1..89526ba64 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -15,6 +15,28 @@ batch: buildspec: codebuild/py37/awses_local.yml env: image: aws/codebuild/standard:5.0 + - identifier: py37_decrypt_dafny_esdk_vectors + buildspec: codebuild/py37/decrypt_dafny_esdk_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py37_decrypt_net_401_vectors + buildspec: codebuild/py37/decrypt_net_401_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py37_generate_decrypt_vectors + buildspec: codebuild/py37/generate_decrypt_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py37_decrypt_generated + depend-on: py37_generate_decrypt_vectors + buildspec: codebuild/py37/decrypt_generated_with_python.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py37_decrypt_generated_js + depend-on: py37_generate_decrypt_vectors + buildspec: codebuild/py37/decrypt_generated_with_js.yml + env: + image: aws/codebuild/standard:5.0 - identifier: py38_integ buildspec: codebuild/py38/integ.yml @@ -28,6 +50,28 @@ batch: buildspec: codebuild/py38/awses_local.yml env: image: aws/codebuild/standard:5.0 + - identifier: py38_decrypt_dafny_esdk_vectors + buildspec: codebuild/py38/decrypt_dafny_esdk_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py38_decrypt_net_401_vectors + buildspec: codebuild/py38/decrypt_net_401_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py38_generate_decrypt_vectors + buildspec: codebuild/py38/generate_decrypt_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py38_decrypt_generated + depend-on: py38_generate_decrypt_vectors + buildspec: codebuild/py38/decrypt_generated_with_python.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py38_decrypt_generated_js + depend-on: py38_generate_decrypt_vectors + buildspec: codebuild/py38/decrypt_generated_with_js.yml + env: + image: aws/codebuild/standard:5.0 - identifier: py39_integ buildspec: codebuild/py39/integ.yml @@ -40,6 +84,28 @@ batch: - identifier: py39_awses_latest env: image: aws/codebuild/standard:5.0 + - identifier: py39_decrypt_dafny_esdk_vectors + buildspec: codebuild/py39/decrypt_dafny_esdk_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_decrypt_net_401_vectors + buildspec: codebuild/py39/decrypt_net_401_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_generate_decrypt_vectors + buildspec: codebuild/py39/generate_decrypt_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_decrypt_generated + depend-on: py39_generate_decrypt_vectors + buildspec: codebuild/py39/decrypt_generated_with_python.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_decrypt_generated_js + depend-on: py39_generate_decrypt_vectors + buildspec: codebuild/py39/decrypt_generated_with_js.yml + env: + image: aws/codebuild/standard:5.0 - identifier: py310_integ buildspec: codebuild/py310/integ.yml @@ -53,6 +119,28 @@ batch: buildspec: codebuild/py310/awses_local.yml env: image: aws/codebuild/standard:6.0 + - identifier: py310_decrypt_dafny_esdk_vectors + buildspec: codebuild/py310/decrypt_dafny_esdk_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py310_decrypt_net_401_vectors + buildspec: codebuild/py310/decrypt_net_401_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py310_generate_decrypt_vectors + buildspec: codebuild/py310/generate_decrypt_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py310_decrypt_generated + depend-on: py310_generate_decrypt_vectors + buildspec: codebuild/py310/decrypt_generated_with_python.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py310_decrypt_generated_js + depend-on: py310_generate_decrypt_vectors + buildspec: codebuild/py310/decrypt_generated_with_js.yml + env: + image: aws/codebuild/standard:5.0 - identifier: py311_integ buildspec: codebuild/py311/integ.yml @@ -82,6 +170,60 @@ batch: buildspec: codebuild/py311/mplawses_local_mpl.yml env: image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_dafny_esdk_vectors_masterkey + buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_dafny_esdk_vectors_keyrings + buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_net_401_vectors_masterkey + buildspec: codebuild/py311/decrypt_net_401_vectors_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_net_401_vectors_keyrings + buildspec: codebuild/py311/decrypt_net_401_vectors_keyrings.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_generate_decrypt_vectors_masterkey + buildspec: codebuild/py311/generate_decrypt_vectors_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_masterkey_with_masterkey + depend-on: py311_generate_decrypt_vectors_masterkey + buildspec: codebuild/py311/decrypt_masterkey_with_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_masterkey_with_keyrings + depend-on: py311_generate_decrypt_vectors_masterkey + buildspec: codebuild/py311/decrypt_masterkey_with_keyrings.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_masterkey_with_js + depend-on: py311_generate_decrypt_vectors_masterkey + buildspec: codebuild/py311/decrypt_masterkey_with_js.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_generate_decrypt_vectors_keyrings + buildspec: codebuild/py311/generate_decrypt_vectors_keyrings.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_keyrings_with_masterkey + depend-on: py311_generate_decrypt_vectors_keyrings + buildspec: codebuild/py311/decrypt_keyrings_with_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_keyrings_with_keyrings + depend-on: py311_generate_decrypt_vectors_keyrings + buildspec: codebuild/py311/decrypt_keyrings_with_keyrings.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_keyrings_with_js + depend-on: py311_generate_decrypt_vectors_keyrings + buildspec: codebuild/py311/decrypt_keyrings_with_js.yml + env: + image: aws/codebuild/standard:5.0 - identifier: py312_integ buildspec: codebuild/py312/integ.yml diff --git a/codebuild/py312/mplawses_local_mpl.yml b/codebuild/py312/mplawses_local_mpl.yml index d932f0461..e11f7523b 100644 --- a/codebuild/py312/mplawses_local_mpl.yml +++ b/codebuild/py312/mplawses_local_mpl.yml @@ -1,4 +1,4 @@ -# Runs MPL-specific test vectors. +# Runs test vectors using MPL constructs. version: 0.2 env: diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 847229b84..1db214525 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -200,7 +200,10 @@ def run_scenario_with_new_provider_info( self, ciphertext_writer, generation_scenario, materials_manager, new_provider_info ): """Run with tampering for a specific new provider info value""" - tampering_materials_manager = ProviderInfoChangingCryptoMaterialsManager(materials_manager, new_provider_info) + if isinstance(materials_manager, CryptoMaterialsManagerFromMPL): + tampering_materials_manager = ProviderInfoChangingCryptoMaterialsManagerFromMPL(materials_manager, new_provider_info) + else: + tampering_materials_manager = ProviderInfoChangingCryptoMaterialsManager(materials_manager, new_provider_info) ciphertext_to_decrypt = generation_scenario.encryption_scenario.run(tampering_materials_manager) expected_result = MessageDecryptionTestResult.expect_error( "Incorrect encrypted data key provider info: " + new_provider_info @@ -239,6 +242,37 @@ def get_encryption_materials(self, request): def decrypt_materials(self, request): """Thunks to the wrapped CMM""" return self.wrapped_cmm.decrypt_materials(request) + + +class ProviderInfoChangingCryptoMaterialsManagerFromMPL(CryptoMaterialsManagerFromMPL): + """ + Custom CMM that modifies the provider info field on EDKS. + + THIS IS ONLY USED TO CREATE INVALID MESSAGES and should never be used in + production! + """ + + wrapped_cmm = attr.ib(validator=attr.validators.instance_of(CryptoMaterialsManager)) + new_provider_info = attr.ib(validator=attr.validators.instance_of(six.string_types)) + + def __init__(self, materials_manager, new_provider_info): + """Create a new CMM that wraps a the given CMM.""" + self.wrapped_cmm = materials_manager + self.new_provider_info = new_provider_info + + def get_encryption_materials(self, request): + """ + Request materials from the wrapped CMM, and then change the provider info + on each EDK. + """ + result = self.wrapped_cmm.get_encryption_materials(request) + for encrypted_data_key in result.encrypted_data_keys: + encrypted_data_key.key_provider.key_info = self.new_provider_info + return result + + def decrypt_materials(self, request): + """Thunks to the wrapped CMM""" + return self.wrapped_cmm.decrypt_materials(request) BITS_PER_BYTE = 8 diff --git a/test_vector_handlers/test/keyrings/__init__.py b/test_vector_handlers/test/keyrings/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/test_vector_handlers/test/keyrings/integration/__init__.py b/test_vector_handlers/test/keyrings/integration/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/test_vector_handlers/test/keyrings/integration/commands/__init__.py b/test_vector_handlers/test/keyrings/integration/commands/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py b/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py deleted file mode 100644 index 6ffd97b60..000000000 --- a/test_vector_handlers/test/keyrings/integration/commands/test_i_full_message_encrypt_keyrings.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. -""" -Integration tests for `awses_test_vectors.commands` with keyrings. -""" -import pytest - -from awses_test_vectors.commands import full_message_decrypt, full_message_decrypt_generate, full_message_encrypt - -from ....integration.integration_test_utils import ( # noqa pylint: disable=unused-import - full_message_decrypt_generation_vectors, - full_message_encrypt_vectors, -) - - -pytestmark = [pytest.mark.integ] - - -def test_full_message_encrypt_canonical_full(full_message_encrypt_vectors): - full_message_encrypt.cli(["--input", full_message_encrypt_vectors]) - full_message_encrypt.cli(["--input", full_message_encrypt_vectors, "--keyrings"]) - - -def test_full_message_cycle_canonical_full(tmpdir, full_message_decrypt_generation_vectors): - # Generate vectors using keyring interfaces - keyring_output_dir = tmpdir.join("output-keyrings") - full_message_decrypt_generate.cli([ - "--output", - str(keyring_output_dir), - "--input", - full_message_decrypt_generation_vectors, - "--keyrings" - ]) - - # Generate vectors using master key interfaces - master_key_output_dir = tmpdir.join("output-master-key") - full_message_decrypt_generate.cli([ - "--output", - str(master_key_output_dir), - "--input", - full_message_decrypt_generation_vectors - ]) - - # Validate that vectors generated using keyring interfaces - # can be decrypted by BOTH keyring and master key interfaces - keyring_decrypt_manifest_file = keyring_output_dir.join("manifest.json") - full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file), "--keyrings"]) - full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file)]) - - # Validate that vectors generated using master key interfaces - # can be decrypted by BOTH keyring and master key interfaces - master_key_decrypt_manifest_file = keyring_output_dir.join("manifest.json") - - full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file), "--keyrings"]) - full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file)]) diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index c2ff913c2..95dc2c9ba 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -55,7 +55,7 @@ deps = .. commands = awses_local: {[testenv:base-command]commands} test/integration - mplvectors: {[testenv:base-command]commands} test/keyrings + mplvectors: {[testenv:base-command]commands} test/mpl [testenv:full-encrypt] basepython = python3 From 33d7bdba32df553cc62438716784dbfae3622f16 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 11:13:34 -0700 Subject: [PATCH 218/422] debug cb --- buildspec.yml | 42 ++++++++++++------- .../full_message/decrypt_generation.py | 2 + 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index 89526ba64..bb5f8767f 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -28,12 +28,14 @@ batch: env: image: aws/codebuild/standard:5.0 - identifier: py37_decrypt_generated - depend-on: py37_generate_decrypt_vectors + depend-on: + - py37_generate_decrypt_vectors buildspec: codebuild/py37/decrypt_generated_with_python.yml env: image: aws/codebuild/standard:5.0 - identifier: py37_decrypt_generated_js - depend-on: py37_generate_decrypt_vectors + depend-on: + - py37_generate_decrypt_vectors buildspec: codebuild/py37/decrypt_generated_with_js.yml env: image: aws/codebuild/standard:5.0 @@ -63,12 +65,14 @@ batch: env: image: aws/codebuild/standard:5.0 - identifier: py38_decrypt_generated - depend-on: py38_generate_decrypt_vectors + depend-on: + - py38_generate_decrypt_vectors buildspec: codebuild/py38/decrypt_generated_with_python.yml env: image: aws/codebuild/standard:5.0 - identifier: py38_decrypt_generated_js - depend-on: py38_generate_decrypt_vectors + depend-on: + - py38_generate_decrypt_vectors buildspec: codebuild/py38/decrypt_generated_with_js.yml env: image: aws/codebuild/standard:5.0 @@ -97,12 +101,14 @@ batch: env: image: aws/codebuild/standard:5.0 - identifier: py39_decrypt_generated - depend-on: py39_generate_decrypt_vectors + depend-on: + - py39_generate_decrypt_vectors buildspec: codebuild/py39/decrypt_generated_with_python.yml env: image: aws/codebuild/standard:5.0 - identifier: py39_decrypt_generated_js - depend-on: py39_generate_decrypt_vectors + depend-on: + - py39_generate_decrypt_vectors buildspec: codebuild/py39/decrypt_generated_with_js.yml env: image: aws/codebuild/standard:5.0 @@ -132,12 +138,14 @@ batch: env: image: aws/codebuild/standard:5.0 - identifier: py310_decrypt_generated - depend-on: py310_generate_decrypt_vectors + depend-on: + - py310_generate_decrypt_vectors buildspec: codebuild/py310/decrypt_generated_with_python.yml env: image: aws/codebuild/standard:5.0 - identifier: py310_decrypt_generated_js - depend-on: py310_generate_decrypt_vectors + depend-on: + - py310_generate_decrypt_vectors buildspec: codebuild/py310/decrypt_generated_with_js.yml env: image: aws/codebuild/standard:5.0 @@ -191,17 +199,20 @@ batch: env: image: aws/codebuild/standard:5.0 - identifier: py311_decrypt_masterkey_with_masterkey - depend-on: py311_generate_decrypt_vectors_masterkey + depend-on: + - py311_generate_decrypt_vectors_masterkey buildspec: codebuild/py311/decrypt_masterkey_with_masterkey.yml env: image: aws/codebuild/standard:5.0 - identifier: py311_decrypt_masterkey_with_keyrings - depend-on: py311_generate_decrypt_vectors_masterkey + depend-on: + - py311_generate_decrypt_vectors_masterkey buildspec: codebuild/py311/decrypt_masterkey_with_keyrings.yml env: image: aws/codebuild/standard:5.0 - identifier: py311_decrypt_masterkey_with_js - depend-on: py311_generate_decrypt_vectors_masterkey + depend-on: + - py311_generate_decrypt_vectors_masterkey buildspec: codebuild/py311/decrypt_masterkey_with_js.yml env: image: aws/codebuild/standard:5.0 @@ -210,17 +221,20 @@ batch: env: image: aws/codebuild/standard:5.0 - identifier: py311_decrypt_keyrings_with_masterkey - depend-on: py311_generate_decrypt_vectors_keyrings + depend-on: + - py311_generate_decrypt_vectors_keyrings buildspec: codebuild/py311/decrypt_keyrings_with_masterkey.yml env: image: aws/codebuild/standard:5.0 - identifier: py311_decrypt_keyrings_with_keyrings - depend-on: py311_generate_decrypt_vectors_keyrings + depend-on: + - py311_generate_decrypt_vectors_keyrings buildspec: codebuild/py311/decrypt_keyrings_with_keyrings.yml env: image: aws/codebuild/standard:5.0 - identifier: py311_decrypt_keyrings_with_js - depend-on: py311_generate_decrypt_vectors_keyrings + depend-on: + - py311_generate_decrypt_vectors_keyrings buildspec: codebuild/py311/decrypt_keyrings_with_js.yml env: image: aws/codebuild/standard:5.0 diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 1db214525..a57d20cb2 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -247,6 +247,8 @@ def decrypt_materials(self, request): class ProviderInfoChangingCryptoMaterialsManagerFromMPL(CryptoMaterialsManagerFromMPL): """ Custom CMM that modifies the provider info field on EDKS. + This extends CryptoMaterialsManagerFromMPL so ESDK-internal checks + follow MPL logic. THIS IS ONLY USED TO CREATE INVALID MESSAGES and should never be used in production! From 6a8a623119fd2626eab35eda68933dcff390b2a7 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 13:08:41 -0700 Subject: [PATCH 219/422] debug new toxes --- buildspec.yml | 530 ++++++++++-------- .../internal/half_signing_mpl_materials.py | 67 --- .../full_message/decrypt_generation.py | 38 +- test_vector_handlers/tox.ini | 2 + 4 files changed, 301 insertions(+), 336 deletions(-) delete mode 100644 test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py diff --git a/buildspec.yml b/buildspec.yml index bb5f8767f..a79eb5d48 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -27,253 +27,313 @@ batch: buildspec: codebuild/py37/generate_decrypt_vectors.yml env: image: aws/codebuild/standard:5.0 - - identifier: py37_decrypt_generated + - identifier: py37_decrypt_masterkey_with_masterkey depend-on: - py37_generate_decrypt_vectors - buildspec: codebuild/py37/decrypt_generated_with_python.yml + buildspec: codebuild/py37/decrypt_masterkey_with_masterkey.yml env: image: aws/codebuild/standard:5.0 - - identifier: py37_decrypt_generated_js + - identifier: py37_decrypt_masterkey_with_js depend-on: - py37_generate_decrypt_vectors - buildspec: codebuild/py37/decrypt_generated_with_js.yml + buildspec: codebuild/py37/decrypt_masterkey_with_js.yml env: image: aws/codebuild/standard:5.0 - - identifier: py38_integ - buildspec: codebuild/py38/integ.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py38_examples - buildspec: codebuild/py38/examples.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py38_awses_local - buildspec: codebuild/py38/awses_local.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py38_decrypt_dafny_esdk_vectors - buildspec: codebuild/py38/decrypt_dafny_esdk_vectors.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py38_decrypt_net_401_vectors - buildspec: codebuild/py38/decrypt_net_401_vectors.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py38_generate_decrypt_vectors - buildspec: codebuild/py38/generate_decrypt_vectors.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py38_decrypt_generated - depend-on: - - py38_generate_decrypt_vectors - buildspec: codebuild/py38/decrypt_generated_with_python.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py38_decrypt_generated_js - depend-on: - - py38_generate_decrypt_vectors - buildspec: codebuild/py38/decrypt_generated_with_js.yml - env: - image: aws/codebuild/standard:5.0 + # - identifier: py38_integ + # buildspec: codebuild/py38/integ.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py38_examples + # buildspec: codebuild/py38/examples.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py38_awses_local + # buildspec: codebuild/py38/awses_local.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py38_decrypt_dafny_esdk_vectors + # buildspec: codebuild/py38/decrypt_dafny_esdk_vectors.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py38_decrypt_net_401_vectors + # buildspec: codebuild/py38/decrypt_net_401_vectors.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py38_generate_decrypt_vectors + # buildspec: codebuild/py38/generate_decrypt_vectors.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py38_decrypt_masterkey_with_masterkey + # depend-on: + # - py38_generate_decrypt_vectors + # buildspec: codebuild/py38/decrypt_masterkey_with_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py38_decrypt_masterkey_with_js + # depend-on: + # - py38_generate_decrypt_vectors + # buildspec: codebuild/py38/decrypt_generated_with_js.yml + # env: + # image: aws/codebuild/standard:5.0 - - identifier: py39_integ - buildspec: codebuild/py39/integ.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py39_examples - buildspec: codebuild/py39/examples.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py39_awses_latest - env: - image: aws/codebuild/standard:5.0 - - identifier: py39_decrypt_dafny_esdk_vectors - buildspec: codebuild/py39/decrypt_dafny_esdk_vectors.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py39_decrypt_net_401_vectors - buildspec: codebuild/py39/decrypt_net_401_vectors.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py39_generate_decrypt_vectors - buildspec: codebuild/py39/generate_decrypt_vectors.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py39_decrypt_generated - depend-on: - - py39_generate_decrypt_vectors - buildspec: codebuild/py39/decrypt_generated_with_python.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py39_decrypt_generated_js - depend-on: - - py39_generate_decrypt_vectors - buildspec: codebuild/py39/decrypt_generated_with_js.yml - env: - image: aws/codebuild/standard:5.0 + # - identifier: py39_integ + # buildspec: codebuild/py39/integ.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py39_examples + # buildspec: codebuild/py39/examples.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py39_awses_latest + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py39_decrypt_dafny_esdk_vectors + # buildspec: codebuild/py39/decrypt_dafny_esdk_vectors.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py39_decrypt_net_401_vectors + # buildspec: codebuild/py39/decrypt_net_401_vectors.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py39_generate_decrypt_vectors + # buildspec: codebuild/py39/generate_decrypt_vectors.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py39_decrypt_masterkey_with_masterkey + # depend-on: + # - py39_generate_decrypt_vectors + # buildspec: codebuild/py39/decrypt_masterkey_with_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py39_decrypt_masterkey_with_js + # depend-on: + # - py39_generate_decrypt_vectors + # buildspec: codebuild/py39/decrypt_generated_with_js.yml + # env: + # image: aws/codebuild/standard:5.0 - - identifier: py310_integ - buildspec: codebuild/py310/integ.yml - env: - image: aws/codebuild/standard:6.0 - - identifier: py310_examples - buildspec: codebuild/py310/examples.yml - env: - image: aws/codebuild/standard:6.0 - - identifier: py310_awses_latest - buildspec: codebuild/py310/awses_local.yml - env: - image: aws/codebuild/standard:6.0 - - identifier: py310_decrypt_dafny_esdk_vectors - buildspec: codebuild/py310/decrypt_dafny_esdk_vectors.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py310_decrypt_net_401_vectors - buildspec: codebuild/py310/decrypt_net_401_vectors.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py310_generate_decrypt_vectors - buildspec: codebuild/py310/generate_decrypt_vectors.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py310_decrypt_generated - depend-on: - - py310_generate_decrypt_vectors - buildspec: codebuild/py310/decrypt_generated_with_python.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py310_decrypt_generated_js - depend-on: - - py310_generate_decrypt_vectors - buildspec: codebuild/py310/decrypt_generated_with_js.yml - env: - image: aws/codebuild/standard:5.0 + # - identifier: py310_integ + # buildspec: codebuild/py310/integ.yml + # env: + # image: aws/codebuild/standard:6.0 + # - identifier: py310_examples + # buildspec: codebuild/py310/examples.yml + # env: + # image: aws/codebuild/standard:6.0 + # - identifier: py310_awses_latest + # buildspec: codebuild/py310/awses_local.yml + # env: + # image: aws/codebuild/standard:6.0 + # - identifier: py310_decrypt_dafny_esdk_vectors + # buildspec: codebuild/py310/decrypt_dafny_esdk_vectors.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py310_decrypt_net_401_vectors + # buildspec: codebuild/py310/decrypt_net_401_vectors.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py310_generate_decrypt_vectors + # buildspec: codebuild/py310/generate_decrypt_vectors.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py310_decrypt_masterkey_with_masterkey + # depend-on: + # - py310_generate_decrypt_vectors + # buildspec: codebuild/py310/decrypt_masterkey_with_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py310_decrypt_masterkey_with_js + # depend-on: + # - py310_generate_decrypt_vectors + # buildspec: codebuild/py310/decrypt_generated_with_js.yml + # env: + # image: aws/codebuild/standard:5.0 - - identifier: py311_integ - buildspec: codebuild/py311/integ.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_integ_mpl - buildspec: codebuild/py311/integ_mpl.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_examples - buildspec: codebuild/py311/examples.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_examples_mpl - buildspec: codebuild/py311/examples_mpl.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_awses_latest - buildspec: codebuild/py311/awses_local.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_awses_latest_mpl - buildspec: codebuild/py311/awses_local_mpl.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_mplawses_latest_mpl - buildspec: codebuild/py311/mplawses_local_mpl.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_dafny_esdk_vectors_masterkey - buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py311_decrypt_dafny_esdk_vectors_keyrings - buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py311_decrypt_net_401_vectors_masterkey - buildspec: codebuild/py311/decrypt_net_401_vectors_masterkey.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py311_decrypt_net_401_vectors_keyrings - buildspec: codebuild/py311/decrypt_net_401_vectors_keyrings.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py311_generate_decrypt_vectors_masterkey - buildspec: codebuild/py311/generate_decrypt_vectors_masterkey.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py311_decrypt_masterkey_with_masterkey - depend-on: - - py311_generate_decrypt_vectors_masterkey - buildspec: codebuild/py311/decrypt_masterkey_with_masterkey.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py311_decrypt_masterkey_with_keyrings - depend-on: - - py311_generate_decrypt_vectors_masterkey - buildspec: codebuild/py311/decrypt_masterkey_with_keyrings.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py311_decrypt_masterkey_with_js - depend-on: - - py311_generate_decrypt_vectors_masterkey - buildspec: codebuild/py311/decrypt_masterkey_with_js.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py311_generate_decrypt_vectors_keyrings - buildspec: codebuild/py311/generate_decrypt_vectors_keyrings.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py311_decrypt_keyrings_with_masterkey - depend-on: - - py311_generate_decrypt_vectors_keyrings - buildspec: codebuild/py311/decrypt_keyrings_with_masterkey.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py311_decrypt_keyrings_with_keyrings - depend-on: - - py311_generate_decrypt_vectors_keyrings - buildspec: codebuild/py311/decrypt_keyrings_with_keyrings.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py311_decrypt_keyrings_with_js - depend-on: - - py311_generate_decrypt_vectors_keyrings - buildspec: codebuild/py311/decrypt_keyrings_with_js.yml - env: - image: aws/codebuild/standard:5.0 + # - identifier: py311_integ + # buildspec: codebuild/py311/integ.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_integ_mpl + # buildspec: codebuild/py311/integ_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_examples + # buildspec: codebuild/py311/examples.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_examples_mpl + # buildspec: codebuild/py311/examples_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_awses_latest + # buildspec: codebuild/py311/awses_local.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_awses_latest_mpl + # buildspec: codebuild/py311/awses_local_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_mplawses_latest_mpl + # buildspec: codebuild/py311/mplawses_local_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_dafny_esdk_vectors_masterkey + # buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py311_decrypt_dafny_esdk_vectors_keyrings + # buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py311_decrypt_net_401_vectors_masterkey + # buildspec: codebuild/py311/decrypt_net_401_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py311_decrypt_net_401_vectors_keyrings + # buildspec: codebuild/py311/decrypt_net_401_vectors_keyrings.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py311_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py311/generate_decrypt_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py311_decrypt_masterkey_with_masterkey + # depend-on: + # - py311_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py311/decrypt_masterkey_with_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py311_decrypt_masterkey_with_keyrings + # depend-on: + # - py311_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py311/decrypt_masterkey_with_keyrings.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py311_decrypt_masterkey_with_js + # depend-on: + # - py311_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py311/decrypt_masterkey_with_js.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py311_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py311/generate_decrypt_vectors_keyrings.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py311_decrypt_keyrings_with_masterkey + # depend-on: + # - py311_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py311/decrypt_keyrings_with_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py311_decrypt_keyrings_with_keyrings + # depend-on: + # - py311_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py311/decrypt_keyrings_with_keyrings.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py311_decrypt_keyrings_with_js + # depend-on: + # - py311_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py311/decrypt_keyrings_with_js.yml + # env: + # image: aws/codebuild/standard:5.0 - - identifier: py312_integ - buildspec: codebuild/py312/integ.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_integ_mpl - buildspec: codebuild/py312/integ_mpl.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_examples - buildspec: codebuild/py312/examples.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_examples_mpl - buildspec: codebuild/py312/examples_mpl.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_awses_latest - buildspec: codebuild/py312/awses_local.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_awses_latest_mpl - buildspec: codebuild/py312/awses_local_mpl.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_mplawses_latest_mpl - buildspec: codebuild/py312/mplawses_local_mpl.yml - env: - image: aws/codebuild/standard:7.0 + # - identifier: py312_integ + # buildspec: codebuild/py312/integ.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_integ_mpl + # buildspec: codebuild/py312/integ_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_examples + # buildspec: codebuild/py312/examples.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_examples_mpl + # buildspec: codebuild/py312/examples_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_awses_latest + # buildspec: codebuild/py312/awses_local.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_awses_latest_mpl + # buildspec: codebuild/py312/awses_local_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_mplawses_latest_mpl + # buildspec: codebuild/py312/mplawses_local_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_decrypt_dafny_esdk_vectors_masterkey + # buildspec: codebuild/py312/decrypt_dafny_esdk_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py312_decrypt_dafny_esdk_vectors_keyrings + # buildspec: codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py312_decrypt_net_401_vectors_masterkey + # buildspec: codebuild/py312/decrypt_net_401_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py312_decrypt_net_401_vectors_keyrings + # buildspec: codebuild/py312/decrypt_net_401_vectors_keyrings.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py312_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py312/generate_decrypt_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py312_decrypt_masterkey_with_masterkey + # depend-on: + # - py312_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py312/decrypt_masterkey_with_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py312_decrypt_masterkey_with_keyrings + # depend-on: + # - py312_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py312/decrypt_masterkey_with_keyrings.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py312_decrypt_masterkey_with_js + # depend-on: + # - py312_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py312/decrypt_masterkey_with_js.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py312_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py312/generate_decrypt_vectors_keyrings.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py312_decrypt_keyrings_with_masterkey + # depend-on: + # - py312_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py312/decrypt_keyrings_with_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py312_decrypt_keyrings_with_keyrings + # depend-on: + # - py312_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py312/decrypt_keyrings_with_keyrings.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py312_decrypt_keyrings_with_js + # depend-on: + # - py312_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py312/decrypt_keyrings_with_js.yml + # env: + # image: aws/codebuild/standard:5.0 - - identifier: code_coverage - buildspec: codebuild/coverage/coverage.yml - - identifier: code_coverage_mpl - buildspec: codebuild/coverage/coverage_mpl.yml - env: - image: aws/codebuild/standard:7.0 + # - identifier: code_coverage + # buildspec: codebuild/coverage/coverage.yml + # - identifier: code_coverage_mpl + # buildspec: codebuild/coverage/coverage_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 - - identifier: compliance - buildspec: codebuild/compliance/compliance.yml + # - identifier: compliance + # buildspec: codebuild/compliance/compliance.yml diff --git a/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py b/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py deleted file mode 100644 index 0f16d57f6..000000000 --- a/test_vector_handlers/src/awses_test_vectors/internal/half_signing_mpl_materials.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Allows overriding the algorithm and signing_key for EncryptionMaterialsFromMPL. -This must ONLY be used in testing and NOT in production.. -This is used in message tampering testing. -""" -# Ignore missing MPL for pylint, but the MPL is required for this class -# pylint: disable=import-error,no-name-in-module -from aws_encryption_sdk.materials_managers.mpl.materials import ( - EncryptionMaterialsFromMPL -) - - -class HalfSigningEncryptionMaterialsFromMPL(EncryptionMaterialsFromMPL): - """Allows overriding the algorithm and signing_key for EncryptionMaterialsFromMPL. - This must ONLY be used in testing and NOT in production.. - This is used in testing malicious message modification (HalfSigningTampering). - """ - - _underlying_materials: EncryptionMaterialsFromMPL - - def __init__(self, underling_materials): - self._underlying_materials = underling_materials - - # pylint thinks EncryptionMaterialsFromMPL.algorithm is a method - # pylint: disable=invalid-overridden-method - @property - def algorithm(self): - """Return any previously-provided overriden algorithm; - if none was provided, returns underlying algorithm from encryption materials. - """ - if hasattr(self, "set_algorithm"): - return self.set_algorithm - return self._underlying_materials.algorithm - - @algorithm.setter - def algorithm(self, algorithm): - self.set_algorithm = algorithm - - # pylint thinks EncryptionMaterialsFromMPL.signing_key is a method - # pylint: disable=invalid-overridden-method - @property - def signing_key(self): - """Return any previously-provided overriden signing_key; - if none was provided, returns underlying signing_key from encryption materials. - """ - if hasattr(self, "set_signing_key"): - return self.set_signing_key - return self._underlying_materials.algorithm - - @signing_key.setter - def signing_key(self, signing_key): - self.set_signing_key = signing_key - - @property - def encryption_context(self): - return self._underlying_materials.encryption_context - - @property - def encrypted_data_keys(self): - return self._underlying_materials.encrypted_data_keys - - @property - def data_encryption_key(self): - return self._underlying_materials.data_encryption_key - - @property - def required_encryption_context_keys(self): - return self._underlying_materials.required_encryption_context_keys diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index a57d20cb2..74985b5d8 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -44,7 +44,10 @@ from aws_encryption_sdk.materials_managers.mpl.materials import ( EncryptionMaterialsFromMPL ) - from awses_test_vectors.internal.half_signing_mpl_materials import HalfSigningEncryptionMaterialsFromMPL + from awses_test_vectors.internal.tampering_mpl_materials import ( + HalfSigningEncryptionMaterialsFromMPL, + ProviderInfoChangingCryptoMaterialsManagerFromMPL, + ) _HAS_MPL = True except ImportError: @@ -244,39 +247,6 @@ def decrypt_materials(self, request): return self.wrapped_cmm.decrypt_materials(request) -class ProviderInfoChangingCryptoMaterialsManagerFromMPL(CryptoMaterialsManagerFromMPL): - """ - Custom CMM that modifies the provider info field on EDKS. - This extends CryptoMaterialsManagerFromMPL so ESDK-internal checks - follow MPL logic. - - THIS IS ONLY USED TO CREATE INVALID MESSAGES and should never be used in - production! - """ - - wrapped_cmm = attr.ib(validator=attr.validators.instance_of(CryptoMaterialsManager)) - new_provider_info = attr.ib(validator=attr.validators.instance_of(six.string_types)) - - def __init__(self, materials_manager, new_provider_info): - """Create a new CMM that wraps a the given CMM.""" - self.wrapped_cmm = materials_manager - self.new_provider_info = new_provider_info - - def get_encryption_materials(self, request): - """ - Request materials from the wrapped CMM, and then change the provider info - on each EDK. - """ - result = self.wrapped_cmm.get_encryption_materials(request) - for encrypted_data_key in result.encrypted_data_keys: - encrypted_data_key.key_provider.key_info = self.new_provider_info - return result - - def decrypt_materials(self, request): - """Thunks to the wrapped CMM""" - return self.wrapped_cmm.decrypt_materials(request) - - BITS_PER_BYTE = 8 diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index 95dc2c9ba..bf4b86724 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -56,6 +56,8 @@ deps = commands = awses_local: {[testenv:base-command]commands} test/integration mplvectors: {[testenv:base-command]commands} test/mpl + full_decrypt_generate: awses-full-message-decrypt-generate {posargs} + full_decrypt: awses-full-message-decrypt {posargs} [testenv:full-encrypt] basepython = python3 From 9fd746852267e19f1a32b1e7a24828b40d551fc5 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 13:11:11 -0700 Subject: [PATCH 220/422] debug new toxes --- buildspec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildspec.yml b/buildspec.yml index a79eb5d48..ca87c41c5 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -2,7 +2,7 @@ version: 0.2 batch: fast-fail: false - build-list: + build-graph: - identifier: py37_integ buildspec: codebuild/py37/integ.yml env: From 5affe9cdf24998e0fd4fd8fd20683aa961631bab Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 13:15:12 -0700 Subject: [PATCH 221/422] add missing files --- codebuild/py37/decrypt_masterkey_with_js.yml | 39 +++++++++++++++++++ .../py37/decrypt_masterkey_with_masterkey.yml | 26 +++++++++++++ codebuild/py37/generate_decrypt_vectors.yml | 27 +++++++++++++ 3 files changed, 92 insertions(+) create mode 100644 codebuild/py37/decrypt_masterkey_with_js.yml create mode 100644 codebuild/py37/decrypt_masterkey_with_masterkey.yml create mode 100644 codebuild/py37/generate_decrypt_vectors.yml diff --git a/codebuild/py37/decrypt_masterkey_with_js.yml b/codebuild/py37/decrypt_masterkey_with_js.yml new file mode 100644 index 000000000..32db1083e --- /dev/null +++ b/codebuild/py37/decrypt_masterkey_with_js.yml @@ -0,0 +1,39 @@ +version: 0.2 + +env: + variables: + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + GENERATE_OUTPUT_DIR: >- + $CODEBUILD_SRC_DIR/generated_vectors/ + + +phases: + install: + runtime-versions: + python: 3.7 + commands: + - n 16 + # Install the Javascript ESDK run test vectors + - npm install -g @aws-crypto/integration-node + + pre_build: + commands: + # Assume Role to access non-prod resources + - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Java-Role-us-west-2" --role-session-name "CB-TestVectorResources") + - export TMP_ROLE + - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') + - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') + - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') + - aws sts get-caller-identity + - cd $CODEBUILD_SRC_DIR + build: + commands: + # Decrypt generated vectors with Javascript ESDK + - integration-node decrypt -v ../tmp/generated/37_masterkey \ No newline at end of file diff --git a/codebuild/py37/decrypt_masterkey_with_masterkey.yml b/codebuild/py37/decrypt_masterkey_with_masterkey.yml new file mode 100644 index 000000000..df7067e60 --- /dev/null +++ b/codebuild/py37/decrypt_masterkey_with_masterkey.yml @@ -0,0 +1,26 @@ +version: 0.2 + +env: + variables: + TOXENV: "py37-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + python: 3.7 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - tox -- \ + --input ../tmp/generated/37_masterkey diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml new file mode 100644 index 000000000..849605b49 --- /dev/null +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -0,0 +1,27 @@ +version: 0.2 + +env: + variables: + TOXENV: "py37-full_decrypt_generate" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + python: 3.7 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - tox -- \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ + --output ../tmp/generated/37_masterkey From 78b817d38296c6d08f8708a188a22a0d2bf5d50c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 13:17:36 -0700 Subject: [PATCH 222/422] debug tox --- codebuild/py37/generate_decrypt_vectors.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml index 849605b49..04a09a47d 100644 --- a/codebuild/py37/generate_decrypt_vectors.yml +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -22,6 +22,7 @@ phases: commands: - pip install "tox < 4.0" - cd test_vector_handlers - - tox -- \ - --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ - --output ../tmp/generated/37_masterkey + - | + tox -- \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ + --output ../tmp/generated/37_masterkey From c817996188a3cfef7c404483122be215b5a93fb1 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 13:24:57 -0700 Subject: [PATCH 223/422] add missing --- codebuild/py37/decrypt_net_401_vectors.yml | 48 ++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 codebuild/py37/decrypt_net_401_vectors.yml diff --git a/codebuild/py37/decrypt_net_401_vectors.yml b/codebuild/py37/decrypt_net_401_vectors.yml new file mode 100644 index 000000000..7053b6c96 --- /dev/null +++ b/codebuild/py37/decrypt_net_401_vectors.yml @@ -0,0 +1,48 @@ +version: 0.2 +# Runs Only the ESDK-NET v4.0.1 Decryption Vectors, testing Required EC CMM + +env: + variables: + TOXENV: "py37-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + java: $JAVA_ENV_VERSION + commands: + # Get Dafny + - curl https://github.com/dafny-lang/dafny/releases/download/v4.2.0/dafny-4.2.0-x64-ubuntu-20.04.zip -L -o dafny.zip + - unzip -qq dafny.zip && rm dafny.zip + - export PATH="$PWD/dafny:$PATH" + pre_build: + commands: + # Assume Role to access non-prod resources + - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Java-Role-us-west-2" --role-session-name "CB-TestVectorResources") + - export TMP_ROLE + - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') + - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') + - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') + - aws sts get-caller-identity + + # Fetch ESDK .NET v4.0.1 Test Vectors + - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip + - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip + - curl -s --output $VECTOR_ZIP --location $VECTORS_URL + build: + commands: + # NOTE: We need to pass the absolute path of the vectors + - pip install "tox < 4.0" + - cd $CODEBUILD_SRC_DIR/test_vector_handlers + - | + tox -- \ + --input $VECTOR_ZIP From c549e393cb7bbb307aa1334470cf27d9bb4ab09f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 13:27:45 -0700 Subject: [PATCH 224/422] debug cb --- codebuild/py37/decrypt_net_401_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_net_401_vectors.yml b/codebuild/py37/decrypt_net_401_vectors.yml index 7053b6c96..7f5c1c8e5 100644 --- a/codebuild/py37/decrypt_net_401_vectors.yml +++ b/codebuild/py37/decrypt_net_401_vectors.yml @@ -18,7 +18,7 @@ env: phases: install: runtime-versions: - java: $JAVA_ENV_VERSION + python: 3.7 commands: # Get Dafny - curl https://github.com/dafny-lang/dafny/releases/download/v4.2.0/dafny-4.2.0-x64-ubuntu-20.04.zip -L -o dafny.zip From 90a18d09460afab78b437fe9f53a3091219f030f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 13:32:05 -0700 Subject: [PATCH 225/422] debug cb --- codebuild/py37/decrypt_net_401_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_net_401_vectors.yml b/codebuild/py37/decrypt_net_401_vectors.yml index 7f5c1c8e5..5ec0365ef 100644 --- a/codebuild/py37/decrypt_net_401_vectors.yml +++ b/codebuild/py37/decrypt_net_401_vectors.yml @@ -27,7 +27,7 @@ phases: pre_build: commands: # Assume Role to access non-prod resources - - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Java-Role-us-west-2" --role-session-name "CB-TestVectorResources") + - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") - export TMP_ROLE - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') From 77fba50f7ed4c7bcee42cbcc7086434d8b74f5b9 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 13:33:07 -0700 Subject: [PATCH 226/422] debug cb --- .../manifests/full_message/decrypt_generation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 74985b5d8..c7cc2be90 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -124,7 +124,7 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, pl materials_manager = DefaultCryptoMaterialsManager( key_provider ) - elif isinstance(key_provider, IKeyring): + elif _HAS_MPL and isinstance(key_provider, IKeyring): mpl = AwsCryptographicMaterialProviders(MaterialProvidersConfig()) mpl_cmm = mpl.create_default_cryptographic_materials_manager( CreateDefaultCryptographicMaterialsManagerInput( @@ -203,7 +203,7 @@ def run_scenario_with_new_provider_info( self, ciphertext_writer, generation_scenario, materials_manager, new_provider_info ): """Run with tampering for a specific new provider info value""" - if isinstance(materials_manager, CryptoMaterialsManagerFromMPL): + if _HAS_MPL and isinstance(materials_manager, CryptoMaterialsManagerFromMPL): tampering_materials_manager = ProviderInfoChangingCryptoMaterialsManagerFromMPL(materials_manager, new_provider_info) else: tampering_materials_manager = ProviderInfoChangingCryptoMaterialsManager(materials_manager, new_provider_info) From 991c55f8795f25a31465814e8772e1a95576872a Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 13:38:12 -0700 Subject: [PATCH 227/422] debug cb --- codebuild/py37/decrypt_net_401_vectors.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/codebuild/py37/decrypt_net_401_vectors.yml b/codebuild/py37/decrypt_net_401_vectors.yml index 5ec0365ef..59e703024 100644 --- a/codebuild/py37/decrypt_net_401_vectors.yml +++ b/codebuild/py37/decrypt_net_401_vectors.yml @@ -38,11 +38,12 @@ phases: - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip - curl -s --output $VECTOR_ZIP --location $VECTORS_URL - build: + - UNZIPPED_VECTORS_DIR = $CODEBUILD_SRC_DIR/test_vector_handlers/tmp/net_401_vectors + - unzip $VECTOR_ZIP -d $UNZIPPED_VECTORS_DIR commands: # NOTE: We need to pass the absolute path of the vectors - pip install "tox < 4.0" - cd $CODEBUILD_SRC_DIR/test_vector_handlers - | tox -- \ - --input $VECTOR_ZIP + --input $UNZIPPED_VECTORS_DIR From 71efaa3653297103df4cd7aee0f53dbdd8183a96 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 13:40:53 -0700 Subject: [PATCH 228/422] debug cb --- codebuild/py37/decrypt_net_401_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_net_401_vectors.yml b/codebuild/py37/decrypt_net_401_vectors.yml index 59e703024..8b388463f 100644 --- a/codebuild/py37/decrypt_net_401_vectors.yml +++ b/codebuild/py37/decrypt_net_401_vectors.yml @@ -38,7 +38,7 @@ phases: - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip - curl -s --output $VECTOR_ZIP --location $VECTORS_URL - - UNZIPPED_VECTORS_DIR = $CODEBUILD_SRC_DIR/test_vector_handlers/tmp/net_401_vectors + - UNZIPPED_VECTORS_DIR=$CODEBUILD_SRC_DIR/test_vector_handlers/tmp/net_401_vectors - unzip $VECTOR_ZIP -d $UNZIPPED_VECTORS_DIR commands: # NOTE: We need to pass the absolute path of the vectors From b1cbf4b69886a1170fb885abc589508a03afc180 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 13:45:47 -0700 Subject: [PATCH 229/422] debug gha and cb --- codebuild/py37/decrypt_net_401_vectors.yml | 2 +- .../manifests/full_message/decrypt_generation.py | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/codebuild/py37/decrypt_net_401_vectors.yml b/codebuild/py37/decrypt_net_401_vectors.yml index 8b388463f..0c04225a6 100644 --- a/codebuild/py37/decrypt_net_401_vectors.yml +++ b/codebuild/py37/decrypt_net_401_vectors.yml @@ -38,7 +38,7 @@ phases: - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip - curl -s --output $VECTOR_ZIP --location $VECTORS_URL - - UNZIPPED_VECTORS_DIR=$CODEBUILD_SRC_DIR/test_vector_handlers/tmp/net_401_vectors + - UNZIPPED_VECTORS_DIR=$CODEBUILD_SRC_DIR/test_vector_handlers/net_401_vectors - unzip $VECTOR_ZIP -d $UNZIPPED_VECTORS_DIR commands: # NOTE: We need to pass the absolute path of the vectors diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index c7cc2be90..a746ac127 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -191,7 +191,7 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, _p cmm = CryptoMaterialsManagerFromMPL(mpl_cmm=mpl_cmm) else: raise TypeError(f"Unrecognized master_key_provider type: {master_key_provider}") - + return [ self.run_scenario_with_new_provider_info( ciphertext_writer, generation_scenario, cmm, new_provider_info @@ -203,10 +203,18 @@ def run_scenario_with_new_provider_info( self, ciphertext_writer, generation_scenario, materials_manager, new_provider_info ): """Run with tampering for a specific new provider info value""" - if _HAS_MPL and isinstance(materials_manager, CryptoMaterialsManagerFromMPL): - tampering_materials_manager = ProviderInfoChangingCryptoMaterialsManagerFromMPL(materials_manager, new_provider_info) + if isinstance(materials_manager, CryptoMaterialsManager): + tampering_materials_manager = ProviderInfoChangingCryptoMaterialsManager( + materials_manager, + new_provider_info + ) + elif _HAS_MPL and isinstance(materials_manager, CryptoMaterialsManagerFromMPL): + tampering_materials_manager = ProviderInfoChangingCryptoMaterialsManagerFromMPL( + materials_manager, + new_provider_info + ) else: - tampering_materials_manager = ProviderInfoChangingCryptoMaterialsManager(materials_manager, new_provider_info) + raise TypeError(f"Unrecognized materials_manager type: {materials_manager}") ciphertext_to_decrypt = generation_scenario.encryption_scenario.run(tampering_materials_manager) expected_result = MessageDecryptionTestResult.expect_error( "Incorrect encrypted data key provider info: " + new_provider_info From 39fcb9b44308602430c5e289ffa72d0fc66cad35 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 13:49:16 -0700 Subject: [PATCH 230/422] debug gha and cb --- codebuild/py37/decrypt_net_401_vectors.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/codebuild/py37/decrypt_net_401_vectors.yml b/codebuild/py37/decrypt_net_401_vectors.yml index 0c04225a6..3711e6130 100644 --- a/codebuild/py37/decrypt_net_401_vectors.yml +++ b/codebuild/py37/decrypt_net_401_vectors.yml @@ -40,6 +40,7 @@ phases: - curl -s --output $VECTOR_ZIP --location $VECTORS_URL - UNZIPPED_VECTORS_DIR=$CODEBUILD_SRC_DIR/test_vector_handlers/net_401_vectors - unzip $VECTOR_ZIP -d $UNZIPPED_VECTORS_DIR + build: commands: # NOTE: We need to pass the absolute path of the vectors - pip install "tox < 4.0" From 5962312488c7243893fc704be7cb9534ffad1c8b Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 13:50:17 -0700 Subject: [PATCH 231/422] debug gha and cb --- .../manifests/full_message/decrypt_generation.py | 2 +- .../src/awses_test_vectors/manifests/full_message/encrypt.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index a746ac127..cef786335 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -253,7 +253,7 @@ def get_encryption_materials(self, request): def decrypt_materials(self, request): """Thunks to the wrapped CMM""" return self.wrapped_cmm.decrypt_materials(request) - + BITS_PER_BYTE = 8 diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 6343b7044..c1ffcdaa0 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -22,6 +22,8 @@ import aws_encryption_sdk import six +from aws_encryption_sdk.key_providers.base import MasterKeyProvider + from awses_test_vectors.internal.defaults import ENCODING from awses_test_vectors.internal.util import ( algorithm_suite_from_string_id, @@ -31,7 +33,6 @@ membership_validator, validate_manifest_type, ) -from aws_encryption_sdk.key_providers.base import MasterKeyProvider from awses_test_vectors.manifests.keys import KeysManifest from awses_test_vectors.manifests.master_key import MasterKeySpec, master_key_provider_from_master_key_specs @@ -45,7 +46,7 @@ from aws_cryptographic_materialproviders.mpl.references import ( IKeyring, ) - + from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_from_master_key_specs _HAS_MPL = True From 0ac5e96d55453204e66cd98f3202f9d0acfe17a2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 13:51:33 -0700 Subject: [PATCH 232/422] debug gha and cb --- codebuild/py37/decrypt_net_401_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_net_401_vectors.yml b/codebuild/py37/decrypt_net_401_vectors.yml index 3711e6130..8188c5ede 100644 --- a/codebuild/py37/decrypt_net_401_vectors.yml +++ b/codebuild/py37/decrypt_net_401_vectors.yml @@ -47,4 +47,4 @@ phases: - cd $CODEBUILD_SRC_DIR/test_vector_handlers - | tox -- \ - --input $UNZIPPED_VECTORS_DIR + --input $UNZIPPED_VECTORS_DIR/manifest.json From 3dcab7fdba982f5ddc9481b46e91d9f23a5f8081 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 13:57:10 -0700 Subject: [PATCH 233/422] debug gha and cb --- src/aws_encryption_sdk/internal/formatting/deserialize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/internal/formatting/deserialize.py b/src/aws_encryption_sdk/internal/formatting/deserialize.py index b06b5ba11..1b46a8b7d 100644 --- a/src/aws_encryption_sdk/internal/formatting/deserialize.py +++ b/src/aws_encryption_sdk/internal/formatting/deserialize.py @@ -475,7 +475,7 @@ def deserialize_frame(stream, header, verifier=None): frame_data["iv"] = frame_iv if final_frame is True: (content_length,) = unpack_values(">I", stream, verifier) - if content_length >= header.frame_length: + if content_length > header.frame_length: raise SerializationError( "Invalid final frame length: {final} >= {normal}".format( final=content_length, normal=header.frame_length From ccb01a2e1d97ec374054c8ea08ce849383e46e48 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 14:59:25 -0700 Subject: [PATCH 234/422] debug cb --- codebuild/py37/decrypt_net_401_vectors.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/codebuild/py37/decrypt_net_401_vectors.yml b/codebuild/py37/decrypt_net_401_vectors.yml index 8188c5ede..5b3925890 100644 --- a/codebuild/py37/decrypt_net_401_vectors.yml +++ b/codebuild/py37/decrypt_net_401_vectors.yml @@ -19,11 +19,6 @@ phases: install: runtime-versions: python: 3.7 - commands: - # Get Dafny - - curl https://github.com/dafny-lang/dafny/releases/download/v4.2.0/dafny-4.2.0-x64-ubuntu-20.04.zip -L -o dafny.zip - - unzip -qq dafny.zip && rm dafny.zip - - export PATH="$PWD/dafny:$PATH" pre_build: commands: # Assume Role to access non-prod resources From 7074f8adc114f8ed904cb37874dd0689da8bb864 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 15:02:57 -0700 Subject: [PATCH 235/422] add missing --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 codebuild/py37/decrypt_dafny_esdk_vectors.yml diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml new file mode 100644 index 000000000..a66c6e2d6 --- /dev/null +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -0,0 +1,60 @@ +version: 0.2 +# Runs Only the ESDK-NET v4.0.1 Decryption Vectors, testing Required EC CMM + +env: + variables: + TOXENV: "py37-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + python: 3.7 + pre_build: + commands: + # Fetch test vectors from Dafny ESDK's most recent run + # (Assuming the first result is most recent; seems to be correct) + - | + MOST_RECENT_RUN_ID=curl -L \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ + | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)' + - | + MOST_RECENT_RUN_DOWNLOAD_URL=curl -L \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" \ + | jq '.artifacts[0].archive_download_url' + - | + curl -L \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }} \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "$MOST_RECENT_RUN_DOWNLOAD_URL" + - unzip ubuntu-latest_test_vector_artifact + + # Assume Role to access non-prod resources + - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") + - export TMP_ROLE + - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') + - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') + - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') + - aws sts get-caller-identity + build: + commands: + # NOTE: We need to pass the absolute path of the vectors + - pip install "tox < 4.0" + - cd $CODEBUILD_SRC_DIR/test_vector_handlers + - | + tox -- \ + --input $UNZIPPED_VECTORS_DIR/manifest.json From 2b36513d6324db308fd2035f550ec05d9754723d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 15:55:20 -0700 Subject: [PATCH 236/422] token --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 18 ++++++++++++++---- .../py37/decrypt_masterkey_with_masterkey.yml | 5 +++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index a66c6e2d6..178d16895 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -3,6 +3,9 @@ version: 0.2 env: variables: + git-credential-helper: yes + secrets-manager: + GH_TOKEN: Github/aws-crypto-tools-ci-bot:personal access token TOXENV: "py37-full_decrypt" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f @@ -21,25 +24,32 @@ phases: python: 3.7 pre_build: commands: + # Authenticate into the CI bot to allow session to download ESDK Dafny GHA artifact + - git config --global user.name "aws-crypto-tools-ci-bot" + - git config --global user.email "no-reply@noemail.local" + - echo $GH_TOKEN > token.txt + # Blank out the token; we're done with it + # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct) - | - MOST_RECENT_RUN_ID=curl -L \ + MOST_RECENT_RUN_ID=curl \ -H "Accept: application/vnd.github+json" \ -H "X-GitHub-Api-Version: 2022-11-28" \ "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)' - | - MOST_RECENT_RUN_DOWNLOAD_URL=curl -L \ + MOST_RECENT_RUN_DOWNLOAD_URL=curl \ -H "Accept: application/vnd.github+json" \ -H "X-GitHub-Api-Version: 2022-11-28" \ "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" \ | jq '.artifacts[0].archive_download_url' - | - curl -L \ + curl \ -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }} \ + -H "Authorization: Bearer $GH_TOKEN \ -H "X-GitHub-Api-Version: 2022-11-28" \ + -o ubuntu-latest_test_vector_artifact.zip "$MOST_RECENT_RUN_DOWNLOAD_URL" - unzip ubuntu-latest_test_vector_artifact diff --git a/codebuild/py37/decrypt_masterkey_with_masterkey.yml b/codebuild/py37/decrypt_masterkey_with_masterkey.yml index df7067e60..357482e1e 100644 --- a/codebuild/py37/decrypt_masterkey_with_masterkey.yml +++ b/codebuild/py37/decrypt_masterkey_with_masterkey.yml @@ -22,5 +22,6 @@ phases: commands: - pip install "tox < 4.0" - cd test_vector_handlers - - tox -- \ - --input ../tmp/generated/37_masterkey + - | + tox -- \ + --input ../tmp/generated/37_masterkey From be6a25363ea74a5fc67d86d37cd06f09438d0dd5 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 15:58:16 -0700 Subject: [PATCH 237/422] token --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 178d16895..1f765daf2 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -3,9 +3,6 @@ version: 0.2 env: variables: - git-credential-helper: yes - secrets-manager: - GH_TOKEN: Github/aws-crypto-tools-ci-bot:personal access token TOXENV: "py37-full_decrypt" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f @@ -17,6 +14,9 @@ env: arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + git-credential-helper: yes + secrets-manager: + GH_TOKEN: Github/aws-crypto-tools-ci-bot:personal access token phases: install: From a431365e91224bdfaf13169b3fff49a6b45458b2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 16:13:19 -0700 Subject: [PATCH 238/422] debug --- codebuild/py37/decrypt_masterkey_with_masterkey.yml | 2 +- codebuild/py37/generate_decrypt_vectors.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/codebuild/py37/decrypt_masterkey_with_masterkey.yml b/codebuild/py37/decrypt_masterkey_with_masterkey.yml index 357482e1e..8e6f916f5 100644 --- a/codebuild/py37/decrypt_masterkey_with_masterkey.yml +++ b/codebuild/py37/decrypt_masterkey_with_masterkey.yml @@ -24,4 +24,4 @@ phases: - cd test_vector_handlers - | tox -- \ - --input ../tmp/generated/37_masterkey + --input tmp/generated/37_masterkey/manifest.json \ No newline at end of file diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml index 04a09a47d..873aac2e6 100644 --- a/codebuild/py37/generate_decrypt_vectors.yml +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -25,4 +25,4 @@ phases: - | tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ - --output ../tmp/generated/37_masterkey + --output tmp/generated/37_masterkey From 4120be3b1ec3f46f4a5e291aaa236f0fa7e73350 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 16:33:26 -0700 Subject: [PATCH 239/422] debug cb --- buildspec.yml | 176 +++++++++++++++++++++++++------------------------- 1 file changed, 88 insertions(+), 88 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index ca87c41c5..3d0f7d684 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -150,94 +150,94 @@ batch: # env: # image: aws/codebuild/standard:5.0 - # - identifier: py311_integ - # buildspec: codebuild/py311/integ.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_integ_mpl - # buildspec: codebuild/py311/integ_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_examples - # buildspec: codebuild/py311/examples.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_examples_mpl - # buildspec: codebuild/py311/examples_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_awses_latest - # buildspec: codebuild/py311/awses_local.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_awses_latest_mpl - # buildspec: codebuild/py311/awses_local_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_mplawses_latest_mpl - # buildspec: codebuild/py311/mplawses_local_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_dafny_esdk_vectors_masterkey - # buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py311_decrypt_dafny_esdk_vectors_keyrings - # buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py311_decrypt_net_401_vectors_masterkey - # buildspec: codebuild/py311/decrypt_net_401_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py311_decrypt_net_401_vectors_keyrings - # buildspec: codebuild/py311/decrypt_net_401_vectors_keyrings.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py311_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py311/generate_decrypt_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py311_decrypt_masterkey_with_masterkey - # depend-on: - # - py311_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py311/decrypt_masterkey_with_masterkey.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py311_decrypt_masterkey_with_keyrings - # depend-on: - # - py311_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py311/decrypt_masterkey_with_keyrings.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py311_decrypt_masterkey_with_js - # depend-on: - # - py311_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py311/decrypt_masterkey_with_js.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py311_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py311/generate_decrypt_vectors_keyrings.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py311_decrypt_keyrings_with_masterkey - # depend-on: - # - py311_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py311/decrypt_keyrings_with_masterkey.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py311_decrypt_keyrings_with_keyrings - # depend-on: - # - py311_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py311/decrypt_keyrings_with_keyrings.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py311_decrypt_keyrings_with_js - # depend-on: - # - py311_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py311/decrypt_keyrings_with_js.yml - # env: - # image: aws/codebuild/standard:5.0 + - identifier: py311_integ + buildspec: codebuild/py311/integ.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_integ_mpl + buildspec: codebuild/py311/integ_mpl.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_examples + buildspec: codebuild/py311/examples.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_examples_mpl + buildspec: codebuild/py311/examples_mpl.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_awses_latest + buildspec: codebuild/py311/awses_local.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_awses_latest_mpl + buildspec: codebuild/py311/awses_local_mpl.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_mplawses_latest_mpl + buildspec: codebuild/py311/mplawses_local_mpl.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_dafny_esdk_vectors_masterkey + buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_dafny_esdk_vectors_keyrings + buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_net_401_vectors_masterkey + buildspec: codebuild/py311/decrypt_net_401_vectors_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_net_401_vectors_keyrings + buildspec: codebuild/py311/decrypt_net_401_vectors_keyrings.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_generate_decrypt_vectors_masterkey + buildspec: codebuild/py311/generate_decrypt_vectors_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_masterkey_with_masterkey + depend-on: + - py311_generate_decrypt_vectors_masterkey + buildspec: codebuild/py311/decrypt_masterkey_with_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_masterkey_with_keyrings + depend-on: + - py311_generate_decrypt_vectors_masterkey + buildspec: codebuild/py311/decrypt_masterkey_with_keyrings.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_masterkey_with_js + depend-on: + - py311_generate_decrypt_vectors_masterkey + buildspec: codebuild/py311/decrypt_masterkey_with_js.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_generate_decrypt_vectors_keyrings + buildspec: codebuild/py311/generate_decrypt_vectors_keyrings.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_keyrings_with_masterkey + depend-on: + - py311_generate_decrypt_vectors_keyrings + buildspec: codebuild/py311/decrypt_keyrings_with_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_keyrings_with_keyrings + depend-on: + - py311_generate_decrypt_vectors_keyrings + buildspec: codebuild/py311/decrypt_keyrings_with_keyrings.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_decrypt_keyrings_with_js + depend-on: + - py311_generate_decrypt_vectors_keyrings + buildspec: codebuild/py311/decrypt_keyrings_with_js.yml + env: + image: aws/codebuild/standard:5.0 # - identifier: py312_integ # buildspec: codebuild/py312/integ.yml From 7391c783ef77073020bf7e50c4f1612207b6330e Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 16:36:11 -0700 Subject: [PATCH 240/422] missing --- .../decrypt_net_401_vectors_keyrings.yml | 46 +++++++++++++++++++ .../decrypt_net_401_vectors_masterkey.yml | 45 ++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 codebuild/py311/decrypt_net_401_vectors_keyrings.yml create mode 100644 codebuild/py311/decrypt_net_401_vectors_masterkey.yml diff --git a/codebuild/py311/decrypt_net_401_vectors_keyrings.yml b/codebuild/py311/decrypt_net_401_vectors_keyrings.yml new file mode 100644 index 000000000..cccf5eda6 --- /dev/null +++ b/codebuild/py311/decrypt_net_401_vectors_keyrings.yml @@ -0,0 +1,46 @@ +version: 0.2 +# Runs Only the ESDK-NET v4.0.1 Decryption Vectors, testing Required EC CMM + +env: + variables: + TOXENV: "py311-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + python: 3.11 + pre_build: + commands: + # Assume Role to access non-prod resources + - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") + - export TMP_ROLE + - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') + - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') + - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') + - aws sts get-caller-identity + + # Fetch ESDK .NET v4.0.1 Test Vectors + - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip + - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip + - curl -s --output $VECTOR_ZIP --location $VECTORS_URL + - UNZIPPED_VECTORS_DIR=$CODEBUILD_SRC_DIR/test_vector_handlers/net_401_vectors + - unzip $VECTOR_ZIP -d $UNZIPPED_VECTORS_DIR + build: + commands: + # NOTE: We need to pass the absolute path of the vectors + - pip install "tox < 4.0" + - cd $CODEBUILD_SRC_DIR/test_vector_handlers + - | + tox -- \ + --input $UNZIPPED_VECTORS_DIR/manifest.json \ + --keyrings diff --git a/codebuild/py311/decrypt_net_401_vectors_masterkey.yml b/codebuild/py311/decrypt_net_401_vectors_masterkey.yml new file mode 100644 index 000000000..f6f0482e7 --- /dev/null +++ b/codebuild/py311/decrypt_net_401_vectors_masterkey.yml @@ -0,0 +1,45 @@ +version: 0.2 +# Runs Only the ESDK-NET v4.0.1 Decryption Vectors, testing Required EC CMM + +env: + variables: + TOXENV: "py311-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + python: 3.11 + pre_build: + commands: + # Assume Role to access non-prod resources + - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") + - export TMP_ROLE + - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') + - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') + - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') + - aws sts get-caller-identity + + # Fetch ESDK .NET v4.0.1 Test Vectors + - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip + - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip + - curl -s --output $VECTOR_ZIP --location $VECTORS_URL + - UNZIPPED_VECTORS_DIR=$CODEBUILD_SRC_DIR/test_vector_handlers/net_401_vectors + - unzip $VECTOR_ZIP -d $UNZIPPED_VECTORS_DIR + build: + commands: + # NOTE: We need to pass the absolute path of the vectors + - pip install "tox < 4.0" + - cd $CODEBUILD_SRC_DIR/test_vector_handlers + - | + tox -- \ + --input $UNZIPPED_VECTORS_DIR/manifest.json \ No newline at end of file From a16be01f77f05bba6ba7864c748d22f709b593e4 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 16:39:15 -0700 Subject: [PATCH 241/422] missing --- buildspec.yml | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index 3d0f7d684..db168cc78 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -181,63 +181,63 @@ batch: - identifier: py311_decrypt_dafny_esdk_vectors_masterkey buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml env: - image: aws/codebuild/standard:5.0 + image: aws/codebuild/standard:7.0 - identifier: py311_decrypt_dafny_esdk_vectors_keyrings buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml env: - image: aws/codebuild/standard:5.0 + image: aws/codebuild/standard:7.0 - identifier: py311_decrypt_net_401_vectors_masterkey buildspec: codebuild/py311/decrypt_net_401_vectors_masterkey.yml env: - image: aws/codebuild/standard:5.0 + image: aws/codebuild/standard:7.0 - identifier: py311_decrypt_net_401_vectors_keyrings buildspec: codebuild/py311/decrypt_net_401_vectors_keyrings.yml env: - image: aws/codebuild/standard:5.0 + image: aws/codebuild/standard:7.0 - identifier: py311_generate_decrypt_vectors_masterkey buildspec: codebuild/py311/generate_decrypt_vectors_masterkey.yml env: - image: aws/codebuild/standard:5.0 + image: aws/codebuild/standard:7.0 - identifier: py311_decrypt_masterkey_with_masterkey depend-on: - py311_generate_decrypt_vectors_masterkey buildspec: codebuild/py311/decrypt_masterkey_with_masterkey.yml env: - image: aws/codebuild/standard:5.0 + image: aws/codebuild/standard:7.0 - identifier: py311_decrypt_masterkey_with_keyrings depend-on: - py311_generate_decrypt_vectors_masterkey buildspec: codebuild/py311/decrypt_masterkey_with_keyrings.yml env: - image: aws/codebuild/standard:5.0 + image: aws/codebuild/standard:7.0 - identifier: py311_decrypt_masterkey_with_js depend-on: - py311_generate_decrypt_vectors_masterkey buildspec: codebuild/py311/decrypt_masterkey_with_js.yml env: - image: aws/codebuild/standard:5.0 + image: aws/codebuild/standard:7.0 - identifier: py311_generate_decrypt_vectors_keyrings buildspec: codebuild/py311/generate_decrypt_vectors_keyrings.yml env: - image: aws/codebuild/standard:5.0 + image: aws/codebuild/standard:7.0 - identifier: py311_decrypt_keyrings_with_masterkey depend-on: - py311_generate_decrypt_vectors_keyrings buildspec: codebuild/py311/decrypt_keyrings_with_masterkey.yml env: - image: aws/codebuild/standard:5.0 + image: aws/codebuild/standard:7.0 - identifier: py311_decrypt_keyrings_with_keyrings depend-on: - py311_generate_decrypt_vectors_keyrings buildspec: codebuild/py311/decrypt_keyrings_with_keyrings.yml env: - image: aws/codebuild/standard:5.0 + image: aws/codebuild/standard:7.0 - identifier: py311_decrypt_keyrings_with_js depend-on: - py311_generate_decrypt_vectors_keyrings buildspec: codebuild/py311/decrypt_keyrings_with_js.yml env: - image: aws/codebuild/standard:5.0 + image: aws/codebuild/standard:7.0 # - identifier: py312_integ # buildspec: codebuild/py312/integ.yml @@ -270,63 +270,63 @@ batch: # - identifier: py312_decrypt_dafny_esdk_vectors_masterkey # buildspec: codebuild/py312/decrypt_dafny_esdk_vectors_masterkey.yml # env: - # image: aws/codebuild/standard:5.0 + # image: aws/codebuild/standard:7.0 # - identifier: py312_decrypt_dafny_esdk_vectors_keyrings # buildspec: codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml # env: - # image: aws/codebuild/standard:5.0 + # image: aws/codebuild/standard:7.0 # - identifier: py312_decrypt_net_401_vectors_masterkey # buildspec: codebuild/py312/decrypt_net_401_vectors_masterkey.yml # env: - # image: aws/codebuild/standard:5.0 + # image: aws/codebuild/standard:7.0 # - identifier: py312_decrypt_net_401_vectors_keyrings # buildspec: codebuild/py312/decrypt_net_401_vectors_keyrings.yml # env: - # image: aws/codebuild/standard:5.0 + # image: aws/codebuild/standard:7.0 # - identifier: py312_generate_decrypt_vectors_masterkey # buildspec: codebuild/py312/generate_decrypt_vectors_masterkey.yml # env: - # image: aws/codebuild/standard:5.0 + # image: aws/codebuild/standard:7.0 # - identifier: py312_decrypt_masterkey_with_masterkey # depend-on: # - py312_generate_decrypt_vectors_masterkey # buildspec: codebuild/py312/decrypt_masterkey_with_masterkey.yml # env: - # image: aws/codebuild/standard:5.0 + # image: aws/codebuild/standard:7.0 # - identifier: py312_decrypt_masterkey_with_keyrings # depend-on: # - py312_generate_decrypt_vectors_masterkey # buildspec: codebuild/py312/decrypt_masterkey_with_keyrings.yml # env: - # image: aws/codebuild/standard:5.0 + # image: aws/codebuild/standard:7.0 # - identifier: py312_decrypt_masterkey_with_js # depend-on: # - py312_generate_decrypt_vectors_masterkey # buildspec: codebuild/py312/decrypt_masterkey_with_js.yml # env: - # image: aws/codebuild/standard:5.0 + # image: aws/codebuild/standard:7.0 # - identifier: py312_generate_decrypt_vectors_keyrings # buildspec: codebuild/py312/generate_decrypt_vectors_keyrings.yml # env: - # image: aws/codebuild/standard:5.0 + # image: aws/codebuild/standard:7.0 # - identifier: py312_decrypt_keyrings_with_masterkey # depend-on: # - py312_generate_decrypt_vectors_keyrings # buildspec: codebuild/py312/decrypt_keyrings_with_masterkey.yml # env: - # image: aws/codebuild/standard:5.0 + # image: aws/codebuild/standard:7.0 # - identifier: py312_decrypt_keyrings_with_keyrings # depend-on: # - py312_generate_decrypt_vectors_keyrings # buildspec: codebuild/py312/decrypt_keyrings_with_keyrings.yml # env: - # image: aws/codebuild/standard:5.0 + # image: aws/codebuild/standard:7.0 # - identifier: py312_decrypt_keyrings_with_js # depend-on: # - py312_generate_decrypt_vectors_keyrings # buildspec: codebuild/py312/decrypt_keyrings_with_js.yml # env: - # image: aws/codebuild/standard:5.0 + # image: aws/codebuild/standard:7.0 # - identifier: code_coverage # buildspec: codebuild/coverage/coverage.yml From 4b2553be0f77628ed83673b629c75d3d52e304a4 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 16:41:05 -0700 Subject: [PATCH 242/422] perms --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 1f765daf2..8478d1c62 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -24,6 +24,14 @@ phases: python: 3.7 pre_build: commands: + # Assume Role to access non-prod resources + - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") + - export TMP_ROLE + - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') + - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') + - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') + - aws sts get-caller-identity + # Authenticate into the CI bot to allow session to download ESDK Dafny GHA artifact - git config --global user.name "aws-crypto-tools-ci-bot" - git config --global user.email "no-reply@noemail.local" @@ -52,14 +60,6 @@ phases: -o ubuntu-latest_test_vector_artifact.zip "$MOST_RECENT_RUN_DOWNLOAD_URL" - unzip ubuntu-latest_test_vector_artifact - - # Assume Role to access non-prod resources - - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") - - export TMP_ROLE - - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') - - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') - - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') - - aws sts get-caller-identity build: commands: # NOTE: We need to pass the absolute path of the vectors From 843ac28c519e5bce05fdc0a322e50ae0b06e101d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 16:50:27 -0700 Subject: [PATCH 243/422] debug cb --- codebuild/py311/decrypt_net_401_vectors_keyrings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py311/decrypt_net_401_vectors_keyrings.yml b/codebuild/py311/decrypt_net_401_vectors_keyrings.yml index cccf5eda6..2b17002e5 100644 --- a/codebuild/py311/decrypt_net_401_vectors_keyrings.yml +++ b/codebuild/py311/decrypt_net_401_vectors_keyrings.yml @@ -3,7 +3,7 @@ version: 0.2 env: variables: - TOXENV: "py311-full_decrypt" + TOXENV: "py311-full_decrypt-mpl" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- From 68c068ecb12931f13d80f16c594156500ca88ab8 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 14 Mar 2024 17:02:37 -0700 Subject: [PATCH 244/422] debug cb --- codebuild/py311/decrypt_net_401_vectors_keyrings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py311/decrypt_net_401_vectors_keyrings.yml b/codebuild/py311/decrypt_net_401_vectors_keyrings.yml index 2b17002e5..6634470c3 100644 --- a/codebuild/py311/decrypt_net_401_vectors_keyrings.yml +++ b/codebuild/py311/decrypt_net_401_vectors_keyrings.yml @@ -21,7 +21,7 @@ phases: python: 3.11 pre_build: commands: - # Assume Role to access non-prod resources + # Assume Role to access non-prod resource - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") - export TMP_ROLE - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') From adf7198bd25d1c7dac13499353dbfbddc5ef32ca Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 11:49:25 -0700 Subject: [PATCH 245/422] files in s3 --- codebuild/py37/generate_decrypt_vectors.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml index 873aac2e6..5ca7d1a4a 100644 --- a/codebuild/py37/generate_decrypt_vectors.yml +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -26,3 +26,6 @@ phases: tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ --output tmp/generated/37_masterkey +artifacts: + files: + - tmp/generated/37_masterkey/**/* \ No newline at end of file From e2464637c3f2db1b04d3941e417edf731b099975 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 11:55:54 -0700 Subject: [PATCH 246/422] files in s3 --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 8478d1c62..9536aa771 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -57,7 +57,7 @@ phases: -H "Accept: application/vnd.github+json" \ -H "Authorization: Bearer $GH_TOKEN \ -H "X-GitHub-Api-Version: 2022-11-28" \ - -o ubuntu-latest_test_vector_artifact.zip + -o ubuntu-latest_test_vector_artifact.zip \ "$MOST_RECENT_RUN_DOWNLOAD_URL" - unzip ubuntu-latest_test_vector_artifact build: From 1aa07e5f1f515349ea27a944bd8d5233b707b343 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 11:58:54 -0700 Subject: [PATCH 247/422] files in s3 --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 9536aa771..bbb58ec30 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -55,7 +55,7 @@ phases: - | curl \ -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $GH_TOKEN \ + -H "Authorization: Bearer $GH_TOKEN" \ -H "X-GitHub-Api-Version: 2022-11-28" \ -o ubuntu-latest_test_vector_artifact.zip \ "$MOST_RECENT_RUN_DOWNLOAD_URL" From 1702822607422ce657abd6df0b6e6149b8119e4e Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 12:01:35 -0700 Subject: [PATCH 248/422] files in s3 --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index bbb58ec30..21bee5367 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -36,10 +36,10 @@ phases: - git config --global user.name "aws-crypto-tools-ci-bot" - git config --global user.email "no-reply@noemail.local" - echo $GH_TOKEN > token.txt - # Blank out the token; we're done with it # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct) + - curl -h - | MOST_RECENT_RUN_ID=curl \ -H "Accept: application/vnd.github+json" \ From 207401aae1660a21b30440472fd9d5b856025d40 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 12:06:36 -0700 Subject: [PATCH 249/422] files in s3 --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 21bee5367..9c9232138 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -39,20 +39,19 @@ phases: # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct) - - curl -h - - | - MOST_RECENT_RUN_ID=curl \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ + - > + MOST_RECENT_RUN_ID=curl + -H "Accept: application/vnd.github+json" + -H "X-GitHub-Api-Version: 2022-11-28" + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)' - - | + - > MOST_RECENT_RUN_DOWNLOAD_URL=curl \ -H "Accept: application/vnd.github+json" \ -H "X-GitHub-Api-Version: 2022-11-28" \ "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" \ | jq '.artifacts[0].archive_download_url' - - | + - > curl \ -H "Accept: application/vnd.github+json" \ -H "Authorization: Bearer $GH_TOKEN" \ From b3e0125577718ab1393399c3c970b13fad8f6823 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 12:09:04 -0700 Subject: [PATCH 250/422] files in s3 --- codebuild/py37/generate_decrypt_vectors.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml index 5ca7d1a4a..be85d73e7 100644 --- a/codebuild/py37/generate_decrypt_vectors.yml +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -25,7 +25,7 @@ phases: - | tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ - --output tmp/generated/37_masterkey + --output 37_masterkey artifacts: files: - - tmp/generated/37_masterkey/**/* \ No newline at end of file + - 37_masterkey/**/* \ No newline at end of file From 644c343d52db17ed87a04e7f27a0114d6eb556a8 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 12:10:37 -0700 Subject: [PATCH 251/422] files in s3 --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 25 +++++-------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 9c9232138..3d921e3fd 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -39,25 +39,12 @@ phases: # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct) - - > - MOST_RECENT_RUN_ID=curl - -H "Accept: application/vnd.github+json" - -H "X-GitHub-Api-Version: 2022-11-28" - "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" - | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)' - - > - MOST_RECENT_RUN_DOWNLOAD_URL=curl \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" \ - | jq '.artifacts[0].archive_download_url' - - > - curl \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $GH_TOKEN" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - -o ubuntu-latest_test_vector_artifact.zip \ - "$MOST_RECENT_RUN_DOWNLOAD_URL" + - | + MOST_RECENT_RUN_ID=curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)' + - | + MOST_RECENT_RUN_DOWNLOAD_URL=curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url' + - | + curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GH_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip "$MOST_RECENT_RUN_DOWNLOAD_URL" - unzip ubuntu-latest_test_vector_artifact build: commands: From e72ab997647f67d3734becc808f4c22ea74163e4 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 13:18:47 -0700 Subject: [PATCH 252/422] files in s3 --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 3d921e3fd..991e3b50d 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -40,9 +40,9 @@ phases: # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct) - | - MOST_RECENT_RUN_ID=curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)' + MOST_RECENT_RUN_ID='curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq "first(.workflow_runs[] | select(.name=="Daily CI") | .id)"' - | - MOST_RECENT_RUN_DOWNLOAD_URL=curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url' + MOST_RECENT_RUN_DOWNLOAD_URL='curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq ".artifacts[0].archive_download_url"' - | curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GH_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip "$MOST_RECENT_RUN_DOWNLOAD_URL" - unzip ubuntu-latest_test_vector_artifact From 36fd56b97a9a288e592543442942eed3bd4d36f6 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 13:27:40 -0700 Subject: [PATCH 253/422] files in s3 --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 991e3b50d..3ee14d3c0 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -41,8 +41,12 @@ phases: # (Assuming the first result is most recent; seems to be correct) - | MOST_RECENT_RUN_ID='curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq "first(.workflow_runs[] | select(.name=="Daily CI") | .id)"' + - | + echo $MOST_RECENT_RUN_ID - | MOST_RECENT_RUN_DOWNLOAD_URL='curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq ".artifacts[0].archive_download_url"' + - | + echo $MOST_RECENT_RUN_DOWNLOAD_URL - | curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GH_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip "$MOST_RECENT_RUN_DOWNLOAD_URL" - unzip ubuntu-latest_test_vector_artifact From 6052b53fee0c225b3d74753ad1ee37b9a577a27d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 13:30:42 -0700 Subject: [PATCH 254/422] files in s3 --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 3ee14d3c0..a8973776f 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -40,11 +40,11 @@ phases: # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct) - | - MOST_RECENT_RUN_ID='curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq "first(.workflow_runs[] | select(.name=="Daily CI") | .id)"' + MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq "first(.workflow_runs[] | select(.name=="Daily CI") | .id)") - | echo $MOST_RECENT_RUN_ID - | - MOST_RECENT_RUN_DOWNLOAD_URL='curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq ".artifacts[0].archive_download_url"' + MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq ".artifacts[0].archive_download_url") - | echo $MOST_RECENT_RUN_DOWNLOAD_URL - | From 457aa8836f04f63830454c1061404406814475a2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 13:38:51 -0700 Subject: [PATCH 255/422] files in s3 --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index a8973776f..8a2483cd6 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -40,11 +40,11 @@ phases: # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct) - | - MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq "first(.workflow_runs[] | select(.name=="Daily CI") | .id)") + MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') - | echo $MOST_RECENT_RUN_ID - | - MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq ".artifacts[0].archive_download_url") + MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url') - | echo $MOST_RECENT_RUN_DOWNLOAD_URL - | From a8b65d3e5809637bd11fe6854f618b92616cf798 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 13:41:36 -0700 Subject: [PATCH 256/422] files in s3 --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 8a2483cd6..d2463e1b7 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -48,7 +48,7 @@ phases: - | echo $MOST_RECENT_RUN_DOWNLOAD_URL - | - curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GH_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip "$MOST_RECENT_RUN_DOWNLOAD_URL" + curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GH_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip $MOST_RECENT_RUN_DOWNLOAD_URL - unzip ubuntu-latest_test_vector_artifact build: commands: From 8ed6cca93c01bbf37f6c8c99ef380c28f6f66c5f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 13:48:02 -0700 Subject: [PATCH 257/422] debug gen --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 4 ++-- codebuild/py37/generate_decrypt_vectors.yml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index d2463e1b7..9294501e9 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -42,11 +42,11 @@ phases: - | MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') - | - echo $MOST_RECENT_RUN_ID + echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" - | MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url') - | - echo $MOST_RECENT_RUN_DOWNLOAD_URL + echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" - | curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GH_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip $MOST_RECENT_RUN_DOWNLOAD_URL - unzip ubuntu-latest_test_vector_artifact diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml index be85d73e7..517544815 100644 --- a/codebuild/py37/generate_decrypt_vectors.yml +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -26,6 +26,8 @@ phases: tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ --output 37_masterkey + - ls + - zip 37_master.zip 37_masterkey artifacts: files: - 37_masterkey/**/* \ No newline at end of file From ef4a9d7605d90718e4a458e4456618cbc5622b57 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 13:51:01 -0700 Subject: [PATCH 258/422] debug gen --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 9294501e9..1433ed114 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -48,7 +48,7 @@ phases: - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" - | - curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GH_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip $MOST_RECENT_RUN_DOWNLOAD_URL + curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip $MOST_RECENT_RUN_DOWNLOAD_URL - unzip ubuntu-latest_test_vector_artifact build: commands: From b564f77150d1d633869b88cab68b31a6d7ccca3f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 13:58:20 -0700 Subject: [PATCH 259/422] debug gen --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 1433ed114..d5e94a7d5 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -44,7 +44,7 @@ phases: - | echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" - | - MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url') + MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url[8:]') - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" - | From 31a58ccea0f86f8e4f2d657e855693ab8bc1deed Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 14:01:42 -0700 Subject: [PATCH 260/422] debug gen --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index d5e94a7d5..00195a437 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -44,11 +44,11 @@ phases: - | echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" - | - MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url[8:]') + MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url') - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" - | - curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip $MOST_RECENT_RUN_DOWNLOAD_URL + curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" $MOST_RECENT_RUN_DOWNLOAD_URL -o ubuntu-latest_test_vector_artifact.zip - unzip ubuntu-latest_test_vector_artifact build: commands: From ed879537d06f4bcbb2599a66a73d5197cabaf6f0 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 14:03:46 -0700 Subject: [PATCH 261/422] debug gen --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 00195a437..9ce80d4a8 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -48,7 +48,7 @@ phases: - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" - | - curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" $MOST_RECENT_RUN_DOWNLOAD_URL -o ubuntu-latest_test_vector_artifact.zip + $(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" $MOST_RECENT_RUN_DOWNLOAD_URL -o ubuntu-latest_test_vector_artifact.zip) - unzip ubuntu-latest_test_vector_artifact build: commands: From ac01f37c0b66962d13ff668323491e16ab8744ca Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 14:05:59 -0700 Subject: [PATCH 262/422] debug gen --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 9ce80d4a8..9ed28658d 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -48,7 +48,7 @@ phases: - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" - | - $(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" $MOST_RECENT_RUN_DOWNLOAD_URL -o ubuntu-latest_test_vector_artifact.zip) + $(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/artifacts/1326417479/zip" -o ubuntu-latest_test_vector_artifact.zip) - unzip ubuntu-latest_test_vector_artifact build: commands: From 69a934c36560b148f2fffa5fb73c45530902d458 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 14:09:58 -0700 Subject: [PATCH 263/422] debug gen --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 9ed28658d..99ee498b9 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -48,7 +48,7 @@ phases: - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" - | - $(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/artifacts/1326417479/zip" -o ubuntu-latest_test_vector_artifact.zip) + curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GH_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/artifacts/1326417479/zip" - unzip ubuntu-latest_test_vector_artifact build: commands: From 65d3acdd5e5e19d46bea860e7814064898cf01ea Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 14:12:44 -0700 Subject: [PATCH 264/422] debug gen --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 99ee498b9..c6401837d 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -38,7 +38,7 @@ phases: - echo $GH_TOKEN > token.txt # Fetch test vectors from Dafny ESDK's most recent run - # (Assuming the first result is most recent; seems to be correct) + # (Assuming the first result is most recent; seems to be correct...) - | MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') - | From 95e8a8bb804ea646065de280884057803914e4ad Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 14:16:39 -0700 Subject: [PATCH 265/422] debug gen --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index c6401837d..99ee498b9 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -38,7 +38,7 @@ phases: - echo $GH_TOKEN > token.txt # Fetch test vectors from Dafny ESDK's most recent run - # (Assuming the first result is most recent; seems to be correct...) + # (Assuming the first result is most recent; seems to be correct) - | MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') - | From 8d484e60cf97f6876ffcc5957c18dad4309af041 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 14:23:58 -0700 Subject: [PATCH 266/422] debug gen --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 99ee498b9..e0580a45e 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -40,7 +40,7 @@ phases: # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct) - | - MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') + MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true") - | echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" - | From 8dcfc3cfd8733960836505d2c6ff10b612826240 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 14:25:56 -0700 Subject: [PATCH 267/422] debug gen --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index e0580a45e..99ee498b9 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -40,7 +40,7 @@ phases: # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct) - | - MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true") + MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') - | echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" - | From a9306bc1a0f9a347d819d73a7fd61175ffce7838 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 14:38:25 -0700 Subject: [PATCH 268/422] debug gen --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 99ee498b9..dbdb1387c 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -35,10 +35,11 @@ phases: # Authenticate into the CI bot to allow session to download ESDK Dafny GHA artifact - git config --global user.name "aws-crypto-tools-ci-bot" - git config --global user.email "no-reply@noemail.local" - - echo $GH_TOKEN > token.txt + - | + echo "DEBUG: $GH_TOKEN" # Fetch test vectors from Dafny ESDK's most recent run - # (Assuming the first result is most recent; seems to be correct) + # (Assuming the first result is most recent; seems to be correct...) - | MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') - | @@ -48,7 +49,7 @@ phases: - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" - | - curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GH_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/artifacts/1326417479/zip" + curl -L -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GH_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip $MOST_RECENT_RUN_DOWNLOAD_URL - unzip ubuntu-latest_test_vector_artifact build: commands: From 894dcee5a2c3e81b78e03493f7db5a7d3a5aac88 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 14:45:49 -0700 Subject: [PATCH 269/422] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index dbdb1387c..3f1a7afee 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -35,15 +35,15 @@ phases: # Authenticate into the CI bot to allow session to download ESDK Dafny GHA artifact - git config --global user.name "aws-crypto-tools-ci-bot" - git config --global user.email "no-reply@noemail.local" - - | - echo "DEBUG: $GH_TOKEN" # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct...) - | - MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') + MOST_RECENT_RUN_STUFF=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') + - | + echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_STUFF" - | - echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" + MOST_RECENT_RUN_ID=$(echo $MOST_RECENT_RUN_STUFF | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') - | MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url') - | From e00ec2ac6aae2f67ac0addcd88949e3b631656f0 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 14:46:15 -0700 Subject: [PATCH 270/422] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 3f1a7afee..03cd87ddf 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -39,7 +39,7 @@ phases: # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct...) - | - MOST_RECENT_RUN_STUFF=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') + MOST_RECENT_RUN_STUFF=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true") - | echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_STUFF" - | From 2f23be33ef1494a6e15854684dbc7990f62947f9 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 14:49:15 -0700 Subject: [PATCH 271/422] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 03cd87ddf..85f9190db 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -35,6 +35,10 @@ phases: # Authenticate into the CI bot to allow session to download ESDK Dafny GHA artifact - git config --global user.name "aws-crypto-tools-ci-bot" - git config --global user.email "no-reply@noemail.local" + - echo $GH_TOKEN > token.txt + gh auth login --with-token < token.txt + rm token.txt + gh auth status # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct...) From 6a7d73250152a54efe9c0bc5d9c86f5971f1c843 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 14:50:06 -0700 Subject: [PATCH 272/422] debug --- codebuild/py37/generate_decrypt_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml index 517544815..f0d4d36a0 100644 --- a/codebuild/py37/generate_decrypt_vectors.yml +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -30,4 +30,4 @@ phases: - zip 37_master.zip 37_masterkey artifacts: files: - - 37_masterkey/**/* \ No newline at end of file + - 37_master.zip \ No newline at end of file From 8cbd3dcd128dfd56964463608b6031735ae38070 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 14:51:28 -0700 Subject: [PATCH 273/422] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 85f9190db..bce139705 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -36,9 +36,9 @@ phases: - git config --global user.name "aws-crypto-tools-ci-bot" - git config --global user.email "no-reply@noemail.local" - echo $GH_TOKEN > token.txt - gh auth login --with-token < token.txt - rm token.txt - gh auth status + - gh auth login --with-token < token.txt + - rm token.txt + - gh auth status # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct...) From 5cb4b13d6cbeb201f767e46c94d2ed4ac8f2407c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 14:55:27 -0700 Subject: [PATCH 274/422] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index bce139705..5f51f8f9a 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -36,6 +36,11 @@ phases: - git config --global user.name "aws-crypto-tools-ci-bot" - git config --global user.email "no-reply@noemail.local" - echo $GH_TOKEN > token.txt + + - type -p yum-config-manager >/dev/null || sudo yum install yum-utils + - sudo yum-config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo + - sudo yum install gh + - gh auth login --with-token < token.txt - rm token.txt - gh auth status From 9d89a742e449ec220ca274de0c4cae7ab45eb618 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 15:03:05 -0700 Subject: [PATCH 275/422] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 5f51f8f9a..e5f733da9 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -37,9 +37,7 @@ phases: - git config --global user.email "no-reply@noemail.local" - echo $GH_TOKEN > token.txt - - type -p yum-config-manager >/dev/null || sudo yum install yum-utils - - sudo yum-config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo - - sudo yum install gh + - sudo apt install gh -y - gh auth login --with-token < token.txt - rm token.txt From 51bf320208d14d3e3d260957b7d48daabc181f35 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 15:05:23 -0700 Subject: [PATCH 276/422] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index e5f733da9..0d608638b 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -37,7 +37,12 @@ phases: - git config --global user.email "no-reply@noemail.local" - echo $GH_TOKEN > token.txt - - sudo apt install gh -y + - | + sudo mkdir -p -m 755 /etc/apt/keyrings && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ + && sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ + && sudo apt update \ + && sudo apt install gh -y - gh auth login --with-token < token.txt - rm token.txt From 3be4969cc23f479877ae585254a55946b7bc1bff Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 15:08:07 -0700 Subject: [PATCH 277/422] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 0d608638b..1b3f9a6f6 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -44,7 +44,7 @@ phases: && sudo apt update \ && sudo apt install gh -y - - gh auth login --with-token < token.txt + - gh auth login - rm token.txt - gh auth status From 9e3358dd9cfa861bfe17b3b88ddc0a3cbb696a5a Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 15:10:39 -0700 Subject: [PATCH 278/422] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 1b3f9a6f6..5bd760579 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -44,20 +44,18 @@ phases: && sudo apt update \ && sudo apt install gh -y - - gh auth login - - rm token.txt - gh auth status # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct...) - | - MOST_RECENT_RUN_STUFF=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true") + MOST_RECENT_RUN_STUFF=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GH_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true") - | echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_STUFF" - | MOST_RECENT_RUN_ID=$(echo $MOST_RECENT_RUN_STUFF | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') - | - MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url') + MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GH_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url') - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" - | From 1b7a54b564542e51e86fb51da67cf26bf104ca84 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 15:21:26 -0700 Subject: [PATCH 279/422] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 5bd760579..75ed3f1a4 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -16,7 +16,7 @@ env: AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" git-credential-helper: yes secrets-manager: - GH_TOKEN: Github/aws-crypto-tools-ci-bot:personal access token + GH_TOKEN: Github/aws-crypto-tools-ci-bot:personal access token (new format) phases: install: From ce59f5777b0d3f37287c4cfd201a70a725b52fd8 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 15:22:26 -0700 Subject: [PATCH 280/422] debug --- codebuild/py37/generate_decrypt_vectors.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml index f0d4d36a0..0d5b18085 100644 --- a/codebuild/py37/generate_decrypt_vectors.yml +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -27,6 +27,8 @@ phases: --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ --output 37_masterkey - ls + - cd 37_masterkey + - ls - zip 37_master.zip 37_masterkey artifacts: files: From 6ef093b63b47785512056a340a269e38bb2799e0 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 15:24:12 -0700 Subject: [PATCH 281/422] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 75ed3f1a4..3e0726b66 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -16,7 +16,7 @@ env: AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" git-credential-helper: yes secrets-manager: - GH_TOKEN: Github/aws-crypto-tools-ci-bot:personal access token (new format) + GH_TOKEN: Github/aws-crypto-tools-ci-bot:personal access token (new token format) phases: install: From ce07e87f3d73d49a9e65ed017d8393505274eb0e Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 15:32:44 -0700 Subject: [PATCH 282/422] Debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 3e0726b66..53aa50c27 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -16,7 +16,7 @@ env: AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" git-credential-helper: yes secrets-manager: - GH_TOKEN: Github/aws-crypto-tools-ci-bot:personal access token (new token format) + GITHUB_TOKEN: Github/aws-crypto-tools-ci-bot:personal access token (new token format) phases: install: @@ -44,8 +44,6 @@ phases: && sudo apt update \ && sudo apt install gh -y - - gh auth status - # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct...) - | From 7225e51bdd76d19714bc192016596655d59161a1 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 15:34:16 -0700 Subject: [PATCH 283/422] debug --- codebuild/py37/generate_decrypt_vectors.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml index 0d5b18085..cdad0861d 100644 --- a/codebuild/py37/generate_decrypt_vectors.yml +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -27,9 +27,7 @@ phases: --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ --output 37_masterkey - ls - - cd 37_masterkey - - ls - - zip 37_master.zip 37_masterkey + - zip -r 37_master.zip 37_masterkey artifacts: files: - 37_master.zip \ No newline at end of file From 549fe07537ba69f36d8e9593c160bf0ef1839136 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 15:38:27 -0700 Subject: [PATCH 284/422] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 53aa50c27..3d5316873 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -35,7 +35,6 @@ phases: # Authenticate into the CI bot to allow session to download ESDK Dafny GHA artifact - git config --global user.name "aws-crypto-tools-ci-bot" - git config --global user.email "no-reply@noemail.local" - - echo $GH_TOKEN > token.txt - | sudo mkdir -p -m 755 /etc/apt/keyrings && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ @@ -47,17 +46,17 @@ phases: # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct...) - | - MOST_RECENT_RUN_STUFF=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GH_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true") + MOST_RECENT_RUN_STUFF=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true") - | echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_STUFF" - | MOST_RECENT_RUN_ID=$(echo $MOST_RECENT_RUN_STUFF | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') - | - MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GH_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url') + MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url') - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" - | - curl -L -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GH_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip $MOST_RECENT_RUN_DOWNLOAD_URL + curl -L -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip $MOST_RECENT_RUN_DOWNLOAD_URL - unzip ubuntu-latest_test_vector_artifact build: commands: From 0f9b66258b12429c0e3f8bedf227fd90ab3120c2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 15:41:36 -0700 Subject: [PATCH 285/422] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 3d5316873..f5dc3126f 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -46,17 +46,17 @@ phases: # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct...) - | - MOST_RECENT_RUN_STUFF=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true") + MOST_RECENT_RUN_STUFF=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: token $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true") - | echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_STUFF" - | MOST_RECENT_RUN_ID=$(echo $MOST_RECENT_RUN_STUFF | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') - | - MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url') + MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: token $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url') - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" - | - curl -L -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip $MOST_RECENT_RUN_DOWNLOAD_URL + curl -L -H "Accept: application/vnd.github+json" -H "Authorization: token $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip $MOST_RECENT_RUN_DOWNLOAD_URL - unzip ubuntu-latest_test_vector_artifact build: commands: From 8dd346883f3f58abfc9d59d844a7d6456198eee0 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 15:45:45 -0700 Subject: [PATCH 286/422] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index f5dc3126f..5aae54440 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -46,7 +46,7 @@ phases: # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct...) - | - MOST_RECENT_RUN_STUFF=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: token $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true") + MOST_RECENT_RUN_STUFF=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: token $(echo ${GITHUB_TOKEN})" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true") - | echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_STUFF" - | From 5378b6282b71c0c3598af44c9b1661279438eaea Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 15:47:59 -0700 Subject: [PATCH 287/422] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 5aae54440..0313c9702 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -36,17 +36,17 @@ phases: - git config --global user.name "aws-crypto-tools-ci-bot" - git config --global user.email "no-reply@noemail.local" - - | - sudo mkdir -p -m 755 /etc/apt/keyrings && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ - && sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \ - && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ - && sudo apt update \ - && sudo apt install gh -y + # - | + # sudo mkdir -p -m 755 /etc/apt/keyrings && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ + # && sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \ + # && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ + # && sudo apt update \ + # && sudo apt install gh -y # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct...) - | - MOST_RECENT_RUN_STUFF=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: token $(echo ${GITHUB_TOKEN})" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true") + MOST_RECENT_RUN_STUFF=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: token ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true") - | echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_STUFF" - | From f3db08a8c5454697558208dbad700baa9245c32e Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 16:20:41 -0700 Subject: [PATCH 288/422] debug --- codebuild/py37/generate_decrypt_vectors.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml index cdad0861d..b379377e9 100644 --- a/codebuild/py37/generate_decrypt_vectors.yml +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -30,4 +30,5 @@ phases: - zip -r 37_master.zip 37_masterkey artifacts: files: - - 37_master.zip \ No newline at end of file + - ./37_master.zip + name: 37_master.zip \ No newline at end of file From 6080556c38432f248f7b5b4fc26de1debfd5b9dc Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 18 Mar 2024 16:47:43 -0700 Subject: [PATCH 289/422] debug --- codebuild/py37/generate_decrypt_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml index b379377e9..aac3d3f38 100644 --- a/codebuild/py37/generate_decrypt_vectors.yml +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -30,5 +30,5 @@ phases: - zip -r 37_master.zip 37_masterkey artifacts: files: - - ./37_master.zip + - test_vector_handlers/37_master.zip name: 37_master.zip \ No newline at end of file From 38623291b4277c6c2777de70511fd458c0fc0ea6 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 09:03:59 -0700 Subject: [PATCH 290/422] consume vectors --- codebuild/py37/decrypt_masterkey_with_js.yml | 6 +++++- codebuild/py37/decrypt_masterkey_with_masterkey.yml | 10 +++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/codebuild/py37/decrypt_masterkey_with_js.yml b/codebuild/py37/decrypt_masterkey_with_js.yml index 32db1083e..fe2f93535 100644 --- a/codebuild/py37/decrypt_masterkey_with_js.yml +++ b/codebuild/py37/decrypt_masterkey_with_js.yml @@ -33,7 +33,11 @@ phases: - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') - aws sts get-caller-identity - cd $CODEBUILD_SRC_DIR + + # Download generated vectors + # TODO rewrite URL + aws s3 cp s3://generated-vectors-artifacts-bucket/py37_generate_decrypt_vectors/test_vector_handlers/37_master.zip 37_master.zip build: commands: # Decrypt generated vectors with Javascript ESDK - - integration-node decrypt -v ../tmp/generated/37_masterkey \ No newline at end of file + - integration-node decrypt -v 37_master.zip \ No newline at end of file diff --git a/codebuild/py37/decrypt_masterkey_with_masterkey.yml b/codebuild/py37/decrypt_masterkey_with_masterkey.yml index 8e6f916f5..5a2347e8f 100644 --- a/codebuild/py37/decrypt_masterkey_with_masterkey.yml +++ b/codebuild/py37/decrypt_masterkey_with_masterkey.yml @@ -18,10 +18,18 @@ phases: install: runtime-versions: python: 3.7 + + + pre-build: + commands: + # Download generated vectors + # TODO rewrite URL + aws s3 cp s3://generated-vectors-artifacts-bucket/py37_generate_decrypt_vectors/test_vector_handlers/37_master.zip 37_masterkey.zip + unzip 37_master.zip build: commands: - pip install "tox < 4.0" - cd test_vector_handlers - | tox -- \ - --input tmp/generated/37_masterkey/manifest.json \ No newline at end of file + --input ../37_masterkey/manifest.json \ No newline at end of file From 30f71529f5a26a05247bd2ce305731d096e87a09 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 09:29:57 -0700 Subject: [PATCH 291/422] rerun ci --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 0313c9702..f7636d33a 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -32,7 +32,7 @@ phases: - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') - aws sts get-caller-identity - # Authenticate into the CI bot to allow session to download ESDK Dafny GHA artifact + # Authenticate into the CI bot to allow session to download ESDK Dafny GHA artifact. - git config --global user.name "aws-crypto-tools-ci-bot" - git config --global user.email "no-reply@noemail.local" From 876ed384f9894a9f65de4386fbee7938e8448d5f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 09:44:25 -0700 Subject: [PATCH 292/422] add missing files --- .../generate_decrypt_vectors_keyrings.yml | 29 +++++++++++++++++++ .../generate_decrypt_vectors_masterkey.yml | 28 ++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 codebuild/py311/generate_decrypt_vectors_keyrings.yml create mode 100644 codebuild/py311/generate_decrypt_vectors_masterkey.yml diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml new file mode 100644 index 000000000..081d944c5 --- /dev/null +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -0,0 +1,29 @@ +version: 0.2 + +env: + variables: + TOXENV: "py311-full_decrypt_generate-mpl" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + python: 3.11 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ + --output tmp/generated/37_masterkey + diff --git a/codebuild/py311/generate_decrypt_vectors_masterkey.yml b/codebuild/py311/generate_decrypt_vectors_masterkey.yml new file mode 100644 index 000000000..873aac2e6 --- /dev/null +++ b/codebuild/py311/generate_decrypt_vectors_masterkey.yml @@ -0,0 +1,28 @@ +version: 0.2 + +env: + variables: + TOXENV: "py37-full_decrypt_generate" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + python: 3.7 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ + --output tmp/generated/37_masterkey From 8a6bf33060d346519451a298409fb807e804866f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 09:46:35 -0700 Subject: [PATCH 293/422] 311 --- codebuild/py311/generate_decrypt_vectors_keyrings.yml | 9 +++++++-- codebuild/py311/generate_decrypt_vectors_masterkey.yml | 7 ++++++- codebuild/py37/generate_decrypt_vectors.yml | 1 - 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml index 081d944c5..1c9f3514a 100644 --- a/codebuild/py311/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -25,5 +25,10 @@ phases: - | tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ - --output tmp/generated/37_masterkey - + --output 311_keyring \ + --keyrings + - zip -r 311_keyring.zip 311_keyring + artifacts: + files: + - test_vector_handlers/311_keyring.zip + name: 311_keyring.zip \ No newline at end of file diff --git a/codebuild/py311/generate_decrypt_vectors_masterkey.yml b/codebuild/py311/generate_decrypt_vectors_masterkey.yml index 873aac2e6..69cbe418b 100644 --- a/codebuild/py311/generate_decrypt_vectors_masterkey.yml +++ b/codebuild/py311/generate_decrypt_vectors_masterkey.yml @@ -25,4 +25,9 @@ phases: - | tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ - --output tmp/generated/37_masterkey + --output 311_masterkey + - zip -r 311_masterkey.zip 311_masterkey + artifacts: + files: + - test_vector_handlers/311_masterkey.zip + name: 311_masterkey.zip \ No newline at end of file diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml index aac3d3f38..6c80a9a1c 100644 --- a/codebuild/py37/generate_decrypt_vectors.yml +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -26,7 +26,6 @@ phases: tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ --output 37_masterkey - - ls - zip -r 37_master.zip 37_masterkey artifacts: files: From 07f3b8f08dd8be8ed3eb4b1e1ca2147f410a81ad Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 09:58:50 -0700 Subject: [PATCH 294/422] cooking --- codebuild/py37/decrypt_masterkey_with_js.yml | 2 +- codebuild/py37/generate_decrypt_vectors.yml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/codebuild/py37/decrypt_masterkey_with_js.yml b/codebuild/py37/decrypt_masterkey_with_js.yml index fe2f93535..22ca730b7 100644 --- a/codebuild/py37/decrypt_masterkey_with_js.yml +++ b/codebuild/py37/decrypt_masterkey_with_js.yml @@ -36,7 +36,7 @@ phases: # Download generated vectors # TODO rewrite URL - aws s3 cp s3://generated-vectors-artifacts-bucket/py37_generate_decrypt_vectors/test_vector_handlers/37_master.zip 37_master.zip + aws s3 cp s3://generated-vectors-artifacts-bucket/GeneratedVectors/py37_generate_decrypt_vectors/test_vector_handlers/37_master.zip 37_master.zip build: commands: # Decrypt generated vectors with Javascript ESDK diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml index 6c80a9a1c..6e578e101 100644 --- a/codebuild/py37/generate_decrypt_vectors.yml +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -30,4 +30,5 @@ phases: artifacts: files: - test_vector_handlers/37_master.zip - name: 37_master.zip \ No newline at end of file + name: $CODEBUILD_BATCH_BUILD_IDENTIFIER/37_master.zip + discard-paths: true From a06684e6da126a991fb8c008fa1bc32624a71fe2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 10:06:16 -0700 Subject: [PATCH 295/422] cooking --- codebuild/py311/generate_decrypt_vectors_keyrings.yml | 9 +++++---- codebuild/py311/generate_decrypt_vectors_masterkey.yml | 9 +++++---- codebuild/py37/generate_decrypt_vectors.yml | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml index 1c9f3514a..f83ac733e 100644 --- a/codebuild/py311/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -28,7 +28,8 @@ phases: --output 311_keyring \ --keyrings - zip -r 311_keyring.zip 311_keyring - artifacts: - files: - - test_vector_handlers/311_keyring.zip - name: 311_keyring.zip \ No newline at end of file +artifacts: + files: + - test_vector_handlers/311_keyring.zip + name: $CODEBUILD_BATCH_BUILD_IDENTIFIER/311_keyring.zip + discard-paths: yes \ No newline at end of file diff --git a/codebuild/py311/generate_decrypt_vectors_masterkey.yml b/codebuild/py311/generate_decrypt_vectors_masterkey.yml index 69cbe418b..0975aa78c 100644 --- a/codebuild/py311/generate_decrypt_vectors_masterkey.yml +++ b/codebuild/py311/generate_decrypt_vectors_masterkey.yml @@ -27,7 +27,8 @@ phases: --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ --output 311_masterkey - zip -r 311_masterkey.zip 311_masterkey - artifacts: - files: - - test_vector_handlers/311_masterkey.zip - name: 311_masterkey.zip \ No newline at end of file +artifacts: + files: + - test_vector_handlers/311_masterkey.zip + name: $CODEBUILD_BATCH_BUILD_IDENTIFIER/311_masterkey.zip + discard-paths: yes \ No newline at end of file diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml index 6e578e101..7aad81e13 100644 --- a/codebuild/py37/generate_decrypt_vectors.yml +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -31,4 +31,4 @@ artifacts: files: - test_vector_handlers/37_master.zip name: $CODEBUILD_BATCH_BUILD_IDENTIFIER/37_master.zip - discard-paths: true + discard-paths: yes From 37fd225cd49a1c1da1dcf3e94e8dcac6905ad459 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 10:11:05 -0700 Subject: [PATCH 296/422] cooking --- codebuild/py311/generate_decrypt_vectors_masterkey.yml | 2 +- .../manifests/full_message/decrypt_generation.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/codebuild/py311/generate_decrypt_vectors_masterkey.yml b/codebuild/py311/generate_decrypt_vectors_masterkey.yml index 0975aa78c..848a19c92 100644 --- a/codebuild/py311/generate_decrypt_vectors_masterkey.yml +++ b/codebuild/py311/generate_decrypt_vectors_masterkey.yml @@ -17,7 +17,7 @@ env: phases: install: runtime-versions: - python: 3.7 + python: 3.11 build: commands: - pip install "tox < 4.0" diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index cef786335..782404704 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -50,7 +50,8 @@ ) _HAS_MPL = True -except ImportError: +except ImportError as e: + print(f"decrypt_generation ImportError: {e}") _HAS_MPL = False From e4590459b5f129a78f3579f9945ad7d0cb543366 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 10:15:33 -0700 Subject: [PATCH 297/422] cooking --- codebuild/py311/generate_decrypt_vectors_masterkey.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py311/generate_decrypt_vectors_masterkey.yml b/codebuild/py311/generate_decrypt_vectors_masterkey.yml index 848a19c92..655fb1985 100644 --- a/codebuild/py311/generate_decrypt_vectors_masterkey.yml +++ b/codebuild/py311/generate_decrypt_vectors_masterkey.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - TOXENV: "py37-full_decrypt_generate" + TOXENV: "py311-full_decrypt_generate" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- From 0e8c6c914495b0c4563de67190a9e0a4fd15be4c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 10:16:31 -0700 Subject: [PATCH 298/422] missing --- .../internal/tampering_mpl_materials.py | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 test_vector_handlers/src/awses_test_vectors/internal/tampering_mpl_materials.py diff --git a/test_vector_handlers/src/awses_test_vectors/internal/tampering_mpl_materials.py b/test_vector_handlers/src/awses_test_vectors/internal/tampering_mpl_materials.py new file mode 100644 index 000000000..4ad948424 --- /dev/null +++ b/test_vector_handlers/src/awses_test_vectors/internal/tampering_mpl_materials.py @@ -0,0 +1,108 @@ +"""Allows overriding the algorithm and signing_key for EncryptionMaterialsFromMPL. +This must ONLY be used in testing and NOT in production.. +This is used in message tampering testing. +""" +import attr +import six + +from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager + +# Ignore missing MPL for pylint, but the MPL is required for this class +# pylint: disable=import-error,no-name-in-module +from aws_encryption_sdk.materials_managers.mpl.materials import ( + EncryptionMaterialsFromMPL +) +from aws_encryption_sdk.materials_managers.mpl.cmm import ( + CryptoMaterialsManagerFromMPL +) + + +class HalfSigningEncryptionMaterialsFromMPL(EncryptionMaterialsFromMPL): + """Allows overriding the algorithm and signing_key for EncryptionMaterialsFromMPL. + This must ONLY be used in testing and NOT in production.. + This is used in testing malicious message modification (HalfSigningTampering). + """ + + _underlying_materials: EncryptionMaterialsFromMPL + + def __init__(self, underling_materials): + self._underlying_materials = underling_materials + + # pylint thinks EncryptionMaterialsFromMPL.algorithm is a method + # pylint: disable=invalid-overridden-method + @property + def algorithm(self): + """Return any previously-provided overriden algorithm; + if none was provided, returns underlying algorithm from encryption materials. + """ + if hasattr(self, "set_algorithm"): + return self.set_algorithm + return self._underlying_materials.algorithm + + @algorithm.setter + def algorithm(self, algorithm): + self.set_algorithm = algorithm + + # pylint thinks EncryptionMaterialsFromMPL.signing_key is a method + # pylint: disable=invalid-overridden-method + @property + def signing_key(self): + """Return any previously-provided overriden signing_key; + if none was provided, returns underlying signing_key from encryption materials. + """ + if hasattr(self, "set_signing_key"): + return self.set_signing_key + return self._underlying_materials.algorithm + + @signing_key.setter + def signing_key(self, signing_key): + self.set_signing_key = signing_key + + @property + def encryption_context(self): + return self._underlying_materials.encryption_context + + @property + def encrypted_data_keys(self): + return self._underlying_materials.encrypted_data_keys + + @property + def data_encryption_key(self): + return self._underlying_materials.data_encryption_key + + @property + def required_encryption_context_keys(self): + return self._underlying_materials.required_encryption_context_keys + + +class ProviderInfoChangingCryptoMaterialsManagerFromMPL(CryptoMaterialsManagerFromMPL): + """ + Custom CMM that modifies the provider info field on EDKS. + This extends CryptoMaterialsManagerFromMPL so ESDK-internal checks + follow MPL logic. + + THIS IS ONLY USED TO CREATE INVALID MESSAGES and should never be used in + production! + """ + + wrapped_cmm = attr.ib(validator=attr.validators.instance_of(CryptoMaterialsManager)) + new_provider_info = attr.ib(validator=attr.validators.instance_of(six.string_types)) + + def __init__(self, materials_manager, new_provider_info): + """Create a new CMM that wraps a the given CMM.""" + self.wrapped_cmm = materials_manager + self.new_provider_info = new_provider_info + + def get_encryption_materials(self, request): + """ + Request materials from the wrapped CMM, and then change the provider info + on each EDK. + """ + result = self.wrapped_cmm.get_encryption_materials(request) + for encrypted_data_key in result.encrypted_data_keys: + encrypted_data_key.key_provider.key_info = self.new_provider_info + return result + + def decrypt_materials(self, request): + """Thunks to the wrapped CMM""" + return self.wrapped_cmm.decrypt_materials(request) From 32446e2f2026751e1c1feb75a66dc785d91b7476 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 10:29:15 -0700 Subject: [PATCH 299/422] tampering mpl --- .../internal/tampering_mpl_materials.py | 64 ++++++++++++++++++- .../full_message/decrypt_generation.py | 31 ++++----- 2 files changed, 79 insertions(+), 16 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/internal/tampering_mpl_materials.py b/test_vector_handlers/src/awses_test_vectors/internal/tampering_mpl_materials.py index 4ad948424..1bb6705fc 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/tampering_mpl_materials.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/tampering_mpl_materials.py @@ -4,6 +4,8 @@ """ import attr import six +from copy import copy + from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager @@ -15,6 +17,66 @@ from aws_encryption_sdk.materials_managers.mpl.cmm import ( CryptoMaterialsManagerFromMPL ) +from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders +from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig +from aws_cryptographic_materialproviders.mpl.models import ( + CreateDefaultCryptographicMaterialsManagerInput, +) + +try: + from aws_encryption_sdk.identifiers import AlgorithmSuite +except ImportError: + from aws_encryption_sdk.identifiers import Algorithm as AlgorithmSuite + +class HalfSigningCryptoMaterialsManagerFromMPL(CryptoMaterialsManagerFromMPL): + """ + Custom CMM that modifies the provider info field on EDKs + This extends CryptoMaterialsManagerFromMPL so ESDK-internal checks + follow MPL logic. + + THIS IS ONLY USED TO CREATE INVALID MESSAGES and should never be used in + production! + """ + + wrapped_default_cmm = attr.ib(validator=attr.validators.instance_of(CryptoMaterialsManagerFromMPL)) + + def __init__(self, master_key_provider): + """Create a new CMM that wraps a the given CMM.""" + mpl = AwsCryptographicMaterialProviders(MaterialProvidersConfig()) + mpl_cmm = mpl.create_default_cryptographic_materials_manager( + CreateDefaultCryptographicMaterialsManagerInput( + keyring=master_key_provider + ) + ) + self.wrapped_default_cmm = CryptoMaterialsManagerFromMPL(mpl_cmm=mpl_cmm) + + def get_encryption_materials(self, request): + """ + Generate half-signing materials by requesting signing materials + from the wrapped default CMM, and then changing the algorithm suite + and removing the signing key from teh result. + """ + if request.algorithm == AlgorithmSuite.AES_256_GCM_HKDF_SHA512_COMMIT_KEY: + signing_request = copy(request) + signing_request.algorithm = AlgorithmSuite.AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384 + + result = HalfSigningEncryptionMaterialsFromMPL( + self.wrapped_default_cmm.get_encryption_materials(signing_request) + ) + + result.algorithm = request.algorithm + result.signing_key = None + + return result + + raise NotImplementedError( + "The half-sign tampering method is only supported on the " + "AES_256_GCM_HKDF_SHA512_COMMIT_KEY algorithm suite." + ) + + def decrypt_materials(self, request): + """Thunks to the wrapped default CMM""" + return self.wrapped_default_cmm.decrypt_materials(request) class HalfSigningEncryptionMaterialsFromMPL(EncryptionMaterialsFromMPL): @@ -77,7 +139,7 @@ def required_encryption_context_keys(self): class ProviderInfoChangingCryptoMaterialsManagerFromMPL(CryptoMaterialsManagerFromMPL): """ - Custom CMM that modifies the provider info field on EDKS. + Custom CMM that modifies the provider info field on EDKs. This extends CryptoMaterialsManagerFromMPL so ESDK-internal checks follow MPL logic. diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 782404704..e2fcc0f65 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -47,6 +47,7 @@ from awses_test_vectors.internal.tampering_mpl_materials import ( HalfSigningEncryptionMaterialsFromMPL, ProviderInfoChangingCryptoMaterialsManagerFromMPL, + HalfSigningCryptoMaterialsManagerFromMPL, ) _HAS_MPL = True @@ -319,9 +320,20 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, _p return: a list of (ciphertext, result) pairs. """ - tampering_materials_manager = HalfSigningCryptoMaterialsManager( - generation_scenario.encryption_scenario.master_key_provider_fn() - ) + if isinstance( + generation_scenario.encryption_scenario.master_key_provider_fn(), + MasterKeyProvider + ): + tampering_materials_manager = HalfSigningCryptoMaterialsManager( + generation_scenario.encryption_scenario.master_key_provider_fn() + ) + elif _HAS_MPL and isinstance( + generation_scenario.encryption_scenario.master_key_provider_fn(), + IKeyring + ): + tampering_materials_manager = HalfSigningCryptoMaterialsManagerFromMPL( + generation_scenario.encryption_scenario.master_key_provider_fn() + ) ciphertext_to_decrypt = generation_scenario.encryption_scenario.run(tampering_materials_manager) expected_result = MessageDecryptionTestResult.expect_error( "Unsigned message using a data key with a public key" @@ -349,18 +361,7 @@ def __init__(self, master_key_provider): Create a new CMM that wraps a new DefaultCryptoMaterialsManager based on the given master key provider. """ - if isinstance(master_key_provider, MasterKeyProvider): - self.wrapped_default_cmm = DefaultCryptoMaterialsManager(master_key_provider) - elif _HAS_MPL and isinstance(master_key_provider, IKeyring): - mpl = AwsCryptographicMaterialProviders(MaterialProvidersConfig()) - mpl_cmm = mpl.create_default_cryptographic_materials_manager( - CreateDefaultCryptographicMaterialsManagerInput( - keyring=master_key_provider - ) - ) - self.wrapped_default_cmm = CryptoMaterialsManagerFromMPL(mpl_cmm=mpl_cmm) - else: - raise TypeError(f"Unrecognized master_key_provider type: {master_key_provider}") + self.wrapped_default_cmm = DefaultCryptoMaterialsManager(master_key_provider) def get_encryption_materials(self, request): """ From f1cd456c4801bdc31300190736b836317f426169 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 10:53:49 -0700 Subject: [PATCH 300/422] more --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index f7636d33a..cc82757de 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -16,7 +16,7 @@ env: AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" git-credential-helper: yes secrets-manager: - GITHUB_TOKEN: Github/aws-crypto-tools-ci-bot:personal access token (new token format) + GITHUB_TOKEN: Github/lucasmcdonald3:actions:read fine-grained PAT phases: install: From a3267bc1d57df45a36d15a3b2a9a41f694dcc72b Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 10:56:43 -0700 Subject: [PATCH 301/422] more --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index cc82757de..62fa6b7e1 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -16,7 +16,7 @@ env: AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" git-credential-helper: yes secrets-manager: - GITHUB_TOKEN: Github/lucasmcdonald3:actions:read fine-grained PAT + GITHUB_TOKEN: Github/lucasmcdonald3:actions\:read fine-grained PAT phases: install: From d4db5ec6fcce95422cf0ac8a68fb44885e9d2d8d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 10:59:18 -0700 Subject: [PATCH 302/422] more --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 62fa6b7e1..939e42185 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -16,7 +16,7 @@ env: AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" git-credential-helper: yes secrets-manager: - GITHUB_TOKEN: Github/lucasmcdonald3:actions\:read fine-grained PAT + GITHUB_TOKEN: "Github/lucasmcdonald3:actions:read fine-grained PAT" phases: install: From bf78061292d3cf1594a22ef96c72011a561de17d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 11:10:06 -0700 Subject: [PATCH 303/422] more --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 939e42185..dc5732552 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -16,7 +16,7 @@ env: AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" git-credential-helper: yes secrets-manager: - GITHUB_TOKEN: "Github/lucasmcdonald3:actions:read fine-grained PAT" + GITHUB_TOKEN: Github/lucasmcdonald3-fgpat:actions read phases: install: From 008ae6ff5f8ebfa26c07192ad0a77ae6a5f01649 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 11:59:25 -0700 Subject: [PATCH 304/422] more --- src/aws_encryption_sdk/streaming_client.py | 1 + .../manifests/full_message/decrypt_generation.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index fb0935ff2..54ce76235 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -582,6 +582,7 @@ def _prep_message(self): else: # MPL verification key is PEM bytes, not DER bytes. # If the underlying CMM is from the MPL, load PEM bytes. + print(f"DEBUG: cmm is {self.config.materials_manager}") if (_HAS_MPL and isinstance(self.config.materials_manager, CryptoMaterialsManagerFromMPL)): self.signer = Signer.from_key_bytes( diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index e2fcc0f65..3fd40271a 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -173,6 +173,8 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, _p """ master_key_provider = generation_scenario.encryption_scenario.master_key_provider_fn() + print(f"DEBUG: mkp gen is {master_key_provider}") + # Use a caching CMM to avoid generating a new data key every time. if isinstance(master_key_provider, MasterKeyProvider): cache = LocalCryptoMaterialsCache(10) @@ -194,6 +196,8 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, _p else: raise TypeError(f"Unrecognized master_key_provider type: {master_key_provider}") + print(f"DEBUG: cmm gen is {cmm}") + return [ self.run_scenario_with_new_provider_info( ciphertext_writer, generation_scenario, cmm, new_provider_info @@ -204,6 +208,7 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, _p def run_scenario_with_new_provider_info( self, ciphertext_writer, generation_scenario, materials_manager, new_provider_info ): + print(f"DEBUG: materials_manager is {materials_manager}") """Run with tampering for a specific new provider info value""" if isinstance(materials_manager, CryptoMaterialsManager): tampering_materials_manager = ProviderInfoChangingCryptoMaterialsManager( From 19a9dad09b96ca773961dab3211861e09578bd27 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 12:49:01 -0700 Subject: [PATCH 305/422] debug --- codebuild/py37/generate_decrypt_vectors.yml | 1 - .../manifests/full_message/decrypt_generation.py | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml index 7aad81e13..7a1acca1b 100644 --- a/codebuild/py37/generate_decrypt_vectors.yml +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -31,4 +31,3 @@ artifacts: files: - test_vector_handlers/37_master.zip name: $CODEBUILD_BATCH_BUILD_IDENTIFIER/37_master.zip - discard-paths: yes diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 3fd40271a..a1fc8fa83 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -210,13 +210,13 @@ def run_scenario_with_new_provider_info( ): print(f"DEBUG: materials_manager is {materials_manager}") """Run with tampering for a specific new provider info value""" - if isinstance(materials_manager, CryptoMaterialsManager): - tampering_materials_manager = ProviderInfoChangingCryptoMaterialsManager( + if _HAS_MPL and isinstance(materials_manager, CryptoMaterialsManagerFromMPL): + tampering_materials_manager = ProviderInfoChangingCryptoMaterialsManagerFromMPL( materials_manager, new_provider_info ) - elif _HAS_MPL and isinstance(materials_manager, CryptoMaterialsManagerFromMPL): - tampering_materials_manager = ProviderInfoChangingCryptoMaterialsManagerFromMPL( + elif isinstance(materials_manager, CryptoMaterialsManager): + tampering_materials_manager = ProviderInfoChangingCryptoMaterialsManager( materials_manager, new_provider_info ) From a4aa0f9dd8e1b86de13d56cd9dda99b32a71bbf8 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 13:05:26 -0700 Subject: [PATCH 306/422] debug --- codebuild/py311/generate_decrypt_vectors_keyrings.yml | 2 +- codebuild/py311/generate_decrypt_vectors_masterkey.yml | 2 +- codebuild/py37/generate_decrypt_vectors.yml | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml index f83ac733e..e9c17c7b9 100644 --- a/codebuild/py311/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -31,5 +31,5 @@ phases: artifacts: files: - test_vector_handlers/311_keyring.zip - name: $CODEBUILD_BATCH_BUILD_IDENTIFIER/311_keyring.zip + name: $CODEBUILD_INITIATOR/311_keyring.zip discard-paths: yes \ No newline at end of file diff --git a/codebuild/py311/generate_decrypt_vectors_masterkey.yml b/codebuild/py311/generate_decrypt_vectors_masterkey.yml index 655fb1985..f4056832a 100644 --- a/codebuild/py311/generate_decrypt_vectors_masterkey.yml +++ b/codebuild/py311/generate_decrypt_vectors_masterkey.yml @@ -30,5 +30,5 @@ phases: artifacts: files: - test_vector_handlers/311_masterkey.zip - name: $CODEBUILD_BATCH_BUILD_IDENTIFIER/311_masterkey.zip + name: $CODEBUILD_INITIATOR/311_masterkey.zip discard-paths: yes \ No newline at end of file diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml index 7a1acca1b..784aaf44d 100644 --- a/codebuild/py37/generate_decrypt_vectors.yml +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -30,4 +30,5 @@ phases: artifacts: files: - test_vector_handlers/37_master.zip - name: $CODEBUILD_BATCH_BUILD_IDENTIFIER/37_master.zip + name: $CODEBUILD_INITIATOR/37_master.zip + discard-paths: yes From d53895de093854b94ff4bdf2c56152aab7b033c3 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 13:14:04 -0700 Subject: [PATCH 307/422] debug --- codebuild/py311/generate_decrypt_vectors_keyrings.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml index e9c17c7b9..1dbfc3b51 100644 --- a/codebuild/py311/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -31,5 +31,4 @@ phases: artifacts: files: - test_vector_handlers/311_keyring.zip - name: $CODEBUILD_INITIATOR/311_keyring.zip - discard-paths: yes \ No newline at end of file + name: $CODEBUILD_INITIATOR/311_keyring.zip \ No newline at end of file From 9c46200e47c1c4c3831ebce1d66868accd043190 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 13:14:30 -0700 Subject: [PATCH 308/422] debug --- codebuild/py311/generate_decrypt_vectors_keyrings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml index 1dbfc3b51..bbc09c89e 100644 --- a/codebuild/py311/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -31,4 +31,4 @@ phases: artifacts: files: - test_vector_handlers/311_keyring.zip - name: $CODEBUILD_INITIATOR/311_keyring.zip \ No newline at end of file + name: $CODEBUILD_INITIATOR \ No newline at end of file From 3af9f3266f5f3c9aa6304c8f92d79d64bfa99714 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 13:22:13 -0700 Subject: [PATCH 309/422] debug --- codebuild/py311/generate_decrypt_vectors_keyrings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml index bbc09c89e..4a178663a 100644 --- a/codebuild/py311/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -31,4 +31,4 @@ phases: artifacts: files: - test_vector_handlers/311_keyring.zip - name: $CODEBUILD_INITIATOR \ No newline at end of file + name: $CODEBUILD_RESOLVED_SOURCE_VERSION \ No newline at end of file From 9e255e487e6e86dbb67453293b8765f0b31ca9f3 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 13:28:54 -0700 Subject: [PATCH 310/422] debug --- codebuild/py311/generate_decrypt_vectors_keyrings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml index 4a178663a..c88001643 100644 --- a/codebuild/py311/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -31,4 +31,4 @@ phases: artifacts: files: - test_vector_handlers/311_keyring.zip - name: $CODEBUILD_RESOLVED_SOURCE_VERSION \ No newline at end of file + name: builds/$CODEBUILD_RESOLVED_SOURCE_VERSION/my-artifacts \ No newline at end of file From 262696b4373980cb727965f403c842431d11c9cf Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 13:36:10 -0700 Subject: [PATCH 311/422] debug --- codebuild/py311/generate_decrypt_vectors_keyrings.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml index c88001643..805179833 100644 --- a/codebuild/py311/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -22,13 +22,13 @@ phases: commands: - pip install "tox < 4.0" - cd test_vector_handlers + - mkdir $CODEBUILD_RESOLVED_SOURCE_VERSION - | tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ - --output 311_keyring \ + --output $CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring \ --keyrings - - zip -r 311_keyring.zip 311_keyring + - zip -r 311_keyring.zip $CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring artifacts: files: - - test_vector_handlers/311_keyring.zip - name: builds/$CODEBUILD_RESOLVED_SOURCE_VERSION/my-artifacts \ No newline at end of file + - test_vector_handlers/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip From fadea8c86ccf7851254b8112650c04c3f6d825df Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 13:42:40 -0700 Subject: [PATCH 312/422] debug --- codebuild/py311/generate_decrypt_vectors_keyrings.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml index 805179833..5f3c52240 100644 --- a/codebuild/py311/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -21,14 +21,14 @@ phases: build: commands: - pip install "tox < 4.0" - - cd test_vector_handlers - mkdir $CODEBUILD_RESOLVED_SOURCE_VERSION + - cd test_vector_handlers - | tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ - --output $CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring \ + --output ../$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring \ --keyrings - - zip -r 311_keyring.zip $CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring + - zip -r 311_keyring.zip ../$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring artifacts: files: - - test_vector_handlers/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip + - $CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip From b8dbb1f68ac1d57985be154543a7b30a8c6f4ae2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 13:45:54 -0700 Subject: [PATCH 313/422] debug --- codebuild/py311/generate_decrypt_vectors_keyrings.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml index 5f3c52240..db996a2fb 100644 --- a/codebuild/py311/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -21,14 +21,14 @@ phases: build: commands: - pip install "tox < 4.0" - - mkdir $CODEBUILD_RESOLVED_SOURCE_VERSION - cd test_vector_handlers + - mkdir $CODEBUILD_RESOLVED_SOURCE_VERSION - | tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ - --output ../$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring \ + --output 311_keyring \ --keyrings - - zip -r 311_keyring.zip ../$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring + - zip -r $CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip 311_keyring artifacts: files: - - $CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip + - test_vector_handlers/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip From f5d6cb7f4f3dcbd59e009f37f5d12030214ecd41 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 13:47:35 -0700 Subject: [PATCH 314/422] debug --- codebuild/py311/generate_decrypt_vectors_keyrings.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml index db996a2fb..9b4e71257 100644 --- a/codebuild/py311/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -22,13 +22,13 @@ phases: commands: - pip install "tox < 4.0" - cd test_vector_handlers - - mkdir $CODEBUILD_RESOLVED_SOURCE_VERSION + - mkdir $CODEBUILD_INITIATOR - | tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ --output 311_keyring \ --keyrings - - zip -r $CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip 311_keyring + - zip -r $CODEBUILD_INITIATOR/311_keyring.zip 311_keyring artifacts: files: - - test_vector_handlers/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip + - test_vector_handlers/$CODEBUILD_INITIATOR/311_keyring.zip From 774abf601694243eb1ede8c43408b946a06cc55f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 14:05:42 -0700 Subject: [PATCH 315/422] debug --- test_vector_handlers/test/mpl/__init__.py | 0 .../test/mpl/integration/__init__.py | 0 .../test/mpl/integration/commands/__init__.py | 0 .../commands/test_i_encrypt_keyrings.py | 64 +++++++++++++++++++ .../commands/test_i_esdk_dafny_keyrings.py | 0 .../commands/test_i_net_401_keyrings.py | 0 6 files changed, 64 insertions(+) create mode 100644 test_vector_handlers/test/mpl/__init__.py create mode 100644 test_vector_handlers/test/mpl/integration/__init__.py create mode 100644 test_vector_handlers/test/mpl/integration/commands/__init__.py create mode 100644 test_vector_handlers/test/mpl/integration/commands/test_i_encrypt_keyrings.py create mode 100644 test_vector_handlers/test/mpl/integration/commands/test_i_esdk_dafny_keyrings.py create mode 100644 test_vector_handlers/test/mpl/integration/commands/test_i_net_401_keyrings.py diff --git a/test_vector_handlers/test/mpl/__init__.py b/test_vector_handlers/test/mpl/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test_vector_handlers/test/mpl/integration/__init__.py b/test_vector_handlers/test/mpl/integration/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test_vector_handlers/test/mpl/integration/commands/__init__.py b/test_vector_handlers/test/mpl/integration/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test_vector_handlers/test/mpl/integration/commands/test_i_encrypt_keyrings.py b/test_vector_handlers/test/mpl/integration/commands/test_i_encrypt_keyrings.py new file mode 100644 index 000000000..077a36d63 --- /dev/null +++ b/test_vector_handlers/test/mpl/integration/commands/test_i_encrypt_keyrings.py @@ -0,0 +1,64 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +""" +Integration tests for `awses_test_vectors.commands` with keyrings. +""" +import pytest + +from awses_test_vectors.commands import full_message_decrypt, full_message_decrypt_generate, full_message_encrypt + +from ....integration.integration_test_utils import ( # noqa pylint: disable=unused-import + full_message_decrypt_generation_vectors, + full_message_encrypt_vectors, +) + + +pytestmark = [pytest.mark.integ] + + +def test_full_message_encrypt_canonical_full(full_message_encrypt_vectors): + full_message_encrypt.cli(["--input", full_message_encrypt_vectors, "--keyrings"]) + + +def test_full_message_cycle_canonical_full(tmpdir, full_message_decrypt_generation_vectors): + # Generate vectors using keyring interfaces + keyring_output_dir = tmpdir.join("output-keyrings") + full_message_decrypt_generate.cli([ + "--output", + str(keyring_output_dir), + "--input", + full_message_decrypt_generation_vectors, + "--keyrings" + ]) + + # Generate vectors using master key interfaces + master_key_output_dir = tmpdir.join("output-master-key") + full_message_decrypt_generate.cli([ + "--output", + str(master_key_output_dir), + "--input", + full_message_decrypt_generation_vectors + ]) + + # Validate that vectors generated using keyring interfaces + # can be decrypted by BOTH keyring and master key interfaces + keyring_decrypt_manifest_file = keyring_output_dir.join("manifest.json") + full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file), "--keyrings"]) + full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file)]) + + # Validate that vectors generated using master key interfaces + # can be decrypted by BOTH keyring and master key interfaces + master_key_decrypt_manifest_file = keyring_output_dir.join("manifest.json") + + full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file), "--keyrings"]) + full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file)]) diff --git a/test_vector_handlers/test/mpl/integration/commands/test_i_esdk_dafny_keyrings.py b/test_vector_handlers/test/mpl/integration/commands/test_i_esdk_dafny_keyrings.py new file mode 100644 index 000000000..e69de29bb diff --git a/test_vector_handlers/test/mpl/integration/commands/test_i_net_401_keyrings.py b/test_vector_handlers/test/mpl/integration/commands/test_i_net_401_keyrings.py new file mode 100644 index 000000000..e69de29bb From e16771ae0a211213d1db5ec3ef140dd16249500e Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 14:06:22 -0700 Subject: [PATCH 316/422] debug --- codebuild/py311/generate_decrypt_vectors_keyrings.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml index 9b4e71257..db996a2fb 100644 --- a/codebuild/py311/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -22,13 +22,13 @@ phases: commands: - pip install "tox < 4.0" - cd test_vector_handlers - - mkdir $CODEBUILD_INITIATOR + - mkdir $CODEBUILD_RESOLVED_SOURCE_VERSION - | tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ --output 311_keyring \ --keyrings - - zip -r $CODEBUILD_INITIATOR/311_keyring.zip 311_keyring + - zip -r $CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip 311_keyring artifacts: files: - - test_vector_handlers/$CODEBUILD_INITIATOR/311_keyring.zip + - test_vector_handlers/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip From 8752df72b0960952b10fffc7def6ab3445f8141d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 14:53:39 -0700 Subject: [PATCH 317/422] Debug --- buildspec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildspec.yml b/buildspec.yml index db168cc78..cc082e284 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -327,7 +327,7 @@ batch: # buildspec: codebuild/py312/decrypt_keyrings_with_js.yml # env: # image: aws/codebuild/standard:7.0 - + # # - identifier: code_coverage # buildspec: codebuild/coverage/coverage.yml # - identifier: code_coverage_mpl From 91d219c608ede8ee53c38609aa8d749f0a6ccb0e Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 15:02:40 -0700 Subject: [PATCH 318/422] Debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index dc5732552..a0e84a369 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -46,11 +46,9 @@ phases: # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct...) - | - MOST_RECENT_RUN_STUFF=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: token ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true") + MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: token ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') - | - echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_STUFF" - - | - MOST_RECENT_RUN_ID=$(echo $MOST_RECENT_RUN_STUFF | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') + echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" - | MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: token $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url') - | From 0178f7a9389292c16ef413fe9493d5dd246f7a77 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 15:12:45 -0700 Subject: [PATCH 319/422] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index a0e84a369..dce57d725 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -54,7 +54,7 @@ phases: - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" - | - curl -L -H "Accept: application/vnd.github+json" -H "Authorization: token $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" -o ubuntu-latest_test_vector_artifact.zip $MOST_RECENT_RUN_DOWNLOAD_URL + curl -L -H "Accept: application/vnd.github+json" -H "Authorization: token $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" $(echo MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip - unzip ubuntu-latest_test_vector_artifact build: commands: From 1060a3f42e01d14d3f818884c71be6123669835b Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 15:13:03 -0700 Subject: [PATCH 320/422] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index dce57d725..d3fb627af 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -54,7 +54,7 @@ phases: - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" - | - curl -L -H "Accept: application/vnd.github+json" -H "Authorization: token $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" $(echo MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip + curl -L -H "Accept: application/vnd.github+json" -H "Authorization: token $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip - unzip ubuntu-latest_test_vector_artifact build: commands: From 9806fba85cecba03789bda0e827893d1a1bd1ddd Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 15:14:41 -0700 Subject: [PATCH 321/422] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index d3fb627af..d3456636d 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -32,29 +32,29 @@ phases: - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') - aws sts get-caller-identity - # Authenticate into the CI bot to allow session to download ESDK Dafny GHA artifact. - - git config --global user.name "aws-crypto-tools-ci-bot" - - git config --global user.email "no-reply@noemail.local" - - # - | - # sudo mkdir -p -m 755 /etc/apt/keyrings && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ - # && sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \ - # && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ - # && sudo apt update \ - # && sudo apt install gh -y - # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct...) - | - MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: token ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') + MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ + | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') - | echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" - | - MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: token $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" | jq '.artifacts[0].archive_download_url') + MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" \ + | jq '.artifacts[0].archive_download_url') - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" - | - curl -L -H "Accept: application/vnd.github+json" -H "Authorization: token $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip + curl -L -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip - unzip ubuntu-latest_test_vector_artifact build: commands: From 3625f98364949445d7f082f0dab7c21ca82a7996 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 15:20:18 -0700 Subject: [PATCH 322/422] debug --- .../py311/generate_decrypt_vectors_keyrings.yml | 12 +++++++----- codebuild/py37/generate_decrypt_vectors.yml | 13 +++++++------ 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml index db996a2fb..3e28dc113 100644 --- a/codebuild/py311/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -22,13 +22,15 @@ phases: commands: - pip install "tox < 4.0" - cd test_vector_handlers - - mkdir $CODEBUILD_RESOLVED_SOURCE_VERSION + # - mkdir $CODEBUILD_RESOLVED_SOURCE_VERSION - | tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ --output 311_keyring \ --keyrings - - zip -r $CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip 311_keyring -artifacts: - files: - - test_vector_handlers/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip + # - zip -r $CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip 311_keyring + - zip -r 311_keyring.zip 311_keyring + - aws s3 cp 311_keyring.zip s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip +# artifacts: +# files: +# - test_vector_handlers/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors.yml index 784aaf44d..f5f12490e 100644 --- a/codebuild/py37/generate_decrypt_vectors.yml +++ b/codebuild/py37/generate_decrypt_vectors.yml @@ -26,9 +26,10 @@ phases: tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ --output 37_masterkey - - zip -r 37_master.zip 37_masterkey -artifacts: - files: - - test_vector_handlers/37_master.zip - name: $CODEBUILD_INITIATOR/37_master.zip - discard-paths: yes + - zip -r 37_masterkey.zip 37_masterkey + - aws s3 cp 37_masterkey.zip s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/37_masterkey.zip +# artifacts: +# files: +# - test_vector_handlers/37_master.zip +# name: $CODEBUILD_INITIATOR/37_master.zip +# discard-paths: yes From 77b9165380cfad5fe8b18809fe72b7ca7accfa1c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 15:22:22 -0700 Subject: [PATCH 323/422] Debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index d3456636d..aef28d67a 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -56,11 +56,12 @@ phases: -H "X-GitHub-Api-Version: 2022-11-28" \ $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip - unzip ubuntu-latest_test_vector_artifact + # This unzips to `net41/` build: commands: # NOTE: We need to pass the absolute path of the vectors - pip install "tox < 4.0" - - cd $CODEBUILD_SRC_DIR/test_vector_handlers + - cd /test_vector_handlers - | tox -- \ - --input $UNZIPPED_VECTORS_DIR/manifest.json + --input ../net41/manifest.json From cfc2681bef96d50a0928767d40b0185b6be9517c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 15:25:27 -0700 Subject: [PATCH 324/422] debug --- .../commands/test_i_encrypt_keyrings.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/test_vector_handlers/test/mpl/integration/commands/test_i_encrypt_keyrings.py b/test_vector_handlers/test/mpl/integration/commands/test_i_encrypt_keyrings.py index 077a36d63..85c94dd22 100644 --- a/test_vector_handlers/test/mpl/integration/commands/test_i_encrypt_keyrings.py +++ b/test_vector_handlers/test/mpl/integration/commands/test_i_encrypt_keyrings.py @@ -41,24 +41,5 @@ def test_full_message_cycle_canonical_full(tmpdir, full_message_decrypt_generati "--keyrings" ]) - # Generate vectors using master key interfaces - master_key_output_dir = tmpdir.join("output-master-key") - full_message_decrypt_generate.cli([ - "--output", - str(master_key_output_dir), - "--input", - full_message_decrypt_generation_vectors - ]) - - # Validate that vectors generated using keyring interfaces - # can be decrypted by BOTH keyring and master key interfaces keyring_decrypt_manifest_file = keyring_output_dir.join("manifest.json") full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file), "--keyrings"]) - full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file)]) - - # Validate that vectors generated using master key interfaces - # can be decrypted by BOTH keyring and master key interfaces - master_key_decrypt_manifest_file = keyring_output_dir.join("manifest.json") - - full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file), "--keyrings"]) - full_message_decrypt.cli(["--input", str(master_key_decrypt_manifest_file)]) From 87bc057d17a6ea9b4e71276d6822bb9f44368c92 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 15:26:09 -0700 Subject: [PATCH 325/422] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index aef28d67a..f7213ac56 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -61,7 +61,7 @@ phases: commands: # NOTE: We need to pass the absolute path of the vectors - pip install "tox < 4.0" - - cd /test_vector_handlers + - cd test_vector_handlers - | tox -- \ --input ../net41/manifest.json From ca6f3a1b61d39af1f1d72f3983af6087680a712d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 15:29:52 -0700 Subject: [PATCH 326/422] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index f7213ac56..7a467a067 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -46,7 +46,7 @@ phases: MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" \ -H "Authorization: token $GITHUB_TOKEN" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/8282993634/artifacts?name=ubuntu-latest_vector_artifact" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/$MOST_RECENT_RUN_ID/artifacts?name=ubuntu-latest_vector_artifact" \ | jq '.artifacts[0].archive_download_url') - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" @@ -55,13 +55,16 @@ phases: -H "Authorization: token $GITHUB_TOKEN" \ -H "X-GitHub-Api-Version: 2022-11-28" \ $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip - - unzip ubuntu-latest_test_vector_artifact # This unzips to `net41/` + - unzip ubuntu-latest_test_vector_artifact + - ls build: commands: # NOTE: We need to pass the absolute path of the vectors - pip install "tox < 4.0" + - ls - cd test_vector_handlers + - ls - | tox -- \ --input ../net41/manifest.json From 3b01d387563790ad6e6b3fea627c7697814376e6 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 15:33:11 -0700 Subject: [PATCH 327/422] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 7a467a067..6c3b532ec 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -55,8 +55,10 @@ phases: -H "Authorization: token $GITHUB_TOKEN" \ -H "X-GitHub-Api-Version: 2022-11-28" \ $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip - # This unzips to `net41/` + # This unzips to `net41.zip` - unzip ubuntu-latest_test_vector_artifact + # This unzips to `net41/` + - unzip net41.zip - ls build: commands: From 68495e83e48192b9ef22b1ffe3b160ba350a2225 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 15:37:26 -0700 Subject: [PATCH 328/422] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 6c3b532ec..5e8fbcb1e 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -56,9 +56,9 @@ phases: -H "X-GitHub-Api-Version: 2022-11-28" \ $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip # This unzips to `net41.zip` - - unzip ubuntu-latest_test_vector_artifact + - unzip ubuntu-latest_test_vector_artifact.zip # This unzips to `net41/` - - unzip net41.zip + # - unzip net41.zip - ls build: commands: From 8a8103a94bd00758c860efd2eadc454dde7d19aa Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 15:41:25 -0700 Subject: [PATCH 329/422] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 5e8fbcb1e..3725ed526 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -56,9 +56,9 @@ phases: -H "X-GitHub-Api-Version: 2022-11-28" \ $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip # This unzips to `net41.zip` - - unzip ubuntu-latest_test_vector_artifact.zip + - unzip ubuntu-latest_test_vector_artifact # This unzips to `net41/` - # - unzip net41.zip + - unzip net41.zip -d net41 - ls build: commands: From 89c39ffea3758797d5fca8bcf36fd78bbfbceb79 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 15:50:45 -0700 Subject: [PATCH 330/422] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index 3725ed526..e60084f94 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -59,14 +59,11 @@ phases: - unzip ubuntu-latest_test_vector_artifact # This unzips to `net41/` - unzip net41.zip -d net41 - - ls build: commands: # NOTE: We need to pass the absolute path of the vectors - pip install "tox < 4.0" - - ls - cd test_vector_handlers - - ls - | tox -- \ --input ../net41/manifest.json From fee0ccece2150a87e259d4a69cd05feaee8b68df Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 15:57:54 -0700 Subject: [PATCH 331/422] debug --- codebuild/py37/decrypt_masterkey_with_js.yml | 4 ++-- codebuild/py37/decrypt_masterkey_with_masterkey.yml | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/codebuild/py37/decrypt_masterkey_with_js.yml b/codebuild/py37/decrypt_masterkey_with_js.yml index 22ca730b7..6fefb12fd 100644 --- a/codebuild/py37/decrypt_masterkey_with_js.yml +++ b/codebuild/py37/decrypt_masterkey_with_js.yml @@ -36,8 +36,8 @@ phases: # Download generated vectors # TODO rewrite URL - aws s3 cp s3://generated-vectors-artifacts-bucket/GeneratedVectors/py37_generate_decrypt_vectors/test_vector_handlers/37_master.zip 37_master.zip + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/37_masterkey.zip 37_masterkey.zip build: commands: # Decrypt generated vectors with Javascript ESDK - - integration-node decrypt -v 37_master.zip \ No newline at end of file + - integration-node decrypt -v 37_masterkey.zip \ No newline at end of file diff --git a/codebuild/py37/decrypt_masterkey_with_masterkey.yml b/codebuild/py37/decrypt_masterkey_with_masterkey.yml index 5a2347e8f..d75c26a27 100644 --- a/codebuild/py37/decrypt_masterkey_with_masterkey.yml +++ b/codebuild/py37/decrypt_masterkey_with_masterkey.yml @@ -18,14 +18,12 @@ phases: install: runtime-versions: python: 3.7 - - pre-build: commands: # Download generated vectors # TODO rewrite URL - aws s3 cp s3://generated-vectors-artifacts-bucket/py37_generate_decrypt_vectors/test_vector_handlers/37_master.zip 37_masterkey.zip - unzip 37_master.zip + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/37_masterkey.zip 37_masterkey.zip + - unzip 37_masterkey.zip build: commands: - pip install "tox < 4.0" From c5ba2fd2b15f959a9e61ec1027a98084ed323594 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 16:00:40 -0700 Subject: [PATCH 332/422] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index e60084f94..bca8a3147 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -12,8 +12,6 @@ env: arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" git-credential-helper: yes secrets-manager: GITHUB_TOKEN: Github/lucasmcdonald3-fgpat:actions read @@ -24,14 +22,6 @@ phases: python: 3.7 pre_build: commands: - # Assume Role to access non-prod resources - - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") - - export TMP_ROLE - - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') - - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') - - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') - - aws sts get-caller-identity - # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct...) - | From 4875dbcbdc81cab69a0be83fb6b86c8ff3f8e380 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 16:09:53 -0700 Subject: [PATCH 333/422] debug --- codebuild/py37/decrypt_dafny_esdk_vectors.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/codebuild/py37/decrypt_dafny_esdk_vectors.yml b/codebuild/py37/decrypt_dafny_esdk_vectors.yml index bca8a3147..ee2d428c7 100644 --- a/codebuild/py37/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py37/decrypt_dafny_esdk_vectors.yml @@ -45,13 +45,12 @@ phases: -H "Authorization: token $GITHUB_TOKEN" \ -H "X-GitHub-Api-Version: 2022-11-28" \ $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip - # This unzips to `net41.zip` + # This unzips to `net41.zip`. - unzip ubuntu-latest_test_vector_artifact - # This unzips to `net41/` + # This unzips to `net41/`. - unzip net41.zip -d net41 build: commands: - # NOTE: We need to pass the absolute path of the vectors - pip install "tox < 4.0" - cd test_vector_handlers - | From 4ed99d371e449f97d167d1cdffe8a3ba72f2ba98 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 17:26:29 -0700 Subject: [PATCH 334/422] debug --- codebuild/py37/decrypt_masterkey_with_js.yml | 2 +- codebuild/py37/decrypt_masterkey_with_masterkey.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/codebuild/py37/decrypt_masterkey_with_js.yml b/codebuild/py37/decrypt_masterkey_with_js.yml index 6fefb12fd..3e5efe8b1 100644 --- a/codebuild/py37/decrypt_masterkey_with_js.yml +++ b/codebuild/py37/decrypt_masterkey_with_js.yml @@ -26,7 +26,7 @@ phases: pre_build: commands: # Assume Role to access non-prod resources - - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Java-Role-us-west-2" --role-session-name "CB-TestVectorResources") + - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") - export TMP_ROLE - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') diff --git a/codebuild/py37/decrypt_masterkey_with_masterkey.yml b/codebuild/py37/decrypt_masterkey_with_masterkey.yml index d75c26a27..c6117b9ba 100644 --- a/codebuild/py37/decrypt_masterkey_with_masterkey.yml +++ b/codebuild/py37/decrypt_masterkey_with_masterkey.yml @@ -18,7 +18,7 @@ phases: install: runtime-versions: python: 3.7 - pre-build: + pre_build: commands: # Download generated vectors # TODO rewrite URL From 9f76cbf0f12d66178bcd64eb0659bd5651fc89f3 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 17:42:25 -0700 Subject: [PATCH 335/422] debug --- buildspec.yml | 176 +++++++++++++++++++++++++------------------------- 1 file changed, 88 insertions(+), 88 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index cc082e284..1ff23ab8b 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -150,94 +150,94 @@ batch: # env: # image: aws/codebuild/standard:5.0 - - identifier: py311_integ - buildspec: codebuild/py311/integ.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_integ_mpl - buildspec: codebuild/py311/integ_mpl.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_examples - buildspec: codebuild/py311/examples.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_examples_mpl - buildspec: codebuild/py311/examples_mpl.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_awses_latest - buildspec: codebuild/py311/awses_local.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_awses_latest_mpl - buildspec: codebuild/py311/awses_local_mpl.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_mplawses_latest_mpl - buildspec: codebuild/py311/mplawses_local_mpl.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_dafny_esdk_vectors_masterkey - buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_dafny_esdk_vectors_keyrings - buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_net_401_vectors_masterkey - buildspec: codebuild/py311/decrypt_net_401_vectors_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_net_401_vectors_keyrings - buildspec: codebuild/py311/decrypt_net_401_vectors_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_generate_decrypt_vectors_masterkey - buildspec: codebuild/py311/generate_decrypt_vectors_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_masterkey_with_masterkey - depend-on: - - py311_generate_decrypt_vectors_masterkey - buildspec: codebuild/py311/decrypt_masterkey_with_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_masterkey_with_keyrings - depend-on: - - py311_generate_decrypt_vectors_masterkey - buildspec: codebuild/py311/decrypt_masterkey_with_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_masterkey_with_js - depend-on: - - py311_generate_decrypt_vectors_masterkey - buildspec: codebuild/py311/decrypt_masterkey_with_js.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_generate_decrypt_vectors_keyrings - buildspec: codebuild/py311/generate_decrypt_vectors_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_keyrings_with_masterkey - depend-on: - - py311_generate_decrypt_vectors_keyrings - buildspec: codebuild/py311/decrypt_keyrings_with_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_keyrings_with_keyrings - depend-on: - - py311_generate_decrypt_vectors_keyrings - buildspec: codebuild/py311/decrypt_keyrings_with_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_keyrings_with_js - depend-on: - - py311_generate_decrypt_vectors_keyrings - buildspec: codebuild/py311/decrypt_keyrings_with_js.yml - env: - image: aws/codebuild/standard:7.0 + # - identifier: py311_integ + # buildspec: codebuild/py311/integ.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_integ_mpl + # buildspec: codebuild/py311/integ_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_examples + # buildspec: codebuild/py311/examples.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_examples_mpl + # buildspec: codebuild/py311/examples_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_awses_latest + # buildspec: codebuild/py311/awses_local.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_awses_latest_mpl + # buildspec: codebuild/py311/awses_local_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_mplawses_latest_mpl + # buildspec: codebuild/py311/mplawses_local_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_dafny_esdk_vectors_masterkey + # buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_dafny_esdk_vectors_keyrings + # buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_net_401_vectors_masterkey + # buildspec: codebuild/py311/decrypt_net_401_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_net_401_vectors_keyrings + # buildspec: codebuild/py311/decrypt_net_401_vectors_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py311/generate_decrypt_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_masterkey_with_masterkey + # depend-on: + # - py311_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py311/decrypt_masterkey_with_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_masterkey_with_keyrings + # depend-on: + # - py311_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py311/decrypt_masterkey_with_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_masterkey_with_js + # depend-on: + # - py311_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py311/decrypt_masterkey_with_js.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py311/generate_decrypt_vectors_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_keyrings_with_masterkey + # depend-on: + # - py311_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py311/decrypt_keyrings_with_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_keyrings_with_keyrings + # depend-on: + # - py311_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py311/decrypt_keyrings_with_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_keyrings_with_js + # depend-on: + # - py311_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py311/decrypt_keyrings_with_js.yml + # env: + # image: aws/codebuild/standard:7.0 # - identifier: py312_integ # buildspec: codebuild/py312/integ.yml From 2f08f0d3a67d1ccb49691cd497cebfc04afadcf9 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 19 Mar 2024 17:46:30 -0700 Subject: [PATCH 336/422] debug --- buildspec.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index 1ff23ab8b..6efc031c6 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -11,14 +11,14 @@ batch: buildspec: codebuild/py37/examples.yml env: image: aws/codebuild/standard:5.0 - - identifier: py37_awses_local - buildspec: codebuild/py37/awses_local.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py37_decrypt_dafny_esdk_vectors - buildspec: codebuild/py37/decrypt_dafny_esdk_vectors.yml - env: - image: aws/codebuild/standard:5.0 + # - identifier: py37_awses_local + # buildspec: codebuild/py37/awses_local.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py37_decrypt_dafny_esdk_vectors + # buildspec: codebuild/py37/decrypt_dafny_esdk_vectors.yml + # env: + # image: aws/codebuild/standard:5.0 - identifier: py37_decrypt_net_401_vectors buildspec: codebuild/py37/decrypt_net_401_vectors.yml env: From 594f2732b947bf79d111276dc19c34407d0f379b Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 09:11:11 -0700 Subject: [PATCH 337/422] debug --- codebuild/py37/decrypt_masterkey_with_js.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/codebuild/py37/decrypt_masterkey_with_js.yml b/codebuild/py37/decrypt_masterkey_with_js.yml index 3e5efe8b1..15dcb35ac 100644 --- a/codebuild/py37/decrypt_masterkey_with_js.yml +++ b/codebuild/py37/decrypt_masterkey_with_js.yml @@ -26,13 +26,13 @@ phases: pre_build: commands: # Assume Role to access non-prod resources - - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") - - export TMP_ROLE - - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') - - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') - - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') - - aws sts get-caller-identity - - cd $CODEBUILD_SRC_DIR + # - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") + # - export TMP_ROLE + # - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') + # - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') + # - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') + # - aws sts get-caller-identity + # - cd $CODEBUILD_SRC_DIR # Download generated vectors # TODO rewrite URL From 108cd03462be6d99b49502b50da33bee59c78b8a Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 09:26:11 -0700 Subject: [PATCH 338/422] refactor --- buildspec.yml | 20 +++++----- codebuild/py37/encrypt_masterkey.yml | 33 +++++++++++++++++ ...=> generate_decrypt_vectors_masterkey.yml} | 0 .../commands/test_i_full_message_encrypt.py | 37 ------------------- .../commands/test_i_esdk_dafny_keyrings.py | 0 .../commands/test_i_net_401_keyrings.py | 0 test_vector_handlers/tox.ini | 1 + 7 files changed, 44 insertions(+), 47 deletions(-) create mode 100644 codebuild/py37/encrypt_masterkey.yml rename codebuild/py37/{generate_decrypt_vectors.yml => generate_decrypt_vectors_masterkey.yml} (100%) delete mode 100644 test_vector_handlers/test/integration/commands/test_i_full_message_encrypt.py delete mode 100644 test_vector_handlers/test/mpl/integration/commands/test_i_esdk_dafny_keyrings.py delete mode 100644 test_vector_handlers/test/mpl/integration/commands/test_i_net_401_keyrings.py diff --git a/buildspec.yml b/buildspec.yml index 6efc031c6..2eb44ccff 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -11,20 +11,20 @@ batch: buildspec: codebuild/py37/examples.yml env: image: aws/codebuild/standard:5.0 - # - identifier: py37_awses_local - # buildspec: codebuild/py37/awses_local.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py37_decrypt_dafny_esdk_vectors - # buildspec: codebuild/py37/decrypt_dafny_esdk_vectors.yml - # env: - # image: aws/codebuild/standard:5.0 + - identifier: py37_decrypt_dafny_esdk_vectors + buildspec: codebuild/py37/decrypt_dafny_esdk_vectors.yml + env: + image: aws/codebuild/standard:5.0 - identifier: py37_decrypt_net_401_vectors buildspec: codebuild/py37/decrypt_net_401_vectors.yml env: image: aws/codebuild/standard:5.0 - - identifier: py37_generate_decrypt_vectors - buildspec: codebuild/py37/generate_decrypt_vectors.yml + - identifier: py37_encrypt_masterkey + buildspec: codebuild/py37/encrypt_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py37_generate_decrypt_vectors_masterkey + buildspec: codebuild/py37/generate_decrypt_vectors_masterkey.yml env: image: aws/codebuild/standard:5.0 - identifier: py37_decrypt_masterkey_with_masterkey diff --git a/codebuild/py37/encrypt_masterkey.yml b/codebuild/py37/encrypt_masterkey.yml new file mode 100644 index 000000000..c6117b9ba --- /dev/null +++ b/codebuild/py37/encrypt_masterkey.yml @@ -0,0 +1,33 @@ +version: 0.2 + +env: + variables: + TOXENV: "py37-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + python: 3.7 + pre_build: + commands: + # Download generated vectors + # TODO rewrite URL + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/37_masterkey.zip 37_masterkey.zip + - unzip 37_masterkey.zip + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../37_masterkey/manifest.json \ No newline at end of file diff --git a/codebuild/py37/generate_decrypt_vectors.yml b/codebuild/py37/generate_decrypt_vectors_masterkey.yml similarity index 100% rename from codebuild/py37/generate_decrypt_vectors.yml rename to codebuild/py37/generate_decrypt_vectors_masterkey.yml diff --git a/test_vector_handlers/test/integration/commands/test_i_full_message_encrypt.py b/test_vector_handlers/test/integration/commands/test_i_full_message_encrypt.py deleted file mode 100644 index 6305a15da..000000000 --- a/test_vector_handlers/test/integration/commands/test_i_full_message_encrypt.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. -""" -Integration tests for ``awses_test_vectors.commands``. -""" -import pytest - -from awses_test_vectors.commands import full_message_decrypt, full_message_decrypt_generate, full_message_encrypt - -from ..integration_test_utils import ( # noqa pylint: disable=unused-import - full_message_decrypt_generation_vectors, - full_message_encrypt_vectors, -) - -pytestmark = [pytest.mark.integ] - - -def test_full_message_encrypt_canonical_full(full_message_encrypt_vectors): - full_message_encrypt.cli(["--input", full_message_encrypt_vectors]) - - -def test_full_message_cycle_canonical_full(tmpdir, full_message_decrypt_generation_vectors): - output_dir = tmpdir.join("output") - full_message_decrypt_generate.cli(["--output", str(output_dir), "--input", full_message_decrypt_generation_vectors]) - - decrypt_manifest_file = output_dir.join("manifest.json") - full_message_decrypt.cli(["--input", str(decrypt_manifest_file)]) diff --git a/test_vector_handlers/test/mpl/integration/commands/test_i_esdk_dafny_keyrings.py b/test_vector_handlers/test/mpl/integration/commands/test_i_esdk_dafny_keyrings.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/test_vector_handlers/test/mpl/integration/commands/test_i_net_401_keyrings.py b/test_vector_handlers/test/mpl/integration/commands/test_i_net_401_keyrings.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index bf4b86724..c002323d3 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -58,6 +58,7 @@ commands = mplvectors: {[testenv:base-command]commands} test/mpl full_decrypt_generate: awses-full-message-decrypt-generate {posargs} full_decrypt: awses-full-message-decrypt {posargs} + full_encrypt: awses-full-message-encrypt {posargs} [testenv:full-encrypt] basepython = python3 From b116b0df644b78f7da0ba2eceea6b153e2d847eb Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 09:27:18 -0700 Subject: [PATCH 339/422] debug --- codebuild/py37/decrypt_masterkey_with_js.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/codebuild/py37/decrypt_masterkey_with_js.yml b/codebuild/py37/decrypt_masterkey_with_js.yml index 15dcb35ac..0652b03bf 100644 --- a/codebuild/py37/decrypt_masterkey_with_js.yml +++ b/codebuild/py37/decrypt_masterkey_with_js.yml @@ -37,7 +37,8 @@ phases: # Download generated vectors # TODO rewrite URL - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/37_masterkey.zip 37_masterkey.zip + - unzip 37_masterkey.zip build: commands: # Decrypt generated vectors with Javascript ESDK - - integration-node decrypt -v 37_masterkey.zip \ No newline at end of file + - integration-node decrypt -v 37_masterkey \ No newline at end of file From 063989feea5db9c5029934a3966435f68a81ac97 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 09:31:02 -0700 Subject: [PATCH 340/422] debug --- buildspec.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index 2eb44ccff..9375821b0 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -29,13 +29,13 @@ batch: image: aws/codebuild/standard:5.0 - identifier: py37_decrypt_masterkey_with_masterkey depend-on: - - py37_generate_decrypt_vectors + - py37_generate_decrypt_vectors_masterkey buildspec: codebuild/py37/decrypt_masterkey_with_masterkey.yml env: image: aws/codebuild/standard:5.0 - identifier: py37_decrypt_masterkey_with_js depend-on: - - py37_generate_decrypt_vectors + - py37_generate_decrypt_vectors_masterkey buildspec: codebuild/py37/decrypt_masterkey_with_js.yml env: image: aws/codebuild/standard:5.0 From 101af711436ec6c1d73b16284597ddd5cbde4521 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 09:32:40 -0700 Subject: [PATCH 341/422] debug --- codebuild/py37/encrypt_masterkey.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/codebuild/py37/encrypt_masterkey.yml b/codebuild/py37/encrypt_masterkey.yml index c6117b9ba..384efc0bb 100644 --- a/codebuild/py37/encrypt_masterkey.yml +++ b/codebuild/py37/encrypt_masterkey.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - TOXENV: "py37-full_decrypt" + TOXENV: "py37-full_encrypt" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- @@ -18,16 +18,15 @@ phases: install: runtime-versions: python: 3.7 - pre_build: - commands: - # Download generated vectors - # TODO rewrite URL - - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/37_masterkey.zip 37_masterkey.zip - - unzip 37_masterkey.zip build: commands: - pip install "tox < 4.0" - cd test_vector_handlers - | tox -- \ - --input ../37_masterkey/manifest.json \ No newline at end of file + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0003-awses-message-encryption.v2.json \ +# artifacts: +# files: +# - test_vector_handlers/37_master.zip +# name: $CODEBUILD_INITIATOR/37_master.zip +# discard-paths: yes From 803ae4d7d3c42357c35041c8d819de46bc947b4e Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 09:49:26 -0700 Subject: [PATCH 342/422] debug --- codebuild/py37/decrypt_masterkey_with_js.yml | 3 +-- .../py37/decrypt_masterkey_with_masterkey.yml | 5 +---- codebuild/py37/decrypt_net_401_vectors.yml | 18 +++++++++--------- codebuild/py37/encrypt_masterkey.yml | 4 ++-- .../generate_decrypt_vectors_masterkey.yml | 4 ++-- 5 files changed, 15 insertions(+), 19 deletions(-) diff --git a/codebuild/py37/decrypt_masterkey_with_js.yml b/codebuild/py37/decrypt_masterkey_with_js.yml index 0652b03bf..e13f3d64a 100644 --- a/codebuild/py37/decrypt_masterkey_with_js.yml +++ b/codebuild/py37/decrypt_masterkey_with_js.yml @@ -34,8 +34,7 @@ phases: # - aws sts get-caller-identity # - cd $CODEBUILD_SRC_DIR - # Download generated vectors - # TODO rewrite URL + # Download previously generated vectors - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/37_masterkey.zip 37_masterkey.zip - unzip 37_masterkey.zip build: diff --git a/codebuild/py37/decrypt_masterkey_with_masterkey.yml b/codebuild/py37/decrypt_masterkey_with_masterkey.yml index c6117b9ba..1774b05f4 100644 --- a/codebuild/py37/decrypt_masterkey_with_masterkey.yml +++ b/codebuild/py37/decrypt_masterkey_with_masterkey.yml @@ -11,8 +11,6 @@ env: arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" phases: install: @@ -20,8 +18,7 @@ phases: python: 3.7 pre_build: commands: - # Download generated vectors - # TODO rewrite URL + # Download previously generated vectors - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/37_masterkey.zip 37_masterkey.zip - unzip 37_masterkey.zip build: diff --git a/codebuild/py37/decrypt_net_401_vectors.yml b/codebuild/py37/decrypt_net_401_vectors.yml index 5b3925890..943f0722a 100644 --- a/codebuild/py37/decrypt_net_401_vectors.yml +++ b/codebuild/py37/decrypt_net_401_vectors.yml @@ -12,8 +12,8 @@ env: arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + # AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + # AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" phases: install: @@ -21,13 +21,13 @@ phases: python: 3.7 pre_build: commands: - # Assume Role to access non-prod resources - - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") - - export TMP_ROLE - - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') - - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') - - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') - - aws sts get-caller-identity + # # Assume Role to access non-prod resources + # - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") + # - export TMP_ROLE + # - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') + # - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') + # - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') + # - aws sts get-caller-identity # Fetch ESDK .NET v4.0.1 Test Vectors - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip diff --git a/codebuild/py37/encrypt_masterkey.yml b/codebuild/py37/encrypt_masterkey.yml index 384efc0bb..70a36c4d0 100644 --- a/codebuild/py37/encrypt_masterkey.yml +++ b/codebuild/py37/encrypt_masterkey.yml @@ -11,8 +11,8 @@ env: arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + # AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + # AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" phases: install: diff --git a/codebuild/py37/generate_decrypt_vectors_masterkey.yml b/codebuild/py37/generate_decrypt_vectors_masterkey.yml index f5f12490e..8c5c9712f 100644 --- a/codebuild/py37/generate_decrypt_vectors_masterkey.yml +++ b/codebuild/py37/generate_decrypt_vectors_masterkey.yml @@ -11,8 +11,8 @@ env: arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + # AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + # AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" phases: install: From 58bd2714e041cca5d1779d2152a1d0b6cda5a072 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 10:24:55 -0700 Subject: [PATCH 343/422] debug --- buildspec.yml | 2 +- codebuild/py37/decrypt_masterkey_with_js.yml | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index 9375821b0..9d588c67b 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -35,7 +35,7 @@ batch: image: aws/codebuild/standard:5.0 - identifier: py37_decrypt_masterkey_with_js depend-on: - - py37_generate_decrypt_vectors_masterkey + # - py37_generate_decrypt_vectors_masterkey buildspec: codebuild/py37/decrypt_masterkey_with_js.yml env: image: aws/codebuild/standard:5.0 diff --git a/codebuild/py37/decrypt_masterkey_with_js.yml b/codebuild/py37/decrypt_masterkey_with_js.yml index e13f3d64a..c1a4196e6 100644 --- a/codebuild/py37/decrypt_masterkey_with_js.yml +++ b/codebuild/py37/decrypt_masterkey_with_js.yml @@ -35,9 +35,12 @@ phases: # - cd $CODEBUILD_SRC_DIR # Download previously generated vectors - - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/37_masterkey.zip 37_masterkey.zip + - aws s3 cp s3://generated-vectors-artifacts-bucket/77b9165380cfad5fe8b18809fe72b7ca7accfa1c/37_masterkey.zip 37_masterkey.zip + # Repackage zip in expected format - unzip 37_masterkey.zip + - cd 37_masterkey + - zip -r vectors.zip . build: commands: # Decrypt generated vectors with Javascript ESDK - - integration-node decrypt -v 37_masterkey \ No newline at end of file + - integration-node decrypt -v vectors.zip \ No newline at end of file From bccd1beb45d332940b9dfe8a7ce8a03f5159c792 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 10:29:53 -0700 Subject: [PATCH 344/422] clean37 --- codebuild/py37/decrypt_masterkey_with_js.yml | 14 +------------- codebuild/py37/decrypt_net_401_vectors.yml | 10 ---------- codebuild/py37/encrypt_masterkey.yml | 7 ------- .../py37/generate_decrypt_vectors_masterkey.yml | 7 ------- 4 files changed, 1 insertion(+), 37 deletions(-) diff --git a/codebuild/py37/decrypt_masterkey_with_js.yml b/codebuild/py37/decrypt_masterkey_with_js.yml index c1a4196e6..8a44e11e7 100644 --- a/codebuild/py37/decrypt_masterkey_with_js.yml +++ b/codebuild/py37/decrypt_masterkey_with_js.yml @@ -10,9 +10,6 @@ env: arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - GENERATE_OUTPUT_DIR: >- - $CODEBUILD_SRC_DIR/generated_vectors/ - phases: install: @@ -25,17 +22,8 @@ phases: pre_build: commands: - # Assume Role to access non-prod resources - # - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") - # - export TMP_ROLE - # - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') - # - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') - # - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') - # - aws sts get-caller-identity - # - cd $CODEBUILD_SRC_DIR - # Download previously generated vectors - - aws s3 cp s3://generated-vectors-artifacts-bucket/77b9165380cfad5fe8b18809fe72b7ca7accfa1c/37_masterkey.zip 37_masterkey.zip + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/37_masterkey.zip 37_masterkey.zip # Repackage zip in expected format - unzip 37_masterkey.zip - cd 37_masterkey diff --git a/codebuild/py37/decrypt_net_401_vectors.yml b/codebuild/py37/decrypt_net_401_vectors.yml index 943f0722a..1dfb48ce8 100644 --- a/codebuild/py37/decrypt_net_401_vectors.yml +++ b/codebuild/py37/decrypt_net_401_vectors.yml @@ -12,8 +12,6 @@ env: arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - # AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" - # AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" phases: install: @@ -21,14 +19,6 @@ phases: python: 3.7 pre_build: commands: - # # Assume Role to access non-prod resources - # - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") - # - export TMP_ROLE - # - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') - # - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') - # - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') - # - aws sts get-caller-identity - # Fetch ESDK .NET v4.0.1 Test Vectors - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip diff --git a/codebuild/py37/encrypt_masterkey.yml b/codebuild/py37/encrypt_masterkey.yml index 70a36c4d0..b5cb57dae 100644 --- a/codebuild/py37/encrypt_masterkey.yml +++ b/codebuild/py37/encrypt_masterkey.yml @@ -11,8 +11,6 @@ env: arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - # AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" - # AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" phases: install: @@ -25,8 +23,3 @@ phases: - | tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0003-awses-message-encryption.v2.json \ -# artifacts: -# files: -# - test_vector_handlers/37_master.zip -# name: $CODEBUILD_INITIATOR/37_master.zip -# discard-paths: yes diff --git a/codebuild/py37/generate_decrypt_vectors_masterkey.yml b/codebuild/py37/generate_decrypt_vectors_masterkey.yml index 8c5c9712f..55ec3e9e4 100644 --- a/codebuild/py37/generate_decrypt_vectors_masterkey.yml +++ b/codebuild/py37/generate_decrypt_vectors_masterkey.yml @@ -11,8 +11,6 @@ env: arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - # AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" - # AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" phases: install: @@ -28,8 +26,3 @@ phases: --output 37_masterkey - zip -r 37_masterkey.zip 37_masterkey - aws s3 cp 37_masterkey.zip s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/37_masterkey.zip -# artifacts: -# files: -# - test_vector_handlers/37_master.zip -# name: $CODEBUILD_INITIATOR/37_master.zip -# discard-paths: yes From 970ca3cc7bf5e86a50095f611370fd0014a733eb Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 10:30:23 -0700 Subject: [PATCH 345/422] clean37 --- buildspec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildspec.yml b/buildspec.yml index 9d588c67b..9375821b0 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -35,7 +35,7 @@ batch: image: aws/codebuild/standard:5.0 - identifier: py37_decrypt_masterkey_with_js depend-on: - # - py37_generate_decrypt_vectors_masterkey + - py37_generate_decrypt_vectors_masterkey buildspec: codebuild/py37/decrypt_masterkey_with_js.yml env: image: aws/codebuild/standard:5.0 From 9ccd01419b944f656b23623b00d84fa5cc2ae268 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 11:04:52 -0700 Subject: [PATCH 346/422] 311 --- buildspec.yml | 158 +++++++++--------- codebuild/py311/decrypt_keyrings_with_js.yml | 34 ++++ .../py311/decrypt_keyrings_with_keyrings.yml | 31 ++++ .../py311/decrypt_keyrings_with_masterkey.yml | 30 ++++ codebuild/py311/decrypt_masterkey_with_js.yml | 34 ++++ .../py311/decrypt_masterkey_with_keyrings.yml | 31 ++++ .../decrypt_masterkey_with_masterkey.yml | 30 ++++ .../decrypt_net_401_vectors_keyrings.yml | 10 -- .../generate_decrypt_vectors_keyrings.yml | 14 +- .../generate_decrypt_vectors_masterkey.yml | 8 +- 10 files changed, 273 insertions(+), 107 deletions(-) create mode 100644 codebuild/py311/decrypt_keyrings_with_js.yml create mode 100644 codebuild/py311/decrypt_keyrings_with_keyrings.yml create mode 100644 codebuild/py311/decrypt_keyrings_with_masterkey.yml create mode 100644 codebuild/py311/decrypt_masterkey_with_js.yml create mode 100644 codebuild/py311/decrypt_masterkey_with_keyrings.yml create mode 100644 codebuild/py311/decrypt_masterkey_with_masterkey.yml diff --git a/buildspec.yml b/buildspec.yml index 9375821b0..3c4b965de 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -150,22 +150,22 @@ batch: # env: # image: aws/codebuild/standard:5.0 - # - identifier: py311_integ - # buildspec: codebuild/py311/integ.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_integ_mpl - # buildspec: codebuild/py311/integ_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_examples - # buildspec: codebuild/py311/examples.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_examples_mpl - # buildspec: codebuild/py311/examples_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 + - identifier: py311_integ + buildspec: codebuild/py311/integ.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_integ_mpl + buildspec: codebuild/py311/integ_mpl.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_examples + buildspec: codebuild/py311/examples.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_examples_mpl + buildspec: codebuild/py311/examples_mpl.yml + env: + image: aws/codebuild/standard:7.0 # - identifier: py311_awses_latest # buildspec: codebuild/py311/awses_local.yml # env: @@ -175,69 +175,69 @@ batch: # env: # image: aws/codebuild/standard:7.0 # - identifier: py311_mplawses_latest_mpl - # buildspec: codebuild/py311/mplawses_local_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_dafny_esdk_vectors_masterkey - # buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_dafny_esdk_vectors_keyrings - # buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_net_401_vectors_masterkey - # buildspec: codebuild/py311/decrypt_net_401_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_net_401_vectors_keyrings - # buildspec: codebuild/py311/decrypt_net_401_vectors_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py311/generate_decrypt_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_masterkey_with_masterkey - # depend-on: - # - py311_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py311/decrypt_masterkey_with_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_masterkey_with_keyrings - # depend-on: - # - py311_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py311/decrypt_masterkey_with_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_masterkey_with_js - # depend-on: - # - py311_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py311/decrypt_masterkey_with_js.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py311/generate_decrypt_vectors_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_keyrings_with_masterkey - # depend-on: - # - py311_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py311/decrypt_keyrings_with_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_keyrings_with_keyrings - # depend-on: - # - py311_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py311/decrypt_keyrings_with_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_keyrings_with_js - # depend-on: - # - py311_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py311/decrypt_keyrings_with_js.yml - # env: - # image: aws/codebuild/standard:7.0 + # buildspec: codebuild/py311/mplawses_local_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_dafny_esdk_vectors_masterkey + buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_dafny_esdk_vectors_keyrings + buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_net_401_vectors_masterkey + buildspec: codebuild/py311/decrypt_net_401_vectors_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_net_401_vectors_keyrings + buildspec: codebuild/py311/decrypt_net_401_vectors_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_generate_decrypt_vectors_masterkey + buildspec: codebuild/py311/generate_decrypt_vectors_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_masterkey_with_masterkey + depend-on: + - py311_generate_decrypt_vectors_masterkey + buildspec: codebuild/py311/decrypt_masterkey_with_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_masterkey_with_keyrings + depend-on: + - py311_generate_decrypt_vectors_masterkey + buildspec: codebuild/py311/decrypt_masterkey_with_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_masterkey_with_js + depend-on: + - py311_generate_decrypt_vectors_masterkey + buildspec: codebuild/py311/decrypt_masterkey_with_js.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_generate_decrypt_vectors_keyrings + buildspec: codebuild/py311/generate_decrypt_vectors_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_keyrings_with_masterkey + depend-on: + - py311_generate_decrypt_vectors_keyrings + buildspec: codebuild/py311/decrypt_keyrings_with_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_keyrings_with_keyrings + depend-on: + - py311_generate_decrypt_vectors_keyrings + buildspec: codebuild/py311/decrypt_keyrings_with_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_keyrings_with_js + depend-on: + - py311_generate_decrypt_vectors_keyrings + buildspec: codebuild/py311/decrypt_keyrings_with_js.yml + env: + image: aws/codebuild/standard:7.0 # - identifier: py312_integ # buildspec: codebuild/py312/integ.yml diff --git a/codebuild/py311/decrypt_keyrings_with_js.yml b/codebuild/py311/decrypt_keyrings_with_js.yml new file mode 100644 index 000000000..578b83cab --- /dev/null +++ b/codebuild/py311/decrypt_keyrings_with_js.yml @@ -0,0 +1,34 @@ +version: 0.2 + +env: + variables: + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.11 + commands: + - n 16 + # Install the Javascript ESDK run test vectors + - npm install -g @aws-crypto/integration-node + + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyrings.zip 311_keyrings.zip + # Repackage zip in expected format + - unzip 311_keyrings.zip + - cd 311_keyrings + - zip -r vectors.zip . + build: + commands: + # Decrypt generated vectors with Javascript ESDK + - integration-node decrypt -v vectors.zip \ No newline at end of file diff --git a/codebuild/py311/decrypt_keyrings_with_keyrings.yml b/codebuild/py311/decrypt_keyrings_with_keyrings.yml new file mode 100644 index 000000000..5478fff38 --- /dev/null +++ b/codebuild/py311/decrypt_keyrings_with_keyrings.yml @@ -0,0 +1,31 @@ +version: 0.2 + +env: + variables: + TOXENV: "py311-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.11 + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyrings.zip 311_keyrings.zip + - unzip 311_keyrings.zip + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../311_keyrings/manifest.json \ + --keyrings \ No newline at end of file diff --git a/codebuild/py311/decrypt_keyrings_with_masterkey.yml b/codebuild/py311/decrypt_keyrings_with_masterkey.yml new file mode 100644 index 000000000..714882c54 --- /dev/null +++ b/codebuild/py311/decrypt_keyrings_with_masterkey.yml @@ -0,0 +1,30 @@ +version: 0.2 + +env: + variables: + TOXENV: "py311-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.11 + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyrings.zip 311_keyrings.zip + - unzip 311_keyrings.zip + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../311_keyrings/manifest.json \ No newline at end of file diff --git a/codebuild/py311/decrypt_masterkey_with_js.yml b/codebuild/py311/decrypt_masterkey_with_js.yml new file mode 100644 index 000000000..a73e93580 --- /dev/null +++ b/codebuild/py311/decrypt_masterkey_with_js.yml @@ -0,0 +1,34 @@ +version: 0.2 + +env: + variables: + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.11 + commands: + - n 16 + # Install the Javascript ESDK run test vectors + - npm install -g @aws-crypto/integration-node + + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_masterkey.zip 311_masterkey.zip + # Repackage zip in expected format + - unzip 311_masterkey.zip + - cd 311_masterkey + - zip -r vectors.zip . + build: + commands: + # Decrypt generated vectors with Javascript ESDK + - integration-node decrypt -v vectors.zip \ No newline at end of file diff --git a/codebuild/py311/decrypt_masterkey_with_keyrings.yml b/codebuild/py311/decrypt_masterkey_with_keyrings.yml new file mode 100644 index 000000000..5479ef16c --- /dev/null +++ b/codebuild/py311/decrypt_masterkey_with_keyrings.yml @@ -0,0 +1,31 @@ +version: 0.2 + +env: + variables: + TOXENV: "py311-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.11 + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_masterkey.zip 311_masterkey.zip + - unzip 311_masterkey.zip + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../311_masterkey/manifest.json \ + --keyrings \ No newline at end of file diff --git a/codebuild/py311/decrypt_masterkey_with_masterkey.yml b/codebuild/py311/decrypt_masterkey_with_masterkey.yml new file mode 100644 index 000000000..dd64d2dff --- /dev/null +++ b/codebuild/py311/decrypt_masterkey_with_masterkey.yml @@ -0,0 +1,30 @@ +version: 0.2 + +env: + variables: + TOXENV: "py311-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.11 + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_masterkey.zip 311_masterkey.zip + - unzip 311_masterkey.zip + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../311_masterkey/manifest.json \ No newline at end of file diff --git a/codebuild/py311/decrypt_net_401_vectors_keyrings.yml b/codebuild/py311/decrypt_net_401_vectors_keyrings.yml index 6634470c3..1a23f0917 100644 --- a/codebuild/py311/decrypt_net_401_vectors_keyrings.yml +++ b/codebuild/py311/decrypt_net_401_vectors_keyrings.yml @@ -12,8 +12,6 @@ env: arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" phases: install: @@ -21,14 +19,6 @@ phases: python: 3.11 pre_build: commands: - # Assume Role to access non-prod resource - - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") - - export TMP_ROLE - - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') - - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') - - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') - - aws sts get-caller-identity - # Fetch ESDK .NET v4.0.1 Test Vectors - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml index 3e28dc113..777a5703f 100644 --- a/codebuild/py311/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -11,8 +11,6 @@ env: arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" phases: install: @@ -22,15 +20,9 @@ phases: commands: - pip install "tox < 4.0" - cd test_vector_handlers - # - mkdir $CODEBUILD_RESOLVED_SOURCE_VERSION - | tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ - --output 311_keyring \ - --keyrings - # - zip -r $CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip 311_keyring - - zip -r 311_keyring.zip 311_keyring - - aws s3 cp 311_keyring.zip s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip -# artifacts: -# files: -# - test_vector_handlers/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyring.zip + --output 311_keyrings + - zip -r 311_keyrings.zip 311_keyrings + - aws s3 cp 311_keyrings.zip s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyrings.zip diff --git a/codebuild/py311/generate_decrypt_vectors_masterkey.yml b/codebuild/py311/generate_decrypt_vectors_masterkey.yml index f4056832a..84db3f176 100644 --- a/codebuild/py311/generate_decrypt_vectors_masterkey.yml +++ b/codebuild/py311/generate_decrypt_vectors_masterkey.yml @@ -11,8 +11,6 @@ env: arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" phases: install: @@ -27,8 +25,4 @@ phases: --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ --output 311_masterkey - zip -r 311_masterkey.zip 311_masterkey -artifacts: - files: - - test_vector_handlers/311_masterkey.zip - name: $CODEBUILD_INITIATOR/311_masterkey.zip - discard-paths: yes \ No newline at end of file + - aws s3 cp 311_masterkey.zip s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_masterkey.zip From 2c3b3800a83e2e6db1d294195aec8be1ffd656b8 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 11:08:51 -0700 Subject: [PATCH 347/422] 311 --- codebuild/py311/decrypt_keyrings_with_keyrings.yml | 2 +- codebuild/py311/decrypt_masterkey_with_keyrings.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/codebuild/py311/decrypt_keyrings_with_keyrings.yml b/codebuild/py311/decrypt_keyrings_with_keyrings.yml index 5478fff38..fec275d48 100644 --- a/codebuild/py311/decrypt_keyrings_with_keyrings.yml +++ b/codebuild/py311/decrypt_keyrings_with_keyrings.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - TOXENV: "py311-full_decrypt" + TOXENV: "py311-full_decrypt-mpl" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- diff --git a/codebuild/py311/decrypt_masterkey_with_keyrings.yml b/codebuild/py311/decrypt_masterkey_with_keyrings.yml index 5479ef16c..8543077bd 100644 --- a/codebuild/py311/decrypt_masterkey_with_keyrings.yml +++ b/codebuild/py311/decrypt_masterkey_with_keyrings.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - TOXENV: "py311-full_decrypt" + TOXENV: "py311-full_decrypt-mpl" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- From c8ca704f4c7b4a79affb9ea4e85652630a305363 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 11:09:01 -0700 Subject: [PATCH 348/422] 311 --- .../decrypt_dafny_esdk_vectors_keyrings.yml | 59 +++++++++++++++++++ .../decrypt_dafny_esdk_vectors_masterkey.yml | 58 ++++++++++++++++++ 2 files changed, 117 insertions(+) create mode 100644 codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml create mode 100644 codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml diff --git a/codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml b/codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml new file mode 100644 index 000000000..d69ce9370 --- /dev/null +++ b/codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml @@ -0,0 +1,59 @@ +version: 0.2 +# Runs Only the ESDK-NET v4.0.1 Decryption Vectors, testing Required EC CMM + +env: + variables: + TOXENV: "py311-full_decrypt-mpl" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + git-credential-helper: yes + secrets-manager: + GITHUB_TOKEN: Github/lucasmcdonald3-fgpat:actions read + +phases: + install: + runtime-versions: + python: 3.11 + pre_build: + commands: + # Fetch test vectors from Dafny ESDK's most recent run + # (Assuming the first result is most recent; seems to be correct...) + - | + MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ + | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') + - | + echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" + - | + MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/$MOST_RECENT_RUN_ID/artifacts?name=ubuntu-latest_vector_artifact" \ + | jq '.artifacts[0].archive_download_url') + - | + echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" + - | + curl -L -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip + # This unzips to `net41.zip`. + - unzip ubuntu-latest_test_vector_artifact + # This unzips to `net41/`. + - unzip net41.zip -d net41 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../net41/manifest.json \ + --keyrings diff --git a/codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml b/codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml new file mode 100644 index 000000000..6106906b5 --- /dev/null +++ b/codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml @@ -0,0 +1,58 @@ +version: 0.2 +# Runs Only the ESDK-NET v4.0.1 Decryption Vectors, testing Required EC CMM + +env: + variables: + TOXENV: "py311-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + git-credential-helper: yes + secrets-manager: + GITHUB_TOKEN: Github/lucasmcdonald3-fgpat:actions read + +phases: + install: + runtime-versions: + python: 3.11 + pre_build: + commands: + # Fetch test vectors from Dafny ESDK's most recent run + # (Assuming the first result is most recent; seems to be correct...) + - | + MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ + | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') + - | + echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" + - | + MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/$MOST_RECENT_RUN_ID/artifacts?name=ubuntu-latest_vector_artifact" \ + | jq '.artifacts[0].archive_download_url') + - | + echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" + - | + curl -L -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip + # This unzips to `net41.zip`. + - unzip ubuntu-latest_test_vector_artifact + # This unzips to `net41/`. + - unzip net41.zip -d net41 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../net41/manifest.json From c8c5a4096e57cb361dc394dc58dd5fdcf5a01a3e Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 11:17:42 -0700 Subject: [PATCH 349/422] py311 --- codebuild/py311/encrypt_keyrings.yml | 26 ++++++++++++++++++++++++++ codebuild/py311/encrypt_masterkey.yml | 25 +++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 codebuild/py311/encrypt_keyrings.yml create mode 100644 codebuild/py311/encrypt_masterkey.yml diff --git a/codebuild/py311/encrypt_keyrings.yml b/codebuild/py311/encrypt_keyrings.yml new file mode 100644 index 000000000..8b7cb94b4 --- /dev/null +++ b/codebuild/py311/encrypt_keyrings.yml @@ -0,0 +1,26 @@ +version: 0.2 + +env: + variables: + TOXENV: "py311-full_encrypt-mpl" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.11 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0003-awses-message-encryption.v2.json \ + --keyrings \ No newline at end of file diff --git a/codebuild/py311/encrypt_masterkey.yml b/codebuild/py311/encrypt_masterkey.yml new file mode 100644 index 000000000..226e1586d --- /dev/null +++ b/codebuild/py311/encrypt_masterkey.yml @@ -0,0 +1,25 @@ +version: 0.2 + +env: + variables: + TOXENV: "py311-full_encrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.11 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0003-awses-message-encryption.v2.json From 8a0ddc4441390c8966e04d593e2b6e9830e8b911 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 11:34:23 -0700 Subject: [PATCH 350/422] 311 --- buildspec.yml | 8 ++++++++ codebuild/py37/encrypt_masterkey.yml | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/buildspec.yml b/buildspec.yml index 3c4b965de..80c04b2ff 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -194,6 +194,14 @@ batch: buildspec: codebuild/py311/decrypt_net_401_vectors_keyrings.yml env: image: aws/codebuild/standard:7.0 + - identifier: py311_encrypt_masterkey + buildspec: codebuild/py311/encrypt_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py311_encrypt_keyrings + buildspec: codebuild/py311/encrypt_keyrings.yml + env: + image: aws/codebuild/standard:5.0 - identifier: py311_generate_decrypt_vectors_masterkey buildspec: codebuild/py311/generate_decrypt_vectors_masterkey.yml env: diff --git a/codebuild/py37/encrypt_masterkey.yml b/codebuild/py37/encrypt_masterkey.yml index b5cb57dae..7cdc7848e 100644 --- a/codebuild/py37/encrypt_masterkey.yml +++ b/codebuild/py37/encrypt_masterkey.yml @@ -22,4 +22,4 @@ phases: - cd test_vector_handlers - | tox -- \ - --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0003-awses-message-encryption.v2.json \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0003-awses-message-encryption.v2.json From 20262d78e9260d8ade44bbf1a34e5f3a1cdc4eff Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 11:38:17 -0700 Subject: [PATCH 351/422] 311 --- buildspec.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index 80c04b2ff..57b86cb82 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -197,11 +197,11 @@ batch: - identifier: py311_encrypt_masterkey buildspec: codebuild/py311/encrypt_masterkey.yml env: - image: aws/codebuild/standard:5.0 + image: aws/codebuild/standard:7.0 - identifier: py311_encrypt_keyrings buildspec: codebuild/py311/encrypt_keyrings.yml env: - image: aws/codebuild/standard:5.0 + image: aws/codebuild/standard:7.0 - identifier: py311_generate_decrypt_vectors_masterkey buildspec: codebuild/py311/generate_decrypt_vectors_masterkey.yml env: From 89efb749c72b8ce49bcabaff3fddb084f4859c6d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 11:56:51 -0700 Subject: [PATCH 352/422] 312 --- .../py312/decrypt_keyrings_with_keyrings.yml | 31 +++++++++++++ .../py312/decrypt_keyrings_with_masterkey.yml | 30 +++++++++++++ codebuild/py312/decrypt_masterkey_with_js.yml | 34 ++++++++++++++ .../py312/decrypt_masterkey_with_keyrings.yml | 31 +++++++++++++ .../decrypt_masterkey_with_masterkey.yml | 30 +++++++++++++ .../decrypt_net_401_vectors_keyrings.yml | 36 +++++++++++++++ .../decrypt_net_401_vectors_masterkey.yml | 45 +++++++++++++++++++ .../{awses_local.yml => encrypt_keyrings.yml} | 16 +++---- codebuild/py312/encrypt_masterkey.yml | 25 +++++++++++ codebuild/py312/examples.yml | 7 +-- codebuild/py312/examples_mpl.yml | 13 ++---- .../generate_decrypt_vectors_keyrings.yml | 28 ++++++++++++ ...=> generate_decrypt_vectors_masterkey.yml} | 21 ++++----- codebuild/py312/integ.yml | 7 +-- codebuild/py312/integ_mpl.yml | 9 +--- codebuild/py312/mplawses_local_mpl.yml | 8 +--- 16 files changed, 311 insertions(+), 60 deletions(-) create mode 100644 codebuild/py312/decrypt_keyrings_with_keyrings.yml create mode 100644 codebuild/py312/decrypt_keyrings_with_masterkey.yml create mode 100644 codebuild/py312/decrypt_masterkey_with_js.yml create mode 100644 codebuild/py312/decrypt_masterkey_with_keyrings.yml create mode 100644 codebuild/py312/decrypt_masterkey_with_masterkey.yml create mode 100644 codebuild/py312/decrypt_net_401_vectors_keyrings.yml create mode 100644 codebuild/py312/decrypt_net_401_vectors_masterkey.yml rename codebuild/py312/{awses_local.yml => encrypt_keyrings.yml} (64%) create mode 100644 codebuild/py312/encrypt_masterkey.yml create mode 100644 codebuild/py312/generate_decrypt_vectors_keyrings.yml rename codebuild/py312/{awses_local_mpl.yml => generate_decrypt_vectors_masterkey.yml} (56%) diff --git a/codebuild/py312/decrypt_keyrings_with_keyrings.yml b/codebuild/py312/decrypt_keyrings_with_keyrings.yml new file mode 100644 index 000000000..3ab7058f9 --- /dev/null +++ b/codebuild/py312/decrypt_keyrings_with_keyrings.yml @@ -0,0 +1,31 @@ +version: 0.2 + +env: + variables: + TOXENV: "py312-full_decrypt-mpl" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.12 + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_keyrings.zip 312_keyrings.zip + - unzip 312_keyrings.zip + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../312_keyrings/manifest.json \ + --keyrings \ No newline at end of file diff --git a/codebuild/py312/decrypt_keyrings_with_masterkey.yml b/codebuild/py312/decrypt_keyrings_with_masterkey.yml new file mode 100644 index 000000000..bb06ba4a2 --- /dev/null +++ b/codebuild/py312/decrypt_keyrings_with_masterkey.yml @@ -0,0 +1,30 @@ +version: 0.2 + +env: + variables: + TOXENV: "py312-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.12 + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_keyrings.zip 312_keyrings.zip + - unzip 312_keyrings.zip + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../312_keyrings/manifest.json \ No newline at end of file diff --git a/codebuild/py312/decrypt_masterkey_with_js.yml b/codebuild/py312/decrypt_masterkey_with_js.yml new file mode 100644 index 000000000..7c57c3111 --- /dev/null +++ b/codebuild/py312/decrypt_masterkey_with_js.yml @@ -0,0 +1,34 @@ +version: 0.2 + +env: + variables: + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.12 + commands: + - n 16 + # Install the Javascript ESDK run test vectors + - npm install -g @aws-crypto/integration-node + + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_masterkey.zip 312_masterkey.zip + # Repackage zip in expected format + - unzip 312_masterkey.zip + - cd 312_masterkey + - zip -r vectors.zip . + build: + commands: + # Decrypt generated vectors with Javascript ESDK + - integration-node decrypt -v vectors.zip \ No newline at end of file diff --git a/codebuild/py312/decrypt_masterkey_with_keyrings.yml b/codebuild/py312/decrypt_masterkey_with_keyrings.yml new file mode 100644 index 000000000..21f646370 --- /dev/null +++ b/codebuild/py312/decrypt_masterkey_with_keyrings.yml @@ -0,0 +1,31 @@ +version: 0.2 + +env: + variables: + TOXENV: "py312-full_decrypt-mpl" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.12 + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_masterkey.zip 312_masterkey.zip + - unzip 312_masterkey.zip + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../312_masterkey/manifest.json \ + --keyrings \ No newline at end of file diff --git a/codebuild/py312/decrypt_masterkey_with_masterkey.yml b/codebuild/py312/decrypt_masterkey_with_masterkey.yml new file mode 100644 index 000000000..0529fd894 --- /dev/null +++ b/codebuild/py312/decrypt_masterkey_with_masterkey.yml @@ -0,0 +1,30 @@ +version: 0.2 + +env: + variables: + TOXENV: "py312-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.12 + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_masterkey.zip 312_masterkey.zip + - unzip 312_masterkey.zip + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../312_masterkey/manifest.json \ No newline at end of file diff --git a/codebuild/py312/decrypt_net_401_vectors_keyrings.yml b/codebuild/py312/decrypt_net_401_vectors_keyrings.yml new file mode 100644 index 000000000..aec3916e5 --- /dev/null +++ b/codebuild/py312/decrypt_net_401_vectors_keyrings.yml @@ -0,0 +1,36 @@ +version: 0.2 +# Runs Only the ESDK-NET v4.0.1 Decryption Vectors, testing Required EC CMM + +env: + variables: + TOXENV: "py312-full_decrypt-mpl" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.12 + pre_build: + commands: + # Fetch ESDK .NET v4.0.1 Test Vectors + - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip + - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip + - curl -s --output $VECTOR_ZIP --location $VECTORS_URL + - UNZIPPED_VECTORS_DIR=$CODEBUILD_SRC_DIR/test_vector_handlers/net_401_vectors + - unzip $VECTOR_ZIP -d $UNZIPPED_VECTORS_DIR + build: + commands: + # NOTE: We need to pass the absolute path of the vectors + - pip install "tox < 4.0" + - cd $CODEBUILD_SRC_DIR/test_vector_handlers + - | + tox -- \ + --input $UNZIPPED_VECTORS_DIR/manifest.json \ + --keyrings diff --git a/codebuild/py312/decrypt_net_401_vectors_masterkey.yml b/codebuild/py312/decrypt_net_401_vectors_masterkey.yml new file mode 100644 index 000000000..5d1ef9d94 --- /dev/null +++ b/codebuild/py312/decrypt_net_401_vectors_masterkey.yml @@ -0,0 +1,45 @@ +version: 0.2 +# Runs Only the ESDK-NET v4.0.1 Decryption Vectors, testing Required EC CMM + +env: + variables: + TOXENV: "py312-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + python: 3.12 + pre_build: + commands: + # Assume Role to access non-prod resources + - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-TestVectorResources") + - export TMP_ROLE + - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') + - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') + - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') + - aws sts get-caller-identity + + # Fetch ESDK .NET v4.0.1 Test Vectors + - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip + - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip + - curl -s --output $VECTOR_ZIP --location $VECTORS_URL + - UNZIPPED_VECTORS_DIR=$CODEBUILD_SRC_DIR/test_vector_handlers/net_401_vectors + - unzip $VECTOR_ZIP -d $UNZIPPED_VECTORS_DIR + build: + commands: + # NOTE: We need to pass the absolute path of the vectors + - pip install "tox < 4.0" + - cd $CODEBUILD_SRC_DIR/test_vector_handlers + - | + tox -- \ + --input $UNZIPPED_VECTORS_DIR/manifest.json \ No newline at end of file diff --git a/codebuild/py312/awses_local.yml b/codebuild/py312/encrypt_keyrings.yml similarity index 64% rename from codebuild/py312/awses_local.yml rename to codebuild/py312/encrypt_keyrings.yml index 0a81984ee..56a389e6f 100644 --- a/codebuild/py312/awses_local.yml +++ b/codebuild/py312/encrypt_keyrings.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - TOXENV: "py312-awses_local" + TOXENV: "py312-full_encrypt-mpl" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- @@ -11,20 +11,16 @@ env: arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" phases: install: runtime-versions: - python: latest + python: 3.12 build: commands: - - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - - - pyenv install --skip-existing 3.12.0 - - pyenv local 3.12.0 - - pip install --upgrade pip - - pip install setuptools - pip install "tox < 4.0" - cd test_vector_handlers - - tox + - | + tox -- \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0003-awses-message-encryption.v2.json \ + --keyrings \ No newline at end of file diff --git a/codebuild/py312/encrypt_masterkey.yml b/codebuild/py312/encrypt_masterkey.yml new file mode 100644 index 000000000..940f336a2 --- /dev/null +++ b/codebuild/py312/encrypt_masterkey.yml @@ -0,0 +1,25 @@ +version: 0.2 + +env: + variables: + TOXENV: "py312-full_encrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.12 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0003-awses-message-encryption.v2.json diff --git a/codebuild/py312/examples.yml b/codebuild/py312/examples.yml index 691ea0e60..855a8fcdb 100644 --- a/codebuild/py312/examples.yml +++ b/codebuild/py312/examples.yml @@ -15,13 +15,8 @@ env: phases: install: runtime-versions: - python: latest + python: 3.12 build: commands: - - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - - - pyenv install --skip-existing 3.12.0 - - pyenv local 3.12.0 - - pip install --upgrade pip - - pip install setuptools - pip install "tox < 4.0" - tox diff --git a/codebuild/py312/examples_mpl.yml b/codebuild/py312/examples_mpl.yml index 366222441..86774df0e 100644 --- a/codebuild/py312/examples_mpl.yml +++ b/codebuild/py312/examples_mpl.yml @@ -1,6 +1,3 @@ -# Runs the same tests as examples in an environment with the MPL installed -# to assert existing tests continue to pass with the MPL installed. -# Then, run MPL-specific tests. version: 0.2 env: @@ -19,19 +16,14 @@ env: phases: install: runtime-versions: - python: latest + python: 3.12 build: commands: - - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - - - pyenv install --skip-existing 3.12.0 - - pyenv local 3.12.0 - - pip install --upgrade pip - - pip install setuptools - pip install "tox < 4.0" # Run non-MPL-specific tests with the MPL installed - tox -e py312-examples-mpl # Assume special role to access keystore - - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-Py311ExamplesMpl") + - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-Py312ExamplesMpl") - export TMP_ROLE - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') @@ -39,3 +31,4 @@ phases: - aws sts get-caller-identity # Run MPL-specific tests with special role - tox -e py312-mplexamples-mpl + diff --git a/codebuild/py312/generate_decrypt_vectors_keyrings.yml b/codebuild/py312/generate_decrypt_vectors_keyrings.yml new file mode 100644 index 000000000..51a1415ee --- /dev/null +++ b/codebuild/py312/generate_decrypt_vectors_keyrings.yml @@ -0,0 +1,28 @@ +version: 0.2 + +env: + variables: + TOXENV: "py312-full_decrypt_generate-mpl" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.12 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ + --output 312_keyrings + - zip -r 312_keyrings.zip 312_keyrings + - aws s3 cp 312_keyrings.zip s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_keyrings.zip diff --git a/codebuild/py312/awses_local_mpl.yml b/codebuild/py312/generate_decrypt_vectors_masterkey.yml similarity index 56% rename from codebuild/py312/awses_local_mpl.yml rename to codebuild/py312/generate_decrypt_vectors_masterkey.yml index 96ca5bc28..1fadba985 100644 --- a/codebuild/py312/awses_local_mpl.yml +++ b/codebuild/py312/generate_decrypt_vectors_masterkey.yml @@ -1,11 +1,8 @@ -# Runs test vectors using native constructs in an environment with the MPL installed. -# This asserts that installing the MPL does not change existing behavior. version: 0.2 env: variables: - TOXENV: "py312-awses_local-mpl" - REGION: "us-west-2" + TOXENV: "py312-full_decrypt_generate" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- @@ -14,20 +11,18 @@ env: arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" phases: install: runtime-versions: - python: latest + python: 3.12 build: commands: - - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - - - pyenv install --skip-existing 3.12.0 - - pyenv local 3.12.0 - - pip install --upgrade pip - - pip install setuptools - pip install "tox < 4.0" - cd test_vector_handlers - - tox + - | + tox -- \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ + --output 312_masterkey + - zip -r 312_masterkey.zip 312_masterkey + - aws s3 cp 312_masterkey.zip s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_masterkey.zip diff --git a/codebuild/py312/integ.yml b/codebuild/py312/integ.yml index 10899f1df..2ccad8913 100644 --- a/codebuild/py312/integ.yml +++ b/codebuild/py312/integ.yml @@ -15,13 +15,8 @@ env: phases: install: runtime-versions: - python: latest + python: 3.12 build: commands: - - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - - - pyenv install --skip-existing 3.12.0 - - pyenv local 3.12.0 - - pip install --upgrade pip - - pip install setuptools - pip install "tox < 4.0" - tox diff --git a/codebuild/py312/integ_mpl.yml b/codebuild/py312/integ_mpl.yml index e292acc57..28bbaa422 100644 --- a/codebuild/py312/integ_mpl.yml +++ b/codebuild/py312/integ_mpl.yml @@ -1,5 +1,3 @@ -# Runs the same tests as integ in an environment with the MPL installed. -# This asserts existing tests continue to pass with the MPL installed. version: 0.2 env: @@ -18,13 +16,8 @@ env: phases: install: runtime-versions: - python: latest + python: 3.12 build: commands: - - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - - - pyenv install --skip-existing 3.12.0 - - pyenv local 3.12.0 - - pip install --upgrade pip - - pip install setuptools - pip install "tox < 4.0" - tox diff --git a/codebuild/py312/mplawses_local_mpl.yml b/codebuild/py312/mplawses_local_mpl.yml index e11f7523b..8a7d5f5c6 100644 --- a/codebuild/py312/mplawses_local_mpl.yml +++ b/codebuild/py312/mplawses_local_mpl.yml @@ -1,4 +1,3 @@ -# Runs test vectors using MPL constructs. version: 0.2 env: @@ -19,14 +18,9 @@ env: phases: install: runtime-versions: - python: latest + python: 3.12 build: commands: - - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - - - pyenv install --skip-existing 3.12.0 - - pyenv local 3.12.0 - - pip install --upgrade pip - - pip install setuptools - pip install "tox < 4.0" - cd test_vector_handlers - tox From e1700b9910e42d342e8c24c05a774af0d020d01f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 12:01:49 -0700 Subject: [PATCH 353/422] ALL --- buildspec.yml | 370 +++++++++++++++++++------------------------------- 1 file changed, 143 insertions(+), 227 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index 57b86cb82..e303d9b35 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -3,152 +3,82 @@ version: 0.2 batch: fast-fail: false build-graph: - - identifier: py37_integ - buildspec: codebuild/py37/integ.yml + + # 3.8 + - identifier: py38_integ + buildspec: codebuild/py38/integ.yml env: image: aws/codebuild/standard:5.0 - - identifier: py37_examples - buildspec: codebuild/py37/examples.yml + - identifier: py38_examples + buildspec: codebuild/py38/examples.yml env: image: aws/codebuild/standard:5.0 - - identifier: py37_decrypt_dafny_esdk_vectors - buildspec: codebuild/py37/decrypt_dafny_esdk_vectors.yml + - identifier: py38_decrypt_dafny_esdk_vectors + buildspec: codebuild/py38/decrypt_dafny_esdk_vectors.yml env: image: aws/codebuild/standard:5.0 - - identifier: py37_decrypt_net_401_vectors - buildspec: codebuild/py37/decrypt_net_401_vectors.yml + - identifier: py38_decrypt_net_401_vectors + buildspec: codebuild/py38/decrypt_net_401_vectors.yml env: image: aws/codebuild/standard:5.0 - - identifier: py37_encrypt_masterkey - buildspec: codebuild/py37/encrypt_masterkey.yml + - identifier: py38_encrypt_masterkey + buildspec: codebuild/py38/encrypt_masterkey.yml env: image: aws/codebuild/standard:5.0 - - identifier: py37_generate_decrypt_vectors_masterkey - buildspec: codebuild/py37/generate_decrypt_vectors_masterkey.yml + - identifier: py38_generate_decrypt_vectors_masterkey + buildspec: codebuild/py38/generate_decrypt_vectors_masterkey.yml env: image: aws/codebuild/standard:5.0 - - identifier: py37_decrypt_masterkey_with_masterkey + - identifier: py38_decrypt_masterkey_with_masterkey depend-on: - - py37_generate_decrypt_vectors_masterkey - buildspec: codebuild/py37/decrypt_masterkey_with_masterkey.yml + - py38_generate_decrypt_vectors_masterkey + buildspec: codebuild/py38/decrypt_masterkey_with_masterkey.yml env: image: aws/codebuild/standard:5.0 - - identifier: py37_decrypt_masterkey_with_js + - identifier: py38_decrypt_masterkey_with_js depend-on: - - py37_generate_decrypt_vectors_masterkey - buildspec: codebuild/py37/decrypt_masterkey_with_js.yml + - py38_generate_decrypt_vectors_masterkey + buildspec: codebuild/py38/decrypt_masterkey_with_js.yml env: image: aws/codebuild/standard:5.0 - # - identifier: py38_integ - # buildspec: codebuild/py38/integ.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py38_examples - # buildspec: codebuild/py38/examples.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py38_awses_local - # buildspec: codebuild/py38/awses_local.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py38_decrypt_dafny_esdk_vectors - # buildspec: codebuild/py38/decrypt_dafny_esdk_vectors.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py38_decrypt_net_401_vectors - # buildspec: codebuild/py38/decrypt_net_401_vectors.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py38_generate_decrypt_vectors - # buildspec: codebuild/py38/generate_decrypt_vectors.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py38_decrypt_masterkey_with_masterkey - # depend-on: - # - py38_generate_decrypt_vectors - # buildspec: codebuild/py38/decrypt_masterkey_with_masterkey.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py38_decrypt_masterkey_with_js - # depend-on: - # - py38_generate_decrypt_vectors - # buildspec: codebuild/py38/decrypt_generated_with_js.yml - # env: - # image: aws/codebuild/standard:5.0 - - # - identifier: py39_integ - # buildspec: codebuild/py39/integ.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py39_examples - # buildspec: codebuild/py39/examples.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py39_awses_latest - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py39_decrypt_dafny_esdk_vectors - # buildspec: codebuild/py39/decrypt_dafny_esdk_vectors.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py39_decrypt_net_401_vectors - # buildspec: codebuild/py39/decrypt_net_401_vectors.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py39_generate_decrypt_vectors - # buildspec: codebuild/py39/generate_decrypt_vectors.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py39_decrypt_masterkey_with_masterkey - # depend-on: - # - py39_generate_decrypt_vectors - # buildspec: codebuild/py39/decrypt_masterkey_with_masterkey.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py39_decrypt_masterkey_with_js - # depend-on: - # - py39_generate_decrypt_vectors - # buildspec: codebuild/py39/decrypt_generated_with_js.yml - # env: - # image: aws/codebuild/standard:5.0 - - # - identifier: py310_integ - # buildspec: codebuild/py310/integ.yml - # env: - # image: aws/codebuild/standard:6.0 - # - identifier: py310_examples - # buildspec: codebuild/py310/examples.yml - # env: - # image: aws/codebuild/standard:6.0 - # - identifier: py310_awses_latest - # buildspec: codebuild/py310/awses_local.yml - # env: - # image: aws/codebuild/standard:6.0 - # - identifier: py310_decrypt_dafny_esdk_vectors - # buildspec: codebuild/py310/decrypt_dafny_esdk_vectors.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py310_decrypt_net_401_vectors - # buildspec: codebuild/py310/decrypt_net_401_vectors.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py310_generate_decrypt_vectors - # buildspec: codebuild/py310/generate_decrypt_vectors.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py310_decrypt_masterkey_with_masterkey - # depend-on: - # - py310_generate_decrypt_vectors - # buildspec: codebuild/py310/decrypt_masterkey_with_masterkey.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py310_decrypt_masterkey_with_js - # depend-on: - # - py310_generate_decrypt_vectors - # buildspec: codebuild/py310/decrypt_generated_with_js.yml - # env: - # image: aws/codebuild/standard:5.0 + # 3.9 + - identifier: py39_integ + buildspec: codebuild/py39/integ.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_examples + buildspec: codebuild/py39/examples.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_decrypt_dafny_esdk_vectors + buildspec: codebuild/py39/decrypt_dafny_esdk_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_decrypt_net_401_vectors + buildspec: codebuild/py39/decrypt_net_401_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_encrypt_masterkey + buildspec: codebuild/py39/encrypt_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_generate_decrypt_vectors_masterkey + buildspec: codebuild/py39/generate_decrypt_vectors_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_decrypt_masterkey_with_masterkey + depend-on: + - py39_generate_decrypt_vectors_masterkey + buildspec: codebuild/py39/decrypt_masterkey_with_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_decrypt_masterkey_with_js + depend-on: + - py39_generate_decrypt_vectors_masterkey + buildspec: codebuild/py39/decrypt_masterkey_with_js.yml + env: + image: aws/codebuild/standard:5.0 - identifier: py311_integ buildspec: codebuild/py311/integ.yml @@ -166,18 +96,6 @@ batch: buildspec: codebuild/py311/examples_mpl.yml env: image: aws/codebuild/standard:7.0 - # - identifier: py311_awses_latest - # buildspec: codebuild/py311/awses_local.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_awses_latest_mpl - # buildspec: codebuild/py311/awses_local_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_mplawses_latest_mpl - # buildspec: codebuild/py311/mplawses_local_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 - identifier: py311_decrypt_dafny_esdk_vectors_masterkey buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml env: @@ -246,95 +164,93 @@ batch: buildspec: codebuild/py311/decrypt_keyrings_with_js.yml env: image: aws/codebuild/standard:7.0 + + + - identifier: py312_integ + buildspec: codebuild/py312/integ.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_integ_mpl + buildspec: codebuild/py312/integ_mpl.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_examples + buildspec: codebuild/py312/examples.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_examples_mpl + buildspec: codebuild/py312/examples_mpl.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_dafny_esdk_vectors_masterkey + buildspec: codebuild/py312/decrypt_dafny_esdk_vectors_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_dafny_esdk_vectors_keyrings + buildspec: codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_net_401_vectors_masterkey + buildspec: codebuild/py312/decrypt_net_401_vectors_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_net_401_vectors_keyrings + buildspec: codebuild/py312/decrypt_net_401_vectors_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_encrypt_masterkey + buildspec: codebuild/py312/encrypt_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_encrypt_keyrings + buildspec: codebuild/py312/encrypt_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_generate_decrypt_vectors_masterkey + buildspec: codebuild/py312/generate_decrypt_vectors_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_masterkey_with_masterkey + depend-on: + - py312_generate_decrypt_vectors_masterkey + buildspec: codebuild/py312/decrypt_masterkey_with_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_masterkey_with_keyrings + depend-on: + - py312_generate_decrypt_vectors_masterkey + buildspec: codebuild/py312/decrypt_masterkey_with_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_masterkey_with_js + depend-on: + - py312_generate_decrypt_vectors_masterkey + buildspec: codebuild/py312/decrypt_masterkey_with_js.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_generate_decrypt_vectors_keyrings + buildspec: codebuild/py312/generate_decrypt_vectors_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_keyrings_with_masterkey + depend-on: + - py312_generate_decrypt_vectors_keyrings + buildspec: codebuild/py312/decrypt_keyrings_with_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_keyrings_with_keyrings + depend-on: + - py312_generate_decrypt_vectors_keyrings + buildspec: codebuild/py312/decrypt_keyrings_with_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_keyrings_with_js + depend-on: + - py312_generate_decrypt_vectors_keyrings + buildspec: codebuild/py312/decrypt_keyrings_with_js.yml + env: + image: aws/codebuild/standard:7.0 - # - identifier: py312_integ - # buildspec: codebuild/py312/integ.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_integ_mpl - # buildspec: codebuild/py312/integ_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_examples - # buildspec: codebuild/py312/examples.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_examples_mpl - # buildspec: codebuild/py312/examples_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_awses_latest - # buildspec: codebuild/py312/awses_local.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_awses_latest_mpl - # buildspec: codebuild/py312/awses_local_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_mplawses_latest_mpl - # buildspec: codebuild/py312/mplawses_local_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_dafny_esdk_vectors_masterkey - # buildspec: codebuild/py312/decrypt_dafny_esdk_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_dafny_esdk_vectors_keyrings - # buildspec: codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_net_401_vectors_masterkey - # buildspec: codebuild/py312/decrypt_net_401_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_net_401_vectors_keyrings - # buildspec: codebuild/py312/decrypt_net_401_vectors_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py312/generate_decrypt_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_masterkey_with_masterkey - # depend-on: - # - py312_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py312/decrypt_masterkey_with_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_masterkey_with_keyrings - # depend-on: - # - py312_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py312/decrypt_masterkey_with_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_masterkey_with_js - # depend-on: - # - py312_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py312/decrypt_masterkey_with_js.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py312/generate_decrypt_vectors_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_keyrings_with_masterkey - # depend-on: - # - py312_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py312/decrypt_keyrings_with_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_keyrings_with_keyrings - # depend-on: - # - py312_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py312/decrypt_keyrings_with_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_keyrings_with_js - # depend-on: - # - py312_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py312/decrypt_keyrings_with_js.yml - # env: - # image: aws/codebuild/standard:7.0 # # - identifier: code_coverage # buildspec: codebuild/coverage/coverage.yml From 1d7fcaee7dad7f31165108066c898b06a6bbe5c1 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 12:01:57 -0700 Subject: [PATCH 354/422] ALL --- .../py310/decrypt_dafny_esdk_vectors.yml | 58 ++++++++++++++++++ codebuild/py310/decrypt_masterkey_with_js.yml | 34 +++++++++++ .../decrypt_masterkey_with_masterkey.yml | 30 ++++++++++ codebuild/py310/decrypt_net_401_vectors.yml | 35 +++++++++++ codebuild/py310/encrypt_masterkey.yml | 25 ++++++++ codebuild/py310/examples copy.yml | 22 +++++++ .../generate_decrypt_vectors_masterkey.yml | 28 +++++++++ codebuild/py310/integ copy.yml | 22 +++++++ codebuild/py312/awses_local.yml | 25 ++++++++ codebuild/py312/awses_local_mpl.yml | 26 ++++++++ .../decrypt_dafny_esdk_vectors_keyrings.yml | 59 +++++++++++++++++++ .../decrypt_dafny_esdk_vectors_masterkey.yml | 58 ++++++++++++++++++ codebuild/py312/decrypt_keyrings_with_js.yml | 34 +++++++++++ codebuild/py38/decrypt_dafny_esdk_vectors.yml | 58 ++++++++++++++++++ codebuild/py38/decrypt_masterkey_with_js.yml | 34 +++++++++++ .../py38/decrypt_masterkey_with_masterkey.yml | 30 ++++++++++ codebuild/py38/decrypt_net_401_vectors.yml | 35 +++++++++++ codebuild/py38/encrypt_masterkey.yml | 25 ++++++++ .../generate_decrypt_vectors_masterkey.yml | 28 +++++++++ codebuild/py39/awses_local.yml | 25 ++++++++ codebuild/py39/decrypt_dafny_esdk_vectors.yml | 58 ++++++++++++++++++ codebuild/py39/decrypt_masterkey_with_js.yml | 34 +++++++++++ .../py39/decrypt_masterkey_with_masterkey.yml | 30 ++++++++++ codebuild/py39/decrypt_net_401_vectors.yml | 35 +++++++++++ codebuild/py39/encrypt_masterkey.yml | 25 ++++++++ codebuild/py39/examples copy.yml | 22 +++++++ .../generate_decrypt_vectors_masterkey.yml | 28 +++++++++ codebuild/py39/integ copy.yml | 22 +++++++ 28 files changed, 945 insertions(+) create mode 100644 codebuild/py310/decrypt_dafny_esdk_vectors.yml create mode 100644 codebuild/py310/decrypt_masterkey_with_js.yml create mode 100644 codebuild/py310/decrypt_masterkey_with_masterkey.yml create mode 100644 codebuild/py310/decrypt_net_401_vectors.yml create mode 100644 codebuild/py310/encrypt_masterkey.yml create mode 100644 codebuild/py310/examples copy.yml create mode 100644 codebuild/py310/generate_decrypt_vectors_masterkey.yml create mode 100644 codebuild/py310/integ copy.yml create mode 100644 codebuild/py312/awses_local.yml create mode 100644 codebuild/py312/awses_local_mpl.yml create mode 100644 codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml create mode 100644 codebuild/py312/decrypt_dafny_esdk_vectors_masterkey.yml create mode 100644 codebuild/py312/decrypt_keyrings_with_js.yml create mode 100644 codebuild/py38/decrypt_dafny_esdk_vectors.yml create mode 100644 codebuild/py38/decrypt_masterkey_with_js.yml create mode 100644 codebuild/py38/decrypt_masterkey_with_masterkey.yml create mode 100644 codebuild/py38/decrypt_net_401_vectors.yml create mode 100644 codebuild/py38/encrypt_masterkey.yml create mode 100644 codebuild/py38/generate_decrypt_vectors_masterkey.yml create mode 100644 codebuild/py39/awses_local.yml create mode 100644 codebuild/py39/decrypt_dafny_esdk_vectors.yml create mode 100644 codebuild/py39/decrypt_masterkey_with_js.yml create mode 100644 codebuild/py39/decrypt_masterkey_with_masterkey.yml create mode 100644 codebuild/py39/decrypt_net_401_vectors.yml create mode 100644 codebuild/py39/encrypt_masterkey.yml create mode 100644 codebuild/py39/examples copy.yml create mode 100644 codebuild/py39/generate_decrypt_vectors_masterkey.yml create mode 100644 codebuild/py39/integ copy.yml diff --git a/codebuild/py310/decrypt_dafny_esdk_vectors.yml b/codebuild/py310/decrypt_dafny_esdk_vectors.yml new file mode 100644 index 000000000..505f3157c --- /dev/null +++ b/codebuild/py310/decrypt_dafny_esdk_vectors.yml @@ -0,0 +1,58 @@ +version: 0.2 +# Runs Only the ESDK-NET v4.0.1 Decryption Vectors, testing Required EC CMM + +env: + variables: + TOXENV: "py310-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + git-credential-helper: yes + secrets-manager: + GITHUB_TOKEN: Github/lucasmcdonald3-fgpat:actions read + +phases: + install: + runtime-versions: + python: 3.10 + pre_build: + commands: + # Fetch test vectors from Dafny ESDK's most recent run + # (Assuming the first result is most recent; seems to be correct...) + - | + MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ + | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') + - | + echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" + - | + MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/$MOST_RECENT_RUN_ID/artifacts?name=ubuntu-latest_vector_artifact" \ + | jq '.artifacts[0].archive_download_url') + - | + echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" + - | + curl -L -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip + # This unzips to `net41.zip`. + - unzip ubuntu-latest_test_vector_artifact + # This unzips to `net41/`. + - unzip net41.zip -d net41 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../net41/manifest.json diff --git a/codebuild/py310/decrypt_masterkey_with_js.yml b/codebuild/py310/decrypt_masterkey_with_js.yml new file mode 100644 index 000000000..fdfb2363c --- /dev/null +++ b/codebuild/py310/decrypt_masterkey_with_js.yml @@ -0,0 +1,34 @@ +version: 0.2 + +env: + variables: + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.10 + commands: + - n 16 + # Install the Javascript ESDK run test vectors + - npm install -g @aws-crypto/integration-node + + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/310_masterkey.zip 310_masterkey.zip + # Repackage zip in expected format + - unzip 310_masterkey.zip + - cd 310_masterkey + - zip -r vectors.zip . + build: + commands: + # Decrypt generated vectors with Javascript ESDK + - integration-node decrypt -v vectors.zip \ No newline at end of file diff --git a/codebuild/py310/decrypt_masterkey_with_masterkey.yml b/codebuild/py310/decrypt_masterkey_with_masterkey.yml new file mode 100644 index 000000000..577e81b9a --- /dev/null +++ b/codebuild/py310/decrypt_masterkey_with_masterkey.yml @@ -0,0 +1,30 @@ +version: 0.2 + +env: + variables: + TOXENV: "py310-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.10 + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/310_masterkey.zip 310_masterkey.zip + - unzip 310_masterkey.zip + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../310_masterkey/manifest.json \ No newline at end of file diff --git a/codebuild/py310/decrypt_net_401_vectors.yml b/codebuild/py310/decrypt_net_401_vectors.yml new file mode 100644 index 000000000..82ac642d9 --- /dev/null +++ b/codebuild/py310/decrypt_net_401_vectors.yml @@ -0,0 +1,35 @@ +version: 0.2 +# Runs Only the ESDK-NET v4.0.1 Decryption Vectors, testing Required EC CMM + +env: + variables: + TOXENV: "py310-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.10 + pre_build: + commands: + # Fetch ESDK .NET v4.0.1 Test Vectors + - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip + - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip + - curl -s --output $VECTOR_ZIP --location $VECTORS_URL + - UNZIPPED_VECTORS_DIR=$CODEBUILD_SRC_DIR/test_vector_handlers/net_401_vectors + - unzip $VECTOR_ZIP -d $UNZIPPED_VECTORS_DIR + build: + commands: + # NOTE: We need to pass the absolute path of the vectors + - pip install "tox < 4.0" + - cd $CODEBUILD_SRC_DIR/test_vector_handlers + - | + tox -- \ + --input $UNZIPPED_VECTORS_DIR/manifest.json diff --git a/codebuild/py310/encrypt_masterkey.yml b/codebuild/py310/encrypt_masterkey.yml new file mode 100644 index 000000000..9cd89fb8f --- /dev/null +++ b/codebuild/py310/encrypt_masterkey.yml @@ -0,0 +1,25 @@ +version: 0.2 + +env: + variables: + TOXENV: "py310-full_encrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.10 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0003-awses-message-encryption.v2.json diff --git a/codebuild/py310/examples copy.yml b/codebuild/py310/examples copy.yml new file mode 100644 index 000000000..b495a327c --- /dev/null +++ b/codebuild/py310/examples copy.yml @@ -0,0 +1,22 @@ +version: 0.2 + +env: + variables: + TOXENV: "py310-examples" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.10 + build: + commands: + - pip install "tox < 4.0" + - tox diff --git a/codebuild/py310/generate_decrypt_vectors_masterkey.yml b/codebuild/py310/generate_decrypt_vectors_masterkey.yml new file mode 100644 index 000000000..640fb72d6 --- /dev/null +++ b/codebuild/py310/generate_decrypt_vectors_masterkey.yml @@ -0,0 +1,28 @@ +version: 0.2 + +env: + variables: + TOXENV: "py310-full_decrypt_generate" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.10 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ + --output 310_masterkey + - zip -r 310_masterkey.zip 310_masterkey + - aws s3 cp 310_masterkey.zip s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/310_masterkey.zip diff --git a/codebuild/py310/integ copy.yml b/codebuild/py310/integ copy.yml new file mode 100644 index 000000000..6b557e709 --- /dev/null +++ b/codebuild/py310/integ copy.yml @@ -0,0 +1,22 @@ +version: 0.2 + +env: + variables: + TOXENV: "py310-integ" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.10 + build: + commands: + - pip install "tox < 4.0" + - tox diff --git a/codebuild/py312/awses_local.yml b/codebuild/py312/awses_local.yml new file mode 100644 index 000000000..844cc7993 --- /dev/null +++ b/codebuild/py312/awses_local.yml @@ -0,0 +1,25 @@ +version: 0.2 + +env: + variables: + TOXENV: "py312-awses_local" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + python: 3.12 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - tox diff --git a/codebuild/py312/awses_local_mpl.yml b/codebuild/py312/awses_local_mpl.yml new file mode 100644 index 000000000..11f995c16 --- /dev/null +++ b/codebuild/py312/awses_local_mpl.yml @@ -0,0 +1,26 @@ +version: 0.2 + +env: + variables: + TOXENV: "py312-awses_local-mpl" + REGION: "us-west-2" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + python: 3.12 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - tox diff --git a/codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml b/codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml new file mode 100644 index 000000000..810d16b74 --- /dev/null +++ b/codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml @@ -0,0 +1,59 @@ +version: 0.2 +# Runs Only the ESDK-NET v4.0.1 Decryption Vectors, testing Required EC CMM + +env: + variables: + TOXENV: "py312-full_decrypt-mpl" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + git-credential-helper: yes + secrets-manager: + GITHUB_TOKEN: Github/lucasmcdonald3-fgpat:actions read + +phases: + install: + runtime-versions: + python: 3.12 + pre_build: + commands: + # Fetch test vectors from Dafny ESDK's most recent run + # (Assuming the first result is most recent; seems to be correct...) + - | + MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ + | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') + - | + echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" + - | + MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/$MOST_RECENT_RUN_ID/artifacts?name=ubuntu-latest_vector_artifact" \ + | jq '.artifacts[0].archive_download_url') + - | + echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" + - | + curl -L -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip + # This unzips to `net41.zip`. + - unzip ubuntu-latest_test_vector_artifact + # This unzips to `net41/`. + - unzip net41.zip -d net41 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../net41/manifest.json \ + --keyrings diff --git a/codebuild/py312/decrypt_dafny_esdk_vectors_masterkey.yml b/codebuild/py312/decrypt_dafny_esdk_vectors_masterkey.yml new file mode 100644 index 000000000..b375651c5 --- /dev/null +++ b/codebuild/py312/decrypt_dafny_esdk_vectors_masterkey.yml @@ -0,0 +1,58 @@ +version: 0.2 +# Runs Only the ESDK-NET v4.0.1 Decryption Vectors, testing Required EC CMM + +env: + variables: + TOXENV: "py312-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + git-credential-helper: yes + secrets-manager: + GITHUB_TOKEN: Github/lucasmcdonald3-fgpat:actions read + +phases: + install: + runtime-versions: + python: 3.12 + pre_build: + commands: + # Fetch test vectors from Dafny ESDK's most recent run + # (Assuming the first result is most recent; seems to be correct...) + - | + MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ + | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') + - | + echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" + - | + MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/$MOST_RECENT_RUN_ID/artifacts?name=ubuntu-latest_vector_artifact" \ + | jq '.artifacts[0].archive_download_url') + - | + echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" + - | + curl -L -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip + # This unzips to `net41.zip`. + - unzip ubuntu-latest_test_vector_artifact + # This unzips to `net41/`. + - unzip net41.zip -d net41 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../net41/manifest.json diff --git a/codebuild/py312/decrypt_keyrings_with_js.yml b/codebuild/py312/decrypt_keyrings_with_js.yml new file mode 100644 index 000000000..9b1ebc270 --- /dev/null +++ b/codebuild/py312/decrypt_keyrings_with_js.yml @@ -0,0 +1,34 @@ +version: 0.2 + +env: + variables: + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.12 + commands: + - n 16 + # Install the Javascript ESDK run test vectors + - npm install -g @aws-crypto/integration-node + + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_keyrings.zip 312_keyrings.zip + # Repackage zip in expected format + - unzip 312_keyrings.zip + - cd 312_keyrings + - zip -r vectors.zip . + build: + commands: + # Decrypt generated vectors with Javascript ESDK + - integration-node decrypt -v vectors.zip \ No newline at end of file diff --git a/codebuild/py38/decrypt_dafny_esdk_vectors.yml b/codebuild/py38/decrypt_dafny_esdk_vectors.yml new file mode 100644 index 000000000..968a74690 --- /dev/null +++ b/codebuild/py38/decrypt_dafny_esdk_vectors.yml @@ -0,0 +1,58 @@ +version: 0.2 +# Runs Only the ESDK-NET v4.0.1 Decryption Vectors, testing Required EC CMM + +env: + variables: + TOXENV: "py38-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + git-credential-helper: yes + secrets-manager: + GITHUB_TOKEN: Github/lucasmcdonald3-fgpat:actions read + +phases: + install: + runtime-versions: + python: 3.8 + pre_build: + commands: + # Fetch test vectors from Dafny ESDK's most recent run + # (Assuming the first result is most recent; seems to be correct...) + - | + MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ + | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') + - | + echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" + - | + MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/$MOST_RECENT_RUN_ID/artifacts?name=ubuntu-latest_vector_artifact" \ + | jq '.artifacts[0].archive_download_url') + - | + echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" + - | + curl -L -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip + # This unzips to `net41.zip`. + - unzip ubuntu-latest_test_vector_artifact + # This unzips to `net41/`. + - unzip net41.zip -d net41 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../net41/manifest.json diff --git a/codebuild/py38/decrypt_masterkey_with_js.yml b/codebuild/py38/decrypt_masterkey_with_js.yml new file mode 100644 index 000000000..953e8818a --- /dev/null +++ b/codebuild/py38/decrypt_masterkey_with_js.yml @@ -0,0 +1,34 @@ +version: 0.2 + +env: + variables: + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.8 + commands: + - n 16 + # Install the Javascript ESDK run test vectors + - npm install -g @aws-crypto/integration-node + + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/38_masterkey.zip 38_masterkey.zip + # Repackage zip in expected format + - unzip 38_masterkey.zip + - cd 38_masterkey + - zip -r vectors.zip . + build: + commands: + # Decrypt generated vectors with Javascript ESDK + - integration-node decrypt -v vectors.zip \ No newline at end of file diff --git a/codebuild/py38/decrypt_masterkey_with_masterkey.yml b/codebuild/py38/decrypt_masterkey_with_masterkey.yml new file mode 100644 index 000000000..6b32dcf15 --- /dev/null +++ b/codebuild/py38/decrypt_masterkey_with_masterkey.yml @@ -0,0 +1,30 @@ +version: 0.2 + +env: + variables: + TOXENV: "py38-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.8 + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/38_masterkey.zip 38_masterkey.zip + - unzip 38_masterkey.zip + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../38_masterkey/manifest.json \ No newline at end of file diff --git a/codebuild/py38/decrypt_net_401_vectors.yml b/codebuild/py38/decrypt_net_401_vectors.yml new file mode 100644 index 000000000..298711975 --- /dev/null +++ b/codebuild/py38/decrypt_net_401_vectors.yml @@ -0,0 +1,35 @@ +version: 0.2 +# Runs Only the ESDK-NET v4.0.1 Decryption Vectors, testing Required EC CMM + +env: + variables: + TOXENV: "py38-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.8 + pre_build: + commands: + # Fetch ESDK .NET v4.0.1 Test Vectors + - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip + - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip + - curl -s --output $VECTOR_ZIP --location $VECTORS_URL + - UNZIPPED_VECTORS_DIR=$CODEBUILD_SRC_DIR/test_vector_handlers/net_401_vectors + - unzip $VECTOR_ZIP -d $UNZIPPED_VECTORS_DIR + build: + commands: + # NOTE: We need to pass the absolute path of the vectors + - pip install "tox < 4.0" + - cd $CODEBUILD_SRC_DIR/test_vector_handlers + - | + tox -- \ + --input $UNZIPPED_VECTORS_DIR/manifest.json diff --git a/codebuild/py38/encrypt_masterkey.yml b/codebuild/py38/encrypt_masterkey.yml new file mode 100644 index 000000000..b05396cc2 --- /dev/null +++ b/codebuild/py38/encrypt_masterkey.yml @@ -0,0 +1,25 @@ +version: 0.2 + +env: + variables: + TOXENV: "py38-full_encrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.8 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0003-awses-message-encryption.v2.json diff --git a/codebuild/py38/generate_decrypt_vectors_masterkey.yml b/codebuild/py38/generate_decrypt_vectors_masterkey.yml new file mode 100644 index 000000000..8705ef57c --- /dev/null +++ b/codebuild/py38/generate_decrypt_vectors_masterkey.yml @@ -0,0 +1,28 @@ +version: 0.2 + +env: + variables: + TOXENV: "py38-full_decrypt_generate" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.8 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ + --output 38_masterkey + - zip -r 38_masterkey.zip 38_masterkey + - aws s3 cp 38_masterkey.zip s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/38_masterkey.zip diff --git a/codebuild/py39/awses_local.yml b/codebuild/py39/awses_local.yml new file mode 100644 index 000000000..e56a9ff45 --- /dev/null +++ b/codebuild/py39/awses_local.yml @@ -0,0 +1,25 @@ +version: 0.2 + +env: + variables: + TOXENV: "py39-awses_local" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" + AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" + +phases: + install: + runtime-versions: + python: 3.9 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - tox diff --git a/codebuild/py39/decrypt_dafny_esdk_vectors.yml b/codebuild/py39/decrypt_dafny_esdk_vectors.yml new file mode 100644 index 000000000..ddb50db1c --- /dev/null +++ b/codebuild/py39/decrypt_dafny_esdk_vectors.yml @@ -0,0 +1,58 @@ +version: 0.2 +# Runs Only the ESDK-NET v4.0.1 Decryption Vectors, testing Required EC CMM + +env: + variables: + TOXENV: "py39-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + git-credential-helper: yes + secrets-manager: + GITHUB_TOKEN: Github/lucasmcdonald3-fgpat:actions read + +phases: + install: + runtime-versions: + python: 3.9 + pre_build: + commands: + # Fetch test vectors from Dafny ESDK's most recent run + # (Assuming the first result is most recent; seems to be correct...) + - | + MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ + | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') + - | + echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" + - | + MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/$MOST_RECENT_RUN_ID/artifacts?name=ubuntu-latest_vector_artifact" \ + | jq '.artifacts[0].archive_download_url') + - | + echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" + - | + curl -L -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip + # This unzips to `net41.zip`. + - unzip ubuntu-latest_test_vector_artifact + # This unzips to `net41/`. + - unzip net41.zip -d net41 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../net41/manifest.json diff --git a/codebuild/py39/decrypt_masterkey_with_js.yml b/codebuild/py39/decrypt_masterkey_with_js.yml new file mode 100644 index 000000000..53f6433f8 --- /dev/null +++ b/codebuild/py39/decrypt_masterkey_with_js.yml @@ -0,0 +1,34 @@ +version: 0.2 + +env: + variables: + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.9 + commands: + - n 16 + # Install the Javascript ESDK run test vectors + - npm install -g @aws-crypto/integration-node + + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/39_masterkey.zip 39_masterkey.zip + # Repackage zip in expected format + - unzip 39_masterkey.zip + - cd 39_masterkey + - zip -r vectors.zip . + build: + commands: + # Decrypt generated vectors with Javascript ESDK + - integration-node decrypt -v vectors.zip \ No newline at end of file diff --git a/codebuild/py39/decrypt_masterkey_with_masterkey.yml b/codebuild/py39/decrypt_masterkey_with_masterkey.yml new file mode 100644 index 000000000..fcd9d3220 --- /dev/null +++ b/codebuild/py39/decrypt_masterkey_with_masterkey.yml @@ -0,0 +1,30 @@ +version: 0.2 + +env: + variables: + TOXENV: "py39-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.9 + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/39_masterkey.zip 39_masterkey.zip + - unzip 39_masterkey.zip + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../39_masterkey/manifest.json \ No newline at end of file diff --git a/codebuild/py39/decrypt_net_401_vectors.yml b/codebuild/py39/decrypt_net_401_vectors.yml new file mode 100644 index 000000000..635abc95b --- /dev/null +++ b/codebuild/py39/decrypt_net_401_vectors.yml @@ -0,0 +1,35 @@ +version: 0.2 +# Runs Only the ESDK-NET v4.0.1 Decryption Vectors, testing Required EC CMM + +env: + variables: + TOXENV: "py39-full_decrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.9 + pre_build: + commands: + # Fetch ESDK .NET v4.0.1 Test Vectors + - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip + - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip + - curl -s --output $VECTOR_ZIP --location $VECTORS_URL + - UNZIPPED_VECTORS_DIR=$CODEBUILD_SRC_DIR/test_vector_handlers/net_401_vectors + - unzip $VECTOR_ZIP -d $UNZIPPED_VECTORS_DIR + build: + commands: + # NOTE: We need to pass the absolute path of the vectors + - pip install "tox < 4.0" + - cd $CODEBUILD_SRC_DIR/test_vector_handlers + - | + tox -- \ + --input $UNZIPPED_VECTORS_DIR/manifest.json diff --git a/codebuild/py39/encrypt_masterkey.yml b/codebuild/py39/encrypt_masterkey.yml new file mode 100644 index 000000000..3bf18fbde --- /dev/null +++ b/codebuild/py39/encrypt_masterkey.yml @@ -0,0 +1,25 @@ +version: 0.2 + +env: + variables: + TOXENV: "py39-full_encrypt" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.9 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0003-awses-message-encryption.v2.json diff --git a/codebuild/py39/examples copy.yml b/codebuild/py39/examples copy.yml new file mode 100644 index 000000000..3d1399251 --- /dev/null +++ b/codebuild/py39/examples copy.yml @@ -0,0 +1,22 @@ +version: 0.2 + +env: + variables: + TOXENV: "py39-examples" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.9 + build: + commands: + - pip install "tox < 4.0" + - tox diff --git a/codebuild/py39/generate_decrypt_vectors_masterkey.yml b/codebuild/py39/generate_decrypt_vectors_masterkey.yml new file mode 100644 index 000000000..eb57d915a --- /dev/null +++ b/codebuild/py39/generate_decrypt_vectors_masterkey.yml @@ -0,0 +1,28 @@ +version: 0.2 + +env: + variables: + TOXENV: "py39-full_decrypt_generate" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.9 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ + --output 39_masterkey + - zip -r 39_masterkey.zip 39_masterkey + - aws s3 cp 39_masterkey.zip s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/39_masterkey.zip diff --git a/codebuild/py39/integ copy.yml b/codebuild/py39/integ copy.yml new file mode 100644 index 000000000..6dec85b07 --- /dev/null +++ b/codebuild/py39/integ copy.yml @@ -0,0 +1,22 @@ +version: 0.2 + +env: + variables: + TOXENV: "py39-integ" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.9 + build: + commands: + - pip install "tox < 4.0" + - tox From bdacdeb455ce1fbe5d44e550054bb9689a8171ad Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 12:06:23 -0700 Subject: [PATCH 355/422] ALL --- buildspec.yml | 94 ++++++++++++++++++++++++++++--- codebuild/py310/examples copy.yml | 22 -------- codebuild/py310/integ copy.yml | 22 -------- codebuild/py39/examples copy.yml | 22 -------- codebuild/py39/integ copy.yml | 22 -------- 5 files changed, 85 insertions(+), 97 deletions(-) delete mode 100644 codebuild/py310/examples copy.yml delete mode 100644 codebuild/py310/integ copy.yml delete mode 100644 codebuild/py39/examples copy.yml delete mode 100644 codebuild/py39/integ copy.yml diff --git a/buildspec.yml b/buildspec.yml index e303d9b35..873e5941e 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -4,6 +4,44 @@ batch: fast-fail: false build-graph: + # 3.7 + - identifier: py37_integ + buildspec: codebuild/py37/integ.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py37_examples + buildspec: codebuild/py37/examples.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py37_decrypt_dafny_esdk_vectors + buildspec: codebuild/py37/decrypt_dafny_esdk_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py37_decrypt_net_401_vectors + buildspec: codebuild/py37/decrypt_net_401_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py37_encrypt_masterkey + buildspec: codebuild/py37/encrypt_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py37_generate_decrypt_vectors_masterkey + buildspec: codebuild/py37/generate_decrypt_vectors_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py37_decrypt_masterkey_with_masterkey + depend-on: + - py37_generate_decrypt_vectors_masterkey + buildspec: codebuild/py37/decrypt_masterkey_with_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py37_decrypt_masterkey_with_js + depend-on: + - py37_generate_decrypt_vectors_masterkey + buildspec: codebuild/py37/decrypt_masterkey_with_js.yml + env: + image: aws/codebuild/standard:5.0 + # 3.8 - identifier: py38_integ buildspec: codebuild/py38/integ.yml @@ -80,6 +118,44 @@ batch: env: image: aws/codebuild/standard:5.0 + # 3.10 + - identifier: py310_integ + buildspec: codebuild/py310/integ.yml + env: + image: aws/codebuild/standard:6.0 + - identifier: py310_examples + buildspec: codebuild/py310/examples.yml + env: + image: aws/codebuild/standard:6.0 + - identifier: py310_decrypt_dafny_esdk_vectors + buildspec: codebuild/py310/decrypt_dafny_esdk_vectors.yml + env: + image: aws/codebuild/standard:6.0 + - identifier: py310_decrypt_net_401_vectors + buildspec: codebuild/py310/decrypt_net_401_vectors.yml + env: + image: aws/codebuild/standard:6.0 + - identifier: py310_encrypt_masterkey + buildspec: codebuild/py310/encrypt_masterkey.yml + env: + image: aws/codebuild/standard:6.0 + - identifier: py310_generate_decrypt_vectors_masterkey + buildspec: codebuild/py310/generate_decrypt_vectors_masterkey.yml + env: + image: aws/codebuild/standard:6.0 + - identifier: py310_decrypt_masterkey_with_masterkey + depend-on: + - py310_generate_decrypt_vectors_masterkey + buildspec: codebuild/py310/decrypt_masterkey_with_masterkey.yml + env: + image: aws/codebuild/standard:6.0 + - identifier: py310_decrypt_masterkey_with_js + depend-on: + - py310_generate_decrypt_vectors_masterkey + buildspec: codebuild/py310/decrypt_masterkey_with_js.yml + env: + image: aws/codebuild/standard:6.0 + - identifier: py311_integ buildspec: codebuild/py311/integ.yml env: @@ -251,13 +327,13 @@ batch: env: image: aws/codebuild/standard:7.0 - # - # - identifier: code_coverage - # buildspec: codebuild/coverage/coverage.yml - # - identifier: code_coverage_mpl - # buildspec: codebuild/coverage/coverage_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 + + - identifier: code_coverage + buildspec: codebuild/coverage/coverage.yml + - identifier: code_coverage_mpl + buildspec: codebuild/coverage/coverage_mpl.yml + env: + image: aws/codebuild/standard:7.0 - # - identifier: compliance - # buildspec: codebuild/compliance/compliance.yml + - identifier: compliance + buildspec: codebuild/compliance/compliance.yml diff --git a/codebuild/py310/examples copy.yml b/codebuild/py310/examples copy.yml deleted file mode 100644 index b495a327c..000000000 --- a/codebuild/py310/examples copy.yml +++ /dev/null @@ -1,22 +0,0 @@ -version: 0.2 - -env: - variables: - TOXENV: "py310-examples" - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- - arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- - arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- - arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- - arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - -phases: - install: - runtime-versions: - python: 3.10 - build: - commands: - - pip install "tox < 4.0" - - tox diff --git a/codebuild/py310/integ copy.yml b/codebuild/py310/integ copy.yml deleted file mode 100644 index 6b557e709..000000000 --- a/codebuild/py310/integ copy.yml +++ /dev/null @@ -1,22 +0,0 @@ -version: 0.2 - -env: - variables: - TOXENV: "py310-integ" - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- - arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- - arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- - arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- - arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - -phases: - install: - runtime-versions: - python: 3.10 - build: - commands: - - pip install "tox < 4.0" - - tox diff --git a/codebuild/py39/examples copy.yml b/codebuild/py39/examples copy.yml deleted file mode 100644 index 3d1399251..000000000 --- a/codebuild/py39/examples copy.yml +++ /dev/null @@ -1,22 +0,0 @@ -version: 0.2 - -env: - variables: - TOXENV: "py39-examples" - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- - arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- - arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- - arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- - arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - -phases: - install: - runtime-versions: - python: 3.9 - build: - commands: - - pip install "tox < 4.0" - - tox diff --git a/codebuild/py39/integ copy.yml b/codebuild/py39/integ copy.yml deleted file mode 100644 index 6dec85b07..000000000 --- a/codebuild/py39/integ copy.yml +++ /dev/null @@ -1,22 +0,0 @@ -version: 0.2 - -env: - variables: - TOXENV: "py39-integ" - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- - arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- - arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- - arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- - arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - -phases: - install: - runtime-versions: - python: 3.9 - build: - commands: - - pip install "tox < 4.0" - - tox From 27838089d0ebe32b78366dd74f55b650e76c02f2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 12:18:47 -0700 Subject: [PATCH 356/422] gen decrypt keyrings --- codebuild/py311/generate_decrypt_vectors_keyrings.yml | 3 ++- codebuild/py312/generate_decrypt_vectors_keyrings.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml index 777a5703f..179ec0f12 100644 --- a/codebuild/py311/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -23,6 +23,7 @@ phases: - | tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ - --output 311_keyrings + --output 311_keyrings \ + --keyrings - zip -r 311_keyrings.zip 311_keyrings - aws s3 cp 311_keyrings.zip s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyrings.zip diff --git a/codebuild/py312/generate_decrypt_vectors_keyrings.yml b/codebuild/py312/generate_decrypt_vectors_keyrings.yml index 51a1415ee..ae79b86ce 100644 --- a/codebuild/py312/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py312/generate_decrypt_vectors_keyrings.yml @@ -23,6 +23,7 @@ phases: - | tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0006-awses-message-decryption-generation.v2.json \ - --output 312_keyrings + --output 312_keyrings \ + --keyrings - zip -r 312_keyrings.zip 312_keyrings - aws s3 cp 312_keyrings.zip s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_keyrings.zip From f690cf7dfca9a18301711aa9ab6e40ac727f07e9 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 15:11:30 -0700 Subject: [PATCH 357/422] cleanup --- buildspec.yml | 574 +++++++++--------- .../materials_managers/mpl/cmm.py | 16 - .../materials_managers/mpl/materials.py | 2 - src/aws_encryption_sdk/streaming_client.py | 1 - .../internal/tampering_mpl_materials.py | 9 +- .../manifests/full_message/decrypt.py | 21 +- .../full_message/decrypt_generation.py | 10 +- .../manifests/full_message/encrypt.py | 139 ----- .../commands/test_i_full_message_encrypt.py} | 24 +- .../integration/integration_test_utils.py | 35 +- test_vector_handlers/tox.ini | 5 + 11 files changed, 322 insertions(+), 514 deletions(-) rename test_vector_handlers/test/{mpl/integration/commands/test_i_encrypt_keyrings.py => integration/commands/test_i_full_message_encrypt.py} (62%) diff --git a/buildspec.yml b/buildspec.yml index 873e5941e..90b5dbfd2 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -42,298 +42,298 @@ batch: env: image: aws/codebuild/standard:5.0 - # 3.8 - - identifier: py38_integ - buildspec: codebuild/py38/integ.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py38_examples - buildspec: codebuild/py38/examples.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py38_decrypt_dafny_esdk_vectors - buildspec: codebuild/py38/decrypt_dafny_esdk_vectors.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py38_decrypt_net_401_vectors - buildspec: codebuild/py38/decrypt_net_401_vectors.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py38_encrypt_masterkey - buildspec: codebuild/py38/encrypt_masterkey.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py38_generate_decrypt_vectors_masterkey - buildspec: codebuild/py38/generate_decrypt_vectors_masterkey.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py38_decrypt_masterkey_with_masterkey - depend-on: - - py38_generate_decrypt_vectors_masterkey - buildspec: codebuild/py38/decrypt_masterkey_with_masterkey.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py38_decrypt_masterkey_with_js - depend-on: - - py38_generate_decrypt_vectors_masterkey - buildspec: codebuild/py38/decrypt_masterkey_with_js.yml - env: - image: aws/codebuild/standard:5.0 + # # 3.8 + # - identifier: py38_integ + # buildspec: codebuild/py38/integ.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py38_examples + # buildspec: codebuild/py38/examples.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py38_decrypt_dafny_esdk_vectors + # buildspec: codebuild/py38/decrypt_dafny_esdk_vectors.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py38_decrypt_net_401_vectors + # buildspec: codebuild/py38/decrypt_net_401_vectors.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py38_encrypt_masterkey + # buildspec: codebuild/py38/encrypt_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py38_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py38/generate_decrypt_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py38_decrypt_masterkey_with_masterkey + # depend-on: + # - py38_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py38/decrypt_masterkey_with_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py38_decrypt_masterkey_with_js + # depend-on: + # - py38_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py38/decrypt_masterkey_with_js.yml + # env: + # image: aws/codebuild/standard:5.0 - # 3.9 - - identifier: py39_integ - buildspec: codebuild/py39/integ.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py39_examples - buildspec: codebuild/py39/examples.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py39_decrypt_dafny_esdk_vectors - buildspec: codebuild/py39/decrypt_dafny_esdk_vectors.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py39_decrypt_net_401_vectors - buildspec: codebuild/py39/decrypt_net_401_vectors.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py39_encrypt_masterkey - buildspec: codebuild/py39/encrypt_masterkey.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py39_generate_decrypt_vectors_masterkey - buildspec: codebuild/py39/generate_decrypt_vectors_masterkey.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py39_decrypt_masterkey_with_masterkey - depend-on: - - py39_generate_decrypt_vectors_masterkey - buildspec: codebuild/py39/decrypt_masterkey_with_masterkey.yml - env: - image: aws/codebuild/standard:5.0 - - identifier: py39_decrypt_masterkey_with_js - depend-on: - - py39_generate_decrypt_vectors_masterkey - buildspec: codebuild/py39/decrypt_masterkey_with_js.yml - env: - image: aws/codebuild/standard:5.0 + # # 3.9 + # - identifier: py39_integ + # buildspec: codebuild/py39/integ.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py39_examples + # buildspec: codebuild/py39/examples.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py39_decrypt_dafny_esdk_vectors + # buildspec: codebuild/py39/decrypt_dafny_esdk_vectors.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py39_decrypt_net_401_vectors + # buildspec: codebuild/py39/decrypt_net_401_vectors.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py39_encrypt_masterkey + # buildspec: codebuild/py39/encrypt_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py39_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py39/generate_decrypt_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py39_decrypt_masterkey_with_masterkey + # depend-on: + # - py39_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py39/decrypt_masterkey_with_masterkey.yml + # env: + # image: aws/codebuild/standard:5.0 + # - identifier: py39_decrypt_masterkey_with_js + # depend-on: + # - py39_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py39/decrypt_masterkey_with_js.yml + # env: + # image: aws/codebuild/standard:5.0 - # 3.10 - - identifier: py310_integ - buildspec: codebuild/py310/integ.yml - env: - image: aws/codebuild/standard:6.0 - - identifier: py310_examples - buildspec: codebuild/py310/examples.yml - env: - image: aws/codebuild/standard:6.0 - - identifier: py310_decrypt_dafny_esdk_vectors - buildspec: codebuild/py310/decrypt_dafny_esdk_vectors.yml - env: - image: aws/codebuild/standard:6.0 - - identifier: py310_decrypt_net_401_vectors - buildspec: codebuild/py310/decrypt_net_401_vectors.yml - env: - image: aws/codebuild/standard:6.0 - - identifier: py310_encrypt_masterkey - buildspec: codebuild/py310/encrypt_masterkey.yml - env: - image: aws/codebuild/standard:6.0 - - identifier: py310_generate_decrypt_vectors_masterkey - buildspec: codebuild/py310/generate_decrypt_vectors_masterkey.yml - env: - image: aws/codebuild/standard:6.0 - - identifier: py310_decrypt_masterkey_with_masterkey - depend-on: - - py310_generate_decrypt_vectors_masterkey - buildspec: codebuild/py310/decrypt_masterkey_with_masterkey.yml - env: - image: aws/codebuild/standard:6.0 - - identifier: py310_decrypt_masterkey_with_js - depend-on: - - py310_generate_decrypt_vectors_masterkey - buildspec: codebuild/py310/decrypt_masterkey_with_js.yml - env: - image: aws/codebuild/standard:6.0 + # # 3.10 + # - identifier: py310_integ + # buildspec: codebuild/py310/integ.yml + # env: + # image: aws/codebuild/standard:6.0 + # - identifier: py310_examples + # buildspec: codebuild/py310/examples.yml + # env: + # image: aws/codebuild/standard:6.0 + # - identifier: py310_decrypt_dafny_esdk_vectors + # buildspec: codebuild/py310/decrypt_dafny_esdk_vectors.yml + # env: + # image: aws/codebuild/standard:6.0 + # - identifier: py310_decrypt_net_401_vectors + # buildspec: codebuild/py310/decrypt_net_401_vectors.yml + # env: + # image: aws/codebuild/standard:6.0 + # - identifier: py310_encrypt_masterkey + # buildspec: codebuild/py310/encrypt_masterkey.yml + # env: + # image: aws/codebuild/standard:6.0 + # - identifier: py310_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py310/generate_decrypt_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:6.0 + # - identifier: py310_decrypt_masterkey_with_masterkey + # depend-on: + # - py310_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py310/decrypt_masterkey_with_masterkey.yml + # env: + # image: aws/codebuild/standard:6.0 + # - identifier: py310_decrypt_masterkey_with_js + # depend-on: + # - py310_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py310/decrypt_masterkey_with_js.yml + # env: + # image: aws/codebuild/standard:6.0 - - identifier: py311_integ - buildspec: codebuild/py311/integ.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_integ_mpl - buildspec: codebuild/py311/integ_mpl.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_examples - buildspec: codebuild/py311/examples.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_examples_mpl - buildspec: codebuild/py311/examples_mpl.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_dafny_esdk_vectors_masterkey - buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_dafny_esdk_vectors_keyrings - buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_net_401_vectors_masterkey - buildspec: codebuild/py311/decrypt_net_401_vectors_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_net_401_vectors_keyrings - buildspec: codebuild/py311/decrypt_net_401_vectors_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_encrypt_masterkey - buildspec: codebuild/py311/encrypt_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_encrypt_keyrings - buildspec: codebuild/py311/encrypt_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_generate_decrypt_vectors_masterkey - buildspec: codebuild/py311/generate_decrypt_vectors_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_masterkey_with_masterkey - depend-on: - - py311_generate_decrypt_vectors_masterkey - buildspec: codebuild/py311/decrypt_masterkey_with_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_masterkey_with_keyrings - depend-on: - - py311_generate_decrypt_vectors_masterkey - buildspec: codebuild/py311/decrypt_masterkey_with_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_masterkey_with_js - depend-on: - - py311_generate_decrypt_vectors_masterkey - buildspec: codebuild/py311/decrypt_masterkey_with_js.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_generate_decrypt_vectors_keyrings - buildspec: codebuild/py311/generate_decrypt_vectors_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_keyrings_with_masterkey - depend-on: - - py311_generate_decrypt_vectors_keyrings - buildspec: codebuild/py311/decrypt_keyrings_with_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_keyrings_with_keyrings - depend-on: - - py311_generate_decrypt_vectors_keyrings - buildspec: codebuild/py311/decrypt_keyrings_with_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py311_decrypt_keyrings_with_js - depend-on: - - py311_generate_decrypt_vectors_keyrings - buildspec: codebuild/py311/decrypt_keyrings_with_js.yml - env: - image: aws/codebuild/standard:7.0 + # - identifier: py311_integ + # buildspec: codebuild/py311/integ.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_integ_mpl + # buildspec: codebuild/py311/integ_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_examples + # buildspec: codebuild/py311/examples.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_examples_mpl + # buildspec: codebuild/py311/examples_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_dafny_esdk_vectors_masterkey + # buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_dafny_esdk_vectors_keyrings + # buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_net_401_vectors_masterkey + # buildspec: codebuild/py311/decrypt_net_401_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_net_401_vectors_keyrings + # buildspec: codebuild/py311/decrypt_net_401_vectors_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_encrypt_masterkey + # buildspec: codebuild/py311/encrypt_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_encrypt_keyrings + # buildspec: codebuild/py311/encrypt_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py311/generate_decrypt_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_masterkey_with_masterkey + # depend-on: + # - py311_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py311/decrypt_masterkey_with_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_masterkey_with_keyrings + # depend-on: + # - py311_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py311/decrypt_masterkey_with_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_masterkey_with_js + # depend-on: + # - py311_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py311/decrypt_masterkey_with_js.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py311/generate_decrypt_vectors_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_keyrings_with_masterkey + # depend-on: + # - py311_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py311/decrypt_keyrings_with_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_keyrings_with_keyrings + # depend-on: + # - py311_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py311/decrypt_keyrings_with_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py311_decrypt_keyrings_with_js + # depend-on: + # - py311_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py311/decrypt_keyrings_with_js.yml + # env: + # image: aws/codebuild/standard:7.0 - - identifier: py312_integ - buildspec: codebuild/py312/integ.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_integ_mpl - buildspec: codebuild/py312/integ_mpl.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_examples - buildspec: codebuild/py312/examples.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_examples_mpl - buildspec: codebuild/py312/examples_mpl.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_decrypt_dafny_esdk_vectors_masterkey - buildspec: codebuild/py312/decrypt_dafny_esdk_vectors_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_decrypt_dafny_esdk_vectors_keyrings - buildspec: codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_decrypt_net_401_vectors_masterkey - buildspec: codebuild/py312/decrypt_net_401_vectors_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_decrypt_net_401_vectors_keyrings - buildspec: codebuild/py312/decrypt_net_401_vectors_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_encrypt_masterkey - buildspec: codebuild/py312/encrypt_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_encrypt_keyrings - buildspec: codebuild/py312/encrypt_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_generate_decrypt_vectors_masterkey - buildspec: codebuild/py312/generate_decrypt_vectors_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_decrypt_masterkey_with_masterkey - depend-on: - - py312_generate_decrypt_vectors_masterkey - buildspec: codebuild/py312/decrypt_masterkey_with_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_decrypt_masterkey_with_keyrings - depend-on: - - py312_generate_decrypt_vectors_masterkey - buildspec: codebuild/py312/decrypt_masterkey_with_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_decrypt_masterkey_with_js - depend-on: - - py312_generate_decrypt_vectors_masterkey - buildspec: codebuild/py312/decrypt_masterkey_with_js.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_generate_decrypt_vectors_keyrings - buildspec: codebuild/py312/generate_decrypt_vectors_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_decrypt_keyrings_with_masterkey - depend-on: - - py312_generate_decrypt_vectors_keyrings - buildspec: codebuild/py312/decrypt_keyrings_with_masterkey.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_decrypt_keyrings_with_keyrings - depend-on: - - py312_generate_decrypt_vectors_keyrings - buildspec: codebuild/py312/decrypt_keyrings_with_keyrings.yml - env: - image: aws/codebuild/standard:7.0 - - identifier: py312_decrypt_keyrings_with_js - depend-on: - - py312_generate_decrypt_vectors_keyrings - buildspec: codebuild/py312/decrypt_keyrings_with_js.yml - env: - image: aws/codebuild/standard:7.0 + # - identifier: py312_integ + # buildspec: codebuild/py312/integ.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_integ_mpl + # buildspec: codebuild/py312/integ_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_examples + # buildspec: codebuild/py312/examples.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_examples_mpl + # buildspec: codebuild/py312/examples_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_decrypt_dafny_esdk_vectors_masterkey + # buildspec: codebuild/py312/decrypt_dafny_esdk_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_decrypt_dafny_esdk_vectors_keyrings + # buildspec: codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_decrypt_net_401_vectors_masterkey + # buildspec: codebuild/py312/decrypt_net_401_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_decrypt_net_401_vectors_keyrings + # buildspec: codebuild/py312/decrypt_net_401_vectors_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_encrypt_masterkey + # buildspec: codebuild/py312/encrypt_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_encrypt_keyrings + # buildspec: codebuild/py312/encrypt_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py312/generate_decrypt_vectors_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_decrypt_masterkey_with_masterkey + # depend-on: + # - py312_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py312/decrypt_masterkey_with_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_decrypt_masterkey_with_keyrings + # depend-on: + # - py312_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py312/decrypt_masterkey_with_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_decrypt_masterkey_with_js + # depend-on: + # - py312_generate_decrypt_vectors_masterkey + # buildspec: codebuild/py312/decrypt_masterkey_with_js.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py312/generate_decrypt_vectors_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_decrypt_keyrings_with_masterkey + # depend-on: + # - py312_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py312/decrypt_keyrings_with_masterkey.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_decrypt_keyrings_with_keyrings + # depend-on: + # - py312_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py312/decrypt_keyrings_with_keyrings.yml + # env: + # image: aws/codebuild/standard:7.0 + # - identifier: py312_decrypt_keyrings_with_js + # depend-on: + # - py312_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py312/decrypt_keyrings_with_js.yml + # env: + # image: aws/codebuild/standard:7.0 - - identifier: code_coverage - buildspec: codebuild/coverage/coverage.yml - - identifier: code_coverage_mpl - buildspec: codebuild/coverage/coverage_mpl.yml - env: - image: aws/codebuild/standard:7.0 + # - identifier: code_coverage + # buildspec: codebuild/coverage/coverage.yml + # - identifier: code_coverage_mpl + # buildspec: codebuild/coverage/coverage_mpl.yml + # env: + # image: aws/codebuild/standard:7.0 - - identifier: compliance - buildspec: codebuild/compliance/compliance.yml + # - identifier: compliance + # buildspec: codebuild/compliance/compliance.yml diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index c398904a9..71e9adf8b 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -69,22 +69,6 @@ def get_encryption_materials( ) mpl_output: MPL_GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) - - # ???????????????????????????? - # kpis = set() - # for edk in mpl_output.encryption_materials.encrypted_data_keys: - # kpis.add(edk.key_provider_info) - - # print(kpis) - # input - - # if len(kpis) == 1: - # for edk in mpl_output.encryption_materials.encrypted_data_keys: - # if edk.key_provider_info == b"rsa-4096-public": - # edk.key_provider_info = b"rsa-4096-private" - - # mpl_output.encryption_materials.encrypted_data_keys[0].key_provider_info = b"rsa-4096-private" - return EncryptionMaterialsFromMPL(mpl_output.encryption_materials) except AwsCryptographicMaterialProvidersException as mpl_exception: # Wrap MPL error into the ESDK error type diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index b70e48efe..54ea21b39 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -75,8 +75,6 @@ def encrypted_data_keys(self) -> List[Native_EncryptedDataKey]: ), encrypted_data_key=mpl_edk.ciphertext, ) for mpl_edk in mpl_edk_list} - # print(f"{key_blob_list=}") - # input() return key_blob_list @property diff --git a/src/aws_encryption_sdk/streaming_client.py b/src/aws_encryption_sdk/streaming_client.py index 54ce76235..fb0935ff2 100644 --- a/src/aws_encryption_sdk/streaming_client.py +++ b/src/aws_encryption_sdk/streaming_client.py @@ -582,7 +582,6 @@ def _prep_message(self): else: # MPL verification key is PEM bytes, not DER bytes. # If the underlying CMM is from the MPL, load PEM bytes. - print(f"DEBUG: cmm is {self.config.materials_manager}") if (_HAS_MPL and isinstance(self.config.materials_manager, CryptoMaterialsManagerFromMPL)): self.signer = Signer.from_key_bytes( diff --git a/test_vector_handlers/src/awses_test_vectors/internal/tampering_mpl_materials.py b/test_vector_handlers/src/awses_test_vectors/internal/tampering_mpl_materials.py index 1bb6705fc..4f7bc658e 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/tampering_mpl_materials.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/tampering_mpl_materials.py @@ -1,6 +1,5 @@ -"""Allows overriding the algorithm and signing_key for EncryptionMaterialsFromMPL. -This must ONLY be used in testing and NOT in production.. -This is used in message tampering testing. +"""Allows using ESDK-MPL interfaces with the tampering tests. +These must ONLY be used in testing and NOT in production. """ import attr import six @@ -30,7 +29,7 @@ class HalfSigningCryptoMaterialsManagerFromMPL(CryptoMaterialsManagerFromMPL): """ - Custom CMM that modifies the provider info field on EDKs + Custom CMM that uses HalfSigningEncryptionMaterialsFromMPL. This extends CryptoMaterialsManagerFromMPL so ESDK-internal checks follow MPL logic. @@ -81,7 +80,7 @@ def decrypt_materials(self, request): class HalfSigningEncryptionMaterialsFromMPL(EncryptionMaterialsFromMPL): """Allows overriding the algorithm and signing_key for EncryptionMaterialsFromMPL. - This must ONLY be used in testing and NOT in production.. + This must ONLY be used in testing and NOT in production. This is used in testing malicious message modification (HalfSigningTampering). """ diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index 4432502c5..2aaaf1bca 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -313,7 +313,8 @@ def master_key_provider_fn(): cmm_type = scenario["cmm"] elif scenario["cmm"] == "RequiredEncryptionContext": # Skip RequiredEncryptionContext CMM for master keys; - # This is unsupported for master keys + # RequiredEncryptionContext is unsupported for master keys. + # Caller logic should expect `None` to mean "no scenario". if keyrings: cmm_type = scenario["cmm"] else: @@ -384,9 +385,9 @@ def _one_shot_decrypt(self): required_ec_cmm: ICryptographicMaterialsManager = \ mpl.create_required_encryption_context_cmm( CreateRequiredEncryptionContextCMMInput( - # Currently, the test vector manifest requires that - # if using the required encryption context CMM, - # both and only "key1" and "key2" are required. + # Currently, the test vector manifest assumes these + # are the only required encryption context keys for any message. + # If this assumption changes, this logic must be augmented. required_encryption_context_keys=["key1", "key2"], underlying_cmm=underlying_cmm, ) @@ -436,9 +437,9 @@ def _streaming_decrypt(self): required_ec_cmm: ICryptographicMaterialsManager = \ mpl.create_required_encryption_context_cmm( CreateRequiredEncryptionContextCMMInput( - # Currently, the test vector manifest requires that - # if using the required encryption context CMM, - # both and only "key1" and "key2" are required. + # Currently, the test vector manifest assumes these + # are the only required encryption context keys for any message. + # If this assumption changes, this logic must be augmented. required_encryption_context_keys=["key1", "key2"], underlying_cmm=underlying_cmm, ) @@ -490,9 +491,9 @@ def _streaming_decrypt_unsigned(self): required_ec_cmm: ICryptographicMaterialsManager = \ mpl.create_required_encryption_context_cmm( CreateRequiredEncryptionContextCMMInput( - # Currently, the test vector manifest requires that - # if using the required encryption context CMM, - # both and only "key1" and "key2" are required. + # Currently, the test vector manifest assumes these + # are the only required encryption context keys for any message. + # If this assumption changes, this logic must be augmented. required_encryption_context_keys=["key1", "key2"], underlying_cmm=underlying_cmm, ) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index a1fc8fa83..f94facf13 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -52,7 +52,6 @@ _HAS_MPL = True except ImportError as e: - print(f"decrypt_generation ImportError: {e}") _HAS_MPL = False @@ -173,8 +172,6 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, _p """ master_key_provider = generation_scenario.encryption_scenario.master_key_provider_fn() - print(f"DEBUG: mkp gen is {master_key_provider}") - # Use a caching CMM to avoid generating a new data key every time. if isinstance(master_key_provider, MasterKeyProvider): cache = LocalCryptoMaterialsCache(10) @@ -185,6 +182,8 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, _p max_messages_encrypted=100, ) cmm = caching_cmm + # No caching CMM in MPL :( + # Use default CMM elif _HAS_MPL and isinstance(master_key_provider, IKeyring): mpl = AwsCryptographicMaterialProviders(MaterialProvidersConfig()) mpl_cmm = mpl.create_default_cryptographic_materials_manager( @@ -196,8 +195,6 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, _p else: raise TypeError(f"Unrecognized master_key_provider type: {master_key_provider}") - print(f"DEBUG: cmm gen is {cmm}") - return [ self.run_scenario_with_new_provider_info( ciphertext_writer, generation_scenario, cmm, new_provider_info @@ -208,7 +205,6 @@ def run_scenario_with_tampering(self, ciphertext_writer, generation_scenario, _p def run_scenario_with_new_provider_info( self, ciphertext_writer, generation_scenario, materials_manager, new_provider_info ): - print(f"DEBUG: materials_manager is {materials_manager}") """Run with tampering for a specific new provider info value""" if _HAS_MPL and isinstance(materials_manager, CryptoMaterialsManagerFromMPL): tampering_materials_manager = ProviderInfoChangingCryptoMaterialsManagerFromMPL( @@ -543,7 +539,7 @@ def _generate_plaintexts(plaintexts_specs): :return: Mapping of plaintext name to randomly generated bytes :rtype: dict """ - return {name: b"a" * size for name, size in plaintexts_specs.items()} + return {name: os.urandom(size) for name, size in plaintexts_specs.items()} @classmethod def from_file(cls, input_file, keyrings): diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index c1ffcdaa0..57de8504c 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -168,145 +168,6 @@ def run(self, materials_manager=None): return ciphertext -# @attr.s -# class MessageEncryptionWithMasterKeysTestScenario(MessageEncryptionTestScenario): -# # pylint: disable=too-many-instance-attributes -# """Data class for a single full message decrypt test scenario that uses master keys. - -# :param master_key_specs: Iterable of loaded master key specifications -# :type master_key_specs: iterable of :class:`MasterKeySpec` -# :param Callable master_key_provider_fn: -# """ - -# master_key_specs = attr.ib(validator=iterable_validator(list, MasterKeySpec)) -# master_key_provider_fn = attr.ib(validator=attr.validators.is_callable()) - -# @classmethod -# def from_scenario(cls, scenario, keys, plaintexts): -# # type: (ENCRYPT_SCENARIO_SPEC, KeysManifest, Dict[str, bytes]) -> MessageEncryptionTestScenario -# """Load from a scenario specification. - -# :param dict scenario: Scenario specification JSON -# :param KeysManifest keys: Loaded keys -# :param dict plaintexts: Mapping of plaintext names to plaintext values -# :return: Loaded test scenario -# :rtype: MessageEncryptionTestScenario -# """ -# algorithm = algorithm_suite_from_string_id(scenario["algorithm"]) -# master_key_specs = [MasterKeySpec.from_scenario(spec) for spec in scenario["master-keys"]] - -# def master_key_provider_fn(): -# return master_key_provider_from_master_key_specs(keys, master_key_specs) - -# return cls( -# plaintext_name=scenario["plaintext"], -# plaintext=plaintexts[scenario["plaintext"]], -# algorithm=algorithm, -# frame_size=scenario["frame-size"], -# encryption_context=scenario["encryption-context"], -# master_key=True, -# master_key_specs=master_key_specs, -# master_key_provider_fn=master_key_provider_fn, -# ) - -# def run(self, materials_manager=None): -# """Run this scenario, writing the resulting ciphertext with ``ciphertext_writer`` and returning -# a :class:`MessageDecryptionTestScenario` that describes the matching decrypt scenario. - -# :param callable ciphertext_writer: Callable that will write the requested named ciphertext and -# return a URI locating the written data -# :param str plaintext_uri: URI locating the written plaintext data for this scenario -# :return: Decrypt test scenario that describes the generated scenario -# :rtype: MessageDecryptionTestScenario -# """ -# commitment_policy = CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT -# if self.algorithm.is_committing(): -# commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT - -# client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=commitment_policy) -# encrypt_kwargs = dict( -# source=self.plaintext, -# algorithm=self.algorithm, -# frame_length=self.frame_size, -# encryption_context=self.encryption_context, -# ) -# if materials_manager: -# encrypt_kwargs["materials_manager"] = materials_manager -# else: -# encrypt_kwargs["key_provider"] = self.master_key_provider_fn() -# ciphertext, _header = client.encrypt(**encrypt_kwargs) -# return ciphertext - -# @attr.s -# class MessageEncryptionWithKeyringsTestScenario(MessageEncryptionTestScenario): -# # pylint: disable=too-many-instance-attributes -# """Data class for a single full message decrypt test scenario that uses keyrings. - -# :param master_key_specs: Iterable of loaded master key specifications -# :type master_key_specs: iterable of :class:`MasterKeySpec` -# :param Callable master_key_provider_fn: -# """ - -# master_key_specs = attr.ib(validator=iterable_validator(list, MasterKeySpec)) -# master_key_provider_fn = attr.ib(validator=attr.validators.is_callable()) - -# @classmethod -# def from_scenario(cls, scenario, keys_uri, plaintexts): -# # type: (ENCRYPT_SCENARIO_SPEC, KeysManifest, Dict[str, bytes]) -> MessageEncryptionTestScenario -# """Load from a scenario specification. - -# :param dict scenario: Scenario specification JSON -# :param KeysManifest keys: Loaded keys -# :param dict plaintexts: Mapping of plaintext names to plaintext values -# :return: Loaded test scenario -# :rtype: MessageEncryptionTestScenario -# """ -# algorithm = algorithm_suite_from_string_id(scenario["algorithm"]) -# # manifest still keys these as `master-keys` even though these are keyrings -# master_key_specs = [KeyringSpec.from_scenario(spec) for spec in scenario["master-keys"]] - -# def keyring_provider_fn(): -# return keyring_from_master_key_specs(keys_uri, master_key_specs) - -# return cls( -# plaintext_name=scenario["plaintext"], -# plaintext=plaintexts[scenario["plaintext"]], -# algorithm=algorithm, -# frame_size=scenario["frame-size"], -# encryption_context=scenario["encryption-context"], -# master_key=True, -# master_key_specs=master_key_specs, -# master_key_provider_fn=keyring_provider_fn, -# ) - -# def run(self, materials_manager=None): -# """Run this scenario, writing the resulting ciphertext with ``ciphertext_writer`` and returning -# a :class:`MessageDecryptionTestScenario` that describes the matching decrypt scenario. - -# :param callable ciphertext_writer: Callable that will write the requested named ciphertext and -# return a URI locating the written data -# :param str plaintext_uri: URI locating the written plaintext data for this scenario -# :return: Decrypt test scenario that describes the generated scenario -# :rtype: MessageDecryptionTestScenario -# """ -# commitment_policy = CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT -# if self.algorithm.is_committing(): -# commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT - -# client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=commitment_policy) -# encrypt_kwargs = dict( -# source=self.plaintext, -# algorithm=self.algorithm, -# frame_length=self.frame_size, -# encryption_context=self.encryption_context, -# ) -# if materials_manager: -# encrypt_kwargs["materials_manager"] = materials_manager -# else: -# encrypt_kwargs["keyring"] = self.keyring_provider_fn() -# ciphertext, _header = client.encrypt(**encrypt_kwargs) -# return ciphertext - @attr.s class MessageEncryptionManifest(object): """AWS Encryption SDK Encrypt Message manifest handler. diff --git a/test_vector_handlers/test/mpl/integration/commands/test_i_encrypt_keyrings.py b/test_vector_handlers/test/integration/commands/test_i_full_message_encrypt.py similarity index 62% rename from test_vector_handlers/test/mpl/integration/commands/test_i_encrypt_keyrings.py rename to test_vector_handlers/test/integration/commands/test_i_full_message_encrypt.py index 85c94dd22..6305a15da 100644 --- a/test_vector_handlers/test/mpl/integration/commands/test_i_encrypt_keyrings.py +++ b/test_vector_handlers/test/integration/commands/test_i_full_message_encrypt.py @@ -11,35 +11,27 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. """ -Integration tests for `awses_test_vectors.commands` with keyrings. +Integration tests for ``awses_test_vectors.commands``. """ import pytest from awses_test_vectors.commands import full_message_decrypt, full_message_decrypt_generate, full_message_encrypt -from ....integration.integration_test_utils import ( # noqa pylint: disable=unused-import +from ..integration_test_utils import ( # noqa pylint: disable=unused-import full_message_decrypt_generation_vectors, full_message_encrypt_vectors, ) - pytestmark = [pytest.mark.integ] def test_full_message_encrypt_canonical_full(full_message_encrypt_vectors): - full_message_encrypt.cli(["--input", full_message_encrypt_vectors, "--keyrings"]) + full_message_encrypt.cli(["--input", full_message_encrypt_vectors]) def test_full_message_cycle_canonical_full(tmpdir, full_message_decrypt_generation_vectors): - # Generate vectors using keyring interfaces - keyring_output_dir = tmpdir.join("output-keyrings") - full_message_decrypt_generate.cli([ - "--output", - str(keyring_output_dir), - "--input", - full_message_decrypt_generation_vectors, - "--keyrings" - ]) - - keyring_decrypt_manifest_file = keyring_output_dir.join("manifest.json") - full_message_decrypt.cli(["--input", str(keyring_decrypt_manifest_file), "--keyrings"]) + output_dir = tmpdir.join("output") + full_message_decrypt_generate.cli(["--output", str(output_dir), "--input", full_message_decrypt_generation_vectors]) + + decrypt_manifest_file = output_dir.join("manifest.json") + full_message_decrypt.cli(["--input", str(decrypt_manifest_file)]) diff --git a/test_vector_handlers/test/integration/integration_test_utils.py b/test_vector_handlers/test/integration/integration_test_utils.py index b8c8beb56..fbe6cf7b7 100644 --- a/test_vector_handlers/test/integration/integration_test_utils.py +++ b/test_vector_handlers/test/integration/integration_test_utils.py @@ -18,47 +18,20 @@ import pytest -here = os.path.abspath(os.path.dirname(__file__)) - - -def legacy_vectors_dir(): +def vectors_dir(): + here = os.path.abspath(os.path.dirname(__file__)) return os.path.abspath(os.path.join(here, "..", "aws-crypto-tools-test-vector-framework")) -def mpl_vectors_dir(): - return os.path.abspath(os.path.join(here, "..", "golden-manifest-TODORENAMEANDGETFROMGHA")) - - -def required_ec_vectors_dir(): - return os.path.abspath(os.path.join(here, "..", "required-ec-TODORENAMEANDGETFROMGHA")) - - @pytest.fixture def full_message_encrypt_vectors(): return os.path.join( - legacy_vectors_dir(), "features", "CANONICAL-GENERATED-MANIFESTS", "0003-awses-message-encryption.v2.json" + vectors_dir(), "features", "CANONICAL-GENERATED-MANIFESTS", "0003-awses-message-encryption.v2.json" ) @pytest.fixture def full_message_decrypt_generation_vectors(): return os.path.join( - legacy_vectors_dir(), - "features", - "CANONICAL-GENERATED-MANIFESTS", - "0006-awses-message-decryption-generation.v2.json" - ) - - -@pytest.fixture -def mpl_decrypt_vectors(): - return os.path.join( - mpl_vectors_dir(), "manifest.json" - ) - - -@pytest.fixture -def required_encryption_context_cmm_decrypt_vectors(): - return os.path.join( - required_ec_vectors_dir(), "manifest.json" + vectors_dir(), "features", "CANONICAL-GENERATED-MANIFESTS", "0006-awses-message-decryption-generation.v2.json" ) diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index c002323d3..654e72189 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -56,6 +56,11 @@ deps = commands = awses_local: {[testenv:base-command]commands} test/integration mplvectors: {[testenv:base-command]commands} test/mpl + + ; full_decrypt_generate: {[testenv:base-command]commands} test/integration/commands/test_i_generate_decrypt_vectors.py + ; full_decrypt: {[testenv:base-command]commands} test/integration/commands/test_i_decrypt_generated_vectors.py + ; full_encrypt: {[testenv:base-command]commands} test/integration/commands/test_i_encrypt_vectors.py + full_decrypt_generate: awses-full-message-decrypt-generate {posargs} full_decrypt: awses-full-message-decrypt {posargs} full_encrypt: awses-full-message-encrypt {posargs} From 446eaa4c93f7c89f9d321d939192ca7dd9df482c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 15:16:42 -0700 Subject: [PATCH 358/422] cleanup --- buildspec.yml | 574 +++++++++++++++++------------------ test_vector_handlers/tox.ini | 8 - 2 files changed, 287 insertions(+), 295 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index 90b5dbfd2..873e5941e 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -42,298 +42,298 @@ batch: env: image: aws/codebuild/standard:5.0 - # # 3.8 - # - identifier: py38_integ - # buildspec: codebuild/py38/integ.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py38_examples - # buildspec: codebuild/py38/examples.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py38_decrypt_dafny_esdk_vectors - # buildspec: codebuild/py38/decrypt_dafny_esdk_vectors.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py38_decrypt_net_401_vectors - # buildspec: codebuild/py38/decrypt_net_401_vectors.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py38_encrypt_masterkey - # buildspec: codebuild/py38/encrypt_masterkey.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py38_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py38/generate_decrypt_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py38_decrypt_masterkey_with_masterkey - # depend-on: - # - py38_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py38/decrypt_masterkey_with_masterkey.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py38_decrypt_masterkey_with_js - # depend-on: - # - py38_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py38/decrypt_masterkey_with_js.yml - # env: - # image: aws/codebuild/standard:5.0 + # 3.8 + - identifier: py38_integ + buildspec: codebuild/py38/integ.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py38_examples + buildspec: codebuild/py38/examples.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py38_decrypt_dafny_esdk_vectors + buildspec: codebuild/py38/decrypt_dafny_esdk_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py38_decrypt_net_401_vectors + buildspec: codebuild/py38/decrypt_net_401_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py38_encrypt_masterkey + buildspec: codebuild/py38/encrypt_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py38_generate_decrypt_vectors_masterkey + buildspec: codebuild/py38/generate_decrypt_vectors_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py38_decrypt_masterkey_with_masterkey + depend-on: + - py38_generate_decrypt_vectors_masterkey + buildspec: codebuild/py38/decrypt_masterkey_with_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py38_decrypt_masterkey_with_js + depend-on: + - py38_generate_decrypt_vectors_masterkey + buildspec: codebuild/py38/decrypt_masterkey_with_js.yml + env: + image: aws/codebuild/standard:5.0 - # # 3.9 - # - identifier: py39_integ - # buildspec: codebuild/py39/integ.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py39_examples - # buildspec: codebuild/py39/examples.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py39_decrypt_dafny_esdk_vectors - # buildspec: codebuild/py39/decrypt_dafny_esdk_vectors.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py39_decrypt_net_401_vectors - # buildspec: codebuild/py39/decrypt_net_401_vectors.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py39_encrypt_masterkey - # buildspec: codebuild/py39/encrypt_masterkey.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py39_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py39/generate_decrypt_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py39_decrypt_masterkey_with_masterkey - # depend-on: - # - py39_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py39/decrypt_masterkey_with_masterkey.yml - # env: - # image: aws/codebuild/standard:5.0 - # - identifier: py39_decrypt_masterkey_with_js - # depend-on: - # - py39_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py39/decrypt_masterkey_with_js.yml - # env: - # image: aws/codebuild/standard:5.0 + # 3.9 + - identifier: py39_integ + buildspec: codebuild/py39/integ.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_examples + buildspec: codebuild/py39/examples.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_decrypt_dafny_esdk_vectors + buildspec: codebuild/py39/decrypt_dafny_esdk_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_decrypt_net_401_vectors + buildspec: codebuild/py39/decrypt_net_401_vectors.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_encrypt_masterkey + buildspec: codebuild/py39/encrypt_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_generate_decrypt_vectors_masterkey + buildspec: codebuild/py39/generate_decrypt_vectors_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_decrypt_masterkey_with_masterkey + depend-on: + - py39_generate_decrypt_vectors_masterkey + buildspec: codebuild/py39/decrypt_masterkey_with_masterkey.yml + env: + image: aws/codebuild/standard:5.0 + - identifier: py39_decrypt_masterkey_with_js + depend-on: + - py39_generate_decrypt_vectors_masterkey + buildspec: codebuild/py39/decrypt_masterkey_with_js.yml + env: + image: aws/codebuild/standard:5.0 - # # 3.10 - # - identifier: py310_integ - # buildspec: codebuild/py310/integ.yml - # env: - # image: aws/codebuild/standard:6.0 - # - identifier: py310_examples - # buildspec: codebuild/py310/examples.yml - # env: - # image: aws/codebuild/standard:6.0 - # - identifier: py310_decrypt_dafny_esdk_vectors - # buildspec: codebuild/py310/decrypt_dafny_esdk_vectors.yml - # env: - # image: aws/codebuild/standard:6.0 - # - identifier: py310_decrypt_net_401_vectors - # buildspec: codebuild/py310/decrypt_net_401_vectors.yml - # env: - # image: aws/codebuild/standard:6.0 - # - identifier: py310_encrypt_masterkey - # buildspec: codebuild/py310/encrypt_masterkey.yml - # env: - # image: aws/codebuild/standard:6.0 - # - identifier: py310_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py310/generate_decrypt_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:6.0 - # - identifier: py310_decrypt_masterkey_with_masterkey - # depend-on: - # - py310_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py310/decrypt_masterkey_with_masterkey.yml - # env: - # image: aws/codebuild/standard:6.0 - # - identifier: py310_decrypt_masterkey_with_js - # depend-on: - # - py310_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py310/decrypt_masterkey_with_js.yml - # env: - # image: aws/codebuild/standard:6.0 + # 3.10 + - identifier: py310_integ + buildspec: codebuild/py310/integ.yml + env: + image: aws/codebuild/standard:6.0 + - identifier: py310_examples + buildspec: codebuild/py310/examples.yml + env: + image: aws/codebuild/standard:6.0 + - identifier: py310_decrypt_dafny_esdk_vectors + buildspec: codebuild/py310/decrypt_dafny_esdk_vectors.yml + env: + image: aws/codebuild/standard:6.0 + - identifier: py310_decrypt_net_401_vectors + buildspec: codebuild/py310/decrypt_net_401_vectors.yml + env: + image: aws/codebuild/standard:6.0 + - identifier: py310_encrypt_masterkey + buildspec: codebuild/py310/encrypt_masterkey.yml + env: + image: aws/codebuild/standard:6.0 + - identifier: py310_generate_decrypt_vectors_masterkey + buildspec: codebuild/py310/generate_decrypt_vectors_masterkey.yml + env: + image: aws/codebuild/standard:6.0 + - identifier: py310_decrypt_masterkey_with_masterkey + depend-on: + - py310_generate_decrypt_vectors_masterkey + buildspec: codebuild/py310/decrypt_masterkey_with_masterkey.yml + env: + image: aws/codebuild/standard:6.0 + - identifier: py310_decrypt_masterkey_with_js + depend-on: + - py310_generate_decrypt_vectors_masterkey + buildspec: codebuild/py310/decrypt_masterkey_with_js.yml + env: + image: aws/codebuild/standard:6.0 - # - identifier: py311_integ - # buildspec: codebuild/py311/integ.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_integ_mpl - # buildspec: codebuild/py311/integ_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_examples - # buildspec: codebuild/py311/examples.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_examples_mpl - # buildspec: codebuild/py311/examples_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_dafny_esdk_vectors_masterkey - # buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_dafny_esdk_vectors_keyrings - # buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_net_401_vectors_masterkey - # buildspec: codebuild/py311/decrypt_net_401_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_net_401_vectors_keyrings - # buildspec: codebuild/py311/decrypt_net_401_vectors_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_encrypt_masterkey - # buildspec: codebuild/py311/encrypt_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_encrypt_keyrings - # buildspec: codebuild/py311/encrypt_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py311/generate_decrypt_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_masterkey_with_masterkey - # depend-on: - # - py311_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py311/decrypt_masterkey_with_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_masterkey_with_keyrings - # depend-on: - # - py311_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py311/decrypt_masterkey_with_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_masterkey_with_js - # depend-on: - # - py311_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py311/decrypt_masterkey_with_js.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py311/generate_decrypt_vectors_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_keyrings_with_masterkey - # depend-on: - # - py311_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py311/decrypt_keyrings_with_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_keyrings_with_keyrings - # depend-on: - # - py311_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py311/decrypt_keyrings_with_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py311_decrypt_keyrings_with_js - # depend-on: - # - py311_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py311/decrypt_keyrings_with_js.yml - # env: - # image: aws/codebuild/standard:7.0 + - identifier: py311_integ + buildspec: codebuild/py311/integ.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_integ_mpl + buildspec: codebuild/py311/integ_mpl.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_examples + buildspec: codebuild/py311/examples.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_examples_mpl + buildspec: codebuild/py311/examples_mpl.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_dafny_esdk_vectors_masterkey + buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_dafny_esdk_vectors_keyrings + buildspec: codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_net_401_vectors_masterkey + buildspec: codebuild/py311/decrypt_net_401_vectors_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_net_401_vectors_keyrings + buildspec: codebuild/py311/decrypt_net_401_vectors_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_encrypt_masterkey + buildspec: codebuild/py311/encrypt_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_encrypt_keyrings + buildspec: codebuild/py311/encrypt_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_generate_decrypt_vectors_masterkey + buildspec: codebuild/py311/generate_decrypt_vectors_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_masterkey_with_masterkey + depend-on: + - py311_generate_decrypt_vectors_masterkey + buildspec: codebuild/py311/decrypt_masterkey_with_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_masterkey_with_keyrings + depend-on: + - py311_generate_decrypt_vectors_masterkey + buildspec: codebuild/py311/decrypt_masterkey_with_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_masterkey_with_js + depend-on: + - py311_generate_decrypt_vectors_masterkey + buildspec: codebuild/py311/decrypt_masterkey_with_js.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_generate_decrypt_vectors_keyrings + buildspec: codebuild/py311/generate_decrypt_vectors_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_keyrings_with_masterkey + depend-on: + - py311_generate_decrypt_vectors_keyrings + buildspec: codebuild/py311/decrypt_keyrings_with_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_keyrings_with_keyrings + depend-on: + - py311_generate_decrypt_vectors_keyrings + buildspec: codebuild/py311/decrypt_keyrings_with_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py311_decrypt_keyrings_with_js + depend-on: + - py311_generate_decrypt_vectors_keyrings + buildspec: codebuild/py311/decrypt_keyrings_with_js.yml + env: + image: aws/codebuild/standard:7.0 - # - identifier: py312_integ - # buildspec: codebuild/py312/integ.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_integ_mpl - # buildspec: codebuild/py312/integ_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_examples - # buildspec: codebuild/py312/examples.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_examples_mpl - # buildspec: codebuild/py312/examples_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_dafny_esdk_vectors_masterkey - # buildspec: codebuild/py312/decrypt_dafny_esdk_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_dafny_esdk_vectors_keyrings - # buildspec: codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_net_401_vectors_masterkey - # buildspec: codebuild/py312/decrypt_net_401_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_net_401_vectors_keyrings - # buildspec: codebuild/py312/decrypt_net_401_vectors_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_encrypt_masterkey - # buildspec: codebuild/py312/encrypt_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_encrypt_keyrings - # buildspec: codebuild/py312/encrypt_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py312/generate_decrypt_vectors_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_masterkey_with_masterkey - # depend-on: - # - py312_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py312/decrypt_masterkey_with_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_masterkey_with_keyrings - # depend-on: - # - py312_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py312/decrypt_masterkey_with_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_masterkey_with_js - # depend-on: - # - py312_generate_decrypt_vectors_masterkey - # buildspec: codebuild/py312/decrypt_masterkey_with_js.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py312/generate_decrypt_vectors_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_keyrings_with_masterkey - # depend-on: - # - py312_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py312/decrypt_keyrings_with_masterkey.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_keyrings_with_keyrings - # depend-on: - # - py312_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py312/decrypt_keyrings_with_keyrings.yml - # env: - # image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_keyrings_with_js - # depend-on: - # - py312_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py312/decrypt_keyrings_with_js.yml - # env: - # image: aws/codebuild/standard:7.0 + - identifier: py312_integ + buildspec: codebuild/py312/integ.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_integ_mpl + buildspec: codebuild/py312/integ_mpl.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_examples + buildspec: codebuild/py312/examples.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_examples_mpl + buildspec: codebuild/py312/examples_mpl.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_dafny_esdk_vectors_masterkey + buildspec: codebuild/py312/decrypt_dafny_esdk_vectors_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_dafny_esdk_vectors_keyrings + buildspec: codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_net_401_vectors_masterkey + buildspec: codebuild/py312/decrypt_net_401_vectors_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_net_401_vectors_keyrings + buildspec: codebuild/py312/decrypt_net_401_vectors_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_encrypt_masterkey + buildspec: codebuild/py312/encrypt_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_encrypt_keyrings + buildspec: codebuild/py312/encrypt_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_generate_decrypt_vectors_masterkey + buildspec: codebuild/py312/generate_decrypt_vectors_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_masterkey_with_masterkey + depend-on: + - py312_generate_decrypt_vectors_masterkey + buildspec: codebuild/py312/decrypt_masterkey_with_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_masterkey_with_keyrings + depend-on: + - py312_generate_decrypt_vectors_masterkey + buildspec: codebuild/py312/decrypt_masterkey_with_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_masterkey_with_js + depend-on: + - py312_generate_decrypt_vectors_masterkey + buildspec: codebuild/py312/decrypt_masterkey_with_js.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_generate_decrypt_vectors_keyrings + buildspec: codebuild/py312/generate_decrypt_vectors_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_keyrings_with_masterkey + depend-on: + - py312_generate_decrypt_vectors_keyrings + buildspec: codebuild/py312/decrypt_keyrings_with_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_keyrings_with_keyrings + depend-on: + - py312_generate_decrypt_vectors_keyrings + buildspec: codebuild/py312/decrypt_keyrings_with_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_keyrings_with_js + depend-on: + - py312_generate_decrypt_vectors_keyrings + buildspec: codebuild/py312/decrypt_keyrings_with_js.yml + env: + image: aws/codebuild/standard:7.0 - # - identifier: code_coverage - # buildspec: codebuild/coverage/coverage.yml - # - identifier: code_coverage_mpl - # buildspec: codebuild/coverage/coverage_mpl.yml - # env: - # image: aws/codebuild/standard:7.0 + - identifier: code_coverage + buildspec: codebuild/coverage/coverage.yml + - identifier: code_coverage_mpl + buildspec: codebuild/coverage/coverage_mpl.yml + env: + image: aws/codebuild/standard:7.0 - # - identifier: compliance - # buildspec: codebuild/compliance/compliance.yml + - identifier: compliance + buildspec: codebuild/compliance/compliance.yml diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index 654e72189..70819dd5f 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -4,7 +4,6 @@ envlist = # so until release we can only effectively test the local version of the ESDK. py{37,38,39,310}-awses_local py{311,312}-awses_local{,-mpl} - py{311,312}-mplvectors-mpl # 1.2.0 and 1.2.max are being difficult because of attrs bandit, doc8, readme, {flake8,pylint}{,-tests}, @@ -54,13 +53,6 @@ deps = mpl: -rrequirements_mpl.txt .. commands = - awses_local: {[testenv:base-command]commands} test/integration - mplvectors: {[testenv:base-command]commands} test/mpl - - ; full_decrypt_generate: {[testenv:base-command]commands} test/integration/commands/test_i_generate_decrypt_vectors.py - ; full_decrypt: {[testenv:base-command]commands} test/integration/commands/test_i_decrypt_generated_vectors.py - ; full_encrypt: {[testenv:base-command]commands} test/integration/commands/test_i_encrypt_vectors.py - full_decrypt_generate: awses-full-message-decrypt-generate {posargs} full_decrypt: awses-full-message-decrypt {posargs} full_encrypt: awses-full-message-encrypt {posargs} From 70b68f9bffe71db82a1b9addaee2d2dfe80bd883 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 15:54:24 -0700 Subject: [PATCH 359/422] cleanup --- .../test/integration/__init__.py | 2 + .../test/integration/commands/__init__.py | 2 + test_vector_handlers/test/keys.json | 214 ------------------ test_vector_handlers/test/mpl/__init__.py | 0 .../test/mpl/integration/__init__.py | 0 .../test/mpl/integration/commands/__init__.py | 0 test_vector_handlers/tox.ini | 3 +- 7 files changed, 6 insertions(+), 215 deletions(-) delete mode 100644 test_vector_handlers/test/keys.json delete mode 100644 test_vector_handlers/test/mpl/__init__.py delete mode 100644 test_vector_handlers/test/mpl/integration/__init__.py delete mode 100644 test_vector_handlers/test/mpl/integration/commands/__init__.py diff --git a/test_vector_handlers/test/integration/__init__.py b/test_vector_handlers/test/integration/__init__.py index e69de29bb..76a5b798a 100644 --- a/test_vector_handlers/test/integration/__init__.py +++ b/test_vector_handlers/test/integration/__init__.py @@ -0,0 +1,2 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 \ No newline at end of file diff --git a/test_vector_handlers/test/integration/commands/__init__.py b/test_vector_handlers/test/integration/commands/__init__.py index e69de29bb..76a5b798a 100644 --- a/test_vector_handlers/test/integration/commands/__init__.py +++ b/test_vector_handlers/test/integration/commands/__init__.py @@ -0,0 +1,2 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 \ No newline at end of file diff --git a/test_vector_handlers/test/keys.json b/test_vector_handlers/test/keys.json deleted file mode 100644 index 304dae5f7..000000000 --- a/test_vector_handlers/test/keys.json +++ /dev/null @@ -1,214 +0,0 @@ -{ - "manifest": { - "type": "keys", - "version": 3 - }, - "keys": { - "aes-128": { - "encrypt": true, - "decrypt": true, - "algorithm": "aes", - "type": "symmetric", - "bits": 128, - "encoding": "base64", - "material": "AAECAwQFBgcICRAREhMUFQ==", - "key-id": "aes-128" - }, - "aes-192": { - "encrypt": true, - "decrypt": true, - "algorithm": "aes", - "type": "symmetric", - "bits": 192, - "encoding": "base64", - "material": "AAECAwQFBgcICRAREhMUFRYXGBkgISIj", - "key-id": "aes-192" - }, - "aes-256": { - "encrypt": true, - "decrypt": true, - "algorithm": "aes", - "type": "symmetric", - "bits": 256, - "encoding": "base64", - "material": "AAECAwQFBgcICRAREhMUFRYXGBkgISIjJCUmJygpMDE=", - "key-id": "aes-256" - }, - "rsa-4096-private": { - "encrypt": true, - "decrypt": true, - "algorithm": "rsa", - "type": "private", - "bits": 4096, - "encoding": "pem", - "material": "-----BEGIN PRIVATE KEY-----\nMIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCztGg1gQ8AjCzz\n1VX6StqtW//jBt2ZQBoApaBa7FmLmdr0YlKaeEKSrItGbvA9tBjgsKhrn8gxTGQc\nuxgM92651jRCbQZyjE6W8kodijhGMXsfKJLfgPp2/I7gZ3dqrSZkejFIYLFb/uF/\nTfAQzNyJUldYdeFojSUPqevMgSAusTgv7dXYt4BCO9mxMp35tgyp5k4vazKJVUgB\nTw87AAYZUGugmi94Wb9JSnqUKI3QzaRN7JADZrHdBO1lIBryfCsjtTnZc7NWZ0yJ\nwmzLY+C5b3y17cy44N0rbjI2QciRhqZ4/9SZ/9ImyFQlB3lr9NSndcT4eE5YC6bH\nba0gOUK9lLXVy6TZ+nRZ4dSddoLX03mpYp+8cQpK6DO3L/PeUY/si0WGsXZfWokd\n4ACwvXWSOjotzjwqwTW8q9udbhUvIHfB02JW+ZQ07b209fBpHRDkZuveOTedTN2Q\nQei4dZDjWW5s4cIIE3dXXeaH8yC02ERIeN+aY6eHngSsP2xoDV3sKNN/yDbCqaMS\nq8ZJbo2rvOFxZHa2nWiV+VLugfO6Xj8jeGeR8vopvbEBZZpAq+Dea2xjY4+XMUQ/\nS1HlRwc9+nkJ5LVfODuE3q9EgJbqbiXe7YckWV3ZqQMybW+dLPxEJs9buOntgHFS\nRYmbKky0bti/ZoZlcZtS0zyjVxlqsQIDAQABAoICAEr3m/GWIXgNAkPGX9PGnmtr\n0dgX6SIhh7d1YOwNZV3DlYAV9HfUa5Fcwc1kQny7QRWbHOepBI7sW2dQ9buTDXIh\nVjPP37yxo6d89EZWfxtpUP+yoXL0D4jL257qCvtJuJZ6E00qaVMDhXbiQKABlo8C\n9sVEiABhwXBDZsctpwtTiykTgv6hrrPy2+H8R8MAm0/VcBCAG9kG5r8FCEmIvQKa\ndgvNxrfiWNZuZ6yfLmpJH54SbhG9Kb4WbCKfvh4ihqyi0btRdSM6fMeLgG9o/zrc\ns54B0kHeLOYNVo0j7FQpZBFeSIbmHfln4RKBh7ntrTke/Ejbh3NbiPvxWSP0P067\nSYWPkQpip2q0ION81wSQZ1haP2GewFFu4IEjG3DlqqpKKGLqXrmjMufnildVFpBx\nir+MgvgQfEBoGEx0aElyO7QuRYaEiXeb/BhMZeC5O65YhJrWSuTVizh3xgJWjgfV\naYwYgxN8SBXBhXLIVvnPhadTqsW1C/aevLOk110eSFWcHf+FCK781ykIzcpXoRGX\nOwWcZzC/fmSABS0yH56ow+I0tjdLIEEMhoa4/kkamioHOJ4yyB+W1DO6/DnMyQlx\ng7y2WsAaIEBoWUARy776k70xPPMtYAxzFXI9KhqRVrPfeaRZ+ojeyLyr3GQGyyoo\ncuGRdMUblsmODv4ixmOxAoIBAQDvkznvVYNdP3Eg5vQeLm/qsP6dLejLijBLeq9i\n7DZH2gRpKcflXZxCkRjsKDDE+fgDcBYEp2zYfRIVvgrxlTQZdaSG+GoDcbjbNQn3\ndjCCtOOACioN/vg2zFlX4Bs6Q+NaV7g5qP5SUaxUBjuHLe7Nc+ZkyheMHuNYVLvk\nHL/IoWyANpZYjMUU3xMbL/J29Gz7CPGr8Si28TihAHGfcNgn8S04OQZhTX+bU805\n/+7B4XW47Mthg/u7hlqFl+YIAaSJYvWkEaVP1A9I7Ve0aMDSMWwzTg9cle2uVaL3\n+PTzWY5coBlHKjqAg9ufhYSDhAqBd/JOSlv8RwcA3PDXJ6C/AoIBAQDABmXXYQky\n7phExXBvkLtJt2TBGjjwulf4R8TC6W5F51jJuoqY/mTqYcLcOn2nYGVwoFvPsy/Q\nCTjfODwJBXzbloXtYFR3PWAeL1Y6+7Cm+koMWIPJyVbD5Fzm+gZStM0GwP8FhDt2\nWt8fWEyXmoLdAy6RAwiEmCagEh8o+13oBfwnBllbz7TxaErsUuR+XVgl/iHwztdv\ncdJKyRgaFfWSh9aiO7EMV2rBGWsoX09SRvprPFAGx8Ffm7YcqIk34QXsQyc45Dyn\nCwkvypxHoaB3ot/48FeFm9IubApb/ctv+EgkBfL4S4bdwRXS1rt+0+QihBoFyP2o\nJ91cdm4hEWCPAoIBAQC6l11hFaYZo0bWDGsHcr2B+dZkzxPoKznQH76n+jeQoLIc\nwgjJkK4afm39yJOrZtEOxGaxu0CgIFFMk9ZsL/wC9EhvQt02z4TdXiLkFK5VrtMd\nr0zv16y06VWQhqBOMf/KJlX6uq9RqADi9HO6pkC+zc0cpPXQEWKaMmygju+kMG2U\nMm/IieMZjWCRJTfgBCE5J88qTsqaKagkZXcZakdAXKwOhQN+F2EStiM6UCZB5PrO\nS8dfrO8ML+ki8Zqck8L1qhiNb5zkXtKExy4u+gNr8khGcT6vqqoSxOoH3mPRgOfL\nJnppne8wlwIf7Vq3H8ka6zPSXEHma999gZcmy9t7AoIBAGbQhiLl79j3a0wXMvZp\nVf5IVYgXFDnAbG2hb7a06bhAAIgyexcjzsC4C2+DWdgOgwHkuoPg+062QV8zauGh\nsJKaa6cHlvIpSJeg3NjD/nfJN3CYzCd0yCIm2Z9Ka6xI5iYhm+pGPNhIG4Na8deS\ngVL46yv1pc/o73VxfoGg5UzgN3xlp97Cva0sHEGguHr4W8Qr59xZw3wGQ4SLW35M\nF6qXVNKUh12GSMCPbZK2RXBWVKqqJmca+WzJoJ6DlsT2lQdFhXCus9L007xlDXxF\nC/hCmw1dEl+VaNo2Ou26W/zdwTKYhNlxBwsg4SB8nPNxXIsmlBBY54froFhriNfn\nx/0CggEAUzz+VMtjoEWw2HSHLOXrO4EmwJniNgiiwfX3DfZE4tMNZgqZwLkq67ns\nT0n3b0XfAOOkLgMZrUoOxPHkxFeyLLf7pAEJe7QNB+Qilw8e2zVqtiJrRk6uDIGJ\nSv+yM52zkImZAe2jOdU3KeUZxSMmb5vIoiPBm+tb2WupAg3YdpKn1/jWTpVmV/+G\nUtTLVE6YpAyFp1gMxhutE9vfIS94ek+vt03AoEOlltt6hqZfv3xmY8vGuAjlnj12\nzHaq+fhCRPsbsZkzJ9nIVdXYnNIEGtMGNnxax7tYRej/UXqyazbxHiJ0iPF4PeDn\ndzxtGxpeTBi+KhKlca8SlCdCqYwG6Q==\n-----END PRIVATE KEY-----", - "key-id": "rsa-4096" - }, - "rsa-4096-public": { - "encrypt": true, - "decrypt": false, - "algorithm": "rsa", - "type": "public", - "bits": 4096, - "encoding": "pem", - "material": "-----BEGIN PUBLIC KEY-----\nMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAs7RoNYEPAIws89VV+kra\nrVv/4wbdmUAaAKWgWuxZi5na9GJSmnhCkqyLRm7wPbQY4LCoa5/IMUxkHLsYDPdu\nudY0Qm0GcoxOlvJKHYo4RjF7HyiS34D6dvyO4Gd3aq0mZHoxSGCxW/7hf03wEMzc\niVJXWHXhaI0lD6nrzIEgLrE4L+3V2LeAQjvZsTKd+bYMqeZOL2syiVVIAU8POwAG\nGVBroJoveFm/SUp6lCiN0M2kTeyQA2ax3QTtZSAa8nwrI7U52XOzVmdMicJsy2Pg\nuW98te3MuODdK24yNkHIkYameP/Umf/SJshUJQd5a/TUp3XE+HhOWAumx22tIDlC\nvZS11cuk2fp0WeHUnXaC19N5qWKfvHEKSugzty/z3lGP7ItFhrF2X1qJHeAAsL11\nkjo6Lc48KsE1vKvbnW4VLyB3wdNiVvmUNO29tPXwaR0Q5Gbr3jk3nUzdkEHouHWQ\n41lubOHCCBN3V13mh/MgtNhESHjfmmOnh54ErD9saA1d7CjTf8g2wqmjEqvGSW6N\nq7zhcWR2tp1olflS7oHzul4/I3hnkfL6Kb2xAWWaQKvg3mtsY2OPlzFEP0tR5UcH\nPfp5CeS1Xzg7hN6vRICW6m4l3u2HJFld2akDMm1vnSz8RCbPW7jp7YBxUkWJmypM\ntG7Yv2aGZXGbUtM8o1cZarECAwEAAQ==\n-----END PUBLIC KEY-----", - "key-id": "rsa-4096" - }, - "us-west-2-decryptable": { - "encrypt": true, - "decrypt": true, - "type": "aws-kms", - "key-id": "arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f" - }, - "us-west-2-encrypt-only": { - "encrypt": true, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws:kms:us-west-2:658956600833:key/590fd781-ddde-4036-abec-3e1ab5a5d2ad" - }, - "us-west-2-mrk": { - "encrypt": true, - "decrypt": true, - "type": "aws-kms", - "key-id": "arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "us-east-1-mrk": { - "encrypt": true, - "decrypt": true, - "type": "aws-kms", - "key-id": "arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "aws:kms:us-west-2:658956600833:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "aws:kms:us-west-2:658956600833:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - ":aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": ":aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "arn-not:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn-not:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "arn:kms:us-west-2:658956600833:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:kms:us-west-2:658956600833:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "arn::kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn::kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "arn:aws-not:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws-not:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "arn:aws:us-west-2:658956600833:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws:us-west-2:658956600833:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "arn:aws::us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws::us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "arn:aws:kms-not:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws:kms-not:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "arn:aws:kms:658956600833:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws:kms:658956600833:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "arn:aws:kms::658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws:kms::658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "arn:aws:kms:us-west-2:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws:kms:us-west-2:key:mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "arn:aws:kms:us-west-2::key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws:kms:us-west-2::key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "arn:aws:kms:us-west-2:658956600833-not:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws:kms:us-west-2:658956600833-not:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "arn:aws:kms:us-west-2:658956600833:mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws:kms:us-west-2:658956600833:mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "arn:aws:kms:us-west-2:658956600833:/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws:kms:us-west-2:658956600833:/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "arn:aws:kms:us-west-2:658956600833:key-not/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws:kms:us-west-2:658956600833:key-not/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "arn:aws:kms:us-west-2:658956600833:key": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws:kms:us-west-2:658956600833:key" - }, - "arn:aws:kms:us-west-2:658956600833:key/": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws:kms:us-west-2:658956600833:key/" - }, - "arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7-not": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7-not" - }, - "arn:aws:kms:us-west-2:658956600833:alias/mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "arn:aws:kms:us-west-2:658956600833:alias/mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - }, - "mrk-80bd8ecdcd4342aebd84b7dc9da498a7": { - "encrypt": false, - "decrypt": false, - "type": "aws-kms", - "key-id": "mrk-80bd8ecdcd4342aebd84b7dc9da498a7" - } - } -} diff --git a/test_vector_handlers/test/mpl/__init__.py b/test_vector_handlers/test/mpl/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/test_vector_handlers/test/mpl/integration/__init__.py b/test_vector_handlers/test/mpl/integration/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/test_vector_handlers/test/mpl/integration/commands/__init__.py b/test_vector_handlers/test/mpl/integration/commands/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index 70819dd5f..cdb1137fb 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -36,7 +36,7 @@ envlist = # release :: Builds dist files and uploads to pypi pypirc profile. [testenv:base-command] -commands = pytest --basetemp={envtmpdir} -l --cov awses_test_vectors {posargs} +commands = pytest --basetemp={envtmpdir} -l --cov awses_test_vectors test/ --ignore test/mpl {posargs} [testenv] passenv = @@ -53,6 +53,7 @@ deps = mpl: -rrequirements_mpl.txt .. commands = + awses_local: {[testenv:base-command]commands} full_decrypt_generate: awses-full-message-decrypt-generate {posargs} full_decrypt: awses-full-message-decrypt {posargs} full_encrypt: awses-full-message-encrypt {posargs} From 67f0179ebccbdb44d9a02545f0276b73d52c08ce Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 15:59:28 -0700 Subject: [PATCH 360/422] cleanup --- .../materials_managers/mpl/cmm.py | 1 - .../internal/keyvectors_provider.py | 27 --- .../internal/tampering_mpl_materials.py | 169 ------------------ .../full_message/decrypt_generation.py | 2 +- .../manifests/mpl_keyring.py | 2 +- test_vector_handlers/test/__init__.py | 0 .../test/integration/__init__.py | 2 - .../test/integration/commands/__init__.py | 2 - 8 files changed, 2 insertions(+), 203 deletions(-) delete mode 100644 test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py delete mode 100644 test_vector_handlers/src/awses_test_vectors/internal/tampering_mpl_materials.py delete mode 100644 test_vector_handlers/test/__init__.py diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index 71e9adf8b..ebef5f7ac 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -67,7 +67,6 @@ def get_encryption_materials( CryptoMaterialsManagerFromMPL._native_to_mpl_get_encryption_materials( request ) - mpl_output: MPL_GetEncryptionMaterialsOutput = self.mpl_cmm.get_encryption_materials(mpl_input) return EncryptionMaterialsFromMPL(mpl_output.encryption_materials) except AwsCryptographicMaterialProvidersException as mpl_exception: diff --git a/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py b/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py deleted file mode 100644 index 305459026..000000000 --- a/test_vector_handlers/src/awses_test_vectors/internal/keyvectors_provider.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Singleton provider for the KeyVectors client.""" -# # Ignore missing MPL TestVectors for pylint, but the MPL TestVectors is required for this file -# pylint: disable=import-error -from aws_cryptography_materialproviderstestvectorkeys.smithygenerated.\ - aws_cryptography_materialproviderstestvectorkeys.client import ( - KeyVectors, - ) -from aws_cryptography_materialproviderstestvectorkeys.smithygenerated.\ - aws_cryptography_materialproviderstestvectorkeys.config import ( - KeyVectorsConfig - ) - -keyvectors_instances = {} - - -# pylint: disable=too-few-public-methods -class KeyVectorsProvider: - """Singleton manager for the KeyVectors client.""" - - instance: KeyVectors - - @classmethod - def get_keyvectors(cls, keys_path): - """Return the singleton KeyVectors client.""" - if keys_path not in keyvectors_instances: - keyvectors_instances[keys_path] = KeyVectors(KeyVectorsConfig(key_manifest_path=keys_path)) - return keyvectors_instances[keys_path] diff --git a/test_vector_handlers/src/awses_test_vectors/internal/tampering_mpl_materials.py b/test_vector_handlers/src/awses_test_vectors/internal/tampering_mpl_materials.py deleted file mode 100644 index 4f7bc658e..000000000 --- a/test_vector_handlers/src/awses_test_vectors/internal/tampering_mpl_materials.py +++ /dev/null @@ -1,169 +0,0 @@ -"""Allows using ESDK-MPL interfaces with the tampering tests. -These must ONLY be used in testing and NOT in production. -""" -import attr -import six -from copy import copy - - -from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager - -# Ignore missing MPL for pylint, but the MPL is required for this class -# pylint: disable=import-error,no-name-in-module -from aws_encryption_sdk.materials_managers.mpl.materials import ( - EncryptionMaterialsFromMPL -) -from aws_encryption_sdk.materials_managers.mpl.cmm import ( - CryptoMaterialsManagerFromMPL -) -from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders -from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig -from aws_cryptographic_materialproviders.mpl.models import ( - CreateDefaultCryptographicMaterialsManagerInput, -) - -try: - from aws_encryption_sdk.identifiers import AlgorithmSuite -except ImportError: - from aws_encryption_sdk.identifiers import Algorithm as AlgorithmSuite - -class HalfSigningCryptoMaterialsManagerFromMPL(CryptoMaterialsManagerFromMPL): - """ - Custom CMM that uses HalfSigningEncryptionMaterialsFromMPL. - This extends CryptoMaterialsManagerFromMPL so ESDK-internal checks - follow MPL logic. - - THIS IS ONLY USED TO CREATE INVALID MESSAGES and should never be used in - production! - """ - - wrapped_default_cmm = attr.ib(validator=attr.validators.instance_of(CryptoMaterialsManagerFromMPL)) - - def __init__(self, master_key_provider): - """Create a new CMM that wraps a the given CMM.""" - mpl = AwsCryptographicMaterialProviders(MaterialProvidersConfig()) - mpl_cmm = mpl.create_default_cryptographic_materials_manager( - CreateDefaultCryptographicMaterialsManagerInput( - keyring=master_key_provider - ) - ) - self.wrapped_default_cmm = CryptoMaterialsManagerFromMPL(mpl_cmm=mpl_cmm) - - def get_encryption_materials(self, request): - """ - Generate half-signing materials by requesting signing materials - from the wrapped default CMM, and then changing the algorithm suite - and removing the signing key from teh result. - """ - if request.algorithm == AlgorithmSuite.AES_256_GCM_HKDF_SHA512_COMMIT_KEY: - signing_request = copy(request) - signing_request.algorithm = AlgorithmSuite.AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384 - - result = HalfSigningEncryptionMaterialsFromMPL( - self.wrapped_default_cmm.get_encryption_materials(signing_request) - ) - - result.algorithm = request.algorithm - result.signing_key = None - - return result - - raise NotImplementedError( - "The half-sign tampering method is only supported on the " - "AES_256_GCM_HKDF_SHA512_COMMIT_KEY algorithm suite." - ) - - def decrypt_materials(self, request): - """Thunks to the wrapped default CMM""" - return self.wrapped_default_cmm.decrypt_materials(request) - - -class HalfSigningEncryptionMaterialsFromMPL(EncryptionMaterialsFromMPL): - """Allows overriding the algorithm and signing_key for EncryptionMaterialsFromMPL. - This must ONLY be used in testing and NOT in production. - This is used in testing malicious message modification (HalfSigningTampering). - """ - - _underlying_materials: EncryptionMaterialsFromMPL - - def __init__(self, underling_materials): - self._underlying_materials = underling_materials - - # pylint thinks EncryptionMaterialsFromMPL.algorithm is a method - # pylint: disable=invalid-overridden-method - @property - def algorithm(self): - """Return any previously-provided overriden algorithm; - if none was provided, returns underlying algorithm from encryption materials. - """ - if hasattr(self, "set_algorithm"): - return self.set_algorithm - return self._underlying_materials.algorithm - - @algorithm.setter - def algorithm(self, algorithm): - self.set_algorithm = algorithm - - # pylint thinks EncryptionMaterialsFromMPL.signing_key is a method - # pylint: disable=invalid-overridden-method - @property - def signing_key(self): - """Return any previously-provided overriden signing_key; - if none was provided, returns underlying signing_key from encryption materials. - """ - if hasattr(self, "set_signing_key"): - return self.set_signing_key - return self._underlying_materials.algorithm - - @signing_key.setter - def signing_key(self, signing_key): - self.set_signing_key = signing_key - - @property - def encryption_context(self): - return self._underlying_materials.encryption_context - - @property - def encrypted_data_keys(self): - return self._underlying_materials.encrypted_data_keys - - @property - def data_encryption_key(self): - return self._underlying_materials.data_encryption_key - - @property - def required_encryption_context_keys(self): - return self._underlying_materials.required_encryption_context_keys - - -class ProviderInfoChangingCryptoMaterialsManagerFromMPL(CryptoMaterialsManagerFromMPL): - """ - Custom CMM that modifies the provider info field on EDKs. - This extends CryptoMaterialsManagerFromMPL so ESDK-internal checks - follow MPL logic. - - THIS IS ONLY USED TO CREATE INVALID MESSAGES and should never be used in - production! - """ - - wrapped_cmm = attr.ib(validator=attr.validators.instance_of(CryptoMaterialsManager)) - new_provider_info = attr.ib(validator=attr.validators.instance_of(six.string_types)) - - def __init__(self, materials_manager, new_provider_info): - """Create a new CMM that wraps a the given CMM.""" - self.wrapped_cmm = materials_manager - self.new_provider_info = new_provider_info - - def get_encryption_materials(self, request): - """ - Request materials from the wrapped CMM, and then change the provider info - on each EDK. - """ - result = self.wrapped_cmm.get_encryption_materials(request) - for encrypted_data_key in result.encrypted_data_keys: - encrypted_data_key.key_provider.key_info = self.new_provider_info - return result - - def decrypt_materials(self, request): - """Thunks to the wrapped CMM""" - return self.wrapped_cmm.decrypt_materials(request) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index f94facf13..e7aa747a7 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -44,7 +44,7 @@ from aws_encryption_sdk.materials_managers.mpl.materials import ( EncryptionMaterialsFromMPL ) - from awses_test_vectors.internal.tampering_mpl_materials import ( + from awses_test_vectors.internal.mpl.tampering_mpl_materials import ( HalfSigningEncryptionMaterialsFromMPL, ProviderInfoChangingCryptoMaterialsManagerFromMPL, HalfSigningCryptoMaterialsManagerFromMPL, diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index 55a9276c9..c05c14714 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -33,7 +33,7 @@ import _dafny import UTF8 -from awses_test_vectors.internal.keyvectors_provider import KeyVectorsProvider +from awses_test_vectors.internal.mpl.keyvectors_provider import KeyVectorsProvider from awses_test_vectors.manifests.keys import KeysManifest # noqa: disable=F401 diff --git a/test_vector_handlers/test/__init__.py b/test_vector_handlers/test/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/test_vector_handlers/test/integration/__init__.py b/test_vector_handlers/test/integration/__init__.py index 76a5b798a..e69de29bb 100644 --- a/test_vector_handlers/test/integration/__init__.py +++ b/test_vector_handlers/test/integration/__init__.py @@ -1,2 +0,0 @@ -# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 \ No newline at end of file diff --git a/test_vector_handlers/test/integration/commands/__init__.py b/test_vector_handlers/test/integration/commands/__init__.py index 76a5b798a..e69de29bb 100644 --- a/test_vector_handlers/test/integration/commands/__init__.py +++ b/test_vector_handlers/test/integration/commands/__init__.py @@ -1,2 +0,0 @@ -# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 \ No newline at end of file From 7d8a515fbb8589243f6538e68f7110994d7727ee Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 16:08:20 -0700 Subject: [PATCH 361/422] cleanup --- .../manifests/full_message/decrypt_generation.py | 2 +- .../src/awses_test_vectors/manifests/mpl_keyring.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index e7aa747a7..6213fc3a6 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -51,7 +51,7 @@ ) _HAS_MPL = True -except ImportError as e: +except ImportError:" _HAS_MPL = False diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index c05c14714..ec35147c0 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -33,6 +33,8 @@ import _dafny import UTF8 +# Ignore pylint not being able to read a module that requires the MPL +# pylint: disable=no-name-in-module from awses_test_vectors.internal.mpl.keyvectors_provider import KeyVectorsProvider from awses_test_vectors.manifests.keys import KeysManifest # noqa: disable=F401 From 48974b051d01003135761a405ec66fe03e68b385 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 16:11:13 -0700 Subject: [PATCH 362/422] debug --- .../src/awses_test_vectors/manifests/full_message/decrypt.py | 3 ++- .../manifests/full_message/decrypt_generation.py | 3 ++- .../src/awses_test_vectors/manifests/full_message/encrypt.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index 2aaaf1bca..6f8b43592 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -46,7 +46,8 @@ ) _HAS_MPL = True -except ImportError: +except ImportError as e: + print(f"ImportError: {e}") _HAS_MPL = False diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 6213fc3a6..45cd3b9d0 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -51,7 +51,8 @@ ) _HAS_MPL = True -except ImportError:" +except ImportError as e: + print(f"ImportError: {e}") _HAS_MPL = False diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 57de8504c..d06d543a2 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -50,7 +50,8 @@ from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_from_master_key_specs _HAS_MPL = True -except ImportError: +except ImportError as e: + print(f"ImportError: {e}") _HAS_MPL = False From 1bb55273a0e9a48e8d9cd29ff4e6ff216184a9ff Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 16:14:28 -0700 Subject: [PATCH 363/422] cleanup --- .../internal/mpl/keyvectors_provider.py | 27 +++ .../internal/mpl/tampering_mpl_materials.py | 172 ++++++++++++++++++ .../manifests/full_message/decrypt.py | 3 +- .../full_message/decrypt_generation.py | 3 +- .../manifests/full_message/encrypt.py | 3 +- 5 files changed, 202 insertions(+), 6 deletions(-) create mode 100644 test_vector_handlers/src/awses_test_vectors/internal/mpl/keyvectors_provider.py create mode 100644 test_vector_handlers/src/awses_test_vectors/internal/mpl/tampering_mpl_materials.py diff --git a/test_vector_handlers/src/awses_test_vectors/internal/mpl/keyvectors_provider.py b/test_vector_handlers/src/awses_test_vectors/internal/mpl/keyvectors_provider.py new file mode 100644 index 000000000..305459026 --- /dev/null +++ b/test_vector_handlers/src/awses_test_vectors/internal/mpl/keyvectors_provider.py @@ -0,0 +1,27 @@ +"""Singleton provider for the KeyVectors client.""" +# # Ignore missing MPL TestVectors for pylint, but the MPL TestVectors is required for this file +# pylint: disable=import-error +from aws_cryptography_materialproviderstestvectorkeys.smithygenerated.\ + aws_cryptography_materialproviderstestvectorkeys.client import ( + KeyVectors, + ) +from aws_cryptography_materialproviderstestvectorkeys.smithygenerated.\ + aws_cryptography_materialproviderstestvectorkeys.config import ( + KeyVectorsConfig + ) + +keyvectors_instances = {} + + +# pylint: disable=too-few-public-methods +class KeyVectorsProvider: + """Singleton manager for the KeyVectors client.""" + + instance: KeyVectors + + @classmethod + def get_keyvectors(cls, keys_path): + """Return the singleton KeyVectors client.""" + if keys_path not in keyvectors_instances: + keyvectors_instances[keys_path] = KeyVectors(KeyVectorsConfig(key_manifest_path=keys_path)) + return keyvectors_instances[keys_path] diff --git a/test_vector_handlers/src/awses_test_vectors/internal/mpl/tampering_mpl_materials.py b/test_vector_handlers/src/awses_test_vectors/internal/mpl/tampering_mpl_materials.py new file mode 100644 index 000000000..7ba471506 --- /dev/null +++ b/test_vector_handlers/src/awses_test_vectors/internal/mpl/tampering_mpl_materials.py @@ -0,0 +1,172 @@ +"""Allows using ESDK-MPL interfaces with the tampering tests. +These must ONLY be used in testing and NOT in production. +""" +import attr +import six +from copy import copy + + +from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager + +# Ignore missing MPL for pylint, but the MPL is required for this class +# pylint: disable=import-error,no-name-in-module +from aws_encryption_sdk.materials_managers.mpl.materials import ( + EncryptionMaterialsFromMPL +) +from aws_encryption_sdk.materials_managers.mpl.cmm import ( + CryptoMaterialsManagerFromMPL +) +from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders +from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig +from aws_cryptographic_materialproviders.mpl.models import ( + CreateDefaultCryptographicMaterialsManagerInput, +) + +try: + from aws_encryption_sdk.identifiers import AlgorithmSuite +except ImportError: + from aws_encryption_sdk.identifiers import Algorithm as AlgorithmSuite + + +class HalfSigningCryptoMaterialsManagerFromMPL(CryptoMaterialsManagerFromMPL): + """ + Custom CMM that uses HalfSigningEncryptionMaterialsFromMPL. + This extends CryptoMaterialsManagerFromMPL so ESDK-internal checks + follow MPL logic. + + THIS IS ONLY USED TO CREATE INVALID MESSAGES and should never be used in + production! + """ + + wrapped_default_cmm = attr.ib(validator=attr.validators.instance_of(CryptoMaterialsManagerFromMPL)) + + def __init__(self, master_key_provider): + """Create a new CMM that wraps a the given CMM.""" + mpl = AwsCryptographicMaterialProviders(MaterialProvidersConfig()) + mpl_cmm = mpl.create_default_cryptographic_materials_manager( + CreateDefaultCryptographicMaterialsManagerInput( + keyring=master_key_provider + ) + ) + self.wrapped_default_cmm = CryptoMaterialsManagerFromMPL(mpl_cmm=mpl_cmm) + + def get_encryption_materials(self, request): + """ + Generate half-signing materials by requesting signing materials + from the wrapped default CMM, and then changing the algorithm suite + and removing the signing key from teh result. + """ + if request.algorithm == AlgorithmSuite.AES_256_GCM_HKDF_SHA512_COMMIT_KEY: + signing_request = copy(request) + signing_request.algorithm = AlgorithmSuite.AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384 + + result = HalfSigningEncryptionMaterialsFromMPL( + self.wrapped_default_cmm.get_encryption_materials(signing_request) + ) + + result.algorithm = request.algorithm + result.signing_key = None + + return result + + raise NotImplementedError( + "The half-sign tampering method is only supported on the " + "AES_256_GCM_HKDF_SHA512_COMMIT_KEY algorithm suite." + ) + + def decrypt_materials(self, request): + """Thunks to the wrapped default CMM""" + return self.wrapped_default_cmm.decrypt_materials(request) + + +class HalfSigningEncryptionMaterialsFromMPL(EncryptionMaterialsFromMPL): + """Allows overriding properties inside the EncryptionMaterialsFromMPL. + The test vectors to this to "tamper" with the messages + and ensure they fail with expected errors. + This must ONLY be used in testing and NOT in production. + This is used in testing malicious message modification (HalfSigningTampering). + """ + + _underlying_materials: EncryptionMaterialsFromMPL + + def __init__(self, underling_materials): + self._underlying_materials = underling_materials + + # pylint thinks EncryptionMaterialsFromMPL.algorithm is a method + # pylint: disable=invalid-overridden-method + @property + def algorithm(self): + """Return any previously-provided overriden algorithm; + if none was provided, returns underlying algorithm from encryption materials. + """ + if hasattr(self, "set_algorithm"): + return self.set_algorithm + return self._underlying_materials.algorithm + + @algorithm.setter + def algorithm(self, algorithm): + self.set_algorithm = algorithm + + # pylint thinks EncryptionMaterialsFromMPL.signing_key is a method + # pylint: disable=invalid-overridden-method + @property + def signing_key(self): + """Return any previously-provided overriden signing_key; + if none was provided, returns underlying signing_key from encryption materials. + """ + if hasattr(self, "set_signing_key"): + return self.set_signing_key + return self._underlying_materials.algorithm + + @signing_key.setter + def signing_key(self, signing_key): + self.set_signing_key = signing_key + + @property + def encryption_context(self): + return self._underlying_materials.encryption_context + + @property + def encrypted_data_keys(self): + return self._underlying_materials.encrypted_data_keys + + @property + def data_encryption_key(self): + return self._underlying_materials.data_encryption_key + + @property + def required_encryption_context_keys(self): + return self._underlying_materials.required_encryption_context_keys + + +class ProviderInfoChangingCryptoMaterialsManagerFromMPL(CryptoMaterialsManagerFromMPL): + """ + Custom CMM that modifies the provider info field on EDKs. + This extends CryptoMaterialsManagerFromMPL so ESDK-internal checks + follow MPL logic. + + THIS IS ONLY USED TO CREATE INVALID MESSAGES and should never be used in + production! + """ + + wrapped_cmm = attr.ib(validator=attr.validators.instance_of(CryptoMaterialsManager)) + new_provider_info = attr.ib(validator=attr.validators.instance_of(six.string_types)) + + def __init__(self, materials_manager, new_provider_info): + """Create a new CMM that wraps a the given CMM.""" + self.wrapped_cmm = materials_manager + self.new_provider_info = new_provider_info + + def get_encryption_materials(self, request): + """ + Request materials from the wrapped CMM, and then change the provider info + on each EDK. + """ + result = self.wrapped_cmm.get_encryption_materials(request) + for encrypted_data_key in result.encrypted_data_keys: + encrypted_data_key.key_provider.key_info = self.new_provider_info + return result + + def decrypt_materials(self, request): + """Thunks to the wrapped CMM""" + return self.wrapped_cmm.decrypt_materials(request) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index 6f8b43592..2aaaf1bca 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -46,8 +46,7 @@ ) _HAS_MPL = True -except ImportError as e: - print(f"ImportError: {e}") +except ImportError: _HAS_MPL = False diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 45cd3b9d0..50c14a091 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -51,8 +51,7 @@ ) _HAS_MPL = True -except ImportError as e: - print(f"ImportError: {e}") +except ImportError: _HAS_MPL = False diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index d06d543a2..57de8504c 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -50,8 +50,7 @@ from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_from_master_key_specs _HAS_MPL = True -except ImportError as e: - print(f"ImportError: {e}") +except ImportError: _HAS_MPL = False From 72de35b7a20e80114001dd061ddb039038c344f2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 16:20:14 -0700 Subject: [PATCH 364/422] cleanup --- .../internal/mpl/tampering_mpl_materials.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/internal/mpl/tampering_mpl_materials.py b/test_vector_handlers/src/awses_test_vectors/internal/mpl/tampering_mpl_materials.py index 7ba471506..8b540f8dd 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/mpl/tampering_mpl_materials.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/mpl/tampering_mpl_materials.py @@ -89,8 +89,10 @@ class HalfSigningEncryptionMaterialsFromMPL(EncryptionMaterialsFromMPL): _underlying_materials: EncryptionMaterialsFromMPL - def __init__(self, underling_materials): - self._underlying_materials = underling_materials + def __init__(self, underlying_materials): + """Creates a HalfSigningEncryptionMaterialsFromMPL wrapper + around underlying_materials.""" + self._underlying_materials = underlying_materials # pylint thinks EncryptionMaterialsFromMPL.algorithm is a method # pylint: disable=invalid-overridden-method @@ -124,18 +126,22 @@ def signing_key(self, signing_key): @property def encryption_context(self): + """Get encryption_context from _underlying_materials.""" return self._underlying_materials.encryption_context @property def encrypted_data_keys(self): + """Get encrypted_data_keys from _underlying_materials.""" return self._underlying_materials.encrypted_data_keys @property def data_encryption_key(self): + """Get data_encryption_key from _underlying_materials.""" return self._underlying_materials.data_encryption_key @property def required_encryption_context_keys(self): + """Get required_encryption_context_keys from _underlying_materials.""" return self._underlying_materials.required_encryption_context_keys From f2792bccea7980b4b990e2bc1cc87b5cc768eba6 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 16:22:27 -0700 Subject: [PATCH 365/422] cleanup --- .../src/awses_test_vectors/internal/mpl/__init__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 test_vector_handlers/src/awses_test_vectors/internal/mpl/__init__.py diff --git a/test_vector_handlers/src/awses_test_vectors/internal/mpl/__init__.py b/test_vector_handlers/src/awses_test_vectors/internal/mpl/__init__.py new file mode 100644 index 000000000..a9f648dff --- /dev/null +++ b/test_vector_handlers/src/awses_test_vectors/internal/mpl/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Internal modules that require the aws-cryptographic-material-providers library.""" From df45d5db58691d70e2aa4f17c7f4ab9ab5887628 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 16:25:45 -0700 Subject: [PATCH 366/422] cleanup --- .../awses_test_vectors/internal/mpl/__init__.py | 14 ++------------ .../internal/mpl/tampering_mpl_materials.py | 7 ++++--- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/internal/mpl/__init__.py b/test_vector_handlers/src/awses_test_vectors/internal/mpl/__init__.py index a9f648dff..11e9569d9 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/mpl/__init__.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/mpl/__init__.py @@ -1,13 +1,3 @@ -# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You -# may not use this file except in compliance with the License. A copy of -# the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is -# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific -# language governing permissions and limitations under the License. +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 """Internal modules that require the aws-cryptographic-material-providers library.""" diff --git a/test_vector_handlers/src/awses_test_vectors/internal/mpl/tampering_mpl_materials.py b/test_vector_handlers/src/awses_test_vectors/internal/mpl/tampering_mpl_materials.py index 8b540f8dd..29a820ecc 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/mpl/tampering_mpl_materials.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/mpl/tampering_mpl_materials.py @@ -1,9 +1,9 @@ """Allows using ESDK-MPL interfaces with the tampering tests. These must ONLY be used in testing and NOT in production. """ +from copy import copy import attr import six -from copy import copy from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager @@ -90,8 +90,9 @@ class HalfSigningEncryptionMaterialsFromMPL(EncryptionMaterialsFromMPL): _underlying_materials: EncryptionMaterialsFromMPL def __init__(self, underlying_materials): - """Creates a HalfSigningEncryptionMaterialsFromMPL wrapper - around underlying_materials.""" + """Create a HalfSigningEncryptionMaterialsFromMPL wrapper + around underlying_materials. + """ self._underlying_materials = underlying_materials # pylint thinks EncryptionMaterialsFromMPL.algorithm is a method From c66938f22081d483b790c2fa00137f0b3759803e Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Mar 2024 16:47:33 -0700 Subject: [PATCH 367/422] cleanup --- codebuild/py311/mplawses_local_mpl.yml | 26 -------------------------- codebuild/py312/mplawses_local_mpl.yml | 26 -------------------------- codebuild/py39/awses_local.yml | 25 ------------------------- 3 files changed, 77 deletions(-) delete mode 100644 codebuild/py311/mplawses_local_mpl.yml delete mode 100644 codebuild/py312/mplawses_local_mpl.yml delete mode 100644 codebuild/py39/awses_local.yml diff --git a/codebuild/py311/mplawses_local_mpl.yml b/codebuild/py311/mplawses_local_mpl.yml deleted file mode 100644 index 92dbdb086..000000000 --- a/codebuild/py311/mplawses_local_mpl.yml +++ /dev/null @@ -1,26 +0,0 @@ -version: 0.2 - -env: - variables: - TOXENV: "py311-mplvectors-mpl" - REGION: "us-west-2" - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- - arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- - arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- - arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- - arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" - -phases: - install: - runtime-versions: - python: 3.11 - build: - commands: - - pip install "tox < 4.0" - - cd test_vector_handlers - - tox diff --git a/codebuild/py312/mplawses_local_mpl.yml b/codebuild/py312/mplawses_local_mpl.yml deleted file mode 100644 index 8a7d5f5c6..000000000 --- a/codebuild/py312/mplawses_local_mpl.yml +++ /dev/null @@ -1,26 +0,0 @@ -version: 0.2 - -env: - variables: - TOXENV: "py312-mplvectors-mpl" - REGION: "us-west-2" - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- - arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- - arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- - arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- - arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" - -phases: - install: - runtime-versions: - python: 3.12 - build: - commands: - - pip install "tox < 4.0" - - cd test_vector_handlers - - tox diff --git a/codebuild/py39/awses_local.yml b/codebuild/py39/awses_local.yml deleted file mode 100644 index e56a9ff45..000000000 --- a/codebuild/py39/awses_local.yml +++ /dev/null @@ -1,25 +0,0 @@ -version: 0.2 - -env: - variables: - TOXENV: "py39-awses_local" - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- - arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- - arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- - arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- - arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_API_DEPLOYMENT_ID: "xi1mwx3ttb" - AWS_ENCRYPTION_SDK_PYTHON_DECRYPT_ORACLE_REGION: "us-west-2" - -phases: - install: - runtime-versions: - python: 3.9 - build: - commands: - - pip install "tox < 4.0" - - cd test_vector_handlers - - tox From b594a38a7aa70ff5c49e48e4e8f0a4be02bdfe26 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 23 Apr 2024 09:40:21 -0700 Subject: [PATCH 368/422] hierarchy --- .../manifests/full_message/decrypt.py | 19 ++++-- .../full_message/decrypt_generation.py | 25 ++++++-- .../manifests/full_message/encrypt.py | 5 +- .../src/awses_test_vectors/manifests/keys.py | 61 ++++++++++++++++++- .../manifests/master_key.py | 18 ++++-- .../manifests/mpl_keyring.py | 30 +++++++++ .../aws-crypto-tools-test-vector-framework | 2 +- 7 files changed, 144 insertions(+), 16 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index 2aaaf1bca..36dad3ed4 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -222,7 +222,7 @@ class MessageDecryptionTestScenario(object): master_key_provider_fn = attr.ib(validator=attr.validators.is_callable()) result = attr.ib(validator=attr.validators.instance_of(MessageDecryptionTestResult)) keyrings = attr.ib(validator=attr.validators.instance_of(bool)) - cmm_type = attr.ib(validator=attr.validators.instance_of(str)) + cmm_type = attr.ib(validator=attr.validators.optional(attr.validators.instance_of(str))) decryption_method = attr.ib( default=None, validator=attr.validators.optional(attr.validators.instance_of(DecryptionMethod)) ) @@ -288,7 +288,8 @@ def from_scenario( ] else: master_key_specs = [ - MasterKeySpec.from_scenario(spec) for spec in raw_master_key_specs + MasterKeySpec.from_scenario(spec) for spec in raw_master_key_specs \ + if spec["type"] != "aws-kms-hierarchy" ] def master_key_provider_fn(): @@ -301,13 +302,14 @@ def master_key_provider_fn(): result_spec = scenario["result"] result = MessageDecryptionTestResult.from_result_spec(result_spec, plaintext_reader) - if "encryption-context" in scenario: + + if hasattr(scenario, "encryption-context"): encryption_context = scenario["encryption-context"] else: encryption_context = {} # MPL test vectors add CMM types to the test vectors manifests - if "cmm" in scenario: + if hasattr(scenario, "cmm"): if scenario["cmm"] == "Default": # Master keys and keyrings can handle default CMM cmm_type = scenario["cmm"] @@ -325,6 +327,12 @@ def master_key_provider_fn(): # If unspecified, set "Default" as the default cmm_type = "Default" + # If this scenario does not have any key providers, + # do not create a scenario. + # Caller logic should expect `None` to mean "no scenario". + if master_key_provider_fn() is None: + return None + return cls( ciphertext_uri=scenario["ciphertext"], ciphertext=ciphertext_reader(scenario["ciphertext"]), @@ -355,6 +363,9 @@ def scenario_spec(self): spec["decryption-method"] = self.decryption_method.value if self.description is not None: spec["description"] = self.description + spec["cmm"] = self.cmm_type + spec["encryption-context"] = self.encryption_context + return spec def _one_shot_decrypt(self): diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 50c14a091..5a7c51b84 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -91,7 +91,7 @@ # We only actually need these imports when running the mypy checks pass -SUPPORTED_VERSIONS = (2,) +SUPPORTED_VERSIONS = (2,4,) class TamperingMethod: @@ -420,6 +420,8 @@ class MessageDecryptionTestScenarioGenerator(object): decryption_master_key_provider_fn = attr.ib(validator=attr.validators.is_callable()) result = attr.ib(validator=attr.validators.optional(attr.validators.instance_of(MessageDecryptionTestResult))) keyrings = attr.ib(validator=attr.validators.instance_of(bool)) + cmm_type = attr.ib(validator=attr.validators.optional(attr.validators.instance_of(str))) + encryption_context = attr.ib(validator=attr.validators.optional(attr.validators.instance_of(dict))) @classmethod def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): @@ -467,6 +469,16 @@ def decryption_master_key_provider_fn(): result_spec = scenario.get("result") result = MessageDecryptionTestResult.from_result_spec(result_spec, None) if result_spec else None + try: + encryption_context = encryption_scenario_spec["encryption-context"] + except KeyError: + encryption_context = None + + try: + cmm_type = encryption_scenario_spec["cmm"] + except KeyError: + cmm_type = None + return cls( encryption_scenario=encryption_scenario, tampering_method=tampering_method, @@ -475,6 +487,8 @@ def decryption_master_key_provider_fn(): decryption_master_key_provider_fn=decryption_master_key_provider_fn, result=result, keyrings=keyrings, + cmm_type=cmm_type, + encryption_context=encryption_context, ) def run(self, ciphertext_writer, plaintext_uri): @@ -504,8 +518,8 @@ def decryption_test_scenario_pair(self, ciphertext_writer, ciphertext_to_decrypt decryption_method=self.decryption_method, result=expected_result, keyrings=self.keyrings, - cmm_type="Default", - encryption_context={} + cmm_type=self.cmm_type, + encryption_context=self.encryption_context, ), ) @@ -573,8 +587,9 @@ def from_file(cls, input_file, keyrings): tests[name] = MessageDecryptionTestScenarioGenerator.from_scenario( scenario=scenario, keys=keys, plaintexts=plaintexts, keyrings=keyrings, keys_uri=keys_abs_path, ) - except NotImplementedError: - continue + except NotImplementedError as e: + # continue + raise e return cls( version=raw_manifest["manifest"]["version"], keys=keys, diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 57de8504c..6b19b921b 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -50,7 +50,8 @@ from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_from_master_key_specs _HAS_MPL = True -except ImportError: +except ImportError as e: + print(e) _HAS_MPL = False @@ -93,6 +94,7 @@ class MessageEncryptionTestScenario(object): master_key_specs = attr.ib(validator=iterable_validator(list, MasterKeySpec)) master_key_provider_fn = attr.ib(validator=attr.validators.is_callable()) keyrings = attr.ib(validator=attr.validators.instance_of(bool)) + cmm = attr.ib(validator=attr.validators.instance_of(str)) @classmethod def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): @@ -133,6 +135,7 @@ def master_key_provider_fn(): master_key_specs=master_key_specs, master_key_provider_fn=master_key_provider_fn, keyrings=keyrings, + cmm=scenario["cmm"], ) def run(self, materials_manager=None): diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/keys.py b/test_vector_handlers/src/awses_test_vectors/manifests/keys.py index cba6b7e25..b2685a11d 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/keys.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/keys.py @@ -29,6 +29,7 @@ from awses_test_vectors.internal.mypy_types import ( # noqa pylint: disable=unused-import AWS_KMS_KEY_SPEC, + AWS_KMS_HIERARCHY_KEY_SPEC, KEY_SPEC, KEYS_MANIFEST, MANIFEST_VERSION, @@ -110,6 +111,50 @@ def manifest_spec(self): } +@attr.s(init=False) +class AwsKmsHierarchyKeySpec(KeySpec): + """AWS KMS hierarchy key specification. + + :param bool encrypt: Key can be used to encrypt + :param bool decrypt: Key can be used to decrypt + :param str type_name: Master key type name (must be "static-branch-key") + :param str key_id: Branch key ID + """ + + # pylint: disable=too-few-public-methods + + type_name = attr.ib(validator=membership_validator(("static-branch-key",))) + + def __init__(self, encrypt, decrypt, type_name, key_id, branch_key_version, branch_key, beacon_key): # noqa=D107 + # type: (bool, bool, str, str) -> None + # Workaround pending resolution of attrs/mypy interaction. + # https://github.com/python/mypy/issues/2088 + # https://github.com/python-attrs/attrs/issues/215 + self.type_name = type_name + self.branch_key_version = branch_key_version + self.branch_key = branch_key + self.beacon_key = beacon_key + super(AwsKmsHierarchyKeySpec, self).__init__(encrypt, decrypt, key_id) + + @property + def manifest_spec(self): + # type: () -> AWS_KMS_HIERARCHY_KEY_SPEC + """Build a key specification describing this key specification. + + :return: Key specification JSON + :rtype: dict + """ + return { + "encrypt": self.encrypt, + "decrypt": self.decrypt, + "type": self.type_name, + "key-id": self.key_id, + "branchKeyVersion": self.branch_key_version, + "branchKey": self.branch_key, + "beaconKey": self.beacon_key, + } + + @attr.s(init=False) class ManualKeySpec(KeySpec): # pylint: disable=too-many-arguments @@ -206,8 +251,22 @@ def key_from_manifest_spec(key_spec): key_id = key_spec["key-id"] # type: str return AwsKmsKeySpec(encrypt=encrypt, decrypt=decrypt, type_name=type_name, key_id=key_id) - algorithm = key_spec["algorithm"] # type: str + elif key_spec["type"] == "static-branch-key": + branch_key_version = key_spec["branchKeyVersion"] # type: str + branch_key = key_spec["branchKey"] # type: str + beacon_key = key_spec["beaconKey"] # type: str + return AwsKmsHierarchyKeySpec( + encrypt=encrypt, + decrypt=decrypt, + type_name=type_name, + key_id=key_id, + branch_key_version=branch_key_version, + branch_key=branch_key, + beacon_key=beacon_key, + ) + bits = key_spec["bits"] # type: int + algorithm = key_spec["algorithm"] encoding = key_spec["encoding"] # type: str material = key_spec["material"] # type: str return ManualKeySpec( diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/master_key.py b/test_vector_handlers/src/awses_test_vectors/manifests/master_key.py index a1a7ae4af..c110effa1 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/master_key.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/master_key.py @@ -44,7 +44,7 @@ # We only actually need these imports when running the mypy checks pass -KNOWN_TYPES = ("aws-kms", "aws-kms-mrk-aware", "aws-kms-mrk-aware-discovery", "raw") +KNOWN_TYPES = ("aws-kms", "aws-kms-mrk-aware", "aws-kms-mrk-aware-discovery", "raw", "aws-kms-hierarchy") KNOWN_ALGORITHMS = ("aes", "rsa") KNOWN_PADDING = ("pkcs1", "oaep-mgf1") KNOWN_PADDING_HASH = ("sha1", "sha256", "sha384", "sha512") @@ -99,8 +99,8 @@ class MasterKeySpec(object): # pylint: disable=too-many-instance-attributes def __attrs_post_init__(self): # type: () -> None """Verify that known types all have loaders and that all required parameters are provided.""" - if set(KNOWN_TYPES) != set(self._MASTER_KEY_LOADERS.keys()): - raise NotImplementedError("Gap found between known master key types and available master key loaders.") + # if set(KNOWN_TYPES) != set(self._MASTER_KEY_LOADERS.keys()): + # raise NotImplementedError("Gap found between known master key types and available master key loaders.") if self.type_name == "raw": if None in (self.provider_id, self.encryption_algorithm): @@ -311,7 +311,17 @@ def master_key_provider_from_master_key_specs(keys, master_key_specs): :return: Master key provider combining all loaded master keys :rtype: MasterKeyProvider """ - master_keys = [spec.master_key(keys) for spec in master_key_specs] + master_keys = [] + for spec in master_key_specs: + try: + master_keys.append(spec.master_key(keys)) + # If spec is not a valid master key + # (e.g. hierarchical keyring) + # do not make a master key + except KeyError: + pass + if len(master_keys) == 0: + return None primary = master_keys[0] others = master_keys[1:] for master_key in others: diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index ec35147c0..2b18291b2 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -30,6 +30,11 @@ from aws_cryptographic_materialproviders.mpl.references import IKeyring from aws_cryptographic_materialproviders.mpl.models import CreateMultiKeyringInput +from .master_key import KNOWN_TYPES as MASTER_KEY_KNOWN_TYPES +from awses_test_vectors.internal.util import membership_validator + +KEYRING_ONLY_KNOWN_TYPES = ("aws-kms-hierarchy") + import _dafny import UTF8 @@ -56,6 +61,15 @@ class KeyringSpec(MasterKeySpec): # pylint: disable=too-many-instance-attribute :param str padding_hash: Wrapping key padding hash (required for raw master keys) """ + # type_name = attr.ib(validator=membership_validator(set(MASTER_KEY_KNOWN_TYPES).union(KEYRING_ONLY_KNOWN_TYPES))) + + def __attrs_post_init__(self): + # type: () -> None + """Verify that known types all have loaders and that all required parameters are provided.""" + # if set(KEYRING_ONLY_KNOWN_TYPES) != set(self._KEYRING_LOADERS.keys()): + # raise NotImplementedError("Gap found between known master key types and available master key loaders.") + # super().__attrs_post_init__() + def keyring(self, keys_uri, mode): # type: (KeysManifest) -> IKeyring """Build a keyring using this specification. @@ -73,6 +87,7 @@ def keyring(self, keys_uri, mode): "key": self.key_name, "provider-id": self.provider_id, "encryption-algorithm": self.encryption_algorithm, + # "keyDescription": } if self.padding_algorithm is not None and self.padding_algorithm != "": @@ -154,6 +169,21 @@ def keyring(self, keys_uri, mode): return keyring + + def _kms_hierarchy_keyring_from_spec(self, keys): + # type: (KeysManifest) -> AwsKmsHierarchyKeyring + """Build an AWS KMS hierarchy keyring using this specification. + + :param KeySpec key_spec: Key specification to use with this master key + :return: AWS KMS hierarchy keyring based on this specification + :rtype: AwsKmsHierarchyKeyring + :raises TypeError: if this is not an AWS KMS master key specification + """ + if not self.type_name == "aws-kms-hierarchy": + raise TypeError("This is not an AWS KMS hierarchy key") + + return keyring_from_master_key_specs(keys_uri, ) + def keyring_from_master_key_specs(keys_uri, master_key_specs, mode): # type: (str, list[KeyringSpec]) -> IKeyring diff --git a/test_vector_handlers/test/aws-crypto-tools-test-vector-framework b/test_vector_handlers/test/aws-crypto-tools-test-vector-framework index c3d73fae2..9eb2fcbbe 160000 --- a/test_vector_handlers/test/aws-crypto-tools-test-vector-framework +++ b/test_vector_handlers/test/aws-crypto-tools-test-vector-framework @@ -1 +1 @@ -Subproject commit c3d73fae260fd9e9cc9e746f09a7ffbab83576e2 +Subproject commit 9eb2fcbbe47ab30c29d6ad9a8125b1064e0db42a From 0da3fe967bbec561b1914af1811ff59318759dc5 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 1 May 2024 13:51:41 -0700 Subject: [PATCH 369/422] ?? --- .../internal/utils/__init__.py | 3 ++ .../materials_managers/mpl/materials.py | 1 + .../manifests/full_message/decrypt.py | 4 +- .../manifests/full_message/encrypt.py | 1 + .../manifests/mpl_keyring.py | 37 +++++++++++++------ 5 files changed, 32 insertions(+), 14 deletions(-) diff --git a/src/aws_encryption_sdk/internal/utils/__init__.py b/src/aws_encryption_sdk/internal/utils/__init__.py index b08121281..bde631473 100644 --- a/src/aws_encryption_sdk/internal/utils/__init__.py +++ b/src/aws_encryption_sdk/internal/utils/__init__.py @@ -115,6 +115,7 @@ def prepare_data_keys(primary_master_key, master_keys, algorithm, encryption_con encrypted_data_encryption_key = None data_encryption_key = primary_master_key.generate_data_key(algorithm, encryption_context) _LOGGER.debug("encryption data generated with master key: %s", data_encryption_key.key_provider) + input(f"{master_keys=}") for master_key in master_keys: # Don't re-encrypt the encryption data key; we already have the ciphertext if master_key is primary_master_key: @@ -128,6 +129,8 @@ def prepare_data_keys(primary_master_key, master_keys, algorithm, encryption_con ) encrypted_data_keys.add(encrypted_key) _LOGGER.debug("encryption key encrypted with master key: %s", master_key.key_provider) + input(f"{data_encryption_key=}") + input(f"{encrypted_data_keys=}") return data_encryption_key, encrypted_data_keys diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index 54ea21b39..0b20af14d 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -68,6 +68,7 @@ def encryption_context(self) -> Dict[str, str]: def encrypted_data_keys(self) -> List[Native_EncryptedDataKey]: """Materials' encrypted data keys.""" mpl_edk_list: List[MPL_EncryptedDataKey] = self.mpl_materials.encrypted_data_keys + input(f"{mpl_edk_list=}") key_blob_list: Set[Native_EncryptedDataKey] = {Native_EncryptedDataKey( key_provider=MasterKeyInfo( provider_id=mpl_edk.key_provider_id, diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index 36dad3ed4..470b99c9e 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -303,13 +303,13 @@ def master_key_provider_fn(): result = MessageDecryptionTestResult.from_result_spec(result_spec, plaintext_reader) - if hasattr(scenario, "encryption-context"): + if "encryption-context" in scenario: encryption_context = scenario["encryption-context"] else: encryption_context = {} # MPL test vectors add CMM types to the test vectors manifests - if hasattr(scenario, "cmm"): + if "cmm" in scenario: if scenario["cmm"] == "Default": # Master keys and keyrings can handle default CMM cmm_type = scenario["cmm"] diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 6b19b921b..22cc8c4a4 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -168,6 +168,7 @@ def run(self, materials_manager=None): else: raise TypeError(f"Unrecognized master_key_provider_fn return type: {self.master_key_provider_fn()}") ciphertext, _header = client.encrypt(**encrypt_kwargs) + input(f"{_header=}") return ciphertext diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index 2b18291b2..1a5a07321 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -106,10 +106,10 @@ def keyring(self, keys_uri, mode): # KeyVectors requires a public key to encrypt. # If this is not done, then keyring.OnEncrypt fails with # "A RawRSAKeyring without a public key cannot provide OnEncrypt" - if input_kwargs["key"] == "rsa-4096-private" \ - and mode in ("decrypt-generate", "encrypt"): - changed_key_name_from_private_to_public = True - input_kwargs["key"] = "rsa-4096-public" + # if input_kwargs["key"] == "rsa-4096-private" \ + # and mode in ("decrypt-generate", "encrypt"): + # changed_key_name_from_private_to_public = True + # input_kwargs["key"] = "rsa-4096-public" # Specify default padding-hash if "padding-hash" not in input_kwargs: input_kwargs["padding-hash"] = "sha1" @@ -159,13 +159,13 @@ def keyring(self, keys_uri, mode): # This configuration seems to be correct, because # all of the test vectors (master keys and MPL) pass with these two hacks. # But this seems weird, and we didn't have to do this in Java. - if hasattr(keyring, "_impl"): # pylint: disable=protected-access - if hasattr(keyring._impl, "_keyName"): # pylint: disable=protected-access - if keyring._impl._keyName == UTF8.default__.Encode(_dafny.Seq("rsa-4096-public")).value \ - and mode in ("decrypt-generate", "encrypt"): # pylint: disable=protected-access - if changed_key_name_from_private_to_public: - # pylint: disable=protected-access - keyring._impl._keyName = UTF8.default__.Encode(_dafny.Seq("rsa-4096-private")).value + # if hasattr(keyring, "_impl"): # pylint: disable=protected-access + # if hasattr(keyring._impl, "_keyName"): # pylint: disable=protected-access + # if keyring._impl._keyName == UTF8.default__.Encode(_dafny.Seq("rsa-4096-public")).value \ + # and mode in ("decrypt-generate", "encrypt"): # pylint: disable=protected-access + # if changed_key_name_from_private_to_public: + # # pylint: disable=protected-access + # keyring._impl._keyName = UTF8.default__.Encode(_dafny.Seq("rsa-4096-private")).value return keyring @@ -196,9 +196,22 @@ def keyring_from_master_key_specs(keys_uri, master_key_specs, mode): :return: Master key provider combining all loaded master keys :rtype: IKeyring """ - keyrings = [spec.keyring(keys_uri, mode) for spec in master_key_specs] + sorted_specs = [] + end_specs = [] + for spec in master_key_specs: + if spec.key_name == "rsa-4096-private": + end_specs.append(spec) + else: + sorted_specs.append(spec) + for end_spec in end_specs: + sorted_specs.append(end_spec) + input(sorted_specs) + + keyrings = [spec.keyring(keys_uri, mode) for spec in sorted_specs] primary = keyrings[0] + input(f"{primary=}") others = keyrings[1:] + input(f"{others=}") mpl: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( MaterialProvidersConfig() From 7cd887431a4880e2b62085461cfc59c9da8d858d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 1 May 2024 16:20:17 -0700 Subject: [PATCH 370/422] found it --- .../internal/crypto/wrapping_keys.py | 9 ++++--- .../materials_managers/mpl/materials.py | 1 - .../manifests/mpl_keyring.py | 26 +++++++------------ 3 files changed, 15 insertions(+), 21 deletions(-) diff --git a/src/aws_encryption_sdk/internal/crypto/wrapping_keys.py b/src/aws_encryption_sdk/internal/crypto/wrapping_keys.py index 91f9fd834..ba6135965 100644 --- a/src/aws_encryption_sdk/internal/crypto/wrapping_keys.py +++ b/src/aws_encryption_sdk/internal/crypto/wrapping_keys.py @@ -98,9 +98,12 @@ def decrypt(self, encrypted_wrapped_data_key, encryption_context): if self.wrapping_key_type is EncryptionKeyType.PUBLIC: raise IncorrectMasterKeyError("Public key cannot decrypt") if self.wrapping_key_type is EncryptionKeyType.PRIVATE: - return self._wrapping_key.decrypt( - ciphertext=encrypted_wrapped_data_key.ciphertext, padding=self.wrapping_algorithm.padding - ) + try: + return self._wrapping_key.decrypt( + ciphertext=encrypted_wrapped_data_key.ciphertext, padding=self.wrapping_algorithm.padding + ) + except ValueError as e: + raise IncorrectMasterKeyError("_wrapping_key cannot decrypt provided ciphertext") serialized_encryption_context = serialize_encryption_context(encryption_context=encryption_context) return decrypt( algorithm=self.wrapping_algorithm.algorithm, diff --git a/src/aws_encryption_sdk/materials_managers/mpl/materials.py b/src/aws_encryption_sdk/materials_managers/mpl/materials.py index 0b20af14d..54ea21b39 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/materials.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/materials.py @@ -68,7 +68,6 @@ def encryption_context(self) -> Dict[str, str]: def encrypted_data_keys(self) -> List[Native_EncryptedDataKey]: """Materials' encrypted data keys.""" mpl_edk_list: List[MPL_EncryptedDataKey] = self.mpl_materials.encrypted_data_keys - input(f"{mpl_edk_list=}") key_blob_list: Set[Native_EncryptedDataKey] = {Native_EncryptedDataKey( key_provider=MasterKeyInfo( provider_id=mpl_edk.key_provider_id, diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index 1a5a07321..917cb7605 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -106,10 +106,11 @@ def keyring(self, keys_uri, mode): # KeyVectors requires a public key to encrypt. # If this is not done, then keyring.OnEncrypt fails with # "A RawRSAKeyring without a public key cannot provide OnEncrypt" - # if input_kwargs["key"] == "rsa-4096-private" \ - # and mode in ("decrypt-generate", "encrypt"): - # changed_key_name_from_private_to_public = True - # input_kwargs["key"] = "rsa-4096-public" + if input_kwargs["key"] == "rsa-4096-private" \ + and mode in ("decrypt-generate", "encrypt"): + changed_key_name_from_private_to_public = True + input("YUP") + input_kwargs["key"] = "rsa-4096-public" # Specify default padding-hash if "padding-hash" not in input_kwargs: input_kwargs["padding-hash"] = "sha1" @@ -123,6 +124,8 @@ def keyring(self, keys_uri, mode): GetKeyDescriptionInput(json=encoded_json) ) + input(f"{output.key_description=}") + keyring: IKeyring = keyvectors.create_test_vector_keyring( TestVectorKeyringInput( key_description=output.key_description @@ -182,7 +185,7 @@ def _kms_hierarchy_keyring_from_spec(self, keys): if not self.type_name == "aws-kms-hierarchy": raise TypeError("This is not an AWS KMS hierarchy key") - return keyring_from_master_key_specs(keys_uri, ) + return keyring_from_master_key_specs(keys, ) def keyring_from_master_key_specs(keys_uri, master_key_specs, mode): @@ -196,18 +199,7 @@ def keyring_from_master_key_specs(keys_uri, master_key_specs, mode): :return: Master key provider combining all loaded master keys :rtype: IKeyring """ - sorted_specs = [] - end_specs = [] - for spec in master_key_specs: - if spec.key_name == "rsa-4096-private": - end_specs.append(spec) - else: - sorted_specs.append(spec) - for end_spec in end_specs: - sorted_specs.append(end_spec) - input(sorted_specs) - - keyrings = [spec.keyring(keys_uri, mode) for spec in sorted_specs] + keyrings = [spec.keyring(keys_uri, mode) for spec in master_key_specs] primary = keyrings[0] input(f"{primary=}") others = keyrings[1:] From b58c5a0cbbc324c2efeef8fd5cfe7729d438a611 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 1 May 2024 16:46:43 -0700 Subject: [PATCH 371/422] more fix --- .../internal/utils/__init__.py | 3 --- .../manifests/full_message/encrypt.py | 1 - .../manifests/mpl_keyring.py | 19 +++++++------------ 3 files changed, 7 insertions(+), 16 deletions(-) diff --git a/src/aws_encryption_sdk/internal/utils/__init__.py b/src/aws_encryption_sdk/internal/utils/__init__.py index bde631473..b08121281 100644 --- a/src/aws_encryption_sdk/internal/utils/__init__.py +++ b/src/aws_encryption_sdk/internal/utils/__init__.py @@ -115,7 +115,6 @@ def prepare_data_keys(primary_master_key, master_keys, algorithm, encryption_con encrypted_data_encryption_key = None data_encryption_key = primary_master_key.generate_data_key(algorithm, encryption_context) _LOGGER.debug("encryption data generated with master key: %s", data_encryption_key.key_provider) - input(f"{master_keys=}") for master_key in master_keys: # Don't re-encrypt the encryption data key; we already have the ciphertext if master_key is primary_master_key: @@ -129,8 +128,6 @@ def prepare_data_keys(primary_master_key, master_keys, algorithm, encryption_con ) encrypted_data_keys.add(encrypted_key) _LOGGER.debug("encryption key encrypted with master key: %s", master_key.key_provider) - input(f"{data_encryption_key=}") - input(f"{encrypted_data_keys=}") return data_encryption_key, encrypted_data_keys diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 22cc8c4a4..6b19b921b 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -168,7 +168,6 @@ def run(self, materials_manager=None): else: raise TypeError(f"Unrecognized master_key_provider_fn return type: {self.master_key_provider_fn()}") ciphertext, _header = client.encrypt(**encrypt_kwargs) - input(f"{_header=}") return ciphertext diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index 917cb7605..67b214d3a 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -109,7 +109,6 @@ def keyring(self, keys_uri, mode): if input_kwargs["key"] == "rsa-4096-private" \ and mode in ("decrypt-generate", "encrypt"): changed_key_name_from_private_to_public = True - input("YUP") input_kwargs["key"] = "rsa-4096-public" # Specify default padding-hash if "padding-hash" not in input_kwargs: @@ -124,8 +123,6 @@ def keyring(self, keys_uri, mode): GetKeyDescriptionInput(json=encoded_json) ) - input(f"{output.key_description=}") - keyring: IKeyring = keyvectors.create_test_vector_keyring( TestVectorKeyringInput( key_description=output.key_description @@ -162,13 +159,13 @@ def keyring(self, keys_uri, mode): # This configuration seems to be correct, because # all of the test vectors (master keys and MPL) pass with these two hacks. # But this seems weird, and we didn't have to do this in Java. - # if hasattr(keyring, "_impl"): # pylint: disable=protected-access - # if hasattr(keyring._impl, "_keyName"): # pylint: disable=protected-access - # if keyring._impl._keyName == UTF8.default__.Encode(_dafny.Seq("rsa-4096-public")).value \ - # and mode in ("decrypt-generate", "encrypt"): # pylint: disable=protected-access - # if changed_key_name_from_private_to_public: - # # pylint: disable=protected-access - # keyring._impl._keyName = UTF8.default__.Encode(_dafny.Seq("rsa-4096-private")).value + if hasattr(keyring, "_impl"): # pylint: disable=protected-access + if hasattr(keyring._impl, "_keyName"): # pylint: disable=protected-access + if keyring._impl._keyName == UTF8.default__.Encode(_dafny.Seq("rsa-4096-public")).value \ + and mode in ("decrypt-generate", "encrypt"): # pylint: disable=protected-access + if changed_key_name_from_private_to_public: + # pylint: disable=protected-access + keyring._impl._keyName = UTF8.default__.Encode(_dafny.Seq("rsa-4096-private")).value return keyring @@ -201,9 +198,7 @@ def keyring_from_master_key_specs(keys_uri, master_key_specs, mode): """ keyrings = [spec.keyring(keys_uri, mode) for spec in master_key_specs] primary = keyrings[0] - input(f"{primary=}") others = keyrings[1:] - input(f"{others=}") mpl: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( MaterialProvidersConfig() From 2c4325ae61d681391e01b947e64ade36f327efe9 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 3 May 2024 10:30:16 -0700 Subject: [PATCH 372/422] tests --- .../py312/decrypt_hkeyring_with_keyrings.yml | 31 +++++++++++++++++ .../py312/decrypt_hkeyring_with_masterkey.yml | 30 +++++++++++++++++ .../generate_hkeyring_decrypt_vectors.yml | 33 +++++++++++++++++++ 3 files changed, 94 insertions(+) create mode 100644 codebuild/py312/decrypt_hkeyring_with_keyrings.yml create mode 100644 codebuild/py312/decrypt_hkeyring_with_masterkey.yml create mode 100644 codebuild/py312/generate_hkeyring_decrypt_vectors.yml diff --git a/codebuild/py312/decrypt_hkeyring_with_keyrings.yml b/codebuild/py312/decrypt_hkeyring_with_keyrings.yml new file mode 100644 index 000000000..49ade991a --- /dev/null +++ b/codebuild/py312/decrypt_hkeyring_with_keyrings.yml @@ -0,0 +1,31 @@ +version: 0.2 + +env: + variables: + TOXENV: "py312-full_decrypt-mpl" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.12 + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_hkeyring_manifest.zip 312_hkeyring_manifest.zip + - unzip 312_hkeyring_manifest.zip + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../312_hkeyring_manifest/manifest.json \ + --keyrings \ No newline at end of file diff --git a/codebuild/py312/decrypt_hkeyring_with_masterkey.yml b/codebuild/py312/decrypt_hkeyring_with_masterkey.yml new file mode 100644 index 000000000..e25882030 --- /dev/null +++ b/codebuild/py312/decrypt_hkeyring_with_masterkey.yml @@ -0,0 +1,30 @@ +version: 0.2 + +env: + variables: + TOXENV: "py312-full_decrypt-mpl" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.12 + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_hkeyring_manifest.zip 312_hkeyring_manifest.zip + - unzip 312_hkeyring_manifest.zip + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers + - | + tox -- \ + --input ../312_hkeyring_manifest/manifest.json diff --git a/codebuild/py312/generate_hkeyring_decrypt_vectors.yml b/codebuild/py312/generate_hkeyring_decrypt_vectors.yml new file mode 100644 index 000000000..8dfa90581 --- /dev/null +++ b/codebuild/py312/generate_hkeyring_decrypt_vectors.yml @@ -0,0 +1,33 @@ +version: 0.2 + +env: + variables: + TOXENV: "py312-full_decrypt_generate-mpl" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.12 + build: + commands: + - pip install "tox < 4.0" + - cd test_vector_handlers/test/aws-crypto-tools-test-vector-framework + # Checkout WIP branch + - git checkout lucmcdon/hierarchy-test-vectors + - git pull + - cd ../.. + - | + tox -- \ + --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0007-hkeyring-reccmm-generate-manifest.json \ + --output 312_hkeyring_manifest \ + --keyrings + - zip -r 312_hkeyring_manifest.zip 312_hkeyring_manifest + - aws s3 cp 312_hkeyring_manifest.zip s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_hkeyring_manifest.zip From e64683cd2ea8059b915b12c99253d4c61706b35a Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 3 May 2024 10:33:27 -0700 Subject: [PATCH 373/422] run tests --- buildspec.yml | 23 +++++++++++++++++++ .../aws-crypto-tools-test-vector-framework | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/buildspec.yml b/buildspec.yml index 873e5941e..4472f2399 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -326,6 +326,29 @@ batch: buildspec: codebuild/py312/decrypt_keyrings_with_js.yml env: image: aws/codebuild/standard:7.0 + - identifier: py312_generate_hkeyring_decrypt_vectors + buildspec: codebuild/py312/py312_generate_hkeyring_decrypt_vectors.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_hkeyring_with_masterkey + depend-on: + - py312_generate_hkeyring_decrypt_vectors + buildspec: codebuild/py312/py312_decrypt_hkeyring_with_masterkey.yml + env: + image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_hkeyring_with_keyrings + depend-on: + - py312_generate_hkeyring_decrypt_vectors + buildspec: codebuild/py312/decrypt_keyrings_with_keyrings.yml + env: + image: aws/codebuild/standard:7.0 + # TODO: turn this into the .NET runner + # - identifier: py312_decrypt_keyrings_with_js + # depend-on: + # - py312_generate_decrypt_vectors_keyrings + # buildspec: codebuild/py312/decrypt_keyrings_with_js.yml + # env: + # image: aws/codebuild/standard:7.0 - identifier: code_coverage diff --git a/test_vector_handlers/test/aws-crypto-tools-test-vector-framework b/test_vector_handlers/test/aws-crypto-tools-test-vector-framework index 9eb2fcbbe..8bb3d9179 160000 --- a/test_vector_handlers/test/aws-crypto-tools-test-vector-framework +++ b/test_vector_handlers/test/aws-crypto-tools-test-vector-framework @@ -1 +1 @@ -Subproject commit 9eb2fcbbe47ab30c29d6ad9a8125b1064e0db42a +Subproject commit 8bb3d9179cacaf50f255eba01b16800e6fb0829c From 66429d2721d0f36a4c666b421793c3dfad92afc4 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 3 May 2024 10:53:46 -0700 Subject: [PATCH 374/422] debug --- buildspec.yml | 6 +++--- .../manifests/full_message/encrypt.py | 21 ++++++++++++++++++- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index 4472f2399..41b3e92fd 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -327,19 +327,19 @@ batch: env: image: aws/codebuild/standard:7.0 - identifier: py312_generate_hkeyring_decrypt_vectors - buildspec: codebuild/py312/py312_generate_hkeyring_decrypt_vectors.yml + buildspec: codebuild/py312/generate_hkeyring_decrypt_vectors.yml env: image: aws/codebuild/standard:7.0 - identifier: py312_decrypt_hkeyring_with_masterkey depend-on: - py312_generate_hkeyring_decrypt_vectors - buildspec: codebuild/py312/py312_decrypt_hkeyring_with_masterkey.yml + buildspec: codebuild/py312/decrypt_hkeyring_with_masterkey.yml env: image: aws/codebuild/standard:7.0 - identifier: py312_decrypt_hkeyring_with_keyrings depend-on: - py312_generate_hkeyring_decrypt_vectors - buildspec: codebuild/py312/decrypt_keyrings_with_keyrings.yml + buildspec: codebuild/py312/decrypt_hkeyring_with_keyrings.yml env: image: aws/codebuild/standard:7.0 # TODO: turn this into the .NET runner diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 6b19b921b..6fffad2c9 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -126,6 +126,25 @@ def master_key_provider_fn(): return keyring_from_master_key_specs(keys_uri, master_key_specs, "encrypt") return master_key_provider_from_master_key_specs(keys, master_key_specs) + # MPL test vectors add CMM types to the test vectors manifests + if "cmm" in scenario: + if scenario["cmm"] == "Default": + # Master keys and keyrings can handle default CMM + cmm_type = scenario["cmm"] + elif scenario["cmm"] == "RequiredEncryptionContext": + # Skip RequiredEncryptionContext CMM for master keys; + # RequiredEncryptionContext is unsupported for master keys. + # Caller logic should expect `None` to mean "no scenario". + if keyrings: + cmm_type = scenario["cmm"] + else: + return None + else: + raise ValueError("Unrecognized cmm_type: " + cmm_type) + else: + # If unspecified, set "Default" as the default + cmm_type = "Default" + return cls( plaintext_name=scenario["plaintext"], plaintext=plaintexts[scenario["plaintext"]], @@ -135,7 +154,7 @@ def master_key_provider_fn(): master_key_specs=master_key_specs, master_key_provider_fn=master_key_provider_fn, keyrings=keyrings, - cmm=scenario["cmm"], + cmm=cmm_type, ) def run(self, materials_manager=None): From 03330eabb0d17d91fdc072fcb88c2ab75e43561a Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 3 May 2024 10:58:21 -0700 Subject: [PATCH 375/422] rerun ci --- .../src/awses_test_vectors/manifests/full_message/encrypt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 6fffad2c9..6ba7599c0 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -142,7 +142,7 @@ def master_key_provider_fn(): else: raise ValueError("Unrecognized cmm_type: " + cmm_type) else: - # If unspecified, set "Default" as the default + # If unspecified, set "Default" as the default. cmm_type = "Default" return cls( From 9fd39cd796bed73c10246eba4719b28472d5458c Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 3 May 2024 11:04:40 -0700 Subject: [PATCH 376/422] fix --- .../test/aws-crypto-tools-test-vector-framework | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_vector_handlers/test/aws-crypto-tools-test-vector-framework b/test_vector_handlers/test/aws-crypto-tools-test-vector-framework index 8bb3d9179..3413afdc6 160000 --- a/test_vector_handlers/test/aws-crypto-tools-test-vector-framework +++ b/test_vector_handlers/test/aws-crypto-tools-test-vector-framework @@ -1 +1 @@ -Subproject commit 8bb3d9179cacaf50f255eba01b16800e6fb0829c +Subproject commit 3413afdc667e9c536f03fe01eb358d4be3b6d269 From 4240cee2a33d5db211505b2d57a62ae9aa894c53 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 3 May 2024 11:07:58 -0700 Subject: [PATCH 377/422] fix --- .../src/awses_test_vectors/manifests/full_message/encrypt.py | 2 +- .../test/aws-crypto-tools-test-vector-framework | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 6ba7599c0..6fffad2c9 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -142,7 +142,7 @@ def master_key_provider_fn(): else: raise ValueError("Unrecognized cmm_type: " + cmm_type) else: - # If unspecified, set "Default" as the default. + # If unspecified, set "Default" as the default cmm_type = "Default" return cls( diff --git a/test_vector_handlers/test/aws-crypto-tools-test-vector-framework b/test_vector_handlers/test/aws-crypto-tools-test-vector-framework index 3413afdc6..fc793e257 160000 --- a/test_vector_handlers/test/aws-crypto-tools-test-vector-framework +++ b/test_vector_handlers/test/aws-crypto-tools-test-vector-framework @@ -1 +1 @@ -Subproject commit 3413afdc667e9c536f03fe01eb358d4be3b6d269 +Subproject commit fc793e257f4a58ae49b92f95a519ba2c31ccff12 From 4adb696fcf42fee1a12809ea49378d10a91929b9 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 3 May 2024 11:29:13 -0700 Subject: [PATCH 378/422] fixes, now run net --- buildspec.yml | 14 ++++---- codebuild/py312/decrypt_hkeyring_with_net.yml | 34 +++++++++++++++++++ .../manifests/full_message/decrypt.py | 2 +- 3 files changed, 41 insertions(+), 9 deletions(-) create mode 100644 codebuild/py312/decrypt_hkeyring_with_net.yml diff --git a/buildspec.yml b/buildspec.yml index 41b3e92fd..e96af6189 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -342,14 +342,12 @@ batch: buildspec: codebuild/py312/decrypt_hkeyring_with_keyrings.yml env: image: aws/codebuild/standard:7.0 - # TODO: turn this into the .NET runner - # - identifier: py312_decrypt_keyrings_with_js - # depend-on: - # - py312_generate_decrypt_vectors_keyrings - # buildspec: codebuild/py312/decrypt_keyrings_with_js.yml - # env: - # image: aws/codebuild/standard:7.0 - + - identifier: py312_decrypt_kkeyring_with_net + depend-on: + - py312_generate_hkeyring_decrypt_vectors + buildspec: codebuild/py312/decrypt_hkeyring_with_net.yml + env: + image: aws/codebuild/standard:7.0 - identifier: code_coverage buildspec: codebuild/coverage/coverage.yml diff --git a/codebuild/py312/decrypt_hkeyring_with_net.yml b/codebuild/py312/decrypt_hkeyring_with_net.yml new file mode 100644 index 000000000..7cf7fa8c2 --- /dev/null +++ b/codebuild/py312/decrypt_hkeyring_with_net.yml @@ -0,0 +1,34 @@ +version: 0.2 + +env: + variables: + TOXENV: "py312-full_decrypt-mpl" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.12 + pre_build: + commands: + # Download previously generated vectors + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_hkeyring_manifest.zip 312_hkeyring_manifest.zip + - unzip 312_hkeyring_manifest.zip + + # Download published .NET ESDK so we don't have to build from source + dotnet add package AWS.Cryptography.EncryptionSDK --version 4.0.1 + + # Clone SDK-Dafny repo to get test vectors runner + - git clone git@github.com:aws/aws-encryption-sdk-dafny.git + - cd aws-encryption-sdk-dafny/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors + - export DAFNY_AWS_ESDK_TEST_VECTOR_MANIFEST_PATH=../../../../../../312_hkeyring_manifest/manifest.json + build: + commands: + - dotnet test --framework net48 \ No newline at end of file diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index 470b99c9e..5e612baa0 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -322,7 +322,7 @@ def master_key_provider_fn(): else: return None else: - raise ValueError("Unrecognized cmm_type: " + cmm_type) + raise ValueError("Unrecognized cmm_type: " + scenario["cmm"]) else: # If unspecified, set "Default" as the default cmm_type = "Default" From 620639f83a0cb61ef3a38ebf1b1c1f475acaa588 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 3 May 2024 11:39:22 -0700 Subject: [PATCH 379/422] fix --- codebuild/py312/decrypt_hkeyring_with_net.yml | 2 +- .../src/awses_test_vectors/manifests/full_message/decrypt.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/codebuild/py312/decrypt_hkeyring_with_net.yml b/codebuild/py312/decrypt_hkeyring_with_net.yml index 7cf7fa8c2..6e8bd1ae1 100644 --- a/codebuild/py312/decrypt_hkeyring_with_net.yml +++ b/codebuild/py312/decrypt_hkeyring_with_net.yml @@ -23,7 +23,7 @@ phases: - unzip 312_hkeyring_manifest.zip # Download published .NET ESDK so we don't have to build from source - dotnet add package AWS.Cryptography.EncryptionSDK --version 4.0.1 + - dotnet add package AWS.Cryptography.EncryptionSDK --version 4.0.1 # Clone SDK-Dafny repo to get test vectors runner - git clone git@github.com:aws/aws-encryption-sdk-dafny.git diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index 5e612baa0..9634e84f1 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -309,7 +309,8 @@ def master_key_provider_fn(): encryption_context = {} # MPL test vectors add CMM types to the test vectors manifests - if "cmm" in scenario: + if "cmm" in scenario \ + and scenario["cmm"] is not None: if scenario["cmm"] == "Default": # Master keys and keyrings can handle default CMM cmm_type = scenario["cmm"] From 559bdd377620bdd124052d98dca009715aff2ad8 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 3 May 2024 11:47:45 -0700 Subject: [PATCH 380/422] fixes --- buildspec.yml | 2 +- codebuild/py312/decrypt_hkeyring_with_net.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index e96af6189..86ec38caa 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -342,7 +342,7 @@ batch: buildspec: codebuild/py312/decrypt_hkeyring_with_keyrings.yml env: image: aws/codebuild/standard:7.0 - - identifier: py312_decrypt_kkeyring_with_net + - identifier: py312_decrypt_hkeyring_with_net depend-on: - py312_generate_hkeyring_decrypt_vectors buildspec: codebuild/py312/decrypt_hkeyring_with_net.yml diff --git a/codebuild/py312/decrypt_hkeyring_with_net.yml b/codebuild/py312/decrypt_hkeyring_with_net.yml index 6e8bd1ae1..85ff7054c 100644 --- a/codebuild/py312/decrypt_hkeyring_with_net.yml +++ b/codebuild/py312/decrypt_hkeyring_with_net.yml @@ -23,7 +23,7 @@ phases: - unzip 312_hkeyring_manifest.zip # Download published .NET ESDK so we don't have to build from source - - dotnet add package AWS.Cryptography.EncryptionSDK --version 4.0.1 + - dotnet add NETTestVectors package AWS.Cryptography.EncryptionSDK --version 4.0.1 # Clone SDK-Dafny repo to get test vectors runner - git clone git@github.com:aws/aws-encryption-sdk-dafny.git From 3ee4086e3e5b2379d06000f134d2ccb5bc599357 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 3 May 2024 14:15:15 -0700 Subject: [PATCH 381/422] fix --- codebuild/py312/decrypt_hkeyring_with_net.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/codebuild/py312/decrypt_hkeyring_with_net.yml b/codebuild/py312/decrypt_hkeyring_with_net.yml index 85ff7054c..fa4871ed0 100644 --- a/codebuild/py312/decrypt_hkeyring_with_net.yml +++ b/codebuild/py312/decrypt_hkeyring_with_net.yml @@ -22,13 +22,14 @@ phases: - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_hkeyring_manifest.zip 312_hkeyring_manifest.zip - unzip 312_hkeyring_manifest.zip - # Download published .NET ESDK so we don't have to build from source - - dotnet add NETTestVectors package AWS.Cryptography.EncryptionSDK --version 4.0.1 - # Clone SDK-Dafny repo to get test vectors runner - git clone git@github.com:aws/aws-encryption-sdk-dafny.git - - cd aws-encryption-sdk-dafny/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors - - export DAFNY_AWS_ESDK_TEST_VECTOR_MANIFEST_PATH=../../../../../../312_hkeyring_manifest/manifest.json + + # Change TestVectors to reference the published .NET ESDK + - cd aws-encryption-sdk-dafny/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectorsLib + - sed -i '' -e 's///g' AWSEncryptionSDKTestVectorLib.csproj + - cd ../TestVectors + build: commands: - - dotnet test --framework net48 \ No newline at end of file + - dotnet test --framework net6.0 \ No newline at end of file From 30ed6fa401df7b3d221e2db7825af703cd511343 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 1 May 2024 16:51:15 -0700 Subject: [PATCH 382/422] fix: Try all master key providers when decrypting raw RSA data key --- src/aws_encryption_sdk/internal/crypto/wrapping_keys.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/aws_encryption_sdk/internal/crypto/wrapping_keys.py b/src/aws_encryption_sdk/internal/crypto/wrapping_keys.py index 91f9fd834..ba6135965 100644 --- a/src/aws_encryption_sdk/internal/crypto/wrapping_keys.py +++ b/src/aws_encryption_sdk/internal/crypto/wrapping_keys.py @@ -98,9 +98,12 @@ def decrypt(self, encrypted_wrapped_data_key, encryption_context): if self.wrapping_key_type is EncryptionKeyType.PUBLIC: raise IncorrectMasterKeyError("Public key cannot decrypt") if self.wrapping_key_type is EncryptionKeyType.PRIVATE: - return self._wrapping_key.decrypt( - ciphertext=encrypted_wrapped_data_key.ciphertext, padding=self.wrapping_algorithm.padding - ) + try: + return self._wrapping_key.decrypt( + ciphertext=encrypted_wrapped_data_key.ciphertext, padding=self.wrapping_algorithm.padding + ) + except ValueError as e: + raise IncorrectMasterKeyError("_wrapping_key cannot decrypt provided ciphertext") serialized_encryption_context = serialize_encryption_context(encryption_context=encryption_context) return decrypt( algorithm=self.wrapping_algorithm.algorithm, From a30bceb0130e9afbfbc2aaa8e29a192d7e2199fc Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 3 May 2024 14:24:07 -0700 Subject: [PATCH 383/422] resovle merge --- .../materials_managers/__init__.py | 5 -- .../materials_managers/mpl/cmm.py | 39 ---------- .../unit/test_material_managers_mpl_cmm.py | 71 ------------------- .../test_material_managers_mpl_materials.py | 6 -- test/unit/test_streaming_client_configs.py | 43 ----------- .../test_streaming_client_stream_encryptor.py | 3 - 6 files changed, 167 deletions(-) diff --git a/src/aws_encryption_sdk/materials_managers/__init__.py b/src/aws_encryption_sdk/materials_managers/__init__.py index 0254381be..950dd87cd 100644 --- a/src/aws_encryption_sdk/materials_managers/__init__.py +++ b/src/aws_encryption_sdk/materials_managers/__init__.py @@ -90,12 +90,7 @@ class DecryptionMaterialsRequest(object): :type encrypted_data_keys: set of `aws_encryption_sdk.structures.EncryptedDataKey` :param dict encryption_context: Encryption context to provide to master keys for underlying decrypt requests :param dict reproduced_encryption_context: Encryption context to provide on decrypt. -<<<<<<< HEAD - This is ONLY processed if using the required encryption context CMM from the - aws-cryptographic-materialproviders library. -======= This is ONLY processed if using a CMM from the aws-cryptographic-materialproviders library. ->>>>>>> mpl-reviewed """ algorithm = attr.ib(validator=attr.validators.instance_of(Algorithm)) diff --git a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py index f3bc60853..49a743f89 100644 --- a/src/aws_encryption_sdk/materials_managers/mpl/cmm.py +++ b/src/aws_encryption_sdk/materials_managers/mpl/cmm.py @@ -7,14 +7,10 @@ # pylint should pass even if the MPL isn't installed # Also thinks these imports aren't used if it can't import them # noqa pylint: disable=import-error,unused-import -<<<<<<< HEAD -from aws_cryptographic_materialproviders.mpl.errors import AwsCryptographicMaterialProvidersException -======= from aws_cryptographic_materialproviders.mpl.errors import ( AwsCryptographicMaterialProvidersException, CollectionOfErrors, ) ->>>>>>> mpl-reviewed from aws_cryptographic_materialproviders.mpl.models import ( AlgorithmSuiteIdESDK as MPL_AlgorithmSuiteIdESDK, CommitmentPolicyESDK as MPL_CommitmentPolicyESDK, @@ -43,11 +39,7 @@ class CryptoMaterialsManagerFromMPL(CryptoMaterialsManager): """ In instances where encryption materials are provided by an implementation of the MPL's `aws_cryptographic_materialproviders.mpl.references.MPL_ICryptographicMaterialsManager`, -<<<<<<< HEAD - this maps the ESDK CMM interfaces to the MPL CMM. -======= this maps the ESDK-Python CMM interfaces to the MPL CMM. ->>>>>>> mpl-reviewed """ mpl_cmm: 'MPL_ICryptographicMaterialsManager' @@ -89,18 +81,6 @@ def get_encryption_materials( def _native_to_mpl_get_encryption_materials( request: EncryptionMaterialsRequest ) -> 'MPL_GetEncryptionMaterialsInput': -<<<<<<< HEAD - commitment_policy = CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy( - request.commitment_policy - ) - mpl_input_kwargs = { - "encryption_context": request.encryption_context, - "commitment_policy": commitment_policy, - "max_plaintext_length": request.plaintext_length, - } - if request.algorithm is not None: - mpl_input_kwargs["algorithm_suite_id"] = \ -======= output_kwargs = { "encryption_context": request.encryption_context, "max_plaintext_length": request.plaintext_length, @@ -111,25 +91,14 @@ def _native_to_mpl_get_encryption_materials( if request.algorithm is not None: output_kwargs["algorithm_suite_id"] = \ ->>>>>>> mpl-reviewed CryptoMaterialsManagerFromMPL._native_algorithm_id_to_mpl_algorithm_id( request.algorithm.algorithm_id ) -<<<<<<< HEAD - output: MPL_GetEncryptionMaterialsInput = MPL_GetEncryptionMaterialsInput( - **mpl_input_kwargs - ) - return output - - @staticmethod - def _native_to_mpl_commmitment_policy( -======= return MPL_GetEncryptionMaterialsInput(**output_kwargs) @staticmethod def _native_to_mpl_commitment_policy( ->>>>>>> mpl-reviewed native_commitment_policy: CommitmentPolicy ) -> 'MPL_CommitmentPolicyESDK': if native_commitment_policy == CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT: @@ -154,11 +123,7 @@ def decrypt_materials( CryptoMaterialsManagerFromMPL._create_mpl_decrypt_materials_input_from_request(request) mpl_output: 'MPL_DecryptMaterialsOutput' = self.mpl_cmm.decrypt_materials(mpl_input) return DecryptionMaterialsFromMPL(mpl_output.decryption_materials) -<<<<<<< HEAD - except AwsCryptographicMaterialProvidersException as mpl_exception: -======= except (AwsCryptographicMaterialProvidersException, CollectionOfErrors) as mpl_exception: ->>>>>>> mpl-reviewed # Wrap MPL error into the ESDK error type # so customers only have to catch ESDK error types. raise AWSEncryptionSDKClientError(mpl_exception) @@ -182,11 +147,7 @@ def _create_mpl_decrypt_materials_input_from_request( algorithm_suite_id=CryptoMaterialsManagerFromMPL._native_algorithm_id_to_mpl_algorithm_id( request.algorithm.algorithm_id ), -<<<<<<< HEAD - commitment_policy=CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy( -======= commitment_policy=CryptoMaterialsManagerFromMPL._native_to_mpl_commitment_policy( ->>>>>>> mpl-reviewed request.commitment_policy ), encrypted_data_keys=list_edks, diff --git a/test/mpl/unit/test_material_managers_mpl_cmm.py b/test/mpl/unit/test_material_managers_mpl_cmm.py index 92f3ba656..603446550 100644 --- a/test/mpl/unit/test_material_managers_mpl_cmm.py +++ b/test/mpl/unit/test_material_managers_mpl_cmm.py @@ -38,10 +38,7 @@ mock_mpl_cmm = MagicMock(__class__=MPL_ICryptographicMaterialsManager) mock_mpl_encryption_materials = MagicMock(__class__=MPL_EncryptionMaterials) mock_mpl_decrypt_materials = MagicMock(__class__=MPL_DecryptionMaterials) -<<<<<<< HEAD -======= mock_reproduced_encryption_context = MagicMock(__class_=dict) ->>>>>>> mpl-reviewed mock_edk = MagicMock(__class__=Native_EncryptedDataKey) @@ -100,12 +97,6 @@ def test_GIVEN_valid_request_WHEN_get_encryption_materials_THEN_return_Encryptio @patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL" -<<<<<<< HEAD - "._native_to_mpl_get_encryption_materials") -def test_GIVEN_mpl_cmm_raises_MPLException_WHEN_get_encryption_materials_THEN_raise_ESDKException( - _ -): -======= "._native_algorithm_id_to_mpl_algorithm_id") @patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL" "._native_to_mpl_commitment_policy") @@ -117,7 +108,6 @@ def test_GIVEN_mpl_cmm_raises_MPLException_WHEN_get_encryption_materials_THEN_ra mock_algorithm_id = "0x1234" # Some fake algorithm ID that fits the format mock_mpl_algorithm_id.return_value = mock_algorithm_id ->>>>>>> mpl-reviewed # Then: Raises AWSEncryptionSDKClientError with pytest.raises(AWSEncryptionSDKClientError): # Given: mpl_cmm.get_encryption_materials raises MPL exception @@ -131,13 +121,6 @@ def test_GIVEN_mpl_cmm_raises_MPLException_WHEN_get_encryption_materials_THEN_ra @patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL" "._native_algorithm_id_to_mpl_algorithm_id") @patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL" -<<<<<<< HEAD - "._native_to_mpl_commmitment_policy") -def test_GIVEN_valid_mpl_commitment_policy_WHEN_native_to_mpl_get_encryption_materials_THEN_returns_MPL_GetEncryptionMaterialsInput( # noqa: E501 - mock_mpl_commitment_policy, - mock_mpl_algorithm, -): -======= "._native_to_mpl_commitment_policy") def test_GIVEN_valid_mpl_commitment_policy_WHEN_native_to_mpl_get_encryption_materials_THEN_returns_MPL_GetEncryptionMaterialsInput( # noqa: E501 mock_mpl_commitment_policy, @@ -147,7 +130,6 @@ def test_GIVEN_valid_mpl_commitment_policy_WHEN_native_to_mpl_get_encryption_mat mock_algorithm_id = "0x1234" # Some fake algorithm ID that fits the format mock_mpl_algorithm_id.return_value = mock_algorithm_id ->>>>>>> mpl-reviewed # Given: commitment policy is some MPL ESDK commitment policy mock_commitment_policy = MagicMock(__class__=MPL_CommitmentPolicyESDK) mock_mpl_commitment_policy.return_value = mock_commitment_policy @@ -162,17 +144,6 @@ def test_GIVEN_valid_mpl_commitment_policy_WHEN_native_to_mpl_get_encryption_mat assert output.encryption_context == mock_encryption_materials_request.encryption_context assert output.commitment_policy == mock_commitment_policy assert output.max_plaintext_length == mock_encryption_materials_request.plaintext_length -<<<<<<< HEAD - assert output.algorithm_suite_id == mock_mpl_algorithm() - - -def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_MPL_CommitmentPolicyESDK_FORBID_ENCRYPT_ALLOW_DECRYPT(): # noqa: E501 - # Given: native FORBID_ENCRYPT_ALLOW_DECRYPT - native_commitment_policy = CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT - - # When: _native_to_mpl_commmitment_policy - output = CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy(native_commitment_policy) -======= def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_commitment_policy_THEN_returns_MPL_CommitmentPolicyESDK_FORBID_ENCRYPT_ALLOW_DECRYPT(): # noqa: E501 @@ -181,72 +152,44 @@ def test_GIVEN_CommitmentPolicy_FORBID_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_ # When: _native_to_mpl_commitment_policy output = CryptoMaterialsManagerFromMPL._native_to_mpl_commitment_policy(native_commitment_policy) ->>>>>>> mpl-reviewed # Then: Returns MPL FORBID_ENCRYPT_ALLOW_DECRYPT assert isinstance(output, MPL_CommitmentPolicyESDK) assert output.value == "FORBID_ENCRYPT_ALLOW_DECRYPT" -<<<<<<< HEAD -def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_MPL_CommitmentPolicyESDK_REQUIRE_ENCRYPT_ALLOW_DECRYPT(): # noqa: E501 - # Given: native REQUIRE_ENCRYPT_ALLOW_DECRYPT - native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT - - # When: _native_to_mpl_commmitment_policy - output = CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy(native_commitment_policy) -======= def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_ALLOW_DECRYPT_WHEN_native_to_mpl_commitment_policy_THEN_returns_MPL_CommitmentPolicyESDK_REQUIRE_ENCRYPT_ALLOW_DECRYPT(): # noqa: E501 # Given: native REQUIRE_ENCRYPT_ALLOW_DECRYPT native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT # When: _native_to_mpl_commitment_policy output = CryptoMaterialsManagerFromMPL._native_to_mpl_commitment_policy(native_commitment_policy) ->>>>>>> mpl-reviewed # Then: Returns MPL REQUIRE_ENCRYPT_ALLOW_DECRYPT assert isinstance(output, MPL_CommitmentPolicyESDK) assert output.value == "REQUIRE_ENCRYPT_ALLOW_DECRYPT" -<<<<<<< HEAD -def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_REQUIRE_DECRYPT_WHEN_native_to_mpl_commmitment_policy_THEN_returns_MPL_CommitmentPolicyESDK_REQUIRE_ENCRYPT_REQUIRE_DECRYPT(): # noqa: E501 - # Given: native REQUIRE_ENCRYPT_REQUIRE_DECRYPT - native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT - - # When: _native_to_mpl_commmitment_policy - output = CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy(native_commitment_policy) -======= def test_GIVEN_CommitmentPolicy_REQUIRE_ENCRYPT_REQUIRE_DECRYPT_WHEN_native_to_mpl_commitment_policy_THEN_returns_MPL_CommitmentPolicyESDK_REQUIRE_ENCRYPT_REQUIRE_DECRYPT(): # noqa: E501 # Given: native REQUIRE_ENCRYPT_REQUIRE_DECRYPT native_commitment_policy = CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT # When: _native_to_mpl_commitment_policy output = CryptoMaterialsManagerFromMPL._native_to_mpl_commitment_policy(native_commitment_policy) ->>>>>>> mpl-reviewed # Then: Returns MPL REQUIRE_ENCRYPT_REQUIRE_DECRYPT assert isinstance(output, MPL_CommitmentPolicyESDK) assert output.value == "REQUIRE_ENCRYPT_REQUIRE_DECRYPT" -<<<<<<< HEAD -def test_GIVEN_CommitmentPolicy_unrecognized_WHEN_native_to_mpl_commmitment_policy_THEN_raise_ValueError(): -======= def test_GIVEN_CommitmentPolicy_unrecognized_WHEN_native_to_mpl_commitment_policy_THEN_raise_ValueError(): ->>>>>>> mpl-reviewed # Given: invalid native commitment policy native_commitment_policy = "not a commitment policy" # Then: Raises ValueError with pytest.raises(ValueError): -<<<<<<< HEAD - # When: _native_to_mpl_commmitment_policy - CryptoMaterialsManagerFromMPL._native_to_mpl_commmitment_policy(native_commitment_policy) -======= # When: _native_to_mpl_commitment_policy CryptoMaterialsManagerFromMPL._native_to_mpl_commitment_policy(native_commitment_policy) ->>>>>>> mpl-reviewed @patch.object(mock_mpl_cmm, "decrypt_materials") @@ -310,11 +253,7 @@ def test_GIVEN_valid_native_algorithm_id_WHEN_native_algorithm_id_to_mpl_algorit @patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL" "._native_algorithm_id_to_mpl_algorithm_id") @patch("aws_encryption_sdk.materials_managers.mpl.cmm.CryptoMaterialsManagerFromMPL" -<<<<<<< HEAD - "._native_to_mpl_commmitment_policy") -======= "._native_to_mpl_commitment_policy") ->>>>>>> mpl-reviewed def test_GIVEN_valid_request_WHEN_create_mpl_decrypt_materials_input_from_request_THEN_returns_MPL_MPL_DecryptMaterialsInput( # noqa: E501 mock_mpl_commitment_policy, mock_mpl_algorithm_id, @@ -323,11 +262,7 @@ def test_GIVEN_valid_request_WHEN_create_mpl_decrypt_materials_input_from_reques mock_algorithm_id = "0x1234" # Some fake algorithm ID that fits the format mock_mpl_algorithm_id.return_value = mock_algorithm_id -<<<<<<< HEAD - # Given: _native_to_mpl_commmitment_policy returns some MPL commitment policy -======= # Given: _native_to_mpl_commitment_policy returns some MPL commitment policy ->>>>>>> mpl-reviewed mock_commitment_policy = MagicMock(__class__=MPL_CommitmentPolicyESDK) mock_mpl_commitment_policy.return_value = mock_commitment_policy @@ -339,10 +274,7 @@ def test_GIVEN_valid_request_WHEN_create_mpl_decrypt_materials_input_from_reques for mock_edks in [no_mock_edks, one_mock_edk, two_mock_edks]: mock_decryption_materials_request.encrypted_data_keys = mock_edks -<<<<<<< HEAD -======= mock_decryption_materials_request.reproduced_encryption_context = mock_reproduced_encryption_context ->>>>>>> mpl-reviewed # When: _create_mpl_decrypt_materials_input_from_request output = CryptoMaterialsManagerFromMPL._create_mpl_decrypt_materials_input_from_request( @@ -355,10 +287,7 @@ def test_GIVEN_valid_request_WHEN_create_mpl_decrypt_materials_input_from_reques assert output.algorithm_suite_id == mock_algorithm_id assert output.commitment_policy == mock_commitment_policy assert output.encryption_context == mock_decryption_materials_request.encryption_context -<<<<<<< HEAD -======= assert output.reproduced_encryption_context == mock_reproduced_encryption_context ->>>>>>> mpl-reviewed assert len(output.encrypted_data_keys) == len(mock_edks) for i in range(len(output.encrypted_data_keys)): diff --git a/test/mpl/unit/test_material_managers_mpl_materials.py b/test/mpl/unit/test_material_managers_mpl_materials.py index 0c31bff17..8d9052c0a 100644 --- a/test/mpl/unit/test_material_managers_mpl_materials.py +++ b/test/mpl/unit/test_material_managers_mpl_materials.py @@ -160,8 +160,6 @@ def test_GIVEN_valid_signing_key_WHEN_EncryptionMaterials_get_signing_key_THEN_r assert output == mock_signing_key -<<<<<<< HEAD -======= def test_GIVEN_valid_required_encryption_context_keys_WHEN_EncryptionMaterials_get_required_encryption_context_keys_THEN_returns_required_encryption_context_keys(): # noqa pylint: disable=line-too-long # Given: valid required encryption context keys mock_required_encryption_context_keys = MagicMock(__class__=bytes) @@ -175,7 +173,6 @@ def test_GIVEN_valid_required_encryption_context_keys_WHEN_EncryptionMaterials_g assert output == mock_required_encryption_context_keys ->>>>>>> mpl-reviewed def test_GIVEN_valid_data_key_WHEN_DecryptionMaterials_get_data_key_THEN_returns_data_key(): # Given: valid MPL data key mock_data_key = MagicMock(__class__=bytes) @@ -203,8 +200,6 @@ def test_GIVEN_valid_verification_key_WHEN_DecryptionMaterials_get_verification_ # Then: returns verification key assert output == mock_verification_key -<<<<<<< HEAD -======= def test_GIVEN_valid_encryption_context_WHEN_DecryptionMaterials_get_encryption_context_THEN_returns_encryption_context(): # noqa pylint: disable=line-too-long @@ -231,4 +226,3 @@ def test_GIVEN_valid_required_encryption_context_keys_WHEN_DecryptionMaterials_g # Then: returns required encryption context keys assert output == mock_required_encryption_context_keys ->>>>>>> mpl-reviewed diff --git a/test/unit/test_streaming_client_configs.py b/test/unit/test_streaming_client_configs.py index 0521139aa..435aff0da 100644 --- a/test/unit/test_streaming_client_configs.py +++ b/test/unit/test_streaming_client_configs.py @@ -15,11 +15,7 @@ import pytest import six -<<<<<<< HEAD -from mock import patch -======= from mock import MagicMock, patch ->>>>>>> mpl-reviewed from aws_encryption_sdk import CommitmentPolicy from aws_encryption_sdk.internal.defaults import ALGORITHM, FRAME_LENGTH, LINE_LENGTH @@ -37,11 +33,7 @@ # Ideally, this logic would be based on mocking imports and testing logic, # but doing that introduces errors that cause other tests to fail. try: -<<<<<<< HEAD - from aws_cryptographic_materialproviders.mpl.references import IKeyring -======= from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager, IKeyring ->>>>>>> mpl-reviewed HAS_MPL = True from aws_encryption_sdk.materials_managers.mpl.cmm import CryptoMaterialsManagerFromMPL @@ -244,13 +236,6 @@ def test_client_configs_with_mpl( assert test.materials_manager is not None # If materials manager was provided, it should be directly used -<<<<<<< HEAD - if hasattr(kwargs, "materials_manager"): - assert kwargs["materials_manager"] == test.materials_manager - - # If MPL keyring was provided, it should be wrapped in MPL materials manager - if hasattr(kwargs, "keyring"): -======= if "materials_manager" in kwargs: assert kwargs["materials_manager"] == test.materials_manager @@ -262,29 +247,17 @@ def test_client_configs_with_mpl( # If MPL keyring was provided, it should be wrapped in MPL materials manager elif "keyring" in kwargs: ->>>>>>> mpl-reviewed assert test.keyring is not None assert test.keyring == kwargs["keyring"] assert isinstance(test.keyring, IKeyring) assert isinstance(test.materials_manager, CryptoMaterialsManagerFromMPL) -<<<<<<< HEAD - # If native key_provider was provided, it should be wrapped in native materials manager - if hasattr(kwargs, "key_provider"): - assert test.key_provider is not None - assert test.key_provider == kwargs["key_provider"] - assert isinstance(test.materials_manager, DefaultCryptoMaterialsManager) - - -# This needs its own test; pytest parametrize cannot use a conditionally-loaded type -======= else: raise ValueError(f"Test did not find materials_manager or key_provider. {kwargs}") # This is an addition to test_client_configs_with_mpl; # This needs its own test; pytest's parametrize cannot use a conditionally-loaded type (IKeyring) ->>>>>>> mpl-reviewed @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") def test_keyring_client_config_with_mpl( ): @@ -296,21 +269,6 @@ def test_keyring_client_config_with_mpl( test = _ClientConfig(**kwargs) -<<<<<<< HEAD - # In all cases, config should have a materials manager - assert test.materials_manager is not None - - # If materials manager was provided, it should be directly used - if hasattr(kwargs, "materials_manager"): - assert kwargs["materials_manager"] == test.materials_manager - - # If MPL keyring was provided, it should be wrapped in MPL materials manager - if hasattr(kwargs, "keyring"): - assert test.keyring is not None - assert test.keyring == kwargs["keyring"] - assert isinstance(test.keyring, IKeyring) - assert isinstance(test.materials_manager, CryptoMaterialsManagerFromMPL) -======= assert test.materials_manager is not None assert test.keyring is not None @@ -338,4 +296,3 @@ def test_mpl_cmm_client_config_with_mpl( assert isinstance(test.materials_manager, CryptoMaterialsManagerFromMPL) # Assert the MPL CMM is used by the native interface assert test.materials_manager.mpl_cmm == mock_mpl_cmm ->>>>>>> mpl-reviewed diff --git a/test/unit/test_streaming_client_stream_encryptor.py b/test/unit/test_streaming_client_stream_encryptor.py index f56ec35a2..4df79e146 100644 --- a/test/unit/test_streaming_client_stream_encryptor.py +++ b/test/unit/test_streaming_client_stream_encryptor.py @@ -452,8 +452,6 @@ def test_GIVEN_has_mpl_AND_has_MPLCMM_AND_uses_signer_WHEN_prep_message_THEN_sig encoding=serialization.Encoding.PEM ) -<<<<<<< HEAD -======= # Given: has MPL @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") def test_GIVEN_has_mpl_AND_encryption_materials_has_required_EC_keys_WHEN_prep_message_THEN_paritions_stored_and_required_EC( # noqa pylint: disable=line-too-long @@ -567,7 +565,6 @@ def test_GIVEN_has_mpl_AND_encryption_materials_does_not_have_required_EC_keys_W # Then: _required_encryption_context is None assert test_encryptor._required_encryption_context is None ->>>>>>> mpl-reviewed def test_prep_message_no_signer(self): self.mock_encryption_materials.algorithm = Algorithm.AES_128_GCM_IV12_TAG16 test_encryptor = StreamEncryptor( From 19712e3521ba40f4d14edd39f82af98f4b7c19d2 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 3 May 2024 14:28:27 -0700 Subject: [PATCH 384/422] fix --- test_vector_handlers/tox.ini | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index 75c9c7a76..cdb1137fb 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -50,11 +50,7 @@ sitepackages = False deps = -rtest/requirements.txt # Install the MPL requirements if the `-mpl` suffix is present -<<<<<<< HEAD mpl: -rrequirements_mpl.txt -======= - mpl: -r../requirements_mpl.txt ->>>>>>> mpl-reviewed .. commands = awses_local: {[testenv:base-command]commands} From befe45d8ad71f0383ce5698265ad6a64aac10caa Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Fri, 3 May 2024 14:31:18 -0700 Subject: [PATCH 385/422] fix --- test/unit/test_streaming_client_stream_decryptor.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/unit/test_streaming_client_stream_decryptor.py b/test/unit/test_streaming_client_stream_decryptor.py index a291cab54..be9304006 100644 --- a/test/unit/test_streaming_client_stream_decryptor.py +++ b/test/unit/test_streaming_client_stream_decryptor.py @@ -195,12 +195,8 @@ def test_read_header(self, mock_derive_datakey, mock_decrypt_materials_request, test_decryptor._stream_length = len(VALUES["data_128"]) # Mock: hasattr(self.config, "encryption_context") returns False -<<<<<<< HEAD - del test_decryptor.config.encryption_context -======= if hasattr(test_decryptor.config, "encryption_context"): del test_decryptor.config.encryption_context ->>>>>>> mpl-reviewed test_header, test_header_auth = test_decryptor._read_header() From d33ff199d971f88e40077f917803cee2a7db088e Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 6 May 2024 13:26:44 -0700 Subject: [PATCH 386/422] remove net for now --- buildspec.yml | 12 ++++++------ codebuild/py312/decrypt_hkeyring_with_net.yml | 6 ++++++ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index 86ec38caa..658d31972 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -342,12 +342,12 @@ batch: buildspec: codebuild/py312/decrypt_hkeyring_with_keyrings.yml env: image: aws/codebuild/standard:7.0 - - identifier: py312_decrypt_hkeyring_with_net - depend-on: - - py312_generate_hkeyring_decrypt_vectors - buildspec: codebuild/py312/decrypt_hkeyring_with_net.yml - env: - image: aws/codebuild/standard:7.0 + # - identifier: py312_decrypt_hkeyring_with_net + # depend-on: + # - py312_generate_hkeyring_decrypt_vectors + # buildspec: codebuild/py312/decrypt_hkeyring_with_net.yml + # env: + # image: aws/codebuild/standard:7.0 - identifier: code_coverage buildspec: codebuild/coverage/coverage.yml diff --git a/codebuild/py312/decrypt_hkeyring_with_net.yml b/codebuild/py312/decrypt_hkeyring_with_net.yml index fa4871ed0..77a1516e5 100644 --- a/codebuild/py312/decrypt_hkeyring_with_net.yml +++ b/codebuild/py312/decrypt_hkeyring_with_net.yml @@ -22,6 +22,12 @@ phases: - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_hkeyring_manifest.zip 312_hkeyring_manifest.zip - unzip 312_hkeyring_manifest.zip + # Approach 1: "The Right Way": + + # TODO: Get published NET TestVectors runner + + # Approach 2: "The Quick Way:" + # Clone SDK-Dafny repo to get test vectors runner - git clone git@github.com:aws/aws-encryption-sdk-dafny.git From 5393825d7a0a269b4ee61c31bd5977e12b166768 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 6 May 2024 13:28:58 -0700 Subject: [PATCH 387/422] cleanup --- test/unit/test_streaming_client_stream_decryptor.py | 4 ---- test_vector_handlers/tox.ini | 4 ---- 2 files changed, 8 deletions(-) diff --git a/test/unit/test_streaming_client_stream_decryptor.py b/test/unit/test_streaming_client_stream_decryptor.py index a291cab54..be9304006 100644 --- a/test/unit/test_streaming_client_stream_decryptor.py +++ b/test/unit/test_streaming_client_stream_decryptor.py @@ -195,12 +195,8 @@ def test_read_header(self, mock_derive_datakey, mock_decrypt_materials_request, test_decryptor._stream_length = len(VALUES["data_128"]) # Mock: hasattr(self.config, "encryption_context") returns False -<<<<<<< HEAD - del test_decryptor.config.encryption_context -======= if hasattr(test_decryptor.config, "encryption_context"): del test_decryptor.config.encryption_context ->>>>>>> mpl-reviewed test_header, test_header_auth = test_decryptor._read_header() diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index 75c9c7a76..ed9ad993e 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -50,11 +50,7 @@ sitepackages = False deps = -rtest/requirements.txt # Install the MPL requirements if the `-mpl` suffix is present -<<<<<<< HEAD - mpl: -rrequirements_mpl.txt -======= mpl: -r../requirements_mpl.txt ->>>>>>> mpl-reviewed .. commands = awses_local: {[testenv:base-command]commands} From b461b64f6d0b36dc7d87a475f5a76acda934b5e5 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 6 May 2024 14:04:05 -0700 Subject: [PATCH 388/422] cleanup --- .../full_message/decrypt_generation.py | 5 +++- .../manifests/full_message/encrypt.py | 3 +- .../manifests/master_key.py | 6 ++-- .../manifests/mpl_keyring.py | 29 ++++++++++++++----- 4 files changed, 29 insertions(+), 14 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 5a7c51b84..48d6784f7 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -444,6 +444,10 @@ def from_scenario(cls, scenario, keys, plaintexts, keyrings, keys_uri): keyrings, keys_uri, ) + + if encryption_scenario is None: + return None + tampering = scenario.get("tampering") tampering_method = TamperingMethod.from_tampering_spec(tampering) decryption_method_spec = scenario.get("decryption-method") @@ -588,7 +592,6 @@ def from_file(cls, input_file, keyrings): scenario=scenario, keys=keys, plaintexts=plaintexts, keyrings=keyrings, keys_uri=keys_abs_path, ) except NotImplementedError as e: - # continue raise e return cls( version=raw_manifest["manifest"]["version"], diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py index 6fffad2c9..120fa6ee4 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/encrypt.py @@ -50,8 +50,7 @@ from awses_test_vectors.manifests.mpl_keyring import KeyringSpec, keyring_from_master_key_specs _HAS_MPL = True -except ImportError as e: - print(e) +except ImportError: _HAS_MPL = False diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/master_key.py b/test_vector_handlers/src/awses_test_vectors/manifests/master_key.py index c110effa1..5c0de0881 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/master_key.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/master_key.py @@ -44,7 +44,7 @@ # We only actually need these imports when running the mypy checks pass -KNOWN_TYPES = ("aws-kms", "aws-kms-mrk-aware", "aws-kms-mrk-aware-discovery", "raw", "aws-kms-hierarchy") +KNOWN_TYPES = ("aws-kms", "aws-kms-mrk-aware", "aws-kms-mrk-aware-discovery", "raw", ) KNOWN_ALGORITHMS = ("aes", "rsa") KNOWN_PADDING = ("pkcs1", "oaep-mgf1") KNOWN_PADDING_HASH = ("sha1", "sha256", "sha384", "sha512") @@ -99,8 +99,8 @@ class MasterKeySpec(object): # pylint: disable=too-many-instance-attributes def __attrs_post_init__(self): # type: () -> None """Verify that known types all have loaders and that all required parameters are provided.""" - # if set(KNOWN_TYPES) != set(self._MASTER_KEY_LOADERS.keys()): - # raise NotImplementedError("Gap found between known master key types and available master key loaders.") + if set(KNOWN_TYPES) != set(self._MASTER_KEY_LOADERS.keys()): + raise NotImplementedError("Gap found between known master key types and available master key loaders.") if self.type_name == "raw": if None in (self.provider_id, self.encryption_algorithm): diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index 67b214d3a..83888b9dc 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -33,7 +33,7 @@ from .master_key import KNOWN_TYPES as MASTER_KEY_KNOWN_TYPES from awses_test_vectors.internal.util import membership_validator -KEYRING_ONLY_KNOWN_TYPES = ("aws-kms-hierarchy") +KEYRING_ONLY_KNOWN_TYPES = ("aws-kms-hierarchy", ) import _dafny import UTF8 @@ -61,14 +61,27 @@ class KeyringSpec(MasterKeySpec): # pylint: disable=too-many-instance-attribute :param str padding_hash: Wrapping key padding hash (required for raw master keys) """ - # type_name = attr.ib(validator=membership_validator(set(MASTER_KEY_KNOWN_TYPES).union(KEYRING_ONLY_KNOWN_TYPES))) + type_name = attr.ib(validator=membership_validator(set(MASTER_KEY_KNOWN_TYPES).union(set(KEYRING_ONLY_KNOWN_TYPES)))) - def __attrs_post_init__(self): - # type: () -> None - """Verify that known types all have loaders and that all required parameters are provided.""" - # if set(KEYRING_ONLY_KNOWN_TYPES) != set(self._KEYRING_LOADERS.keys()): - # raise NotImplementedError("Gap found between known master key types and available master key loaders.") - # super().__attrs_post_init__() + @classmethod + def from_scenario(cls, spec): + # type: (MASTER_KEY_SPEC) -> MasterKeySpec + """Load from a keyring specification. + + :param dict spec: Master key specification JSON + :return: Loaded master key specification + :rtype: MasterKeySpec + """ + return cls( + type_name=spec["type"], + key_name=spec.get("key"), + default_mrk_region=spec.get("default-mrk-region"), + discovery_filter=cls._discovery_filter_from_spec(spec.get("aws-kms-discovery-filter")), + provider_id=spec.get("provider-id"), + encryption_algorithm=spec.get("encryption-algorithm"), + padding_algorithm=spec.get("padding-algorithm"), + padding_hash=spec.get("padding-hash"), + ) def keyring(self, keys_uri, mode): # type: (KeysManifest) -> IKeyring From 709fb3ae46ed22af340c604aab84b0ce69d1214d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 6 May 2024 14:06:31 -0700 Subject: [PATCH 389/422] cleanup --- .../test_streaming_client_stream_decryptor.py | 39 ------------------- 1 file changed, 39 deletions(-) diff --git a/test/unit/test_streaming_client_stream_decryptor.py b/test/unit/test_streaming_client_stream_decryptor.py index be9304006..83ce22c29 100644 --- a/test/unit/test_streaming_client_stream_decryptor.py +++ b/test/unit/test_streaming_client_stream_decryptor.py @@ -365,45 +365,6 @@ def test_GIVEN_verification_key_AND_has_mpl_AND_has_MPLCMM_WHEN_read_header_THEN algorithm=self.mock_header.algorithm, encoded_point=mock_b64encoding() ) - @patch("aws_encryption_sdk.streaming_client.derive_data_encryption_key") - @patch("aws_encryption_sdk.streaming_client.DecryptionMaterialsRequest") - @patch("aws_encryption_sdk.streaming_client.Verifier") - # Given: no MPL - @pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") - def test_GIVEN_decrypt_config_has_ec_WHEN_read_header_THEN_calls_decrypt_materials_with_reproduced_ec( - self, - mock_verifier, - mock_decrypt_materials_request, - *_, - ): - - mock_verifier_instance = MagicMock() - mock_verifier.from_key_bytes.return_value = mock_verifier_instance - ct_stream = io.BytesIO(VALUES["data_128"]) - mock_commitment_policy = MagicMock(__class__=CommitmentPolicy) - test_decryptor = StreamDecryptor( - materials_manager=self.mock_materials_manager, - source=ct_stream, - commitment_policy=mock_commitment_policy, - ) - test_decryptor.source_stream = ct_stream - test_decryptor._stream_length = len(VALUES["data_128"]) - # Given: self.config has "encryption_context" - any_reproduced_ec = {"some": "ec"} - test_decryptor.config.encryption_context = any_reproduced_ec - - # When: read header - test_decryptor._read_header() - - # Then: calls decrypt_materials with reproduced_encryption_context - mock_decrypt_materials_request.assert_called_once_with( - encrypted_data_keys=self.mock_header.encrypted_data_keys, - algorithm=self.mock_header.algorithm, - encryption_context=sentinel.encryption_context, - commitment_policy=mock_commitment_policy, - reproduced_encryption_context=any_reproduced_ec, - ) - @patch("aws_encryption_sdk.streaming_client.DecryptionMaterialsRequest") @patch("aws_encryption_sdk.streaming_client.derive_data_encryption_key") @patch("aws_encryption_sdk.streaming_client.Verifier") From d64dc81796519189f625f26f1d633f75a00e3ac7 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 6 May 2024 14:08:53 -0700 Subject: [PATCH 390/422] cleanup --- .../test_streaming_client_stream_decryptor.py | 97 ------------------- 1 file changed, 97 deletions(-) diff --git a/test/unit/test_streaming_client_stream_decryptor.py b/test/unit/test_streaming_client_stream_decryptor.py index 83ce22c29..ce3d6ee3c 100644 --- a/test/unit/test_streaming_client_stream_decryptor.py +++ b/test/unit/test_streaming_client_stream_decryptor.py @@ -365,103 +365,6 @@ def test_GIVEN_verification_key_AND_has_mpl_AND_has_MPLCMM_WHEN_read_header_THEN algorithm=self.mock_header.algorithm, encoded_point=mock_b64encoding() ) - @patch("aws_encryption_sdk.streaming_client.DecryptionMaterialsRequest") - @patch("aws_encryption_sdk.streaming_client.derive_data_encryption_key") - @patch("aws_encryption_sdk.streaming_client.Verifier") - # Given: no MPL - @pytest.mark.skipif(HAS_MPL, reason="Test should only be executed without MPL in installation") - def test_GIVEN_verification_key_AND_no_mpl_WHEN_read_header_THEN_calls_from_key_bytes( - self, - mock_verifier, - *_, - ): - # Given: verification key - mock_verifier_instance = MagicMock() - mock_verifier.from_key_bytes.return_value = mock_verifier_instance - ct_stream = io.BytesIO(VALUES["data_128"]) - mock_commitment_policy = MagicMock(__class__=CommitmentPolicy) - test_decryptor = StreamDecryptor( - materials_manager=self.mock_materials_manager, - source=ct_stream, - commitment_policy=mock_commitment_policy, - ) - test_decryptor.source_stream = ct_stream - test_decryptor._stream_length = len(VALUES["data_128"]) - - # When: read header - test_decryptor._read_header() - - # Then: calls from_key_bytes - mock_verifier.from_key_bytes.assert_called_once_with( - algorithm=self.mock_header.algorithm, key_bytes=sentinel.verification_key - ) - - @patch("aws_encryption_sdk.streaming_client.DecryptionMaterialsRequest") - @patch("aws_encryption_sdk.streaming_client.derive_data_encryption_key") - @patch("aws_encryption_sdk.streaming_client.Verifier") - # Given: has MPL - @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") - def test_GIVEN_verification_key_AND_has_mpl_AND_not_MPLCMM_WHEN_read_header_THEN_calls_from_key_bytes( - self, - mock_verifier, - *_, - ): - # Given: verification key - mock_verifier_instance = MagicMock() - mock_verifier.from_key_bytes.return_value = mock_verifier_instance - ct_stream = io.BytesIO(VALUES["data_128"]) - mock_commitment_policy = MagicMock(__class__=CommitmentPolicy) - test_decryptor = StreamDecryptor( - # Given: native CMM - materials_manager=self.mock_materials_manager, - source=ct_stream, - commitment_policy=mock_commitment_policy, - ) - test_decryptor.source_stream = ct_stream - test_decryptor._stream_length = len(VALUES["data_128"]) - - # When: read_header - test_decryptor._read_header() - - # Then: calls from_key_bytess - mock_verifier.from_key_bytes.assert_called_once_with( - algorithm=self.mock_header.algorithm, key_bytes=sentinel.verification_key - ) - - @patch("aws_encryption_sdk.streaming_client.DecryptionMaterialsRequest") - @patch("aws_encryption_sdk.streaming_client.derive_data_encryption_key") - @patch("aws_encryption_sdk.streaming_client.Verifier") - @patch("base64.b64encode") - # Given: has MPL - @pytest.mark.skipif(not HAS_MPL, reason="Test should only be executed with MPL in installation") - def test_GIVEN_verification_key_AND_has_mpl_AND_has_MPLCMM_WHEN_read_header_THEN_calls_from_encoded_point( - self, - mock_b64encoding, - mock_verifier, - *_, - ): - # Given: Verification key - mock_verifier_instance = MagicMock() - mock_verifier.from_key_bytes.return_value = mock_verifier_instance - ct_stream = io.BytesIO(VALUES["data_128"]) - mock_commitment_policy = MagicMock(__class__=CommitmentPolicy) - test_decryptor = StreamDecryptor( - # Given: MPL CMM - materials_manager=self.mock_mpl_materials_manager, - source=ct_stream, - commitment_policy=mock_commitment_policy, - ) - test_decryptor.source_stream = ct_stream - test_decryptor._stream_length = len(VALUES["data_128"]) - - # When: read header - test_decryptor._read_header() - - # Then: calls from_encoded_point - mock_verifier.from_encoded_point.assert_called_once_with( - algorithm=self.mock_header.algorithm, encoded_point=mock_b64encoding() - ) - @patch("aws_encryption_sdk.streaming_client.derive_data_encryption_key") def test_read_header_frame_too_large(self, mock_derive_datakey): self.mock_header.content_type = ContentType.FRAMED_DATA From c953b215494162055bdbd40f3d1df4f3fa948f24 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 6 May 2024 14:12:36 -0700 Subject: [PATCH 391/422] fix --- src/aws_encryption_sdk/internal/crypto/wrapping_keys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aws_encryption_sdk/internal/crypto/wrapping_keys.py b/src/aws_encryption_sdk/internal/crypto/wrapping_keys.py index ba6135965..da9bc9b6b 100644 --- a/src/aws_encryption_sdk/internal/crypto/wrapping_keys.py +++ b/src/aws_encryption_sdk/internal/crypto/wrapping_keys.py @@ -102,7 +102,7 @@ def decrypt(self, encrypted_wrapped_data_key, encryption_context): return self._wrapping_key.decrypt( ciphertext=encrypted_wrapped_data_key.ciphertext, padding=self.wrapping_algorithm.padding ) - except ValueError as e: + except ValueError: raise IncorrectMasterKeyError("_wrapping_key cannot decrypt provided ciphertext") serialized_encryption_context = serialize_encryption_context(encryption_context=encryption_context) return decrypt( From c7c6a5648333c51bb075f0d347f4c3d60422613d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 6 May 2024 14:14:07 -0700 Subject: [PATCH 392/422] fix --- test_vector_handlers/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index ed9ad993e..cdb1137fb 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -50,7 +50,7 @@ sitepackages = False deps = -rtest/requirements.txt # Install the MPL requirements if the `-mpl` suffix is present - mpl: -r../requirements_mpl.txt + mpl: -rrequirements_mpl.txt .. commands = awses_local: {[testenv:base-command]commands} From 990e2b8a37f33474b3d74052bfbe152d1d0b389d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 6 May 2024 14:15:04 -0700 Subject: [PATCH 393/422] rv vectors --- .../test/aws-crypto-tools-test-vector-framework | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_vector_handlers/test/aws-crypto-tools-test-vector-framework b/test_vector_handlers/test/aws-crypto-tools-test-vector-framework index fc793e257..c3d73fae2 160000 --- a/test_vector_handlers/test/aws-crypto-tools-test-vector-framework +++ b/test_vector_handlers/test/aws-crypto-tools-test-vector-framework @@ -1 +1 @@ -Subproject commit fc793e257f4a58ae49b92f95a519ba2c31ccff12 +Subproject commit c3d73fae260fd9e9cc9e746f09a7ffbab83576e2 From 6979419a3f81ec5aecfa51ba98b214abc21d007f Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 6 May 2024 14:18:05 -0700 Subject: [PATCH 394/422] fix --- .../test/aws-crypto-tools-test-vector-framework | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_vector_handlers/test/aws-crypto-tools-test-vector-framework b/test_vector_handlers/test/aws-crypto-tools-test-vector-framework index c3d73fae2..9eb2fcbbe 160000 --- a/test_vector_handlers/test/aws-crypto-tools-test-vector-framework +++ b/test_vector_handlers/test/aws-crypto-tools-test-vector-framework @@ -1 +1 @@ -Subproject commit c3d73fae260fd9e9cc9e746f09a7ffbab83576e2 +Subproject commit 9eb2fcbbe47ab30c29d6ad9a8125b1064e0db42a From dbed9b4db0c48deec14c70f98412b7b512341985 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 6 May 2024 15:04:31 -0700 Subject: [PATCH 395/422] fixes --- .../manifests/mpl_keyring.py | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index 83888b9dc..e8c54c7c8 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -16,9 +16,18 @@ """ import json import attr - # Ignore missing MPL for pylint, but the MPL is required for this example # noqa pylint: disable=import-error +import _dafny +import UTF8 + +# Ignore pylint not being able to read a module that requires the MPL +# pylint: disable=no-name-in-module +from awses_test_vectors.internal.mpl.keyvectors_provider import KeyVectorsProvider +from awses_test_vectors.manifests.keys import KeysManifest # noqa: disable=F401 +from .master_key import KNOWN_TYPES as MASTER_KEY_KNOWN_TYPES + + from aws_cryptography_materialproviderstestvectorkeys.smithygenerated.\ aws_cryptography_materialproviderstestvectorkeys.models import ( GetKeyDescriptionInput, @@ -30,21 +39,13 @@ from aws_cryptographic_materialproviders.mpl.references import IKeyring from aws_cryptographic_materialproviders.mpl.models import CreateMultiKeyringInput -from .master_key import KNOWN_TYPES as MASTER_KEY_KNOWN_TYPES from awses_test_vectors.internal.util import membership_validator -KEYRING_ONLY_KNOWN_TYPES = ("aws-kms-hierarchy", ) - -import _dafny -import UTF8 +from .master_key import MasterKeySpec -# Ignore pylint not being able to read a module that requires the MPL -# pylint: disable=no-name-in-module -from awses_test_vectors.internal.mpl.keyvectors_provider import KeyVectorsProvider -from awses_test_vectors.manifests.keys import KeysManifest # noqa: disable=F401 +KEYRING_ONLY_KNOWN_TYPES = ("aws-kms-hierarchy", ) -from .master_key import MasterKeySpec @attr.s @@ -61,7 +62,9 @@ class KeyringSpec(MasterKeySpec): # pylint: disable=too-many-instance-attribute :param str padding_hash: Wrapping key padding hash (required for raw master keys) """ - type_name = attr.ib(validator=membership_validator(set(MASTER_KEY_KNOWN_TYPES).union(set(KEYRING_ONLY_KNOWN_TYPES)))) + type_name = attr.ib(validator=membership_validator( + set(MASTER_KEY_KNOWN_TYPES).union(set(KEYRING_ONLY_KNOWN_TYPES)) + )) @classmethod def from_scenario(cls, spec): @@ -100,9 +103,8 @@ def keyring(self, keys_uri, mode): "key": self.key_name, "provider-id": self.provider_id, "encryption-algorithm": self.encryption_algorithm, - # "keyDescription": - } + if self.padding_algorithm is not None and self.padding_algorithm != "": input_kwargs["padding-algorithm"] = self.padding_algorithm if self.padding_hash is not None: @@ -182,7 +184,7 @@ def keyring(self, keys_uri, mode): return keyring - + def _kms_hierarchy_keyring_from_spec(self, keys): # type: (KeysManifest) -> AwsKmsHierarchyKeyring """Build an AWS KMS hierarchy keyring using this specification. @@ -194,7 +196,7 @@ def _kms_hierarchy_keyring_from_spec(self, keys): """ if not self.type_name == "aws-kms-hierarchy": raise TypeError("This is not an AWS KMS hierarchy key") - + return keyring_from_master_key_specs(keys, ) From 0137a48b0bc11db840070874ff32ddc53b4bedec Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 6 May 2024 15:11:12 -0700 Subject: [PATCH 396/422] clean --- .../manifests/full_message/decrypt.py | 2 +- .../full_message/decrypt_generation.py | 3 +- .../src/awses_test_vectors/manifests/keys.py | 4 +- .../manifests/mpl_keyring.py | 47 ++++++------------- 4 files changed, 19 insertions(+), 37 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index 9634e84f1..9ca14b81d 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -366,7 +366,7 @@ def scenario_spec(self): spec["description"] = self.description spec["cmm"] = self.cmm_type spec["encryption-context"] = self.encryption_context - + return spec def _one_shot_decrypt(self): diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index 48d6784f7..a0979f666 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -561,6 +561,7 @@ def _generate_plaintexts(plaintexts_specs): @classmethod def from_file(cls, input_file, keyrings): + # noqa: R0914 # type: (IO) -> MessageDecryptionGenerationManifest """Load from a file containing a full message encrypt manifest. @@ -592,7 +593,7 @@ def from_file(cls, input_file, keyrings): scenario=scenario, keys=keys, plaintexts=plaintexts, keyrings=keyrings, keys_uri=keys_abs_path, ) except NotImplementedError as e: - raise e + continue return cls( version=raw_manifest["manifest"]["version"], keys=keys, diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/keys.py b/test_vector_handlers/src/awses_test_vectors/manifests/keys.py index b2685a11d..089607899 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/keys.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/keys.py @@ -125,7 +125,7 @@ class AwsKmsHierarchyKeySpec(KeySpec): type_name = attr.ib(validator=membership_validator(("static-branch-key",))) - def __init__(self, encrypt, decrypt, type_name, key_id, branch_key_version, branch_key, beacon_key): # noqa=D107 + def __init__(self, encrypt, decrypt, type_name, key_id, branch_key_version, branch_key, beacon_key): # noqa=D107,R0913 # type: (bool, bool, str, str) -> None # Workaround pending resolution of attrs/mypy interaction. # https://github.com/python/mypy/issues/2088 @@ -251,7 +251,7 @@ def key_from_manifest_spec(key_spec): key_id = key_spec["key-id"] # type: str return AwsKmsKeySpec(encrypt=encrypt, decrypt=decrypt, type_name=type_name, key_id=key_id) - elif key_spec["type"] == "static-branch-key": + if key_spec["type"] == "static-branch-key": branch_key_version = key_spec["branchKeyVersion"] # type: str branch_key = key_spec["branchKey"] # type: str beacon_key = key_spec["beaconKey"] # type: str diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index e8c54c7c8..712734a7c 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -15,34 +15,30 @@ This REQUIRES the aws-cryptographic-material-providers library. """ import json -import attr + # Ignore missing MPL for pylint, but the MPL is required for this example # noqa pylint: disable=import-error import _dafny +import attr import UTF8 - -# Ignore pylint not being able to read a module that requires the MPL -# pylint: disable=no-name-in-module -from awses_test_vectors.internal.mpl.keyvectors_provider import KeyVectorsProvider -from awses_test_vectors.manifests.keys import KeysManifest # noqa: disable=F401 -from .master_key import KNOWN_TYPES as MASTER_KEY_KNOWN_TYPES - - -from aws_cryptography_materialproviderstestvectorkeys.smithygenerated.\ - aws_cryptography_materialproviderstestvectorkeys.models import ( - GetKeyDescriptionInput, - GetKeyDescriptionOutput, - TestVectorKeyringInput, - ) from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig -from aws_cryptographic_materialproviders.mpl.references import IKeyring from aws_cryptographic_materialproviders.mpl.models import CreateMultiKeyringInput +from aws_cryptographic_materialproviders.mpl.references import IKeyring +from aws_cryptography_materialproviderstestvectorkeys.smithygenerated.\ + aws_cryptography_materialproviderstestvectorkeys.models import ( + GetKeyDescriptionInput, + GetKeyDescriptionOutput, + TestVectorKeyringInput, +) +# Ignore pylint not being able to read a module that requires the MPL +# pylint: disable=no-name-in-module +from awses_test_vectors.internal.mpl.keyvectors_provider import KeyVectorsProvider from awses_test_vectors.internal.util import membership_validator +from awses_test_vectors.manifests.keys import KeysManifest # noqa: disable=F401 -from .master_key import MasterKeySpec - +from .master_key import KNOWN_TYPES as MASTER_KEY_KNOWN_TYPES, MasterKeySpec KEYRING_ONLY_KNOWN_TYPES = ("aws-kms-hierarchy", ) @@ -185,21 +181,6 @@ def keyring(self, keys_uri, mode): return keyring - def _kms_hierarchy_keyring_from_spec(self, keys): - # type: (KeysManifest) -> AwsKmsHierarchyKeyring - """Build an AWS KMS hierarchy keyring using this specification. - - :param KeySpec key_spec: Key specification to use with this master key - :return: AWS KMS hierarchy keyring based on this specification - :rtype: AwsKmsHierarchyKeyring - :raises TypeError: if this is not an AWS KMS master key specification - """ - if not self.type_name == "aws-kms-hierarchy": - raise TypeError("This is not an AWS KMS hierarchy key") - - return keyring_from_master_key_specs(keys, ) - - def keyring_from_master_key_specs(keys_uri, master_key_specs, mode): # type: (str, list[KeyringSpec]) -> IKeyring """Build and combine all keyrings identified by the provided specs and From 50f16cdaeb12fc094db34f7f6f9d5c2dba5c8745 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 6 May 2024 15:14:34 -0700 Subject: [PATCH 397/422] pylint --- .../manifests/full_message/decrypt_generation.py | 4 ++-- test_vector_handlers/src/awses_test_vectors/manifests/keys.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index a0979f666..d39ec8eb9 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -561,7 +561,7 @@ def _generate_plaintexts(plaintexts_specs): @classmethod def from_file(cls, input_file, keyrings): - # noqa: R0914 + # pylint: disable=too-many-locals # type: (IO) -> MessageDecryptionGenerationManifest """Load from a file containing a full message encrypt manifest. @@ -592,7 +592,7 @@ def from_file(cls, input_file, keyrings): tests[name] = MessageDecryptionTestScenarioGenerator.from_scenario( scenario=scenario, keys=keys, plaintexts=plaintexts, keyrings=keyrings, keys_uri=keys_abs_path, ) - except NotImplementedError as e: + except NotImplementedError: continue return cls( version=raw_manifest["manifest"]["version"], diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/keys.py b/test_vector_handlers/src/awses_test_vectors/manifests/keys.py index 089607899..ce6161431 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/keys.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/keys.py @@ -125,7 +125,8 @@ class AwsKmsHierarchyKeySpec(KeySpec): type_name = attr.ib(validator=membership_validator(("static-branch-key",))) - def __init__(self, encrypt, decrypt, type_name, key_id, branch_key_version, branch_key, beacon_key): # noqa=D107,R0913 + # noqa pylint: disable=line-too-long,too-many-arguments + def __init__(self, encrypt, decrypt, type_name, key_id, branch_key_version, branch_key, beacon_key): # noqa=D107 # type: (bool, bool, str, str) -> None # Workaround pending resolution of attrs/mypy interaction. # https://github.com/python/mypy/issues/2088 From e3431999c64164f7dab44eaaed5af96d72f2b549 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 6 May 2024 15:18:12 -0700 Subject: [PATCH 398/422] flake8 --- .../manifests/full_message/decrypt.py | 5 ++--- .../manifests/full_message/decrypt_generation.py | 2 +- .../src/awses_test_vectors/manifests/mpl_keyring.py | 11 +++++------ 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index 9ca14b81d..e2ebc9faa 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -288,8 +288,8 @@ def from_scenario( ] else: master_key_specs = [ - MasterKeySpec.from_scenario(spec) for spec in raw_master_key_specs \ - if spec["type"] != "aws-kms-hierarchy" + MasterKeySpec.from_scenario(spec) for spec in raw_master_key_specs + if spec["type"] != "aws-kms-hierarchy" ] def master_key_provider_fn(): @@ -302,7 +302,6 @@ def master_key_provider_fn(): result_spec = scenario["result"] result = MessageDecryptionTestResult.from_result_spec(result_spec, plaintext_reader) - if "encryption-context" in scenario: encryption_context = scenario["encryption-context"] else: diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py index d39ec8eb9..041e7d98d 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt_generation.py @@ -91,7 +91,7 @@ # We only actually need these imports when running the mypy checks pass -SUPPORTED_VERSIONS = (2,4,) +SUPPORTED_VERSIONS = (2, 4, ) class TamperingMethod: diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index 712734a7c..d48e6f111 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -26,11 +26,11 @@ from aws_cryptographic_materialproviders.mpl.models import CreateMultiKeyringInput from aws_cryptographic_materialproviders.mpl.references import IKeyring from aws_cryptography_materialproviderstestvectorkeys.smithygenerated.\ - aws_cryptography_materialproviderstestvectorkeys.models import ( - GetKeyDescriptionInput, - GetKeyDescriptionOutput, - TestVectorKeyringInput, -) + aws_cryptography_materialproviderstestvectorkeys.models import ( + GetKeyDescriptionInput, + GetKeyDescriptionOutput, + TestVectorKeyringInput, + ) # Ignore pylint not being able to read a module that requires the MPL # pylint: disable=no-name-in-module @@ -43,7 +43,6 @@ KEYRING_ONLY_KNOWN_TYPES = ("aws-kms-hierarchy", ) - @attr.s class KeyringSpec(MasterKeySpec): # pylint: disable=too-many-instance-attributes """AWS Encryption SDK master key specification utilities. From af0e37261da56bb8a27d683e7d6d840cbc197798 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 6 May 2024 15:24:58 -0700 Subject: [PATCH 399/422] flake8 --- .../src/awses_test_vectors/manifests/mpl_keyring.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index d48e6f111..640e7edaf 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -40,6 +40,14 @@ from .master_key import KNOWN_TYPES as MASTER_KEY_KNOWN_TYPES, MasterKeySpec +try: # Python 3.5.0 and 3.5.1 have incompatible typing modules + from typing import Iterable # noqa pylint: disable=unused-import + + from awses_test_vectors.internal.mypy_types import MASTER_KEY_SPEC # noqa pylint: disable=unused-import +except ImportError: # pragma: no cover + # We only actually need these imports when running the mypy checks + pass + KEYRING_ONLY_KNOWN_TYPES = ("aws-kms-hierarchy", ) @@ -63,7 +71,7 @@ class KeyringSpec(MasterKeySpec): # pylint: disable=too-many-instance-attribute @classmethod def from_scenario(cls, spec): - # type: (MASTER_KEY_SPEC) -> MasterKeySpec + # type: (MASTER_KEY_SPEC) -> KeyringSpec """Load from a keyring specification. :param dict spec: Master key specification JSON From 0c51f3f2c13d3a1b71b86f3b1004c2a01ea2ff01 Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Tue, 2 Jul 2024 13:55:42 -0700 Subject: [PATCH 400/422] add py312_generate_hkeyring_decrypt_vectors in codebuild --- buildspec.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index 95e4d0a17..b2999d01b 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -304,12 +304,12 @@ batch: buildspec: codebuild/py312/decrypt_hkeyring_with_keyrings.yml env: image: aws/codebuild/standard:7.0 - # - identifier: py312_decrypt_hkeyring_with_net - # depend-on: - # - py312_generate_hkeyring_decrypt_vectors - # buildspec: codebuild/py312/decrypt_hkeyring_with_net.yml - # env: - # image: aws/codebuild/standard:7.0 + - identifier: py312_decrypt_hkeyring_with_net + depend-on: + - py312_generate_hkeyring_decrypt_vectors + buildspec: codebuild/py312/decrypt_hkeyring_with_net.yml + env: + image: aws/codebuild/standard:7.0 - identifier: code_coverage buildspec: codebuild/coverage/coverage.yml From 2b0de545e799ef41e833098567f22873b6a2a297 Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Tue, 2 Jul 2024 14:18:04 -0700 Subject: [PATCH 401/422] fix --- codebuild/py312/decrypt_hkeyring_with_net.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py312/decrypt_hkeyring_with_net.yml b/codebuild/py312/decrypt_hkeyring_with_net.yml index 77a1516e5..20eab5ebc 100644 --- a/codebuild/py312/decrypt_hkeyring_with_net.yml +++ b/codebuild/py312/decrypt_hkeyring_with_net.yml @@ -29,7 +29,7 @@ phases: # Approach 2: "The Quick Way:" # Clone SDK-Dafny repo to get test vectors runner - - git clone git@github.com:aws/aws-encryption-sdk-dafny.git + - git clone https://github.com/aws/aws-encryption-sdk-dafny.git # Change TestVectors to reference the published .NET ESDK - cd aws-encryption-sdk-dafny/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectorsLib From f670df350bcc4a85fe44b6a20a2844580d91adea Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Tue, 2 Jul 2024 14:27:14 -0700 Subject: [PATCH 402/422] updated decrypt_hkeyring_with_net.yml --- codebuild/py312/decrypt_hkeyring_with_net.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codebuild/py312/decrypt_hkeyring_with_net.yml b/codebuild/py312/decrypt_hkeyring_with_net.yml index 20eab5ebc..209ac5748 100644 --- a/codebuild/py312/decrypt_hkeyring_with_net.yml +++ b/codebuild/py312/decrypt_hkeyring_with_net.yml @@ -32,8 +32,8 @@ phases: - git clone https://github.com/aws/aws-encryption-sdk-dafny.git # Change TestVectors to reference the published .NET ESDK - - cd aws-encryption-sdk-dafny/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectorsLib - - sed -i '' -e 's///g' AWSEncryptionSDKTestVectorLib.csproj + - cd aws-encryption-sdk-dafny/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectorLib + - sed -i -e 's///g' AWSEncryptionSDKTestVectorLib.csproj - cd ../TestVectors build: From 89cb9f1e35f12c8a3e27bbcba8aac0de12b0a521 Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Tue, 2 Jul 2024 14:46:00 -0700 Subject: [PATCH 403/422] set env variable DAFNY_AWS_ESDK_TEST_VECTOR_MANIFEST_PATH --- codebuild/py312/decrypt_hkeyring_with_net.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/codebuild/py312/decrypt_hkeyring_with_net.yml b/codebuild/py312/decrypt_hkeyring_with_net.yml index 209ac5748..f8bd4a4a2 100644 --- a/codebuild/py312/decrypt_hkeyring_with_net.yml +++ b/codebuild/py312/decrypt_hkeyring_with_net.yml @@ -21,6 +21,9 @@ phases: # Download previously generated vectors - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_hkeyring_manifest.zip 312_hkeyring_manifest.zip - unzip 312_hkeyring_manifest.zip + - export DAFNY_AWS_ESDK_TEST_VECTOR_MANIFEST_PATH="${PWD}/312_hkeyring_manifest/manifest.json" + - ls 312_hkeyring_manifest/ + - pwd # Approach 1: "The Right Way": From 3e43ffd30a1287a8619d2ba634e605bfaa5c896e Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Tue, 2 Jul 2024 15:39:19 -0700 Subject: [PATCH 404/422] debug --- codebuild/py312/decrypt_hkeyring_with_net.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/codebuild/py312/decrypt_hkeyring_with_net.yml b/codebuild/py312/decrypt_hkeyring_with_net.yml index f8bd4a4a2..8da9a41bd 100644 --- a/codebuild/py312/decrypt_hkeyring_with_net.yml +++ b/codebuild/py312/decrypt_hkeyring_with_net.yml @@ -23,6 +23,7 @@ phases: - unzip 312_hkeyring_manifest.zip - export DAFNY_AWS_ESDK_TEST_VECTOR_MANIFEST_PATH="${PWD}/312_hkeyring_manifest/manifest.json" - ls 312_hkeyring_manifest/ + - cat 312_hkeyring_manifest/keys.json - pwd # Approach 1: "The Right Way": From 059df059448b09ebcc140642e9698d99ecbdc96c Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Tue, 2 Jul 2024 15:48:48 -0700 Subject: [PATCH 405/422] debug --- codebuild/py312/decrypt_hkeyring_with_net.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/codebuild/py312/decrypt_hkeyring_with_net.yml b/codebuild/py312/decrypt_hkeyring_with_net.yml index 8da9a41bd..c1baf7e20 100644 --- a/codebuild/py312/decrypt_hkeyring_with_net.yml +++ b/codebuild/py312/decrypt_hkeyring_with_net.yml @@ -21,7 +21,6 @@ phases: # Download previously generated vectors - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_hkeyring_manifest.zip 312_hkeyring_manifest.zip - unzip 312_hkeyring_manifest.zip - - export DAFNY_AWS_ESDK_TEST_VECTOR_MANIFEST_PATH="${PWD}/312_hkeyring_manifest/manifest.json" - ls 312_hkeyring_manifest/ - cat 312_hkeyring_manifest/keys.json - pwd @@ -42,4 +41,4 @@ phases: build: commands: - - dotnet test --framework net6.0 \ No newline at end of file + - DAFNY_AWS_ESDK_TEST_VECTOR_MANIFEST_PATH="${PWD}/312_hkeyring_manifest/manifest.json" dotnet test --framework net6.0 \ No newline at end of file From 5360da96c1e97dc3d35f5ecc772f45b763e8f14b Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Tue, 2 Jul 2024 16:11:43 -0700 Subject: [PATCH 406/422] change branch for hkeyring decrypt --- codebuild/py312/decrypt_hkeyring_with_net.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/codebuild/py312/decrypt_hkeyring_with_net.yml b/codebuild/py312/decrypt_hkeyring_with_net.yml index c1baf7e20..0a5a15da9 100644 --- a/codebuild/py312/decrypt_hkeyring_with_net.yml +++ b/codebuild/py312/decrypt_hkeyring_with_net.yml @@ -21,6 +21,7 @@ phases: # Download previously generated vectors - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_hkeyring_manifest.zip 312_hkeyring_manifest.zip - unzip 312_hkeyring_manifest.zip + - export DAFNY_AWS_ESDK_TEST_VECTOR_MANIFEST_PATH="${PWD}/312_hkeyring_manifest/manifest.json" - ls 312_hkeyring_manifest/ - cat 312_hkeyring_manifest/keys.json - pwd @@ -33,6 +34,9 @@ phases: # Clone SDK-Dafny repo to get test vectors runner - git clone https://github.com/aws/aws-encryption-sdk-dafny.git + # TODO: Change branch to published when available + - git checkout lucmcdon/hkeyring-vectors + - git pull # Change TestVectors to reference the published .NET ESDK - cd aws-encryption-sdk-dafny/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectorLib @@ -41,4 +45,4 @@ phases: build: commands: - - DAFNY_AWS_ESDK_TEST_VECTOR_MANIFEST_PATH="${PWD}/312_hkeyring_manifest/manifest.json" dotnet test --framework net6.0 \ No newline at end of file + - dotnet test --framework net6.0 \ No newline at end of file From 72574f9bec41d2ff6d51b79e6d798e2a4aca11d5 Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Tue, 2 Jul 2024 16:19:02 -0700 Subject: [PATCH 407/422] fix --- codebuild/py312/decrypt_hkeyring_with_net.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/codebuild/py312/decrypt_hkeyring_with_net.yml b/codebuild/py312/decrypt_hkeyring_with_net.yml index 0a5a15da9..e59b0be42 100644 --- a/codebuild/py312/decrypt_hkeyring_with_net.yml +++ b/codebuild/py312/decrypt_hkeyring_with_net.yml @@ -35,8 +35,10 @@ phases: # Clone SDK-Dafny repo to get test vectors runner - git clone https://github.com/aws/aws-encryption-sdk-dafny.git # TODO: Change branch to published when available + - cd aws-encryption-sdk-dafny - git checkout lucmcdon/hkeyring-vectors - git pull + - cd .. # Change TestVectors to reference the published .NET ESDK - cd aws-encryption-sdk-dafny/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectorLib From 3e91d5c92442ec6b8ba754da6291bc9e5e056c0a Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Tue, 2 Jul 2024 16:27:59 -0700 Subject: [PATCH 408/422] debug --- codebuild/py312/decrypt_hkeyring_with_net.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/codebuild/py312/decrypt_hkeyring_with_net.yml b/codebuild/py312/decrypt_hkeyring_with_net.yml index e59b0be42..3e702c05a 100644 --- a/codebuild/py312/decrypt_hkeyring_with_net.yml +++ b/codebuild/py312/decrypt_hkeyring_with_net.yml @@ -22,9 +22,6 @@ phases: - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_hkeyring_manifest.zip 312_hkeyring_manifest.zip - unzip 312_hkeyring_manifest.zip - export DAFNY_AWS_ESDK_TEST_VECTOR_MANIFEST_PATH="${PWD}/312_hkeyring_manifest/manifest.json" - - ls 312_hkeyring_manifest/ - - cat 312_hkeyring_manifest/keys.json - - pwd # Approach 1: "The Right Way": @@ -41,9 +38,9 @@ phases: - cd .. # Change TestVectors to reference the published .NET ESDK - - cd aws-encryption-sdk-dafny/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectorLib - - sed -i -e 's///g' AWSEncryptionSDKTestVectorLib.csproj - - cd ../TestVectors + # - cd aws-encryption-sdk-dafny/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectorLib + # - sed -i -e 's///g' AWSEncryptionSDKTestVectorLib.csproj + # - cd ../TestVectors build: commands: From b70fd8c5fa47a6c7b15e852426bf46642be69fa8 Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Tue, 2 Jul 2024 16:33:32 -0700 Subject: [PATCH 409/422] debug --- codebuild/py312/decrypt_hkeyring_with_net.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py312/decrypt_hkeyring_with_net.yml b/codebuild/py312/decrypt_hkeyring_with_net.yml index 3e702c05a..3efcfa0a7 100644 --- a/codebuild/py312/decrypt_hkeyring_with_net.yml +++ b/codebuild/py312/decrypt_hkeyring_with_net.yml @@ -38,7 +38,7 @@ phases: - cd .. # Change TestVectors to reference the published .NET ESDK - # - cd aws-encryption-sdk-dafny/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectorLib + - cd aws-encryption-sdk-dafny/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors # - sed -i -e 's///g' AWSEncryptionSDKTestVectorLib.csproj # - cd ../TestVectors From ef7660ecc03c5a9b0eb0c4bde9632245acb1495b Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Wed, 3 Jul 2024 12:19:30 -0700 Subject: [PATCH 410/422] make net --- codebuild/py312/decrypt_hkeyring_with_net.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codebuild/py312/decrypt_hkeyring_with_net.yml b/codebuild/py312/decrypt_hkeyring_with_net.yml index 3efcfa0a7..fa6863716 100644 --- a/codebuild/py312/decrypt_hkeyring_with_net.yml +++ b/codebuild/py312/decrypt_hkeyring_with_net.yml @@ -35,10 +35,10 @@ phases: - cd aws-encryption-sdk-dafny - git checkout lucmcdon/hkeyring-vectors - git pull - - cd .. + - make transpile_net # Change TestVectors to reference the published .NET ESDK - - cd aws-encryption-sdk-dafny/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors + - cd AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors # - sed -i -e 's///g' AWSEncryptionSDKTestVectorLib.csproj # - cd ../TestVectors From a2f5505c24fd98edfcf6d9f4f5e55f972b66c336 Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Wed, 3 Jul 2024 13:55:47 -0700 Subject: [PATCH 411/422] submodules; dafny --- codebuild/py312/decrypt_hkeyring_with_net.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/codebuild/py312/decrypt_hkeyring_with_net.yml b/codebuild/py312/decrypt_hkeyring_with_net.yml index fa6863716..15c4d85c2 100644 --- a/codebuild/py312/decrypt_hkeyring_with_net.yml +++ b/codebuild/py312/decrypt_hkeyring_with_net.yml @@ -29,16 +29,22 @@ phases: # Approach 2: "The Quick Way:" + # Download dafny + - curl https://github.com/dafny-lang/dafny/releases/download/v$dafnyVersion/dafny-$dafnyVersion-x64-ubuntu-20.04.zip -L -o dafny.zip + - unzip -qq dafny.zip && rm dafny.zip + - export PATH="$PWD/dafny:$PATH" + # Clone SDK-Dafny repo to get test vectors runner - - git clone https://github.com/aws/aws-encryption-sdk-dafny.git + - git clone --recurse-submodules https://github.com/aws/aws-encryption-sdk-dafny.git # TODO: Change branch to published when available - cd aws-encryption-sdk-dafny - git checkout lucmcdon/hkeyring-vectors - git pull + - cd AwsEncryptionSDK/ - make transpile_net # Change TestVectors to reference the published .NET ESDK - - cd AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors + - cd runtimes/net/TestVectorsNative/TestVectors # - sed -i -e 's///g' AWSEncryptionSDKTestVectorLib.csproj # - cd ../TestVectors From 61257d0050987c24c88dfbe802c42f021edec20d Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Wed, 3 Jul 2024 14:08:02 -0700 Subject: [PATCH 412/422] fix --- codebuild/py312/decrypt_hkeyring_with_net.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py312/decrypt_hkeyring_with_net.yml b/codebuild/py312/decrypt_hkeyring_with_net.yml index 15c4d85c2..529db92a1 100644 --- a/codebuild/py312/decrypt_hkeyring_with_net.yml +++ b/codebuild/py312/decrypt_hkeyring_with_net.yml @@ -30,7 +30,7 @@ phases: # Approach 2: "The Quick Way:" # Download dafny - - curl https://github.com/dafny-lang/dafny/releases/download/v$dafnyVersion/dafny-$dafnyVersion-x64-ubuntu-20.04.zip -L -o dafny.zip + - curl https://github.com/dafny-lang/dafny/releases/download/v4.7.0/dafny-4.7.0-x64-ubuntu-20.04.zip -L -o dafny.zip - unzip -qq dafny.zip && rm dafny.zip - export PATH="$PWD/dafny:$PATH" From 45cc6891945e4f2f4730a197bd243df3c6ff01be Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Wed, 3 Jul 2024 14:28:57 -0700 Subject: [PATCH 413/422] fix - mpl transpile_net --- codebuild/py312/decrypt_hkeyring_with_net.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/codebuild/py312/decrypt_hkeyring_with_net.yml b/codebuild/py312/decrypt_hkeyring_with_net.yml index 529db92a1..20d4f47da 100644 --- a/codebuild/py312/decrypt_hkeyring_with_net.yml +++ b/codebuild/py312/decrypt_hkeyring_with_net.yml @@ -42,6 +42,8 @@ phases: - git pull - cd AwsEncryptionSDK/ - make transpile_net + - cd ../mpl/TestVectorsAwsCryptographicMaterialProviders/ + - make transpile_net # Change TestVectors to reference the published .NET ESDK - cd runtimes/net/TestVectorsNative/TestVectors From 2264cb7606226f6c3d8da64a5a0cf735652746a8 Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Wed, 3 Jul 2024 14:38:41 -0700 Subject: [PATCH 414/422] debug --- codebuild/py312/decrypt_hkeyring_with_net.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codebuild/py312/decrypt_hkeyring_with_net.yml b/codebuild/py312/decrypt_hkeyring_with_net.yml index 20d4f47da..8f28b9f76 100644 --- a/codebuild/py312/decrypt_hkeyring_with_net.yml +++ b/codebuild/py312/decrypt_hkeyring_with_net.yml @@ -46,7 +46,7 @@ phases: - make transpile_net # Change TestVectors to reference the published .NET ESDK - - cd runtimes/net/TestVectorsNative/TestVectors + - cd ../../AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors # - sed -i -e 's///g' AWSEncryptionSDKTestVectorLib.csproj # - cd ../TestVectors From 4c5d4adaf0d7b3d4566969a6dac16665a7cf40e6 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 5 Aug 2024 20:33:21 -0700 Subject: [PATCH 415/422] Update codebuild/py312/decrypt_hkeyring_with_net.yml --- codebuild/py312/decrypt_hkeyring_with_net.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/codebuild/py312/decrypt_hkeyring_with_net.yml b/codebuild/py312/decrypt_hkeyring_with_net.yml index 8f28b9f76..37c2cf0d2 100644 --- a/codebuild/py312/decrypt_hkeyring_with_net.yml +++ b/codebuild/py312/decrypt_hkeyring_with_net.yml @@ -23,11 +23,6 @@ phases: - unzip 312_hkeyring_manifest.zip - export DAFNY_AWS_ESDK_TEST_VECTOR_MANIFEST_PATH="${PWD}/312_hkeyring_manifest/manifest.json" - # Approach 1: "The Right Way": - - # TODO: Get published NET TestVectors runner - - # Approach 2: "The Quick Way:" # Download dafny - curl https://github.com/dafny-lang/dafny/releases/download/v4.7.0/dafny-4.7.0-x64-ubuntu-20.04.zip -L -o dafny.zip From 2f46a86c97b4a75693e8f13a5321e1c6d1447a58 Mon Sep 17 00:00:00 2001 From: Ritvik Kapila <61410899+RitvikKapila@users.noreply.github.com> Date: Wed, 5 Jun 2024 10:37:00 -0700 Subject: [PATCH 416/422] chore: performance tests for ESDK-python (#680) --- .gitignore | 6 + buildspec.yml | 8 + codebuild/py311/performance_tests_mpl.yml | 38 +++ codebuild/py312/performance_tests_mpl.yml | 38 +++ performance_tests/README.rst | 219 ++++++++++++++++++ performance_tests/__init__.py | 3 + performance_tests/consolidate_results.py | 49 ++++ performance_tests/pylintrc | 45 ++++ performance_tests/requirements.txt | 5 + performance_tests/requirements_mpl.txt | 1 + performance_tests/results/.gitkeep | 0 performance_tests/setup.cfg | 41 ++++ performance_tests/setup.py | 34 +++ .../__init__.py | 4 + .../keyrings/__init__.py | 3 + .../keyrings/aws_kms_keyring.py | 131 +++++++++++ .../keyrings/raw_aes_keyring.py | 85 +++++++ .../keyrings/raw_rsa_keyring.py | 79 +++++++ .../master_key_providers/__init__.py | 3 + .../aws_kms_master_key_provider.py | 69 ++++++ .../raw_aes_master_key_provider.py | 101 ++++++++ .../raw_rsa_master_key_provider.py | 101 ++++++++ .../utils/__init__.py | 3 + .../utils/util.py | 116 ++++++++++ performance_tests/test/keyrings/__init__.py | 3 + .../test/keyrings/test_aws_kms_keyring.py | 206 ++++++++++++++++ .../test/keyrings/test_raw_aes_keyring.py | 156 +++++++++++++ .../test/keyrings/test_raw_rsa_keyring.py | 163 +++++++++++++ .../test/master_key_providers/__init__.py | 3 + .../test_aws_kms_master_key_provider.py | 165 +++++++++++++ .../test_raw_aes_master_key_provider.py | 156 +++++++++++++ .../test_raw_rsa_master_key_provider.py | 156 +++++++++++++ performance_tests/test/resources/__init__.py | 3 + .../ciphertext/kms/ciphertext-data-empty.ct | Bin 0 -> 587 bytes .../ciphertext/kms/ciphertext-data-large.ct | Bin 0 -> 8619 bytes .../ciphertext/kms/ciphertext-data-medium.ct | Bin 0 -> 4587 bytes .../ciphertext/kms/ciphertext-data-small.ct | Bin 0 -> 623 bytes .../raw_aes/ciphertext-data-empty.ct | Bin 0 -> 454 bytes .../raw_aes/ciphertext-data-large.ct | Bin 0 -> 8486 bytes .../raw_aes/ciphertext-data-medium.ct | Bin 0 -> 4454 bytes .../raw_aes/ciphertext-data-small.ct | Bin 0 -> 490 bytes .../raw_rsa/ciphertext-data-empty.ct | Bin 0 -> 899 bytes .../raw_rsa/ciphertext-data-large.ct | Bin 0 -> 8931 bytes .../raw_rsa/ciphertext-data-medium.ct | Bin 0 -> 4899 bytes .../raw_rsa/ciphertext-data-small.ct | Bin 0 -> 935 bytes .../plaintext/plaintext-data-empty.dat | 0 .../plaintext/plaintext-data-large.dat | 21 ++ .../plaintext/plaintext-data-medium.dat | 11 + .../plaintext/plaintext-data-small.dat | 1 + performance_tests/tox.ini | 215 +++++++++++++++++ 50 files changed, 2441 insertions(+) create mode 100644 codebuild/py311/performance_tests_mpl.yml create mode 100644 codebuild/py312/performance_tests_mpl.yml create mode 100644 performance_tests/README.rst create mode 100644 performance_tests/__init__.py create mode 100644 performance_tests/consolidate_results.py create mode 100644 performance_tests/pylintrc create mode 100644 performance_tests/requirements.txt create mode 100644 performance_tests/requirements_mpl.txt create mode 100644 performance_tests/results/.gitkeep create mode 100644 performance_tests/setup.cfg create mode 100644 performance_tests/setup.py create mode 100644 performance_tests/src/aws_encryption_sdk_performance_tests/__init__.py create mode 100644 performance_tests/src/aws_encryption_sdk_performance_tests/keyrings/__init__.py create mode 100644 performance_tests/src/aws_encryption_sdk_performance_tests/keyrings/aws_kms_keyring.py create mode 100644 performance_tests/src/aws_encryption_sdk_performance_tests/keyrings/raw_aes_keyring.py create mode 100644 performance_tests/src/aws_encryption_sdk_performance_tests/keyrings/raw_rsa_keyring.py create mode 100644 performance_tests/src/aws_encryption_sdk_performance_tests/master_key_providers/__init__.py create mode 100644 performance_tests/src/aws_encryption_sdk_performance_tests/master_key_providers/aws_kms_master_key_provider.py create mode 100644 performance_tests/src/aws_encryption_sdk_performance_tests/master_key_providers/raw_aes_master_key_provider.py create mode 100644 performance_tests/src/aws_encryption_sdk_performance_tests/master_key_providers/raw_rsa_master_key_provider.py create mode 100644 performance_tests/src/aws_encryption_sdk_performance_tests/utils/__init__.py create mode 100644 performance_tests/src/aws_encryption_sdk_performance_tests/utils/util.py create mode 100644 performance_tests/test/keyrings/__init__.py create mode 100644 performance_tests/test/keyrings/test_aws_kms_keyring.py create mode 100644 performance_tests/test/keyrings/test_raw_aes_keyring.py create mode 100644 performance_tests/test/keyrings/test_raw_rsa_keyring.py create mode 100644 performance_tests/test/master_key_providers/__init__.py create mode 100644 performance_tests/test/master_key_providers/test_aws_kms_master_key_provider.py create mode 100644 performance_tests/test/master_key_providers/test_raw_aes_master_key_provider.py create mode 100644 performance_tests/test/master_key_providers/test_raw_rsa_master_key_provider.py create mode 100644 performance_tests/test/resources/__init__.py create mode 100644 performance_tests/test/resources/ciphertext/kms/ciphertext-data-empty.ct create mode 100644 performance_tests/test/resources/ciphertext/kms/ciphertext-data-large.ct create mode 100644 performance_tests/test/resources/ciphertext/kms/ciphertext-data-medium.ct create mode 100644 performance_tests/test/resources/ciphertext/kms/ciphertext-data-small.ct create mode 100644 performance_tests/test/resources/ciphertext/raw_aes/ciphertext-data-empty.ct create mode 100644 performance_tests/test/resources/ciphertext/raw_aes/ciphertext-data-large.ct create mode 100644 performance_tests/test/resources/ciphertext/raw_aes/ciphertext-data-medium.ct create mode 100644 performance_tests/test/resources/ciphertext/raw_aes/ciphertext-data-small.ct create mode 100644 performance_tests/test/resources/ciphertext/raw_rsa/ciphertext-data-empty.ct create mode 100644 performance_tests/test/resources/ciphertext/raw_rsa/ciphertext-data-large.ct create mode 100644 performance_tests/test/resources/ciphertext/raw_rsa/ciphertext-data-medium.ct create mode 100644 performance_tests/test/resources/ciphertext/raw_rsa/ciphertext-data-small.ct create mode 100644 performance_tests/test/resources/plaintext/plaintext-data-empty.dat create mode 100644 performance_tests/test/resources/plaintext/plaintext-data-large.dat create mode 100644 performance_tests/test/resources/plaintext/plaintext-data-medium.dat create mode 100644 performance_tests/test/resources/plaintext/plaintext-data-small.dat create mode 100644 performance_tests/tox.ini diff --git a/.gitignore b/.gitignore index 24df397ed..2843404d0 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,12 @@ __pycache__ .pytest_cache # Ignore key materials generated by examples or tests test_keyrings/ +# Ignore results of performance test +performance_tests/results/*.csv +performance_tests/results/*.pstats +performance_tests/results/*.png +# Ignore the memory profile logs +mprofile_* # PyCharm .idea/ diff --git a/buildspec.yml b/buildspec.yml index b2999d01b..4665ac89e 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -126,6 +126,10 @@ batch: buildspec: codebuild/py311/integ_mpl.yml env: image: aws/codebuild/standard:7.0 + - identifier: py311_performance_tests_mpl + buildspec: codebuild/py311/performance_tests_mpl.yml + env: + image: aws/codebuild/standard:7.0 - identifier: py311_examples buildspec: codebuild/py311/examples.yml env: @@ -212,6 +216,10 @@ batch: buildspec: codebuild/py312/integ_mpl.yml env: image: aws/codebuild/standard:7.0 + - identifier: py312_performance_tests_mpl + buildspec: codebuild/py312/performance_tests_mpl.yml + env: + image: aws/codebuild/standard:7.0 - identifier: py312_examples buildspec: codebuild/py312/examples.yml env: diff --git a/codebuild/py311/performance_tests_mpl.yml b/codebuild/py311/performance_tests_mpl.yml new file mode 100644 index 000000000..2debb1185 --- /dev/null +++ b/codebuild/py311/performance_tests_mpl.yml @@ -0,0 +1,38 @@ +# Runs the performance tests for the MPL in an environment with the MPL installed +version: 0.2 + +env: + variables: + # No TOXENV. This runs multiple environments. + REGION: "us-west-2" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.11 + build: + commands: + - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - + - pyenv install --skip-existing 3.11.0 + - pyenv local 3.11.0 + - pip install --upgrade pip + - pip install setuptools + - pip install "tox < 4.0" + # Assume special role to access keystore + - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-Py312ExamplesMpl") + - export TMP_ROLE + - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') + - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') + - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') + - aws sts get-caller-identity + # Run MPL-specific tests with special role + - cd performance_tests/ + - tox -e py311-performance_tests-mpl diff --git a/codebuild/py312/performance_tests_mpl.yml b/codebuild/py312/performance_tests_mpl.yml new file mode 100644 index 000000000..97dbf359f --- /dev/null +++ b/codebuild/py312/performance_tests_mpl.yml @@ -0,0 +1,38 @@ +# Runs the performance tests for the MPL in an environment with the MPL installed +version: 0.2 + +env: + variables: + # No TOXENV. This runs multiple environments. + REGION: "us-west-2" + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- + arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- + arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >- + arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >- + arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7 + +phases: + install: + runtime-versions: + python: 3.12 + build: + commands: + - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - + - pyenv install --skip-existing 3.12.0 + - pyenv local 3.12.0 + - pip install --upgrade pip + - pip install setuptools + - pip install "tox < 4.0" + # Assume special role to access keystore + - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-Py312ExamplesMpl") + - export TMP_ROLE + - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId') + - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey') + - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken') + - aws sts get-caller-identity + # Run MPL-specific tests with special role + - cd performance_tests/ + - tox -e py312-performance_tests-mpl diff --git a/performance_tests/README.rst b/performance_tests/README.rst new file mode 100644 index 000000000..ba9b89f15 --- /dev/null +++ b/performance_tests/README.rst @@ -0,0 +1,219 @@ +##################################### +aws-encryption-sdk performance tests +##################################### + +This module runs performance tests for the `AWS Encryption SDK Python`_. + +******** +Overview +******** + +This module tests the following keyrings / master key providers: + +1. KMS Keyring / KMS Master Key Provider +2. Raw AES Keyring / AES Master Key Provider +3. Raw RSA Keyring / RSA Master Key Provider +4. Hierarchy Keyring +5. Caching CMM + +For each test on the above keyrings / master key providers, this package measures: + +1. Execution time +2. Total memory consumption + +For each keyring / master key provider, the execution time and memory consumption +is measured for three operations: + +1. Create keyring / master key provider +2. Encrypt +3. Decrypt + +The usage of the performance tests is demonstrated through an `AWS KMS Keyring`_. +However, the procedure is the same for any keyring / master key provider, with slight +changes in the input arguments. + +The results for the performance test will be available in the results folder in the +performance_tests directory. + +********************** +Required Prerequisites +********************** + +* Python 3.8+ +* aws-encryption-sdk +* boto3 >= 1.10.0 +* click +* tqdm +* pytest + +Recommended Prerequisites +========================= + +* aws-cryptographic-material-providers: >= 1.0.0 + * Requires Python 3.11+. + +***** +Usage +***** + +Execution Time +============== + +Create Keyring +-------------- +To run the performance test for execution time, please use the +following commands in the performance_tests directory. + +.. code:: + + usage: python test/keyrings/test_aws_kms_keyring.py create + + Create a keyring to use for encryption and decryption. + + optional arguments: + -h, --help show this help message and exit. + --kms_key_id KMS_KEY_ID The KMS key ID you want to use. + --n_iters N_ITERS Number of iterations you want to + run the test for. For instance, + if n_iters = 100, this performance + test script will run the create_keyring + method 100 times and report the + execution time of each of the calls. + --output_file OUTPUT_FILE The output file for execution times + for each function call, + default='kms_keyring_create' in the + results folder. + +Encrypt +------- + +To run the performance test for execution time, please use the following +commands in the performance_tests directory: + +.. code:: + + usage: python test/keyrings/test_aws_kms_keyring.py encrypt + + optional arguments: + -h, --help show this help message and exit. + --plaintext_data_filename PLAINTEXT_DATA_FILENAME Filename containing plaintext data + you want to encrypt. + default='test/resources/plaintext/plaintext-data-medium.dat'. + You can choose to use any other plaintext + file as well. Some example plaintext + data files are present in the + 'test/resources' directory. + --kms_key_id KMS_KEY_ID The KMS key ID you want to use. + --n_iters N_ITERS Number of iterations you want to + run the test for. For instance, + if n_iters = 100, this performance + test script will run the create_keyring + method 100 times and report the + execution time of each of the calls. + --output_file OUTPUT_FILE The output file for execution times + for each function call, + default='kms_keyring_create' in the + results folder. + +Decrypt +------- + +To run the performance test for execution time, please use the +following commands in the performance_tests directory + +.. code:: + + usage: python test/keyrings/test_aws_kms_keyring.py decrypt + + optional arguments: + -h, --help show this help message and exit. + --ciphertext_data_filename CIPHERTEXT_DATA_FILENAME Filename containing ciphertext data + you want to decrypt. + default='test/resources/ciphertext/kms/ciphertext-data-medium.ct'. + You can choose to use any other + ciphertext file as well. Some example + ciphertext data files are present in + the 'test/resources' directory. + --kms_key_id KMS_KEY_ID The KMS key ID you want to use. + --n_iters N_ITERS Number of iterations you want to + run the test for. For instance, + if n_iters = 100, this performance + test script will run the create_keyring + method 100 times and report the + execution time of each of the calls. + --output_file OUTPUT_FILE The output file for execution times + for each function call, + default='kms_keyring_create' in the + results folder. + +Consolidate Time Results +======================== + +In order to find the minimum, maximum, average, 99th percentile and bottom +99th percentile trimmed average times from the n_iters runs, please use the +following script from the performance_tests directory with the csv file +containing times for each of the n_iters runs generated in the previous +"Execution Time" section: + +.. code:: + + usage: python consolidate_results.py results/kms_keyring_decrypt.csv + +Memory Consumption +================== + +To get the memory consumption, simply replace 'python' +with 'mprof run' in the previously mentioned commands. + +For example, if you want to calculate the memory consumption +of the encrypt function of a AWS KMS Keyring, simply write: + +.. code:: + + usage: mprof run test/keyrings/test_aws_kms_keyring.py encrypt + + +This should generate an mprofile log file in your current directory. +This mprofile log file contains the total memory consumed by the program +with respect to time elapsed. +To plot the memory consumption with respect to time, please use the following +command from the same directory + +.. code:: + + usage: mprof plot + + +This 'mprof plot' command will plot the most recent mprofile log file. + + +Performance Graph +================= + +To generate a performance graph, please use the following command +to generate the pstats log file by specifying the output pstats file +path. Here, 'results/kms_keyring_create.pstats' is set as the default +output file. + +.. code:: + + usage: python -m cProfile -o results/kms_keyring_create.pstats test/keyrings/test_aws_kms_keyring.py create + + +After generating the pstats file, please run the following command +to generate the performance graph. The output performance graph will +be a .png file that you specify. Here, 'results/kms_keyring_create.png' +is set as the default output file. + +.. code:: + + usage: gprof2dot -f pstats results/kms_keyring_create.pstats | dot -Tpng -o results/kms_keyring_create.png && eog results/kms_keyring_create.png + + +Note: This project does not adhere to semantic versioning; as such it +makes no guarantees that functionality will persist across major, +minor, or patch versions. +**DO NOT** take a standalone dependency on this library. + +.. _AWS Encryption SDK Python: https://github.com/aws/aws-encryption-sdk-python/ +.. _AWS KMS Keyring: https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/use-kms-keyring.html diff --git a/performance_tests/__init__.py b/performance_tests/__init__.py new file mode 100644 index 000000000..120179eda --- /dev/null +++ b/performance_tests/__init__.py @@ -0,0 +1,3 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Stub module indicator to make linter configuration simpler.""" diff --git a/performance_tests/consolidate_results.py b/performance_tests/consolidate_results.py new file mode 100644 index 000000000..2601417cc --- /dev/null +++ b/performance_tests/consolidate_results.py @@ -0,0 +1,49 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Script for consolidating results for execution times""" + +import argparse +import csv + +import numpy as np + + +def calculate_statistics(_csv_file): + """Calculate average, trimmed average, minimum, maximum and p99 statistics for execution times in a CSV file.""" + with open(_csv_file, 'r', encoding='utf-8') as file: + reader = csv.reader(file) + data = [float(row[0]) for row in reader] + + output_stats = {} + + # Calculate statistics + if data: + data = np.sort(data) + output_stats['total_entries'] = len(data) + output_stats['average'] = np.mean(data) + output_stats['trimmed_average_99_bottom'] = np.mean(data[0:int(0.99 * len(data))]) + output_stats['minimum'] = min(data) + output_stats['maximum'] = max(data) + output_stats['perc_99'] = np.percentile(data, 99) + return output_stats + + return None + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('csv_file', + help='csv file containing the outputs of execution times for n_iter iterations') + args = parser.parse_args() + + statistics = calculate_statistics(args.csv_file) + if statistics: + print("CSV File:", args.csv_file) + print("Total Entries:", statistics['total_entries']) + print("Average:", statistics['average']) + print("Bottom 99th percentile trimmed average:", statistics['trimmed_average_99_bottom']) + print("Minimum:", statistics['minimum']) + print("Maximum:", statistics['maximum']) + print("99th percentile:", statistics['perc_99']) + else: + print("No data found in the CSV file.") diff --git a/performance_tests/pylintrc b/performance_tests/pylintrc new file mode 100644 index 000000000..8ed5cb105 --- /dev/null +++ b/performance_tests/pylintrc @@ -0,0 +1,45 @@ +[MESSAGE CONTROL] +# Disabling messages that either we don't care about we intentionally break. +disable = + import-error, # ignore mpl import errors + invalid-name, # we prefer long, descriptive, names for examples + bad-continuation, # we let black handle this + ungrouped-imports, # we let isort handle this + no-member, # breaks with attrs + no-self-use, # interesting to keep in mind for later refactoring, but not blocking + useless-object-inheritance, # we need to support Python 2, so no, not useless + duplicate-code, # some examples may be similar + too-few-public-methods, # does not allow value stores + too-many-locals, # examples may sometimes have more locals defined for clarity than would be appropriate in code + no-else-return, # we omit this on purpose for brevity where it would add no value + attribute-defined-outside-init, # breaks with attrs_post_init + abstract-method, # throws false positives on io.BaseIO grandchildren + redefined-outer-name, # we do this on purpose in multiple places + consider-using-f-string # disable until 2022-05-05; 6 months after 3.5 deprecation + +[BASIC] +# Allow function names up to 50 characters +function-rgx = [a-z_][a-z0-9_]{2,50}$ +# Allow method names up to 50 characters +method-rgx = [a-z_][a-z0-9_]{2,50}$ +# Allow class attribute names up to 50 characters +# Whitelist class attribute names: iv +class-attribute-rgx = (([A-Za-z_][A-Za-z0-9_]{2,50}|(__.*__))$)|(^iv$) +# Whitelist attribute names: iv +attr-rgx = ([a-z_][a-z0-9_]{2,30}$)|(^iv$) +# Whitelist argument names: iv, b +argument-rgx = ([a-z_][a-z0-9_]{2,30}$)|(^iv$)|(^b$) +# Whitelist variable names: iv, b, _b, x, y, r, s +variable-rgx = ([a-z_][a-z0-9_]{2,30}$)|(^iv$)|(^b$)|(^_b$)|(^x$)|(^y$)|(^r$)|(^s$) + +[VARIABLES] +additional-builtins = raw_input + +[DESIGN] +max-args = 10 + +[FORMAT] +max-line-length = 120 + +[REPORTS] +msg-template = {path}:{line}: [{msg_id}({symbol}), {obj}] {msg} diff --git a/performance_tests/requirements.txt b/performance_tests/requirements.txt new file mode 100644 index 000000000..0b879647f --- /dev/null +++ b/performance_tests/requirements.txt @@ -0,0 +1,5 @@ +attrs >= 17.4.0 +aws-encryption-sdk>=2.3.0 +pytest>=3.3.1 +tqdm +click diff --git a/performance_tests/requirements_mpl.txt b/performance_tests/requirements_mpl.txt new file mode 100644 index 000000000..209e10f2c --- /dev/null +++ b/performance_tests/requirements_mpl.txt @@ -0,0 +1 @@ +aws-cryptographic-material-providers @ git+https://github.com/aws/aws-cryptographic-material-providers-library.git@lucmcdon/python-mpl#subdirectory=AwsCryptographicMaterialProviders/runtimes/python \ No newline at end of file diff --git a/performance_tests/results/.gitkeep b/performance_tests/results/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/performance_tests/setup.cfg b/performance_tests/setup.cfg new file mode 100644 index 000000000..c584a25bd --- /dev/null +++ b/performance_tests/setup.cfg @@ -0,0 +1,41 @@ +[wheel] +universal = 1 + +[metadata] +license_file = LICENSE + +[coverage:run] +branch = True + +[coverage:report] +show_missing = True + +[mypy] +ignore_missing_imports = True + +[flake8] +max_complexity = 10 +max_line_length = 120 +import_order_style = google +application_import_names = aws_encryption_sdk_cli +builtins = raw_input +ignore = + # Ignoring D205 and D400 because of false positives + D205, D400, + # E203 is not PEP8 compliant https://github.com/ambv/black#slices + E203, + # W503 is not PEP8 compliant https://github.com/ambv/black#line-breaks--binary-operators + W503 + +[doc8] +max-line-length = 120 + +[isort] +line_length = 120 +# https://github.com/timothycrosley/isort#multi-line-output-modes +multi_line_output = 3 +include_trailing_comma = True +force_grid_wrap = 0 +combine_as_imports = True +not_skip = __init__.py +known_third_party = attr,aws_encryption_sdk,pytest,setuptools,six diff --git a/performance_tests/setup.py b/performance_tests/setup.py new file mode 100644 index 000000000..702813509 --- /dev/null +++ b/performance_tests/setup.py @@ -0,0 +1,34 @@ +"""Performance test for the AWS Encryption SDK for Python.""" +import os +import re + +from setuptools import find_packages, setup + +VERSION_RE = re.compile(r"""__version__ = ['"]([0-9.]+)['"]""") +HERE = os.path.abspath(os.path.dirname(__file__)) + + +def read(*args): + """Read complete file contents.""" + return open(os.path.join(HERE, *args), encoding="utf-8").read() # pylint: disable=consider-using-with + + +def get_version(): + """Read the version from this module.""" + init = read("src", "aws_encryption_sdk_performance_tests", "__init__.py") + return VERSION_RE.search(init).group(1) + + +setup( + name="aws-encryption-sdk-performance-tests", + packages=find_packages("src"), + package_dir={"": "src"}, + author="Amazon Web Services", + maintainer="Amazon Web Services", + author_email="aws-cryptools@amazon.com", + url="https://github.com/awslabs/aws-encryption-sdk-python", + description="Performance tests for the AWS Encryption SDK for Python", + keywords="aws-encryption-sdk aws kms encryption", + license="Apache License 2.0", + version=get_version(), +) diff --git a/performance_tests/src/aws_encryption_sdk_performance_tests/__init__.py b/performance_tests/src/aws_encryption_sdk_performance_tests/__init__.py new file mode 100644 index 000000000..cf1bf0fb4 --- /dev/null +++ b/performance_tests/src/aws_encryption_sdk_performance_tests/__init__.py @@ -0,0 +1,4 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Stub module indicator to make linter configuration simpler.""" +__version__ = "0.1.0" diff --git a/performance_tests/src/aws_encryption_sdk_performance_tests/keyrings/__init__.py b/performance_tests/src/aws_encryption_sdk_performance_tests/keyrings/__init__.py new file mode 100644 index 000000000..120179eda --- /dev/null +++ b/performance_tests/src/aws_encryption_sdk_performance_tests/keyrings/__init__.py @@ -0,0 +1,3 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Stub module indicator to make linter configuration simpler.""" diff --git a/performance_tests/src/aws_encryption_sdk_performance_tests/keyrings/aws_kms_keyring.py b/performance_tests/src/aws_encryption_sdk_performance_tests/keyrings/aws_kms_keyring.py new file mode 100644 index 000000000..e846ec695 --- /dev/null +++ b/performance_tests/src/aws_encryption_sdk_performance_tests/keyrings/aws_kms_keyring.py @@ -0,0 +1,131 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Performance tests for the AWS KMS keyring.""" + +import aws_encryption_sdk +import boto3 +from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders +from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig +from aws_cryptographic_materialproviders.mpl.models import CreateAwsKmsKeyringInput +from aws_cryptographic_materialproviders.mpl.references import IKeyring + + +def create_keyring( + kms_key_id: str +): + """Demonstrate how to create an AWS KMS keyring. + + Usage: create_keyring(kms_key_id) + :param kms_key_id: KMS Key identifier for the KMS key you want to use. + :type kms_key_id: string + + For more information on KMS Key identifiers, see + https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#key-id + """ + # Create a boto3 client for KMS. + kms_client = create_kms_client() + + # Create a KMS keyring + mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + config=MaterialProvidersConfig() + ) + + keyring_input: CreateAwsKmsKeyringInput = CreateAwsKmsKeyringInput( + kms_key_id=kms_key_id, + kms_client=kms_client + ) + + keyring: IKeyring = mat_prov.create_aws_kms_keyring( + input=keyring_input + ) + + return keyring + + +def create_kms_client(aws_region="us-west-2"): + """Create an AWS KMS client. + + Usage: create_kms_client(aws_region) + :param aws_region: AWS region to use for KMS client. + :type aws_region: string + """ + # Create a boto3 client for KMS. + kms_client = boto3.client('kms', region_name=aws_region) + + return kms_client + + +def create_keyring_given_kms_client( + kms_key_id: str, + kms_client: boto3.client, +): + """Demonstrate how to create an AWS KMS keyring with given KMS client. + + Usage: create_keyring(kms_key_id, kms_client) + :param kms_key_id: KMS Key identifier for the KMS key you want to use. + :type kms_key_id: string + :param kms_client: boto3 client for KMS. + :type kms_client: boto3.client + + For more information on KMS Key identifiers, see + https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#key-id + """ + # Create a KMS keyring + mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + config=MaterialProvidersConfig() + ) + + keyring_input: CreateAwsKmsKeyringInput = CreateAwsKmsKeyringInput( + kms_key_id=kms_key_id, + kms_client=kms_client + ) + + keyring: IKeyring = mat_prov.create_aws_kms_keyring( + input=keyring_input + ) + + return keyring + + +def encrypt_using_keyring( + plaintext_data: bytes, + keyring: IKeyring +): + """Demonstrate how to encrypt plaintext data using an AWS KMS keyring. + + Usage: encrypt_using_keyring(plaintext_data, keyring) + :param plaintext_data: plaintext data you want to encrypt + :type: bytes + :param keyring: Keyring to use for encryption. + :type keyring: IKeyring + """ + client = aws_encryption_sdk.EncryptionSDKClient() + + ciphertext_data, _ = client.encrypt( + source=plaintext_data, + keyring=keyring + ) + + return ciphertext_data + + +def decrypt_using_keyring( + ciphertext_data: bytes, + keyring: IKeyring +): + """Demonstrate how to decrypt ciphertext data using an AWS KMS keyring. + + Usage: decrypt_using_keyring(ciphertext_data, keyring) + :param ciphertext_data: ciphertext data you want to decrypt + :type: bytes + :param keyring: Keyring to use for decryption. + :type keyring: IKeyring + """ + client = aws_encryption_sdk.EncryptionSDKClient() + + decrypted_plaintext_data, _ = client.decrypt( + source=ciphertext_data, + keyring=keyring + ) + + return decrypted_plaintext_data diff --git a/performance_tests/src/aws_encryption_sdk_performance_tests/keyrings/raw_aes_keyring.py b/performance_tests/src/aws_encryption_sdk_performance_tests/keyrings/raw_aes_keyring.py new file mode 100644 index 000000000..a849b1a7f --- /dev/null +++ b/performance_tests/src/aws_encryption_sdk_performance_tests/keyrings/raw_aes_keyring.py @@ -0,0 +1,85 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Performance tests for the Raw AES keyring.""" + +import aws_encryption_sdk +from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders +from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig +from aws_cryptographic_materialproviders.mpl.models import AesWrappingAlg, CreateRawAesKeyringInput +from aws_cryptographic_materialproviders.mpl.references import IKeyring + +from ..utils.util import PerfTestUtils + + +def create_keyring(): + """Demonstrate how to create a Raw AES keyring. + + Usage: create_keyring() + """ + key_name_space = "Some managed raw keys" + key_name = "My 256-bit AES wrapping key" + + # Here, the input to secrets.token_bytes() = 32 bytes = 256 bits + # We fix the static key in order to make the test deterministic + static_key = PerfTestUtils.DEFAULT_AES_256_STATIC_KEY + + mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + config=MaterialProvidersConfig() + ) + + keyring_input: CreateRawAesKeyringInput = CreateRawAesKeyringInput( + key_namespace=key_name_space, + key_name=key_name, + wrapping_key=static_key, + wrapping_alg=AesWrappingAlg.ALG_AES256_GCM_IV12_TAG16 + ) + + keyring: IKeyring = mat_prov.create_raw_aes_keyring( + input=keyring_input + ) + + return keyring + + +def encrypt_using_keyring( + plaintext_data: bytes, + keyring: IKeyring +): + """Demonstrate how to encrypt plaintext data using a Raw AES keyring. + + Usage: encrypt_using_keyring(plaintext_data, keyring) + :param plaintext_data: plaintext data you want to encrypt + :type: bytes + :param keyring: Keyring to use for encryption. + :type keyring: IKeyring + """ + client = aws_encryption_sdk.EncryptionSDKClient() + + ciphertext_data, _ = client.encrypt( + source=plaintext_data, + keyring=keyring + ) + + return ciphertext_data + + +def decrypt_using_keyring( + ciphertext_data: bytes, + keyring: IKeyring +): + """Demonstrate how to decrypt ciphertext data using a Raw AES keyring. + + Usage: decrypt_using_keyring(ciphertext_data, keyring) + :param ciphertext_data: ciphertext data you want to decrypt + :type: bytes + :param keyring: Keyring to use for decryption. + :type keyring: IKeyring + """ + client = aws_encryption_sdk.EncryptionSDKClient() + + decrypted_plaintext_data, _ = client.decrypt( + source=ciphertext_data, + keyring=keyring + ) + + return decrypted_plaintext_data diff --git a/performance_tests/src/aws_encryption_sdk_performance_tests/keyrings/raw_rsa_keyring.py b/performance_tests/src/aws_encryption_sdk_performance_tests/keyrings/raw_rsa_keyring.py new file mode 100644 index 000000000..6eed281bd --- /dev/null +++ b/performance_tests/src/aws_encryption_sdk_performance_tests/keyrings/raw_rsa_keyring.py @@ -0,0 +1,79 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Performance tests for the Raw RSA keyring.""" +import aws_encryption_sdk +from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders +from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig +from aws_cryptographic_materialproviders.mpl.models import CreateRawRsaKeyringInput, PaddingScheme +from aws_cryptographic_materialproviders.mpl.references import IKeyring + + +def create_keyring(public_key, private_key): + """Demonstrate how to create a Raw RSA keyring using the key pair. + + Usage: create_keyring(public_key, private_key) + """ + key_name_space = "Some managed raw keys" + key_name = "My 4096-bit RSA wrapping key" + + mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + config=MaterialProvidersConfig() + ) + + keyring_input: CreateRawRsaKeyringInput = CreateRawRsaKeyringInput( + key_namespace=key_name_space, + key_name=key_name, + padding_scheme=PaddingScheme.OAEP_SHA256_MGF1, + public_key=public_key, + private_key=private_key + ) + + keyring: IKeyring = mat_prov.create_raw_rsa_keyring( + input=keyring_input + ) + + return keyring + + +def encrypt_using_keyring( + plaintext_data: bytes, + keyring: IKeyring +): + """Demonstrate how to encrypt plaintext data using a Raw RSA keyring. + + Usage: encrypt_using_keyring(plaintext_data, keyring) + :param plaintext_data: plaintext data you want to encrypt + :type: bytes + :param keyring: Keyring to use for encryption. + :type keyring: IKeyring + """ + client = aws_encryption_sdk.EncryptionSDKClient() + + ciphertext_data, _ = client.encrypt( + source=plaintext_data, + keyring=keyring + ) + + return ciphertext_data + + +def decrypt_using_keyring( + ciphertext_data: bytes, + keyring: IKeyring +): + """Demonstrate how to decrypt ciphertext data using a Raw RSA keyring. + + Usage: decrypt_using_keyring(ciphertext_data, keyring) + :param ciphertext_data: ciphertext data you want to decrypt + :type: bytes + :param keyring: Keyring to use for decryption. + :type keyring: IKeyring + """ + client = aws_encryption_sdk.EncryptionSDKClient() + + decrypted_plaintext_data, _ = client.decrypt( + source=ciphertext_data, + keyring=keyring + ) + + return decrypted_plaintext_data diff --git a/performance_tests/src/aws_encryption_sdk_performance_tests/master_key_providers/__init__.py b/performance_tests/src/aws_encryption_sdk_performance_tests/master_key_providers/__init__.py new file mode 100644 index 000000000..120179eda --- /dev/null +++ b/performance_tests/src/aws_encryption_sdk_performance_tests/master_key_providers/__init__.py @@ -0,0 +1,3 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Stub module indicator to make linter configuration simpler.""" diff --git a/performance_tests/src/aws_encryption_sdk_performance_tests/master_key_providers/aws_kms_master_key_provider.py b/performance_tests/src/aws_encryption_sdk_performance_tests/master_key_providers/aws_kms_master_key_provider.py new file mode 100644 index 000000000..c3136a5c7 --- /dev/null +++ b/performance_tests/src/aws_encryption_sdk_performance_tests/master_key_providers/aws_kms_master_key_provider.py @@ -0,0 +1,69 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Performance tests for the AWS KMS master key provider.""" + +import aws_encryption_sdk + + +def create_key_provider( + kms_key_id: str +): + """Demonstrate how to create an AWS KMS master key-provider. + + Usage: create_key_provider(kms_key_id) + :param kms_key_id: KMS Key identifier for the KMS key you want to use. + :type kms_key_id: string + + For more information on KMS Key identifiers, see + https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#key-id + """ + # Create a KMS master key-provider. + key_provider = aws_encryption_sdk.StrictAwsKmsMasterKeyProvider(key_ids=[ + kms_key_id, + ]) + + return key_provider + + +def encrypt_using_key_provider( + plaintext_data: bytes, + key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider +): + """Demonstrate how to encrypt plaintext data using an AWS KMS master key-provider. + + Usage: encrypt_using_key_provider(plaintext_data, key_provider) + :param plaintext_data: plaintext data you want to encrypt + :type: bytes + :param key_provider: Master key provider to use for encryption. + :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider + """ + client = aws_encryption_sdk.EncryptionSDKClient() + + ciphertext_data, _ = client.encrypt( + source=plaintext_data, + key_provider=key_provider + ) + + return ciphertext_data + + +def decrypt_using_key_provider( + ciphertext_data: bytes, + key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider +): + """Demonstrate how to decrypt ciphertext data using an AWS KMS master key-provider. + + Usage: decrypt_using_key_provider(ciphertext_data, key_provider) + :param ciphertext_data: ciphertext data you want to decrypt + :type: bytes + :param key_provider: Master key provider to use for decryption. + :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider + """ + client = aws_encryption_sdk.EncryptionSDKClient() + + decrypted_plaintext_data, _ = client.decrypt( + source=ciphertext_data, + key_provider=key_provider + ) + + return decrypted_plaintext_data diff --git a/performance_tests/src/aws_encryption_sdk_performance_tests/master_key_providers/raw_aes_master_key_provider.py b/performance_tests/src/aws_encryption_sdk_performance_tests/master_key_providers/raw_aes_master_key_provider.py new file mode 100644 index 000000000..42d071dcf --- /dev/null +++ b/performance_tests/src/aws_encryption_sdk_performance_tests/master_key_providers/raw_aes_master_key_provider.py @@ -0,0 +1,101 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Performance tests for the Raw AES master key provider.""" + +import aws_encryption_sdk +from aws_encryption_sdk.identifiers import EncryptionKeyType, WrappingAlgorithm +from aws_encryption_sdk.internal.crypto.wrapping_keys import WrappingKey +from aws_encryption_sdk.key_providers.raw import RawMasterKeyProvider + +from ..utils.util import PerfTestUtils + + +class StaticRandomMasterKeyProvider(RawMasterKeyProvider): + """Generates 256-bit keys for each unique key ID.""" + + # The Provider ID (or Provider) field in the JceMasterKey and RawMasterKey is + # equivalent to key namespace in the Raw keyrings + provider_id = "Some managed raw keys" + + def __init__(self, **kwargs): # pylint: disable=unused-argument + """Initialize empty map of keys.""" + self._static_keys = {} + + def _get_raw_key(self, key_id): + """Returns a static, randomly-generated symmetric key for the specified key ID. + + :param str key_id: Key ID + :returns: Wrapping key that contains the specified static key + :rtype: :class:`aws_encryption_sdk.internal.crypto.WrappingKey` + """ + try: + static_key = self._static_keys[key_id] + except KeyError: + # We fix the static key in order to make the test deterministic + # In practice, you should get this key from a secure key management system such as an HSM. + static_key = PerfTestUtils.DEFAULT_AES_256_STATIC_KEY + self._static_keys[key_id] = static_key + return WrappingKey( + wrapping_algorithm=WrappingAlgorithm.AES_256_GCM_IV12_TAG16_NO_PADDING, + wrapping_key=static_key, + wrapping_key_type=EncryptionKeyType.SYMMETRIC, + ) + + +def create_key_provider(): + """Demonstrate how to create a Raw AES master key-provider. + + Usage: create_key_provider() + """ + # Create a Raw AES master key-provider. + + # The Key ID field in the JceMasterKey and RawMasterKey is equivalent to key name in the Raw keyrings + key_id = "My 256-bit AES wrapping key" + key_provider = StaticRandomMasterKeyProvider() + key_provider.add_master_key(key_id) + + return key_provider + + +def encrypt_using_key_provider( + plaintext_data: bytes, + key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider +): + """Demonstrate how to encrypt plaintext data using a Raw AES master key-provider. + + Usage: encrypt_using_key_provider(plaintext_data, key_provider) + :param plaintext_data: plaintext data you want to encrypt + :type: bytes + :param key_provider: Master key provider to use for encryption. + :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider + """ + client = aws_encryption_sdk.EncryptionSDKClient() + + ciphertext_data, _ = client.encrypt( + source=plaintext_data, + key_provider=key_provider + ) + + return ciphertext_data + + +def decrypt_using_key_provider( + ciphertext_data: bytes, + key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider +): + """Demonstrate how to decrypt ciphertext data using a Raw AES master key-provider. + + Usage: decrypt_using_key_provider(ciphertext_data, key_provider) + :param ciphertext_data: ciphertext data you want to decrypt + :type: bytes + :param key_provider: Master key provider to use for decryption. + :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider + """ + client = aws_encryption_sdk.EncryptionSDKClient() + + decrypted_plaintext_data, _ = client.decrypt( + source=ciphertext_data, + key_provider=key_provider + ) + + return decrypted_plaintext_data diff --git a/performance_tests/src/aws_encryption_sdk_performance_tests/master_key_providers/raw_rsa_master_key_provider.py b/performance_tests/src/aws_encryption_sdk_performance_tests/master_key_providers/raw_rsa_master_key_provider.py new file mode 100644 index 000000000..b52b78735 --- /dev/null +++ b/performance_tests/src/aws_encryption_sdk_performance_tests/master_key_providers/raw_rsa_master_key_provider.py @@ -0,0 +1,101 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Performance tests for the Raw RSA master key provider.""" + +import aws_encryption_sdk +from aws_encryption_sdk.identifiers import EncryptionKeyType, WrappingAlgorithm +from aws_encryption_sdk.internal.crypto.wrapping_keys import WrappingKey +from aws_encryption_sdk.key_providers.raw import RawMasterKeyProvider + +from aws_encryption_sdk_performance_tests.utils.util import PerfTestUtils + + +class StaticRandomMasterKeyProvider(RawMasterKeyProvider): + """Randomly generates and provides 4096-bit RSA keys consistently per unique key id.""" + + # The Provider ID (or Provider) field in the JceMasterKey and RawMasterKey is + # equivalent to key namespace in the Raw keyrings + provider_id = "Some managed raw keys" + + def __init__(self, **kwargs): # pylint: disable=unused-argument + """Initialize empty map of keys.""" + self._static_keys = {} + + def _get_raw_key(self, key_id): + """Retrieves a static, randomly generated, RSA key for the specified key id. + + :param str key_id: User-defined ID for the static key + :returns: Wrapping key that contains the specified static key + :rtype: :class:`aws_encryption_sdk.internal.crypto.WrappingKey` + """ + try: + static_key = self._static_keys[key_id] + except KeyError: + # We fix the static key in order to make the test deterministic + # In practice, you should get this key from a secure key management system such as an HSM. + static_key = PerfTestUtils.DEFAULT_RSA_PRIVATE_KEY + self._static_keys[key_id] = static_key + return WrappingKey( + wrapping_algorithm=WrappingAlgorithm.RSA_OAEP_SHA256_MGF1, + wrapping_key=static_key, + wrapping_key_type=EncryptionKeyType.PRIVATE, + ) + + +def create_key_provider(): + """Demonstrate how to create a Raw RSA master key-provider. + + Usage: create_key_provider() + """ + # Create a Raw RSA master key-provider. + + # The Key ID field in the JceMasterKey and RawMasterKey is equivalent to key name in the Raw keyrings + key_id = "My 4096-bit RSA wrapping key" + key_provider = StaticRandomMasterKeyProvider() + key_provider.add_master_key(key_id) + + return key_provider + + +def encrypt_using_key_provider( + plaintext_data: bytes, + key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider +): + """Demonstrate how to encrypt plaintext data using a Raw RSA master key-provider. + + Usage: encrypt_using_key_provider(plaintext_data, key_provider) + :param plaintext_data: plaintext data you want to encrypt + :type: bytes + :param key_provider: Master key provider to use for encryption. + :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider + """ + client = aws_encryption_sdk.EncryptionSDKClient() + + ciphertext_data, _ = client.encrypt( + source=plaintext_data, + key_provider=key_provider + ) + + return ciphertext_data + + +def decrypt_using_key_provider( + ciphertext_data: bytes, + key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider +): + """Demonstrate how to decrypt ciphertext data using a Raw RSA master key-provider. + + Usage: decrypt_using_key_provider(ciphertext_data, key_provider) + :param ciphertext_data: ciphertext data you want to decrypt + :type: bytes + :param key_provider: Master key provider to use for decryption. + :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider + """ + client = aws_encryption_sdk.EncryptionSDKClient() + + decrypted_plaintext_data, _ = client.decrypt( + source=ciphertext_data, + key_provider=key_provider + ) + + return decrypted_plaintext_data diff --git a/performance_tests/src/aws_encryption_sdk_performance_tests/utils/__init__.py b/performance_tests/src/aws_encryption_sdk_performance_tests/utils/__init__.py new file mode 100644 index 000000000..120179eda --- /dev/null +++ b/performance_tests/src/aws_encryption_sdk_performance_tests/utils/__init__.py @@ -0,0 +1,3 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Stub module indicator to make linter configuration simpler.""" diff --git a/performance_tests/src/aws_encryption_sdk_performance_tests/utils/util.py b/performance_tests/src/aws_encryption_sdk_performance_tests/utils/util.py new file mode 100644 index 000000000..52914b76a --- /dev/null +++ b/performance_tests/src/aws_encryption_sdk_performance_tests/utils/util.py @@ -0,0 +1,116 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Utility functions for AWS Encryption SDK performance tests.""" + + +class PerfTestUtils: + """Utility functions for AWS Encryption SDK performance tests.""" + DEFAULT_N_ITERS = 100 + DEFAULT_TESTING_N_ITERS = 1 + DEFAULT_FILE_SIZE = 'medium' + DEFAULT_AES_256_STATIC_KEY = \ + b'_\xcf"\x82\x03\x12\x9d\x00\x8a\xed\xaf\xe4\x80\x1d\x00t\xa6P\xac\xb6\xfe\xc5\xf6/{\xe7\xaaO\x01\x13W\x85' + DEFAULT_RSA_PUBLIC_KEY = bytes("-----BEGIN PUBLIC KEY-----\n" + + "MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxwWEEtEofjwaoo3WO79D\n" + + "hntoPf2APlY5yzlqm6ZvMyaazlwetkAzLSn5GB4hjKZaf043BfADJEdwXMHn8/UN\n" + + "up0BfUj8PfGn/b8cL78CTnvFZd/7WxQh6tUnfLX7BMiccHMb9OHhRy5PrTSuj6Um\n" + + "wwhBadL+Lc23DGl2cyN9SjGuYWWQ1IHGFA4/2EQr+Ez4LpebZqwXgv0iLuApte1q\n" + + "vGl6zOhByxi1N/ORVEscLT82+L+F3STgeTYA1CaoLFQ0y9ybx+7UUfEfKxhGoGEO\n" + + "XEOTuRBdLE2Jm8xaBODLqfiXr0z62VhTpRs4CYYTGHTLFCJHqeH7R2fwvwoG1nIg\n" + + "QzWSyyapK7d5MLn3rF3ManjZhvlyHK1wqa7nWVpo+jq1Py+HWLAtU8FY0br6wnOR\n" + + "3jjPGk0N4//iDnxNN+kpDxFnHEvxe3eJKWnbw0GR9+BGj32O+wRMtGyfRTzkoD/E\n" + + "EqIRlDzdtYCAtFW0HUsdQwL+ssDjEQ0+lqvEQrwTU1WBZiBQhEmzksAowHAcNIT+\n" + + "Fz7mvIlpEETNOQbsJkoXdEkhJXljh5UYmH1cB5al1MJf/5ea5Xb2HfH5WkMy4+eS\n" + + "V68V+tXv3ZthTe2bCk9rQTH9FWKLIYJyZfv8WAIxSWEEsyk5b+7WUGmvtm/nPJ4Z\n" + + "RfzkXoBJqJiSiPYCM0+jG4sCAwEAAQ==\n" + + "-----END PUBLIC KEY-----\n", 'utf-8') + + DEFAULT_RSA_PRIVATE_KEY = bytes("-----BEGIN PRIVATE KEY-----\n" + + "MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDHBYQS0Sh+PBqi\n" + + "jdY7v0OGe2g9/YA+VjnLOWqbpm8zJprOXB62QDMtKfkYHiGMplp/TjcF8AMkR3Bc\n" + + "wefz9Q26nQF9SPw98af9vxwvvwJOe8Vl3/tbFCHq1Sd8tfsEyJxwcxv04eFHLk+t\n" + + "NK6PpSbDCEFp0v4tzbcMaXZzI31KMa5hZZDUgcYUDj/YRCv4TPgul5tmrBeC/SIu\n" + + "4Cm17Wq8aXrM6EHLGLU385FUSxwtPzb4v4XdJOB5NgDUJqgsVDTL3JvH7tRR8R8r\n" + + "GEagYQ5cQ5O5EF0sTYmbzFoE4Mup+JevTPrZWFOlGzgJhhMYdMsUIkep4ftHZ/C/\n" + + "CgbWciBDNZLLJqkrt3kwufesXcxqeNmG+XIcrXCprudZWmj6OrU/L4dYsC1TwVjR\n" + + "uvrCc5HeOM8aTQ3j/+IOfE036SkPEWccS/F7d4kpadvDQZH34EaPfY77BEy0bJ9F\n" + + "POSgP8QSohGUPN21gIC0VbQdSx1DAv6ywOMRDT6Wq8RCvBNTVYFmIFCESbOSwCjA\n" + + "cBw0hP4XPua8iWkQRM05BuwmShd0SSEleWOHlRiYfVwHlqXUwl//l5rldvYd8fla\n" + + "QzLj55JXrxX61e/dm2FN7ZsKT2tBMf0VYoshgnJl+/xYAjFJYQSzKTlv7tZQaa+2\n" + + "b+c8nhlF/ORegEmomJKI9gIzT6MbiwIDAQABAoICAAi9ysfCzQsCW88g+LRmGbKp\n" + + "7/GtFTlnsyEkc/TDMiYmf20p6aVqm3TT3596D1IsqlPmHQ+TM6gfxSUl1SjHbiNw\n" + + "qvSURJP57b186+GC+7hzwj9Pv6wH7ddxJktZeN2EbC6aN7OhSjJEq/Y5FqOzhsjR\n" + + "L4JU5Joha3VNmojDGcks9nJLsjlLO+Z8m7xFfkLpKottWEOBsoSr1pkFen+FnocJ\n" + + "AP5IAz/G5YrAFXWE2Qd5u9HgI6KLcJqSTyYCTqenySvdFDCLYmL4+rv7VHrN2IIf\n" + + "67iYqeb8vtsLdja5ouhjxVHLSUdLlFzvnZ35eBQ+aP8I5GnnRZCk1ZOmfpdjqtwE\n" + + "4mQRJU44DtGH/aySgQEAjn5BAxjrflSBpgAJs8HxTIoGXEEtGgJQeJcvSxv/1fTy\n" + + "EJSmwzepxDT1kAK0BPEllSHNLlHTEeJ8FMCGaEofDXPvJsJP/UvWxGmyRQXtG68m\n" + + "WAy27OsAQ2z6Iqn2829lUnJERjtFUHJDu5ZlJHRPz6d7FTbmI5jFOGGTDWKtHqFI\n" + + "88JZTwby55KyYLwDyxbqcDrRSOtzZ4N0rV2tLIMRoMDpjhJ8CopuxuQyxeuP3/7V\n" + + "tcW4IbNTqEDKL4TFZkZhb+govAvFAkRFjBWu7kZpSEGNVvR+O1pTgXxWsfaAb+3K\n" + + "EZ0lXelzaGCMCbwysAhxAoIBAQD+AfzgIva7GelSKujRO8rlhtPxoVNTMDbRo9QX\n" + + "NtztLHvrxyaZqM5nqf4rMjrbU7vPdT5Fn/3/iupaBkZk8IqqdKpdmgi+Pr+aFvOB\n" + + "LU2blEY8zWZCOwYerrwEPbQKblLdkIhDvOGpx1g4JuAlqIqJWW/RvMODf9Makwyq\n" + + "fxkG+y2Cr8TIsM3jKXprOkgeE7sB97r2OvkSuL/xP2cedCt0dI1vnk2QvUWw6af4\n" + + "Fs4xzqntS3KG3PHM9Jhljadm6da3cFnQxTIYpS0qT+Dv07NnTn1Ysjb2iCXEjvW2\n" + + "vZEjrcLO4dWfZXVIXAjKhG+e/MbCcjEmbhd480SvDjImzk47AoIBAQDIlR+afYw+\n" + + "UHaaQJiqnkY8E/ju4emgwVDZN3QJGEQS1q+HrCM0QAD410cwEBiyhuciYN27lEfU\n" + + "3NXNb4TKYLN9u/Alj0Em+UFN/cPdUEvgrqQXS5E5GWOX3ehG3LYI/a4n6nlo/zdu\n" + + "GSqHU93i8PoKweQFS23oCqnCkH5xBRcyvC3J/T4G/fl8FrnoVn9HLs3vM0gYMZSl\n" + + "Ej2XZJXbitpqS3QyK51ULePVwaC3Zjot3YxsAzpdcSG1/6VNj1QWr9KAr8YdXTu7\n" + + "VcStCElDksVbfMgYahpBYlU4xipPA101ll1KPom1ECI/F6ku5b2H2vnewy2TNzsY\n" + + "QX0R4NFofQLxAoIBAG2af/pbO+naMXKSL2nxighmmFfATAsuV8k4DxGBS+1Pb516\n" + + "jq5pR781fAY5o2n2hKjtJ1S1x80XrS3xXTi7Dqqkssq256TnwJeF5cbMvJswbOpZ\n" + + "mxFjFK3yqhCOa3zAxCL09cd83kb7TJbWN4woYLcJj5WKBTdd1cK2xxVeyHbZtXaZ\n" + + "z6jlmcG2qStRt8K6sswTkGolYkpwy+oWeLGMYR/cFxed0ExvT34aJK+Jb6nQSkSp\n" + + "dJ67Ad91f7j6WcyvhEYdRbQvEwHNbGLAmwgBan1eQfoe1Famwt1A7sfOnq0tkkzg\n" + + "5+PizKvPgr+YS+3nlwBac9joUlqPZgi/cGaMSPcCggEBALbTLZ4sJyM5RhFtJXoG\n" + + "j6/86F4cbk1HRwDmSY5snsepBQ8duGzMldY6qrlFQq2expgQQKrUCfEcZIg+yIOK\n" + + "RrApGEez3ke+02ZaEifsI20k4Y4WI8UuvhdTfX7xd76UMyRQ1N7+GTDyIVB+AfXz\n" + + "fYVGmya0TPY+meMsvwMXB8EHwpikid/nqHoRYNxD0vk30R7g2CqtLnaTPK58URdt\n" + + "5Y0TP1LnbBypQ0y3k1z3AbqCgJaHDrDTCE4SOUKLjLKtCaqgDG0BaQtkrsKkldrQ\n" + + "sbCk+OE//LRyA4mfHjssrs3EQz4D6JKvpPdrApsrbmihEDWaIzVXFzcRogUkrNqX\n" + + "b5ECggEBAKGW7doJEm0MjyvrJj/Tj4Zx3S8UjMgheBEIUZtMjewtNL0pn70O2AxN\n" + + "aEa4zHaNS0yTgMdbObImzYgat+asJbmFcv0UJy/e4CN+rrZlCHW2D9v9U+O0wKLB\n" + + "e5AmmFwaT/vVIy4gmBTcKGxV90ZF799gmKSoHAlrgjPFSRB/WcJsMwsGEyXl/C4Z\n" + + "4/xCqJgr0VJvuwrCiWf1QKn9AHuytit27E2R52n4FjU5nJ+CJEQqU1XDgF0x+txw\n" + + "PXUuRjOxKO6MzldzqJSUrTir8uqCwBIR9x9GOrGDp//ZbRw2TK4EbkyjNYO7KtOF\n" + + "A/DHJmMI5bKETJyj1GhBE9LqypAI1Bo=\n" + + "-----END PRIVATE KEY-----\n", "utf-8") + + DEFAULT_ENCRYPTION_CONTEXT = { + "tenant": "TenantA", + "encryption": "context", + "is not": "secret", + "but adds": "useful metadata", + "that can help you": "be confident that", + "the data you are handling": "is what you think it is", + } + + DEFAULT_BRANCH_KEY_ID_A = 'a52dfaad-7dbd-4430-a1fd-abaa5299da07' + + DEFAULT_BRANCH_KEY_ID_B = '8ba79cef-581c-4125-9292-b057a29d42d7' + + @staticmethod + def read_file(filename): + """Returns the contents of the file.""" + with open(filename, 'rb') as file: + return file.read() + + @staticmethod + def get_rsa_key_from_file(filename): + """Returns the RSA key""" + with open(filename, "r", encoding='utf-8') as f: + key = f.read() + + # Convert the key from a string to bytes + key = bytes(key, 'utf-8') + + return key + + @staticmethod + def write_time_list_to_csv(time_list, filename): + """Writes the time list to a CSV file.""" + with open(filename + '.csv', 'w', encoding='utf-8') as myfile: + for time in time_list: + myfile.write(str(time) + '\n') diff --git a/performance_tests/test/keyrings/__init__.py b/performance_tests/test/keyrings/__init__.py new file mode 100644 index 000000000..120179eda --- /dev/null +++ b/performance_tests/test/keyrings/__init__.py @@ -0,0 +1,3 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Stub module indicator to make linter configuration simpler.""" diff --git a/performance_tests/test/keyrings/test_aws_kms_keyring.py b/performance_tests/test/keyrings/test_aws_kms_keyring.py new file mode 100644 index 000000000..950a2a82e --- /dev/null +++ b/performance_tests/test/keyrings/test_aws_kms_keyring.py @@ -0,0 +1,206 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""This is a performance test for creating the AWS KMS keyring.""" + +import os +import time + +import click +import click.testing +import pytest +from tqdm import tqdm + +from aws_encryption_sdk_performance_tests.keyrings.aws_kms_keyring import ( + create_keyring, + create_keyring_given_kms_client, + create_kms_client, + decrypt_using_keyring, + encrypt_using_keyring, +) +from aws_encryption_sdk_performance_tests.utils.util import PerfTestUtils + +MODULE_ABS_PATH = os.path.abspath(__file__) + + +@click.group() +def create_kms_keyring(): + """Click group helper function""" + + +@create_kms_keyring.command() +@click.option('--kms_key_id', + default='arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f') +@click.option('--n_iters', + default=PerfTestUtils.DEFAULT_N_ITERS) +@click.option('--output_file', + default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/kms_keyring_create') +def create( + kms_key_id: str, + n_iters: int, + output_file: str +): + """Performance test for the create_keyring function.""" + time_list = [] + for _ in tqdm(range(n_iters)): + curr_time = time.time() + + create_keyring(kms_key_id) + + # calculate elapsed time in milliseconds + elapsed_time = (time.time() - curr_time) * 1000 + time_list.append(elapsed_time) + PerfTestUtils.write_time_list_to_csv(time_list, output_file) + + +@click.group() +def create_kms_keyring_given_kms_client(): + """Click group helper function""" + + +@create_kms_keyring_given_kms_client.command() +@click.option('--kms_key_id', + default='arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f') +@click.option('--n_iters', + default=PerfTestUtils.DEFAULT_N_ITERS) +@click.option('--output_file', + default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/kms_keyring_create_given_kms_client') +def create_given_kms_client( + kms_key_id: str, + n_iters: int, + output_file: str +): + """Performance test for the create_keyring function.""" + kms_client = create_kms_client() + time_list = [] + for _ in tqdm(range(n_iters)): + curr_time = time.time() + + create_keyring_given_kms_client(kms_key_id, kms_client) + + # calculate elapsed time in milliseconds + elapsed_time = (time.time() - curr_time) * 1000 + time_list.append(elapsed_time) + + PerfTestUtils.write_time_list_to_csv(time_list, output_file) + + +@click.group() +def encrypt_kms_keyring(): + """Click group helper function""" + + +@encrypt_kms_keyring.command() +@click.option('--plaintext_data_filename', + default='/'.join(MODULE_ABS_PATH.split("/")[:-2]) + '/resources/plaintext/plaintext-data-' + + PerfTestUtils.DEFAULT_FILE_SIZE + '.dat') +@click.option('--kms_key_id', + default='arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f') +@click.option('--n_iters', + default=PerfTestUtils.DEFAULT_N_ITERS) +@click.option('--output_file', + default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/kms_keyring_encrypt') +def encrypt( + plaintext_data_filename: str, + kms_key_id: str, + n_iters: int, + output_file: str +): + """Performance test for the encrypt_using_keyring function.""" + plaintext_data = PerfTestUtils.read_file(plaintext_data_filename) + + keyring = create_keyring(kms_key_id) + time_list = [] + + for _ in tqdm(range(n_iters)): + curr_time = time.time() + + encrypt_using_keyring(plaintext_data, keyring) + + # calculate elapsed time in milliseconds + elapsed_time = (time.time() - curr_time) * 1000 + time_list.append(elapsed_time) + + PerfTestUtils.write_time_list_to_csv(time_list, output_file) + + +@click.group() +def decrypt_kms_keyring(): + """Click group helper function""" + + +@decrypt_kms_keyring.command() +@click.option('--ciphertext_data_filename', + default='/'.join(MODULE_ABS_PATH.split("/")[:-2]) + '/resources/ciphertext/kms/ciphertext-data-' + + PerfTestUtils.DEFAULT_FILE_SIZE + '.ct') +@click.option('--kms_key_id', + default='arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f') +@click.option('--n_iters', + default=PerfTestUtils.DEFAULT_N_ITERS) +@click.option('--output_file', + default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/kms_keyring_decrypt') +def decrypt( + ciphertext_data_filename: str, + kms_key_id: str, + n_iters: int, + output_file: str +): + """Performance test for the decrypt_using_keyring function.""" + ciphertext_data = PerfTestUtils.read_file(ciphertext_data_filename) + + keyring = create_keyring(kms_key_id) + time_list = [] + + for _ in tqdm(range(n_iters)): + curr_time = time.time() + + decrypt_using_keyring(ciphertext_data, keyring) + + # calculate elapsed time in milliseconds + elapsed_time = (time.time() - curr_time) * 1000 + time_list.append(elapsed_time) + + PerfTestUtils.write_time_list_to_csv(time_list, output_file) + + +kms_keyring_test = click.CommandCollection(sources=[create_kms_keyring, + create_kms_keyring_given_kms_client, + encrypt_kms_keyring, + decrypt_kms_keyring]) + + +@pytest.fixture +def runner(): + """Click runner""" + return click.testing.CliRunner() + + +def test_create(runner): + """Test the create_keyring function""" + result = runner.invoke(create_kms_keyring.commands['create'], + ['--n_iters', PerfTestUtils.DEFAULT_TESTING_N_ITERS]) + assert result.exit_code == 0 + + +def test_create_given_kms_client(runner): + """Test the create_keyring_given_kms_client function""" + result = runner.invoke(create_kms_keyring_given_kms_client.commands['create-given-kms-client'], + ['--n_iters', PerfTestUtils.DEFAULT_TESTING_N_ITERS]) + assert result.exit_code == 0 + + +def test_encrypt(runner): + """Test the encrypt_using_keyring function""" + result = runner.invoke(encrypt_kms_keyring.commands['encrypt'], + ['--n_iters', PerfTestUtils.DEFAULT_TESTING_N_ITERS]) + assert result.exit_code == 0 + + +def test_decrypt(runner): + """Test the decrypt_using_keyring function""" + result = runner.invoke(decrypt_kms_keyring.commands['decrypt'], + ['--n_iters', PerfTestUtils.DEFAULT_TESTING_N_ITERS]) + assert result.exit_code == 0 + + +if __name__ == "__main__": + kms_keyring_test() diff --git a/performance_tests/test/keyrings/test_raw_aes_keyring.py b/performance_tests/test/keyrings/test_raw_aes_keyring.py new file mode 100644 index 000000000..1e1da428a --- /dev/null +++ b/performance_tests/test/keyrings/test_raw_aes_keyring.py @@ -0,0 +1,156 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""This is a performance test for creating the Raw AES keyring.""" + +import os +import time + +import click +import click.testing +import pytest +from tqdm import tqdm + +from aws_encryption_sdk_performance_tests.keyrings.raw_aes_keyring import ( + create_keyring, + decrypt_using_keyring, + encrypt_using_keyring, +) +from aws_encryption_sdk_performance_tests.utils.util import PerfTestUtils + +MODULE_ABS_PATH = os.path.abspath(__file__) + + +@click.group() +def create_raw_aes_keyring(): + """Click group helper function""" + + +@create_raw_aes_keyring.command() +@click.option('--n_iters', + default=PerfTestUtils.DEFAULT_N_ITERS) +@click.option('--output_file', + default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/raw_aes_keyring_create') +def create( + n_iters: int, + output_file: str +): + """Performance test for the create_keyring function.""" + time_list = [] + for _ in tqdm(range(n_iters)): + curr_time = time.time() + + create_keyring() + + # calculate elapsed time in milliseconds + elapsed_time = (time.time() - curr_time) * 1000 + time_list.append(elapsed_time) + + PerfTestUtils.write_time_list_to_csv(time_list, output_file) + + +@click.group() +def encrypt_raw_aes_keyring(): + """Click group helper function""" + + +@encrypt_raw_aes_keyring.command() +@click.option('--plaintext_data_filename', + default='/'.join(MODULE_ABS_PATH.split("/")[:-2]) + '/resources/plaintext/plaintext-data-' + + PerfTestUtils.DEFAULT_FILE_SIZE + '.dat') +@click.option('--n_iters', + default=PerfTestUtils.DEFAULT_N_ITERS) +@click.option('--output_file', + default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/raw_aes_keyring_encrypt') +def encrypt( + plaintext_data_filename: str, + n_iters: int, + output_file: str +): + """Performance test for the encrypt_using_keyring function.""" + plaintext_data = PerfTestUtils.read_file(plaintext_data_filename) + + keyring = create_keyring() + time_list = [] + + for _ in tqdm(range(n_iters)): + curr_time = time.time() + + encrypt_using_keyring(plaintext_data, keyring) + + # calculate elapsed time in milliseconds + elapsed_time = (time.time() - curr_time) * 1000 + time_list.append(elapsed_time) + + PerfTestUtils.write_time_list_to_csv(time_list, output_file) + + +@click.group() +def decrypt_raw_aes_keyring(): + """Click group helper function""" + + +@decrypt_raw_aes_keyring.command() +@click.option('--ciphertext_data_filename', + default='/'.join(MODULE_ABS_PATH.split("/")[:-2]) + '/resources/ciphertext/raw_aes/ciphertext-data-' + + PerfTestUtils.DEFAULT_FILE_SIZE + '.ct') +@click.option('--n_iters', + default=PerfTestUtils.DEFAULT_N_ITERS) +@click.option('--output_file', + default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/raw_aes_keyring_decrypt') +def decrypt( + ciphertext_data_filename: str, + n_iters: int, + output_file: str +): + """Performance test for the decrypt_using_keyring function.""" + ciphertext_data = PerfTestUtils.read_file(ciphertext_data_filename) + + keyring = create_keyring() + time_list = [] + + for _ in tqdm(range(n_iters)): + curr_time = time.time() + + decrypt_using_keyring(ciphertext_data, keyring) + + # calculate elapsed time in milliseconds + elapsed_time = (time.time() - curr_time) * 1000 + time_list.append(elapsed_time) + + PerfTestUtils.write_time_list_to_csv(time_list, output_file) + + +raw_aes_keyring_test = click.CommandCollection(sources=[create_raw_aes_keyring, + encrypt_raw_aes_keyring, + decrypt_raw_aes_keyring]) + + +@pytest.fixture +def runner(): + """Click runner""" + return click.testing.CliRunner() + + +def test_create(runner): + """Test the create_keyring function""" + result = runner.invoke(create_raw_aes_keyring.commands['create'], + ['--n_iters', PerfTestUtils.DEFAULT_TESTING_N_ITERS]) + assert result.exit_code == 0 + + +def test_encrypt(runner): + """Test the encrypt_using_keyring function""" + result = runner.invoke(encrypt_raw_aes_keyring.commands['encrypt'], + ['--n_iters', PerfTestUtils.DEFAULT_TESTING_N_ITERS]) + assert result.exit_code == 0 + + +def test_decrypt(runner): + """Test the decrypt_using_keyring function""" + result = runner.invoke(decrypt_raw_aes_keyring.commands['decrypt'], + ['--n_iters', PerfTestUtils.DEFAULT_TESTING_N_ITERS]) + assert result.exit_code == 0 + + +if __name__ == "__main__": + raw_aes_keyring_test() diff --git a/performance_tests/test/keyrings/test_raw_rsa_keyring.py b/performance_tests/test/keyrings/test_raw_rsa_keyring.py new file mode 100644 index 000000000..476701ac0 --- /dev/null +++ b/performance_tests/test/keyrings/test_raw_rsa_keyring.py @@ -0,0 +1,163 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""This is a performance test for creating the Raw RSA keyring.""" + +import os +import time + +import click +import click.testing +import pytest +from tqdm import tqdm + +from aws_encryption_sdk_performance_tests.keyrings.raw_rsa_keyring import ( + create_keyring, + decrypt_using_keyring, + encrypt_using_keyring, +) +from aws_encryption_sdk_performance_tests.utils.util import PerfTestUtils + +MODULE_ABS_PATH = os.path.abspath(__file__) + + +@click.group() +def create_raw_rsa_keyring(): + """Click group helper function""" + + +@create_raw_rsa_keyring.command() +@click.option('--n_iters', + default=PerfTestUtils.DEFAULT_N_ITERS) +@click.option('--output_file', + default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/raw_rsa_keyring_create') +def create( + n_iters: int, + output_file: str +): + """Performance test for the create_keyring function.""" + public_key = PerfTestUtils.DEFAULT_RSA_PUBLIC_KEY + private_key = PerfTestUtils.DEFAULT_RSA_PRIVATE_KEY + + time_list = [] + for _ in tqdm(range(n_iters)): + curr_time = time.time() + + create_keyring(public_key, private_key) + + # calculate elapsed time in milliseconds + elapsed_time = (time.time() - curr_time) * 1000 + time_list.append(elapsed_time) + + PerfTestUtils.write_time_list_to_csv(time_list, output_file) + + +@click.group() +def encrypt_raw_rsa_keyring(): + """Click group helper function""" + + +@encrypt_raw_rsa_keyring.command() +@click.option('--plaintext_data_filename', + default='/'.join(MODULE_ABS_PATH.split("/")[:-2]) + '/resources/plaintext/plaintext-data-' + + PerfTestUtils.DEFAULT_FILE_SIZE + '.dat') +@click.option('--n_iters', + default=PerfTestUtils.DEFAULT_N_ITERS) +@click.option('--output_file', + default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/raw_rsa_keyring_encrypt') +def encrypt( + plaintext_data_filename: str, + n_iters: int, + output_file: str +): + """Performance test for the encrypt_using_keyring function.""" + public_key = PerfTestUtils.DEFAULT_RSA_PUBLIC_KEY + private_key = PerfTestUtils.DEFAULT_RSA_PRIVATE_KEY + plaintext_data = PerfTestUtils.read_file(plaintext_data_filename) + + keyring = create_keyring(public_key, private_key) + time_list = [] + + for _ in tqdm(range(n_iters)): + curr_time = time.time() + + encrypt_using_keyring(plaintext_data, keyring) + + # calculate elapsed time in milliseconds + elapsed_time = (time.time() - curr_time) * 1000 + time_list.append(elapsed_time) + + PerfTestUtils.write_time_list_to_csv(time_list, output_file) + + +@click.group() +def decrypt_raw_rsa_keyring(): + """Click group helper function""" + + +@decrypt_raw_rsa_keyring.command() +@click.option('--ciphertext_data_filename', + default='/'.join(MODULE_ABS_PATH.split("/")[:-2]) + '/resources/ciphertext/raw_rsa/ciphertext-data-' + + PerfTestUtils.DEFAULT_FILE_SIZE + '.ct') +@click.option('--n_iters', + default=PerfTestUtils.DEFAULT_N_ITERS) +@click.option('--output_file', + default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/raw_rsa_keyring_decrypt') +def decrypt( + ciphertext_data_filename: str, + n_iters: int, + output_file: str +): + """Performance test for the decrypt_using_keyring function.""" + public_key = PerfTestUtils.DEFAULT_RSA_PUBLIC_KEY + private_key = PerfTestUtils.DEFAULT_RSA_PRIVATE_KEY + ciphertext_data = PerfTestUtils.read_file(ciphertext_data_filename) + + keyring = create_keyring(public_key, private_key) + time_list = [] + + for _ in tqdm(range(n_iters)): + curr_time = time.time() + + decrypt_using_keyring(ciphertext_data, keyring) + + # calculate elapsed time in milliseconds + elapsed_time = (time.time() - curr_time) * 1000 + time_list.append(elapsed_time) + + PerfTestUtils.write_time_list_to_csv(time_list, output_file) + + +raw_rsa_keyring_test = click.CommandCollection(sources=[create_raw_rsa_keyring, + encrypt_raw_rsa_keyring, + decrypt_raw_rsa_keyring]) + + +@pytest.fixture +def runner(): + """Click runner""" + return click.testing.CliRunner() + + +def test_create(runner): + """Test the create_keyring function""" + result = runner.invoke(create_raw_rsa_keyring.commands['create'], + ['--n_iters', PerfTestUtils.DEFAULT_TESTING_N_ITERS]) + assert result.exit_code == 0 + + +def test_encrypt(runner): + """Test the encrypt_using_keyring function""" + result = runner.invoke(encrypt_raw_rsa_keyring.commands['encrypt'], + ['--n_iters', PerfTestUtils.DEFAULT_TESTING_N_ITERS]) + assert result.exit_code == 0 + + +def test_decrypt(runner): + """Test the decrypt_using_keyring function""" + result = runner.invoke(decrypt_raw_rsa_keyring.commands['decrypt'], + ['--n_iters', PerfTestUtils.DEFAULT_TESTING_N_ITERS]) + assert result.exit_code == 0 + + +if __name__ == "__main__": + raw_rsa_keyring_test() diff --git a/performance_tests/test/master_key_providers/__init__.py b/performance_tests/test/master_key_providers/__init__.py new file mode 100644 index 000000000..120179eda --- /dev/null +++ b/performance_tests/test/master_key_providers/__init__.py @@ -0,0 +1,3 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Stub module indicator to make linter configuration simpler.""" diff --git a/performance_tests/test/master_key_providers/test_aws_kms_master_key_provider.py b/performance_tests/test/master_key_providers/test_aws_kms_master_key_provider.py new file mode 100644 index 000000000..b869245b5 --- /dev/null +++ b/performance_tests/test/master_key_providers/test_aws_kms_master_key_provider.py @@ -0,0 +1,165 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""This is a performance test for creating the AWS KMS Master key-provider.""" + +import os +import time + +import click +import click.testing +import pytest +from tqdm import tqdm + +from aws_encryption_sdk_performance_tests.master_key_providers.aws_kms_master_key_provider import ( + create_key_provider, + decrypt_using_key_provider, + encrypt_using_key_provider, +) +from aws_encryption_sdk_performance_tests.utils.util import PerfTestUtils + +MODULE_ABS_PATH = os.path.abspath(__file__) + + +@click.group() +def create_kms_key_provider(): + """Click group helper function""" + + +@create_kms_key_provider.command() +@click.option('--kms_key_id', + default='arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f') +@click.option('--n_iters', + default=PerfTestUtils.DEFAULT_N_ITERS) +@click.option('--output_file', + default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/kms_key_provider_create') +def create( + kms_key_id: str, + n_iters: int, + output_file: str +): + """Performance test for the create_key_provider function.""" + time_list = [] + for _ in tqdm(range(n_iters)): + curr_time = time.time() + + create_key_provider(kms_key_id) + + # calculate elapsed time in milliseconds + elapsed_time = (time.time() - curr_time) * 1000 + time_list.append(elapsed_time) + + PerfTestUtils.write_time_list_to_csv(time_list, output_file) + + +@click.group() +def encrypt_kms_key_provider(): + """Click group helper function""" + + +@encrypt_kms_key_provider.command() +@click.option('--plaintext_data_filename', + default='/'.join(MODULE_ABS_PATH.split("/")[:-2]) + '/resources/plaintext/plaintext-data-' + + PerfTestUtils.DEFAULT_FILE_SIZE + '.dat') +@click.option('--kms_key_id', + default='arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f') +@click.option('--n_iters', + default=PerfTestUtils.DEFAULT_N_ITERS) +@click.option('--output_file', + default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/kms_key_provider_encrypt') +def encrypt( + plaintext_data_filename: str, + kms_key_id: str, + n_iters: int, + output_file: str +): + """Performance test for the encrypt_using_key_provider function.""" + plaintext_data = PerfTestUtils.read_file(plaintext_data_filename) + + key_provider = create_key_provider(kms_key_id) + time_list = [] + + for _ in tqdm(range(n_iters)): + curr_time = time.time() + + encrypt_using_key_provider(plaintext_data, key_provider) + + # calculate elapsed time in milliseconds + elapsed_time = (time.time() - curr_time) * 1000 + time_list.append(elapsed_time) + + PerfTestUtils.write_time_list_to_csv(time_list, output_file) + + +@click.group() +def decrypt_kms_key_provider(): + """Click group helper function""" + + +@decrypt_kms_key_provider.command() +@click.option('--ciphertext_data_filename', + default='/'.join(MODULE_ABS_PATH.split("/")[:-2]) + '/resources/ciphertext/kms/ciphertext-data-' + + PerfTestUtils.DEFAULT_FILE_SIZE + '.ct') +@click.option('--kms_key_id', + default='arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f') +@click.option('--n_iters', + default=PerfTestUtils.DEFAULT_N_ITERS) +@click.option('--output_file', + default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/kms_key_provider_decrypt') +def decrypt( + ciphertext_data_filename: str, + kms_key_id: str, + n_iters: int, + output_file: str +): + """Performance test for the decrypt_using_key_provider function.""" + ciphertext_data = PerfTestUtils.read_file(ciphertext_data_filename) + + key_provider = create_key_provider(kms_key_id) + time_list = [] + + for _ in tqdm(range(n_iters)): + curr_time = time.time() + + decrypt_using_key_provider(ciphertext_data, key_provider) + + # calculate elapsed time in milliseconds + elapsed_time = (time.time() - curr_time) * 1000 + time_list.append(elapsed_time) + + PerfTestUtils.write_time_list_to_csv(time_list, output_file) + + +kms_key_provider_test = click.CommandCollection(sources=[create_kms_key_provider, + encrypt_kms_key_provider, + decrypt_kms_key_provider]) + + +@pytest.fixture +def runner(): + """Click runner""" + return click.testing.CliRunner() + + +def test_create(runner): + """Test the create_key_provider function""" + result = runner.invoke(create_kms_key_provider.commands['create'], + ['--n_iters', PerfTestUtils.DEFAULT_TESTING_N_ITERS]) + assert result.exit_code == 0 + + +def test_encrypt(runner): + """Test the encrypt_using_key_provider function""" + result = runner.invoke(encrypt_kms_key_provider.commands['encrypt'], + ['--n_iters', PerfTestUtils.DEFAULT_TESTING_N_ITERS]) + assert result.exit_code == 0 + + +def test_decrypt(runner): + """Test the decrypt_using_key_provider function""" + result = runner.invoke(decrypt_kms_key_provider.commands['decrypt'], + ['--n_iters', PerfTestUtils.DEFAULT_TESTING_N_ITERS]) + assert result.exit_code == 0 + + +if __name__ == "__main__": + kms_key_provider_test() diff --git a/performance_tests/test/master_key_providers/test_raw_aes_master_key_provider.py b/performance_tests/test/master_key_providers/test_raw_aes_master_key_provider.py new file mode 100644 index 000000000..375eca2ef --- /dev/null +++ b/performance_tests/test/master_key_providers/test_raw_aes_master_key_provider.py @@ -0,0 +1,156 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""This is a performance test for creating the Raw AES Master key-provider.""" + +import os +import time + +import click +import click.testing +import pytest +from tqdm import tqdm + +from aws_encryption_sdk_performance_tests.master_key_providers.raw_aes_master_key_provider import ( + create_key_provider, + decrypt_using_key_provider, + encrypt_using_key_provider, +) +from aws_encryption_sdk_performance_tests.utils.util import PerfTestUtils + +MODULE_ABS_PATH = os.path.abspath(__file__) + + +@click.group() +def create_raw_aes_key_provider(): + """Click group helper function""" + + +@create_raw_aes_key_provider.command() +@click.option('--n_iters', + default=PerfTestUtils.DEFAULT_N_ITERS) +@click.option('--output_file', + default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/raw_aes_key_provider_create') +def create( + n_iters: int, + output_file: str +): + """Performance test for the create_key_provider function.""" + time_list = [] + for _ in tqdm(range(n_iters)): + curr_time = time.time() + + create_key_provider() + + # calculate elapsed time in milliseconds + elapsed_time = (time.time() - curr_time) * 1000 + time_list.append(elapsed_time) + + PerfTestUtils.write_time_list_to_csv(time_list, output_file) + + +@click.group() +def encrypt_raw_aes_key_provider(): + """Click group helper function""" + + +@encrypt_raw_aes_key_provider.command() +@click.option('--plaintext_data_filename', + default='/'.join(MODULE_ABS_PATH.split("/")[:-2]) + '/resources/plaintext/plaintext-data-' + + PerfTestUtils.DEFAULT_FILE_SIZE + '.dat') +@click.option('--n_iters', + default=PerfTestUtils.DEFAULT_N_ITERS) +@click.option('--output_file', + default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/raw_aes_key_provider_encrypt') +def encrypt( + plaintext_data_filename: str, + n_iters: int, + output_file: str +): + """Performance test for the encrypt_using_key_provider function.""" + plaintext_data = PerfTestUtils.read_file(plaintext_data_filename) + + key_provider = create_key_provider() + time_list = [] + + for _ in tqdm(range(n_iters)): + curr_time = time.time() + + encrypt_using_key_provider(plaintext_data, key_provider) + + # calculate elapsed time in milliseconds + elapsed_time = (time.time() - curr_time) * 1000 + time_list.append(elapsed_time) + + PerfTestUtils.write_time_list_to_csv(time_list, output_file) + + +@click.group() +def decrypt_raw_aes_key_provider(): + """Click group helper function""" + + +@decrypt_raw_aes_key_provider.command() +@click.option('--ciphertext_data_filename', + default='/'.join(MODULE_ABS_PATH.split("/")[:-2]) + '/resources/ciphertext/raw_aes/ciphertext-data-' + + PerfTestUtils.DEFAULT_FILE_SIZE + '.ct') +@click.option('--n_iters', + default=PerfTestUtils.DEFAULT_N_ITERS) +@click.option('--output_file', + default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/raw_aes_key_provider_decrypt') +def decrypt( + ciphertext_data_filename: str, + n_iters: int, + output_file: str +): + """Performance test for the decrypt_using_key_provider function.""" + ciphertext_data = PerfTestUtils.read_file(ciphertext_data_filename) + + key_provider = create_key_provider() + time_list = [] + + for _ in tqdm(range(n_iters)): + curr_time = time.time() + + decrypt_using_key_provider(ciphertext_data, key_provider) + + # calculate elapsed time in milliseconds + elapsed_time = (time.time() - curr_time) * 1000 + time_list.append(elapsed_time) + + PerfTestUtils.write_time_list_to_csv(time_list, output_file) + + +raw_aes_key_provider_test = click.CommandCollection(sources=[create_raw_aes_key_provider, + encrypt_raw_aes_key_provider, + decrypt_raw_aes_key_provider]) + + +@pytest.fixture +def runner(): + """Click runner""" + return click.testing.CliRunner() + + +def test_create(runner): + """Test the create_key_provider function""" + result = runner.invoke(create_raw_aes_key_provider.commands['create'], + ['--n_iters', PerfTestUtils.DEFAULT_TESTING_N_ITERS]) + assert result.exit_code == 0 + + +def test_encrypt(runner): + """Test the encrypt_using_key_provider function""" + result = runner.invoke(encrypt_raw_aes_key_provider.commands['encrypt'], + ['--n_iters', PerfTestUtils.DEFAULT_TESTING_N_ITERS]) + assert result.exit_code == 0 + + +def test_decrypt(runner): + """Test the decrypt_using_key_provider function""" + result = runner.invoke(decrypt_raw_aes_key_provider.commands['decrypt'], + ['--n_iters', PerfTestUtils.DEFAULT_TESTING_N_ITERS]) + assert result.exit_code == 0 + + +if __name__ == "__main__": + raw_aes_key_provider_test() diff --git a/performance_tests/test/master_key_providers/test_raw_rsa_master_key_provider.py b/performance_tests/test/master_key_providers/test_raw_rsa_master_key_provider.py new file mode 100644 index 000000000..5d6db861a --- /dev/null +++ b/performance_tests/test/master_key_providers/test_raw_rsa_master_key_provider.py @@ -0,0 +1,156 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""This is a performance test for creating the Raw RSA Master key-provider.""" + +import os +import time + +import click +import click.testing +import pytest +from tqdm import tqdm + +from aws_encryption_sdk_performance_tests.master_key_providers.raw_rsa_master_key_provider import ( + create_key_provider, + decrypt_using_key_provider, + encrypt_using_key_provider, +) +from aws_encryption_sdk_performance_tests.utils.util import PerfTestUtils + +MODULE_ABS_PATH = os.path.abspath(__file__) + + +@click.group() +def create_raw_rsa_key_provider(): + """Click group helper function""" + + +@create_raw_rsa_key_provider.command() +@click.option('--n_iters', + default=PerfTestUtils.DEFAULT_N_ITERS) +@click.option('--output_file', + default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/raw_rsa_key_provider_create') +def create( + n_iters: int, + output_file: str +): + """Performance test for the create_key_provider function.""" + time_list = [] + for _ in tqdm(range(n_iters)): + curr_time = time.time() + + create_key_provider() + + # calculate elapsed time in milliseconds + elapsed_time = (time.time() - curr_time) * 1000 + time_list.append(elapsed_time) + + PerfTestUtils.write_time_list_to_csv(time_list, output_file) + + +@click.group() +def encrypt_raw_rsa_key_provider(): + """Click group helper function""" + + +@encrypt_raw_rsa_key_provider.command() +@click.option('--plaintext_data_filename', + default='/'.join(MODULE_ABS_PATH.split("/")[:-2]) + '/resources/plaintext/plaintext-data-' + + PerfTestUtils.DEFAULT_FILE_SIZE + '.dat') +@click.option('--n_iters', + default=PerfTestUtils.DEFAULT_N_ITERS) +@click.option('--output_file', + default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/raw_rsa_key_provider_encrypt') +def encrypt( + plaintext_data_filename: str, + n_iters: int, + output_file: str +): + """Performance test for the encrypt_using_key_provider function.""" + plaintext_data = PerfTestUtils.read_file(plaintext_data_filename) + + key_provider = create_key_provider() + time_list = [] + + for _ in tqdm(range(n_iters)): + curr_time = time.time() + + encrypt_using_key_provider(plaintext_data, key_provider) + + # calculate elapsed time in milliseconds + elapsed_time = (time.time() - curr_time) * 1000 + time_list.append(elapsed_time) + + PerfTestUtils.write_time_list_to_csv(time_list, output_file) + + +@click.group() +def decrypt_raw_rsa_key_provider(): + """Click group helper function""" + + +@decrypt_raw_rsa_key_provider.command() +@click.option('--ciphertext_data_filename', + default='/'.join(MODULE_ABS_PATH.split("/")[:-2]) + '/resources/ciphertext/raw_rsa/ciphertext-data-' + + PerfTestUtils.DEFAULT_FILE_SIZE + '.ct') +@click.option('--n_iters', + default=PerfTestUtils.DEFAULT_N_ITERS) +@click.option('--output_file', + default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/raw_rsa_key_provider_decrypt') +def decrypt( + ciphertext_data_filename: str, + n_iters: int, + output_file: str +): + """Performance test for the decrypt_using_key_provider function.""" + ciphertext_data = PerfTestUtils.read_file(ciphertext_data_filename) + + key_provider = create_key_provider() + time_list = [] + + for _ in tqdm(range(n_iters)): + curr_time = time.time() + + decrypt_using_key_provider(ciphertext_data, key_provider) + + # calculate elapsed time in milliseconds + elapsed_time = (time.time() - curr_time) * 1000 + time_list.append(elapsed_time) + + PerfTestUtils.write_time_list_to_csv(time_list, output_file) + + +raw_rsa_key_provider_test = click.CommandCollection(sources=[create_raw_rsa_key_provider, + encrypt_raw_rsa_key_provider, + decrypt_raw_rsa_key_provider]) + + +@pytest.fixture +def runner(): + """Click runner""" + return click.testing.CliRunner() + + +def test_create(runner): + """Test the create_key_provider function""" + result = runner.invoke(create_raw_rsa_key_provider.commands['create'], + ['--n_iters', PerfTestUtils.DEFAULT_TESTING_N_ITERS]) + assert result.exit_code == 0 + + +def test_encrypt(runner): + """Test the encrypt_using_key_provider function""" + result = runner.invoke(encrypt_raw_rsa_key_provider.commands['encrypt'], + ['--n_iters', PerfTestUtils.DEFAULT_TESTING_N_ITERS]) + assert result.exit_code == 0 + + +def test_decrypt(runner): + """Test the decrypt_using_key_provider function""" + result = runner.invoke(decrypt_raw_rsa_key_provider.commands['decrypt'], + ['--n_iters', PerfTestUtils.DEFAULT_TESTING_N_ITERS]) + assert result.exit_code == 0 + + +if __name__ == "__main__": + raw_rsa_key_provider_test() diff --git a/performance_tests/test/resources/__init__.py b/performance_tests/test/resources/__init__.py new file mode 100644 index 000000000..120179eda --- /dev/null +++ b/performance_tests/test/resources/__init__.py @@ -0,0 +1,3 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Stub module indicator to make linter configuration simpler.""" diff --git a/performance_tests/test/resources/ciphertext/kms/ciphertext-data-empty.ct b/performance_tests/test/resources/ciphertext/kms/ciphertext-data-empty.ct new file mode 100644 index 0000000000000000000000000000000000000000..18f1696874059d7bc64600dfae68f89a2a262f9e GIT binary patch literal 587 zcmZQ#typC1*|SBNH>m&Yq8iz2~2yypTmG|C1r-_NVh0;u#njL=(%4b(4!K z3rg~J3rdr6GLv<)Q!5!<94)ky%KZWg(o-#5z0520vs?^=!i=IS(j85c%~JF7GQxvB zEz14;OFb%#!m*9;5@mrPl^_`QtcOP}rqd-u%c-y*`m zz))vU!^WxA=F#?@myw-qLB2sQ6C;B`211O{K#q+wq0NIam6?T6&p?ocr}KZ(hSe`K z7P;)|=wf0NXs{NjFR|-v581ub%aHqAxQDC6)a4CM)6QNze!z1>v%vb9>jbTwPESkj zyFAzGjMzkroSHpLbx&<*{9>%%!ouHU>2x zTIZp7RIqJRuj$>%SL+|lc^kQY%BN#Tbp8VY(CLgo3)1 literal 0 HcmV?d00001 diff --git a/performance_tests/test/resources/ciphertext/kms/ciphertext-data-large.ct b/performance_tests/test/resources/ciphertext/kms/ciphertext-data-large.ct new file mode 100644 index 0000000000000000000000000000000000000000..076926bfb29ea579b6decde9b556366fc845ec1b GIT binary patch literal 8619 zcmV;cAynQ11$cscq)#6Mw!A0C+Zk8^WOE<(NYH7ajV|+M>=YB&GuHrL0096MVRv&a zV{&qK~IaOFjXGL>yHceAcHBCw@IC42NcXUciFKRG&OJZ_# zaX~~&LNQU&LNQU%!R1j+8c$)IJmt69+=t<;7=Qz;B^ji`ih|8X*CJz`mD18oqj<$a}Ry}@#*N%nd@ zpT5!D*-jlT8``i;z%+<8vp~Cj?G^Nk-*ZjcCh&xf=`ZgX>_t51mi5cEvZXph<@=Zv z@N5mr_NeM0;3{{vIom0Ylfdtzt~%(+6g1TLQc;pt=lhev16&=Ef^7pESZqX6_`dr2ok8KrTBc^B(WP@{Dr zo-!V|`sE6D4P>Y6{=&@La03l*2zwmRqQW5jtojo%IkAZZ!^Q@?IgxcC9_&A&a8SqY z*VrTqI1%c^>22v{xqj!3G}-UuvUfSgu*YgTO=kUMXGvE%g%jcrlQp0u(~1Z~Q9UYl zMNXQ>uLef^I!q1!9Q0n;(3c7)S6o}!}zuuwryzWwkRsTn5;XN;2a zop*ztCukC(G2VAT@%#ul6cn_JG9!Pr83SwSy1QEb@S%|)HLURuKI~-XAepqGR&m7O zj?XE5JUcs5j64CEc?c&HBqI=HN(k3ODv-6!*yKN}jr))x0{hEc2fYyp=~D( zNB+_}f;lV`G*_nu6_RcQ3J?MI!umn9_Bw1yA)AYF>iLCFXOxHNFnQ@)9 zPKd5@cH1!1gd?pc`Ai5K9mR2m!)fVsEtiGvJwkdx?@AWCErZ<Fd38VhgE-VCMfSR8Jj>u|G_*n&Gp zkDMQmTcjZR&QZna^94zicT-8{P)h{dNF`HVx21-Y&BS@r=G!|UMG^VOY|rP+t+8*+ zn;0_MtK(KdRgdK@wl4k=-7!j+haUqr0!VbxsGK=96?OMJrD^bQ3ow5ar_P?=ZSpIu z7ycLC^sLi5_8R->KWmSWs6F4`qQK+Nwm=TDt4Slv9xh?p(Rmlcpv8{bg87oFwf`>m z9L;(#aLqsxmZ>sW6wn+t&v)Ke1xUG!sWo#$w2Aus4+t+8^{}lGc%(6ci z#HCHo)9Zz$bM-$~@2il31vTTwGY6Z=XYqok{c*bWT^JN})xq5B{gj;V8tuk*fDm$V zvxqaB$5NYNWm^?D?cHS!)e@1mNA$1-!K)_uMYyMJ1sQ4l0f`YKa~lPm1cwSmUj1?NB??6V-)`&-C?)gR6)EtJ_dv%*s#+0J6ILZ1e}>VNn#`7Yct zMe)3*bDFk+#WO~J*78jdt|GWj76^(C*J`dE?HE%c+kmGg$irHnFq#) z`qI)UJ3!+x%fapGJdpYfi7S}Nl=)8RoA{)*m&E4c``JrN14GVazsL?lD(?wNe>7Iu z>RF%S*t_}>5!R~TSfU~Up}VlgYw{ZH&lZE_0PADqB{g8+vCl&&H)9z#@&wlkAu^j{J2$GB2wy{?3l!`z8r|n$)T!) z6VEE!TN+eDF>-)M-UK!0p|wey6eE2$70{r0}}i<#oA~ zI8tdRVdJ_PhfwqU6R(k`B&SK3u^&X5pL2M{#~MOz9`pR1!_AS~p11LnF?uUkqL76S z62&jxahNiuc)F+kDqkkR&d-cEH_Jj~GIqCZy; z561{#!sUO!w9F>$%2 zD1Ar~f7NpmU*7dPRO}@$HjGxS*x-VAC@cZ#ftl?kX3%5lfr@^EH<9(<+fbQ!_ax_X zFZw}IR?omZYg77Uvf&sxqYa0M)Te&%cd>_a`99tmI-BC_B1A?8w zpWb7)Ma$i-KESJU+FOllO;%Dw*RpqWR@rA9G-;UqAtyd#Z`FtyyNdMQoUI=FXI%dQ z+f)!Xuqd;=`yW(q%DPyn-A>dwTKvB2@xux)=&^&z!>CopsJo5LeoYSSFIf--;l7c~ z3+SK1pn#17L!j{!4jm5NN1v!>w!+Vc5%CAh6SBdwUoOJ)Xw?9WMYr%4nV0lOsa!lw zn}hWb$`4G^8*~6?5euCTSGK8VYl#bYx#5`=WZ2OHr$M2RseQhyj4DRTzxnV!D^Hpy z3SP1Z^@)`d?RVQB6fZTX_|yVcA1hf3-+~L~1F5HUU<88iRNuG8(&uf+S#c?1Dd$A# zL-Q7dIry6KhBFs(VyN1&?4AU#b_Xk+k#yJ2ue2c>3AR)VG=v}>F7UR=y|y9CB>*8952Ivb9fnKr zH>ufEr#%jF#GOoW$MuqGXe>{m>(b8-$O$_;-&-HC)$P!Md9gtGjl%&a6$yUhV>3SbjUYCQy;Aj#vUgG6b>ZNpF#T)2F#2Rgvp zwjuZBrz$Ze{eBy|Bd0kbge2ejr+^J<5uA3h(qpR;aDKx}dUg#8x5c?fK7%PB$Cr`? z+YdhO&=W^OxC$U$5g^@P8PyJO<7p|=m6-`!m~+m6&g}}7xFh+>s~Q|0J#4Tfd`mo< zP>m}->nDwD8C!l}=b$eXNyzZjG&s6- zAB7Fehs}sj94Xw>u9&iH3Hyic)XB5)hwIb%AA(t{O6&ThGZdPQ%olSO!4;OD5yUguERfTaFo1c||oXqMRA#mqQ_GuAnR4X`dl#D6mpy?LoHFuUY$4 zz0g2lCNdelSA6qYmv)@cjJ`Klr9jkT5N4=jHSr((fChSuN_qS2K9X`B8B=kJuv>j( z5&`sTh^&JbZ4rp%%u!_054K#X(G_l=H#_fRsN`#S8Je#W9VT(ExclZGe84D!aU`MU zz}=xS2jNY|Uh==EZn5DVQiz#Bo37!c6{+`yQHUKlRSQ((28mAFw@*y%u%XuD-@x_p zW-{-e*S7e-`<}R?AZ(!C>!+$l7kn?#7xWzJ_Tni?DQF=Hn>gG(X3~Jp(SfyU08Q%3 zagqINn+fW08$Cw*dk`N-ZLG0ScYITBqN5HQ zQenq?N=%nWQz?X<3Mm1ubzop)=NfjYh0y{>5)mF*+PrlYfD}FBd^L(8Zc8zZ4)xo; zgOb;D%T)*|xmxKG?Tc;2w{(jgVPn6m4V`w0sAqRJ_6EMH@km6sB$$MQi)vcb1KT(? z5_mB%38*;lu=W|uQz95lGOx{;EeEYllMYu%yEFVJn6`>|u?Kj0+RzYwg!;YTW|=_o z#BOwOZ2{E|QdB=D1z@VNV)peu%{27(k~u6+;7C}VLR-Jkv)0_l%p#bdD4Z5aMQ&NA zXjfPp*endlWo=*sWQ%G>gxSSBb3h$Ry;NzQ%zyN4v@!vm=43etIv`S%j$llrzmy}; z5@e8UWOK7<*2r%RRz!sJ7#aEv{-8%(P>|hI_GvWV%+b@)=f7KxC-(iiSKpNW(?E~W z7OOR)Q=grEHAz#(-hm{LJ$iR-AQi_WZjjCxIIS@oLJQ#cR1y(>c6y5$?r@=OUIQin zX#fBJ{{R300ssI200000000000ssIHKx$Az$_w$G_y7jPv>+wlxK!1=>~x^7g}IV8 z$!rdmg#QB(1iP{dhlk<`vbSDirS*>;Z5^B8*El`0$K$d;eAN~o?)b}g*XdW=EG;Vu!{k71Xk^__ZzWt|SGBtu7Aa!DF{ZOl>Ho$&eEZi` zJL&qZjz+2ow>exb!_2*@weKB~V{n+&I~UXgmM33tK_ll{w3_|HA+;6|J0aX-(QKGn z-*Z+JB_28RHcL%s}%{MFgOty*5>c^lP zkq;$Ihn-!^5dH4Ssa<~Fdf#k4l2GH1 zfVcqtlH`>ikFS^{h~jkry`N$F=@3{jJO9xOT_r4CCA}Joq_t4hf_=DqDUHZ=C|_aJ zmvq=qh*phM1MbI<1DGpa6j}X6g?3S*RSsj^2Owb~McYBZ4|M^K0&DH0;l$LPdHeb9 z^M|(RiDt2dID`~=C3Txpl~`#xrIYrmM+UW|MHN>WaQK#X|JHE@SPi%*uc3J06y7Bd z=HtxJbsL>K5?-qi4H3n*|EJpx_fYX<2>mqS-RUE8in>3>?d?;B*M0k{-x5rdxhXyF z2}k%cfmyd)F85FYzze!FDPtD9Qr(U#Y-yqL_RXSf?4#RGS9~>cHa+x{dk~FqNk#5z zl}y^StXys2no=id0Z`%VFWbo{ju~|dP%|n^4aI56{lqDkGCTDs-wzwMdfL}`m&cmM zDT|Xks-RA~{%ySYIuJ?Q?-742>L}r6WfCOs0Emn*63A066@AOAtKMvU#GFatRk=}v z&$vviA99%k-hlKGCU|-5T7`+@jFdIXOw&GsvLMQntbQR&hO{ftl$9DADv4`1BDK5sQpJU*w*5B3wUG6o@O(E6^U^Irl>5NoFv4e%NvHwLnfcj@-Gl#*US4( zW1fOdmSQ~lr02;ZB(=$($T_@J`MkB@53GIN*aNeLd?wwrsuu=-SbLQ&&i`z98m`hY zp#XLe=Y#(3RQ_t#>J!`LEQizS4qVq+&Vi%|&IW#_UT*B+rgR1CFVNro-f9D2fLL#^ z4GdsdC0btH*`5Ynrn05Mw@9~A>|lQaG76umWtk=6{oTL>T{elSs;by2*j(l?7i~a$ zdT)XLSwkWAB<=>j?AYyv?3n_|usGOmpq6P3`Ys{|b@wzYJNkr5+RLH929Kw)CbKCsoDH<4n7v_C!_XyKuEu;p4Rw zG-KrS0Wya4ZY#gL>=@r?-3oj~w_p%02KD&K5KY%rT|q9$-8u=xqHhOMqz51UOPCUT zn6qE67m*>$H1xIw6=}Nf;*9t_vZ61yi`_MqQSSK}WbA__O_D^#mCuCH$e87># zW7r~U%n@qVee7h2u<3`f7ZA)#bqEhDLN2T>(`)eq!S8HfP#&Z94 z4**W{A)r^PoZpO&9)oF2W!{$|F2l-J90 zYwcEy8@OFsb&6tRAo@{Hi2}`RE{lc%DGC|>WoHsh*?Ef;{B+E!$U^`nT zWL9%IHHt(B8+F=*q*{z%+|aFw#YJzJFiBON_;TOJdpLBoj#N;VDnX-MU_UHk)@6^k z0Q*Mdg&={TBXxe)-`*L+e1U6VHu=mzk2%e?(X}3ubHs(4<)6^=Yu|Ffz9QdRr#ls(QywjT(fmBVKmos`6VvX zeI(X|sXry05uN39?hUQFYod3uN9WA!x@(x7dlzs_&-z_|MVx7n14j#l1K_eWm&GVD zX(8#*Ey}Q7b7UWfiePQm>5~+L-v}l~Nb99+S>1MY*Sso@)&gM#&DqsObl+W-MTA*k z?Q^+*w)VSS74j}>C%2x?yR7SUO7;(y>q^0uZK~furReu|ybQV-i((lUN`~s(tYivm zlKs35=#fPXm2DI$z`q{O3fBV`C|ROX72hp7jVIm$Z+gdH{9DbnpV!2?hQb*w?E9|_ z=Fu)3BtVEC3v4?*zpnN8Qu18tAlq$`8UFKj+a&wPK%sC^h0x}rND_%PAG!+_-8B$D z8XvUp&sdM?v+pe{rbaN%2=m>hyJ@caYCrvS7Xr9C;FXLkiKw)hx8ErH1?yHYVRrfY z(T1BRO48z`V*s4ttrirEbvQ%H(`Zac-PM!9w54Q2cfPraXW-`bNThvn6%zCQR3bOF zBPbpn*Hsh%&4O0ncs1uQdz9}*Mk*0ZF@JBiYpCFTqECPNQ8>5F-2Sg}-9A55>2sVFH0>pOC;|m?e`?MR z>#7bXo491pgwf+X^z*_$wde`J)0~Sg5<}HPK}d=}BeS5fYWTDF(7pPNJbU^fn+L3F zj***_qSeC-C(ZDb%xxylC278$|GghL>!KEqgucL^&ex8)jCu#uZ6n=T?+A| zP`%PW1d_I&NwQdVJk(4p)Z}J^PD5LiU{FcMp<)K2z~@HbKCV-|3okTK4cB0x5HPY} zZoOvGjd((3yl`5{a-In(!e(mj5XiW?8DkO0RCN!sCT$qbD7p4K%n4mPVxU7Rf%)T` ze{tU*gbu|9#@gX_?|0Sss)O&7wZs;S0jnYXXUBTg8NW=rW3ut=4fvdDg^Omuf2gYp zjR9JMQk8+%PH>69+hmz5T21lTS(VsqfeS~T8{m}nifYe?KzTV)@!j`ggiy@xelvn@oB`9I$5PlL_sj8ahfR4}Q6jM!W{SoMjSL#G3N}%42wfvK)&en}D_K)y)ji$Xz zclc4kM+K`dkG;Ne%?&xL@Qc@;jXm@z+=XjvXj_>Pujq*R1Zh|lGAEWg}3tYTqr z(n9ngh)SUQ#YOW>iC>$IHwzmA3!P^0%6I?-`WU_~%1a+G-(8TTCNE z&Lb!aRzlX5|1ouz4ece?HR#>X`6p>_GZS$img3?Dq~S_M1?zY$Q3sQesn|B{t^|sg z@N6I!N$!DNsY+rg2c->$Od2sX3d)@b&$cxaC#v|UGez07?ZpK=&JSSfv0HmkR5x&G z4@0xEX?yZd;QK>@{+A+s7jRDQ|G092i4u`lBQw`(Z+blL5fZ2$(5U4eS zG?Sfv;_v)-}{!fi!ELmQL zM}6FhAQ#ztLZLY{T&x}rZm~u|xbl)-q?w}A)BB}$e%!2F?iH%Mx*8?RiM*&jt>KlH z40KVHO$DFinB5JWCTc5p!^30)dh|NiOoNJW7RgM_Kg*PJ zui8y0*RYmlx$h}`VrnU01Q_$(%(+}2-hY!$O>C2;rDYcWsa*R4DHER_IY-+D*$n|2 z0?v5|g8EGfhxtRXp>C1)>GJx#+pG=DSv8V%iyg!wlhp*9F1kVi{`-21nY{!@eYisF zDOnBU{?oKdq$3>V>F@QcQ(Es1PzpPq5Qe1nP~-CtqF|}IE9g#ktB9q(Yr^{ZCr4yu zG)K9`TWfjN?d9{t!L7cqmWtVzWT-_R8Dx=hDs0u_vqXLYG~R+OUx5pMGH3CRAm!Ia zx}kF#*e!2149v(XzQ7@~Fjthr*~w8L>1HOlPkY~eS~Ddo4vNQk5A1OMg)RU9 literal 0 HcmV?d00001 diff --git a/performance_tests/test/resources/ciphertext/kms/ciphertext-data-medium.ct b/performance_tests/test/resources/ciphertext/kms/ciphertext-data-medium.ct new file mode 100644 index 0000000000000000000000000000000000000000..a954c7134cf35f43723c446ecd912460ba8213ec GIT binary patch literal 4587 zcmVx;cCA?n7YZsT~ff|j%*k!ke>fw2X4((J^Srq_Z0096MVRv&a zV{&qK~ZfJ8kSTA`+W>ZTwLQY{$H8N#KYg$rgaCTZ(ICD!m zGfYNmPDXiZWkYpxRAWwaV@hLbWJ!22Y&c&LNQU&LNQU3q z6f1qdT(+bL4TTwB!zA;(p+SygPQ3Q#q$L`tn`Wo)RsaA0{{R300RR910000000000 z0RR9Gpbd~A-}^wWBbwUHPQGuzO;_MBam39)y3aM zuUJ$vgtJh>l5w$zszpUNYrWCuMaFrS&JF=w)F-xhe<>-x38_!{>}GK+5h^nKRiw95 zm#kZ>5+Z0de6*LRxWSxsE5Ba|=BPM=dpUi6VzTkauK#~r({ z6O6Lt#qu7f$Y%gwyHb3note{-WbLD21~nJ|1lgMod>IH6u_mC{&KP1BdZ^U693Bj_ zMH%aKRO%Bn-|AdxSa&!gu05y^c48aP8ZO-P;@pfHLZ{uFABW^J6MaGaq=NSbYYGSP zqE0vOHcNM_!w3WZ?T{6qP~A1f&ooa~!V&WsOV;0kK9t@sQkfY%O2sg5=U?v%0j2&^ zfZB@pQIFs$Vd#|@3eoV+9m#D znwB`NIg-7ud9}o&Ih@n4r?6_EspX_J3NFI{kA9{^m#{#k!)`mJcPK;iTPQzk@3V$7 z)^qKI&r{Z`u~3Y;afdO9O7aTp?98E5f_iLJ1dp!fP1QA)e-sUzdyXlt^*?aSTp)@r zfw1^n1S>W(zGLyipb#VN$op!<#9u%~f0_^`bltAAk-`St=_V^IsH8Y4^JC?6@cg?~ zmviT7vsF3j_-!F~mblR_TcDEEJcB;#j}JxT0UmnB15gYmM}e~F1lE6Bnz`78WGqbP z#|Tbgk{up1%Ta}zbUlEPm)Ga!4d^t-N8jaR^O4o+XZo- z*mgNSO3aoYax`tZ8%6yLQa0vdOy6WN!ZMVVK%*MtW%jeC`0}v&$iSO*b1vC`m4%IQ z%tMEj{Wrjw0TLgR9rMNe@1l=OFolg#A&xNr<@;Fj-mKt67R1Z(0 z9)8LYRYj5fR82J#XKlDLcWh%N--^-3-CgE^&aARE*3hr0zAy|u^B`d@d0qoj8kN_L z*tH_2xp};=ox)=38WOkq2ax$u0z-nf@nIl`N$iIDUm233cF9ww7=S)k(p;4na_AC8 z=CwST(k*oPRyvdS^=Pt_Kjq7ZX(2e031xBv2UCOO3?SDUe|h34m2)!aW4IdW5bZ76 z|C|ARdtAc7&Pt#JBnU50ehiTQSfK`hXCTFQ#9#?2pJVXQay##2@onp`$&#c*-^@3w zo#iRLXi+Cs`w*$PU`zhGXZb0rafCTk?aB#XiSb7C9(mjIxd{-tAt`2}%0Mh<=D6|0 zZ_A>4V1c#IN?Wx!4{&)$L-WS4_Gc>x{dqF<*={NSh z%-b2yLU(+AUkLSQ2izU0@J9vyAYvsRVbEFg`=SNK&1@})WZZB#V*0HfUzFO2zI7li zdk%dQAt6JsMatV&bcUH5b(H90d8t+78jRNk%D>SuUQQJbh(xK-=J+?q+*F@`@ z|I3h3K(32K#Y8q02@9)C>{XB`=9*QpH9Oo6(shM`;%T2sV~vWV0J}72)2`^07=U`> zGo4G54utE+(qxvb?7%vnU$ag`1SOc|Ng(hn{0WHg{CqK%ge~YqB+ZVb=^1U4brfAL zgznt7ZZyuDk$Qs)PK=kx?AFR&o@V;$Suni)Gi9^+L~%8_Cs&n1-&vq zm=C0fFJ+2J{p8P`D>2IFKt+gh5NHz9jWN``YMUA4^kR|8P|IaVGcp=(EJI+78o(G3VKG%|28%3pK&@rj)ocT)*0c#;&SG_-AV~{zx z5N?2&P!3+IIL5M)Q(=uYv1-j$?Up=K;i4kl+|q;wBj0Z*%0Lo}eTF7O&-_((LhSmd zP%rf(4VE}}vAZMhkHcE)!touo9~SXpL&O8HYZ=vUvWpvsH0g`kG`#KMNWUmt9#@UU z9{=nlSx;pV1bOQi@n+S340*uDZ1{n`O}~Fmp{N%S`XpPJ%@2KR>U|tPrD^bLz0FH&vVDHxwAz{a=54Ok<$uz13$pM)*!Pq36)Eq!$- z1r&!cU~csYsK`7o_qi1BjmEbrk9^(|Qm#uJ|Rym50dM;DVTlQ#Kqp zi9L1>j#vS#O?P^K>R`GGzHL?|EQEpgYOS^hbbu5*5o*5cGI*szCPbn*zelmh_w?ao zR`TJy`gTAU2;HF@Y`$I|Pe6*H*kLj+t69)wX`Sr~;>S?GhPMrZl zNVgL+X-yK)W3!b)AQZ?|N`D*a#=}jBJr2vO#}zNGP+&4_DPnM4e({JRH-c6198*4c z_4j3OYixOj!S5TBCG09bI@L)jkO%S@F?=0EIJQ^piyPrD#BC>m)uL9qRQ3~7obwjq z^bu3otnt?Qei2mx%7yZ83yj(_Ff>KbEaoreKC{mqBeUq-|5md7=p$}7U9hGS&yP4j zlv93dLsY9Gk!$#>@_Ez-JSn2OlF8yvxy%)TR+1gk)8Ksgwgk0hlMSLg8?_eiX&Adx z(VKcF{Ayb9G13{cMZ+3Ha_^!$s#CEz{-#5?-ThFo%NUt@?xTQ~mlg#%xY3Mu+Yd-=iIph*9W${~GNh{s5J_;Aw zp34zHmRhf#n$G1aQ4?B4_4W0wKFGSOajKo@&jW7eXrUGayXXUEtaR68k>HSg2#MN* z5+JRv19;@26P9*3Vh|y7l8&)#*aZmhI=~axE{_seRxfGdiT2dQ22J*3+v4DO)ICVC z;+-DW{fcXO+%cC>hQPYopUO*mx8ly+|`$tTEv1dDDm7e zr7PF8%l&Ut4vsj*7`((fM;O!TE1B=@l1(7h*!WnOrRSDLn5D*bNlCNKv$nDxqC9y< zf*>cn*={?8iwQ_#gbdmh5#*BQbW?}B?xkP)ro*`}COITxdxkhiZ0eIeR?+goB9N4Y zfAmQC6X!Y(f-(^baVXEr@K2DQTP5uXGy9i<+g{=7i()qv73l+Me3{SkOLd8q9M|~j zpMT=gKIo$$DN~t(fL(OND`?W&Irv4FZ_>KBqM!qk_$>ByOG2~r{5X5jzGMIZ1#C;v^5o328;6U=xx(DOdW{(t>?TkfykO{eGf;D%u|ZiwD;lAcioJ!O+n zmMD&o_`64;TKu#Z&fqwX%XbO+`wF8m>-#`+j2HT0zqx zr^t#^4d1^}+*l3iK$i5PI|_27+Q*u?Hbdq&W4P37eY|w$Q3b(KiAQYQV~Bj|53FYj zrYvgLqm^g!>as9TN@^v@&mqE_i$;Z@_iP}9OHIf|*uZbEt(OHXACv154kzbcnw*9w zhKBZ79ST{H&oPLV=W<&v=8sl4`$2Jb?>w;-%-cpuR&+GObm}y;#3_piJmxfR?uHTN z|AE4s1FZaqWyV%=k>rqM7nyku=$;(`t}i$6aeip%egIg4jou2}&OJG7k1{BPF^QzK z#${96B=Mw~$)-nS+e7cl%mxWRAd7G6pLBYW;0_>0rvf$j%U0w52*9Ukx8^L#kZj)f zwS*lihL5 zq!ZRz5ZTYw`jx*1UAKHg8eIM2)svOGljG*UyvnB|dHp!`b6W_x)U`r=o~zeR<)3AO zwzEPPePzTj9K>Dy`BZRSQ{jk(#_}eAA#lSE8qC0v_VSs5WJx!9C$m}Fe$P(E4r3{I zp_pRiI(8GHTCK47&_$UyeGdQ1IGA&Y#)qIgfpqsk0!wni+bg-Tashtbf;A#@P{A(t z7EUFl6rAgj_F?!e@;R?}%O#31${?R^>=b}fl-#TT2`;N+%P{W-p%M-8+&ADe=#9^y z8PKN_2S0ebfJPBTkX6t-MV?Ie5cmJQ`4SK!dNgExA(psc;ibgVcwDG`NYeG>D0r?Z zRI%|Yih`LjI}07Ow~a|{fjoaX8_6_zJA4;!JstNDf#^>3+7TgM8rGQicLD;FdE;9i0QhnKn5IWwDiVS~doY26 z0#3KTHJnC;O=&<=T{6>eo!cr|F%xR7o=WOfe?saxm{Un4Pb!a;kkY$D{n`eetPFcf z=7^DUf*!G&hI=&k;?NX5N{2Lvel4MsaOz#;L$H?u2c3DDv52sZx|a7M|4s>p-lEf{ zEKP2O=Iqfcs=wRqI5y`?1D+bK6>2t!#L=c^j!O*3{V6W=WnbpWxkSXG9_{dCTZyYg ze-4_Rahm5fc1KijGj(GL_(pQO$_Tq5Q%%ZfbUkx;j&b|Ei`Qw!G_{!Qo4_OR#hY;jDDM%}o= z;%xO|c`8hft0OpI$*9;5@mrPl^_`QtcOP}rqd-u%c-y*`m zz))vU!^WxA=F#?@myw-qLB2sQ6C;B`211O{K#q+wq0NIam6?T6&p?oc$0jS}{H>pU zSskxtE@Wa9Xs`~dNt|hMa>l$=^V8YqrMN?4H92`2CdL0b^k%25tjSTcid$?vuP)R_ zgdOoZssFz?#A{2eM&$Dold`TZV`5+sU|@;0S;l!+T%W7acX9Qvx0I^jTMF$zi-~MRu-u7in=ds%G zhd&~w>c#f{bXA}AbYkzz7pENJ9CN8>bu7cGBp^|4N{p5Yz>!fed}?= z-^Vh@&O#_gAjvV`d%9iIF6R%8OT*F`3xejJG(7(3u*~{b8=R)w>%L$zWH^yiRb{R4 we9;}@ZG7#r6P~g^U}Pv#&+rQUwB+T{+ z)ipT4GTp;0E2^l(rOY*|D8n(zG{7*(+}$j)xYW_s7G!*Ier~ElZem_ydTNS7QDV6Q z(DY&kecwt2BU3Zoq|6cpN7rD5@}k6og3P>hkSqfOLjw@=gty-7zcQZF~shgG~>LdHBC5c}slAnzeSyucXe45jW2m33>M}komKHK{yiw zg8;+w^e5+iISi!N@4huXRAlm<*+)JqE?jdjq?aL6zT_)s!=!a<9D<`y*`0jT6uPuG zI_f_V0Ig;OVi*9k_iQznYX7d$sy;c$nTt4wEJ$#u_e!RjN@Kq z%*oOEzgPQcLCEJkuPA=%yn1$cP}?p`}qHQcP!Nej@PY5YvmoGUNR9BIn39VFr`t119r0096MVRv&a zV{&qK~XfRrAZ(2t~Yb#@FM?zM3V?}LPIazOLGiz^hdRbyu zXEt+aQgByGPBdvbbYWI%LuOV*Xh(H0Zf#_3bTdjeQ9V5X0RRATl*JEn;bOAVEb_Aa`n+afB*mh3@`7Tgqm}s z`Z1N@;{Y&V3i2xVq&i*c!^#;)6+v;WRIF_qz1%hP*ZsDSrh$v}X+);|iZ zJ;;k{HeLN@Y~pdDwtR1!Abq<=GYLc2v)iWbK*%5RfYu^sVQEI@8DCf>$!0u~Ka2ANm$M|YLh#2lL5WU8W%h5h30%v6h@&eSf6K7(>@%A$5#x`m5-BjH zdL?7UU6N1khdvN_OHqZ~)T5)=U)JfC%xJC#e9<}q93PLyy?-fN@Uai zAl>Gz{sSpVpXA5P3VYuCP5(UV!S#kseEq9H3>?)o-y$zu4n_93>t~4>vFY+8w?_w{ zT7kkEyIV_^XZAKLoy&6z5iX&Ct)rf{{MvTfrKN3cv>^7R5*pNaJmgg+Pt{qz5iYeB zof-q>rdKkJs_<6mEMe@=mu1JSX1H5|)sm>b4TTt1UK*rM7#Kuc|7&VkVM?=L z45zzw`^fEe%7-)NC5jUOeO5)Qohp-o8x1rG^k7+?YX*lRyX+S)5Y zpkx|!#AHr(Yym79&xt$CB$28_Gm|Fr2&l8g21g80(#d#ZK(55QlttHjFYnp1%?Qq( z3eS;KAVCai#d(is+u~QvTII5E4Q$KtjIYfKmWZT>)ke6 z7I@hv*R=NU?rBn_qT)aIVul`rMY( zzG1YAsNF7z;|-sHGA{O;N19s0T+l?Z^W8sNGPB@}GZKVT0e>=Hkc33ae_SSNdf3FV!fEb`nffD!5l0d;P;w3u;4)yKIZ9Q~JgR z1ioA1?YzTNCnU24AOQCF6gOhd+3KQ=>)wXvqb^anHiYE~+Y?XTRK?yZ2@%1UuH1W8 zrp_^!m`B*j>Sudupr(<`c+gM9Cu*=Q5y83I3G%2?aMEGS(~zyWAwNElgWB*JyWjg# zS116#kV-;;H};8l`_4ro3SW`wBuRDTs_sM~^#OP(p%p%FGcOZ@_HHJ*y&KW>52!3r zy;NTTFDbX^e?mg-zMdD>9b-qJM$cc4q&&IGew$WX|F>3YrWXRmV7wKJ+=kAGX zvghL%W3&r%zL_pL)Srie3+=H!MRk#{`0LV~APG|mn~NW?xq@&YPU>Zujh>n2MXJRPFnAQuxXu7{PpF1%ZgBNY z<7Je=0l=~%3iD%ucvQSH+Dw&`Va+;Hv!hK;Q2m=iv;Z>7ou*^}+l8A%T`;yGH*^aM z_!gp-`v-`Fi>o`YB;^|KY}w{b;={sm2Dz=nJfWolRlPVwN~E(*ev^u;xBHv3bL?`M zP#J5b6+__qVu7^ejoIO7?InYveXUkIt9_YjvVot-Ua|yX0o%?K@wfZxV-7qL=b02= zpqyj0-bXAl#^!Sb`DapmAqe3cwU9iro(Jgl(v!Fs82r7!w~{uI1CC1C^?4Pzj7*8t zN12u`u&{L3&iohumaU2z{KmiGv3{9EE@>WBmL+FprNqI74@#g$HX12dat#$}u^lWJ z-FR85+kXgM#B1L*RzHp?ae6Qtjddp2Cb7qd)z`+_EQd!vcC&`J(MEIo38uS|1wmSD zI^~nrQfmTYBkTAQO%ao0H~a!FtidOPw?D zH0lKESiJSKn?uu)S`G{PnMh>O7dUPUlO4FISJl%3qzKiR{b0ZV%#1{zPi zW@u1*#Ui7*$0?9Au;ur*aH8}h6@gqxF3g=^F=54;BEy@Q>2`L9>02!cx#{%PiH)z+ zVL*R2_x?naZNv=KJveBRP2XcJ1~FJdL;QZA?Io%Zb8Jb#l}%N@II~B;7KdQHwKj=d zrcs;_L!5wxg1?qQ<8x_)!D41pH{nxwLL|MOA%6Z+4f32%#-z^C&bF!Lm!uDF1~s%kS|n3m12 zZM;`u%?&d=9{yEI9XHz0R2uHJIJq~#LLiHK?trl397^d;TV`fdUgOVxZ6f4NHO-zN z5Tg{*Djc#LO?1(Cu#d5TtEr%`vkC(y;tJ6zc4iPY%XF_tUPr3d=teQ0Bn$z5OPb!- zwId-yjG(~Ad;Y~Fp%7=sEl9>Mzp0seL!=}&O;7A`J8 z`!Dkq1#+{o6S{l1UmW}ODy-0b;=$eVq*oAW@=!(Vu}g6wlHHrXSiJ0?qX?G*`sDEe zAM6QqjBQC(z-EWa0At4@24FqoKmGtbU6F*iqQE=U!WJ}g8fSYlPF0X+)av0uGJ~-m zPV%{lmGC;ox!-(nvd75K9cuD|QC}lncFiiK@^!~TQn|YYDRRA zXy4^3A_Gnu$7AmQ4T*FQ9}jb>o9Fcc691)J)%iv)3j9+Ley^>wKukiD%2FZ6*7Jd* z<a(j{0e6%W6(LlPQ19}SQMsX`gvr6=6FXBlB$_UF)=s!-=^jB(r3j-1`Ut${ZF6c%{vB#QMeqJdIml02?H>zbR(yoYV{A|D2~yQb@lt)ba(URkK+)eXV^ZAmlFtv4|x@9 zt~{kc3~Gx|o3#TW0?@46c#Vod4yed>c_^d0#=5LqY0ss1D@iaM0a#zN{b_D`p~``g z{n>I~^5a~~BHf9Q=&QlXh7WhvLULWwFzykYTIqE4qsv&Iz0}}VVC8q_GG_0>QP=?F&9kce&Pe*R8>}1wr&Zjo5om@ zD20!AwDDXWfX;XqT(!3%;>G|^cR*LbbUmA`_3XDn^Zq>_=MtAJ}H(U&Bx$4wokwiq1cZ`BJd@;Y=n zXJZts>W{|+rIzWZzuesI*`P8u8dgW5m6;p@Bi=~%no2815|23#O@PD; ziwOR?H*JA|^*gR*!c1XKF9nP=?7C@9c>l;I{&qA{PAI<1sFj`CvA9B@s_gG^;tsd>dcLV31m)5l0`3hx-cQ*2^Zk(4i z?AIk6@dtQiLbM1}BWQ|=}Q_}L{6X+4{R>QoU{7t=Q zJuxfzU?5>YF8MnuyCcpM2e>`P_tmL|B~Fy=ms3@I&3cJ3q4QO(90`7|W?`h0w|z>e zYL+1ctZ=B=hP}N$YhynRW! zs+}(}tpM9^vqfFWjtOQJM8Z!);?=gV;np9v+8HM+rfKy;LzGN=M)I8h5&md;NZ^{z zkfW}Z)w$>>)^tv>{M_M?!3M1+~5BLF%5jXOQde%Eb^y8iYpl~Gq1j$Z?FIungX?7K<{4ELiC zxy(c%Hvy!7|KR0QQq@971YpUS z{A9g2wDr4=ql*T#jFOnFKK7MW7-ZSETLBbg674cZwA=bf1lbZD>Cy+vM=-M(PywBf zte(Gm$He?JoF4OH2=@(Y%3|oB3_HOKY6hfAzR}Lbn}9I8?_NM}%GNhpBUKUjF3d7C zwWF-*<4H;4qogG&)Qr8Vr#o3$XOE5_Hvl^pK;X`dAqwtkP)Geb*0c(NhxY`Q>UYkYF*4PM-bra=+~ze(6|oFvn2PxtE+%o zdU#c1q(2tCa(eTPWsJUsMhR6{6lt+U|NsC000002000000000000002000j_6wa)N zKVaMtE!0W+Dj{==cZ8|bs_0(Kcoc|!z|xMFpq?oKM*{|XW4^}<x#gxgbNdAee(HOp}0MjL;BllnPP&TceXIi`d{IDcy!U0R8*r%&Ox$%91}Z(Ua#@q zCA2(5?V)bkjiF9Q+G%m*HhPzp zd^9yIP+#$@&Mjpik9%{7t4vgx$TpPNp?8<$C~bglOj9N-3(l3_2g9t9ngThSD7OmC zE2J>xi4i=`&;duW{?H5DbYHm_SM<-*B|3lleBR;o{1u*!xaUpZE^+hOL@O*B6jRnu zJJ&GhU-0W_jXC#auUZsOAllD37jCV zEemui#x+qi7<(Ux``y1TW$EbrQ??gU47i7}7Z05O%ugH=_oeoYWA1B@co361)Vo3V zqr7G`UE^P8fX7{rSBTUQw)jV1MOXHo?mpDn%ANU$^K;O^S8HW49ib=CO0NE`m_SjB)=brtfrl8t$W_#R z;d`j`tL<8I^ywI4D`fC@$Xjs}{6cPGqWtA2Zob0Und{KAp8DQ8p`6=_ZCnRQo$JSw zG`sQvwXt)CdCpJ<4r_z?iu$&Cad2&Li`P_(8NRWt>Yfk?n-GW{redvq_h)rPrY0&b zNdk%;-;;!&U(8#bFv+*wEhru2cwVy86>7Gx;F!4UfWE&2*@lRoCvZ5R&R|zMc%y}! zZPw$FlfYm$UJ~+Ok9#W z`y6z;Zl>KgM`$C?;;w935E*rDvpoQLf+vVY3F5I!B^c|m!*~7;>(4=`PbvL7Z5&dZKb(I$RkoM;;+PIA)Ph!$lr3!XG4v)84{R-D`Wcu#Bu~V_57DdvW-mW*pf+4BxbCc5sE#1Mo0mSgxPmt;^kluvIuNLNJCJ2Bp2SyMovIt0OsOOHdBiNB2>lEq6h4PxojiW<1zh9u*X z0ez9TOmB7aRdJaw)1sau!U8eA$FG<<7u-;bJjkW75KdKa!xxUAEDWKf4e+M0^Pmp_ zN<4{JV*F7PTtT93jdeQeecm}@xRacM>*^2@`0hG#JamCQVV!=Kt(R8rVcW~ zG%o*tG-)d@l-V&kt;9Xm5$tE`SRS~*x<@xr47d9M}QlF7=kkqVa7bsu+xiIfS3u|wD!(4*#zt`Ti)tO$0EWX zb3YzzT}CH=R10jBM><7cpL6XIMnd~IlERt3ojqREH8~hje~fAgc&~Cd+Fx9I@jB({*HWVMHORX)aS5P=t(8?$`FFxLPrh6C(OE-u%62X!BoR&=HX2f4$9p zY(pXC>2?>clB8SVNBWMCNmQS#W}&ysg2q1bp~MnB3F=|i-zj9FSLDO96MNSi(ysrt z5;~o+LNYsC;~`|)PSDw*KRnGCmlG#ncGdnfCw@K67ouF#nOkE|Lp!w4bKZpc&{ULH9 zF%Yqx4^uachr>L~Oa6}_PP1goUQKNyQd3DSm(y*^frgo%PyOb5LrEp9jEt3w4jr?a z#NcZs5fjyc2yPaxgnL!k;9VhKFHUJ&mq@#}uXru*PQn)p>EvpS#c1(@XE-d0skA6U zs-Jjk(M6$g3(%zVEg(8O2U8#<@;T2B*$+X@g5T@!EO~oLfXS%pcbX3cff>oF#b3c9 z^~Wy^-&4BB&qWl}0c#zcWCJlVbWSgf=b?vo{Ox8r$=tZZ7Pyblt}Ry*pc-H z;Xyhy&pw8o8i%7v%zjdKH+qS_>Gk)Riw#T9cj+vOiVX##lqQ0VfCPoofB9QxZUT0? zi>GEj2sTF$&)9d55wAPvzX1b0I1#R{UJJ;_uP)fF7XIEFXwcz)LvfS3=yQZ9Q(YY4PfJU1fd#Uzx*sZzKcxC%eg z<0C}s9;9BVum@7KK=o+cIoJhc$tl+khI3HMRO0-(2^CQF)z;hLqAR~TJKembU}?scfT?*XDSBkn)^oAEmLpoR4}IfAqtUGY$+EK(h$d_}p|Kc_CFpI8@B+TFME)esADb`x z3?`#rtF!G#*@4>uiC6h)Uy-1#!F_(Y{Iev9im29n{BpibV$+v|O%rHtm96>;F%;PF z6XFIye_0ddP{-i!F*9?V=?^dX_iW*))tS#Zp5G(Jv^KRUopU{QymL&LjH7bf90vx( zd$x<8rRu52a?l_^V4VM!;m=0PWfK&5q^~^~?<}OtBzfcbjkJ*^7D0i^!q}+&F!w+q{NM`Cn&Kc=e1bYga@B$yBl*U0^E}i zS3}vUYTtof#M&?Pj*l^+xKp1?TiSi$xBs@M#)F@lSF5w>o(LhI6)d#d+MF;8Vvs6G zg=B^ml3s79p+#BaR&T&?f72$bY7b&v%=jZEY9WJ$fns3*lQy<036blc<0{b_p^E;F zsf;0MFeghWtC?eHKN@G30B0~|0x=M<7`OFg7;N8bkyEU8U;LUP?zFxfy-=5r^1cMmjI?P_yKB UqpSaVSDwuHE*52Gm|5trYCM)!R{#J2 literal 0 HcmV?d00001 diff --git a/performance_tests/test/resources/ciphertext/raw_aes/ciphertext-data-medium.ct b/performance_tests/test/resources/ciphertext/raw_aes/ciphertext-data-medium.ct new file mode 100644 index 0000000000000000000000000000000000000000..e62387249c756424277b3029e6db0ad58591826f GIT binary patch literal 4454 zcmV-s5t;4+1$Yw{{ts}sCFsTUNLf1Af0V}r7mhVx!S`DW(lB|@hF$<)0096MVRv&a zV{&qK~c4BQyZ&gDwYjIabLV7P`NmFBTLUTAqM`|@fQ%E^+ zYeP|TLn~!YHa1CBG&E^tW_36@a$ATl*JEn;bOAVEb_Aa`n+afB*mh35S`Ee(a(p5CW@-b(;NU_S^ zKn&jZ$cEFi5mSqAdIvo&c@Wg&3v`d@N{&2yxJ#TG3Atx98IX2}Kecc7EAI$D-;)hz z&bzZ;cTxBNHP=1KEa02|WDA+;L0Eev7!y4Qm>yeG6fz*a1|n#4b0M=gMV=$(t>HwE z-xcj=S<>i+qKzHVs9H&U5qEOlOKFO^=lP*2*J`}_lzudtDXbXIj{{e^!ouQIlc+{q zgYV|`ei-&#cDnbYm~msNFi72Z(%pI0f)vim0&(pC=)}WOClhfs`*OATpPoG9;IcWF z%s}Af^jL>U&WbSAm;B8BWA`?(#u4YnxRaC+c-|P`jHAxqArpC2R6U)pH%i%KrBA!c z5j#DJkmFBJ)pNK;(Wc%?D?xpE~?#jaj(97#%DZrLneeu2+ z|9aN$8JbAWN&hL0UXgU-^Q1A~^`u@UKX`Dy+!DOVTE!N&ycjs)%gZZ-GYLd-7APnv zBOTqA!cYIuaJn}on8^kCiLR6-4l*Eoqni1OQ8W{bu##N#ij**kr~*B_B{+!WSyTSw zw9y;Vv5|>ROL^*uA66?Z5r1!zkmSGlt(0Js;4|BGk3fz~sG=kN4bD;9_W=+)uj%7t zQip?zg|Ac^7OPw;O1IsuaLse1p?>U7`XF%nGXD20fi1CouMAb#qSBK|YxU}LO-Unn z{}z4GU7}34c@ZZ^xkFqT3z>AF&~|$%ptYjPDA4&YeS-t+&C>2jIW-bHHvV@xMm*LX zG8m|vZqTeJsl!EhZJxMJ7D<*sED#5q#Q57h$P$AytX()zKT5 zw(T3KtP&=yAQtTZV(~i4l-af7^ESRWXl&aF8`0m5o={|asP!~$m517{H0$)ydUX2w4wOQyMd9mz#kyqy$E-fMbV+#t0W*80uva;;Z@MK$#>hAyEU%@dmb6Z88W5BUQ|%ie6T{UNTl-N6o$3Cu@0qp0 z?jhQ8LoJI82bK>~Hj%gGCW>}Y_Kyg+Kc5cGRkGtpgvPdrlN(WPZB@nrK$It~Emfy= z&6eCIEr*GO4I_9IvL2u|DM05nWeCtoqyWNTxdkGH3bq|FA&xsGBQrgdn>Pc7g)l+( ze=?$K9@CoUD0X8^&@g#sh+uJj(o`8Jb0|DDED6#VY;UTKR2Niu!qNB{2?*h=)96zZ zdF-)L)9FqFlDc4Z(mlIq*|8i>CERA^uQl1iXml``U|{?H;z|POI26`s8p!kkOGoou ze?NpX9S{wA#yxk&mDJ-=_ZdDhrn;dGfk9JVK7%HMU$b~G%25}#FFO%}qg=1=L=%g; zL1TYa)859m6qj;T0F^9GS@P$Sg*}2tCsL12*nZ5U(| zZ8{bd73?1K#vJCFPIBACyE)%ssik#6`+O6bk_C^frCr|ID!Nn|b2kfN=sK?#Ucjg4 z9wf&+XJWu-Qc|N@^xyok>@$48RgvPJWf>Ld;EL(O5e&0uG`;DYcHxy0hkWeGDy1wQ z3BTEynsRyK--^u+k;oH*VcmkEXdC;HNPi_+%F3;P3e_b%Bvrr&Ybh$4-G$ewxPBi#HO4_BF;#*6dHnk}C)y`pQ_CjH)ZeJ*jvDpFA+t7U62w=3=|n4J zqs(9e=8!jwOPOQ3P>alUakD+x%6xBtcHlksxZ3;uvAgYe!Zd+kFX-O!t`ev-t5ItTEmpbkgnuBXY%}Mg z#*Vo1*~_L|fJ~vCCXOC%ErE=rt^$=60rYSd-$QSL(>MQCR{myRSUZ;vGpX>A)3rZn zQ|P$8N(OrE+$wLIu|+NP7|*zLIet zLo4pyuTtf&$NOeOyZ$4r9#SOUvLCNbOEFQF1>ATcA0Hwrridb-XxXEp0}me!|r-i6kAVz62Ak z`8q~(!e7dAXj;QrM+!Y~*zJ-!t?Rh@S>r*>Ia!GDGm>@CjK1=5pIDc=*aPMBkwtaj zQbLj_PWNpJm-Wu$DDN|zLXF=b%Ey)e68?^7$M7G~yk0pty{ zVeWd?H>--Wpz162w*`9blE)^KVTTOsb8T_5$P} zv?>jGvNT{}xeI})I$*aGQ*uJ+jJqF1Li_Nf--qaQ1{?U7jrZ1*TtjV@(|i zYj?(wd-=amVS;=;X)?q++&0R+`W88dVGi+=&t+a96{yGLCSGsdhA_SL;rPj(+1Wxb z{lQOmjjQMD>6lbCQA{ZnSBt;y>td(yP<;-L7z+RGq1C?c%n1|bWx~1ll)3-yQfEoU zRXd~-{QDZ~AJu^XsOs+=`9xz~^IC8mcd1Bfy21ppx{g=hjrpW1{+At3{tr9I3Sdd}!gf1k zknVAcdcvdmWb)8VWF`Tp2hi}cMVY7!)3A=;5f;@#mwW-AR{%j9MWZ6(8TBC~nf*-s zaMM60Pp77i!F&u8boM|)g`{mFPkDFE-dt!>Wba$SCr=n-pl1(j<P5xLdFtE*jeE-0JwKxHhkY;_~KCw z-{QA=GY3YPi(PCu-bZ7%f}gg3p&z}-nIozn&m+_1T#V~TBP`v>B1Houn${gbEnb&J z6B}@q{`F1)uyWvMe#88#XA&(6BuC28Hys=gPsTR2NOBQIS(PH9%kvCpIMalFqD1Jv zwOmC^I|8;4NCNxo_3d*_>vTt$92s~cD)Uh6M68$&&R&i49!yH5MzTjuDNco~c#D(& zLpSxvEqOOylKgS$d)fmlFw-z0Ubd*5wK%ptn|(HnDY+mf9%s2uCYw-$i0dg+O!Ey7 zRiVLuQA$kRxd%$|ZPk?vUFJ|7#SSJ5!+I{x&p=e(_2}N*geP;Dckfp^3YpYE#4@b} zu;VOfVfd8?D5;(=A)3ktA^LFoLp>6BI~%;Qm5k{o;udFQHUITU0GJSLzY*n1!xqkg z7#Yz6iavDk$2*bq0TZXQR^P@$609)rhho`Z9z12lQI z^8H4;$TyeU$Y0bRLr`_5_O=dfweZ;)SIkc}pkE-NKbNT#KOKxgZWQI-^8aVf-4Mo# zJ4|Drkqa-&bL!K78!WqBS9(l!d*iU@{Qcq*?FyZI{5KHKYC&T8Rql{?|G#Wx6ItUY zyE0wy5E^M;=aQh_A5`4ve0XeUZh~SDMY25H_UBXD1edZ%Lm_LfLMLccNcE*1LGNV|iXzHzuhb*hMxHt6919WL%6I?6~1pOrBI)DZV!}NtQ9-bIh14nNA7^bpZVHb-%0B0~|0xTZee@l;HH_$Qh*i$n3l+R&|PRunpy8Oka`IEu@)gOA3_&ctyWGt`R+-Ecm_rW(Zup%-Q=Rm zf|7jQg3_d%%w*l{)Jg^yM>D^m($Fwt{enoB(g;HXXA6^vuu`w^aEqk8^w1Qyie#5c zpI{>|@1%5>D36j%BV(s<{{k~_|IlO)ud)J*f^u71knzF!xv2`diFt|XsVNFYiRB7F z(~BAOeJd4=OwDwYGD{R3U4s?MixLY8GV{_wvJ4Ci4M5EE<(GoTms>X?V=ppxGZ;*q zx&D9niFehe8NpYjyW|)a_pXzVIQno`+qDnNWPTL22_@Zlz}L1>`Q*E`m!?!zzh+`! z5Ma0z>>Ct+k@LDf-!!Ku$KG8qoseo|6Y}?8dnebwbDq}hrxMx!AO1B@&DP)k?5Ar1 zQU8GeXf-1c!vKh_vax2#ku`4}4`gh4r}^A%-wizj4Gz)2XLi3yxy~cINdK;uz~5~r zY4yzk4A^*|@o!#QZ7_3s9`H}7EfxUcjmj(l literal 0 HcmV?d00001 diff --git a/performance_tests/test/resources/ciphertext/raw_rsa/ciphertext-data-empty.ct b/performance_tests/test/resources/ciphertext/raw_rsa/ciphertext-data-empty.ct new file mode 100644 index 0000000000000000000000000000000000000000..202643e9a47c3d902e31641767e837dbab65aaaa GIT binary patch literal 899 zcmV-}1AP1f1$a;k*Y6C|%Vu~O9PME8#TMGv-D56&of(URe4oUhO)&sp0096MVRv&a zV{&qK~dUZK(NlRj9I7%>bPEKiKNH=6ectT_^S4Ar}YDY3h zD`rGTX*5}6c5G-yWi(-7M=>ixVMc3Xd16;raCJ&yQ9V5X0RRVQ_G1Zf78CWqASsUpQlUu^)lm4R@Xf z3()wGI&o41BK?E3)<7oi^$-i}~GmK=OFN?HzBcx4;+(`3b@*q0?IB4*n5? z>9V$X=XZUpek$EzX&HROH09VgaGQY%PeoC{IMv&rDxyR(gC9tKIdcJ!@x9>?NU2ma zxF6|b+RLMB%Uu1@*L|9USt6u}NFiZL5ZQBwY*X;@T_-}I1%b{ZJP6cwyD$Du0}6Ub3(Gb$GZz}CX7^U? zN6SYN#JbaH(HM@Ffm?F5W|bxjOR}~7>e$$X8WnblyBzJYuidn-vzg@u#OcHqJEXRrfQ8q!b*QKi36DF~>EC^s z5>($+#m=(GPG46}z^aY*wc^nS&YJnlT2#!}-$qum)QPd>;_>sNL%y{O{{ktH;aBa0 zj8;i|0kc`yjGG7Wnit~6)?RL>tS9|^enhQy*94~+0ssII0I8@V{w5l%yy=9znPr}4 z(CW?RNfD!_?Vwv%cmW|R^O-LUpXR@Mp4PBCzlA6xXaE2I{{R300RR910000000000 z0RR910PvOk%5-wF>4(^>4Dnz@AOL4DWdbn(nI{|Z=WlyJzsROUf%}!7NJDYF5tf_- za^SO;L`WdC%_g6;r^9HSV{j@1$F+cl0x*4r?|Sl{+298nR<-|k#Esi>(tjA=Y#&$k Z4Q=Y&cGne<@Mn1}3)%96i?ANx>G00Sq__Y8 literal 0 HcmV?d00001 diff --git a/performance_tests/test/resources/ciphertext/raw_rsa/ciphertext-data-large.ct b/performance_tests/test/resources/ciphertext/raw_rsa/ciphertext-data-large.ct new file mode 100644 index 0000000000000000000000000000000000000000..259d83ea5ae956148020b4f61061e0932dfc0d6e GIT binary patch literal 8931 zcmV<9A{^ZU1$Z7^6is}J`L|i_hTTxthx@jqK~FH>1eN=8g#YF14!H!E~>b!K{4Xkk!QH&t|WZcTMU zX>fO9Fj{7IYc)bkD_KrxI5}=vZA@-iF)?B?dT>NecRf7-0RRVQ_G1Zf78CWqASsHuf&#Cv~zrlE-)e zUfZV6-nAVObj4#1!JmAU+mqu@az9t$OXxnk5803-@QD< zz?bdfo}l+!fMRPxRAH+2<*^W{TSI)~=tnThz~HB}TH@hmu%=z=2xBQjEBWm^VGIm%OCpynJO`K%*6TnNLGuhUxAXZO)v^?#);rWP8<-Q;~otGXEn@W7o!-a+)Hbejn2 z4IY(e-)(n)b;eNUA||0dar4YS8+)vqyV+RxvWPo|6qI~APTmcGTbzCGExld-3#bGKhgAiI` znkxWo!_Cv&pknArq=FYHh_KSCH%&UOa6y1)hF0ssII09~KT_!sJ2+z1X{9=(p- zis}lXO$9%#-tScm5V{4nmrovqBG~Fwuu(M7xv0h-S^xk50RR9100000000000p^KM zW>9rd{&@wb@&)jk=ks=7er(-uc8;R-5T)@&kn7n*I6-H2Ig`-|J#=KE1%kDs&GIC? zvvF|LYq;N2wnnLkUvuXHFR)L&M!4%1pcVykxzn7L4zg_b9HJ4eAc2u4L7_T(QOpUl z{dmPC-w%L{9PFAB=s0Q;^qc_)!P}es$+ua3=2{VjYpJq86PyZtQ%M0QV3hQ{$Dw6T zhwZ!79|wc)y~ucD-#;eb%roe_+ETRH`~Y#rpU3~0h&i66FHA(`^9&*>y?c9CuO+fj zQsR~=UqUR+aTnztum4X^1#cTyqN9G$Vg^59yBJJrjQLYf2@MZqbhrgMIXvzJ^Pi%; zomU#hR}U%jkY5`GT-AkPA#gHc(0Xuq;hbbK{0Iz;*%B9q<@rX4^%I!ct*cCj$@;1JRm6F^uvs`>jp1$46NHLqf+k@NmwbENiO z2rAJzm6?EjslI=BT+gdQ*zGvwV!WP=#3aw7390?qFpxVjiFMcV$0&i>&^u|C1r>hcw8+bWm4e8MR-6*uLBg{;zhP@0DU$ zXc;-QQ#5?%S4jo;%em8~jB(P7h90yu`ozY>W1et*RRGvNmFrXWy^ z8e;qx+LtZbUn{Ap zLD{4MN&%$x_p-+@x{gTn>Hn#Oe{C+9Gy8+iXcl^leFrb+rh6Okf9AQ9GR5~*utN{a z@^!up{}B!=g->i8c0cKpR+-~m%AX1zE%!EQkqdwo|A2X(GM903PJ4?PU$gQ;>gG zov$;0M4@9!J6g56aR`6(6O)qZZAwY6Jrzjn&d~P)6!h?1we^qO6f)XuXn%>iY0fq+ z3Sd2WRh_1)-x~7gDrl@}+=|;o1hP^YpjWcBVt#_6VyHbQW`v}25M=yxb8PNvjt9}x z)rCOPCzRHQ2%>KH>~Yf1$ZoV;vT*M*X(!59RQaAp`T7ye5_7;<{e1lO1?>@EjG|Oi zzgI@9j8~5{ZzmF5LV{0sR3r>(i0!?p2dYtUnlD*~CpWm}*DMX45r_}ac&6#?z8nGoL z=>veeZSrnILW`?G1wr+k^n_?!F7=G2yK7=&QIudH^ah%J>{o&8QbTAMHCc_IH#mI% z+6r-erMOilQ$2-1vOGGFn0Kjk_MyuKhQEXa!8+YWrb6qo3TP;IL{}&8Jg-j6i(Hl3 zhe)tKN@@)_p>Rj}-D0Hj8;$E>b;qD{%s424lT8E%C6 zy2IWCJY)?4OZQZtFHqO61s)Bpns-STOo_47rjefjh>uw%POmE+VfTnglDL#4TYXY) zES?Sku?Xd3W~gWLFfg2BI1Ss6idDTp><7lQU8xx3lJl@=_PVsb`bg&`yb)o)hL ztsk*+tLx2jjjM#IXLF>tCwmV*%T3hsgMBYlTy)I=2PHPGNNeop`ttgZsV~=dzuGIv zgY#F00lJ6UVAiw3Xn11eEYI5z*6Pz(d(@OwE4Bv;9md-7si|yY7ZB%;H18d54!7H+ zfzh}BSgLmON!^$-R7^+9$|&WHD>a_i`?_esT!OkIcT$Z7QE(eC(RE}McdV4}>+h_w zW>NiQTw>KJy5>#!>*1et3aKUR;dF9IL8{?VEAvB7r8fq#kiO ziCSKmN8h_RMQb888eo9w?~Ne)4U{{m*a&@iZ-`(;uN+U?i zF3XKKQ!0^}u;K?rAjExbjuJ1A?O3cMq6rLDU>PsiO&|EJ%(~$jBpv}8#$7m0d}i&( z7S4d4u245T9CKIes-{L1AY@AaufLJ);~j}d`I&qS2j45^OK4z`?$pHNMt=Sw(FXc- z5_r4{8}{UDX;T_jSQ^^Fz>VmNZlAb>1~n3J&-ScT-nj_qrYLaw1|314ZkQSojcO2t zf7uXhk?kjPNQf;(&kpZZ)HG)dFNbfDR|y4S8tmP3&c~+lOk-W(p>k7Op9c@6e}h0l z$hx^2IVh52Ro4sQ>xY+U1PS2K!8lOY^dBwDD|102d#f`+Lw!0wVER8J)>~6bW{OZ3 zkLs|*Vj(e4p|UbNU2Ww+W8Y7mIZ%=A(?zR{anbFe$_ya^0N~rO%)*O1_Ym8(MT870 z;b!KM3qPn-Vr#}Kcbl)7AYddM9bLAd`X5&Oz88?>9W3BB$fjU!Pv~Gy*`^G4^_=_f<#sEP5i zL~(3At2V~Mb0NltK%$bFV|BF(NMqU_(2XDo)V$ez3FK&6>-TP^ZdxY zy+Y{W1T|*6N*iE!CW)qNC7NLGY*b1vO*sMW9TyLg$XHGa|J8gwn>7RQG?^o*r--Wj z1v-Q?Z-vZK1F*4s+tvouLx|d3qjo$r*9VDrPiP#T8lR4xJ0v_kQ2g~sdF$`+w$vMZ zCUy9(FuJ7{Mn19hNT5}~GOVL>g>TiZl(&)=M^J}*rqy+Gu!TgnD7}&TSuYkEo1p)L z^f%G7YRBKXI6-b`$JYJlO4Rkr6(6)v3R(!A)ZUsGR9-LDDa)xHA|3V;${Q-`m_7n- zq>v3k=pu)m;p96EWh(a=$nhGmeFuph3Sq59TK@_K@I*WGCr7ofAQURf;%L^Rzv#+ubT;!RvZ7XZV_2IuHOg zwF%H->j(QS0q|#YIYvpUluOCgL@kRy^Dcjd*=GVvuD2OYBlxDUC4?zBR1Q7e8Fra$ zkcE=DzsVI_&@8i*xNR)y?SPDsSg-UQongo77gl0B6qhD~;WFQwEk}>d6wj`(=_{~s zZ+Jrr)&a@3nRiA=+od9IehhmMqBfIu;*RpSdQ5BOj97Y7rKKm#ulcpA7MlA-_p-AG zvpc--#fslwGxLR9&Y%XmGM$F5t5q%xSf@MPtFX$x_f-yQAQ>%V^c*HI5OJFK`CEF$ zQ1ELK(5&G_vi9YBvvJ%@K!n*=+sz1|Ovw@YASBr~tdV*`^s< ztiB$ho^@R6({wF|A>DS1h#8>qSW=Gnc-A_}-yxK;i(_`Cvg3^p{Yw51)T)u2Y;>!*tKp2+6AYh)c(#`ijr9HP8EuKQHRv zv{AMeT7G7Gi^;M!)>3@L3^BFz>C>X&%cD1*YQxifxET&HAKsaSr*TXD(*B0=o5*Hm zJoikMsxvL^z_B)#(qKUnKs}O$Nq&E9SWdWs|NFAl=^`V-Y0mE{hmRUZnFa2~NG$YK zsVtfQYEd7xp*V~(EP1xF4cjHq9&zvU56>V%>=AoBI+db*T-XA zQ6}oOH6*p)6x2>H>oGOD99s(gP}Ot34JtP6v=;UjPJpB=;c#<((1OqSZG zhbsS(HAiAA04S2~A>DFjdcXnoM)~Q+4|v@SA(+v! z1H#;tK_Y|=#7Y9&A{LZ6nhjKGybrv+;S~a(sFFQZk5sNm+?R;F7U4Q;Jcj3>IvJUP6}e?c>Mp~m0lV{w;fc_rZZ(zQpBLVE zk-y$#kEyoG?h~Z0feK$Z)MxKraEpH#!+vQgO9AjFigqmPinNCLyZH~jH9$DdniW*v zeXbw>5-f(v2d!y+j&!g{13?bNkDHkdl5IaTliMqr5~mSCul!#`xUt7%uA41JE9O_` zyS+`mHgUl@U!3(E-N^cPB)B6^asNCI@7yK(Nx0F&P;H-P2_SJM`0QG4%2jc)GPpLC z3U6E-*wPacd{#;x+`Xyiha@e>i}8K)ud4UTajue+wV zs02HpJuY0$3`_~4cq|;`shno;q9+&a^FnohtOj2nw8`L=%K}&Lr7Z$c!s>;W`#Fjq zF#yQE*ppK8OJrE(*uFqqa(KUJsfGSZ+F7gEwVYID(>_+Fg&8Sy4{$bdkX5+#TqW^9 z0RR90{{R300ssI200000000000ssIHK>nEUzxKP?W!0vj*0p*Ck~X^JL^?zjfp1`E z9`&AVPSe5~aO0?m;l&)u=}CWlm=b3&)?eNUJr;i#+C`MQ=w?DoKGlw-(BW4n)gVVT zqt0E_iX*~XPMt=P`*iH9E6qe=5@l;a+(66`c@1G`D-&Y$J6f1PCUN|}R)h(~djGXG zhN~aQt27BTAgM3MqJGe^ES$rI6)&%3oCg1jdhq`RH`v;Bb$vQ z=#D^va0BNLX+;GiN#NUlXUx4hqK!+BN&Z{ycM(zEkj;hCg8dC8oWH^Nps4(yFYMku zsUW~#>Ebo!AbW6;J6+(!OEK%5k)s+pGFfMsdTQ4UYu|xId2buo^`YCygj~D=<(%I> zZ0KfkJ%?1^p_oc`LDWG2J{}X?TK)|GDW`x)7pwO!OINOx-!<_0R=dH~6@VZKqcMpA zaD)^!!zX;oKaTORPmp!EGG|BqdY}>2fA2yw^s^A+5@wJ#j*Y_iMuyPtfoDn+ceGmX zz0ET=DgUD*zG<1;@uCUR>?+hH(n>p^<0{$0Jr@#$Cd?wh%!!93b!XaJKH~BUp`nCz z293cO+*P>F@gajr?ppMde{_l(qh@X-b|8cR(~{a?iFNKxz;mI3TbT7pvk zJH1C``qnh(xzO8;P2v|WP?W~M#_Vs)*9ip$k5v7fQ45(;3phc}#_;XqRtfp#IUk4? zsK%5bwT}a6b`3cu@-otnI8wMfRKq@{y&5A$X>EAC_Br@?zbXWrpn-(ia@mfxvCj$0 zz*3guB3cCN<(ppIu6M%MK3Hi&AY<4u7rKH6fA`6t46=Ih+i)luY=8LN^bR-cY`!JOSs19*=csXl5x$3p5Z4 zoLm{Fzm>-dFDpuUjZUISi6mJ@%HB~5K!ck9uKmNzA_+C{Md_m3GM7CABz`D`Y=E_X z;s6vKa1r*nk-!9H-9ASMBNX)D7mPd>Up72fcZUlwxo+d8O4qekYfNITC-aXL0~AsI zhdat<%p#4YnK+!Y-D=ZDM4IBJu@KSe&5EA-H!c->=6Hsy3UX?)ZaP0R2ZN>fSh3vU zYvSK>kDF805(3=pf$bZsgURLOKMiy1{?o_}iXWy#y;pK6XCnF(eA2~|gnY|GhBiq{ z>YTSMS_r5Xhy^sb!*i5h$mF7ZKMth-CN_fQmUK732Sn}r)m2}g3nDbS){EIlU_sZd z#cYX<_$@bsum8eLpDORjLP9^EFT^)T5F>5r?TGQa0RqI2T&J4W3_EWHBJpIe`Bfcq-4)Sc3iO&5if5QX4 z)ZEI`us5lwBxZuP@~T8sFlhiYlXhoSQBu|H?}Gda0|ew7p*LWKsi z3cle*g@qASfuC`n+K~8u34l68q0WGl_T0Q>y(N?l{+@QLyK}FnE`wLm)qu+i(MdE0 z+ojYsX!c@?pEzNeO-|%k>wYIPv4+fz3xIPh*+W&c&VBk z03gsO)6yBxUj<*Jq@rPet3>}Aqph-|nWCv0ldc7;LC37q<;rn_z3z9~pWZge6=GYnY0pdcpSkgD?cdNdHPkZgD05hvSoI$F%=`z+O)k zBU?>3PM~aPCEK@|Bg2pLsV1Q8|l{v*T9eAG^40(lHFq(W+~SRW>+0%tdSPCT#6dcDZO zb3owLZjKMVr$N8RvQlSCVg>wtXqTV2laD~Dm%mioQ?iv&y^$feS|!8Mb$KQo(n(Z_ zvml$WEe88#I2q|^t%arvEAGhVl!c)`FHIG%@?2jz&okB@1szXJ069|{v|V3g5DE8> zHpLw2V9!&JwT!~%2*5~rQ)!*`IE=>GWFX}BxUfKcQw25}Jbed|`_c#~ia` zgXKozgWU2yaP}sue{tG-f-zEi6g|NeLfl%o<1>f4D3A&(i(%<;V(HZf{uta3x7uS# z=orWk-?4GEP#M=lC;T(3=8v?5^ilEw#*PYZCI!3%!Bg0(UQ2qCo?@b5Gm zY4x<9MM9s>Uh~>CxQQ_5=241#E5iXPL9R)Xxm?zon`LGhYq{uMfyM2?x#qM?V z!FsH3CXcLOSp390TDVyhcW;ZvE-{1ybKHI~OSv9u??%njX=TeyGCq|_{_rWKyOdff7|Gtu$f#pnc}^%IG<*xz!cr@k-k)%>vp-!6J>auR zDQo~Bq}b)}XMP9dpwXdOl{ugZ<#O&nfd~AI?Vn@3e1z&p#P5L3wC@|Aw@wPBHOeXv zGdxhkY{0QaQ}e{AC|6tws@*~^u)`_(b|x~D>=$B%WFIu(!zY2vzuh@~5QDCf!;w-eagG9-&Vq-YHPxVeA9jdT1S!fAkBUw~GMER`;!bEp7ZOJvKvu^_*3#F9Q|svmH0V zU^YRTj1<<@dU8(yQ5wmt>jDPfN%!Bm&R7@w7*8C`zcP%omp$-#;;2dnD}>)2KU@O0 zbJk|Ks%VGE8Ej;HC+e*T`W6IeAzq;)Zl#;I zB%h;JyW6uvuNVc-`8nL%k;!``?-$O&{-KXGdgS{4r@@p_k|nae7*<7o=u6M@V3GW$ zFm7UWOc|wZ7e%O-LBDi@TCSbc(C(h#($6At??&Rk7S@vWHotk%nrcv{{|2pW(5c)U z*Qc+wzjt5f^b0R@h@yHw!;m`qWS%yX$7pKLCysJ8k82NAUqZ5rOoRp5yu06UAP(@` zo>g~NEq%hZG5vEX;Pi06Jf#v~!;Yvcv5-;tyW|Z41i?*;FSIscTkH4wz8ZU0d8Z{R z0?QkkTUcNYaY(_5>)E+OR}^4^V6os4;iF01YL^gXKFmVEWz>p6SklMTh76;cUF*&^ zV>2eV@R{sZiP6j5uyiIzyG8-`;h-8 z?1f*IE-4vwsWx~~l8^XMgaH_s$-3L{^16l@aEEw0LF~&~w*p;w$aG6nBK8}NQ2flb zyHdqNVk`}y!I@QCI8E$T9D0(I0Pd+;HmLt>{oWxncR!+%8PTyXWXywOf~T!botZKppjW zYJq?D7hf%gPcZMGfv7P88Bs07;8|4Z?C@ToH`B+8u~yu>1Bl{cX!AXvea>EH2ou=q z=Ekz7e8aAA&WVV7rt{%z{e{z(zVSHJ6 z0T{&A+y`xXQBboPplB#1t|xts>v^X!(UJo{faN{Hhn+7_AqAqi_}YCkx$B+B;g>z} z+(0Vk7KAMkV@%;h+T@30RRifL%oYSZ_^D&{70)2CuoA1~052Xj>)59+GmiE5`u6Jt zwB4^t?lL@3M4bcLud|*N)e{mx;{gE)*o;*_eS`d4;8Tq$Xj_FW49#u4$9~1XvveNL z0>LQIAZm7P3~v-MTFjywa zC(y}_ji!??i77xoIjZM_v{mbJuX4-}|3l%wn&KvviR3&%H~)ddHRif0=m2LhWdbk= zuA&qRJZa$_cU;a2Wbh48ZTjZrKIB4mi=R~WQ=a=;YF#ynI xq@t>M7SYVamtH=W2|kPqVaEp8*fp2Kz4IecwUdB8OqK~Z%KDEG;L8#Nl#%kZFEp*QEqcoMMFh&S~Yb|Q*Ks4 zOn5L^dO|ctXn0IYMsGrCb9G8rQ+7^MNO~_}MRi6-cRf7-0RRVQ_G1Zf78CWqASsZ7qKC2GZHvm=xt| zc@6g7I?-4VrU4UnraGtx_lcngQ3s-m2J?|xGu%g+uo^9Z!3o8wN2DNsx|Rfu+~#qB*6NQn$2M+CpY$-Z=(Y=Fv*AVc#8q;TvEe3MouUTvA`+|jsk za^$8Sihyx6zq4C7oB5T5>2Wve@NZ;aeYRnsUurH|4FU?T!BCV#gIQ5u4#O}>o{EQ` z&_htiVu77)H0P=VYyVBUP#t2(jAawMd#Z)x-nwsibYc+Jl}CR{oz>Bp zx-Db*rJWMx(AhU>LAFIuseBQ*|3*2*SaW4R`OLNLy9YTTl}k{d%F!wwKiGL@(zRV` z9oD-IP80DEwB-WSGUvx#IWV#?*SU2>q$?F_91c_z3V-4=5GPAC$4JL^1|k#)k2&-- za)eSEe9YjzhY@l#l^2fb&mu1Hx%>ah2nV<+iky)_?%(P=|KAA9Z~X5~43($z3A@HQ zIc;c}jbFAZx&tqIPOna5-IBKRLGde7G~iCdf!e|^0ssII0QRMjN+L z&oB%yn?otT?cMX6sw~J(K@2I>m0fL+V@w+Ef8x~z!~g&P{{R300RR910000000000 z0RR9GpjuDeihXg|%tNRVr;Z}E$G(@?0Vao(jJ*E;NJaDjcr-t9`scot)bm!05vsQs z(wm<1G5;Ne+PE8VoFnLK^BuL((Kj8*H6&NTni{V?BbG0y!B%a^ptw%JYJ~8Wd6!?0 z)j%A8Qpgjh{gFZJ^hmTbX7^$iA~X{>ypm{UaOlCjYmTVvV* z=#Qn6az^ikF@&v-*{LXip)6n4R@NECoJLUthVIFF>{~&t?HYn z7F{H+>jDQXOU@lNAR%F?>Rsr7UGGEX(J}i34?mMF$*yoksh@OxZ>q9HTu`E4|MMYl z^Zl``wMT&XZRVqUw^$(!w_DSAAzVG~GV;YdU-wDWez)MN}y; z%F#Vf4&ztv08bNs(BD-3>-M>(K5>+0sj(Xv=gWKJ^7er7%Luw}CRzU1PXJ1C>o*S()|z9ZZ>Elco9&%>PENO*$rinOlc6TnQA)PO=F*OH>fJj#e*}*7=-TYn8EX%gn7(n zaYelh7u9V(6ZkEu$9)7VenMPI_YZ{59^M?-%AOq8F1!22hK2@gu+sEl7?@KMA*AY{ zNX2^`tt<^p8tPdxiiw33$ehgu#F(x*7s~8uP)c`N z!d3s}l#AsiK8!dxL{;cefx@^irrH^+K%5N{P3W94q9hg$ldKjfjEtNIN`Kkv>wRHi zV*a*OgV5M#0SVWgT z-tKVel-pI2689V$sw-u_mr1fz$xxH>3TQ;voa5kDk{v2_%B^!t6;g)5NCylq`^JGh zOG^FCEJA1&iYv!+>Oi| z!)s53YS_NF&Go>o)35Xa7;n09s`d#oE3c>RUa4C?Ijy^<-kJi58s-8Gxgj`^k-@Ty zx9)X6gP>m);L$>R4#4&R|lTPu3^Q> zF-a4?%VD*Y?7}fUzy<&Pv>ykm=}lKuc*$PyWiGb84|~D<8cq84EZ@~GzaOiNRNBbu z**@fOi+hhaV^IH_#j=yNA$LKCIk5e=(y>g4VwDtaHJF)13*%S+h1>nzG@WC*osfY* z8wC;zT{l9bTk&^+-}adtk}^q4%~?Nz{BD-&3QcX!9KCto8otv+lGXFu7>uImAmAEU zQXg0<$Z!huG;Eb1-jfLB1b~B82`3pZ?UH4Gxb6D14smb8Wnu|zgtmxw zb8wakjf0a#-znBwqFk=pF2(*&ZzsV@($Y+{K(rTl((H(F zVgD4z;}_eYGlYkSA~VK^(RetQ@_N9z@_|OW*Fy!U=B*|@FLp2qUH;wU9O&D2UzDa5 zis#fF+ZWdtTsX%Rv3|=3{g7-rUfuO56^}lKE{Sw|7FMT0^PWf6zDq0okw2r?6#SG1h&ts%Ix!$4`i%7Vn)XvP?L!5>K~Yf=a^jGLm&Lg z#@sd>dV3PnrVO;(@D$fFX(OS*e?tJOSi6v zHt_|0I&=2g)`GaIc&_4MCk$JKaOT%0MVri+;(##xHyNr-_WH(-4lSZf}g_sCc zo}?gE_VbcfsVXq2T^{>qRlW zFmq6*1S5OdeW9kNGd3Aex1N*Ix^BM`g6%7|$>HW%*^!Ns>BS-{Z15y|f?IVpa&#sQ#9wGlOVJ=NXs*WKKMu|MZAeZXy(Zb`DW zBAz6RGAWC4K~C)i?tl(7bx0)-j7mKVU@r*Z1Zu1lq39gCF78KELbrEO?YLS6RJc}V zk*XPNGl|htmDi7qqLX0QQO6mw^p%Yu2sMf(iz-Fqu62ST8)q=F!4Hh4 zT)8zjncqUowROozJ4}%rWPjV+;!w-UPdugOQh*!>}Hdk$>GngKYzH-IPHtu9&eL_E$)%Otl|{wf^sH=J)#MPID0onM*GmUS$s z^9M?BaFZ57N8}YVMJI#u6BCAo=#$PZ_2SVso>EGjJhTPhI{R<7v{?2Xi1&-{o}y)% zTyY@8`hD_}0{ALLue^(6rLGT6oyPVJ)B{MWQ z^J$*p%mXXGbICZXcAADyzfF$U6+`ETLsD7Du(9qy`u+k8@Dvn@vBElGv2?HO19GPO ztrx2li&U_}#j7R}i>RMG8$~UPuUTET>Ep`Ec4(pD93M3QFUcq245%SJ=m}eZ+ZMLc zdXH>6pvlP=t=9<~nikH-+{P`U?+L5uvtvUSS-b(CvIFJO{6vwrT{}7+U*a$gY5E>3 z&xdC8k3QPu@dsB#97R5bs}mtheGFS>Sq$NC#BwLNpa8UcDgFhWmew7xY)K3lT^xzq zYLC=B%l^=qKh`mo#k%axHKUDiT5vfSb6K}mi)N?I%(WUgwQ?R21t=R%-Ht`UGZ z_BkuU6XncrG2dC|)=~55R4UW+F-6nb$lViZuONU9A&7MFgF9Xi&u(ixyNd1On`FC+*2pHr&YNVny56|#)K%Ehx>t2f2Ijh-gtn*iJB z#q)&CNQ{r)UfgBDGC(O2s6}Krx*kjLYj!kCHhlM=qbD7d-7@R?T%J*%esd>L${nnU z8LUQzmD1=r5NVC>Bf_qS8*Sy-+{CBrIPJh+UB~Pk2x_EZN;Mwx1$7k!=6RMNceO=b zcLW@AV{;dKcN37YR2vHBL;=&=V^E-9trMWIoY;hAu}61sRLxJ zRkLNXBk$p#9RM=FpjOaQI?m=0=cJLBQF?)Pw7cv5t||iYz6{8&94%zWV5Zr-mBlh+ zKo7=d_N%ngk_7^Wk`HqtDJ?%09ct4+E)#590M<|Q**FBm1>+^0A|$_zREHAa5eHL$ z9eE;fz3I{C=*E|lsyV$z?TL85M#A>IC2?} zv+ReN2#3@6X0+%bawmBJWo5tS-S|mQGb)3M7Snx*yl1pv$J!k(-1RPsNpLbPV*uBZ z6Z5#T2r$xVMr7ETHRfhN8K5ZukA&30nD}}aa?x4?fUZdVW<;@Wvkd&}$CPdiKSCaA zAsw%WIQZh7r*1+WQfC1w={t%JwZCirzbxyUl>z7wN#)H>3LBv4W1Sv*n3DKAtWeH+ zO3C;AltpJYYp$o5^3)|Z=KY}0Yc0g~An~b#XK7Bm5QaFghCGz9-=%f<0Lz`SA)JXE zOq$ffxS9`b{#ZCVpg2oFNhVP2U13?C%f#^k(O{e+=$6x)f;rs1JDV){iEa@WLe1#~ zTOJbj{cNa*Qg!#XOL@3K8$XD&uaAj>hI1vGx;_oe8)I7oO5HxFuUm^QLVoD2(&u3P z@xsU#qK~aCl=+b5?IgFGNf^R!>lQNjYmdT3SXyG-!Hwc4t9P zSypT{QZ+d-HF<1qcS&w-FVQ_G1Zf78CWqASs58naU+q=!I4UR#E z@T20HH+nZaMAp`4JhKekumPA%phX{?uTeZe$oqq!el!e^# zIH_IBCY9m@J7o*;2OC%`cAD(D?i00n;%&OUSBqG!`KXn@TfI7^2~yWX0a*62u{jVZav7z4X15V zhIrO{S!5RSER@7uBEMgC__rBIn8LF_)sGk|#~FYen}TFDaH);EvL@Ffi3tNfi=J?s z9|<(5w>~re7U07Ir*W6aiu(~#RbV1f@UTR_%BA~BD5tyPuY z2eYvgOxb&1oLbDeXvuODQnVuaa}rKesODduMtWdbl)hp%P=^`2C{O%sJkpjW_=_@oHiK>`0^*gtFIt|&j;{ITCYa2k}> z*<$OY3~uWJF#y5N|4Yk34eG>&1W+~RHWJDP;bPg83Z02S;I!(o!^u(-LK^&X(@=yK J#Rbn%t$1*ovReQE literal 0 HcmV?d00001 diff --git a/performance_tests/test/resources/plaintext/plaintext-data-empty.dat b/performance_tests/test/resources/plaintext/plaintext-data-empty.dat new file mode 100644 index 000000000..e69de29bb diff --git a/performance_tests/test/resources/plaintext/plaintext-data-large.dat b/performance_tests/test/resources/plaintext/plaintext-data-large.dat new file mode 100644 index 000000000..22bad9f3f --- /dev/null +++ b/performance_tests/test/resources/plaintext/plaintext-data-large.dat @@ -0,0 +1,21 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Et netus et malesuada fames. Bibendum enim facilisis gravida neque convallis. Tortor consequat id porta nibh venenatis cras. Lacus sed viverra tellus in hac habitasse platea dictumst. Ipsum dolor sit amet consectetur adipiscing elit pellentesque. Risus pretium quam vulputate dignissim suspendisse in. Ante in nibh mauris cursus mattis molestie a iaculis at. Nulla porttitor massa id neque aliquam vestibulum morbi blandit. Urna et pharetra pharetra massa massa ultricies mi quis. Leo duis ut diam quam nulla porttitor massa id. Vitae suscipit tellus mauris a diam maecenas sed enim ut. + +Non nisi est sit amet facilisis magna etiam tempor. Mi bibendum neque egestas congue quisque egestas diam in arcu. Volutpat est velit egestas dui id ornare arcu odio ut. Amet tellus cras adipiscing enim eu turpis egestas. Tortor dignissim convallis aenean et tortor at risus viverra. Interdum consectetur libero id faucibus nisl tincidunt eget. In eu mi bibendum neque egestas congue quisque egestas. Pellentesque habitant morbi tristique senectus et netus et malesuada fames. Leo vel orci porta non pulvinar. Amet massa vitae tortor condimentum. Malesuada fames ac turpis egestas maecenas pharetra. Feugiat nibh sed pulvinar proin gravida hendrerit lectus a. Quam elementum pulvinar etiam non quam lacus. Accumsan in nisl nisi scelerisque eu ultrices. Pretium aenean pharetra magna ac placerat vestibulum. Dui faucibus in ornare quam viverra orci sagittis eu volutpat. Amet venenatis urna cursus eget nunc scelerisque viverra mauris in. Penatibus et magnis dis parturient montes nascetur ridiculus mus mauris. Quis commodo odio aenean sed adipiscing diam donec adipiscing tristique. + +Leo a diam sollicitudin tempor id. Tempor orci eu lobortis elementum nibh. Sit amet commodo nulla facilisi nullam vehicula. Hendrerit gravida rutrum quisque non tellus. Diam vulputate ut pharetra sit. Semper feugiat nibh sed pulvinar proin gravida hendrerit lectus a. Tempus iaculis urna id volutpat lacus laoreet non curabitur gravida. Porttitor lacus luctus accumsan tortor posuere ac ut consequat semper. Tortor vitae purus faucibus ornare. Risus viverra adipiscing at in tellus integer. Suspendisse potenti nullam ac tortor vitae purus faucibus ornare suspendisse. Facilisis sed odio morbi quis commodo odio aenean sed. At auctor urna nunc id. Ac tortor vitae purus faucibus ornare suspendisse sed nisi. Suspendisse interdum consectetur libero id faucibus nisl. Tellus id interdum velit laoreet id. Mus mauris vitae ultricies leo integer. Metus vulputate eu scelerisque felis imperdiet proin. + +Venenatis cras sed felis eget velit aliquet sagittis id. Turpis cursus in hac habitasse. Magna fringilla urna porttitor rhoncus dolor purus non. In tellus integer feugiat scelerisque varius morbi. Tortor consequat id porta nibh venenatis cras sed. Ut sem viverra aliquet eget sit amet tellus cras. Semper risus in hendrerit gravida. Libero enim sed faucibus turpis in. Ultricies leo integer malesuada nunc vel risus commodo. Ipsum dolor sit amet consectetur adipiscing elit pellentesque habitant. Varius duis at consectetur lorem donec massa sapien faucibus et. Ornare arcu dui vivamus arcu felis bibendum ut tristique et. + +Diam quis enim lobortis scelerisque fermentum dui faucibus in. Cursus mattis molestie a iaculis at erat. Diam sit amet nisl suscipit adipiscing. Ultrices dui sapien eget mi proin sed libero. Purus ut faucibus pulvinar elementum integer enim neque. Ultricies mi quis hendrerit dolor magna eget. Morbi tincidunt ornare massa eget. Mauris cursus mattis molestie a iaculis at erat pellentesque. Phasellus vestibulum lorem sed risus. Sodales ut etiam sit amet nisl purus in. Habitant morbi tristique senectus et netus et. Consectetur lorem donec massa sapien faucibus et molestie. + +Montes nascetur ridiculus mus mauris vitae ultricies. Lectus urna duis convallis convallis. Pulvinar proin gravida hendrerit lectus. Semper auctor neque vitae tempus quam pellentesque nec. Donec pretium vulputate sapien nec. Id aliquet lectus proin nibh nisl. Enim ut sem viverra aliquet eget sit amet tellus. Vel orci porta non pulvinar neque laoreet suspendisse interdum consectetur. Feugiat vivamus at augue eget arcu dictum varius. Venenatis tellus in metus vulputate eu. Aliquam vestibulum morbi blandit cursus risus at ultrices mi tempus. Id venenatis a condimentum vitae. Lorem mollis aliquam ut porttitor leo a diam sollicitudin tempor. Rhoncus est pellentesque elit ullamcorper dignissim cras tincidunt lobortis. + +Nisi scelerisque eu ultrices vitae. Eu feugiat pretium nibh ipsum consequat nisl vel pretium. Commodo ullamcorper a lacus vestibulum sed arcu non odio. Sit amet cursus sit amet dictum. Fermentum iaculis eu non diam. Quis imperdiet massa tincidunt nunc pulvinar. Tempor commodo ullamcorper a lacus vestibulum sed. Venenatis urna cursus eget nunc scelerisque viverra mauris in. Vulputate mi sit amet mauris commodo quis imperdiet massa. Non nisi est sit amet facilisis magna etiam tempor orci. Consectetur libero id faucibus nisl tincidunt eget nullam. Sit amet risus nullam eget felis eget nunc. Aliquet porttitor lacus luctus accumsan. Vitae congue eu consequat ac felis donec. Vehicula ipsum a arcu cursus vitae congue mauris rhoncus. Fringilla phasellus faucibus scelerisque eleifend donec pretium vulputate. + +Massa eget egestas purus viverra. Nunc sed augue lacus viverra vitae congue eu consequat. Lectus quam id leo in. Augue eget arcu dictum varius duis. Nulla facilisi cras fermentum odio eu feugiat pretium. Adipiscing diam donec adipiscing tristique risus. Imperdiet dui accumsan sit amet. Volutpat commodo sed egestas egestas fringilla phasellus faucibus scelerisque. Dolor sit amet consectetur adipiscing elit duis. In fermentum et sollicitudin ac orci phasellus egestas tellus rutrum. Ridiculus mus mauris vitae ultricies leo integer malesuada. Nulla pharetra diam sit amet nisl. Nec dui nunc mattis enim ut tellus. Morbi non arcu risus quis varius quam quisque. Ac auctor augue mauris augue neque gravida in fermentum et. Morbi tincidunt augue interdum velit euismod. Sem viverra aliquet eget sit amet tellus cras adipiscing enim. Volutpat commodo sed egestas egestas fringilla phasellus faucibus scelerisque. Odio facilisis mauris sit amet. Amet mattis vulputate enim nulla aliquet. + +Nisi vitae suscipit tellus mauris a diam maecenas sed enim. Venenatis urna cursus eget nunc scelerisque viverra. Neque egestas congue quisque egestas diam in. Adipiscing vitae proin sagittis nisl. Sodales neque sodales ut etiam sit. Non consectetur a erat nam at. Ac felis donec et odio. Adipiscing tristique risus nec feugiat in fermentum posuere urna. Ultrices in iaculis nunc sed augue lacus viverra vitae. Enim sit amet venenatis urna. Amet consectetur adipiscing elit pellentesque. + +Venenatis a condimentum vitae sapien pellentesque. Ut faucibus pulvinar elementum integer enim neque. Nisl nunc mi ipsum faucibus vitae aliquet. Netus et malesuada fames ac. Et odio pellentesque diam volutpat commodo sed egestas egestas fringilla. Ut diam quam nulla porttitor massa id. Id donec ultrices tincidunt arcu non sodales neque sodales ut. Viverra ipsum nunc aliquet bibendum enim. Lacus vestibulum sed arcu non odio. Lobortis mattis aliquam faucibus purus in massa tempor. Tortor at auctor urna nunc id cursus metus aliquam eleifend. Ornare suspendisse sed nisi lacus sed viverra tellus in. Tristique magna sit amet purus gravida quis. At ultrices mi tempus imperdiet nulla malesuada. Erat imperdiet sed euismod nisi. Eleifend donec pretium vulputate sapien nec sagittis aliquam malesuada. Fermentum dui faucibus in ornare quam viverra orci sagittis. Nec dui nunc mattis enim ut tellus elementum sagittis. + +Suspendisse interdum consectetur libero id faucibus nisl. Bibendum enim facilisis gravida neque convallis. Nisi vitae suscipit tellus mauris a. Massa ultricies mi quis hendreri. \ No newline at end of file diff --git a/performance_tests/test/resources/plaintext/plaintext-data-medium.dat b/performance_tests/test/resources/plaintext/plaintext-data-medium.dat new file mode 100644 index 000000000..3313ba519 --- /dev/null +++ b/performance_tests/test/resources/plaintext/plaintext-data-medium.dat @@ -0,0 +1,11 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Et netus et malesuada fames. Bibendum enim facilisis gravida neque convallis. Tortor consequat id porta nibh venenatis cras. Lacus sed viverra tellus in hac habitasse platea dictumst. Ipsum dolor sit amet consectetur adipiscing elit pellentesque. Risus pretium quam vulputate dignissim suspendisse in. Ante in nibh mauris cursus mattis molestie a iaculis at. Nulla porttitor massa id neque aliquam vestibulum morbi blandit. Urna et pharetra pharetra massa massa ultricies mi quis. Leo duis ut diam quam nulla porttitor massa id. Vitae suscipit tellus mauris a diam maecenas sed enim ut. + +Non nisi est sit amet facilisis magna etiam tempor. Mi bibendum neque egestas congue quisque egestas diam in arcu. Volutpat est velit egestas dui id ornare arcu odio ut. Amet tellus cras adipiscing enim eu turpis egestas. Tortor dignissim convallis aenean et tortor at risus viverra. Interdum consectetur libero id faucibus nisl tincidunt eget. In eu mi bibendum neque egestas congue quisque egestas. Pellentesque habitant morbi tristique senectus et netus et malesuada fames. Leo vel orci porta non pulvinar. Amet massa vitae tortor condimentum. Malesuada fames ac turpis egestas maecenas pharetra. Feugiat nibh sed pulvinar proin gravida hendrerit lectus a. Quam elementum pulvinar etiam non quam lacus. Accumsan in nisl nisi scelerisque eu ultrices. Pretium aenean pharetra magna ac placerat vestibulum. Dui faucibus in ornare quam viverra orci sagittis eu volutpat. Amet venenatis urna cursus eget nunc scelerisque viverra mauris in. Penatibus et magnis dis parturient montes nascetur ridiculus mus mauris. Quis commodo odio aenean sed adipiscing diam donec adipiscing tristique. + +Leo a diam sollicitudin tempor id. Tempor orci eu lobortis elementum nibh. Sit amet commodo nulla facilisi nullam vehicula. Hendrerit gravida rutrum quisque non tellus. Diam vulputate ut pharetra sit. Semper feugiat nibh sed pulvinar proin gravida hendrerit lectus a. Tempus iaculis urna id volutpat lacus laoreet non curabitur gravida. Porttitor lacus luctus accumsan tortor posuere ac ut consequat semper. Tortor vitae purus faucibus ornare. Risus viverra adipiscing at in tellus integer. Suspendisse potenti nullam ac tortor vitae purus faucibus ornare suspendisse. Facilisis sed odio morbi quis commodo odio aenean sed. At auctor urna nunc id. Ac tortor vitae purus faucibus ornare suspendisse sed nisi. Suspendisse interdum consectetur libero id faucibus nisl. Tellus id interdum velit laoreet id. Mus mauris vitae ultricies leo integer. Metus vulputate eu scelerisque felis imperdiet proin. + +Venenatis cras sed felis eget velit aliquet sagittis id. Turpis cursus in hac habitasse. Magna fringilla urna porttitor rhoncus dolor purus non. In tellus integer feugiat scelerisque varius morbi. Tortor consequat id porta nibh venenatis cras sed. Ut sem viverra aliquet eget sit amet tellus cras. Semper risus in hendrerit gravida. Libero enim sed faucibus turpis in. Ultricies leo integer malesuada nunc vel risus commodo. Ipsum dolor sit amet consectetur adipiscing elit pellentesque habitant. Varius duis at consectetur lorem donec massa sapien faucibus et. Ornare arcu dui vivamus arcu felis bibendum ut tristique et. + +Diam quis enim lobortis scelerisque fermentum dui faucibus in. Cursus mattis molestie a iaculis at erat. Diam sit amet nisl suscipit adipiscing. Ultrices dui sapien eget mi proin sed libero. Purus ut faucibus pulvinar elementum integer enim neque. Ultricies mi quis hendrerit dolor magna eget. Morbi tincidunt ornare massa eget. Mauris cursus mattis molestie a iaculis at erat pellentesque. Phasellus vestibulum lorem sed risus. Sodales ut etiam sit amet nisl purus in. Habitant morbi tristique senectus et netus et. Consectetur lorem donec massa sapien faucibus et molestie. + +Montes nascetur ridiculus mus mauris vitae ultricies. Lectus urna duis convallis convallis. Pulvinar pr. \ No newline at end of file diff --git a/performance_tests/test/resources/plaintext/plaintext-data-small.dat b/performance_tests/test/resources/plaintext/plaintext-data-small.dat new file mode 100644 index 000000000..b8475e61f --- /dev/null +++ b/performance_tests/test/resources/plaintext/plaintext-data-small.dat @@ -0,0 +1 @@ +Lorem ipsum dolor sit amet, consect. \ No newline at end of file diff --git a/performance_tests/tox.ini b/performance_tests/tox.ini new file mode 100644 index 000000000..1b7d073aa --- /dev/null +++ b/performance_tests/tox.ini @@ -0,0 +1,215 @@ +[tox] +envlist = + # The performance tests only work for python 3.11 and 3.12 + py{311,312}-performance_tests-mpl + bandit, doc8 + ; {flake8, pylint}{,-tests}, + isort-check, black-check, + # prone to false positives + vulture + +# Additional test environments: +# +# linters :: Runs all linters over all source code. +# linters-tests :: Runs all linters over all tests. + +# Autoformatter helper environments: +# +# autoformat : Apply all autoformatters +# +# black-check : Check for "black" issues +# blacken : Fix all "black" issues +# +# isort-seed : Generate a known_third_party list for isort. +# NOTE: make the "known_third_party = " line in setup.cfg before running this +# NOTE: currently it incorrectly identifies this library too; make sure you remove it +# isort-check : Check for isort issues +# isort : Fix isort issues + +# Operational helper environments: +# +# build :: Builds source and wheel dist files. +# test-release :: Builds dist files and uploads to testpypi pypirc profile. +# release :: Builds dist files and uploads to pypi pypirc profile. + +[testenv:base-command] +commands = pytest test/ +deps = + click + + +[testenv] +passenv = + # Pass through AWS credentials + AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN \ + # AWS Role access in CodeBuild is via the contaner URI + AWS_CONTAINER_CREDENTIALS_RELATIVE_URI \ + # Pass through AWS profile name (useful for local testing) + AWS_PROFILE +sitepackages = False +deps = + -rrequirements.txt + # Install the MPL requirements if the `-mpl` suffix is present + mpl: -rrequirements_mpl.txt + .. +commands = + performance_tests: {[testenv:base-command]commands} + +[testenv:blacken-src] +basepython = python3 +deps = -r../dev_requirements/linter-requirements.txt +commands = + black --line-length 120 \ + src/aws_encryption_sdk_performance_tests/ \ + setup.py \ + test/ \ + {posargs} + +# Linters +[testenv:flake8] +basepython = python3 +deps = -r../dev_requirements/linter-requirements.txt +commands = + flake8 \ + src/aws_encryption_sdk_performance_tests/ \ + setup.py \ + {posargs} + +[testenv:flake8-tests] +basepython = {[testenv:flake8]basepython} +deps = -r../dev_requirements/linter-requirements.txt +commands = + flake8 \ + # Ignore F811 redefinition errors in tests (breaks with pytest-mock use) + # E203 is not PEP8 compliant https://github.com/ambv/black#slices + # W503 is not PEP8 compliant https://github.com/ambv/black#line-breaks--binary-operators + --ignore F811,E203,W503,D \ + test/ + +[testenv:pylint] +basepython = python3 +deps = + -r../dev_requirements/linter-requirements.txt +commands = + pylint \ + --rcfile=pylintrc \ + src/aws_encryption_sdk_performance_tests/ \ + setup.py \ + {posargs} + +[testenv:pylint-tests] +basepython = {[testenv:pylint]basepython} +deps = {[testenv:pylint]deps} +commands = + pylint \ + --rcfile=pylintrc \ + test/ \ + {posargs} + +[testenv:blacken] +basepython = python3 +deps = + {[testenv:blacken-src]deps} +commands = + {[testenv:blacken-src]commands} + +[testenv:black-check] +basepython = python3 +deps = + {[testenv:blacken]deps} +commands = + {[testenv:blacken-src]commands} --diff + +[testenv:isort-seed] +basepython = python3 +deps = -r../dev_requirements/linter-requirements.txt +commands = seed-isort-config + +[testenv:isort] +basepython = python3 +deps = -r../dev_requirements/linter-requirements.txt +commands = isort -rc \ + src \ + test \ + setup.py \ + {posargs} + +[testenv:isort-check] +basepython = python3 +deps = {[testenv:isort]deps} +commands = {[testenv:isort]commands} -c + +[testenv:autoformat] +basepython = python3 +deps = + {[testenv:blacken]deps} + {[testenv:isort]deps} + .. +commands = + {[testenv:blacken]commands} + {[testenv:isort]commands} + +[testenv:doc8] +basepython = python3 +deps = -r../dev_requirements/linter-requirements.txt +commands = doc8 README.rst + +[testenv:readme] +basepython = python3 +deps = -r../dev_requirements/linter-requirements.txt +commands = python setup.py check -r -s + +[testenv:bandit] +basepython = python3 +deps = -r../dev_requirements/linter-requirements.txt +commands = bandit -r src/aws_encryption_sdk_performance_tests/ + +[testenv:linters] +basepython = python3 +deps = + {[testenv:flake8]deps} + {[testenv:pylint]deps} + {[testenv:doc8]deps} + {[testenv:readme]deps} + {[testenv:bandit]deps} +commands = + {[testenv:flake8]commands} + {[testenv:pylint]commands} + {[testenv:doc8]commands} + {[testenv:readme]commands} + {[testenv:bandit]commands} + +# Release tooling +[testenv:park] +basepython = python3 +skip_install = true +deps = -r../dev_requirements/release-requirements.txt +commands = python setup.py park + +[testenv:build] +basepython = python3 +skip_install = true +deps = + -r../dev_requirements/release-requirements.txt +commands = + python setup.py sdist bdist_wheel + +[testenv:test-release] +basepython = python3 +skip_install = true +deps = + {[testenv:build]deps} + twine +commands = + {[testenv:build]commands} + twine upload --skip-existing --repository testpypi dist/* + +[testenv:release] +basepython = python3 +skip_install = true +deps = + {[testenv:build]deps} + twine +commands = + {[testenv:build]commands} + twine upload --skip-existing --repository pypi dist/* From dee26cd7ec9bcb2c9faa1a6793c475d8dd5a01cf Mon Sep 17 00:00:00 2001 From: Ritvik Kapila <61410899+RitvikKapila@users.noreply.github.com> Date: Fri, 7 Jun 2024 13:57:34 -0700 Subject: [PATCH 417/422] chore(performance_tests): added hierarchy keyring and caching cmm tests (#686) --- .../keyrings/hierarchy_keyring.py | 128 ++++++++++++ .../keyrings/raw_aes_keyring.py | 1 - .../aws_kms_master_key_provider.py | 8 +- .../master_key_providers/caching_cmm.py | 85 ++++++++ .../raw_aes_master_key_provider.py | 10 +- .../raw_rsa_master_key_provider.py | 10 +- .../utils/util.py | 4 +- .../test/keyrings/test_hierarchy_keyring.py | 174 +++++++++++++++++ .../test_aws_kms_master_key_provider.py | 2 +- .../master_key_providers/test_caching_cmm.py | 183 ++++++++++++++++++ .../test_raw_aes_master_key_provider.py | 2 +- .../test_raw_rsa_master_key_provider.py | 2 +- .../caching_cmm/ciphertext-data-empty.ct | Bin 0 -> 587 bytes .../caching_cmm/ciphertext-data-large.ct | Bin 0 -> 8619 bytes .../caching_cmm/ciphertext-data-medium.ct | Bin 0 -> 4587 bytes .../caching_cmm/ciphertext-data-small.ct | Bin 0 -> 623 bytes .../hierarchy/ciphertext-data-empty.ct | Bin 0 -> 654 bytes .../hierarchy/ciphertext-data-large.ct | Bin 0 -> 8686 bytes .../hierarchy/ciphertext-data-medium.ct | Bin 0 -> 4654 bytes .../hierarchy/ciphertext-data-small.ct | Bin 0 -> 690 bytes 20 files changed, 588 insertions(+), 21 deletions(-) create mode 100644 performance_tests/src/aws_encryption_sdk_performance_tests/keyrings/hierarchy_keyring.py create mode 100644 performance_tests/src/aws_encryption_sdk_performance_tests/master_key_providers/caching_cmm.py create mode 100644 performance_tests/test/keyrings/test_hierarchy_keyring.py create mode 100644 performance_tests/test/master_key_providers/test_caching_cmm.py create mode 100644 performance_tests/test/resources/ciphertext/caching_cmm/ciphertext-data-empty.ct create mode 100644 performance_tests/test/resources/ciphertext/caching_cmm/ciphertext-data-large.ct create mode 100644 performance_tests/test/resources/ciphertext/caching_cmm/ciphertext-data-medium.ct create mode 100644 performance_tests/test/resources/ciphertext/caching_cmm/ciphertext-data-small.ct create mode 100644 performance_tests/test/resources/ciphertext/hierarchy/ciphertext-data-empty.ct create mode 100644 performance_tests/test/resources/ciphertext/hierarchy/ciphertext-data-large.ct create mode 100644 performance_tests/test/resources/ciphertext/hierarchy/ciphertext-data-medium.ct create mode 100644 performance_tests/test/resources/ciphertext/hierarchy/ciphertext-data-small.ct diff --git a/performance_tests/src/aws_encryption_sdk_performance_tests/keyrings/hierarchy_keyring.py b/performance_tests/src/aws_encryption_sdk_performance_tests/keyrings/hierarchy_keyring.py new file mode 100644 index 000000000..b1bdc6913 --- /dev/null +++ b/performance_tests/src/aws_encryption_sdk_performance_tests/keyrings/hierarchy_keyring.py @@ -0,0 +1,128 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Performance tests for the hierarchy keyring.""" + +import aws_encryption_sdk +import boto3 +from aws_cryptographic_materialproviders.keystore import KeyStore +from aws_cryptographic_materialproviders.keystore.config import KeyStoreConfig +from aws_cryptographic_materialproviders.keystore.models import KMSConfigurationKmsKeyArn +from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders +from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig +from aws_cryptographic_materialproviders.mpl.models import ( + CacheTypeDefault, + CreateAwsKmsHierarchicalKeyringInput, + DefaultCache, +) +from aws_cryptographic_materialproviders.mpl.references import IKeyring + +from ..utils.util import PerfTestUtils + + +def create_keyring( + key_store_table_name: str, + logical_key_store_name: str, + kms_key_id: str, + branch_key_id: str = PerfTestUtils.DEFAULT_BRANCH_KEY_ID +): + """Demonstrate how to create a hierarchy keyring. + + Usage: create_keyring(key_store_table_name, logical_key_store_name, kms_key_id, branch_key_id) + :param key_store_table_name: Name of the KeyStore DynamoDB table. + :type key_store_table_name: string + :param logical_key_store_name: Logical name of the KeyStore. + :type logical_key_store_name: string + :param kms_key_id: KMS Key identifier for the KMS key you want to use. + :type kms_key_id: string + :param branch_key_id: Branch key you want to use for the hierarchy keyring. + :type branch_key_id: string + + For more information on KMS Key identifiers, see + https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#key-id + """ + # Create boto3 clients for DynamoDB and KMS. + ddb_client = boto3.client('dynamodb', region_name="us-west-2") + kms_client = boto3.client('kms', region_name="us-west-2") + + # Configure your KeyStore resource. + # This SHOULD be the same configuration that you used + # to initially create and populate your KeyStore. + keystore: KeyStore = KeyStore( + config=KeyStoreConfig( + ddb_client=ddb_client, + ddb_table_name=key_store_table_name, + logical_key_store_name=logical_key_store_name, + kms_client=kms_client, + kms_configuration=KMSConfigurationKmsKeyArn( + value=kms_key_id + ), + ) + ) + + # Create the Hierarchical Keyring. + mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + config=MaterialProvidersConfig() + ) + + keyring_input: CreateAwsKmsHierarchicalKeyringInput = CreateAwsKmsHierarchicalKeyringInput( + key_store=keystore, + branch_key_id=branch_key_id, + ttl_seconds=600, + cache=CacheTypeDefault( + value=DefaultCache( + entry_capacity=100 + ) + ), + ) + + keyring: IKeyring = mat_prov.create_aws_kms_hierarchical_keyring( + input=keyring_input + ) + + return keyring + + +def encrypt_using_keyring( + plaintext_data: bytes, + keyring: IKeyring +): + """Demonstrate how to encrypt plaintext data using a hierarchy keyring. + + Usage: encrypt_using_keyring(plaintext_data, keyring) + :param plaintext_data: plaintext data you want to encrypt + :type: bytes + :param keyring: Keyring to use for encryption. + :type keyring: IKeyring + """ + client = aws_encryption_sdk.EncryptionSDKClient() + + ciphertext_data, _ = client.encrypt( + source=plaintext_data, + keyring=keyring, + encryption_context=PerfTestUtils.DEFAULT_ENCRYPTION_CONTEXT + ) + + return ciphertext_data + + +def decrypt_using_keyring( + ciphertext_data: bytes, + keyring: IKeyring +): + """Demonstrate how to decrypt ciphertext data using a hierarchy keyring. + + Usage: decrypt_using_keyring(ciphertext_data, keyring) + :param ciphertext_data: ciphertext data you want to decrypt + :type: bytes + :param keyring: Keyring to use for decryption. + :type keyring: IKeyring + """ + client = aws_encryption_sdk.EncryptionSDKClient() + + decrypted_plaintext_data, _ = client.decrypt( + source=ciphertext_data, + keyring=keyring, + encryption_context=PerfTestUtils.DEFAULT_ENCRYPTION_CONTEXT + ) + + return decrypted_plaintext_data diff --git a/performance_tests/src/aws_encryption_sdk_performance_tests/keyrings/raw_aes_keyring.py b/performance_tests/src/aws_encryption_sdk_performance_tests/keyrings/raw_aes_keyring.py index a849b1a7f..a2a3c9ab1 100644 --- a/performance_tests/src/aws_encryption_sdk_performance_tests/keyrings/raw_aes_keyring.py +++ b/performance_tests/src/aws_encryption_sdk_performance_tests/keyrings/raw_aes_keyring.py @@ -19,7 +19,6 @@ def create_keyring(): key_name_space = "Some managed raw keys" key_name = "My 256-bit AES wrapping key" - # Here, the input to secrets.token_bytes() = 32 bytes = 256 bits # We fix the static key in order to make the test deterministic static_key = PerfTestUtils.DEFAULT_AES_256_STATIC_KEY diff --git a/performance_tests/src/aws_encryption_sdk_performance_tests/master_key_providers/aws_kms_master_key_provider.py b/performance_tests/src/aws_encryption_sdk_performance_tests/master_key_providers/aws_kms_master_key_provider.py index c3136a5c7..023cd5942 100644 --- a/performance_tests/src/aws_encryption_sdk_performance_tests/master_key_providers/aws_kms_master_key_provider.py +++ b/performance_tests/src/aws_encryption_sdk_performance_tests/master_key_providers/aws_kms_master_key_provider.py @@ -8,7 +8,7 @@ def create_key_provider( kms_key_id: str ): - """Demonstrate how to create an AWS KMS master key-provider. + """Demonstrate how to create an AWS KMS master key provider. Usage: create_key_provider(kms_key_id) :param kms_key_id: KMS Key identifier for the KMS key you want to use. @@ -17,7 +17,7 @@ def create_key_provider( For more information on KMS Key identifiers, see https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#key-id """ - # Create a KMS master key-provider. + # Create a KMS master key provider. key_provider = aws_encryption_sdk.StrictAwsKmsMasterKeyProvider(key_ids=[ kms_key_id, ]) @@ -29,7 +29,7 @@ def encrypt_using_key_provider( plaintext_data: bytes, key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider ): - """Demonstrate how to encrypt plaintext data using an AWS KMS master key-provider. + """Demonstrate how to encrypt plaintext data using an AWS KMS master key provider. Usage: encrypt_using_key_provider(plaintext_data, key_provider) :param plaintext_data: plaintext data you want to encrypt @@ -51,7 +51,7 @@ def decrypt_using_key_provider( ciphertext_data: bytes, key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider ): - """Demonstrate how to decrypt ciphertext data using an AWS KMS master key-provider. + """Demonstrate how to decrypt ciphertext data using an AWS KMS master key provider. Usage: decrypt_using_key_provider(ciphertext_data, key_provider) :param ciphertext_data: ciphertext data you want to decrypt diff --git a/performance_tests/src/aws_encryption_sdk_performance_tests/master_key_providers/caching_cmm.py b/performance_tests/src/aws_encryption_sdk_performance_tests/master_key_providers/caching_cmm.py new file mode 100644 index 000000000..7199c50bf --- /dev/null +++ b/performance_tests/src/aws_encryption_sdk_performance_tests/master_key_providers/caching_cmm.py @@ -0,0 +1,85 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Performance tests for the Caching Cryptographic Materials Manager (CMM) with KMS Master Key Provider.""" + +import aws_encryption_sdk + + +def create_cmm( + kms_key_id: str, + max_age_in_cache: float, + cache_capacity: int +): + """Demonstrate how to create a Caching CMM. + + Usage: create_cmm(kms_key_id, max_age_in_cache, cache_capacity) + :param kms_key_id: Amazon Resource Name (ARN) of the KMS customer master key + :type kms_key_id: str + :param max_age_in_cache: Maximum time in seconds that a cached entry can be used + :type max_age_in_cache: float + :param cache_capacity: Maximum number of entries to retain in cache at once + :type cache_capacity: int + """ + # Security thresholds + # Max messages (or max bytes per) data key are optional + max_messages_encrypted = 100 + + # Create a master key provider for the KMS customer master key (CMK) + key_provider = aws_encryption_sdk.StrictAwsKmsMasterKeyProvider(key_ids=[kms_key_id]) + + # Create a local cache + cache = aws_encryption_sdk.LocalCryptoMaterialsCache(cache_capacity) + + # Create a caching CMM + caching_cmm = aws_encryption_sdk.CachingCryptoMaterialsManager( + master_key_provider=key_provider, + cache=cache, + max_age=max_age_in_cache, + max_messages_encrypted=max_messages_encrypted, + ) + + return caching_cmm + + +def encrypt_using_cmm( + plaintext_data: bytes, + caching_cmm: aws_encryption_sdk.materials_managers.base.CryptoMaterialsManager +): + """Demonstrate how to encrypt plaintext data using a Caching CMM. + + Usage: encrypt_using_cmm(plaintext_data, caching_cmm) + :param plaintext_data: plaintext data you want to encrypt + :type: bytes + :param caching_cmm: Crypto Materials Manager to use for encryption. + :type caching_cmm: aws_encryption_sdk.materials_managers.base.CryptoMaterialsManager + """ + client = aws_encryption_sdk.EncryptionSDKClient() + + ciphertext_data, _ = client.encrypt( + source=plaintext_data, + materials_manager=caching_cmm + ) + + return ciphertext_data + + +def decrypt_using_cmm( + ciphertext_data: bytes, + caching_cmm: aws_encryption_sdk.materials_managers.base.CryptoMaterialsManager +): + """Demonstrate how to decrypt ciphertext data using a Caching CMM. + + Usage: decrypt_using_cmm(ciphertext_data, caching_cmm) + :param ciphertext_data: ciphertext data you want to decrypt + :type: bytes + :param caching_cmm: Crypto Materials Manager to use for encryption. + :type caching_cmm: aws_encryption_sdk.materials_managers.base.CryptoMaterialsManager + """ + client = aws_encryption_sdk.EncryptionSDKClient() + + decrypted_plaintext_data, _ = client.decrypt( + source=ciphertext_data, + materials_manager=caching_cmm + ) + + return decrypted_plaintext_data diff --git a/performance_tests/src/aws_encryption_sdk_performance_tests/master_key_providers/raw_aes_master_key_provider.py b/performance_tests/src/aws_encryption_sdk_performance_tests/master_key_providers/raw_aes_master_key_provider.py index 42d071dcf..de0188c5e 100644 --- a/performance_tests/src/aws_encryption_sdk_performance_tests/master_key_providers/raw_aes_master_key_provider.py +++ b/performance_tests/src/aws_encryption_sdk_performance_tests/master_key_providers/raw_aes_master_key_provider.py @@ -43,13 +43,13 @@ def _get_raw_key(self, key_id): def create_key_provider(): - """Demonstrate how to create a Raw AES master key-provider. + """Demonstrate how to create a Raw AES master key provider. Usage: create_key_provider() """ - # Create a Raw AES master key-provider. + # Create a Raw AES master key provider. - # The Key ID field in the JceMasterKey and RawMasterKey is equivalent to key name in the Raw keyrings + # The Key ID field in the JceMasterKey and RawMasterKey is equivalent to key name in the Raw keyrings key_id = "My 256-bit AES wrapping key" key_provider = StaticRandomMasterKeyProvider() key_provider.add_master_key(key_id) @@ -61,7 +61,7 @@ def encrypt_using_key_provider( plaintext_data: bytes, key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider ): - """Demonstrate how to encrypt plaintext data using a Raw AES master key-provider. + """Demonstrate how to encrypt plaintext data using a Raw AES master key provider. Usage: encrypt_using_key_provider(plaintext_data, key_provider) :param plaintext_data: plaintext data you want to encrypt @@ -83,7 +83,7 @@ def decrypt_using_key_provider( ciphertext_data: bytes, key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider ): - """Demonstrate how to decrypt ciphertext data using a Raw AES master key-provider. + """Demonstrate how to decrypt ciphertext data using a Raw AES master key provider. Usage: decrypt_using_key_provider(ciphertext_data, key_provider) :param ciphertext_data: ciphertext data you want to decrypt diff --git a/performance_tests/src/aws_encryption_sdk_performance_tests/master_key_providers/raw_rsa_master_key_provider.py b/performance_tests/src/aws_encryption_sdk_performance_tests/master_key_providers/raw_rsa_master_key_provider.py index b52b78735..cdbe24110 100644 --- a/performance_tests/src/aws_encryption_sdk_performance_tests/master_key_providers/raw_rsa_master_key_provider.py +++ b/performance_tests/src/aws_encryption_sdk_performance_tests/master_key_providers/raw_rsa_master_key_provider.py @@ -43,13 +43,13 @@ def _get_raw_key(self, key_id): def create_key_provider(): - """Demonstrate how to create a Raw RSA master key-provider. + """Demonstrate how to create a Raw RSA master key provider. Usage: create_key_provider() """ - # Create a Raw RSA master key-provider. + # Create a Raw RSA master key provider. - # The Key ID field in the JceMasterKey and RawMasterKey is equivalent to key name in the Raw keyrings + # The Key ID field in the JceMasterKey and RawMasterKey is equivalent to key name in the Raw keyrings key_id = "My 4096-bit RSA wrapping key" key_provider = StaticRandomMasterKeyProvider() key_provider.add_master_key(key_id) @@ -61,7 +61,7 @@ def encrypt_using_key_provider( plaintext_data: bytes, key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider ): - """Demonstrate how to encrypt plaintext data using a Raw RSA master key-provider. + """Demonstrate how to encrypt plaintext data using a Raw RSA master key provider. Usage: encrypt_using_key_provider(plaintext_data, key_provider) :param plaintext_data: plaintext data you want to encrypt @@ -83,7 +83,7 @@ def decrypt_using_key_provider( ciphertext_data: bytes, key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider ): - """Demonstrate how to decrypt ciphertext data using a Raw RSA master key-provider. + """Demonstrate how to decrypt ciphertext data using a Raw RSA master key provider. Usage: decrypt_using_key_provider(ciphertext_data, key_provider) :param ciphertext_data: ciphertext data you want to decrypt diff --git a/performance_tests/src/aws_encryption_sdk_performance_tests/utils/util.py b/performance_tests/src/aws_encryption_sdk_performance_tests/utils/util.py index 52914b76a..9100ef12e 100644 --- a/performance_tests/src/aws_encryption_sdk_performance_tests/utils/util.py +++ b/performance_tests/src/aws_encryption_sdk_performance_tests/utils/util.py @@ -87,9 +87,7 @@ class PerfTestUtils: "the data you are handling": "is what you think it is", } - DEFAULT_BRANCH_KEY_ID_A = 'a52dfaad-7dbd-4430-a1fd-abaa5299da07' - - DEFAULT_BRANCH_KEY_ID_B = '8ba79cef-581c-4125-9292-b057a29d42d7' + DEFAULT_BRANCH_KEY_ID = 'a52dfaad-7dbd-4430-a1fd-abaa5299da07' @staticmethod def read_file(filename): diff --git a/performance_tests/test/keyrings/test_hierarchy_keyring.py b/performance_tests/test/keyrings/test_hierarchy_keyring.py new file mode 100644 index 000000000..e890c2a18 --- /dev/null +++ b/performance_tests/test/keyrings/test_hierarchy_keyring.py @@ -0,0 +1,174 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""This is a performance test for creating the hierarchy keyring.""" + +import os +import time + +import click +import click.testing +import pytest +from tqdm import tqdm + +from aws_encryption_sdk_performance_tests.keyrings.hierarchy_keyring import ( + create_keyring, + decrypt_using_keyring, + encrypt_using_keyring, +) +from aws_encryption_sdk_performance_tests.utils.util import PerfTestUtils + +MODULE_ABS_PATH = os.path.abspath(__file__) + + +@click.group() +def create_hierarchy_keyring(): + """Click group helper function""" + + +@create_hierarchy_keyring.command() +@click.option('--key_store_table_name', + default='KeyStoreDdbTable') +@click.option('--kms_key_id', + default='arn:aws:kms:us-west-2:370957321024:key/9d989aa2-2f9c-438c-a745-cc57d3ad0126') +@click.option('--n_iters', + default=PerfTestUtils.DEFAULT_N_ITERS) +@click.option('--output_file', + default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/hierarchy_keyring_create') +def create( + key_store_table_name: str, + kms_key_id: str, + n_iters: int, + output_file: str +): + """Performance test for the create_keyring function.""" + time_list = [] + for _ in tqdm(range(n_iters)): + curr_time = time.time() + + create_keyring(key_store_table_name, key_store_table_name, kms_key_id) + + # calculate elapsed time in milliseconds + elapsed_time = (time.time() - curr_time) * 1000 + time_list.append(elapsed_time) + + PerfTestUtils.write_time_list_to_csv(time_list, output_file) + + +@click.group() +def encrypt_hierarchy_keyring(): + """Click group helper function""" + + +@encrypt_hierarchy_keyring.command() +@click.option('--plaintext_data_filename', + default='/'.join(MODULE_ABS_PATH.split("/")[:-2]) + '/resources/plaintext/plaintext-data-' + + PerfTestUtils.DEFAULT_FILE_SIZE + '.dat') +@click.option('--key_store_table_name', + default='KeyStoreDdbTable') +@click.option('--kms_key_id', + default='arn:aws:kms:us-west-2:370957321024:key/9d989aa2-2f9c-438c-a745-cc57d3ad0126') +@click.option('--n_iters', + default=PerfTestUtils.DEFAULT_N_ITERS) +@click.option('--output_file', + default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/hierarchy_keyring_encrypt') +def encrypt( + plaintext_data_filename: str, + key_store_table_name: str, + kms_key_id: str, + n_iters: int, + output_file: str +): + """Performance test for the encrypt_using_keyring function.""" + plaintext_data = PerfTestUtils.read_file(plaintext_data_filename) + + keyring = create_keyring(key_store_table_name, key_store_table_name, kms_key_id) + time_list = [] + + for _ in tqdm(range(n_iters)): + curr_time = time.time() + + encrypt_using_keyring(plaintext_data, keyring) + + # calculate elapsed time in milliseconds + elapsed_time = (time.time() - curr_time) * 1000 + time_list.append(elapsed_time) + + PerfTestUtils.write_time_list_to_csv(time_list, output_file) + + +@click.group() +def decrypt_hierarchy_keyring(): + """Click group helper function""" + + +@decrypt_hierarchy_keyring.command() +@click.option('--ciphertext_data_filename', + default='/'.join(MODULE_ABS_PATH.split("/")[:-2]) + '/resources/ciphertext/hierarchy/ciphertext-data-' + + PerfTestUtils.DEFAULT_FILE_SIZE + '.ct') +@click.option('--key_store_table_name', + default='KeyStoreDdbTable') +@click.option('--kms_key_id', + default='arn:aws:kms:us-west-2:370957321024:key/9d989aa2-2f9c-438c-a745-cc57d3ad0126') +@click.option('--n_iters', + default=PerfTestUtils.DEFAULT_N_ITERS) +@click.option('--output_file', + default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/hierarchy_keyring_decrypt') +def decrypt( + ciphertext_data_filename: str, + key_store_table_name: str, + kms_key_id: str, + n_iters: int, + output_file: str +): + """Performance test for the decrypt_using_keyring function.""" + ciphertext_data = PerfTestUtils.read_file(ciphertext_data_filename) + + keyring = create_keyring(key_store_table_name, key_store_table_name, kms_key_id) + time_list = [] + + for _ in tqdm(range(n_iters)): + curr_time = time.time() + + decrypt_using_keyring(ciphertext_data, keyring) + + # calculate elapsed time in milliseconds + elapsed_time = (time.time() - curr_time) * 1000 + time_list.append(elapsed_time) + + PerfTestUtils.write_time_list_to_csv(time_list, output_file) + + +hierarchy_keyring_test = click.CommandCollection(sources=[create_hierarchy_keyring, + encrypt_hierarchy_keyring, + decrypt_hierarchy_keyring]) + + +@pytest.fixture +def runner(): + """Click runner""" + return click.testing.CliRunner() + + +def test_create(runner): + """Test the create_keyring function""" + result = runner.invoke(create_hierarchy_keyring.commands['create'], + ['--n_iters', PerfTestUtils.DEFAULT_TESTING_N_ITERS]) + assert result.exit_code == 0 + + +def test_encrypt(runner): + """Test the encrypt_using_keyring function""" + result = runner.invoke(encrypt_hierarchy_keyring.commands['encrypt'], + ['--n_iters', PerfTestUtils.DEFAULT_TESTING_N_ITERS]) + assert result.exit_code == 0 + + +def test_decrypt(runner): + """Test the decrypt_using_keyring function""" + result = runner.invoke(decrypt_hierarchy_keyring.commands['decrypt'], + ['--n_iters', PerfTestUtils.DEFAULT_TESTING_N_ITERS]) + assert result.exit_code == 0 + + +if __name__ == "__main__": + hierarchy_keyring_test() diff --git a/performance_tests/test/master_key_providers/test_aws_kms_master_key_provider.py b/performance_tests/test/master_key_providers/test_aws_kms_master_key_provider.py index b869245b5..c7b665857 100644 --- a/performance_tests/test/master_key_providers/test_aws_kms_master_key_provider.py +++ b/performance_tests/test/master_key_providers/test_aws_kms_master_key_provider.py @@ -1,6 +1,6 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""This is a performance test for creating the AWS KMS Master key-provider.""" +"""This is a performance test for creating the AWS KMS Master key provider.""" import os import time diff --git a/performance_tests/test/master_key_providers/test_caching_cmm.py b/performance_tests/test/master_key_providers/test_caching_cmm.py new file mode 100644 index 000000000..f8552f96e --- /dev/null +++ b/performance_tests/test/master_key_providers/test_caching_cmm.py @@ -0,0 +1,183 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""This is a performance test for creating a Caching CMM.""" + +import os +import time + +import click +import click.testing +import pytest +from tqdm import tqdm + +from aws_encryption_sdk_performance_tests.master_key_providers.caching_cmm import ( + create_cmm, + decrypt_using_cmm, + encrypt_using_cmm, +) +from aws_encryption_sdk_performance_tests.utils.util import PerfTestUtils + +MODULE_ABS_PATH = os.path.abspath(__file__) + + +@click.group() +def create_caching_cmm(): + """Click group helper function""" + + +@create_caching_cmm.command() +@click.option('--kms_key_id', + default='arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f') +@click.option('--max_age_in_cache', + default=10.0) +@click.option('--cache_capacity', + default=10) +@click.option('--n_iters', + default=PerfTestUtils.DEFAULT_N_ITERS) +@click.option('--output_file', + default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/caching_cmm_create') +def create( + kms_key_id: str, + max_age_in_cache: float, + cache_capacity: int, + n_iters: int, + output_file: str +): + """Performance test for the create_cmm function.""" + time_list = [] + for _ in tqdm(range(n_iters)): + curr_time = time.time() + + create_cmm(kms_key_id, max_age_in_cache, cache_capacity) + + # calculate elapsed time in milliseconds + elapsed_time = (time.time() - curr_time) * 1000 + time_list.append(elapsed_time) + + PerfTestUtils.write_time_list_to_csv(time_list, output_file) + + +@click.group() +def encrypt_caching_cmm(): + """Click group helper function""" + + +@encrypt_caching_cmm.command() +@click.option('--plaintext_data_filename', + default='/'.join(MODULE_ABS_PATH.split("/")[:-2]) + '/resources/plaintext/plaintext-data-' + + PerfTestUtils.DEFAULT_FILE_SIZE + '.dat') +@click.option('--kms_key_id', + default='arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f') +@click.option('--max_age_in_cache', + default=10.0) +@click.option('--cache_capacity', + default=10) +@click.option('--n_iters', + default=PerfTestUtils.DEFAULT_N_ITERS) +@click.option('--output_file', + default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/caching_cmm_encrypt') +def encrypt( + plaintext_data_filename: str, + kms_key_id: str, + max_age_in_cache: float, + cache_capacity: int, + n_iters: int, + output_file: str +): + """Performance test for the encrypt_using_cmm function.""" + plaintext_data = PerfTestUtils.read_file(plaintext_data_filename) + + caching_cmm = create_cmm(kms_key_id, max_age_in_cache, cache_capacity) + time_list = [] + + for _ in tqdm(range(n_iters)): + curr_time = time.time() + + encrypt_using_cmm(plaintext_data, caching_cmm) + + # calculate elapsed time in milliseconds + elapsed_time = (time.time() - curr_time) * 1000 + time_list.append(elapsed_time) + + PerfTestUtils.write_time_list_to_csv(time_list, output_file) + + +@click.group() +def decrypt_caching_cmm(): + """Click group helper function""" + + +@decrypt_caching_cmm.command() +@click.option('--ciphertext_data_filename', + default='/'.join(MODULE_ABS_PATH.split("/")[:-2]) + '/resources/ciphertext/caching_cmm/ciphertext-data-' + + PerfTestUtils.DEFAULT_FILE_SIZE + '.ct') +@click.option('--kms_key_id', + default='arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f') +@click.option('--max_age_in_cache', + default=10.0) +@click.option('--cache_capacity', + default=10) +@click.option('--n_iters', + default=PerfTestUtils.DEFAULT_N_ITERS) +@click.option('--output_file', + default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/caching_cmm_decrypt') +def decrypt( + ciphertext_data_filename: str, + kms_key_id: str, + max_age_in_cache: float, + cache_capacity: int, + n_iters: int, + output_file: str +): + """Performance test for the decrypt_using_cmm function.""" + ciphertext_data = PerfTestUtils.read_file(ciphertext_data_filename) + + caching_cmm = create_cmm(kms_key_id, max_age_in_cache, cache_capacity) + time_list = [] + + for _ in tqdm(range(n_iters)): + curr_time = time.time() + + decrypt_using_cmm(ciphertext_data, caching_cmm) + + # calculate elapsed time in milliseconds + elapsed_time = (time.time() - curr_time) * 1000 + time_list.append(elapsed_time) + + PerfTestUtils.write_time_list_to_csv(time_list, output_file) + + +caching_cmm_test = click.CommandCollection(sources=[create_caching_cmm, + encrypt_caching_cmm, + decrypt_caching_cmm]) + + +@pytest.fixture +def runner(): + """Click runner""" + return click.testing.CliRunner() + + +def test_create(runner): + """Test the create_cmm function""" + result = runner.invoke(create_caching_cmm.commands['create'], + ['--n_iters', PerfTestUtils.DEFAULT_TESTING_N_ITERS]) + assert result.exit_code == 0 + + +def test_encrypt(runner): + """Test the encrypt_using_cmm function""" + result = runner.invoke(encrypt_caching_cmm.commands['encrypt'], + ['--n_iters', PerfTestUtils.DEFAULT_TESTING_N_ITERS]) + assert result.exit_code == 0 + + +def test_decrypt(runner): + """Test the decrypt_using_cmm function""" + result = runner.invoke(decrypt_caching_cmm.commands['decrypt'], + ['--n_iters', PerfTestUtils.DEFAULT_TESTING_N_ITERS]) + assert result.exit_code == 0 + + +if __name__ == "__main__": + caching_cmm_test() diff --git a/performance_tests/test/master_key_providers/test_raw_aes_master_key_provider.py b/performance_tests/test/master_key_providers/test_raw_aes_master_key_provider.py index 375eca2ef..cf39963bf 100644 --- a/performance_tests/test/master_key_providers/test_raw_aes_master_key_provider.py +++ b/performance_tests/test/master_key_providers/test_raw_aes_master_key_provider.py @@ -1,6 +1,6 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""This is a performance test for creating the Raw AES Master key-provider.""" +"""This is a performance test for creating the Raw AES Master key provider.""" import os import time diff --git a/performance_tests/test/master_key_providers/test_raw_rsa_master_key_provider.py b/performance_tests/test/master_key_providers/test_raw_rsa_master_key_provider.py index 5d6db861a..63b00a0e3 100644 --- a/performance_tests/test/master_key_providers/test_raw_rsa_master_key_provider.py +++ b/performance_tests/test/master_key_providers/test_raw_rsa_master_key_provider.py @@ -1,6 +1,6 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""This is a performance test for creating the Raw RSA Master key-provider.""" +"""This is a performance test for creating the Raw RSA Master key provider.""" import os import time diff --git a/performance_tests/test/resources/ciphertext/caching_cmm/ciphertext-data-empty.ct b/performance_tests/test/resources/ciphertext/caching_cmm/ciphertext-data-empty.ct new file mode 100644 index 0000000000000000000000000000000000000000..5bfb39a4ef910764c629a850f8f51b4631306ee9 GIT binary patch literal 587 zcmZQ#tzbVM?_qb%i0znq(}fRz{R+42cH8reiPKE2{fb70cm_rW(Zup%-Q=Rm zf|7jQg3_d%%w*l{)Jg^yM@#4MEaNgam#AzrWADT!mgC1MBJH3I{}B~#WeelMf=(x-dD-aT{qw}>z> zFw_~;uyJa&d9;1!Wn^btkZ+L7#K>Tffe>RfkYnRaX!Br9WoBX2GZ19qnUZUGa|-_- zGqGUHHB5{G4c50ElaJooozL{2{Rr3bH`U8jI?AtG+fQZD`WUn~WzVBZ9p-skTKkw+ zW;mTsTJqzA4X4DSWwGt2T%JA(VPaqqU=VQOo&2BYeoEVhsb}6Got&S zF{B%$G8r=TpR$;rsGs&bJZPiO)acDgsoeWd%y_ViN4rmb<<%aqQ>;P18}nu)N35ME z!142F*H$J28H2(tH;s3&yojn3?H3js_Td0u0096MVRv&a zV{&qK~X>W2bO-xWwbZ$ydI4dt>WkEq`IAw5TR9bRNS#VS` zM@dn5Xish{QbTxZcv^F8M{7xKV@YyYHB(17L@;&LNQU&LNQUDGuXAkdZ;^lmGq6^nh8CK~UjHZ*5PYkDfTip1Ty-~z@)xq9 z8gN8}DW#8Ux^Oc|zuxC2IVmU~`a!M)9?wTcX`#F1wKlT^<`i$Si5iLpv&9G%C^GYL zLo9X~+qQQuc3^Hav{llgMwWx3X0zpWtw~5NzXo{U4Q9eQAc` z{gX$#3R1R728j)mJtE(kQR#*glh@r8yock`u1XS2JImPwVA9%R@?^(%iWZPixvF+_rRj`p&_8;Tac2~{pK*EI0apq?707A`T zxU-xqvF6>>(Pz&gQL#O1EEjCuCzqCL>chyKz53;Vs^l63m@ZIVJR$oisKrF1e` zoY>(Jei}jL20BV_J-NpW6$2mwG*&wowcRRHQuBViT&LSs}QgY^m`sA=#Y8GAEPpb&SI}dq? z+Q{Erw6$*%3y1v}fuYQ^)48-M$k5SbXr3Xhcw4SMSS`^v0>WA-Tgc3r>5_Hh3857F zfJ+y_nGbLnTPp)i>m-CzpwLw-ebze}NI!5l)~ysYoWm$c5*{n~;_@SEe8HzM$DqQ^ zVwxem5-Qv#Yj+3Z^=W%01_XFB%6$b)#zu)~i;0@}nzK#KZavX>-SseEzuc1CL*qL~5knoR=MV){?C|i6h44_rG zNn3D=?usD?8}aTh8R=yFB9$R_BB0nTv=MNv(mGgO{m0$nKU^uSmq;8a8(d58?|sfO zgmWAZ@GJZwx7N0Mt^h+qMs;2v4h~O@Pt6{-6L& zgW4?z0PB@Akft%_B&2R3p|e`PM^5Pb0iY0nl~b;jwt9XW!MQe$0(}HT(ihW{R!-ol zLN}-H#pB%X%vY=)7d_o}QcEJc!m+6nfkIZVt#|fqrjiLhxC?sQJ=-+oIc@ro5a$0g z$;^hiSRWA`GzmOzi(^urh&yaKsZyY-*fQmvyRBBW=`=h2USAjrgFy2ABiK;;9V<*QLZ zhtqU<=ZWXTHc_cz&!V}ytV*Z`#`RPN4e?1Cs3DfQ20n$_pRBMTx{2kl;+FR7ars3( zeoR1V?u#&WI*Cq||FE8(5`XmF8$}ZK%|}X*sfyO_rr8jxj8?R+2q1dTM|D<<6iLB( z#t_}V=&e%AlG;J)?-m%R!Md%!@BL}f1uKC6yMC4>Y3(*ILG7ypa$Y%?|D#`(o1^&J=>%_-RV0^tfg6)COD!=eZwq_G_Tp2@kyPb8`PDZq5}K~C11G#1f2 zdxSP~0YwEqt&2ZsC){Y0f4!&ovT}}2!AY`GNI!{v$#!{|Kyt+7@O5b>?|@=sqw%uX zoMSd+RhS(4?Ftu^>TP)+Uyy{*FcJXn%uXbzLNz9uX7b6pR(PVN?}d027)T6~#3)9*A*1L?AtcMKZY;U>5`M4hVDdN70eVDsX1}C>p z`ha^;fh5ou{OWdb@u8{Ph-I=PRYA&EQ-^Kp6L~1VC{klw2+j>PjnB(&m;fE{T#Qrw zuh1(d{W93fU(#Yl9~t)CzyLeJ^{3bc^JC8CG;Wn{p z;am8jQCX5g5xB-)DPeOf7Se3(r3)-z96q%meS{^4Xhgy}5XX=?3y;ERPf{=Pg`$hM zr)Kj@s z()>ZY2d0AF4^}~GTS#*1f_EhVNpl2`Y;Ay^U}#h%JGD2jMqrLeaz5rK)634oHxfrq z#a5aH1(2I%Wxue4`)=E^cRSfkN=(OB8}lv8jtZ#Z)2kC3SGS`gO~OZ8#4>w^Gc_K9 zac{>ZTZYEmaahUZXj2brvbJA5p;O77!+=ICn@? zs*BGubL|oi(|`1Zn=+YZ6yKx*WBgWv9IZ?F&4tBc#g4ArI?i78 z#2<3*;}X8E&@?!=5-6vMCtNV$Bb>#7?$~{mId;a3Gk&6n6@HwuwN@>KBT&puB>Bai z>qKKPI#Q&Kf+rjz9Ty8dAE$W@ln#7Xq3P$t*(^JEiQr{sO0;gI-hk;BQIE;M8kdZP zmV!QjT3m=u)koHWBqt&T+bZm!D}3D505nOUsYipM1BLTO9RX!QZ$tk~@)jbrgA;O+ zs{~@lOy$g@HOOdnz(&uUe^XJ*PP6Dl06}100wt^A(5Ez2R@;oabXdzhf?=e6#?p`* zF6*y;(rPKA2>AN!{*ygFMM!-Ul+!e1oy`7Jk9{vdlGN>9UHw6q1&fg#v*LeIV+_TP z>~2J@2Gy5WRGos>x`fB2m5}#{mM4kKZK9dFUL{Qvt__FA8_^-#gPgJf8KaD)NN~&P z9|oskVNp8d?CnCuCnV&!TvkQtn?%U2Lm1CZV#K}C^{#`X0S$%1z(nR@F#2`_?DfzI zJk+h_PJES}SI?U6H{WdvRU`jO@@U-8;rTj%6x(CzsCtPjW$m8z*)Qf)fj zZz++*MN2Lj12RxSOxi5+Beq8N7LudHTYwgqxtVXN%HxpKjypMQ%6L)gY6LJ-;2Y6L zKDUR>^pvz>tl$M^>0qXISB2I5ecl_waKOC@K#Tf*XR)9fKSbK}?d7S7=-QIS-UyWd zTlutUgU_-dKLY*hzcj{PuIQR4(p!%aMNV%RV0~99t$i(rM)X}vOf-!Px_5LuftS=_y+V9 z=DluBzpgpIg~gH>&4DoJslmDR#^33zJ^7eD%l&5UVQ0-TH#onWsF!gdy12mk?|&?H zPVJjgp2;?hfOKk+&b~lbl7AHz9KpNG&^UUd%O)5WbY>*k6{v|FjbF=qq5DO=fyj*- zG^VRv+q2w`lAz_GlCkMAQ|@(ea(xAKq-cIRBk zse0o-(UD}m)#|D%wB!~S7uFrd>geOWMWF%M)nGA)l|Y?6TP9*Jk6F6a16oM z-J^0oK)?u=;%#4($RUyK%^?`VF*K3Ninyn|#ak{m*Tsn_NJAoAqd|@poK!a3;S%L1 z`c!#0TE291(>k?^cw~02Hwj`>71vx612fFt{-aoMbhI75<1voT?!zi;i2<$-!?l(o9f-f52YE7*#@CO+jgOTYX~$K% z!T+@KoE0^pEl@VG`pb}U7(n^M5i;9nCkGu+H0hBWZqSPjh6YB@QAiMt-BROa?7t29 zVF#JFZ-k)7O29VqQW%NuA)__q$s8KT8P%l#B+wT<)#f^|HAO08Pu{tX2WPIU%LHKp zK&3d666LA70BR_cGq1u{fH1nEAZY$g%~Y!volP1DX8~+94qDW%RBh8AO`*l& zB5S`Q1-}Fk!=v)6R$*=&$FQ{2mi^;Iw&*)F9DTbVfU&Z$MfM>^zVF6qRe;prcvE`I_OxH{57h=O;ho16n1<=$1lDaoN zObT4mI={)-im;J9QZ4PQ78r%FBQI+?zNd{z;js+vhjqk#Q>tb=mT!dZkugqQU`=Am zivR!r{{R300ssI200000000000ssIHKxF0)yIOdUn5DCdJkt==Kz>lg%K#ZsQ}MxP zwJO@=_E-4*T7APFSQr$oe}7%8c7iK|@WT%tEufvC)@dxo_5ey@yqe5pKyeAaiX3M$ zJBULdDm>(_#^PK5kYOyYzMRz76TR39m!zzdk|g=Fg-V#a+mrj@K~(@(lCIji=$1?N z_fWSWqDNoZGNmxs*8SY@5>vGQHT5oSC~`=dW=$n$`gkoKL%YU=m1j;uh0=4mM;kV* zxX=4lrf^e6^5Z5{F;NiCrd)dY-1eRrdQ8Ao;p)Mlz?})Zm_Ve?R#Eq7^=rnZ1b%CrL9CarBKo|mO5hyXj4EcZaDOYV@!`6 zfOH&Ein)=6Vx#($;Y7|{O zQV`$m9)PNWj=g6C&cHaa`hIqb*+cnFxFNk)Ql<}?2+FvgeMwp3z+i3QDQc1&#_Unk zU0S)t3A!&VbO;yV#@+r>%!%l@j1AqSaJdDX^jRR)Kml^^<0&(JRet5aeAhfW^fov< z8W-53^z)C((0c_jG3KRDf<@w)0D5wlv(GtEkqw&0NV@cnjEn!d0tDk=uYUU#2;3T9 zP6jrg3;}eoS;bq=uu>Bt0DiR59PNLg%Rm!9vFnl;2gA5>4tHBM z7iqHYBaG6(B&4myu@vfP)WM&v@QZT|wm%I~75WgiRa>jrF5A|50eq_f-V1AV*uzHI zlgBW48|SZV!yT$vQ!#lS&;N@&*DQu_UUf~7VA{KI%-2W;e=FTmYx&NrRX$jvpFGa{ z#UC4%WV2hWX}DW#~zEFJ!6p z5!3unRP!k--BYW-bWrN64}KD^UxREm&&LmN8g=Khdki%cnYNVKQT~Oi!wtL-NjCXu zX;wz+^b^i2P=_#A$}Wma2WA6>OHeuO2E2L27JBPzJsB0-)mWaZ=b(TiL^~U(D)f7x z)t+FfvuhXH*d0WW#n!P4bZziog*gwhQ%Ku&EEyvaX5i?~i`^E(&U6Eu+?Y_n$XT6r z9CtL(#6D+uIfxxRD+1nGdYm1$+t{siisVZ8|0BDrr}oZ-=jNQHLB*Kmp4Z9mQxdil zgHF%e*ytdlUj$WM;a2lw2lq|%>aG|%Q#gyUB`U;&!~vCb6iES+Z9xjs#XBSTtQ!uj ztcO37hGY4f_#;BW%;0+Mhi%ZNl6i;@Pk^aR@&)%vo=mS=N0#hWYq+Y_iKt&80R!>~ zxixLVtO7x-#R*#{V9>;Ws1q_H)iEJU{6i z`DjuxSnXeC{K?;D+?xU79JD0v5JRbA+EW}96dS>7EkWHfz*zKh8G(Q&`Co*tZ_W)f zz1VxEE2=$j0BMD#uBNa$$~_8JBMm@61)$g`U&96>Qwv_sXoo4acfV6uuRuDb z$dP|TDYJ>T%o1TKFV9h#;fM;m#SLcwKTU!i_^_8 zXixb*y3m$h0nZ7uCxHsMt$V?R2NT%VrFNw?c4agW_}dVw0sB(1K<6;RBsqQ`#p-`H zq;R}RGIFN>`e1$mpv<9mvxiI>KZYiq(8kJRYAkpFTct3>3)O@(-EIZ%K2lNtWvl|E zS&_0Hmb;_2Zq7MOZbD!thnXL%`e(1{6)AWqy#d4rcm6|-3dji=rpBC~4mU;C0nm~L z2>tE{P&FP(1&QwINMEFr1`=oB63(#i6BY23l<|umW`6Y_3S-c`>>!%7z%w6k0oy2| zr1O9rPu<4O9>RpLWEQabWailqov6>}&)znIj(=a$=3JR4EiN9#e4HZu%ai^tWAIee zPQPcIX9SvhnW!t&mg+E#20i|0H<^;i5XfDtAl_L@qS(F#2RQzM?~TvB6WQL(`nN1h z%#BAghTRjOB7#m7G^;_2uxar35vp_l*juAZEy9GNk|kM@?5uxddy~f$omyM$@hl1o z)bF_st${#+@s}TEQi2O4*#a4pqkM)%%xBV)4htiP3ms&EfbVAXZuG5LVWAKq3h$4& z6cka(HmZ0I_`R_HCon(v)5hRnBq}C94Qeb}hD&ppDuYhTO}*p=&U4s`=@Efz@t(^e zIC-&%RVd~teB=M;JePrg3rgCX-dAd+^3>aFC-e5dG&<~4xckzVa^r}WA6ALtwl&at(h3Ee~>^Ue?ZH8(e)hGN)*Vj8Q zES#AiKD?L7q9m^UaUq(v4`PxvM*W!qdl%@0#)|W7Q)I)6l7eQ(4K8;HHu8Ckb~Fwv zZ3zsMFRncgpVK{*=KJKmGC_@+XzwK{tgJ^aHshBxcP|}`bnO^n6aU;UC z6OS>y>~rDGn~R=c;a@%JQ1;57XD?E@!Y;R-OlEoH-)mwt9jU_>B__GC*}@WU-3RYw zyI|Tg6!f*)9NR*eTMEPJx}DhqCuD>;9%>JRbOfpL}Hvo{xWn)E&tY zN$2v#gSE_Mq8O4_8Ps!@p~yD~444$qKo00}vtRHh`%g6=q5(O0#$4?I~ zHhm8+31IB^Q$s%Ow-;6w0><~%wCK-I>pEwn&U>~(>6sDj3#2ltkXXGavc344;YI7! zU8Ksjbq45^BA1`kfzaMxm36Ley!x(~EuC`9rOY3_{j8_4!dh#VyHRS1Pb`$RZEo?( zgAL@`cHs@Cv?9?NUs~fy$iJ5Om(^@slCCv2u_iFBIJ@T;S#Jt^dN$7RN zhyPvm`~!hI9(9v3;yx2WfE#Kkbk^D*_M7+6?1-AEobdEB-SCIdZQw_Pwu?|J%Z3jY zURZ}XO!Y-DM_Kq1yzgl%HX>eQR+L*#2u__}d!r6h76GJ{k!6{0f=bsWj^>unls*jO zPsNWC7fz_wSmg{Val$2OoOC~`tb3kZiYl*&$MK1F^&MYos#GL!Tw~H-KeS7{WfnqL zPj66J(Ng6w0Pu>S)PHNsz8V_d;9{gL86Q?j76Y`0@3#u1nfLAvk$WHALLJ43@M@?u zh|b7f_3rb`6N18NM?&8_iItjz;?Ap}k=BI6d4hFNkR^%<+U%AI6>@h>Ydet#J5-Cn zDSrq&o)(XhU}Zc`g!{DlZv&=q<`~JKC#}8s9cJ}<&KR2>0zx{DCb4yl&tXd$xThFm zWIfWe_{Enbc!;@n+HnCI$-5g;e9fF1*Mz}`u7N1q`C9KT~p`1~OrW1+uj zeGDInZ?0vqt?}Y6H!^@fFef`c3uHD(df-|Z0%*_MHcy?0mj912`n?@E3-a~_@QU&Z z2A~yHJwEAK4IR^i*R0P~^Mk4l_=`IV>SAB`NjW-JifMM0(LZ4S_vpZ25&7gH%QsgB zA7fmyT{?hz@Ac$+AbL5o6*ci?Qrg~F#qZ0lnj$%O9?9b5je>syJ=^5d$*isVa`ID| zk;;+pY1UpYr5j3~JkP!@E7`5|67@PU-umh|ina1;O5h%N)jV;x?jvX$!*_;1CVchY zd-~k|0L_{<`d@N}H^0$Mvb8>#fEZ#1qPrJTsD_byO$upS=T6 zZx9_ExSZC-e6l_|3oA!)hN{e6e)xF!AmO)c` zX;}GABta)Cb(uAJyHmWO!ejg#=7LvB@*)D*3&CA?r~VU`ATZlcic-gNb^vEEWdblQ z^<_)EcwmhPH&&sEN&{;nAH+ZAe3_5iAIl>j+wa@Qv7(|l4LmiOMuDqCT}QeCF#x95 xT++!fKiILtuGVo)z~Q|A((p++W>=Gp*c+8!nuL@7x9399X>uoGcKsiHtkznqK~b51x+Y;<>8ct}-EMLA4bQfyE`cyDlIcuY@rLParc zK}JtdLS$BCG%syKQ9@c-cT-PUcXvo&LNQU&LNQU~)(l@DEVG$8l z(r!Tgo^_VKS*|Vq<=WeQsDVaN1Yb9{9wsk>4e(v!0ssII08_jBm{>g$JmnAfGnA)0 z^SFzRXmdJ%qF+3FB@h-oN#=(Vb)z2i!%BSTd$iElF8}}k{{R300RR910000000000 z0RR9Gpi^ge3INpzr>~kt{!NHZ52J?wf-N(O0PanRmPsJ9`TzwJ8pBF?_ktDsi7><6 ze$j)ZCFxGmh)a_grcWo$uvJM_{trv1a)>?;y)EqxLSlDeax&M6&4L%Ot5&_cNv@>{ z9_AHIKWG~~`b`hQAy?55-D7{83L&sw{mrtgcL(A-aO}C)#erv0Yb@0+oH;n(Cqa%c zK1A7kw?R@YaN@aihL5Zp_tW#}ZrrpZ7&7JpAu{2b0sq(9;epeM&jNgmJ}=Qnz0xRw z0$S8_f!p}P1)TR8C#~Bw)pj^!#fw;xaW5qzE#YE6)WbbQhZ0#{hJ?_7Yc&i*zI#G_ zvjSJ0Yu!UqzL`U<*i+c#-emx>lEBlL>4{VeeHjwRM2WAx03p7i$Km+4^zmi*qo&RnLw7tjW zYf&E2;D84$#c~GvM=2ZN^=Z%Xl-JW=JFyv>;U?Oth3n-S!GC$J)GETBBZ~%8yCTBo zH2s+T<5|}k_UsVRKX1wQjE4Kw;S)5rw>s3%_7-IyD@MnklFRU*QOJLm@l-Q&6Wl6r zH1V+B9{kbS_72TzZKiiIdVkZVF?S>D;pAzwYt-N>=WY zPB%aF&wlWXYrd!l1Rh4(T3!cE|M<5|T~qPwHF@LtqV%cWf(7fC1P6K1MO!PSF5v)7 z*^zu+UT_eorCSMfD$;qkGRexp_kg?s3j#8Wtr#nNx-Pr?Iyf$*Q=GrGK$TFM@%t%^ zGv7CP0I#Pf@M1PBKt_^qL>EtRkkM~Q4iR=Y^5iNGN2^70^EOK$9^F64*jizskaWas zs%giPl{A0?g4Nz*PI?z42G1ZaJkA+j&3$o0bnECRQiABgZYLNobE*yXq^yO3RPidRf ziVF~4=Tv*bgmYheBau9>GaLG8dPHC7#Qe>L>N@68lWPbAXoz?VNb7tajRqcWS&M@W z{L7$$AMthl>%Dg+qXA6a{A3raFk?VLH|Wxjg9ogp`FAmC`V>F-!m_ux6M?7;E5j9# zu-H~3a%%QXtw$%0o?5k7zvGLtqd8Ffky-I(74h}dHK*S*08f;Bq_8I@Eky_l%eLL< zL0T=|m(!1kI#Y_ks~PynuE?XUw<#T$4Z%ATa)PMfDh8Pfm0WWyvTe856$lnTWdo~^Ix>zZY7Jn7b=#NLFby=^kjqfintu2TWKVBF`F5ZF8M&(yb(Ml* zRIUep+=R^+e^4qTD6bW644_4Pc_9skIYZf`HlKo;VW8qUk-2V)BpolcwVc<)8{@w< zuuHUEy``Sw)&oqv(7Ze4Y9g`BcJ>FwZ6dZRtWRE27DdLRyTSHs^Nd+mO2P|-hLQjP z=cmX#02C$&KwOERB-koQYY680Fa$9mDMJ7OLBf>d@7+bg>03eCPPqt82`x5p?|PbTU{6RT9aWc4OS| z$pl2CtkeH{J2cu!-rec2L6Z!%8*&s%D4xmO*fw6GGG!~Bz_X49fMp?B(Mb3jN0Rb zPzCvyK7~Zy@Hr;==ADiP2L>yd`H*b&jf_iB(6?wbZ1;WZ|D z_A6_n$+PX#V1(Dgo7Kud;Kiog)PLT?^75NJ}fb8)Q2df_2+VXk9PIAt9o|#QA zn<`7w^=JS+ZYwRg;!`gdU!$u2B|1)uv+{F_$e#=nv!>c? zRD~LX|2gDAKGbQe5+{-@i5@&Z9fBZ!llEl>Pj{L*xErpbcXq*K>`1zvC`~=ce@Xu7 zt|uHco6|fSR&VPT(JAIKbBAJ0BB+M+7DgLq4QB30q;di^3s(~XV9J9~vxuzj4`#^5 z1l6vUXuvxF_9s}ncfS(RMFK3hwUO=_ODkCe?{+G!qC%0*!q9c^@&AY(GF zJgADM0xFI2VlK}-f%3g8S2rDPDjjCNsgLHM6rluTR?zmO(ZKxIJVefhkIiQGIRcb> zq8>;n$NH}?bKO0pIB%$J$_uijumF9veVMlkjk}jh`BYDJirdJ98wWu7tL0q#{vM5V zAbz=f#utc_(1 zD`Rg@9WvvxFXJ6gt6KFH&j6f>5S&bSt3XMNi5`VJeb&AIU^rK4fdl;(S`ZavB#jPu zWUq^oRD^anA_zn8u%BSm%thhipDAnkAoL}3wEUT79{hMKilv0iUX#TRxXjB+=|wQP z0JUHbKV@ToIhQn;*^Sz(_iT=0e;QWzZ?2P87X7;5B!4Rfj{nVGUh=( zT&eY%lC^*-lhBmA{eO-(7;s^L?;s)-UNLE;LJR&HyhoExGaM)yD9H*Jg_K-=PjWQj%K z9ExbDqP);aBi0E6JGWAgW~t7_;a`hBC;gE!KEHW-vW$nZ^7E@XwaykynzM zGD~g{(m9|MHS##IV92hSW_JHb8zDmOkSI>Wy@Su8!TkyntL!oqF-TMUaa;WGBf-YS zvwX>1A;~=uvdwa+d^0Ij?C_63hZR6A?L$G@99sP?bN5Hvy?zgf``PvpNU`+wJ>?B@ z=x7Bn5rkG0*)FMG)UcNR_)T>(#*Xk&ed8*;`jpiSHc2QR7&hD+3O_^}#~`S^x4CC2 zEXN=!OIt+%@i%+4&imMO+V#1K#EJZ}yb&IBAtZR!3A5yZ>S3@oVBs(`X%kl}vhC6moU?bQRS6%31Y^HZB%M(k;G!=6=d7c`>#|&!_5$+$qU|7`O`BF6>d?Qrw zI|{IaAP)xsfP=zQYFy?WaI{IoOs+n~oba|P5` zU16QKh(mJ#RA)V#DJ;@O{W4US39)wZY`AG--Ge}lnx;~3kM+7LgU3RZE8;Wgx*j59 zBT@nok26b;hzWWfnTTuj%e*zDz_%gu6p9aMU{3K0etzA#3Ri}BJpE=y4v9JzZh3o? zKBo)OqD$HJ({d_g!zL1tG`8r_m<6dkH-qY&6>ufj@lXg~0d>rGS{`uC3!88+pW?aC z#u7aYkudtqpQ-F&k|XKRN2IOlgxOz+mym1>=K5t=&s787v~VH}QoC7Umq~PGXo0h3 za+E<4&6r^yrOy`uJ8h+EbMCzj61$GOyCQ^U0B-%Tfc3_a#*GLzFg98V{rLaeC}ccL z>!9!|#F8LqVQdB}R0^#Dm`h24>=c2F57LZ{MzYWZ8gN%`V;xArST`%bcPGCK(8Ftb6AOtNp3a|2hM9I-{zLC7)|9&@1bZS8I{0dGZ=*{!#6_!)_vJB_GW? z>S>|rv|Rgj1HXoQrZdB51oP4E3m-Z0y@V^gw|E?JfmUT!Euy30iK+NE{1jSV$5qBR zarJ5P2PyipP{?_bSXA?~cl{zH(4xV1#6oBHZ*Pf%Mv0hODHgCS4OTVnq}3Z)v9kj= zyDGDK<8+!3;g)0hDHp{}^ab3X(*Hg0cc5Ptb38H;w~+Q0z`TZV0|~zypdcd$H48dL zSpL6t_V$Qn9>Yj7-<45GI4&82${Hv4v5lKJpFR^b_0KSjdDA8{>+@otxIR`7$Wn{? zWDYqvic5m_KoAXt^{I@5sccRT&l4qVq(@bXS9SvfqKmaPz_V_N!6WIqlAE|Hc_l zT(FmGn#vSc>cw~U#JT%v&2O#7APzo@D2**?B!BDKH@eeV)@?&CV7j@xJ?NuP&9?8} zVSz1kbfuWvq$%ARHpFY5R}shVJ}>o(RvyAP>eL0%?imy*x=KM@r#|LWiEC1cqSRa& zlePmxuP6s9DHl~V0^J~(9=|~?ZkYNaBT%SO6Lo-5oy;o?ON0}p&hgHz3sg5Aa{naF z?O7V(RO-yE%n-6Ubn$A8j0ANx{AaAU0pKbVELyRXA2AT`UK9Cg>ml96x@hFH2Bv8_ zXX$fka9B4m89Wi?p>}}P3tqi6<)(cr#1k5-rm}RbFcx&9#xbArr5F{Xpqjz{gHC}3 z&mc+#^Z;_hPDx7()uzPzjk#x~K(VZrI16DiTGGJ0Gs+^;M+~1^J^%g)|HwCKr#i8F z{JcFtdk8W2R)NV&JouMMyH&%(oAdH690b~Nr%$b){>Hw~>a>Xl)6pz_6V|vMi^4s! zC91uSeWx?B>tsA@%zZmOwBiNs6EBGLrdNv$!ay{UaT*Dep|D(}uABIf%=1qnE8d5k1u=zy z%&l0qfuDK{#O`_MdnuwAWC$}Hpc8;n!->1v?y V-X-*L3)Ww3h#M2;#hI&ZLd6QKdcyz! literal 0 HcmV?d00001 diff --git a/performance_tests/test/resources/ciphertext/caching_cmm/ciphertext-data-small.ct b/performance_tests/test/resources/ciphertext/caching_cmm/ciphertext-data-small.ct new file mode 100644 index 0000000000000000000000000000000000000000..63dac90b7e752819bd5740db6d3f7d3a3cb73a65 GIT binary patch literal 623 zcmZQ#t+=k}wKe7Pg1*9K(sA3qFHtz?I*0eWX1hk=@r;>w_SWxWh-YAA5KSyE)=e&| zEGWs>EhtUO$xPPGPOW5caWvPj3`>b9Eh@}$3QjX9^>lVkinKH{tSs|!^U5?Sa&#{7 ztTa!IOwIB4&36j&aQ8_oG&Rlg&oS2yC`oqHcX70}1scx|GCn)En87=-D9;MWu>x|e zN{e;NQ;SPMHB~WkNP7HcRa02?-eQSQNEN?w`lp z4|5KyO#D!C+apI}G|Om=?aWx{i|wE zRLcrKli9d#?UkG-U;a{~S zkJTGO@9vR2(6qNp?xnok;r~lx&htKhW98{NuW_^8{*rmq-DW&tNH<7jGVs!B&^7YR zIqQ45-#?rCY{JPW7kd(I96X$d)4F<%OUxL$&g{W+`Wy7 y`)l`{P%)ijlvKB4V)ON&#}N^=x) zQ%e$45=#;pxKi`Lwr1w%F|a4+=ar;ZlrXSm7AxfC1L@+_yhyOc;Z6n2cwOfmXx)W2k>~wK4xNI5MT(Mto1Wc_{SZg9~tG&wZ1X8B^Ca< zI(?q}XJ3rXQa$tMsj*R)y;h~qIQe(Y(QEQ5o&SM=fdS|z1OT(|we0+ROKt1TAb01N z43D4ZFr*u#G8r)DuF;8~-?QM);oy+ZC*)`JvDNDS&wAIZ-2PHiaNXA{@&y7$&~r~RHY%?PsUW~ PN2N7q=$;rmEe^6F+=&Sg_E%s8Iyvjc~z_FjqK~D_BfWYA{k&K}l;_G;wWFYHLMlWm;HJb3;T*cSUt^ zRC!KVR(3BmL@;)AR8cT#MPf^1Zc<`SICE)lZ)8$PXFWXt2x4_~AYo)=a{v!@b7f|A zY#?oAbYWy+bYTDrWo~0~d2n=TZ*BkwV{dMBWq5P|25EC3Zf|q|26JU&a%FS?26Sa^ zVQzE)2UKNlVQzFm01$Mp z01;t#b1iFab1i6TWpZJ1V`zB*Bw;l&WM*MuWGy#jVq`5eG&3+QVKHW8En#9|VKp*2 zIb>llHvn8qVEAa6K9sGyWr*f{FZg&AR}+tCM2x~(uKn;4t4M8;{z}-JQb7~4l}}(U z`sQKxm(sm6%L5KaMcTDHcu-*>lUVZB)Fg|xxs}%`>eQLTsAJDC4qYKnZr`o~000mG zJY(`y*ayl12Ign5os^h3)<)WSvQY0iw!xUJY}hZ4!m9YJRl+dsmnQJoGe6#Yq`nL7!9DNfOdn zs$wRt^|ith=xKvVI8|T_-bKDLds+&=*r-jVL-lpQJ_!2LuCsY!HwpWT@9{V*mwMCo z-FS2EJ&aQq#j^#|DRmaW7X>r}FpW`|O>0J(Bke~Q$~ajPRmXCYUMc$KwMeytFcxMc zpGe!)=Lhc3p99KiLwTn%AWdwNMxaE_fBh@V7GAeIcMX#NBiEvw?Z`b@7!=jR@xnyT zRqaV?5{;LdC!v;42m8ewJavyPtM5V;sNyrmieoWPEyLb6=`mH@VJMU0kF+i;Cxca$ z(H_2eT6F22BS&(e*QUY+3tE({YV1PF@0#R8#@6=idl=;_1zWo1$`ns#k2H_9gm8ON z8gZDacwY&8x}a2aO=d1%EFR`=7j*sgzk<7}jj`+8Ngg%z6Shi&4rftH8tV7eFCuVN z?!cj)+ELdrI$iBZ8bWpCChNq5>6NuB-yiBcR1WwJAc^@7)EeBy#o*xF#Boo z4d{snP8&rsXl!oEEC08>^z@ssZ`B+mt}F?fDay^Z=h(`wx>kc<(oStm9IHnv2fs0= zQj>2^TjwC;^fu@O)Uask8eoBc`y?^|Lfos8NL`q5x=LIHE>6glDOM@!N+@La_V+(_ znK~Akd@AR?K_oNSZh==qDKsM%gK0eJtCw_n9<%6S?&Zxa|0^5K$tm&J;Ddps{fN+E zqPe~hw$s|{obnC@;HgK`N>e6mG25B?$wjG`Ff+y|lpqy$D&d0=WLMF zRzwjCyCNJkj zU38v8f@0F{3=A*)iG&Cib$FaBIqA@p5jpSP8{$z=w01@N&3gX$<5Nl-j^5%aazR9L z4(6}Ny^#5Ti(jsq=V%R#yEy@wlc7}(V5=`q;I{oqtTo&JRo4V=(^U4Y!aXBLKbGWr z4x)hL|B40g-%6xnnfXs79qzW_-et^-1!jHh+$6IqkGCU_bg2WvR}d;Ge*ab)X2>Bh zbaS(d#IdV`rahE%8Ioz3y1l(dAObyg?cQ`T++%6#lt}R83GiWQc&%zJ`v!e z(mIhpVv3-ED}s#8!;?ZSTU_KGv|H`1ci7rnzd8yo;rdwv$Nei4_U3q>3>GV2o>*AA zI>lD<^e7wYdGJZUo+h)O)tq!A|>NHWH&3`EQ5s!R2u9~P+Vz&8^lrFQQ z|FMFH10*U(5_nao0wwCdPD~vCd0-JH{!qM#MRYi0p{8?js?ls@x@o?v2hj+ZNXkc0#NNUz zbJLk(%bR4<>Dne1)M4s2C`*OZEn^$A%IZt7oc`L!%aP^MfDMnBqYa^)2u>NI3>fuZ4RU5R?XE$fervux>fbr2XZvp zk*K5v5U|K!3`R!C@&O-1;I!#BuaT*{?P5kt;0~-YI0Z9Gy-mUn)zn$Ww>dk*fXkAI zOzEIn-ll={!QPyd@Fg&Zw+6sI#EwQ6GsL6M7)x=2mFx`lWtJ7RMSMFvDqUXnLW!rP z=>amwMB35AVgM~0SQ@?dF$A~CjH(6XtA3+VvsD`DH>$VMCdsr=Afd0VM)(Ynpc6G^ z-FQ64hx7Q@T}1!byt0~nl&3@FW-cGsx6npev4Q)&bSBh1Ugdm9}Yk|@acEIj6=VV zg9+AnYcA3P6%95qj104jHcf;;p!)&uk9EKMnbO51XodVoO2^o~5X>K!hzN++<*t`T zauPtltyIEIy3k%kFlWBW&&2i6rMhMNU3}+o9~0?$g^fUyaSP#n+ZCYKd~8z9!W-;) zjbO)E-yuhdbvduBWQskB4~F-57{Z}v>r_+K4Je6m{kr*-7W^3TZJQa<6%5uX--9HW zyYwL6d*}PJS`9xlM@LDYW&jwy4~vQ)?eF*m+o!Xx#GI-AURd&kKm@U)A9$-}5baL^ zjdBIGA)?6wM{o&n943r<&eY`muV%cUZRK2lw`<21r8W%rd7UyzL~!NHcb80Pnq$^o zoJ(0qV7?nheE8?F2VM>cm=gh#JUNm%`Lu}}Iy*=>kPn_KO@SlcIrV3vWY*$S7G9-~ zm1)tgxczsQc_zl!BW|nR93%z{Y(o$}l&Ftl`W%!ji>V#LCX2aTjDw@2TM?)fCnEz{l_IN6sVQ$6M9(%!K}}_CD)7=Htt^c+@o`dLMNmJS|qmouNg@@ z`CeypQ?&LU%_z+HW7;M@$Pp+XHG!u#1+LV|HHewGNfGuPQ5g^k1dH*7MloS6QBM#bQ2C*WP7HE>+z#2xYQrIsK(?sJX86c7ee$Ut1J zWCrj9Sco-Sf8m;|GLS9{u!8$yiTAMg8W0-Mp9r69c|-#JnZq#z!q+3sWnuyIEqE-` zPgU{gC#9rI9wE5-YEjx0t#-c{Sc2;emFZ871{6^=-E-~0N{xcgC6V0O7Xr8p=ujfq zO((KIpiH{)V0s8C46}Jo#n%GQx1w8+q9M+MlY(dkv8o~SbxgNk$~znV=wmp+u$tcz z`s$g$y1kPCM1b2;LzEu%t$cL3s4}irxVGPj6;}wTnY7X@d+A`@&g8*Q}{@R8o*7yp1`S6(acAY)oq}>++64u!t(}m!~)h9-u!0BLJiMDcL zgdwshO27{LpT`_YRE${Rgi-o)o-RTLfgo?f2G!8o*2p$y+h3zx_)=)YjCU#P+-He< zjFg;u2UwkEH7L>pMnL8aK?VVF+?kaYPC@k+y@5J1kc+dujN!daqOmAg_KwIN{dUe~ z!rz@aJ=BDXtCLxJQQ7OTJ)lx7fQ)q*pKm^-PO_L3;Q55(PDvW$+b=HLBM z*AOWJqnyAx@5Caqa$bx%;p@OPDM$HjUAMFx+vih&Q^e^NG$S7!T#VCog?S&m?2DA>Su-mvM}%6r9H0}l2#mOdmv$Jz>~64DDUhdjIG)w# zTOjyIoD(>$n#d=iM5rLBj^b#-@i{eqJnFRJHm(* zLuel+1BFLB;09L*TVkUesVjKIjeeC#%sMa~&Nd$h%to`}UCn_YK&@glbf^NtP!p~+ z8HNp=tD?+4^B8O4-~1x0km0)U6@hPb7kja&i^rzwwo>&xw|4FL(QXY5y7jIVq65>1 zO2X`tIIoR0P!kZewzz+W;vBxvyg-E^XqVh9N}kc{K_ z6C~qX3p6NBmSL^!w|>R%Xw#S+jRRVgWW_S}x^tQU%o1>2H-q-_H}+ooA^&4vPxITEiN=cJC=R3RmFp5tll4zV{KYYx`p zz3_i9vxGG5H|qs_1iMNwbdmT^g8Os0RB?||qh_Yge`vrANjU5yQ7gxayNfwS#1-(S zcsL)h_i4}G&*c+-yhR~JX|n%zzyB{DhB|9tB-kzs>{6SVRd|dfHAM zam82-0u3$cN(A~SgIWULqZw2t5zMUC*~Z3MRzGZ$mA-RM39|rU{17aDi+;-$4vPml zL3(wWPaSdlr}AY@=#HO7ZEGDf+Q4^ch1h7WSl9Tre`$9R4Q5&%`My>y76-l-Qc57& z8f%&agM;CQ9M)>$kZQdA#Nxq%Qd|*-8uVTQjJk<=6+rft5reBwwioDSAO9_`LVcwn zv9OEgQ|(UAjABK7y6qk&ZN|0Um@bS0+qyGPod%bR0I+m91Nb*@sJKJFO63y@=-o;_ z`3#-)(+D6bCb#hSYdi25|NsC000002000000000000002000j_PxC#|DmeW}osYQQ zEA6i9va~G0`044K3o@z)jg+?m>XcPt1r-ADKq$zzH_-;i@s#LP&|jg=h4pU%#EPQ@ z838#s9Mu&R+(ywP>}!0PpO{oWQcY$!5p*Aq)uU)d6*1mYuvfK2qBvMB79N^L-I$Qs zo3683)S6MUDyEUapiLSUZ>Wl_E8t07fH~)Z4

JYO(+TJbO_}u7^|V49O9TF zzpZ#;P~puS4-{r%)V0U3NiFWTK0U1VWgw$n7dD2aq2;;eo=FY=C=$myGClqJ%QR)+ z+ZB6^;)1VDN#!C4&`#%2Q<0A$few3>y%*&;dr7jrXuoQ5<{)A}h+tNV$?nm&O-M{{0SaT^5GXf;ubC* zcA?%;?``nI5NKF1(hVzf7LTPCd}K;DRv8U!)Tsi$#W#xkt5Y{gLq(93s!`?o19&2t zJro3M^M~bpqhCVbB&+%#gBwB#fTpq@SIF*(lC($?IMkyRzjK2U$I}~Q1GA);%?na} zL5$e`TR=`29ro3)(iKo)h+SJGuhYBhi*Hiz?fO-}!a)W2#F0fQVAXNHn*zf)1^+Bp z;LmwW)07LU^94bvIY<8FwHFwwYzdp3;5Z0w-^iX;sf-B^IaTZ1ksQ@v`B-%{o|UI+ z)TtLO(YBl3r+i|)Nt_rn$9WQUdXJp@e%Xdg3<81{|D|USE@}f8@lL(zR3MjhnH74i2oPRc5Wrx%md$~OB6KQk>`MlC z{q1cZWk0&vTH4bV`47Q_L(&D5N0FMVXy1!&IKK5EKaq!t4cC0C*MZ&h6;b+0h+n_h zVlB}NkJjM76KePSYOjsHZIvp{*02GO?Qq6^{JHp#?3JlMP(@99lEVX_G(AP4TT<7$ zZ)h<^N7l%784j(w$9m;+fG*ZJCNHS~Z|RX0Hy=&c2qDULbzXRjXcy@zq%QazavV?Y z>Hm_E8Cs<(U@=_Xdas)TGZmMR=oyRZhC|=1Q5&kKfLFD(Z8nRZZW!&Yw&0Bul~oR9 zud$7W`(hKo8+2h90r#&A*rW91_9nqk)!{1r3K*uczBmN|?c~$1noeHW8F%x%20`3}9Ga3FQPoh3KI)qFl9}{$} z3+{0OI*}}$swSwTk+XjhWwCUYBUp+W#?bYI8?Xuamf=zxQ(R|{FJX76#EYcnyu&`M zt~_{k^wy@WpB%{%;#@0K~IYvKI z>&LcscK>>_tZoVrG73F9<6w!Vfa_>1Ty^rwA4_aU>N0Q+y6Hm|qO`KWw*vj4hS0^= zf5@|ZOBZPJ-6>N#wz80B@(m4b_paL0T+avr*3uzGN}Y|i(u-p{zPURV(Rqw|m5dZ+ zzEmE%%_-AxmT(JI-;9@+_NhfF#se|?@fa)kUU;#apF>LcDr%?YvUwkSST;A!$s>;$7FdQKdY)TQS(P)jLp!AV}6j z)vhRmO3&a_QKI&omQp(tgnEpFSBRG-lKz>So%rtwjwG%L~#}d ze;K}H7w)DlQEx&f5Zd+;&TNvL!~Ru;Np2qTF%dtp;}!;&07&wF;ik)xjpN_Om&US`oInnn>Hfdo1Mef zbQqZ&cPFXv>LPu?fTS5ui!*-$GlsbhQBQbX!|tEkW_SyB!Gh|V#{Hu0I-U=JwPndu z=$oAM0JHc=Vq~ukQfT3;!Nu9noz9x44Sdx-vNcwNkioat%c)q0bx zCTCuMxqcO!f+hT%(#8n6Dk(v6WChGG!xtEbukTLj3drnW}wyP4JxRR8U`7&0yZGbYobQy7- zz_SIOdy9>`W~Hd^FP&aqy5P-G(OT8muGBiN37mfOr=K#JYjor!_0+B8XIBXQJ{1mp zqvqR>xkAw;Mhv@416}^-+l^h@+~>KMY+I6N8vK_5E{1 zRG5YfiSDlM0I|Dt=V4oxt;@Gs_m7A~M8%l|;2c(bXin}0o@P}P-dS%bv2y3?_P4%P zHN($;@h?~ylc2Q66*F6&hpn@2=Hv+O%W;*F&IM89+`D|EJKzloS4jDtMowaeEw;2C zjQY+;+T{$1VP>>&y0d9rLS`4JZA4dL=}K<1X4>>!_d}7r5@C+w4ma~!Z;&*W_I31$ zv^!mnx4$B|+P2|oW&G1%(__#D_JYel<4Kn#9gb<8m*y}=@}Pf1b)!4{n^Mi+Z0 ze0FCT^{XYc_X($_eQ3leF~(=1WRWImy6&?aSZq!hJ(hmX7J_KWqV_WacNXphC_<@* zMi<8oP&{T<&~7`X)}381k1j;(O= zwXLaf7T5eTV8|Ql0b{c7N#V$GgG2YjABi!$%cjYl*$1pf-8SF7SL$cHMQQQ?PFJR% zjv@PRsbeqQPQu=>YJ&B=AuCNs!<3U8u1Jy(yf?<4U5=8^2%ZNqad;SEp(ZIJ)iR8P zOeUx?0f{t3b5sFp%*nQI*>A6je+91^Sfwz6rRe$5+aI`%`9>80O9KuLuY34|ZN6$s z)B#84!LcQX%@3l@)7ou^^u^LV8)Q55Sep`UE(42Nsli~O1 zPBJMA&Jbl8aoY9L=r>-hUwyR)LjCH`%UFG+9^-~+z!N>%&Up7$ZenTl9C|NQhAJG4 zH_ExY>ybn5q{o>ghn(qUdS?;{PR<&Ax)>Oua6%UhJBURu-(uuyD~@0*p7OuL`I;>0 z3d6RxV6kl>T~=_2$q7yzd&wjN(4(xBqOlNN8|_(E_L=A+ZBm2yTXKx&U#XqZ4j=jc zOmu=X&dC^`?iUPOVEc;TC3bG>cK}Fo@|j={A5;iRSLEkSUmGrB68FtirDI<^#W;xf zprg9^YEa1NZL<{F$JPcRFi#U89`&n#3jf*)`>A?q!m}qrsp0OvBN|R7!a$y#fVz03 zewEK_rQXm>(_{pS6U_H=HC@5LcF`#R-^ORU-nLp`a}4`0Toy%<%Ou=_0YQ7Dh|Rx$9jQG3!kR8g4$+PHPvixkw?tQg)J z5VY6+kJN!}xkiMHNR}h<#Q&Aqk%NwMWFZ?;A2L0~=y7*x)!$b&DI38^6una1L9R&S z(+WUz())v$g(z5qYB-&W0-T<{7Q9lX9wz$kRw;uF1#F89)TR|FSK(g_qhAwpAP4~p00#gSVRv&a zV{&qK~c4T8_R54mwWJ)+fWk@$_FE2t@VKi7`XHY{^aZ^$_ zWm7q9a931XOjb!^Xi+n2b!#_4P)%b_R%d5NH#atLXFWXt2x4_~AYo)=a{v!@b7f|A zY#?oAbYWy+bYTDrWo~0~d2n=TZ*BkwV{dMBWq5P|25EC3Zf|q|26JU&a%FS?26Sa^ zVQzE)2UKNlVQzFm01$Mp z01;t#b1iFab1i6TWpZJ1V`zB*Bw;l&WM*MuWGy#jVq`5eG&3+QVKHW8En#9|VKp*2 zIb>llHvn9G1IO7~-X!h0_bL;vo%ASH%16V;FcgN$oif#z>rUuwFCX1gXLFXBmLeU7|ACHEsrK?Yua#UF|NsC0 z00001000000000000001000l5Lj(Q$+)w?{@T_lLB-gi2`Iq&)a2=v-o_5YIra~B{ z&8{{ig<-)RN`2c9jrU0IydEyPoXXMnlX*i0Fq{c3O_Io=_ z>kVEiTP^&h1Jw~SGSZmGLcDmFBnC%VH<%tJMob_ z+~Z+Hxsa_AT5!8X&E>P8;td`%OpADFc=x01PZ8f=Xq94Z z2fT6!bZ`d#;lQ?wKtLN#7TRnixo6XTo+W@0kQ$jUdO;Xf9=_VaEG&fQ`^!gfPR`7` z26bbadf9Et9TgMjJtmD%ewA$@E3*SAy_Rh4sbz|6Fc|O-0aoyb6n-y822tC8&Iz># zHOd#_DloDp08fZ*UO;O@9OH+H5l&^HP86qpr9D%nMbPmai^lPTqGN1)tjNW|EF#O_ zan+$rwL;HdJ_TPM56o}rB%P$y4Dc>}0=n+|!l}Y{YMGfwqT;hw31=culsKy0tb!f% z2(=XE{}cNR?<7g{RY3_P7Lpf>ahSIVHjP!|sxK+UwAV@?obT=b)jz275RK-Bi^Gg| zZwYX%MGfzPRz`#2KbIL$2WWer*U88|d82u(gln`|4A@>gVbvsRQd#!)Yu0BBI@FLm zf)rBJAek~3vasHDt?^W1Rm|jubwXtMhli)P*^J8=X_mYZb0^eJSz?#-?pFMM>JwPP z*b$+MNX~)5O`mT~!6FLrCONW{-uwvl?YZIA&y5i=xNm4K75}zCS_VCn;OAeH?|&+% z=Fo)C9MUy?4qWOs1h&y0`&oILZpf8V!|d_Qi}agfK(V3ouh?j944CbGr^9U!u2sew z-r2oW2ece)NBkuP0pypM3@s7fDi^fz%1gkdk^hYY1mNnTM>Gv|kMVDDyEDC)5uUHm z99J4~-Vn$$e6zyp70tXc&Vl^r+fWw>L=1!92anX`HwfqoKklBblyYXV>yJKZfoBPY z)aljJs4Km7_nweu<2Fx3k3!y^A^m=9RVmX2oryzC;Eo|MfSY$6ib}!PGT0anl zgrCK#gM*|yy^66{fNnBJ2^X)G-YEL@&AwlvE^tonGear&uj6px%x9InwSDZ=h&KE3 zPVrcgvblJ!@U&nl27ov52pc3k-?^GODgwCQ6tm?BsH1zTMR)Y221iV2F>rM4N?cZLdu~^8vU>SHT!#EZJY23 zhx<6!xj0%by-aIZ3sIaWE?Tu3oC=5JTV$(-^hz!Fw|KQ>u>%4T*Cq1F9?fvjg6sqTiW)S zjaa+Xk$i!jMu5zpHdc2R{GZ&&|J%0+q#gbkoT;Ih|%q~`m`04oH`#{i&$2bmWP@H-xzwG4dGycva zn0~&6{b?G4m+H3)#!8g7>(Rp7%a1c?>x8IFpoaN8T@nyQHERn>z?QxrRwv^g|5{cX z1~&nurLy1AH#P&VmlY`JELq7pV*=IJhG6+jY}v?(n@c*;K*RPot3;mD5Pg;pLCOUz_=f9Zh!G7b%dIN~3Kh!C4{D>~d4wi)8_n4^2 zEUd|ouG#LxF;e1GXrYN!qk9EEu3}KbE#4(CP+RnowRlp0_}%br zcpY!Q0;k5P0}IBf0JQV(bw|R(WXn7ns9Kr>#>vrs-?gh%AxOa4da8uMk({%)yVRFJ zOGlk#c*+#67Y|3-aWpsXYVNzV8Wt6fbxHVB#Am^l?rj@(y%SXkGcDk6qSSTKm=U|k zi|3W7W5nB`{E{iE>c4<4u^m|i#xZvXVVJBrYzJfyDHTL{(4X$J zYl}5-y-sr^_s1=>#I)_zy~cr~&2;qq|EU2<5l*yvxLWsud_E?0RdfXia9U{E4RCBj z_Wzins1FE}W-J~jr_e1U%aWd~=D{-D@+7LsoY{S*E%=xB7^wdQ`>yyZaxFPyQWFqD ze=Q{Pwq>q#!rWS$!4rks%0+jg=}sgpq3CX`$~-PGW)g9jiqESqZ@f%K^j$xSfs|iM zgM{8e0Gwk#h-k@pjza=InDlSS%sj3S@xw0vBNRtqoJ<*vI5FK?8bYmamrK=9f)qcI zLuMM)*K$q1^-N?|+eRqhc#z?!gUL&anXp~FembaWI~DvI3Ql=)758-=uV_pi^WGNx zb~N58`G=rj|6|3NTEXOa?vMLv48Z@_hCS0m;WDx1rci4g=vddh$;knLKAhk7BSfQT zg(L+EnVnIu1l;0Qrk)#KskcHr(o>1EM_!&BP36(_o`)Z}*E#fB5QX5_ynsV0;g=ZP z$N3priE#g?wMVftw>1o0B_IfzkE?oN0b;jO^0GB%G*r}B%IM?yVYo8wBrATjrL8fy zTyf;ImI;jAFK_I4KfkeCyY8&7vAsnu35)KB&+Xi-tGqGSI8ch;F+CpAdWU7P z@OfcHFSA$bu#cJp0TG7EB=@72KUPNlB@KU{q>ooY38M%h=(A(D#xv4D0ntnzMs$g$ z$h@L(n4rx(bSx+cZEcf_ zCzq2WIgH35}DAUYk?0bko`B6Um9)TrLkS|H>3#}jCoP@S^r|5Gb4c(#^C~J z%A35YXwKx0=BX^bZ%$2QsMFtYd10%u1QVNT_r~71=h5FlXT$9Gii&K$ZXfOJuy;mH zHa8YEWxD&8i;)>Ezh`<8RA$!|7q8g&M%kmreP1`+u@$ZKWyIfi>8UffpjwE|?(X21 z&=VuY&61?~0^M3rk#S!q$R1lml$3!O|4p#{ZO|2ORHn!ie0l0YQ`G#3FcUZEESwcL zZu$nLO@i>K6lE%2D5qCmTAiESWN~Ek@p-`uOIZ{(^;WaXJO`Dw3!3RqrL2=Pk4>F5 z1x0>`-!X=ht$Jv?hFkJueQMXjKdg(MN z%#$tby||i)2e2)}7tT0yJd>Hn4})yKM>oq8WeB`sPuL+Y;|`somd5POCS(pBU1!+jb>Q}A5=$Z8$*LEJ4+3HkH24bPHoV_ulZiuRx$v!SamE~CE9rCu9 zS!uu~P_!nx>R9<3%?|7cWn7%A_%7pu8b(zRnW8@9$*Zbm3+%w+bDIh~# zihbLLmy|(XJH9l;r%j*(@Gja=iNtoyPa}Ny+p{00kcQ82sI|zTA5rC&ne~Sdsn8>y z$nr}TCJZdByK~*|!;SW@4e0~w8U<_$`6k;+&HAdw$-9%_aB#p};&*zNuK=HZV&UP7 z3*!0<8Z`++`_wlFEgEzb%Xe>wO3+v-fb{p^xX_7y)Fm?=td{v+?rE>cvsjL=Dc0tL z)o3s@cYVWse8z0+03I<-*RT;BS(fuU7hk8YSrhoJs5jIUJfIwPY@1HuM9Gt*(6zpf z?-_#UQi~-u#!-}IZw#$+lQ%H2o$=lca}}_gCM?m2oOmzjtv7;KF;`U0jc0E^U`6Ze z)Em7*3Z>|X(iEC{3qOnEqV^ZJBvIAafMlqlb=b(2+cIvIzxsn7x|>ozzxv|A3qj z3^IkgZ!fF!B>&k5i3%DFEZ?Ys(_yYT)c01vJEyNNB@Z&-Uq6oyX!YiDw(8>gt3htw z-n#d%AO3ny&80p;o_>EtSwU6LV|vvRhEe?wXe2vFy?{KqHUHNOH69j8f|#^2wRqmu z3mB$8EQhi7Wk^3zDo-abJom{Uf$aj%HN>PR0Hy*-2r!c^2kQnMV4~c}Aw5`H^9{frcg7#*vw! z+WMIhNl7N|h9+fsCC)C6E-6(hAz76c?l}P-Zh^M83>-cZ%DXDoS3Ltp~$&!pz1&}Qu1qz8psR|j1 zc_}%WdFc$|KpVVvb0P|G%#m~`D#{Oa!kW!5wDi4T-c*1=Y21$ zpBJrr(UH2_V2a>skKBp>yl%`6aunV))jz>}>A}ms?MI&*{fb>W$KqedsvBG0{4uD0 z)st1X;GFdBw0ps8%w>;>+I@Rcrgp@Q$&`tKL4bj2?^K_e-E#Z4L@{!-e){st?Bbt$ zEy4*Mum20psd3M0Z`^Dx@$zNDm1PetR(bC*IQSn37#M(lLI4n3Ww!a>txXT?)mOiG z^h8CZ;O71KbSd@Nokh#bmp6X2nf5=@z9=)}+qRUwM|d(+?Y~wQT4gaYq#L9%8TgiV z_pUhpddJgw1y7sTAD2JRrEZ*}C?uZ%=7WOV@tlA5hV&eRN?+OwoZ&?sYQOW&pIW B55NEb literal 0 HcmV?d00001 From 9a912d382dff6e18841dbe2f19036c181932fc39 Mon Sep 17 00:00:00 2001 From: Ritvik Kapila <61410899+RitvikKapila@users.noreply.github.com> Date: Thu, 13 Jun 2024 10:58:52 -0700 Subject: [PATCH 418/422] =?UTF-8?q?chore(migration=20examples):=20added=20?= =?UTF-8?q?KMS,=20raw=20AES=20and=20raw=20RSA=20keyring/MKP=E2=80=A6=20(#6?= =?UTF-8?q?87)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/src/migration/README.rst | 19 ++ .../migration_aws_kms_key_example.py | 166 +++++++++++ .../migration_raw_aes_key_example.py | 207 ++++++++++++++ .../migration_raw_rsa_key_example.py | 259 ++++++++++++++++++ ...migration_set_commitment_policy_example.py | 0 examples/test/migration/__init__.py | 3 + .../test_i_migration_aws_kms_key_example.py | 16 ++ .../test_i_migration_raw_aes_key_example.py | 15 + .../test_i_migration_raw_rsa_key_example.py | 15 + ...migration_set_commitment_policy_example.py | 2 +- 10 files changed, 701 insertions(+), 1 deletion(-) create mode 100644 examples/src/migration/README.rst create mode 100644 examples/src/migration/migration_aws_kms_key_example.py create mode 100644 examples/src/migration/migration_raw_aes_key_example.py create mode 100644 examples/src/migration/migration_raw_rsa_key_example.py rename examples/src/{ => migration}/migration_set_commitment_policy_example.py (100%) create mode 100644 examples/test/migration/__init__.py create mode 100644 examples/test/migration/test_i_migration_aws_kms_key_example.py create mode 100644 examples/test/migration/test_i_migration_raw_aes_key_example.py create mode 100644 examples/test/migration/test_i_migration_raw_rsa_key_example.py rename examples/test/{ => migration}/test_i_migration_set_commitment_policy_example.py (83%) diff --git a/examples/src/migration/README.rst b/examples/src/migration/README.rst new file mode 100644 index 000000000..41d3f5515 --- /dev/null +++ b/examples/src/migration/README.rst @@ -0,0 +1,19 @@ +################## +Migration Examples +################## + +The `Encryption SDK for Python`_ now uses the `AWS Cryptographic Material Providers Library`_. The MPL abstracts lower +level cryptographic materials management of encryption and decryption materials. + +This directory contains migration examples for: + +#. Moving to Keyrings from Master Key Providers: + * Migration example to AWS KMS keyring from AWS KMS Master Key Provider. + * Migration example to Raw AES keyring from Raw AES Master Key Provider. + * Migration example to Raw RSA keyring from Raw RSA Master Key Provider. + +#. Migration to newer versions of the ESDK (4.x+) from 1.x versions: + * Setting a 'CommitmentPolicy' during migration - If you have messages encrypted with 1.x versions of the ESDK (i.e. not using key commitment) and want to migrate to encrypt with key commitment using the keyring providers introduced in ESDK 4.x, this example will guide you on how to decrypt those messages using the new version of the ESDK. + +.. _AWS Cryptographic Material Providers Library: https://github.com/aws/aws-cryptographic-material-providers-library +.. _Encryption SDK for Python: https://github.com/aws/aws-encryption-sdk-python/tree/9c34aad60fc918c1a9186ec5215a451e8bfd0f65 \ No newline at end of file diff --git a/examples/src/migration/migration_aws_kms_key_example.py b/examples/src/migration/migration_aws_kms_key_example.py new file mode 100644 index 000000000..28b8193e3 --- /dev/null +++ b/examples/src/migration/migration_aws_kms_key_example.py @@ -0,0 +1,166 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +""" +This is a migration example for moving to the AWS KMS Keyring from AWS KMS master key provider (MKP) + +The AWS KMS keyring uses symmetric encryption KMS keys to generate, encrypt and +decrypt data keys. This example creates a KMS Keyring and KMS MKP and +then encrypts a custom input EXAMPLE_DATA with the same encryption context using both +the keyring and MKP. The example then decrypts the ciphertexts using both keyring and MKPs. +This example also includes some sanity checks for demonstration: +1. Decryption of these ciphertexts encrypted using keyring and MKP + is possible using both KMS keyring and KMS MKP +2. Both decrypted plaintexts are same and match EXAMPLE_DATA +These sanity checks are for demonstration in the example only. You do not need these in your code. + +Note: The ciphertexts obtained by encrypting EXAMPLE_DATA using keyring and MKP are not +the same because the ESDK generates different data keys each time for encryption of the data. +But both ciphertexts when decrypted using keyring and MKP should give the same plaintext result. + +For more information on how to use KMS keyrings, see +https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/use-kms-keyring.html +""" +import boto3 +from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders +from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig +from aws_cryptographic_materialproviders.mpl.models import CreateAwsKmsKeyringInput +from aws_cryptographic_materialproviders.mpl.references import IKeyring +from typing import Dict # noqa pylint: disable=wrong-import-order + +import aws_encryption_sdk + +EXAMPLE_DATA: bytes = b"Hello World" + +DEFAULT_ENCRYPTION_CONTEXT : Dict[str, str] = { + "encryption": "context", + "is not": "secret", + "but adds": "useful metadata", + "that can help you": "be confident that", + "the data you are handling": "is what you think it is", +} + + +def create_keyring( + kms_key_id: str, + aws_region="us-west-2" +): + """Demonstrate how to create an AWS KMS keyring. + + Usage: create_keyring(kms_key_id) + :param kms_key_id: KMS Key identifier for the KMS key you want to use. + :type kms_key_id: string + + For more information on KMS Key identifiers, see + https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#key-id + """ + # Create a boto3 client for KMS. + kms_client = boto3.client('kms', region_name=aws_region) + + # Create a KMS keyring + mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + config=MaterialProvidersConfig() + ) + + keyring_input: CreateAwsKmsKeyringInput = CreateAwsKmsKeyringInput( + kms_key_id=kms_key_id, + kms_client=kms_client + ) + + keyring: IKeyring = mat_prov.create_aws_kms_keyring( + input=keyring_input + ) + + return keyring + + +def create_key_provider( + kms_key_id: str +): + """Demonstrate how to create an AWS KMS master key provider. + + Usage: create_key_provider(kms_key_id) + :param kms_key_id: KMS Key identifier for the KMS key you want to use. + :type kms_key_id: string + + For more information on KMS Key identifiers, see + https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#key-id + """ + # Create a KMS master key provider. + key_provider = aws_encryption_sdk.StrictAwsKmsMasterKeyProvider(key_ids=[ + kms_key_id, + ]) + + return key_provider + + +def migration_aws_kms_key( + kms_key_id: str +): + """Demonstrate a migration example for moving to an AWS KMS keyring from AWS KMS MKP. + + Usage: migration_aws_kms_key(kms_key_id) + :param kms_key_id: KMS Key identifier for the KMS key you want to use for encryption and + decryption of your data keys. + :type kms_key_id: string + + For more information on KMS Key identifiers, see + https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#key-id + """ + client = aws_encryption_sdk.EncryptionSDKClient() + + # 1a. Create a AWS KMS Keyring + aws_kms_keyring = create_keyring(kms_key_id=kms_key_id) + + # 1b. Create a AWS KMS Master Key Provider + aws_kms_master_key_provider = create_key_provider(kms_key_id=kms_key_id) + + # 2a. Encrypt EXAMPLE_DATA using AWS KMS Keyring + ciphertext_keyring, _ = client.encrypt( + source=EXAMPLE_DATA, + keyring=aws_kms_keyring, + encryption_context=DEFAULT_ENCRYPTION_CONTEXT + ) + + # 2b. Encrypt EXAMPLE_DATA using AWS KMS Master Key Provider + ciphertext_mkp, _ = client.encrypt( + source=EXAMPLE_DATA, + key_provider=aws_kms_master_key_provider, + encryption_context=DEFAULT_ENCRYPTION_CONTEXT + ) + + # Note: The ciphertexts obtained by encrypting EXAMPLE_DATA using keyring and MKP + # (that is ciphertext_keyring and ciphertext_mkp) are not the same because the ESDK + # generates different data keys each time for encryption of the data. But both + # ciphertexts when decrypted using keyring and MKP should give the same plaintext result. + + # 3. Decrypt the ciphertext_keyring using both the keyring and MKP and ensure the + # resulting plaintext is the same and also equal to EXAMPLE_DATA + decrypted_ciphertext_keyring_using_keyring, _ = client.decrypt( + source=ciphertext_keyring, + keyring=aws_kms_keyring + ) + + decrypted_ciphertext_keyring_using_mkp, _ = client.decrypt( + source=ciphertext_keyring, + key_provider=aws_kms_master_key_provider + ) + + assert decrypted_ciphertext_keyring_using_keyring == decrypted_ciphertext_keyring_using_mkp \ + and decrypted_ciphertext_keyring_using_keyring == EXAMPLE_DATA, \ + "Decrypted outputs using keyring and master key provider are not the same" + + # 4. Decrypt the ciphertext_mkp using both the keyring and MKP and ensure the + # resulting plaintext is the same and also equal to EXAMPLE_DATA + decrypted_ciphertext_mkp_using_keyring, _ = client.decrypt( + source=ciphertext_mkp, + keyring=aws_kms_keyring + ) + + decrypted_ciphertext_mkp_using_mkp, _ = client.decrypt( + source=ciphertext_mkp, + key_provider=aws_kms_master_key_provider + ) + + assert decrypted_ciphertext_mkp_using_keyring == decrypted_ciphertext_mkp_using_mkp \ + and decrypted_ciphertext_mkp_using_keyring == EXAMPLE_DATA, \ + "Decrypted outputs using keyring and master key provider are not the same" diff --git a/examples/src/migration/migration_raw_aes_key_example.py b/examples/src/migration/migration_raw_aes_key_example.py new file mode 100644 index 000000000..772f83cf5 --- /dev/null +++ b/examples/src/migration/migration_raw_aes_key_example.py @@ -0,0 +1,207 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +""" +This is a migration example for moving to the Raw AES Keyring from Raw AES master key provider (MKP) + +The Raw AES keyring lets you use an AES symmetric key that you provide as a wrapping key that +protects your data key. You need to generate, store, and protect the key material, +preferably in a hardware security module (HSM) or key management system. Use a Raw AES keyring +when you need to provide the wrapping key and encrypt the data keys locally or offline. + +This example creates a Raw AES Keyring and Raw AES MKP and +then encrypts a custom input EXAMPLE_DATA with the same encryption context using both +the keyring and MKP. The example then decrypts the ciphertexts using both keyring and MKPs. +This example also includes some sanity checks for demonstration: +1. Decryption of these ciphertexts encrypted using keyring and MKP + is possible using both Raw AES keyring and Raw AES MKP +2. Both decrypted plaintexts are same and match EXAMPLE_DATA +These sanity checks are for demonstration in the example only. You do not need these in your code. + +Note: The ciphertexts obtained by encrypting EXAMPLE_DATA using keyring and MKP are not +the same because the ESDK generates different data keys each time for encryption of the data. +But both ciphertexts when decrypted using keyring and MKP will give the same plaintext result. + +For more information on how to use Raw AES keyrings, see +https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/use-raw-aes-keyring.html +""" +import secrets + +from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders +from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig +from aws_cryptographic_materialproviders.mpl.models import AesWrappingAlg, CreateRawAesKeyringInput +from aws_cryptographic_materialproviders.mpl.references import IKeyring +from typing import Dict # noqa pylint: disable=wrong-import-order + +import aws_encryption_sdk +from aws_encryption_sdk.identifiers import EncryptionKeyType, WrappingAlgorithm +from aws_encryption_sdk.internal.crypto.wrapping_keys import WrappingKey +from aws_encryption_sdk.key_providers.raw import RawMasterKeyProvider + +EXAMPLE_DATA: bytes = b"Hello World" + +DEFAULT_ENCRYPTION_CONTEXT : Dict[str, str] = { + "encryption": "context", + "is not": "secret", + "but adds": "useful metadata", + "that can help you": "be confident that", + "the data you are handling": "is what you think it is", +} + +DEFAULT_AES_256_STATIC_KEY = secrets.token_bytes(32) + +# The key namespace in the Raw keyrings is equivalent to Provider ID (or Provider) field +# in the Raw Master Key Providers +DEFAULT_KEY_NAME_SPACE = "Some managed raw keys" + +# The key name in the Raw keyrings is equivalent to the Key ID field +# in the Raw Master Key Providers +DEFAULT_KEY_NAME = "My 256-bit AES wrapping key" + + +def create_keyring(): + """Demonstrate how to create a Raw AES keyring. + + Usage: create_keyring() + """ + # We fix the static key in order to make the test deterministic + static_key = DEFAULT_AES_256_STATIC_KEY + + mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + config=MaterialProvidersConfig() + ) + + # The key namespace in the Raw keyrings is equivalent to Provider ID (or Provider) field + # in the Raw Master Key Providers + # The key name in the Raw keyrings is equivalent to the Key ID field + # in the Raw Master Key Providers + keyring_input: CreateRawAesKeyringInput = CreateRawAesKeyringInput( + key_namespace=DEFAULT_KEY_NAME_SPACE, + key_name=DEFAULT_KEY_NAME, + wrapping_key=static_key, + wrapping_alg=AesWrappingAlg.ALG_AES256_GCM_IV12_TAG16 + ) + + keyring: IKeyring = mat_prov.create_raw_aes_keyring( + input=keyring_input + ) + + return keyring + + +# This is a helper class necessary for the Raw AES master key provider +# In the StaticMasterKeyProvider, we fix the static key to +# DEFAULT_AES_256_STATIC_KEY in order to make the test deterministic. +# Thus, both the Raw AES keyring and Raw AES MKP have the same key +# and we are able to encrypt data using keyrings and decrypt using MKP and vice versa +# In practice, users should generate a new random key for each key id. +class StaticMasterKeyProvider(RawMasterKeyProvider): + """Generates 256-bit keys for each unique key ID.""" + + # The key namespace in the Raw keyrings is equivalent to Provider ID (or Provider) field + # in the Raw Master Key Providers + provider_id = DEFAULT_KEY_NAME_SPACE + + def __init__(self, **kwargs): # pylint: disable=unused-argument + """Initialize empty map of keys.""" + self._static_keys = {} + + def _get_raw_key(self, key_id): + """Returns a static, symmetric key for the specified key ID. + + :param str key_id: Key ID + :returns: Wrapping key that contains the specified static key + :rtype: :class:`aws_encryption_sdk.internal.crypto.WrappingKey` + """ + try: + static_key = self._static_keys[key_id] + except KeyError: + # We fix the static key in order to make the test deterministic + # In practice, you should get this key from a secure key management system such as an HSM. + static_key = DEFAULT_AES_256_STATIC_KEY + self._static_keys[key_id] = static_key + return WrappingKey( + wrapping_algorithm=WrappingAlgorithm.AES_256_GCM_IV12_TAG16_NO_PADDING, + wrapping_key=static_key, + wrapping_key_type=EncryptionKeyType.SYMMETRIC, + ) + + +def create_key_provider(): + """Demonstrate how to create a Raw AES master key provider. + + Usage: create_key_provider() + """ + # Create a Raw AES master key provider. + + # The key name in the Raw keyrings is equivalent to the Key ID field + # in the Raw Master Key Providers + key_id = DEFAULT_KEY_NAME + key_provider = StaticMasterKeyProvider() + key_provider.add_master_key(key_id) + + return key_provider + + +def migration_raw_aes_key(): + """Demonstrate a migration example for moving to a Raw AES keyring from Raw AES MKP. + + Usage: migration_raw_aes_key() + """ + client = aws_encryption_sdk.EncryptionSDKClient() + + # 1a. Create a Raw AES Keyring + raw_aes_keyring = create_keyring() + + # 1b. Create a Raw AES Master Key Provider + raw_aes_master_key_provider = create_key_provider() + + # 2a. Encrypt EXAMPLE_DATA using Raw AES Keyring + ciphertext_keyring, _ = client.encrypt( + source=EXAMPLE_DATA, + keyring=raw_aes_keyring, + encryption_context=DEFAULT_ENCRYPTION_CONTEXT + ) + + # 2b. Encrypt EXAMPLE_DATA using Raw AES Master Key Provider + ciphertext_mkp, _ = client.encrypt( + source=EXAMPLE_DATA, + key_provider=raw_aes_master_key_provider, + encryption_context=DEFAULT_ENCRYPTION_CONTEXT + ) + + # Note: The ciphertexts obtained by encrypting EXAMPLE_DATA using keyring and MKP + # (that is ciphertext_keyring and ciphertext_mkp) are not the same because the ESDK + # generates different data keys each time for encryption of the data. But both + # ciphertexts when decrypted using keyring and MKP will give the same plaintext result. + + # 3. Decrypt the ciphertext_keyring using both the keyring and MKP and ensure the + # resulting plaintext is the same and also equal to EXAMPLE_DATA + decrypted_ciphertext_keyring_using_keyring, _ = client.decrypt( + source=ciphertext_keyring, + keyring=raw_aes_keyring + ) + + decrypted_ciphertext_keyring_using_mkp, _ = client.decrypt( + source=ciphertext_keyring, + key_provider=raw_aes_master_key_provider + ) + + assert decrypted_ciphertext_keyring_using_keyring == decrypted_ciphertext_keyring_using_mkp \ + and decrypted_ciphertext_keyring_using_keyring == EXAMPLE_DATA, \ + "Decrypted outputs using keyring and master key provider are not the same" + + # 4. Decrypt the ciphertext_mkp using both the keyring and MKP and ensure the + # resulting plaintext is the same and also equal to EXAMPLE_DATA + decrypted_ciphertext_mkp_using_keyring, _ = client.decrypt( + source=ciphertext_mkp, + keyring=raw_aes_keyring + ) + + decrypted_ciphertext_mkp_using_mkp, _ = client.decrypt( + source=ciphertext_mkp, + key_provider=raw_aes_master_key_provider + ) + + assert decrypted_ciphertext_mkp_using_keyring == decrypted_ciphertext_mkp_using_mkp \ + and decrypted_ciphertext_mkp_using_keyring == EXAMPLE_DATA, \ + "Decrypted outputs using keyring and master key provider are not the same" diff --git a/examples/src/migration/migration_raw_rsa_key_example.py b/examples/src/migration/migration_raw_rsa_key_example.py new file mode 100644 index 000000000..7c6020a53 --- /dev/null +++ b/examples/src/migration/migration_raw_rsa_key_example.py @@ -0,0 +1,259 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +""" +This is a migration example for moving to the Raw RSA Keyring from Raw RSA master key provider (MKP) + +The Raw RSA keyring performs asymmetric encryption and decryption of data keys in local memory +with RSA public and private keys that you provide. In this example, we define the RSA keys to +encrypt and decrypt the data keys. + +You need to generate, store, and protect the private key, preferably in a +hardware security module (HSM) or key management system. +The encryption function encrypts the data key under the RSA public key. The decryption function +decrypts the data key using the private key. + +This example creates a Raw RSA Keyring and Raw RSA MKP and +then encrypts a custom input EXAMPLE_DATA with the same encryption context using both +the keyring and MKP. The example then decrypts the ciphertexts using both keyring and MKPs. +This example also includes some sanity checks for demonstration: +1. Decryption of these ciphertexts encrypted using keyring and MKP + is possible using both Raw RSA keyring and Raw RSA MKP +2. Both decrypted plaintexts are same and match EXAMPLE_DATA +These sanity checks are for demonstration in the example only. You do not need these in your code. + +Note: The ciphertexts obtained by encrypting EXAMPLE_DATA using keyring and MKP are not +the same because the ESDK generates different data keys each time for encryption of the data. +But both ciphertexts when decrypted using keyring and MKP will give the same plaintext result. + +For more information on how to use Raw RSA keyrings, see +https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/use-raw-rsa-keyring.html +""" +from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders +from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig +from aws_cryptographic_materialproviders.mpl.models import CreateRawRsaKeyringInput, PaddingScheme +from aws_cryptographic_materialproviders.mpl.references import IKeyring +from cryptography.hazmat.backends import default_backend as crypto_default_backend +from cryptography.hazmat.primitives import serialization as crypto_serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from typing import Dict # noqa pylint: disable=wrong-import-order + +import aws_encryption_sdk +from aws_encryption_sdk.identifiers import EncryptionKeyType, WrappingAlgorithm +from aws_encryption_sdk.internal.crypto.wrapping_keys import WrappingKey +from aws_encryption_sdk.key_providers.raw import RawMasterKeyProvider + +EXAMPLE_DATA: bytes = b"Hello World" + +DEFAULT_ENCRYPTION_CONTEXT : Dict[str, str] = { + "encryption": "context", + "is not": "secret", + "but adds": "useful metadata", + "that can help you": "be confident that", + "the data you are handling": "is what you think it is", +} + +# The key namespace in the Raw keyrings is equivalent to Provider ID (or Provider) field +# in the Raw Master Key Providers +DEFAULT_KEY_NAME_SPACE = "Some managed raw keys" + +# The key name in the Raw keyrings is equivalent to the Key ID field +# in the Raw Master Key Providers +DEFAULT_KEY_NAME = "My 4096-bit RSA wrapping key" + + +def generate_rsa_keys_helper(): + """Generates a 4096-bit RSA public and private key pair + + Usage: generate_rsa_keys_helper() + """ + ssh_rsa_exponent = 65537 + bit_strength = 4096 + key = rsa.generate_private_key( + backend=crypto_default_backend(), + public_exponent=ssh_rsa_exponent, + key_size=bit_strength + ) + + # This example choses a particular type of encoding, format and encryption_algorithm + # Users can choose the PublicFormat, PrivateFormat and encryption_algorithm that align most + # with their use-cases + public_key = key.public_key().public_bytes( + encoding=crypto_serialization.Encoding.PEM, + format=crypto_serialization.PublicFormat.SubjectPublicKeyInfo + ) + private_key = key.private_bytes( + encoding=crypto_serialization.Encoding.PEM, + format=crypto_serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=crypto_serialization.NoEncryption() + ) + + return public_key, private_key + + +DEFAULT_RSA_PUBLIC_KEY, DEFAULT_RSA_PRIVATE_KEY = generate_rsa_keys_helper() + + +def create_keyring(public_key, private_key): + """Demonstrate how to create a Raw RSA keyring using the key pair. + + Usage: create_keyring(public_key, private_key) + """ + mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + config=MaterialProvidersConfig() + ) + + # The key namespace in the Raw keyrings is equivalent to Provider ID (or Provider) field + # in the Raw Master Key Providers + # The key name in the Raw keyrings is equivalent to the Key ID field + # in the Raw Master Key Providers + keyring_input: CreateRawRsaKeyringInput = CreateRawRsaKeyringInput( + key_namespace=DEFAULT_KEY_NAME_SPACE, + key_name=DEFAULT_KEY_NAME, + padding_scheme=PaddingScheme.OAEP_SHA256_MGF1, + public_key=public_key, + private_key=private_key + ) + + keyring: IKeyring = mat_prov.create_raw_rsa_keyring( + input=keyring_input + ) + + return keyring + + +# This is a helper class necessary for the Raw RSA master key provider. +# In the StaticMasterKeyProvider, we fix the static key to +# DEFAULT_RSA_PRIVATE_KEY in order to make the test deterministic. +# Thus, both the Raw RSA keyring and Raw RSA MKP have the same private_key +# and we are able to encrypt data using keyrings and decrypt using MKP and vice versa +# In practice, users should generate a new random key pair for each key id. +class StaticMasterKeyProvider(RawMasterKeyProvider): + """Provides 4096-bit RSA keys consistently per unique key id.""" + + # The key namespace in the Raw keyrings is equivalent to Provider ID (or Provider) field + # in the Raw Master Key Providers + provider_id = DEFAULT_KEY_NAME_SPACE + + def __init__(self, **kwargs): # pylint: disable=unused-argument + """Initialize empty map of keys.""" + self._static_keys = {} + + def _get_raw_key(self, key_id): + """Retrieves a static, RSA key for the specified key id. + + :param str key_id: User-defined ID for the static key + :returns: Wrapping key that contains the specified static key + :rtype: :class:`aws_encryption_sdk.internal.crypto.WrappingKey` + """ + try: + static_key = self._static_keys[key_id] + except KeyError: + # We fix the static key in order to make the test deterministic + # In practice, you should get this key from a secure key management system such as an HSM. + # Also, in practice, users should generate a new key pair for each key id in + # the StaticMasterKeyProvider. + static_key = DEFAULT_RSA_PRIVATE_KEY + self._static_keys[key_id] = static_key + return WrappingKey( + wrapping_algorithm=WrappingAlgorithm.RSA_OAEP_SHA256_MGF1, + wrapping_key=static_key, + wrapping_key_type=EncryptionKeyType.PRIVATE, + ) + + +def create_key_provider(): + """Demonstrate how to create a Raw RSA master key provider. + + Usage: create_key_provider() + """ + # Create a Raw RSA master key provider. + + # The key name in the Raw keyrings is equivalent to the Key ID field + # in the Raw Master Key Providers + key_id = DEFAULT_KEY_NAME + + # In this example, we fix the static key to DEFAULT_RSA_PRIVATE_KEY in both the keyring + # and MKP (for MKP, we fix the static key in StaticMasterKeyProvider) in order to make + # the test deterministic. Thus, both the Raw RSA keyring and Raw RSA MKP have the same + # private_key and we are able to encrypt data using keyrings and decrypt using MKP + # and vice versa. In practice, users should generate a new key pair for each key id in + # the StaticMasterKeyProvider. + key_provider = StaticMasterKeyProvider() + key_provider.add_master_key(key_id) + + return key_provider + + +def migration_raw_rsa_key( + public_key=DEFAULT_RSA_PUBLIC_KEY, + private_key=DEFAULT_RSA_PRIVATE_KEY +): + """Demonstrate a migration example for moving to a Raw RSA keyring from Raw RSA MKP. + + Usage: migration_raw_rsa_key(public_key, private_key) + """ + client = aws_encryption_sdk.EncryptionSDKClient() + + # 1a. Create a Raw RSA Keyring + raw_rsa_keyring = create_keyring(public_key=public_key, private_key=private_key) + + # 1b. Create a Raw RSA Master Key Provider + + # In this example, we fix the static key to DEFAULT_RSA_PRIVATE_KEY in both the keyring + # and MKP (for MKP, we fix the static key in StaticMasterKeyProvider) in order to make + # the test deterministic. Thus, both the Raw RSA keyring and Raw RSA MKP have the same + # private_key and we are able to encrypt data using keyrings and decrypt using MKP + # and vice versa. In practice, users should generate a new key pair for each key id in + # the StaticMasterKeyProvider. + raw_rsa_master_key_provider = create_key_provider() + + # 2a. Encrypt EXAMPLE_DATA using Raw RSA Keyring + ciphertext_keyring, _ = client.encrypt( + source=EXAMPLE_DATA, + keyring=raw_rsa_keyring, + encryption_context=DEFAULT_ENCRYPTION_CONTEXT + ) + + # 2b. Encrypt EXAMPLE_DATA using Raw RSA Master Key Provider + ciphertext_mkp, _ = client.encrypt( + source=EXAMPLE_DATA, + key_provider=raw_rsa_master_key_provider, + encryption_context=DEFAULT_ENCRYPTION_CONTEXT + ) + + # Note: The ciphertexts obtained by encrypting EXAMPLE_DATA using keyring and MKP + # (that is ciphertext_keyring and ciphertext_mkp) are not the same because the ESDK + # generates different data keys each time for encryption of the data. But both + # ciphertexts when decrypted using keyring and MKP will give the same plaintext result. + + # 3. Decrypt the ciphertext_keyring using both the keyring and MKP and ensure the + # resulting plaintext is the same and also equal to EXAMPLE_DATA + decrypted_ciphertext_keyring_using_keyring, _ = client.decrypt( + source=ciphertext_keyring, + keyring=raw_rsa_keyring + ) + + decrypted_ciphertext_keyring_using_mkp, _ = client.decrypt( + source=ciphertext_keyring, + key_provider=raw_rsa_master_key_provider + ) + + assert decrypted_ciphertext_keyring_using_keyring == decrypted_ciphertext_keyring_using_mkp \ + and decrypted_ciphertext_keyring_using_keyring == EXAMPLE_DATA, \ + "Decrypted outputs using keyring and master key provider are not the same" + + # 4. Decrypt the ciphertext_mkp using both the keyring and MKP and ensure the + # resulting plaintext is the same and also equal to EXAMPLE_DATA + decrypted_ciphertext_mkp_using_keyring, _ = client.decrypt( + source=ciphertext_mkp, + keyring=raw_rsa_keyring + ) + + decrypted_ciphertext_mkp_using_mkp, _ = client.decrypt( + source=ciphertext_mkp, + key_provider=raw_rsa_master_key_provider + ) + + assert decrypted_ciphertext_mkp_using_keyring == decrypted_ciphertext_mkp_using_mkp \ + and decrypted_ciphertext_mkp_using_keyring == EXAMPLE_DATA, \ + "Decrypted outputs using keyring and master key provider are not the same" diff --git a/examples/src/migration_set_commitment_policy_example.py b/examples/src/migration/migration_set_commitment_policy_example.py similarity index 100% rename from examples/src/migration_set_commitment_policy_example.py rename to examples/src/migration/migration_set_commitment_policy_example.py diff --git a/examples/test/migration/__init__.py b/examples/test/migration/__init__.py new file mode 100644 index 000000000..120179eda --- /dev/null +++ b/examples/test/migration/__init__.py @@ -0,0 +1,3 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Stub module indicator to make linter configuration simpler.""" diff --git a/examples/test/migration/test_i_migration_aws_kms_key_example.py b/examples/test/migration/test_i_migration_aws_kms_key_example.py new file mode 100644 index 000000000..d4cf49ce9 --- /dev/null +++ b/examples/test/migration/test_i_migration_aws_kms_key_example.py @@ -0,0 +1,16 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Test suite for the migration_aws_kms_key_example.""" +import pytest + +from ...src.migration.migration_aws_kms_key_example import ( + migration_aws_kms_key, +) + +pytestmark = [pytest.mark.examples] + + +def test_migration_aws_kms_key(): + """Test function for migration of AWS KMS Keys.""" + kms_key_id = "arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f" + migration_aws_kms_key(kms_key_id) diff --git a/examples/test/migration/test_i_migration_raw_aes_key_example.py b/examples/test/migration/test_i_migration_raw_aes_key_example.py new file mode 100644 index 000000000..7601e7dc0 --- /dev/null +++ b/examples/test/migration/test_i_migration_raw_aes_key_example.py @@ -0,0 +1,15 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Test suite for the migration_raw_aes_key_example.""" +import pytest + +from ...src.migration.migration_raw_aes_key_example import ( + migration_raw_aes_key, +) + +pytestmark = [pytest.mark.examples] + + +def test_migration_raw_aes_key(): + """Test function for migration of Raw AES keys.""" + migration_raw_aes_key() diff --git a/examples/test/migration/test_i_migration_raw_rsa_key_example.py b/examples/test/migration/test_i_migration_raw_rsa_key_example.py new file mode 100644 index 000000000..9e111d25a --- /dev/null +++ b/examples/test/migration/test_i_migration_raw_rsa_key_example.py @@ -0,0 +1,15 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Test suite for the migration_raw_rsa_key_example.""" +import pytest + +from ...src.migration.migration_raw_rsa_key_example import ( + migration_raw_rsa_key, +) + +pytestmark = [pytest.mark.examples] + + +def test_migration_raw_rsa_key(): + """Test function for migration of Raw RSA keys.""" + migration_raw_rsa_key() diff --git a/examples/test/test_i_migration_set_commitment_policy_example.py b/examples/test/migration/test_i_migration_set_commitment_policy_example.py similarity index 83% rename from examples/test/test_i_migration_set_commitment_policy_example.py rename to examples/test/migration/test_i_migration_set_commitment_policy_example.py index afbc758bc..4620d64df 100644 --- a/examples/test/test_i_migration_set_commitment_policy_example.py +++ b/examples/test/migration/test_i_migration_set_commitment_policy_example.py @@ -3,7 +3,7 @@ """Test suite for the migration_set_commitment_policy_example.""" import pytest -from ..src.migration_set_commitment_policy_example import encrypt_and_decrypt_with_keyring +from ...src.migration.migration_set_commitment_policy_example import encrypt_and_decrypt_with_keyring pytestmark = [pytest.mark.examples] From 89042834dcd3f332d28d6399ca08e8a1d5a2166d Mon Sep 17 00:00:00 2001 From: Ritvik Kapila <61410899+RitvikKapila@users.noreply.github.com> Date: Wed, 26 Jun 2024 13:32:15 -0700 Subject: [PATCH 419/422] chore(custom_cmm_example.py): added test for custom_cmm_example.py (#690) --- examples/src/custom_mpl_cmm_example.py | 125 ++++++++++++++ ...cryptographic_materials_manager_example.py | 133 +++++++++++++++ examples/src/legacy/custom_cmm_example.py | 87 ++++++++++ examples/test/legacy/examples_test_utils.py | 2 +- .../test/legacy/test_i_custom_cmm_example.py | 51 ++++++ examples/test/legacy/v3_default_cmm.py | 159 ++++++++++++++++++ .../test_i_migration_aws_kms_key_example.py | 4 +- .../test_i_migration_raw_aes_key_example.py | 4 +- .../test_i_migration_raw_rsa_key_example.py | 4 +- .../test/test_i_custom_mpl_cmm_example.py | 39 +++++ ...cryptographic_materials_manager_example.py | 14 ++ tox.ini | 4 +- 12 files changed, 615 insertions(+), 11 deletions(-) create mode 100644 examples/src/custom_mpl_cmm_example.py create mode 100644 examples/src/default_cryptographic_materials_manager_example.py create mode 100644 examples/src/legacy/custom_cmm_example.py create mode 100644 examples/test/legacy/test_i_custom_cmm_example.py create mode 100644 examples/test/legacy/v3_default_cmm.py create mode 100644 examples/test/test_i_custom_mpl_cmm_example.py create mode 100644 examples/test/test_i_default_cryptographic_materials_manager_example.py diff --git a/examples/src/custom_mpl_cmm_example.py b/examples/src/custom_mpl_cmm_example.py new file mode 100644 index 000000000..544ca5f00 --- /dev/null +++ b/examples/src/custom_mpl_cmm_example.py @@ -0,0 +1,125 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +""" +Example to create a custom implementation of the MPL's ICryptographicMaterialsManager class and use it with the ESDK. + +The cryptographic materials manager (CMM) assembles the cryptographic materials that are used +to encrypt and decrypt data. The cryptographic materials include plaintext and encrypted data keys, +and an optional message signing key. + +Cryptographic Materials Managers (CMMs) are composable; if you just want to extend the behavior of +the default CMM, you can do this as demonstrated in this example. This is the easiest approach if +you are just adding a small check to the CMM methods, as in this example. + +If your use case calls for fundamentally changing aspects of the default CMM, you can also write +your own implementation without extending an existing CMM. The default CMM's implementation is a +good reference to use if you need to write a custom CMM implementation from scratch. +Custom implementations of CMMs must implement get_encryption_materials and decrypt_materials. + +For more information on a default implementation of a CMM, +please look at the default_cryptographic_materials_manager_example.py example. + +For more information on Cryptographic Material Managers, see +https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/concepts.html#crypt-materials-manager +""" + +from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders +from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig +from aws_cryptographic_materialproviders.mpl.models import ( + CreateDefaultCryptographicMaterialsManagerInput, + SignatureAlgorithmNone, +) +from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager, IKeyring + +import aws_encryption_sdk +from aws_encryption_sdk import CommitmentPolicy + + +# Custom CMM implementation using the MPL. +# This CMM only allows encryption/decryption using signing algorithms. +# It wraps an underlying CMM implementation and checks its materials +# to ensure that it is only using signed encryption algorithms. +class MPLCustomSigningSuiteOnlyCMM(ICryptographicMaterialsManager): + """Example custom crypto materials manager class.""" + + def __init__(self, keyring: IKeyring, cmm: ICryptographicMaterialsManager = None) -> None: + """Constructor for MPLCustomSigningSuiteOnlyCMM class.""" + if cmm is not None: + self.underlying_cmm = cmm + else: + mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + config=MaterialProvidersConfig() + ) + + # Create a CryptographicMaterialsManager for encryption and decryption + cmm_input: CreateDefaultCryptographicMaterialsManagerInput = \ + CreateDefaultCryptographicMaterialsManagerInput( + keyring=keyring + ) + + self.underlying_cmm: ICryptographicMaterialsManager = \ + mat_prov.create_default_cryptographic_materials_manager( + input=cmm_input + ) + + def get_encryption_materials(self, param): + """Provides encryption materials appropriate for the request for the custom CMM. + + :param aws_cryptographic_materialproviders.mpl.models.GetEncryptionMaterialsInput param: Input object to + provide to a crypto material manager's `get_encryption_materials` method. + :returns: Encryption materials output + :rtype: aws_cryptographic_materialproviders.mpl.models.GetEncryptionMaterialsOutput + """ + materials = self.underlying_cmm.get_encryption_materials(param) + if isinstance(materials.encryption_materials.algorithm_suite.signature, SignatureAlgorithmNone): + raise ValueError( + "Algorithm provided to MPLCustomSigningSuiteOnlyCMM" + + " is not a supported signing algorithm: " + str(materials.encryption_materials.algorithm_suite) + ) + return materials + + def decrypt_materials(self, param): + """Provides decryption materials appropriate for the request for the custom CMM. + + :param aws_cryptographic_materialproviders.mpl.models.DecryptMaterialsInput param: Input object to provide + to a crypto material manager's `decrypt_materials` method. + :returns: Decryption materials output + :rtype: aws_cryptographic_materialproviders.mpl.models.GetDecryptionMaterialsOutput + """ + materials = self.underlying_cmm.decrypt_materials(param) + if isinstance(materials.decryption_materials.algorithm_suite.signature, SignatureAlgorithmNone): + raise ValueError( + "Algorithm provided to MPLCustomSigningSuiteOnlyCMM" + + " is not a supported signing algorithm: " + str(materials.decryption_materials.algorithm_suite) + ) + return materials + + +EXAMPLE_DATA: bytes = b"Hello World" + + +def encrypt_decrypt_with_cmm( + cmm: ICryptographicMaterialsManager +): + """Encrypts and decrypts a string using a custom CMM. + + :param ICryptographicMaterialsManager cmm: CMM to use for encryption and decryption + """ + # Set up an encryption client with an explicit commitment policy. Note that if you do not explicitly choose a + # commitment policy, REQUIRE_ENCRYPT_REQUIRE_DECRYPT is used by default. + client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + + # Encrypt the plaintext source data + ciphertext, _ = client.encrypt( + source=EXAMPLE_DATA, + materials_manager=cmm + ) + + # Decrypt the ciphertext + cycled_plaintext, _ = client.decrypt( + source=ciphertext, + materials_manager=cmm + ) + + # Verify that the "cycled" (encrypted, then decrypted) plaintext is identical to the source plaintext + assert cycled_plaintext == EXAMPLE_DATA diff --git a/examples/src/default_cryptographic_materials_manager_example.py b/examples/src/default_cryptographic_materials_manager_example.py new file mode 100644 index 000000000..bfffc36d9 --- /dev/null +++ b/examples/src/default_cryptographic_materials_manager_example.py @@ -0,0 +1,133 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +""" +This example sets up the default Cryptographic Material Managers (CMM). + +The default cryptographic materials manager (CMM) assembles the cryptographic materials +that are used to encrypt and decrypt data. The cryptographic materials include +plaintext and encrypted data keys, and an optional message signing key. +This example creates a CMM and then encrypts a custom input EXAMPLE_DATA +with an encryption context. Creating a CMM involves taking a keyring as input, +and we use an AWS KMS Keyring for this example. +This example also includes some sanity checks for demonstration: +1. Ciphertext and plaintext data are not the same +2. Encryption context is correct in the decrypted message header +3. Decrypted plaintext value matches EXAMPLE_DATA +These sanity checks are for demonstration in the example only. You do not need these in your code. + +For more information on Cryptographic Material Managers, see +https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/concepts.html#crypt-materials-manager +""" +import sys + +import boto3 +from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders +from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig +from aws_cryptographic_materialproviders.mpl.models import ( + CreateAwsKmsKeyringInput, + CreateDefaultCryptographicMaterialsManagerInput, +) +from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager, IKeyring +from typing import Dict # noqa pylint: disable=wrong-import-order + +import aws_encryption_sdk +from aws_encryption_sdk import CommitmentPolicy + +# TODO-MPL: Remove this as part of removing PYTHONPATH hacks. +MODULE_ROOT_DIR = '/'.join(__file__.split("/")[:-1]) + +sys.path.append(MODULE_ROOT_DIR) + +EXAMPLE_DATA: bytes = b"Hello World" + + +def encrypt_and_decrypt_with_default_cmm( + kms_key_id: str +): + """Demonstrate an encrypt/decrypt cycle using default Cryptographic Material Managers. + + Usage: encrypt_and_decrypt_with_default_cmm(kms_key_id) + :param kms_key_id: KMS Key identifier for the KMS key you want to use for encryption and + decryption of your data keys. + :type kms_key_id: string + + For more information on KMS Key identifiers, see + https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#key-id + """ + # 1. Instantiate the encryption SDK client. + # This builds the client with the REQUIRE_ENCRYPT_REQUIRE_DECRYPT commitment policy, + # which enforces that this client only encrypts using committing algorithm suites and enforces + # that this client will only decrypt encrypted messages that were created with a committing + # algorithm suite. + # This is the default commitment policy if you were to build the client as + # `client = aws_encryption_sdk.EncryptionSDKClient()`. + client = aws_encryption_sdk.EncryptionSDKClient( + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT + ) + + # 2. Create encryption context. + # Remember that your encryption context is NOT SECRET. + # For more information, see + # https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/concepts.html#encryption-context + encryption_context: Dict[str, str] = { + "encryption": "context", + "is not": "secret", + "but adds": "useful metadata", + "that can help you": "be confident that", + "the data you are handling": "is what you think it is", + } + + # 3. Create a KMS keyring to use with the CryptographicMaterialsManager + kms_client = boto3.client('kms', region_name="us-west-2") + + mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + config=MaterialProvidersConfig() + ) + + keyring_input: CreateAwsKmsKeyringInput = CreateAwsKmsKeyringInput( + kms_key_id=kms_key_id, + kms_client=kms_client + ) + + kms_keyring: IKeyring = mat_prov.create_aws_kms_keyring( + input=keyring_input + ) + + # 4. Create a CryptographicMaterialsManager for encryption and decryption + cmm_input: CreateDefaultCryptographicMaterialsManagerInput = \ + CreateDefaultCryptographicMaterialsManagerInput( + keyring=kms_keyring + ) + + cmm: ICryptographicMaterialsManager = mat_prov.create_default_cryptographic_materials_manager( + input=cmm_input + ) + + # 5. Encrypt the data with the encryptionContext. + ciphertext, _ = client.encrypt( + source=EXAMPLE_DATA, + materials_manager=cmm, + encryption_context=encryption_context + ) + + # 6. Demonstrate that the ciphertext and plaintext are different. + # (This is an example for demonstration; you do not need to do this in your own code.) + assert ciphertext != EXAMPLE_DATA, \ + "Ciphertext and plaintext data are the same. Invalid encryption" + + # 7. Decrypt your encrypted data using the same cmm you used on encrypt. + plaintext_bytes, dec_header = client.decrypt( + source=ciphertext, + materials_manager=cmm + ) + + # 8. Demonstrate that the encryption context is correct in the decrypted message header + # (This is an example for demonstration; you do not need to do this in your own code.) + for k, v in encryption_context.items(): + assert v == dec_header.encryption_context[k], \ + "Encryption context does not match expected values" + + # 9. Demonstrate that the decrypted plaintext is identical to the original plaintext. + # (This is an example for demonstration; you do not need to do this in your own code.) + assert plaintext_bytes == EXAMPLE_DATA, \ + "Decrypted plaintext should be identical to the original plaintext. Invalid decryption" diff --git a/examples/src/legacy/custom_cmm_example.py b/examples/src/legacy/custom_cmm_example.py new file mode 100644 index 000000000..07e8ca50b --- /dev/null +++ b/examples/src/legacy/custom_cmm_example.py @@ -0,0 +1,87 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Example to create a custom implementation of the native ESDK CryptoMaterialsManager class.""" + +import aws_encryption_sdk +from aws_encryption_sdk import CommitmentPolicy, StrictAwsKmsMasterKeyProvider +from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager +from aws_encryption_sdk.materials_managers.default import DefaultCryptoMaterialsManager + + +# Custom CMM implementation. +# This CMM only allows encryption/decryption using signing algorithms. +# It wraps an underlying CMM implementation and checks its materials +# to ensure that it is only using signed encryption algorithms. +class CustomSigningSuiteOnlyCMM(CryptoMaterialsManager): + """Example custom crypto materials manager class.""" + + def __init__(self, master_key_provider: StrictAwsKmsMasterKeyProvider) -> None: + """Constructor for CustomSigningSuiteOnlyCMM class.""" + self.underlying_cmm = DefaultCryptoMaterialsManager(master_key_provider) + + def get_encryption_materials(self, request): + """Provides encryption materials appropriate for the request for the custom CMM. + + :param EncryptionMaterialsRequest request: Request object to provide to a + crypto material manager's `get_encryption_materials` method. + :returns: Encryption materials + :rtype: EncryptionMaterials + """ + materials = self.underlying_cmm.get_encryption_materials(request) + if not materials.algorithm.is_signing(): + raise ValueError( + "Algorithm provided to CustomSigningSuiteOnlyCMM" + + " is not a supported signing algorithm: " + materials.algorithm + ) + return materials + + def decrypt_materials(self, request): + """Provides decryption materials appropriate for the request for the custom CMM. + + :param DecryptionMaterialsRequest request: Request object to provide to a + crypto material manager's `decrypt_materials` method. + """ + if not request.algorithm.is_signing(): + raise ValueError( + "Algorithm provided to CustomSigningSuiteOnlyCMM" + + " is not a supported signing algorithm: " + request.algorithm + ) + return self.underlying_cmm.decrypt_materials(request) + + +def encrypt_decrypt_with_cmm( + cmm: CryptoMaterialsManager, + source_plaintext: str +): + """Encrypts and decrypts a string using a custom CMM. + + :param CryptoMaterialsManager cmm: CMM to use for encryption and decryption + :param bytes source_plaintext: Data to encrypt + """ + # Set up an encryption client with an explicit commitment policy. Note that if you do not explicitly choose a + # commitment policy, REQUIRE_ENCRYPT_REQUIRE_DECRYPT is used by default. + client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + + # Encrypt the plaintext source data + ciphertext, encryptor_header = client.encrypt( + source=source_plaintext, + materials_manager=cmm + ) + + # Decrypt the ciphertext + cycled_plaintext, decrypted_header = client.decrypt( + source=ciphertext, + materials_manager=cmm + ) + + # Verify that the "cycled" (encrypted, then decrypted) plaintext is identical to the source plaintext + assert cycled_plaintext == source_plaintext + + # Verify that the encryption context used in the decrypt operation includes all key pairs from + # the encrypt operation. (The SDK can add pairs, so don't require an exact match.) + # + # In production, always use a meaningful encryption context. In this sample, we omit the + # encryption context (no key pairs). + assert all( + pair in decrypted_header.encryption_context.items() for pair in encryptor_header.encryption_context.items() + ) diff --git a/examples/test/legacy/examples_test_utils.py b/examples/test/legacy/examples_test_utils.py index 3f877e301..8787e0f38 100644 --- a/examples/test/legacy/examples_test_utils.py +++ b/examples/test/legacy/examples_test_utils.py @@ -39,7 +39,7 @@ from integration_test_utils import ( # noqa pylint: disable=unused-import,import-error get_cmk_arn, - get_second_cmk_arn, get_mrk_arn, + get_second_cmk_arn, get_second_mrk_arn, ) diff --git a/examples/test/legacy/test_i_custom_cmm_example.py b/examples/test/legacy/test_i_custom_cmm_example.py new file mode 100644 index 000000000..397230090 --- /dev/null +++ b/examples/test/legacy/test_i_custom_cmm_example.py @@ -0,0 +1,51 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Test suite for encryption and decryption using custom CMM.""" +import botocore.session +import pytest + +import aws_encryption_sdk + +from ...src.legacy.custom_cmm_example import CustomSigningSuiteOnlyCMM, encrypt_decrypt_with_cmm +from .examples_test_utils import get_cmk_arn, static_plaintext +from .v3_default_cmm import V3DefaultCryptoMaterialsManager + +pytestmark = [pytest.mark.examples] + + +def test_custom_cmm_example(): + """Test method for encryption and decryption using V3 default CMM.""" + plaintext = static_plaintext + cmk_arn = get_cmk_arn() + botocore_session = botocore.session.Session() + + # Create a KMS master key provider. + kms_kwargs = dict(key_ids=[cmk_arn]) + if botocore_session is not None: + kms_kwargs["botocore_session"] = botocore_session + master_key_provider = aws_encryption_sdk.StrictAwsKmsMasterKeyProvider(**kms_kwargs) + + # Create the custom signing CMM using the master_key_provider + cmm = CustomSigningSuiteOnlyCMM(master_key_provider=master_key_provider) + + encrypt_decrypt_with_cmm(cmm=cmm, + source_plaintext=plaintext) + + +def test_v3_default_cmm(): + """Test method for encryption and decryption using V3 default CMM.""" + plaintext = static_plaintext + cmk_arn = get_cmk_arn() + botocore_session = botocore.session.Session() + + # Create a KMS master key provider. + kms_kwargs = dict(key_ids=[cmk_arn]) + if botocore_session is not None: + kms_kwargs["botocore_session"] = botocore_session + master_key_provider = aws_encryption_sdk.StrictAwsKmsMasterKeyProvider(**kms_kwargs) + + # Create the V3 default CMM (V3DefaultCryptoMaterialsManager) using the master_key_provider + cmm = V3DefaultCryptoMaterialsManager(master_key_provider=master_key_provider) + + encrypt_decrypt_with_cmm(cmm=cmm, + source_plaintext=plaintext) diff --git a/examples/test/legacy/v3_default_cmm.py b/examples/test/legacy/v3_default_cmm.py new file mode 100644 index 000000000..f077e26c9 --- /dev/null +++ b/examples/test/legacy/v3_default_cmm.py @@ -0,0 +1,159 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Copy-paste of the V3 default CMM with small changes to pass linters.""" +import logging + +import attr + +from aws_encryption_sdk.exceptions import MasterKeyProviderError, SerializationError +from aws_encryption_sdk.identifiers import CommitmentPolicy +from aws_encryption_sdk.internal.crypto.authentication import Signer, Verifier +from aws_encryption_sdk.internal.crypto.elliptic_curve import generate_ecc_signing_key +from aws_encryption_sdk.internal.defaults import ALGORITHM, ALGORITHM_COMMIT_KEY, ENCODED_SIGNER_KEY +from aws_encryption_sdk.internal.str_ops import to_str +from aws_encryption_sdk.internal.utils import prepare_data_keys +from aws_encryption_sdk.internal.utils.commitment import ( + validate_commitment_policy_on_decrypt, + validate_commitment_policy_on_encrypt, +) +from aws_encryption_sdk.key_providers.base import MasterKeyProvider +from aws_encryption_sdk.materials_managers import DecryptionMaterials, EncryptionMaterials +from aws_encryption_sdk.materials_managers.base import CryptoMaterialsManager + +_LOGGER = logging.getLogger(__name__) + + +@attr.s(hash=False) +class V3DefaultCryptoMaterialsManager(CryptoMaterialsManager): + """Copy of the default crypto material manager for ESDK V3. + + This is a copy-paste of the DefaultCryptoMaterialsManager implementation + from the V3 ESDK commit: 98b5eb7c2bd7d1b2a3380aacfa508e8721c4d8a9 + This CMM is used to explicitly assert that the V3 implementation of + the DefaultCMM is compatible with future version's logic, + which implicitly asserts that custom implementations of V3-compatible CMMs + are also compatible with future version's logic. + + :param master_key_provider: Master key provider to use + :type master_key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider + """ + + master_key_provider = attr.ib(validator=attr.validators.instance_of(MasterKeyProvider)) + +# pylint: disable=no-self-use + def _generate_signing_key_and_update_encryption_context(self, algorithm, encryption_context): + """Generates a signing key based on the provided algorithm. + + :param algorithm: Algorithm for which to generate signing key + :type algorithm: aws_encryption_sdk.identifiers.Algorithm + :param dict encryption_context: Encryption context from request + :returns: Signing key bytes + :rtype: bytes or None + """ + _LOGGER.debug("Generating signing key") + if algorithm.signing_algorithm_info is None: + return None + + signer = Signer(algorithm=algorithm, key=generate_ecc_signing_key(algorithm=algorithm)) + encryption_context[ENCODED_SIGNER_KEY] = to_str(signer.encoded_public_key()) + return signer.key_bytes() + + def get_encryption_materials(self, request): + """Creates encryption materials using underlying master key provider. + + :param request: encryption materials request + :type request: aws_encryption_sdk.materials_managers.EncryptionMaterialsRequest + :returns: encryption materials + :rtype: aws_encryption_sdk.materials_managers.EncryptionMaterials + :raises MasterKeyProviderError: if no master keys are available from the underlying master key provider + :raises MasterKeyProviderError: if the primary master key provided by the underlying master key provider + is not included in the full set of master keys provided by that provider + :raises ActionNotAllowedError: if the commitment policy in the request is violated by the algorithm being + used + """ + default_algorithm = ALGORITHM + if request.commitment_policy in ( + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT, + ): + default_algorithm = ALGORITHM_COMMIT_KEY + algorithm = request.algorithm if request.algorithm is not None else default_algorithm + + validate_commitment_policy_on_encrypt(request.commitment_policy, request.algorithm) + + encryption_context = request.encryption_context.copy() + + signing_key = self._generate_signing_key_and_update_encryption_context(algorithm, encryption_context) + + primary_master_key, master_keys = self.master_key_provider.master_keys_for_encryption( + encryption_context=encryption_context, + plaintext_rostream=request.plaintext_rostream, + plaintext_length=request.plaintext_length, + ) + if not master_keys: + raise MasterKeyProviderError("No Master Keys available from Master Key Provider") + if primary_master_key not in master_keys: + raise MasterKeyProviderError("Primary Master Key not in provided Master Keys") + + data_encryption_key, encrypted_data_keys = prepare_data_keys( + primary_master_key=primary_master_key, + master_keys=master_keys, + algorithm=algorithm, + encryption_context=encryption_context, + ) + + _LOGGER.debug("Post-encrypt encryption context: %s", encryption_context) + + return EncryptionMaterials( + algorithm=algorithm, + data_encryption_key=data_encryption_key, + encrypted_data_keys=encrypted_data_keys, + encryption_context=encryption_context, + signing_key=signing_key, + ) + +# pylint: disable=no-self-use + def _load_verification_key_from_encryption_context(self, algorithm, encryption_context): + """Loads the verification key from the encryption context if used by algorithm suite. + + :param algorithm: Algorithm for which to generate signing key + :type algorithm: aws_encryption_sdk.identifiers.Algorithm + :param dict encryption_context: Encryption context from request + :returns: Raw verification key + :rtype: bytes + :raises SerializationError: if algorithm suite requires message signing and no verification key is found + """ + encoded_verification_key = encryption_context.get(ENCODED_SIGNER_KEY, None) + + if algorithm.signing_algorithm_info is not None and encoded_verification_key is None: + raise SerializationError("No signature verification key found in header for signed algorithm.") + + if algorithm.signing_algorithm_info is None: + if encoded_verification_key is not None: + raise SerializationError("Signature verification key found in header for non-signed algorithm.") + return None + + verifier = Verifier.from_encoded_point(algorithm=algorithm, encoded_point=encoded_verification_key) + return verifier.key_bytes() + + def decrypt_materials(self, request): + """Obtains a plaintext data key from one or more encrypted data keys + using underlying master key provider. + + :param request: decrypt materials request + :type request: aws_encryption_sdk.materials_managers.DecryptionMaterialsRequest + :returns: decryption materials + :rtype: aws_encryption_sdk.materials_managers.DecryptionMaterials + """ + validate_commitment_policy_on_decrypt(request.commitment_policy, request.algorithm) + + data_key = self.master_key_provider.decrypt_data_key_from_list( + encrypted_data_keys=request.encrypted_data_keys, + algorithm=request.algorithm, + encryption_context=request.encryption_context, + ) + verification_key = self._load_verification_key_from_encryption_context( + algorithm=request.algorithm, encryption_context=request.encryption_context + ) + + return DecryptionMaterials(data_key=data_key, verification_key=verification_key) diff --git a/examples/test/migration/test_i_migration_aws_kms_key_example.py b/examples/test/migration/test_i_migration_aws_kms_key_example.py index d4cf49ce9..612a896ba 100644 --- a/examples/test/migration/test_i_migration_aws_kms_key_example.py +++ b/examples/test/migration/test_i_migration_aws_kms_key_example.py @@ -3,9 +3,7 @@ """Test suite for the migration_aws_kms_key_example.""" import pytest -from ...src.migration.migration_aws_kms_key_example import ( - migration_aws_kms_key, -) +from ...src.migration.migration_aws_kms_key_example import migration_aws_kms_key pytestmark = [pytest.mark.examples] diff --git a/examples/test/migration/test_i_migration_raw_aes_key_example.py b/examples/test/migration/test_i_migration_raw_aes_key_example.py index 7601e7dc0..d5e4f7789 100644 --- a/examples/test/migration/test_i_migration_raw_aes_key_example.py +++ b/examples/test/migration/test_i_migration_raw_aes_key_example.py @@ -3,9 +3,7 @@ """Test suite for the migration_raw_aes_key_example.""" import pytest -from ...src.migration.migration_raw_aes_key_example import ( - migration_raw_aes_key, -) +from ...src.migration.migration_raw_aes_key_example import migration_raw_aes_key pytestmark = [pytest.mark.examples] diff --git a/examples/test/migration/test_i_migration_raw_rsa_key_example.py b/examples/test/migration/test_i_migration_raw_rsa_key_example.py index 9e111d25a..238dcbaab 100644 --- a/examples/test/migration/test_i_migration_raw_rsa_key_example.py +++ b/examples/test/migration/test_i_migration_raw_rsa_key_example.py @@ -3,9 +3,7 @@ """Test suite for the migration_raw_rsa_key_example.""" import pytest -from ...src.migration.migration_raw_rsa_key_example import ( - migration_raw_rsa_key, -) +from ...src.migration.migration_raw_rsa_key_example import migration_raw_rsa_key pytestmark = [pytest.mark.examples] diff --git a/examples/test/test_i_custom_mpl_cmm_example.py b/examples/test/test_i_custom_mpl_cmm_example.py new file mode 100644 index 000000000..d98b6b6b9 --- /dev/null +++ b/examples/test/test_i_custom_mpl_cmm_example.py @@ -0,0 +1,39 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Test suite for encryption and decryption using custom CMM.""" +import boto3 +import pytest +from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders +from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig +from aws_cryptographic_materialproviders.mpl.models import CreateAwsKmsKeyringInput +from aws_cryptographic_materialproviders.mpl.references import IKeyring + +from ..src.custom_mpl_cmm_example import MPLCustomSigningSuiteOnlyCMM, encrypt_decrypt_with_cmm + +pytestmark = [pytest.mark.examples] + + +def test_custom_cmm_example(): + """Test method for encryption and decryption using V3 default CMM.""" + kms_key_id = "arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f" + + # Create KMS keyring to use with the CMM + kms_client = boto3.client('kms', region_name="us-west-2") + + mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + config=MaterialProvidersConfig() + ) + + keyring_input: CreateAwsKmsKeyringInput = CreateAwsKmsKeyringInput( + kms_key_id=kms_key_id, + kms_client=kms_client + ) + + kms_keyring: IKeyring = mat_prov.create_aws_kms_keyring( + input=keyring_input + ) + + # Create the custom MPL signing CMM using the keyring + cmm = MPLCustomSigningSuiteOnlyCMM(keyring=kms_keyring) + + encrypt_decrypt_with_cmm(cmm=cmm) diff --git a/examples/test/test_i_default_cryptographic_materials_manager_example.py b/examples/test/test_i_default_cryptographic_materials_manager_example.py new file mode 100644 index 000000000..8a18f655d --- /dev/null +++ b/examples/test/test_i_default_cryptographic_materials_manager_example.py @@ -0,0 +1,14 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Test suite for the default Cryptographic Materials Manager example.""" +import pytest + +from ..src.default_cryptographic_materials_manager_example import encrypt_and_decrypt_with_default_cmm + +pytestmark = [pytest.mark.examples] + + +def test_encrypt_and_decrypt_with_default_cmm(): + """Test function for encrypt and decrypt using the default Cryptographic Materials Manager example.""" + kms_key_id = "arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f" + encrypt_and_decrypt_with_default_cmm(kms_key_id) diff --git a/tox.ini b/tox.ini index 9152f51a6..28be94a63 100644 --- a/tox.ini +++ b/tox.ini @@ -169,10 +169,12 @@ deps = {[testenv:flake8]deps} commands = flake8 examples/src/ flake8 \ - # Ingore D103 missing docstring errors in tests (test names should be self-documenting) + # Ignore D103 missing docstring errors in tests (test names should be self-documenting) # E203 is not PEP8 compliant https://github.com/ambv/black#slices # W503 is not PEP8 compliant https://github.com/ambv/black#line-breaks--binary-operators --ignore D103,E203,W503 \ + # copy-paste test for v3_default_cmm; intentionally not changing code + --per-file-ignores 'examples/test/legacy/v3_default_cmm.py: D205,D400,D401' \ examples/test/ [testenv:pylint] From abede7443e7942fda1c8478fb29308ddf64e8ac1 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Tue, 30 Jul 2024 10:56:50 -0700 Subject: [PATCH 420/422] chore(MPL): Update README and primary pydocs (#658) --- README.rst | 284 +++++++----------- src/aws_encryption_sdk/__init__.py | 88 ++++-- .../internal/deprecation.py | 32 ++ 3 files changed, 204 insertions(+), 200 deletions(-) create mode 100644 src/aws_encryption_sdk/internal/deprecation.py diff --git a/README.rst b/README.rst index 6cd8c6824..fa6effb1f 100644 --- a/README.rst +++ b/README.rst @@ -39,6 +39,12 @@ Required Prerequisites * boto3 >= 1.10.0 * attrs +Recommended Prerequisites +========================= + +* aws-cryptographic-material-providers: >= TODO.TODO.TODO (TODO-MPL: versionme) + * Requires Python 3.11+. + Installation ============ @@ -49,42 +55,71 @@ Installation .. code:: - $ pip install aws-encryption-sdk + $ pip install "aws-encryption-sdk[MPL]" +The `[MPL]` suffix also installs the `AWS Cryptographic Material Providers Library (MPL)`_. +This is a library that contains constructs for encrypting and decrypting your data. +We highly recommend installing the MPL. +However, if you do not wish to install the MPL, omit the `[MPL]` suffix. Concepts ======== -There are four main concepts that you need to understand to use this library: +There are three main concepts that you need to understand to use this library: + +Data Keys +--------- +Data keys are the encryption keys that are used to encrypt your data. If your algorithm suite +uses a key derivation function, the data key is used to generate the key that directly encrypts the data. + +Keyrings +-------- +Keyrings are resources that generate, encrypt, and decrypt data keys. +You specify a keyring when encrypting and the same or a different keyring when decrypting. + +Note: You must also install the `AWS Cryptographic Material Providers Library (MPL)`_ to create and use keyrings. + +For more information, see the `AWS Documentation for Keyrings`_. Cryptographic Materials Managers -------------------------------- Cryptographic materials managers (CMMs) are resources that collect cryptographic materials and prepare them for use by the Encryption SDK core logic. -An example of a CMM is the default CMM, which is automatically generated anywhere a caller provides a master -key provider. The default CMM collects encrypted data keys from all master keys referenced by the master key -provider. +An example of a CMM is the default CMM, +which is automatically generated anywhere a caller provides a keyring. + +Note: You must also install the `AWS Cryptographic Material Providers Library (MPL)`_ +to create and use CMMs that use keyrings. +CMMs that use master key providers have been marked as legacy since v4 of this library. -An example of a more advanced CMM is the caching CMM, which caches cryptographic materials provided by another CMM. +Legacy Concepts +=============== +This section describes legacy concepts introduced in earlier versions of this library. +These components have been superseded by new components in the `AWS Cryptographic Material Providers Library (MPL)`_. +Please avoid using these components, and instead use components in the MPL. Master Key Providers -------------------- Master key providers are resources that provide master keys. -An example of a master key provider is `AWS KMS`_. To encrypt data in this client, a ``MasterKeyProvider`` object must contain at least one ``MasterKey`` object. ``MasterKeyProvider`` objects can also contain other ``MasterKeyProvider`` objects. +NOTE: Master key providers are legacy components +and have been superseded by keyrings +provided by the `AWS Cryptographic Material Providers Library (MPL)`_. +Please install this library and migrate master key providers to keyring interfaces. + Master Keys ----------- Master keys generate, encrypt, and decrypt data keys. -An example of a master key is a `KMS customer master key (CMK)`_. +An example of a master key is an `AWS KMS key`_. -Data Keys ---------- -Data keys are the encryption keys that are used to encrypt your data. If your algorithm suite -uses a key derivation function, the data key is used to generate the key that directly encrypts the data. +NOTE: Master keys are legacy constructs +and have been superseded by keyrings +provided by the `AWS Cryptographic Material Providers Library (MPL)`_. +Please install this library and migrate master key providers to keyring interfaces. ***** Usage @@ -110,147 +145,71 @@ version of the AWS Encryption SDK, we recommend using the default value. ) -You must then create an instance of either a master key provider or a CMM. The examples in this -readme use the ``StrictAwsKmsMasterKeyProvider`` class. +You must then create an instance of either a keyring (with the MPL installed) or a CMM. +Note: You must also install the `AWS Cryptographic Material Providers Library (MPL)`_ to use keyrings. +(You may also provide an instance of a legacy master key provider, but this is not recommended.) + +AwsKmsMultiKeyring +================== -StrictAwsKmsMasterKeyProvider -============================= -A ``StrictAwsKmsMasterKeyProvider`` is configured with an explicit list of AWS KMS -CMKs with which to encrypt and decrypt data. On encryption, it encrypts the plaintext with all -configured CMKs. On decryption, it only attempts to decrypt ciphertexts that have been wrapped -with a CMK that matches one of the configured CMK ARNs. +An ``AwsKmsMultiKeyring`` is configured with a generator keyring and a list of +child keyrings of type ``AwsKmsKeyring``. The effect is like using several keyrings +in a series. When you use a multi-keyring to encrypt data, any of the wrapping keys +in any of its keyrings can decrypt that data. -To create a ``StrictAwsKmsMasterKeyProvider`` you must provide one or more CMKs. For providers that will only -be used for encryption, you can use any valid `KMS key identifier`_. For providers that will be used for decryption, you -must use the key ARN; key ids, alias names, and alias ARNs are not supported. +On encryption, the generator keyring generates and encrypts the plaintext data key. +Then, all of the wrapping keys in all of the child keyrings encrypt the same plaintext data key. +The final `encrypted message`_ will include a copy of the data key encrypted by each configured key. +On decryption, the AWS Encryption SDK uses the keyrings to try to decrypt one of the encrypted data keys. +The keyrings are called in the order that they are specified in the multi-keyring. +Processing stops as soon as any key in any keyring can decrypt an encrypted data key. -Because the ``StrictAwsKmsMasterKeyProvider`` uses the `boto3 SDK`_ to interact with `AWS KMS`_, +An individual ``AwsKmsKeyring`` in an ``AwsKmsMultiKeyring`` is configured with an +AWS KMS key ARN. +For keyrings that will only be used for encryption, +you can use any valid `KMS key identifier`_. +For providers that will be used for decryption, +you must use the key ARN. +Key ids, alias names, and alias ARNs are not supported for decryption. + +Because the ``AwsKmsMultiKeyring`` uses the `boto3 SDK`_ to interact with `AWS KMS`_, it requires AWS Credentials. To provide these credentials, use the `standard means by which boto3 locates credentials`_ or provide a -pre-existing instance of a ``botocore session`` to the ``StrictAwsKmsMasterKeyProvider``. +pre-existing instance of a ``botocore session`` to the ``AwsKmsMultiKeyring``. This latter option can be useful if you have an alternate way to store your AWS credentials or you want to reuse an existing instance of a botocore session in order to decrease startup costs. +You can also add KMS keys from multiple regions to the ``AwsKmsMultiKeyring``. -If you configure the the ``StrictAwsKmsMasterKeyProvider`` with multiple CMKs, the `final message`_ -will include a copy of the data key encrypted by each configured CMK. +See `examples/src/aws_kms_multi_keyring_example.py`_ for a code example configuring and using +a ``AwsKmsMultiKeyring`` with the ``EncryptionSDKClient``. -.. code:: python - - import aws_encryption_sdk - - kms_key_provider = aws_encryption_sdk.StrictAwsKmsMasterKeyProvider(key_ids=[ - 'arn:aws:kms:us-east-1:2222222222222:key/22222222-2222-2222-2222-222222222222', - 'arn:aws:kms:us-east-1:3333333333333:key/33333333-3333-3333-3333-333333333333' - ]) - -You can add CMKs from multiple regions to the ``StrictAwsKmsMasterKeyProvider``. - -.. code:: python - - import aws_encryption_sdk - - kms_key_provider = aws_encryption_sdk.StrictAwsKmsMasterKeyProvider(key_ids=[ - 'arn:aws:kms:us-east-1:2222222222222:key/22222222-2222-2222-2222-222222222222', - 'arn:aws:kms:us-west-2:3333333333333:key/33333333-3333-3333-3333-333333333333', - 'arn:aws:kms:ap-northeast-1:4444444444444:key/44444444-4444-4444-4444-444444444444' - ]) - - -DiscoveryAwsKmsMasterKeyProvider -================================ -We recommend using a ``StrictAwsKmsMasterKeyProvider`` in order to ensure that you can only -encrypt and decrypt data using the AWS KMS CMKs you expect. However, if you are unable to -explicitly identify the AWS KMS CMKs that should be used for decryption, you can instead -use a ``DiscoveryAwsKmsMasterKeyProvider`` for decryption operations. This provider +AwsKmsDiscoveryKeyring +====================== +We recommend using an ``AwsKmsMultiKeyring`` in order to ensure that you can only +encrypt and decrypt data using the AWS KMS key ARN you expect. However, if you are unable to +explicitly identify the AWS KMS key ARNs that should be used for decryption, you can instead +use an ``AwsKmsDiscoveryKeyring`` for decryption operations. This provider attempts decryption of any ciphertexts as long as they match a ``DiscoveryFilter`` that you configure. A ``DiscoveryFilter`` consists of a list of AWS account ids and an AWS partition. +If you do not want to filter the set of allowed accounts, you can also omit the ``discovery_filter`` argument. -.. code:: python +Note that an ``AwsKmsDiscoveryKeyring`` cannot be used for encryption operations. - import aws_encryption_sdk - from aws_encryption_sdk.key_providers.kms import DiscoveryFilter +See `examples/src/aws_kms_discovery_keyring_example.py`_ for a code example configuring and using +an ``AwsKmsDiscoveryKeyring`` with the ``EncryptionSDKClient``. - discovery_filter = DiscoveryFilter( - account_ids=['222222222222', '333333333333'], - partition='aws' - ) - kms_key_provider = aws_encryption_sdk.DiscoveryAwsKmsMasterKeyProvider( - discovery_filter=discovery_filter - ) - -If you do not want to filter the set of allowed accounts, you can also omit the ``discovery_filter`` argument. - -Note that a ``DiscoveryAwsKmsMasterKeyProvider`` cannot be used for encryption operations. Encryption and Decryption ========================= -After you create an instance of an ``EncryptionSDKClient`` and a ``MasterKeyProvider``, you can use either of -the client's two ``encrypt``/``decrypt`` functions to encrypt and decrypt your data. - -.. code:: python - - import aws_encryption_sdk - from aws_encryption_sdk.identifiers import CommitmentPolicy - - client = aws_encryption_sdk.EncryptionSDKClient( - commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT - ) - - kms_key_provider = aws_encryption_sdk.StrictAwsKmsMasterKeyProvider(key_ids=[ - 'arn:aws:kms:us-east-1:2222222222222:key/22222222-2222-2222-2222-222222222222', - 'arn:aws:kms:us-east-1:3333333333333:key/33333333-3333-3333-3333-333333333333' - ]) - my_plaintext = b'This is some super secret data! Yup, sure is!' - - my_ciphertext, encryptor_header = client.encrypt( - source=my_plaintext, - key_provider=kms_key_provider - ) - - decrypted_plaintext, decryptor_header = client.decrypt( - source=my_ciphertext, - key_provider=kms_key_provider - ) - - assert my_plaintext == decrypted_plaintext - assert encryptor_header.encryption_context == decryptor_header.encryption_context - -You can provide an `encryption context`_: a form of additional authenticating information. - -.. code:: python - - import aws_encryption_sdk - from aws_encryption_sdk.identifiers import CommitmentPolicy - - client = aws_encryption_sdk.EncryptionSDKClient( - commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT - ) +After you create an instance of an ``EncryptionSDKClient`` and a ``Keyring``, you can use +the client's ``encrypt`` and ``decrypt`` functions to encrypt and decrypt your data. - kms_key_provider = aws_encryption_sdk.StrictAwsKmsMasterKeyProvider(key_ids=[ - 'arn:aws:kms:us-east-1:2222222222222:key/22222222-2222-2222-2222-222222222222', - 'arn:aws:kms:us-east-1:3333333333333:key/33333333-3333-3333-3333-333333333333' - ]) - my_plaintext = b'This is some super secret data! Yup, sure is!' - - my_ciphertext, encryptor_header = client.encrypt( - source=my_plaintext, - key_provider=kms_key_provider, - encryption_context={ - 'not really': 'a secret', - 'but adds': 'some authentication' - } - ) - - decrypted_plaintext, decryptor_header = client.decrypt( - source=my_ciphertext, - key_provider=kms_key_provider - ) - - assert my_plaintext == decrypted_plaintext - assert encryptor_header.encryption_context == decryptor_header.encryption_context +You can also provide an `encryption context`_: a form of additional authenticating information. +See code in the `examples/src/`_ directory for code examples configuring and using +keyrings and encryption context with the ``EncryptionSDKClient``. Streaming ========= @@ -259,57 +218,19 @@ memory at once, you can use this library's streaming clients directly. The strea file-like objects, and behave exactly as you would expect a Python file object to behave, offering context manager and iteration support. -.. code:: python - - import aws_encryption_sdk - from aws_encryption_sdk.identifiers import CommitmentPolicy - import filecmp - - client = aws_encryption_sdk.EncryptionSDKClient( - commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT - ) - - kms_key_provider = aws_encryption_sdk.StrictAwsKmsMasterKeyProvider(key_ids=[ - 'arn:aws:kms:us-east-1:2222222222222:key/22222222-2222-2222-2222-222222222222', - 'arn:aws:kms:us-east-1:3333333333333:key/33333333-3333-3333-3333-333333333333' - ]) - plaintext_filename = 'my-secret-data.dat' - ciphertext_filename = 'my-encrypted-data.ct' - - with open(plaintext_filename, 'rb') as pt_file, open(ciphertext_filename, 'wb') as ct_file: - with client.stream( - mode='e', - source=pt_file, - key_provider=kms_key_provider - ) as encryptor: - for chunk in encryptor: - ct_file.write(chunk) - - new_plaintext_filename = 'my-decrypted-data.dat' - - with open(ciphertext_filename, 'rb') as ct_file, open(new_plaintext_filename, 'wb') as pt_file: - with client.stream( - mode='d', - source=ct_file, - key_provider=kms_key_provider - ) as decryptor: - for chunk in decryptor: - pt_file.write(chunk) - - assert filecmp.cmp(plaintext_filename, new_plaintext_filename) - assert encryptor.header.encryption_context == decryptor.header.encryption_context +See `examples/src/file_streaming_example.py`_ for a code example streaming data to and from files. Performance Considerations ========================== Adjusting the frame size can significantly improve the performance of encrypt/decrypt operations with this library. -Processing each frame in a framed message involves a certain amount of overhead. If you are encrypting a large file, -increasing the frame size can offer potentially significant performance gains. We recommend that you tune these values +Processing each frame in a framed message involves a certain amount of overhead. If you are encrypting a large file, +increasing the frame size can offer potentially significant performance gains. We recommend that you tune these values to your use-case in order to obtain peak performance. Thread safety ========================== -The ``EncryptionSDKClient`` and all provided ``CryptoMaterialsManager`` are thread safe. +The ``EncryptionSDKClient`` and all provided ``CryptoMaterialsManager`` in this library are thread safe. But instances of ``BaseKMSMasterKeyProvider`` MUST not be shared between threads, for the reasons outlined in `the boto3 docs `_. @@ -323,17 +244,28 @@ Finally, while the ``CryptoMaterialsCache`` is thread safe, sharing entries in that cache across threads needs to be done carefully (see the !Note about partition name `in the API Docs `_). +**Important:** Components from the `AWS Cryptographic Material Providers Library (MPL)`_ +have separate thread safety considerations. +For more information, see the note on thread safety in that project's README (TODO-MPL: link) + + .. _AWS Encryption SDK: https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/introduction.html .. _cryptography: https://cryptography.io/en/latest/ .. _cryptography installation guide: https://cryptography.io/en/latest/installation/ .. _Read the Docs: http://aws-encryption-sdk-python.readthedocs.io/en/latest/ .. _GitHub: https://github.com/aws/aws-encryption-sdk-python/ .. _AWS KMS: https://docs.aws.amazon.com/kms/latest/developerguide/overview.html -.. _KMS customer master key (CMK): https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#master_keys +.. _AWS KMS key: https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#master_keys .. _KMS key identifier: https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#key-id .. _boto3 SDK: https://boto3.readthedocs.io/en/latest/ .. _standard means by which boto3 locates credentials: https://boto3.readthedocs.io/en/latest/guide/configuration.html -.. _final message: https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/message-format.html +.. _encrypted message: https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/message-format.html .. _encryption context: https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#encrypt_context .. _Security issue notifications: ./CONTRIBUTING.md#security-issue-notifications .. _Support Policy: ./SUPPORT_POLICY.rst +.. _AWS Cryptographic Material Providers Library (MPL): https://github.com/aws/aws-cryptographic-material-providers-library +.. _AWS Documentation for Keyrings: https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/choose-keyring.html +.. _examples/src/aws_kms_multi_keyring_example.py: https://github.com/aws/aws-encryption-sdk-python/blob/master/examples/src/aws_kms_multi_keyring_example.py +.. _examples/src/aws_kms_discovery_keyring_example.py: https://github.com/aws/aws-encryption-sdk-python/blob/master/examples/src/aws_kms_discovery_keyring_example.py +.. _examples/src/: https://github.com/aws/aws-encryption-sdk-python/tree/master/examples/src/ +.. _examples/src/file_streaming_example.py: https://github.com/aws/aws-encryption-sdk-python/blob/master/examples/src/file_streaming_example.py diff --git a/src/aws_encryption_sdk/__init__.py b/src/aws_encryption_sdk/__init__.py index 21d3084a1..9f3525349 100644 --- a/src/aws_encryption_sdk/__init__.py +++ b/src/aws_encryption_sdk/__init__.py @@ -35,7 +35,8 @@ class EncryptionSDKClientConfig(object): :param commitment_policy: The commitment policy to apply to encryption and decryption requests :type commitment_policy: aws_encryption_sdk.materials_manager.identifiers.CommitmentPolicy - :param max_encrypted_data_keys: The maximum number of encrypted data keys to allow during encryption and decryption + :param max_encrypted_data_keys: The maximum number of encrypted data keys to allow during + encryption and decryption :type max_encrypted_data_keys: None or positive int """ @@ -94,15 +95,26 @@ def encrypt(self, **kwargs): .. code:: python + >>> import boto3 + >>> from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders + >>> from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig + >>> from aws_cryptographic_materialproviders.mpl.models import CreateAwsKmsKeyringInput + >>> from aws_cryptographic_materialproviders.mpl.references import IKeyring >>> import aws_encryption_sdk >>> client = aws_encryption_sdk.EncryptionSDKClient() - >>> kms_key_provider = aws_encryption_sdk.StrictAwsKmsMasterKeyProvider(key_ids=[ - ... 'arn:aws:kms:us-east-1:2222222222222:key/22222222-2222-2222-2222-222222222222', - ... 'arn:aws:kms:us-east-1:3333333333333:key/33333333-3333-3333-3333-333333333333' - ... ]) + >>> mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + ... config=MaterialProvidersConfig() + ... ) + >>> keyring_input: CreateAwsKmsKeyringInput = CreateAwsKmsKeyringInput( + ... kms_key_id='arn:aws:kms:us-east-1:2222222222222:key/22222222-2222-2222-2222-222222222222', + ... kms_client=boto3.client('kms', region_name="us-west-2") + ... ) + >>> kms_keyring: IKeyring = mat_prov.create_aws_kms_keyring( + ... input=keyring_input + ... ) >>> my_ciphertext, encryptor_header = client.encrypt( ... source=my_plaintext, - ... key_provider=kms_key_provider + ... keyring=kms_keyring ... ) :param config: Client configuration object (config or individual parameters required) @@ -110,11 +122,14 @@ def encrypt(self, **kwargs): :param source: Source data to encrypt or decrypt :type source: str, bytes, io.IOBase, or file :param materials_manager: `CryptoMaterialsManager` that returns cryptographic materials - (requires either `materials_manager` or `key_provider`) + (requires either `materials_manager` or `keyring`) :type materials_manager: aws_encryption_sdk.materials_managers.base.CryptoMaterialsManager :param key_provider: `MasterKeyProvider` that returns data keys for encryption (requires either `materials_manager` or `key_provider`) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider + :param keyring: `IKeyring` that returns keyring for encryption + (requires either `materials_manager` or `keyring`) + :type keyring: aws_cryptographic_materialproviders.mpl.references.IKeyring :param int source_length: Length of source data (optional) .. note:: @@ -124,7 +139,7 @@ def encrypt(self, **kwargs): .. note:: If `source_length` and `materials_manager` are both provided, the total plaintext bytes encrypted will not be allowed to exceed `source_length`. To maintain backwards compatibility, - this is not enforced if a `key_provider` is provided. + this is not enforced if a `keyring` is provided. :param dict encryption_context: Dictionary defining encryption context :param algorithm: Algorithm to use for encryption @@ -148,15 +163,26 @@ def decrypt(self, **kwargs): .. code:: python + >>> import boto3 + >>> from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders + >>> from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig + >>> from aws_cryptographic_materialproviders.mpl.models import CreateAwsKmsKeyringInput + >>> from aws_cryptographic_materialproviders.mpl.references import IKeyring >>> import aws_encryption_sdk >>> client = aws_encryption_sdk.EncryptionSDKClient() - >>> kms_key_provider = aws_encryption_sdk.StrictAwsKmsMasterKeyProvider(key_ids=[ - ... 'arn:aws:kms:us-east-1:2222222222222:key/22222222-2222-2222-2222-222222222222', - ... 'arn:aws:kms:us-east-1:3333333333333:key/33333333-3333-3333-3333-333333333333' - ... ]) + >>> mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + ... config=MaterialProvidersConfig() + ... ) + >>> keyring_input: CreateAwsKmsKeyringInput = CreateAwsKmsKeyringInput( + ... kms_key_id='arn:aws:kms:us-east-1:2222222222222:key/22222222-2222-2222-2222-222222222222', + ... kms_client=boto3.client('kms', region_name="us-west-2") + ... ) + >>> kms_keyring: IKeyring = mat_prov.create_aws_kms_keyring( + ... input=keyring_input + ... ) >>> my_plaintext, decryptor_header = client.decrypt( ... source=my_ciphertext, - ... key_provider=kms_key_provider + ... keyring=kms_keyring ... ) :param config: Client configuration object (config or individual parameters required) @@ -164,11 +190,14 @@ def decrypt(self, **kwargs): :param source: Source data to encrypt or decrypt :type source: str, bytes, io.IOBase, or file :param materials_manager: `CryptoMaterialsManager` that returns cryptographic materials - (requires either `materials_manager` or `key_provider`) + (requires either `materials_manager` or `keyring`) :type materials_manager: aws_encryption_sdk.materials_managers.base.CryptoMaterialsManager :param key_provider: `MasterKeyProvider` that returns data keys for decryption (requires either `materials_manager` or `key_provider`) :type key_provider: aws_encryption_sdk.key_providers.base.MasterKeyProvider + :param keyring: `IKeyring` that returns keyring for encryption + (requires either `materials_manager` or `keyring`) + :type keyring: aws_cryptographic_materialproviders.mpl.references.IKeyring :param int source_length: Length of source data (optional) .. note:: @@ -208,28 +237,39 @@ def stream(self, **kwargs): .. code:: python + >>> import boto3 + >>> from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders + >>> from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig + >>> from aws_cryptographic_materialproviders.mpl.models import CreateAwsKmsKeyringInput + >>> from aws_cryptographic_materialproviders.mpl.references import IKeyring >>> import aws_encryption_sdk >>> client = aws_encryption_sdk.EncryptionSDKClient() - >>> kms_key_provider = aws_encryption_sdk.StrictAwsKmsMasterKeyProvider(key_ids=[ - ... 'arn:aws:kms:us-east-1:2222222222222:key/22222222-2222-2222-2222-222222222222', - ... 'arn:aws:kms:us-east-1:3333333333333:key/33333333-3333-3333-3333-333333333333' - ... ]) + >>> mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders( + ... config=MaterialProvidersConfig() + ... ) + >>> keyring_input: CreateAwsKmsKeyringInput = CreateAwsKmsKeyringInput( + ... kms_key_id='arn:aws:kms:us-east-1:2222222222222:key/22222222-2222-2222-2222-222222222222', + ... kms_client=boto3.client('kms', region_name="us-west-2") + ... ) + >>> kms_keyring: IKeyring = mat_prov.create_aws_kms_keyring( + ... input=keyring_input + ... ) >>> plaintext_filename = 'my-secret-data.dat' >>> ciphertext_filename = 'my-encrypted-data.ct' >>> with open(plaintext_filename, 'rb') as pt_file, open(ciphertext_filename, 'wb') as ct_file: - ... with client.stream( + ... with client.stream( ... mode='e', ... source=pt_file, - ... key_provider=kms_key_provider + ... keyring=kms_keyring ... ) as encryptor: ... for chunk in encryptor: - ... ct_file.write(chunk) - >>> new_plaintext_filename = 'my-decrypted-data.dat' - >>> with open(ciphertext_filename, 'rb') as ct_file, open(new_plaintext_filename, 'wb') as pt_file: + ... ct_file.write(chunk) + >>> decrypted_filename = 'my-decrypted-data.dat' + >>> with open(ciphertext_filename, 'rb') as ct_file, open(decrypted_filename, 'wb') as pt_file: ... with client.stream( ... mode='d', ... source=ct_file, - ... key_provider=kms_key_provider + ... keyring=kms_keyring ... ) as decryptor: ... for chunk in decryptor: ... pt_file.write(chunk) diff --git a/src/aws_encryption_sdk/internal/deprecation.py b/src/aws_encryption_sdk/internal/deprecation.py new file mode 100644 index 000000000..18e587237 --- /dev/null +++ b/src/aws_encryption_sdk/internal/deprecation.py @@ -0,0 +1,32 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Module containing utilities for deprecated components.""" +import functools +import warnings + + +def deprecated(reason): + """Decorator to apply to classes to emit deprecation warnings.""" + def decorator(cls): + # If class does not define init, + # its default init it Python's object.__init__, + # which does nothing, but cannot be wrapped. + if cls.__init__ is object.__init__: + # Make a new init that just emits this deprecation warning. + def new_init(self, *args, **kwargs): # pylint: disable=unused-argument + warnings.warn(f"{cls.__name__} is deprecated: {reason}", + category=DeprecationWarning, stacklevel=2) + else: + original_init = cls.__init__ + + # Wrap the original init method with a deprecation warning. + @functools.wraps(cls.__init__) + def new_init(self, *args, **kwargs): + warnings.warn(f"{cls.__name__} is deprecated: {reason}", + category=DeprecationWarning, stacklevel=2) + original_init(self, *args, **kwargs) + + cls.__init__ = new_init + return cls + + return decorator From 864d29b253fe69f11a58bcc01303b76d66900023 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 5 Aug 2024 12:24:30 -0700 Subject: [PATCH 421/422] chore: Change MPL branch, remove PYTHONPATH workarounds (#683) --- .github/workflows/ci_tests.yaml | 6 +----- examples/src/aws_kms_discovery_keyring_example.py | 6 ------ .../aws_kms_discovery_multi_keyring_example.py | 6 ------ examples/src/aws_kms_keyring_example.py | 6 ------ .../src/aws_kms_mrk_discovery_keyring_example.py | 6 ------ ...aws_kms_mrk_discovery_multi_keyring_example.py | 6 ------ examples/src/aws_kms_mrk_keyring_example.py | 6 ------ examples/src/aws_kms_mrk_multi_keyring_example.py | 6 ------ examples/src/aws_kms_multi_keyring_example.py | 6 ------ examples/src/aws_kms_rsa_keyring_example.py | 6 ------ examples/src/branch_key_id_supplier_example.py | 8 ++++---- ...ult_cryptographic_materials_manager_example.py | 7 ------- examples/src/file_streaming_example.py | 6 ------ examples/src/hierarchical_keyring_example.py | 6 ------ examples/src/legacy/module_.py | 1 - .../migration_set_commitment_policy_example.py | 6 ------ examples/src/module_.py | 1 - examples/src/multi_keyring_example.py | 6 ------ examples/src/raw_aes_keyring_example.py | 6 ------ examples/src/raw_rsa_keyring_example.py | 6 ------ examples/src/required_encryption_context_cmm.py | 6 ------ .../src/set_encryption_algorithm_suite_example.py | 6 ------ performance_tests/requirements_mpl.txt | 2 +- requirements_mpl.txt | 2 +- setup.py | 2 +- test_vector_handlers/requirements_mpl.txt | 2 +- .../manifests/full_message/decrypt.py | 15 ++++++++++++++- .../awses_test_vectors/manifests/mpl_keyring.py | 2 +- 28 files changed, 24 insertions(+), 126 deletions(-) delete mode 100644 examples/src/legacy/module_.py delete mode 100644 examples/src/module_.py diff --git a/.github/workflows/ci_tests.yaml b/.github/workflows/ci_tests.yaml index 7ec273ba2..1a3229683 100644 --- a/.github/workflows/ci_tests.yaml +++ b/.github/workflows/ci_tests.yaml @@ -25,11 +25,7 @@ jobs: matrix: os: - ubuntu-latest - # Windows fails due to "No module named 'Wrappers'" - # This SHOULD be fixed once Dafny generates fully-qualified import statements - # (i.e. doo files, per-package module names) - # Disable for now - # - windows-latest + - windows-latest - macos-12 python: - 3.8 diff --git a/examples/src/aws_kms_discovery_keyring_example.py b/examples/src/aws_kms_discovery_keyring_example.py index 4695e8783..d78121bc3 100644 --- a/examples/src/aws_kms_discovery_keyring_example.py +++ b/examples/src/aws_kms_discovery_keyring_example.py @@ -32,7 +32,6 @@ For more information on how to use KMS Discovery keyrings, see https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/use-kms-keyring.html#kms-keyring-discovery """ -import sys import boto3 from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders @@ -49,11 +48,6 @@ from aws_encryption_sdk import CommitmentPolicy from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError -# TODO-MPL: Remove this as part of removing PYTHONPATH hacks. -MODULE_ROOT_DIR = '/'.join(__file__.split("/")[:-1]) - -sys.path.append(MODULE_ROOT_DIR) - EXAMPLE_DATA: bytes = b"Hello World" diff --git a/examples/src/aws_kms_discovery_multi_keyring_example.py b/examples/src/aws_kms_discovery_multi_keyring_example.py index 60d74ff31..9381a740b 100644 --- a/examples/src/aws_kms_discovery_multi_keyring_example.py +++ b/examples/src/aws_kms_discovery_multi_keyring_example.py @@ -29,7 +29,6 @@ For more information on how to use KMS Discovery keyrings, see https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/use-kms-keyring.html#kms-keyring-discovery """ -import sys import boto3 from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders @@ -45,11 +44,6 @@ import aws_encryption_sdk from aws_encryption_sdk import CommitmentPolicy -# TODO-MPL: Remove this as part of removing PYTHONPATH hacks. -MODULE_ROOT_DIR = '/'.join(__file__.split("/")[:-1]) - -sys.path.append(MODULE_ROOT_DIR) - EXAMPLE_DATA: bytes = b"Hello World" diff --git a/examples/src/aws_kms_keyring_example.py b/examples/src/aws_kms_keyring_example.py index b166aab99..8977e3750 100644 --- a/examples/src/aws_kms_keyring_example.py +++ b/examples/src/aws_kms_keyring_example.py @@ -17,7 +17,6 @@ For more information on how to use KMS keyrings, see https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/use-kms-keyring.html """ -import sys import boto3 from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders @@ -29,11 +28,6 @@ import aws_encryption_sdk from aws_encryption_sdk import CommitmentPolicy -# TODO-MPL: Remove this as part of removing PYTHONPATH hacks. -MODULE_ROOT_DIR = '/'.join(__file__.split("/")[:-1]) - -sys.path.append(MODULE_ROOT_DIR) - EXAMPLE_DATA: bytes = b"Hello World" diff --git a/examples/src/aws_kms_mrk_discovery_keyring_example.py b/examples/src/aws_kms_mrk_discovery_keyring_example.py index 8ba80b621..23d6cb322 100644 --- a/examples/src/aws_kms_mrk_discovery_keyring_example.py +++ b/examples/src/aws_kms_mrk_discovery_keyring_example.py @@ -34,7 +34,6 @@ For more information on how to use KMS Discovery keyrings, see https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/use-kms-keyring.html#kms-keyring-discovery """ -import sys import boto3 from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders @@ -50,11 +49,6 @@ import aws_encryption_sdk from aws_encryption_sdk import CommitmentPolicy -# TODO-MPL: Remove this as part of removing PYTHONPATH hacks. -MODULE_ROOT_DIR = '/'.join(__file__.split("/")[:-1]) - -sys.path.append(MODULE_ROOT_DIR) - EXAMPLE_DATA: bytes = b"Hello World" diff --git a/examples/src/aws_kms_mrk_discovery_multi_keyring_example.py b/examples/src/aws_kms_mrk_discovery_multi_keyring_example.py index 47cb80f1d..adb249e2a 100644 --- a/examples/src/aws_kms_mrk_discovery_multi_keyring_example.py +++ b/examples/src/aws_kms_mrk_discovery_multi_keyring_example.py @@ -36,7 +36,6 @@ For more information on how to use KMS Discovery keyrings, see https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/use-kms-keyring.html#kms-keyring-discovery """ -import sys import boto3 from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders @@ -52,11 +51,6 @@ import aws_encryption_sdk from aws_encryption_sdk import CommitmentPolicy -# TODO-MPL: Remove this as part of removing PYTHONPATH hacks. -MODULE_ROOT_DIR = '/'.join(__file__.split("/")[:-1]) - -sys.path.append(MODULE_ROOT_DIR) - EXAMPLE_DATA: bytes = b"Hello World" diff --git a/examples/src/aws_kms_mrk_keyring_example.py b/examples/src/aws_kms_mrk_keyring_example.py index 65d8be71b..edb3cc410 100644 --- a/examples/src/aws_kms_mrk_keyring_example.py +++ b/examples/src/aws_kms_mrk_keyring_example.py @@ -21,7 +21,6 @@ For more info on KMS MRK (multi-region keys), see the KMS documentation: https://docs.aws.amazon.com/kms/latest/developerguide/multi-region-keys-overview.html """ -import sys import boto3 from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders @@ -33,11 +32,6 @@ import aws_encryption_sdk from aws_encryption_sdk import CommitmentPolicy -# TODO-MPL: Remove this as part of removing PYTHONPATH hacks. -MODULE_ROOT_DIR = '/'.join(__file__.split("/")[:-1]) - -sys.path.append(MODULE_ROOT_DIR) - EXAMPLE_DATA: bytes = b"Hello World" diff --git a/examples/src/aws_kms_mrk_multi_keyring_example.py b/examples/src/aws_kms_mrk_multi_keyring_example.py index 9c87008fe..6b1e64eec 100644 --- a/examples/src/aws_kms_mrk_multi_keyring_example.py +++ b/examples/src/aws_kms_mrk_multi_keyring_example.py @@ -27,7 +27,6 @@ For more info on KMS MRK (multi-region keys), see the KMS documentation: https://docs.aws.amazon.com/kms/latest/developerguide/multi-region-keys-overview.html """ -import sys import boto3 from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders @@ -39,11 +38,6 @@ import aws_encryption_sdk from aws_encryption_sdk import CommitmentPolicy -# TODO-MPL: Remove this as part of removing PYTHONPATH hacks. -MODULE_ROOT_DIR = '/'.join(__file__.split("/")[:-1]) - -sys.path.append(MODULE_ROOT_DIR) - EXAMPLE_DATA: bytes = b"Hello World" diff --git a/examples/src/aws_kms_multi_keyring_example.py b/examples/src/aws_kms_multi_keyring_example.py index 715181646..7cba36167 100644 --- a/examples/src/aws_kms_multi_keyring_example.py +++ b/examples/src/aws_kms_multi_keyring_example.py @@ -36,7 +36,6 @@ For more information on how to use Multi keyrings, see https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/use-multi-keyring.html """ -import sys import boto3 from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders @@ -48,11 +47,6 @@ import aws_encryption_sdk from aws_encryption_sdk import CommitmentPolicy -# TODO-MPL: Remove this as part of removing PYTHONPATH hacks. -MODULE_ROOT_DIR = '/'.join(__file__.split("/")[:-1]) - -sys.path.append(MODULE_ROOT_DIR) - EXAMPLE_DATA: bytes = b"Hello World" diff --git a/examples/src/aws_kms_rsa_keyring_example.py b/examples/src/aws_kms_rsa_keyring_example.py index 337dd14b6..fd05fc20b 100644 --- a/examples/src/aws_kms_rsa_keyring_example.py +++ b/examples/src/aws_kms_rsa_keyring_example.py @@ -14,7 +14,6 @@ # For more information on how to use KMS keyrings, see # https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/use-kms-keyring.html """ -import sys import boto3 from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders @@ -27,11 +26,6 @@ from aws_encryption_sdk import CommitmentPolicy from aws_encryption_sdk.identifiers import AlgorithmSuite -# TODO-MPL: Remove this as part of removing PYTHONPATH hacks. -MODULE_ROOT_DIR = '/'.join(__file__.split("/")[:-1]) - -sys.path.append(MODULE_ROOT_DIR) - EXAMPLE_DATA: bytes = b"Hello World" diff --git a/examples/src/branch_key_id_supplier_example.py b/examples/src/branch_key_id_supplier_example.py index b3d26497b..b25d9abeb 100644 --- a/examples/src/branch_key_id_supplier_example.py +++ b/examples/src/branch_key_id_supplier_example.py @@ -25,15 +25,15 @@ def get_branch_key_id( """Returns branch key ID from the tenant ID in input's encryption context.""" encryption_context: Dict[str, str] = param.encryption_context - if b"tenant" not in encryption_context: + if "tenant" not in encryption_context: raise ValueError("EncryptionContext invalid, does not contain expected tenant key value pair.") - tenant_key_id: str = encryption_context.get(b"tenant") + tenant_key_id: str = encryption_context.get("tenant") branch_key_id: str - if tenant_key_id == b"TenantA": + if tenant_key_id == "TenantA": branch_key_id = self.branch_key_id_for_tenant_A - elif tenant_key_id == b"TenantB": + elif tenant_key_id == "TenantB": branch_key_id = self.branch_key_id_for_tenant_B else: raise ValueError(f"Item does not contain valid tenant ID: {tenant_key_id=}") diff --git a/examples/src/default_cryptographic_materials_manager_example.py b/examples/src/default_cryptographic_materials_manager_example.py index bfffc36d9..15a9f22cf 100644 --- a/examples/src/default_cryptographic_materials_manager_example.py +++ b/examples/src/default_cryptographic_materials_manager_example.py @@ -18,8 +18,6 @@ For more information on Cryptographic Material Managers, see https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/concepts.html#crypt-materials-manager """ -import sys - import boto3 from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig @@ -33,11 +31,6 @@ import aws_encryption_sdk from aws_encryption_sdk import CommitmentPolicy -# TODO-MPL: Remove this as part of removing PYTHONPATH hacks. -MODULE_ROOT_DIR = '/'.join(__file__.split("/")[:-1]) - -sys.path.append(MODULE_ROOT_DIR) - EXAMPLE_DATA: bytes = b"Hello World" diff --git a/examples/src/file_streaming_example.py b/examples/src/file_streaming_example.py index 72decde60..3f547d220 100644 --- a/examples/src/file_streaming_example.py +++ b/examples/src/file_streaming_example.py @@ -25,7 +25,6 @@ """ import filecmp import secrets -import sys from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig @@ -36,11 +35,6 @@ import aws_encryption_sdk from aws_encryption_sdk import CommitmentPolicy -# TODO-MPL: Remove this as part of removing PYTHONPATH hacks. -MODULE_ROOT_DIR = '/'.join(__file__.split("/")[:-1]) - -sys.path.append(MODULE_ROOT_DIR) - def encrypt_and_decrypt_with_keyring( plaintext_filename: str, diff --git a/examples/src/hierarchical_keyring_example.py b/examples/src/hierarchical_keyring_example.py index c781f4f40..00dadf9d8 100644 --- a/examples/src/hierarchical_keyring_example.py +++ b/examples/src/hierarchical_keyring_example.py @@ -31,7 +31,6 @@ This example also requires using a KMS Key. You need the following access on this key: - GenerateDataKeyWithoutPlaintext - Decrypt """ -import sys import boto3 # Ignore missing MPL for pylint, but the MPL is required for this example @@ -55,11 +54,6 @@ from .branch_key_id_supplier_example import ExampleBranchKeyIdSupplier -# TODO-MPL: Remove this as part of removing PYTHONPATH hacks. -module_root_dir = '/'.join(__file__.split("/")[:-1]) - -sys.path.append(module_root_dir) - EXAMPLE_DATA: bytes = b"Hello World" diff --git a/examples/src/legacy/module_.py b/examples/src/legacy/module_.py deleted file mode 100644 index 3e8d3062a..000000000 --- a/examples/src/legacy/module_.py +++ /dev/null @@ -1 +0,0 @@ -"""Should remove this once PYTHONPATH issues are resolved by adding doo files.""" diff --git a/examples/src/migration/migration_set_commitment_policy_example.py b/examples/src/migration/migration_set_commitment_policy_example.py index 4bd0bc372..3851df0e2 100644 --- a/examples/src/migration/migration_set_commitment_policy_example.py +++ b/examples/src/migration/migration_set_commitment_policy_example.py @@ -20,7 +20,6 @@ For more information on setting your commitment policy, see https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/concepts.html#commitment-policy """ -import sys import boto3 from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders @@ -32,11 +31,6 @@ import aws_encryption_sdk from aws_encryption_sdk import CommitmentPolicy -# TODO-MPL: Remove this as part of removing PYTHONPATH hacks. -MODULE_ROOT_DIR = '/'.join(__file__.split("/")[:-1]) - -sys.path.append(MODULE_ROOT_DIR) - EXAMPLE_DATA: bytes = b"Hello World" diff --git a/examples/src/module_.py b/examples/src/module_.py deleted file mode 100644 index 3e8d3062a..000000000 --- a/examples/src/module_.py +++ /dev/null @@ -1 +0,0 @@ -"""Should remove this once PYTHONPATH issues are resolved by adding doo files.""" diff --git a/examples/src/multi_keyring_example.py b/examples/src/multi_keyring_example.py index b12ab61a3..20af7ba81 100644 --- a/examples/src/multi_keyring_example.py +++ b/examples/src/multi_keyring_example.py @@ -37,7 +37,6 @@ https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/use-multi-keyring.html """ import secrets -import sys import boto3 from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders @@ -54,11 +53,6 @@ import aws_encryption_sdk from aws_encryption_sdk import CommitmentPolicy -# TODO-MPL: Remove this as part of removing PYTHONPATH hacks. -MODULE_ROOT_DIR = '/'.join(__file__.split("/")[:-1]) - -sys.path.append(MODULE_ROOT_DIR) - EXAMPLE_DATA: bytes = b"Hello World" diff --git a/examples/src/raw_aes_keyring_example.py b/examples/src/raw_aes_keyring_example.py index d8634f774..ab9603af6 100644 --- a/examples/src/raw_aes_keyring_example.py +++ b/examples/src/raw_aes_keyring_example.py @@ -23,7 +23,6 @@ https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/use-raw-aes-keyring.html """ import secrets -import sys from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig @@ -34,11 +33,6 @@ import aws_encryption_sdk from aws_encryption_sdk import CommitmentPolicy -# TODO-MPL: Remove this as part of removing PYTHONPATH hacks. -MODULE_ROOT_DIR = '/'.join(__file__.split("/")[:-1]) - -sys.path.append(MODULE_ROOT_DIR) - EXAMPLE_DATA: bytes = b"Hello World" diff --git a/examples/src/raw_rsa_keyring_example.py b/examples/src/raw_rsa_keyring_example.py index 4a38fd166..1200a7c72 100644 --- a/examples/src/raw_rsa_keyring_example.py +++ b/examples/src/raw_rsa_keyring_example.py @@ -33,7 +33,6 @@ For more information on how to use Raw RSA keyrings, see https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/use-raw-rsa-keyring.html """ -import sys from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig @@ -48,11 +47,6 @@ from aws_encryption_sdk import CommitmentPolicy from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError -# TODO-MPL: Remove this as part of removing PYTHONPATH hacks. -MODULE_ROOT_DIR = '/'.join(__file__.split("/")[:-1]) - -sys.path.append(MODULE_ROOT_DIR) - EXAMPLE_DATA: bytes = b"Hello World" diff --git a/examples/src/required_encryption_context_cmm.py b/examples/src/required_encryption_context_cmm.py index 1d5140e6d..edbb3f414 100644 --- a/examples/src/required_encryption_context_cmm.py +++ b/examples/src/required_encryption_context_cmm.py @@ -6,7 +6,6 @@ on encrypt such that they will not be stored on the message, but WILL be included in the header signature. On decrypt, the client MUST supply the key/value pair(s) that were not stored to successfully decrypt the message. """ -import sys import boto3 # Ignore missing MPL for pylint, but the MPL is required for this example @@ -25,11 +24,6 @@ from aws_encryption_sdk import CommitmentPolicy from aws_encryption_sdk.exceptions import AWSEncryptionSDKClientError -# TODO-MPL: Remove this as part of removing PYTHONPATH hacks -module_root_dir = '/'.join(__file__.split("/")[:-1]) - -sys.path.append(module_root_dir) - EXAMPLE_DATA: bytes = b"Hello World" diff --git a/examples/src/set_encryption_algorithm_suite_example.py b/examples/src/set_encryption_algorithm_suite_example.py index 200570083..75eaee85a 100644 --- a/examples/src/set_encryption_algorithm_suite_example.py +++ b/examples/src/set_encryption_algorithm_suite_example.py @@ -39,7 +39,6 @@ https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/use-raw-aes-keyring.html """ import secrets -import sys from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig @@ -51,11 +50,6 @@ from aws_encryption_sdk import CommitmentPolicy from aws_encryption_sdk.identifiers import AlgorithmSuite -# TODO-MPL: Remove this as part of removing PYTHONPATH hacks. -MODULE_ROOT_DIR = '/'.join(__file__.split("/")[:-1]) - -sys.path.append(MODULE_ROOT_DIR) - EXAMPLE_DATA: bytes = b"Hello World" diff --git a/performance_tests/requirements_mpl.txt b/performance_tests/requirements_mpl.txt index 209e10f2c..28d7dc356 100644 --- a/performance_tests/requirements_mpl.txt +++ b/performance_tests/requirements_mpl.txt @@ -1 +1 @@ -aws-cryptographic-material-providers @ git+https://github.com/aws/aws-cryptographic-material-providers-library.git@lucmcdon/python-mpl#subdirectory=AwsCryptographicMaterialProviders/runtimes/python \ No newline at end of file +aws-cryptographic-material-providers @ git+https://github.com/aws/aws-cryptographic-material-providers-library.git@lucmcdon/python-mpl-v2#subdirectory=AwsCryptographicMaterialProviders/runtimes/python \ No newline at end of file diff --git a/requirements_mpl.txt b/requirements_mpl.txt index 209e10f2c..28d7dc356 100644 --- a/requirements_mpl.txt +++ b/requirements_mpl.txt @@ -1 +1 @@ -aws-cryptographic-material-providers @ git+https://github.com/aws/aws-cryptographic-material-providers-library.git@lucmcdon/python-mpl#subdirectory=AwsCryptographicMaterialProviders/runtimes/python \ No newline at end of file +aws-cryptographic-material-providers @ git+https://github.com/aws/aws-cryptographic-material-providers-library.git@lucmcdon/python-mpl-v2#subdirectory=AwsCryptographicMaterialProviders/runtimes/python \ No newline at end of file diff --git a/setup.py b/setup.py index 8ed2fe4e0..c069bade1 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ def get_requirements(): extras_require={ "MPL": ["aws-cryptographic-material-providers @" \ "git+https://github.com/aws/aws-cryptographic-material-providers-library.git@" \ - "lucmcdon/python-mpl#subdirectory=AwsCryptographicMaterialProviders/runtimes/python"], + "lucmcdon/python-mpl-v2#subdirectory=AwsCryptographicMaterialProviders/runtimes/python"], }, classifiers=[ "Development Status :: 5 - Production/Stable", diff --git a/test_vector_handlers/requirements_mpl.txt b/test_vector_handlers/requirements_mpl.txt index c7927a851..1aab5f534 100644 --- a/test_vector_handlers/requirements_mpl.txt +++ b/test_vector_handlers/requirements_mpl.txt @@ -1 +1 @@ -amazon-cryptographic-material-providers-test-vectors @ git+https://github.com/aws/aws-cryptographic-material-providers-library.git@lucmcdon/python-mpl#subdirectory=TestVectorsAwsCryptographicMaterialProviders/runtimes/python \ No newline at end of file +amazon-cryptographic-material-providers-test-vectors @ git+https://github.com/aws/aws-cryptographic-material-providers-library.git@lucmcdon/python-mpl-v2#subdirectory=TestVectorsAwsCryptographicMaterialProviders/runtimes/python \ No newline at end of file diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py index fac44bb71..79df4f2cf 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/full_message/decrypt.py @@ -8,6 +8,8 @@ import json import os from enum import Enum +import contextlib +import io import attr import aws_encryption_sdk @@ -126,8 +128,19 @@ def match(self, name, decrypt_fn): # The ESDK implementations are not consistent in the types of errors they produce # or the exact error messages they use. The most important thing to test is that decryption # fails in some way, and hence the overly-broad implicit try/catch here. + with pytest.raises(Exception): - decrypt_fn() + # Here, an exception is expected. + # However, when the expected exception is raised, + # the Python environment will write stderrs to console. + # Redirect stderr to null-like sink + # so local and CI build logs are cleaner, + # and any actual issues are easier to see. + # If an exception is not raised as expected, + # `pytest.raises` will fail. + tmp_file = io.StringIO() + with contextlib.redirect_stderr(tmp_file): + decrypt_fn() except BaseException: # Translate the exception just to attach context. raise RuntimeError( diff --git a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py index 776810d78..dbabeb3c7 100644 --- a/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py +++ b/test_vector_handlers/src/awses_test_vectors/manifests/mpl_keyring.py @@ -31,7 +31,7 @@ from aws_cryptographic_materialproviders.mpl.models import CreateMultiKeyringInput import _dafny -import UTF8 +from standard_library.internaldafny.generated import UTF8 # Ignore pylint not being able to read a module that requires the MPL # pylint: disable=no-name-in-module From 46de750449b3a3b22bc1447f7604e78ad4db7e2e Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 7 Aug 2024 10:21:23 -0700 Subject: [PATCH 422/422] cleanup --- codebuild/py312/decrypt_hkeyring_with_keyrings.yml | 7 ++++--- codebuild/py312/decrypt_hkeyring_with_masterkey.yml | 7 ++++--- codebuild/py312/decrypt_hkeyring_with_net.yml | 8 ++++---- codebuild/py312/generate_hkeyring_decrypt_vectors.yml | 8 ++++---- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/codebuild/py312/decrypt_hkeyring_with_keyrings.yml b/codebuild/py312/decrypt_hkeyring_with_keyrings.yml index 49ade991a..5bcd26738 100644 --- a/codebuild/py312/decrypt_hkeyring_with_keyrings.yml +++ b/codebuild/py312/decrypt_hkeyring_with_keyrings.yml @@ -19,13 +19,14 @@ phases: pre_build: commands: # Download previously generated vectors - - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_hkeyring_manifest.zip 312_hkeyring_manifest.zip - - unzip 312_hkeyring_manifest.zip + # This manifest has coverage for both HKeyring and required encryption context CMM + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_hkeyring_reccmm_manifest.zip 312_hkeyring_reccmm_manifest.zip + - unzip 312_hkeyring_reccmm_manifest.zip build: commands: - pip install "tox < 4.0" - cd test_vector_handlers - | tox -- \ - --input ../312_hkeyring_manifest/manifest.json \ + --input ../312_hkeyring_reccmm_manifest/manifest.json \ --keyrings \ No newline at end of file diff --git a/codebuild/py312/decrypt_hkeyring_with_masterkey.yml b/codebuild/py312/decrypt_hkeyring_with_masterkey.yml index e25882030..be67235d7 100644 --- a/codebuild/py312/decrypt_hkeyring_with_masterkey.yml +++ b/codebuild/py312/decrypt_hkeyring_with_masterkey.yml @@ -19,12 +19,13 @@ phases: pre_build: commands: # Download previously generated vectors - - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_hkeyring_manifest.zip 312_hkeyring_manifest.zip - - unzip 312_hkeyring_manifest.zip + # This manifest has coverage for both HKeyring and required encryption context CMM + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_hkeyring_reccmm_manifest.zip 312_hkeyring_reccmm_manifest.zip + - unzip 312_hkeyring_reccmm_manifest.zip build: commands: - pip install "tox < 4.0" - cd test_vector_handlers - | tox -- \ - --input ../312_hkeyring_manifest/manifest.json + --input ../312_hkeyring_reccmm_manifest/manifest.json diff --git a/codebuild/py312/decrypt_hkeyring_with_net.yml b/codebuild/py312/decrypt_hkeyring_with_net.yml index 37c2cf0d2..6a3b321eb 100644 --- a/codebuild/py312/decrypt_hkeyring_with_net.yml +++ b/codebuild/py312/decrypt_hkeyring_with_net.yml @@ -19,10 +19,10 @@ phases: pre_build: commands: # Download previously generated vectors - - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_hkeyring_manifest.zip 312_hkeyring_manifest.zip - - unzip 312_hkeyring_manifest.zip - - export DAFNY_AWS_ESDK_TEST_VECTOR_MANIFEST_PATH="${PWD}/312_hkeyring_manifest/manifest.json" - + # This manifest has coverage for both HKeyring and required encryption context CMM + - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_hkeyring_reccmm_manifest.zip 312_hkeyring_reccmm_manifest.zip + - unzip 312_hkeyring_reccmm_manifest.zip + - export DAFNY_AWS_ESDK_TEST_VECTOR_MANIFEST_PATH="${PWD}/312_hkeyring_reccmm_manifest/manifest.json" # Download dafny - curl https://github.com/dafny-lang/dafny/releases/download/v4.7.0/dafny-4.7.0-x64-ubuntu-20.04.zip -L -o dafny.zip diff --git a/codebuild/py312/generate_hkeyring_decrypt_vectors.yml b/codebuild/py312/generate_hkeyring_decrypt_vectors.yml index 8dfa90581..b0a755360 100644 --- a/codebuild/py312/generate_hkeyring_decrypt_vectors.yml +++ b/codebuild/py312/generate_hkeyring_decrypt_vectors.yml @@ -20,14 +20,14 @@ phases: commands: - pip install "tox < 4.0" - cd test_vector_handlers/test/aws-crypto-tools-test-vector-framework - # Checkout WIP branch + # Checkout WIP branch with manifest containing HKeyring and required EC CMM test cases - git checkout lucmcdon/hierarchy-test-vectors - git pull - cd ../.. - | tox -- \ --input test/aws-crypto-tools-test-vector-framework/features/CANONICAL-GENERATED-MANIFESTS/0007-hkeyring-reccmm-generate-manifest.json \ - --output 312_hkeyring_manifest \ + --output 312_hkeyring_reccmm_manifest \ --keyrings - - zip -r 312_hkeyring_manifest.zip 312_hkeyring_manifest - - aws s3 cp 312_hkeyring_manifest.zip s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_hkeyring_manifest.zip + - zip -r 312_hkeyring_reccmm_manifest.zip 312_hkeyring_reccmm_manifest + - aws s3 cp 312_hkeyring_reccmm_manifest.zip s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_hkeyring_reccmm_manifest.zip