Merge branch 'mpl-reviewed' into lucmcdon/mpl-v2

Author: lucasmcdonald3

.gitignore

@@ -33,6 +33,12 @@ __pycache__
 .pytest_cache
 # Ignore key materials generated by examples or tests
 test_keyrings/
+# Ignore results of performance test
+performance_tests/results/*.csv
+performance_tests/results/*.pstats
+performance_tests/results/*.png
+# Ignore the memory profile logs
+mprofile_*
 
 # PyCharm
 .idea/

README.rst

@@ -126,6 +126,10 @@ batch:
       buildspec: codebuild/py311/integ_mpl.yml
       env:
         image: aws/codebuild/standard:7.0
+    - identifier: py311_performance_tests_mpl
+      buildspec: codebuild/py311/performance_tests_mpl.yml
+      env:
+        image: aws/codebuild/standard:7.0
     - identifier: py311_examples
       buildspec: codebuild/py311/examples.yml
       env:
@@ -212,6 +216,10 @@ batch:
       buildspec: codebuild/py312/integ_mpl.yml
       env:
         image: aws/codebuild/standard:7.0
+    - identifier: py312_performance_tests_mpl
+      buildspec: codebuild/py312/performance_tests_mpl.yml
+      env:
+        image: aws/codebuild/standard:7.0
     - identifier: py312_examples
       buildspec: codebuild/py312/examples.yml
       env:

buildspec.yml

@@ -0,0 +1,38 @@
+# Runs the performance tests for the MPL in an environment with the MPL installed
+version: 0.2
+
+env:
+  variables:
+    # No TOXENV. This runs multiple environments.
+    REGION: "us-west-2"
+    AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >-
+      arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f
+    AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >-
+      arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2
+    AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >-
+      arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7
+    AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >-
+      arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7
+
+phases:
+  install:
+    runtime-versions:
+      python: 3.11
+  build:
+    commands:
+      - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd -
+      - pyenv install --skip-existing 3.11.0
+      - pyenv local 3.11.0
+      - pip install --upgrade pip
+      - pip install setuptools
+      - pip install "tox < 4.0"
+      # Assume special role to access keystore
+      - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-Py312ExamplesMpl")
+      - export TMP_ROLE
+      - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId')
+      - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey')
+      - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken')
+      - aws sts get-caller-identity
+      # Run MPL-specific tests with special role
+      - cd performance_tests/
+      - tox -e py311-performance_tests-mpl

codebuild/py311/performance_tests_mpl.yml

@@ -0,0 +1,38 @@
+# Runs the performance tests for the MPL in an environment with the MPL installed
+version: 0.2
+
+env:
+  variables:
+    # No TOXENV. This runs multiple environments.
+    REGION: "us-west-2"
+    AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >-
+      arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f
+    AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >-
+      arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2
+    AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >-
+      arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7
+    AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >-
+      arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7
+
+phases:
+  install:
+    runtime-versions:
+      python: 3.12
+  build:
+    commands:
+      - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd -
+      - pyenv install --skip-existing 3.12.0
+      - pyenv local 3.12.0
+      - pip install --upgrade pip
+      - pip install setuptools
+      - pip install "tox < 4.0"
+      # Assume special role to access keystore
+      - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-Py312ExamplesMpl")
+      - export TMP_ROLE
+      - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId')
+      - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey')
+      - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken')
+      - aws sts get-caller-identity
+      # Run MPL-specific tests with special role
+      - cd performance_tests/
+      - tox -e py312-performance_tests-mpl

codebuild/py312/performance_tests_mpl.yml

@@ -0,0 +1,125 @@
+# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""
+Example to create a custom implementation of the MPL's ICryptographicMaterialsManager class and use it with the ESDK.
+
+The cryptographic materials manager (CMM) assembles the cryptographic materials that are used
+to encrypt and decrypt data. The cryptographic materials include plaintext and encrypted data keys,
+and an optional message signing key.
+
+Cryptographic Materials Managers (CMMs) are composable; if you just want to extend the behavior of
+the default CMM, you can do this as demonstrated in this example. This is the easiest approach if
+you are just adding a small check to the CMM methods, as in this example.
+
+If your use case calls for fundamentally changing aspects of the default CMM, you can also write
+your own implementation without extending an existing CMM. The default CMM's implementation is a
+good reference to use if you need to write a custom CMM implementation from scratch.
+Custom implementations of CMMs must implement get_encryption_materials and decrypt_materials.
+
+For more information on a default implementation of a CMM,
+please look at the default_cryptographic_materials_manager_example.py example.
+
+For more information on Cryptographic Material Managers, see
+https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/concepts.html#crypt-materials-manager
+"""
+
+from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders
+from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig
+from aws_cryptographic_materialproviders.mpl.models import (
+    CreateDefaultCryptographicMaterialsManagerInput,
+    SignatureAlgorithmNone,
+)
+from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager, IKeyring
+
+import aws_encryption_sdk
+from aws_encryption_sdk import CommitmentPolicy
+
+
+# Custom CMM implementation using the MPL.
+# This CMM only allows encryption/decryption using signing algorithms.
+# It wraps an underlying CMM implementation and checks its materials
+# to ensure that it is only using signed encryption algorithms.
+class MPLCustomSigningSuiteOnlyCMM(ICryptographicMaterialsManager):
+    """Example custom crypto materials manager class."""
+
+    def __init__(self, keyring: IKeyring, cmm: ICryptographicMaterialsManager = None) -> None:
+        """Constructor for MPLCustomSigningSuiteOnlyCMM class."""
+        if cmm is not None:
+            self.underlying_cmm = cmm
+        else:
+            mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders(
+                config=MaterialProvidersConfig()
+            )
+
+            # Create a CryptographicMaterialsManager for encryption and decryption
+            cmm_input: CreateDefaultCryptographicMaterialsManagerInput = \
+                CreateDefaultCryptographicMaterialsManagerInput(
+                    keyring=keyring
+                )
+
+            self.underlying_cmm: ICryptographicMaterialsManager = \
+                mat_prov.create_default_cryptographic_materials_manager(
+                    input=cmm_input
+                )
+
+    def get_encryption_materials(self, param):
+        """Provides encryption materials appropriate for the request for the custom CMM.
+
+        :param aws_cryptographic_materialproviders.mpl.models.GetEncryptionMaterialsInput param: Input object to
+        provide to a crypto material manager's `get_encryption_materials` method.
+        :returns: Encryption materials output
+        :rtype: aws_cryptographic_materialproviders.mpl.models.GetEncryptionMaterialsOutput
+        """
+        materials = self.underlying_cmm.get_encryption_materials(param)
+        if isinstance(materials.encryption_materials.algorithm_suite.signature, SignatureAlgorithmNone):
+            raise ValueError(
+                "Algorithm provided to MPLCustomSigningSuiteOnlyCMM"
+                  + " is not a supported signing algorithm: " + str(materials.encryption_materials.algorithm_suite)
+                  )
+        return materials
+
+    def decrypt_materials(self, param):
+        """Provides decryption materials appropriate for the request for the custom CMM.
+
+        :param aws_cryptographic_materialproviders.mpl.models.DecryptMaterialsInput param: Input object to provide
+        to a crypto material manager's `decrypt_materials` method.
+        :returns: Decryption materials output
+        :rtype: aws_cryptographic_materialproviders.mpl.models.GetDecryptionMaterialsOutput
+        """
+        materials = self.underlying_cmm.decrypt_materials(param)
+        if isinstance(materials.decryption_materials.algorithm_suite.signature, SignatureAlgorithmNone):
+            raise ValueError(
+                "Algorithm provided to MPLCustomSigningSuiteOnlyCMM"
+                  + " is not a supported signing algorithm: " + str(materials.decryption_materials.algorithm_suite)
+                  )
+        return materials
+
+
+EXAMPLE_DATA: bytes = b"Hello World"
+
+
+def encrypt_decrypt_with_cmm(
+    cmm: ICryptographicMaterialsManager
+):
+    """Encrypts and decrypts a string using a custom CMM.
+
+    :param ICryptographicMaterialsManager cmm: CMM to use for encryption and decryption
+    """
+    # Set up an encryption client with an explicit commitment policy. Note that if you do not explicitly choose a
+    # commitment policy, REQUIRE_ENCRYPT_REQUIRE_DECRYPT is used by default.
+    client = aws_encryption_sdk.EncryptionSDKClient(commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT)
+
+    # Encrypt the plaintext source data
+    ciphertext, _ = client.encrypt(
+        source=EXAMPLE_DATA,
+        materials_manager=cmm
+    )
+
+    # Decrypt the ciphertext
+    cycled_plaintext, _ = client.decrypt(
+        source=ciphertext,
+        materials_manager=cmm
+    )
+
+    # Verify that the "cycled" (encrypted, then decrypted) plaintext is identical to the source plaintext
+    assert cycled_plaintext == EXAMPLE_DATA

examples/src/custom_mpl_cmm_example.py

@@ -0,0 +1,133 @@
+# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""
+This example sets up the default Cryptographic Material Managers (CMM).
+
+The default cryptographic materials manager (CMM) assembles the cryptographic materials
+that are used to encrypt and decrypt data. The cryptographic materials include
+plaintext and encrypted data keys, and an optional message signing key.
+This example creates a CMM and then encrypts a custom input EXAMPLE_DATA
+with an encryption context. Creating a CMM involves taking a keyring as input,
+and we use an AWS KMS Keyring for this example.
+This example also includes some sanity checks for demonstration:
+1. Ciphertext and plaintext data are not the same
+2. Encryption context is correct in the decrypted message header
+3. Decrypted plaintext value matches EXAMPLE_DATA
+These sanity checks are for demonstration in the example only. You do not need these in your code.
+
+For more information on Cryptographic Material Managers, see
+https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/concepts.html#crypt-materials-manager
+"""
+import sys
+
+import boto3
+from aws_cryptographic_materialproviders.mpl import AwsCryptographicMaterialProviders
+from aws_cryptographic_materialproviders.mpl.config import MaterialProvidersConfig
+from aws_cryptographic_materialproviders.mpl.models import (
+    CreateAwsKmsKeyringInput,
+    CreateDefaultCryptographicMaterialsManagerInput,
+)
+from aws_cryptographic_materialproviders.mpl.references import ICryptographicMaterialsManager, IKeyring
+from typing import Dict  # noqa pylint: disable=wrong-import-order
+
+import aws_encryption_sdk
+from aws_encryption_sdk import CommitmentPolicy
+
+# TODO-MPL: Remove this as part of removing PYTHONPATH hacks.
+MODULE_ROOT_DIR = '/'.join(__file__.split("/")[:-1])
+
+sys.path.append(MODULE_ROOT_DIR)
+
+EXAMPLE_DATA: bytes = b"Hello World"
+
+
+def encrypt_and_decrypt_with_default_cmm(
+    kms_key_id: str
+):
+    """Demonstrate an encrypt/decrypt cycle using default Cryptographic Material Managers.
+
+    Usage: encrypt_and_decrypt_with_default_cmm(kms_key_id)
+    :param kms_key_id: KMS Key identifier for the KMS key you want to use for encryption and
+    decryption of your data keys.
+    :type kms_key_id: string
+
+    For more information on KMS Key identifiers, see
+    https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#key-id
+    """
+    # 1. Instantiate the encryption SDK client.
+    # This builds the client with the REQUIRE_ENCRYPT_REQUIRE_DECRYPT commitment policy,
+    # which enforces that this client only encrypts using committing algorithm suites and enforces
+    # that this client will only decrypt encrypted messages that were created with a committing
+    # algorithm suite.
+    # This is the default commitment policy if you were to build the client as
+    # `client = aws_encryption_sdk.EncryptionSDKClient()`.
+    client = aws_encryption_sdk.EncryptionSDKClient(
+        commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT
+    )
+
+    # 2. Create encryption context.
+    # Remember that your encryption context is NOT SECRET.
+    # For more information, see
+    # https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/concepts.html#encryption-context
+    encryption_context: Dict[str, str] = {
+        "encryption": "context",
+        "is not": "secret",
+        "but adds": "useful metadata",
+        "that can help you": "be confident that",
+        "the data you are handling": "is what you think it is",
+    }
+
+    # 3. Create a KMS keyring to use with the CryptographicMaterialsManager
+    kms_client = boto3.client('kms', region_name="us-west-2")
+
+    mat_prov: AwsCryptographicMaterialProviders = AwsCryptographicMaterialProviders(
+        config=MaterialProvidersConfig()
+    )
+
+    keyring_input: CreateAwsKmsKeyringInput = CreateAwsKmsKeyringInput(
+        kms_key_id=kms_key_id,
+        kms_client=kms_client
+    )
+
+    kms_keyring: IKeyring = mat_prov.create_aws_kms_keyring(
+        input=keyring_input
+    )
+
+    # 4. Create a CryptographicMaterialsManager for encryption and decryption
+    cmm_input: CreateDefaultCryptographicMaterialsManagerInput = \
+        CreateDefaultCryptographicMaterialsManagerInput(
+            keyring=kms_keyring
+        )
+
+    cmm: ICryptographicMaterialsManager = mat_prov.create_default_cryptographic_materials_manager(
+        input=cmm_input
+    )
+
+    # 5. Encrypt the data with the encryptionContext.
+    ciphertext, _ = client.encrypt(
+        source=EXAMPLE_DATA,
+        materials_manager=cmm,
+        encryption_context=encryption_context
+    )
+
+    # 6. Demonstrate that the ciphertext and plaintext are different.
+    # (This is an example for demonstration; you do not need to do this in your own code.)
+    assert ciphertext != EXAMPLE_DATA, \
+        "Ciphertext and plaintext data are the same. Invalid encryption"
+
+    # 7. Decrypt your encrypted data using the same cmm you used on encrypt.
+    plaintext_bytes, dec_header = client.decrypt(
+        source=ciphertext,
+        materials_manager=cmm
+    )
+
+    # 8. Demonstrate that the encryption context is correct in the decrypted message header
+    # (This is an example for demonstration; you do not need to do this in your own code.)
+    for k, v in encryption_context.items():
+        assert v == dec_header.encryption_context[k], \
+            "Encryption context does not match expected values"
+
+    # 9. Demonstrate that the decrypted plaintext is identical to the original plaintext.
+    # (This is an example for demonstration; you do not need to do this in your own code.)
+    assert plaintext_bytes == EXAMPLE_DATA, \
+        "Decrypted plaintext should be identical to the original plaintext. Invalid decryption"