chore: performance tests for ESDK-python

.gitignore

@@ -33,6 +33,12 @@ __pycache__
 .pytest_cache
 # Ignore key materials generated by examples or tests
 test_keyrings/
+# Ignore results of performance test
+performance_tests/results/*
+# To track the results folder, keep the __init__.py file
+!performance_tests/results/__init__.py
+# Ignore the memory profile logs
+mprofile_*
 
 # PyCharm
 .idea/

buildspec.yml

@@ -164,6 +164,10 @@ batch:
       buildspec: codebuild/py311/integ_mpl.yml
       env:
         image: aws/codebuild/standard:7.0
+    - identifier: py311_performance_tests_mpl
+      buildspec: codebuild/py311/performance_tests_mpl.yml
+      env:
+        image: aws/codebuild/standard:7.0
     - identifier: py311_examples
       buildspec: codebuild/py311/examples.yml
       env:
@@ -250,6 +254,10 @@ batch:
       buildspec: codebuild/py312/integ_mpl.yml
       env:
         image: aws/codebuild/standard:7.0
+    - identifier: py312_performance_tests_mpl
+      buildspec: codebuild/py312/performance_tests_mpl.yml
+      env:
+        image: aws/codebuild/standard:7.0
     - identifier: py312_examples
       buildspec: codebuild/py312/examples.yml
       env:

codebuild/py311/performance_tests_mpl.yml

@@ -0,0 +1,38 @@
+# Runs the performance tests for the MPL in an environment with the MPL installed
+version: 0.2
+
+env:
+  variables:
+    # No TOXENV. This runs multiple environments.
+    REGION: "us-west-2"
+    AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >-
+      arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f
+    AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >-
+      arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2
+    AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >-
+      arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7
+    AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >-
+      arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7
+
+phases:
+  install:
+    runtime-versions:
+      python: 3.11
+  build:
+    commands:
+      - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd -
+      - pyenv install --skip-existing 3.11.0
+      - pyenv local 3.11.0
+      - pip install --upgrade pip
+      - pip install setuptools
+      - pip install "tox < 4.0"
+      # Assume special role to access keystore
+      - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-Py312ExamplesMpl")
+      - export TMP_ROLE
+      - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId')
+      - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey')
+      - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken')
+      - aws sts get-caller-identity
+      # Run MPL-specific tests with special role
+      - cd performance_tests/
+      - tox -e py311-performance_tests-mpl

codebuild/py312/performance_tests_mpl.yml

@@ -0,0 +1,38 @@
+# Runs the performance tests for the MPL in an environment with the MPL installed
+version: 0.2
+
+env:
+  variables:
+    # No TOXENV. This runs multiple environments.
+    REGION: "us-west-2"
+    AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >-
+      arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f
+    AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >-
+      arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2
+    AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >-
+      arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7
+    AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >-
+      arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7
+
+phases:
+  install:
+    runtime-versions:
+      python: 3.12
+  build:
+    commands:
+      - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd -
+      - pyenv install --skip-existing 3.12.0
+      - pyenv local 3.12.0
+      - pip install --upgrade pip
+      - pip install setuptools
+      - pip install "tox < 4.0"
+      # Assume special role to access keystore
+      - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-Py312ExamplesMpl")
+      - export TMP_ROLE
+      - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId')
+      - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey')
+      - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken')
+      - aws sts get-caller-identity
+      # Run MPL-specific tests with special role
+      - cd performance_tests/
+      - tox -e py312-performance_tests-mpl

performance_tests/README.rst

@@ -0,0 +1,135 @@
+# aws-encryption-sdk performance tests
+
+## License
+
+This project is licensed under the Apache-2.0 License.
+
+## Overview
+
+This library tests the following keyrings / master key-providers:
+
+1. KMS Keyring / KMS Master Key Provider
+2. Raw AES Keyring / AES Master Key Provider
+3. Raw RSA Keyring / RSA Master Key Provider
+4. Hierarchy Keyring
+5. Caching CMM
+
+For each test on the above keyrings / master key-providers, this package measures the execution time and memory consumption.
+
+For each keyring / master key-provider, the execution time and memory consumption time is measured for three operations:
+1. Create keyring / master key-provider
+2. Encrypt
+3. Decrypt
+
+The usage of the performance tests is demonstrated through an [AWS KMS Keyring](https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/use-kms-keyring.html). However, the procedure is the same for any keyring / master key-provider, with slight change in the input arguments.
+
+The results for the performance test will be available in the results folder in the performance_tests directory.
+
+## Usage: Execution Time
+
+### Create Keyring
+To run the performance test for execution time, please use the following commands in the performance_tests directory
+```
+python test/keyrings/test_aws_kms_keyring.py create
+```
+
+#### Optional Arguments
+* kms_key_id: The KMS key ID you want to use
+* n_iters: Number of iterations you want to run the test for. For instance, if n_iters = 100, this performance test script will run the create_keyring method 100 times and report the execution time of each of the calls.
+* output_file: The output file for execution times for each function call, default='kms_keyring_create' in the results folder
+
+#### Consolidate Results
+
+In order to find the minimum, maximum and average times from the n_iters runs, please use the following script from the performance_tests directory:
+```
+python consolidate_results.py results/kms_keyring_create.csv
+```
+
+### Encrypt
+To run the performance test for execution time, please use the following commands in the performance_tests directory
+```
+python test/keyrings/test_aws_kms_keyring.py encrypt
+```
+
+Here, you will receive a prompt on the terminal to specify the plaintext file you want to encrypt. Some example plaintext data files are present in the 'test/resources' directory.
+
+Alternatively, if you want to provide the arguments as flags without using the interactive CLI, you can run the command in the following manner:
+
+```
+python test/keyrings/test_aws_kms_keyring.py encrypt --plaintext_data_filename test/resources/plaintext-data-medium.dat
+```
+
+You can choose to use any other plaintext file as well.
+
+#### Arguments
+* plaintext_data_filename: Filename containing plaintext data you want to encrypt
+
+#### Optional Arguments
+* kms_key_id: The KMS key ID you want to use to encrypt the data
+* n_iters: Number of iterations you want to run the test for. For instance, if n_iters = 100, this performance test script will run the encrypt method 100 times and report the execution time of each of the calls.
+* output_file: The output file for execution times for each function call, default='kms_keyring_encrypt'
+
+#### Consolidate Results
+
+In order to find the minimum, maximum and average times from the n_iters runs, please use the following script from the performance_tests directory:
+```
+python consolidate_results.py results/kms_keyring_encrypt.csv
+```
+
+### Decrypt
+To run the performance test for execution time, please use the following commands in the performance_tests directory
+```
+python test/keyrings/test_aws_kms_keyring.py decrypt
+```
+
+Here, you will receive a prompt on the terminal to specify the ciphertext file you want to decrypt. Some example ciphertext data files are present in the 'test/resources' directory.
+
+Alternatively, if you want to provide the arguments as flags without using the interactive CLI, you can run the command in the following manner:
+
+```
+python test/keyrings/test_aws_kms_keyring.py decrypt --ciphertext_data_filename test/resources/ciphertext-data-medium.ct
+```
+
+You can choose to use any other ciphertext file as well.
+
+#### Arguments
+* ciphertext_data_filename: Filename containing ciphertext data you want to decrypt
+
+#### Optional Arguments
+* kms_key_id: The KMS key ID you want to use to decrypt the data
+* n_iters: Number of iterations you want to run the test for. For instance, if n_iters = 100, this performance test script will run the decrypt method 100 times and report the execution time of each of the calls.
+* output_file: The output file for execution times for each function call, default='kms_keyring_decrypt'
+
+#### Consolidate Results
+
+In order to find the minimum, maximum and average times from the n_iters runs, please use the following script from the performance_tests directory:
+```
+python consolidate_results.py results/kms_keyring_decrypt.csv
+```
+
+## Usage: Memory Consumption
+To get the memory consumption, simply use 'mprof run' instead of 'python' in the previously mentioned commands.
+
+For example, if you want to calculate the memory consumption of the encrypt function of a AWS KMS Keyring, simply write:
+```
+mprof run test/keyrings/test_aws_kms_keyring.py encrypt --plaintext_data_filename test/resources/plaintext-data-medium.dat
+```
+
+This should generate an mprofile log file in your current directory. To plot the memory consumption with time, please use the following command from the same directory
+```
+mprof plot
+```
+
+This 'mprof plot' command will plot the most recent mprofile log file.
+
+## Usage: Performance Graph
+To generate a performance graph, please use the following command to generate the pstats log file by specifying the output pstats file path. Here, 'results/kms_keyring_create.pstats' is set as the default output file.
+
+```
+python -m cProfile -o results/kms_keyring_create.pstats test/keyrings/test_aws_kms_keyring.py create
+```
+
+After generating the pstats file, please run the following command to generate the performance graph. The output performance graph will be a .png file that you specify. Here, 'results/kms_keyring_create.png' is set as the default output file.
+```
+gprof2dot -f pstats results/kms_keyring_create.pstats | dot -Tpng -o results/kms_keyring_create.png && eog results/kms_keyring_create.png 
+```

performance_tests/__init__.py

@@ -0,0 +1,3 @@
+# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""Stub module indicator to make linter configuration simpler."""

performance_tests/consolidate_results.py

@@ -0,0 +1,48 @@
+# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""Script for consolidating results for execution times"""
+
+import argparse
+import csv
+
+import numpy as np
+
+
+def calculate_statistics(_csv_file):
+    """Calculate average, trimmed average, minimum, maximum and p99 statistics for execution times in a CSV file."""
+    with open(_csv_file, 'r', encoding='utf-8') as file:
+        reader = csv.reader(file)
+        data = [float(row[0]) for row in reader]
+
+    # Calculate statistics
+    if data:
+        data = np.sort(data)
+        _total_entries = len(data)
+        _average = np.mean(data)
+        _trimmed_average_99_bottom = np.mean(data[0:int(0.99 * len(data))])
+        _minimum = min(data)
+        _maximum = max(data)
+        _perc_99 = np.percentile(data, 99)
+        return _total_entries, _average, _trimmed_average_99_bottom, _minimum, _maximum, _perc_99
+
+    return None
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('csv_file',
+                        help='csv file containing the outputs of execution times for n_iter iterations')
+    args = parser.parse_args()
+
+    statistics = calculate_statistics(args.csv_file)
+    if statistics:
+        total_entries, average, trimmend_average_99_bottom, minimum, maximum, perc_99 = statistics
+        print("CSV File:", args.csv_file)
+        print("Total Entries:", total_entries)
+        print("Average:", average)
+        print("Bottom 99th percentile trimmed average:", trimmend_average_99_bottom)
+        print("Minimum:", minimum)
+        print("Maximum:", maximum)
+        print("99th percentile:", perc_99)
+    else:
+        print("No data found in the CSV file.")

performance_tests/pylintrc

@@ -0,0 +1,45 @@
+[MESSAGE CONTROL]
+# Disabling messages that either we don't care about we intentionally break.
+disable =
+    import-error, # ignore mpl import errors
+    invalid-name,  # we prefer long, descriptive, names for examples
+    bad-continuation,  # we let black handle this
+    ungrouped-imports,  # we let isort handle this
+    no-member,  # breaks with attrs
+    no-self-use,  # interesting to keep in mind for later refactoring, but not blocking
+    useless-object-inheritance,  # we need to support Python 2, so no, not useless
+    duplicate-code,  # some examples may be similar
+    too-few-public-methods,  # does not allow value stores
+    too-many-locals,  # examples may sometimes have more locals defined for clarity than would be appropriate in code
+    no-else-return,  # we omit this on purpose for brevity where it would add no value
+    attribute-defined-outside-init,  # breaks with attrs_post_init
+    abstract-method,  # throws false positives on io.BaseIO grandchildren
+    redefined-outer-name,  # we do this on purpose in multiple places
+    consider-using-f-string # disable until 2022-05-05; 6 months after 3.5 deprecation
+
+[BASIC]
+# Allow function names up to 50 characters
+function-rgx = [a-z_][a-z0-9_]{2,50}$
+# Allow method names up to 50 characters
+method-rgx = [a-z_][a-z0-9_]{2,50}$
+# Allow class attribute names up to 50 characters
+# Whitelist class attribute names: iv
+class-attribute-rgx = (([A-Za-z_][A-Za-z0-9_]{2,50}|(__.*__))$)|(^iv$)
+# Whitelist attribute names: iv
+attr-rgx = ([a-z_][a-z0-9_]{2,30}$)|(^iv$)
+# Whitelist argument names: iv, b
+argument-rgx = ([a-z_][a-z0-9_]{2,30}$)|(^iv$)|(^b$)
+# Whitelist variable names: iv, b, _b, x, y, r, s
+variable-rgx = ([a-z_][a-z0-9_]{2,30}$)|(^iv$)|(^b$)|(^_b$)|(^x$)|(^y$)|(^r$)|(^s$)
+
+[VARIABLES]
+additional-builtins = raw_input
+
+[DESIGN]
+max-args = 10
+
+[FORMAT]
+max-line-length = 120
+
+[REPORTS]
+msg-template = {path}:{line}: [{msg_id}({symbol}), {obj}] {msg}

performance_tests/requirements.txt

@@ -0,0 +1,5 @@
+attrs >= 17.4.0
+aws-encryption-sdk>=2.3.0
+pytest>=3.3.1
+tqdm
+click

performance_tests/requirements_mpl.txt

@@ -0,0 +1 @@
+aws-cryptographic-material-providers @ git+https://github.com/aws/aws-cryptographic-material-providers-library.git@lucmcdon/python-mpl#subdirectory=AwsCryptographicMaterialProviders/runtimes/python

performance_tests/results/__init__.py

@@ -0,0 +1,3 @@
+# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""Stub module indicator to make linter configuration simpler."""