refactoring and fixes

Author: RitvikKapila

performance_tests/README.rst

@@ -8,28 +8,28 @@ This module runs performance tests for the `AWS Encryption SDK Python`_.
 Overview
 ********
 
-This module tests the following keyrings / master key-providers:
+This module tests the following keyrings / master key providers:
 
 1. KMS Keyring / KMS Master Key Provider
 2. Raw AES Keyring / AES Master Key Provider
 3. Raw RSA Keyring / RSA Master Key Provider
 4. Hierarchy Keyring
 5. Caching CMM
 
-For each test on the above keyrings / master key-providers, this package measures:
+For each test on the above keyrings / master key providers, this package measures:
 
 1. Execution time
 2. Total memory consumption
 
-For each keyring / master key-provider, the execution time and memory consumption
+For each keyring / master key provider, the execution time and memory consumption
 is measured for three operations:
 
-1. Create keyring / master key-provider
+1. Create keyring / master key provider
 2. Encrypt
 3. Decrypt
 
 The usage of the performance tests is demonstrated through an `AWS KMS Keyring`_.
-However, the procedure is the same for any keyring / master key-provider, with slight
+However, the procedure is the same for any keyring / master key provider, with slight
 changes in the input arguments.
 
 The results for the performance test will be available in the results folder in the
@@ -39,8 +39,18 @@ performance_tests directory.
 Required Prerequisites
 **********************
 
-* Python 3.11+
+* Python 3.8+
 * aws-encryption-sdk
+* boto3 >= 1.10.0
+* click
+* tqdm
+* pytest
+
+Recommended Prerequisites
+=========================
+
+* aws-cryptographic-material-providers: >= 1.0.0
+  * Requires Python 3.11+.
 
 *****
 Usage
@@ -74,19 +84,6 @@ following commands in the performance_tests directory.
                                     default='kms_keyring_create' in the
                                     results folder.
 
-
-Consolidate Results
-~~~~~~~~~~~~~~~~~~~
-
-In order to find the minimum, maximum, average, 99th percentile and bottom
-99th percentile trimmed average times from the n_iters runs, please use the
-following script from the performance_tests directory:
-
-.. code::
-
-    usage: python consolidate_results.py results/kms_keyring_create.csv
-
-
 Encrypt
 -------
 
@@ -118,18 +115,6 @@ commands in the performance_tests directory:
                                                             default='kms_keyring_create' in the
                                                             results folder.
 
-Consolidate Results
-~~~~~~~~~~~~~~~~~~~
-
-In order to find the minimum, maximum, average, 99th percentile and bottom
-99th percentile trimmed average times from the n_iters runs, please use the
-following script from the performance_tests directory:
-
-.. code::
-
-    usage: python consolidate_results.py results/kms_keyring_encrypt.csv
-
-
 Decrypt
 -------
 
@@ -161,12 +146,14 @@ following commands in the performance_tests directory
                                                             default='kms_keyring_create' in the
                                                             results folder.
 
-Consolidate Results
-~~~~~~~~~~~~~~~~~~~
+Consolidate Time Results
+========================
 
 In order to find the minimum, maximum, average, 99th percentile and bottom
 99th percentile trimmed average times from the n_iters runs, please use the
-following script from the performance_tests directory:
+following script from the performance_tests directory with the csv file
+containing times for each of the n_iters runs generated in the previous
+"Execution Time" section:
 
 .. code::
 

performance_tests/consolidate_results.py

@@ -14,16 +14,18 @@ def calculate_statistics(_csv_file):
         reader = csv.reader(file)
         data = [float(row[0]) for row in reader]
 
+    output_stats = {}
+
     # Calculate statistics
     if data:
         data = np.sort(data)
-        _total_entries = len(data)
-        _average = np.mean(data)
-        _trimmed_average_99_bottom = np.mean(data[0:int(0.99 * len(data))])
-        _minimum = min(data)
-        _maximum = max(data)
-        _perc_99 = np.percentile(data, 99)
-        return _total_entries, _average, _trimmed_average_99_bottom, _minimum, _maximum, _perc_99
+        output_stats['total_entries'] = len(data)
+        output_stats['average'] = np.mean(data)
+        output_stats['trimmed_average_99_bottom'] = np.mean(data[0:int(0.99 * len(data))])
+        output_stats['minimum'] = min(data)
+        output_stats['maximum'] = max(data)
+        output_stats['perc_99'] = np.percentile(data, 99)
+        return output_stats
 
     return None
 
@@ -36,13 +38,12 @@ def calculate_statistics(_csv_file):
 
     statistics = calculate_statistics(args.csv_file)
     if statistics:
-        total_entries, average, trimmend_average_99_bottom, minimum, maximum, perc_99 = statistics
         print("CSV File:", args.csv_file)
-        print("Total Entries:", total_entries)
-        print("Average:", average)
-        print("Bottom 99th percentile trimmed average:", trimmend_average_99_bottom)
-        print("Minimum:", minimum)
-        print("Maximum:", maximum)
-        print("99th percentile:", perc_99)
+        print("Total Entries:", statistics['total_entries'])
+        print("Average:", statistics['average'])
+        print("Bottom 99th percentile trimmed average:", statistics['trimmed_average_99_bottom'])
+        print("Minimum:", statistics['minimum'])
+        print("Maximum:", statistics['maximum'])
+        print("99th percentile:", statistics['perc_99'])
     else:
         print("No data found in the CSV file.")

performance_tests/test/keyrings/test_aws_kms_keyring.py

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 """This is a performance test for creating the AWS KMS keyring."""
 
+import os
 import time
 
 import click
@@ -18,6 +19,8 @@
 )
 from aws_encryption_sdk_performance_tests.utils.util import PerfTestUtils
 
+MODULE_ABS_PATH = os.path.abspath(__file__)
+
 
 @click.group()
 def create_kms_keyring():
@@ -30,7 +33,7 @@ def create_kms_keyring():
 @click.option('--n_iters',
               default=PerfTestUtils.DEFAULT_N_ITERS)
 @click.option('--output_file',
-              default='/'.join(__file__.split("/")[:-3]) + '/results/kms_keyring_create')
+              default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/kms_keyring_create')
 def create(
     kms_key_id: str,
     n_iters: int,
@@ -60,7 +63,7 @@ def create_kms_keyring_given_kms_client():
 @click.option('--n_iters',
               default=PerfTestUtils.DEFAULT_N_ITERS)
 @click.option('--output_file',
-              default='/'.join(__file__.split("/")[:-3]) + '/results/kms_keyring_create_given_kms_client')
+              default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/kms_keyring_create_given_kms_client')
 def create_given_kms_client(
     kms_key_id: str,
     n_iters: int,
@@ -88,15 +91,14 @@ def encrypt_kms_keyring():
 
 @encrypt_kms_keyring.command()
 @click.option('--plaintext_data_filename',
-              default='/'.join(__file__.split("/")[:-2]) + '/resources/plaintext/plaintext-data-'
-              + PerfTestUtils.DEFAULT_FILE_SIZE + '.dat',
-              prompt='Filename containing plaintext data you want to encrypt')
+              default='/'.join(MODULE_ABS_PATH.split("/")[:-2]) + '/resources/plaintext/plaintext-data-'
+              + PerfTestUtils.DEFAULT_FILE_SIZE + '.dat')
 @click.option('--kms_key_id',
               default='arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f')
 @click.option('--n_iters',
               default=PerfTestUtils.DEFAULT_N_ITERS)
 @click.option('--output_file',
-              default='/'.join(__file__.split("/")[:-3]) + '/results/kms_keyring_encrypt')
+              default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/kms_keyring_encrypt')
 def encrypt(
     plaintext_data_filename: str,
     kms_key_id: str,
@@ -128,15 +130,14 @@ def decrypt_kms_keyring():
 
 @decrypt_kms_keyring.command()
 @click.option('--ciphertext_data_filename',
-              default='/'.join(__file__.split("/")[:-2]) + '/resources/ciphertext/kms/ciphertext-data-'
-              + PerfTestUtils.DEFAULT_FILE_SIZE + '.ct',
-              prompt='Filename containing ciphertext data you want to decrypt')
+              default='/'.join(MODULE_ABS_PATH.split("/")[:-2]) + '/resources/ciphertext/kms/ciphertext-data-'
+              + PerfTestUtils.DEFAULT_FILE_SIZE + '.ct')
 @click.option('--kms_key_id',
               default='arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f')
 @click.option('--n_iters',
               default=PerfTestUtils.DEFAULT_N_ITERS)
 @click.option('--output_file',
-              default='/'.join(__file__.split("/")[:-3]) + '/results/kms_keyring_decrypt')
+              default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/kms_keyring_decrypt')
 def decrypt(
     ciphertext_data_filename: str,
     kms_key_id: str,

performance_tests/test/keyrings/test_raw_aes_keyring.py

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 """This is a performance test for creating the Raw AES keyring."""
 
+import os
 import time
 
 import click
@@ -16,6 +17,8 @@
 )
 from aws_encryption_sdk_performance_tests.utils.util import PerfTestUtils
 
+MODULE_ABS_PATH = os.path.abspath(__file__)
+
 
 @click.group()
 def create_raw_aes_keyring():
@@ -26,7 +29,7 @@ def create_raw_aes_keyring():
 @click.option('--n_iters',
               default=PerfTestUtils.DEFAULT_N_ITERS)
 @click.option('--output_file',
-              default='/'.join(__file__.split("/")[:-3]) + '/results/raw_aes_keyring_create')
+              default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/raw_aes_keyring_create')
 def create(
     n_iters: int,
     output_file: str
@@ -52,13 +55,12 @@ def encrypt_raw_aes_keyring():
 
 @encrypt_raw_aes_keyring.command()
 @click.option('--plaintext_data_filename',
-              default='/'.join(__file__.split("/")[:-2]) + '/resources/plaintext/plaintext-data-'
-              + PerfTestUtils.DEFAULT_FILE_SIZE + '.dat',
-              prompt='Filename containing plaintext data you want to encrypt')
+              default='/'.join(MODULE_ABS_PATH.split("/")[:-2]) + '/resources/plaintext/plaintext-data-'
+              + PerfTestUtils.DEFAULT_FILE_SIZE + '.dat')
 @click.option('--n_iters',
               default=PerfTestUtils.DEFAULT_N_ITERS)
 @click.option('--output_file',
-              default='/'.join(__file__.split("/")[:-3]) + '/results/raw_aes_keyring_encrypt')
+              default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/raw_aes_keyring_encrypt')
 def encrypt(
     plaintext_data_filename: str,
     n_iters: int,
@@ -89,13 +91,12 @@ def decrypt_raw_aes_keyring():
 
 @decrypt_raw_aes_keyring.command()
 @click.option('--ciphertext_data_filename',
-              default='/'.join(__file__.split("/")[:-2]) + '/resources/ciphertext/raw_aes/ciphertext-data-'
-              + PerfTestUtils.DEFAULT_FILE_SIZE + '.ct',
-              prompt='Filename containing ciphertext data you want to decrypt')
+              default='/'.join(MODULE_ABS_PATH.split("/")[:-2]) + '/resources/ciphertext/raw_aes/ciphertext-data-'
+              + PerfTestUtils.DEFAULT_FILE_SIZE + '.ct')
 @click.option('--n_iters',
               default=PerfTestUtils.DEFAULT_N_ITERS)
 @click.option('--output_file',
-              default='/'.join(__file__.split("/")[:-3]) + '/results/raw_aes_keyring_decrypt')
+              default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/raw_aes_keyring_decrypt')
 def decrypt(
     ciphertext_data_filename: str,
     n_iters: int,

performance_tests/test/keyrings/test_raw_rsa_keyring.py

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 """This is a performance test for creating the Raw RSA keyring."""
 
+import os
 import time
 
 import click
@@ -16,6 +17,8 @@
 )
 from aws_encryption_sdk_performance_tests.utils.util import PerfTestUtils
 
+MODULE_ABS_PATH = os.path.abspath(__file__)
+
 
 @click.group()
 def create_raw_rsa_keyring():
@@ -26,7 +29,7 @@ def create_raw_rsa_keyring():
 @click.option('--n_iters',
               default=PerfTestUtils.DEFAULT_N_ITERS)
 @click.option('--output_file',
-              default='/'.join(__file__.split("/")[:-3]) + '/results/raw_rsa_keyring_create')
+              default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/raw_rsa_keyring_create')
 def create(
     n_iters: int,
     output_file: str
@@ -55,13 +58,12 @@ def encrypt_raw_rsa_keyring():
 
 @encrypt_raw_rsa_keyring.command()
 @click.option('--plaintext_data_filename',
-              default='/'.join(__file__.split("/")[:-2]) + '/resources/plaintext/plaintext-data-'
-              + PerfTestUtils.DEFAULT_FILE_SIZE + '.dat',
-              prompt='Filename containing plaintext data you want to encrypt')
+              default='/'.join(MODULE_ABS_PATH.split("/")[:-2]) + '/resources/plaintext/plaintext-data-'
+              + PerfTestUtils.DEFAULT_FILE_SIZE + '.dat')
 @click.option('--n_iters',
               default=PerfTestUtils.DEFAULT_N_ITERS)
 @click.option('--output_file',
-              default='/'.join(__file__.split("/")[:-3]) + '/results/raw_rsa_keyring_encrypt')
+              default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/raw_rsa_keyring_encrypt')
 def encrypt(
     plaintext_data_filename: str,
     n_iters: int,
@@ -94,13 +96,12 @@ def decrypt_raw_rsa_keyring():
 
 @decrypt_raw_rsa_keyring.command()
 @click.option('--ciphertext_data_filename',
-              default='/'.join(__file__.split("/")[:-2]) + '/resources/ciphertext/raw_rsa/ciphertext-data-'
-              + PerfTestUtils.DEFAULT_FILE_SIZE + '.ct',
-              prompt='Filename containing ciphertext data you want to decrypt')
+              default='/'.join(MODULE_ABS_PATH.split("/")[:-2]) + '/resources/ciphertext/raw_rsa/ciphertext-data-'
+              + PerfTestUtils.DEFAULT_FILE_SIZE + '.ct')
 @click.option('--n_iters',
               default=PerfTestUtils.DEFAULT_N_ITERS)
 @click.option('--output_file',
-              default='/'.join(__file__.split("/")[:-3]) + '/results/raw_rsa_keyring_decrypt')
+              default='/'.join(MODULE_ABS_PATH.split("/")[:-3]) + '/results/raw_rsa_keyring_decrypt')
 def decrypt(
     ciphertext_data_filename: str,
     n_iters: int,