Skip to content

Commit 2f46a86

Browse files
RitvikKapilalucasmcdonald3
RitvikKapila
authored and
lucasmcdonald3
committed
chore: performance tests for ESDK-python (#680)
1 parent 4c5d4ad commit 2f46a86

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+2441
-0
lines changed

‎.gitignore

+6
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,12 @@ __pycache__
3333
.pytest_cache
3434
# Ignore key materials generated by examples or tests
3535
test_keyrings/
36+
# Ignore results of performance test
37+
performance_tests/results/*.csv
38+
performance_tests/results/*.pstats
39+
performance_tests/results/*.png
40+
# Ignore the memory profile logs
41+
mprofile_*
3642

3743
# PyCharm
3844
.idea/

‎buildspec.yml

+8
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,10 @@ batch:
126126
buildspec: codebuild/py311/integ_mpl.yml
127127
env:
128128
image: aws/codebuild/standard:7.0
129+
- identifier: py311_performance_tests_mpl
130+
buildspec: codebuild/py311/performance_tests_mpl.yml
131+
env:
132+
image: aws/codebuild/standard:7.0
129133
- identifier: py311_examples
130134
buildspec: codebuild/py311/examples.yml
131135
env:
@@ -212,6 +216,10 @@ batch:
212216
buildspec: codebuild/py312/integ_mpl.yml
213217
env:
214218
image: aws/codebuild/standard:7.0
219+
- identifier: py312_performance_tests_mpl
220+
buildspec: codebuild/py312/performance_tests_mpl.yml
221+
env:
222+
image: aws/codebuild/standard:7.0
215223
- identifier: py312_examples
216224
buildspec: codebuild/py312/examples.yml
217225
env:

‎codebuild/py311/performance_tests_mpl.yml

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# Runs the performance tests for the MPL in an environment with the MPL installed
2+
version: 0.2
3+
4+
env:
5+
variables:
6+
# No TOXENV. This runs multiple environments.
7+
REGION: "us-west-2"
8+
AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >-
9+
arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f
10+
AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >-
11+
arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2
12+
AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >-
13+
arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7
14+
AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >-
15+
arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7
16+
17+
phases:
18+
install:
19+
runtime-versions:
20+
python: 3.11
21+
build:
22+
commands:
23+
- cd /root/.pyenv/plugins/python-build/../.. && git pull && cd -
24+
- pyenv install --skip-existing 3.11.0
25+
- pyenv local 3.11.0
26+
- pip install --upgrade pip
27+
- pip install setuptools
28+
- pip install "tox < 4.0"
29+
# Assume special role to access keystore
30+
- TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-Py312ExamplesMpl")
31+
- export TMP_ROLE
32+
- export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId')
33+
- export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey')
34+
- export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken')
35+
- aws sts get-caller-identity
36+
# Run MPL-specific tests with special role
37+
- cd performance_tests/
38+
- tox -e py311-performance_tests-mpl

‎codebuild/py312/performance_tests_mpl.yml

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# Runs the performance tests for the MPL in an environment with the MPL installed
2+
version: 0.2
3+
4+
env:
5+
variables:
6+
# No TOXENV. This runs multiple environments.
7+
REGION: "us-west-2"
8+
AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >-
9+
arn:aws:kms:us-west-2:658956600833:key/b3537ef1-d8dc-4780-9f5a-55776cbb2f7f
10+
AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >-
11+
arn:aws:kms:eu-central-1:658956600833:key/75414c93-5285-4b57-99c9-30c1cf0a22c2
12+
AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_1: >-
13+
arn:aws:kms:us-west-2:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7
14+
AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_MRK_KEY_ID_2: >-
15+
arn:aws:kms:us-east-1:658956600833:key/mrk-80bd8ecdcd4342aebd84b7dc9da498a7
16+
17+
phases:
18+
install:
19+
runtime-versions:
20+
python: 3.12
21+
build:
22+
commands:
23+
- cd /root/.pyenv/plugins/python-build/../.. && git pull && cd -
24+
- pyenv install --skip-existing 3.12.0
25+
- pyenv local 3.12.0
26+
- pip install --upgrade pip
27+
- pip install setuptools
28+
- pip install "tox < 4.0"
29+
# Assume special role to access keystore
30+
- TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Python-Role-us-west-2" --role-session-name "CB-Py312ExamplesMpl")
31+
- export TMP_ROLE
32+
- export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId')
33+
- export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey')
34+
- export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken')
35+
- aws sts get-caller-identity
36+
# Run MPL-specific tests with special role
37+
- cd performance_tests/
38+
- tox -e py312-performance_tests-mpl

‎performance_tests/README.rst

+219
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,219 @@
1+
#####################################
2+
aws-encryption-sdk performance tests
3+
#####################################
4+
5+
This module runs performance tests for the `AWS Encryption SDK Python`_.
6+
7+
********
8+
Overview
9+
********
10+
11+
This module tests the following keyrings / master key providers:
12+
13+
1. KMS Keyring / KMS Master Key Provider
14+
2. Raw AES Keyring / AES Master Key Provider
15+
3. Raw RSA Keyring / RSA Master Key Provider
16+
4. Hierarchy Keyring
17+
5. Caching CMM
18+
19+
For each test on the above keyrings / master key providers, this package measures:
20+
21+
1. Execution time
22+
2. Total memory consumption
23+
24+
For each keyring / master key provider, the execution time and memory consumption
25+
is measured for three operations:
26+
27+
1. Create keyring / master key provider
28+
2. Encrypt
29+
3. Decrypt
30+
31+
The usage of the performance tests is demonstrated through an `AWS KMS Keyring`_.
32+
However, the procedure is the same for any keyring / master key provider, with slight
33+
changes in the input arguments.
34+
35+
The results for the performance test will be available in the results folder in the
36+
performance_tests directory.
37+
38+
**********************
39+
Required Prerequisites
40+
**********************
41+
42+
* Python 3.8+
43+
* aws-encryption-sdk
44+
* boto3 >= 1.10.0
45+
* click
46+
* tqdm
47+
* pytest
48+
49+
Recommended Prerequisites
50+
=========================
51+
52+
* aws-cryptographic-material-providers: >= 1.0.0
53+
* Requires Python 3.11+.
54+
55+
*****
56+
Usage
57+
*****
58+
59+
Execution Time
60+
==============
61+
62+
Create Keyring
63+
--------------
64+
To run the performance test for execution time, please use the
65+
following commands in the performance_tests directory.
66+
67+
.. code::
68+
69+
usage: python test/keyrings/test_aws_kms_keyring.py create
70+
71+
Create a keyring to use for encryption and decryption.
72+
73+
optional arguments:
74+
-h, --help show this help message and exit.
75+
--kms_key_id KMS_KEY_ID The KMS key ID you want to use.
76+
--n_iters N_ITERS Number of iterations you want to
77+
run the test for. For instance,
78+
if n_iters = 100, this performance
79+
test script will run the create_keyring
80+
method 100 times and report the
81+
execution time of each of the calls.
82+
--output_file OUTPUT_FILE The output file for execution times
83+
for each function call,
84+
default='kms_keyring_create' in the
85+
results folder.
86+
87+
Encrypt
88+
-------
89+
90+
To run the performance test for execution time, please use the following
91+
commands in the performance_tests directory:
92+
93+
.. code::
94+
95+
usage: python test/keyrings/test_aws_kms_keyring.py encrypt
96+
97+
optional arguments:
98+
-h, --help show this help message and exit.
99+
--plaintext_data_filename PLAINTEXT_DATA_FILENAME Filename containing plaintext data
100+
you want to encrypt.
101+
default='test/resources/plaintext/plaintext-data-medium.dat'.
102+
You can choose to use any other plaintext
103+
file as well. Some example plaintext
104+
data files are present in the
105+
'test/resources' directory.
106+
--kms_key_id KMS_KEY_ID The KMS key ID you want to use.
107+
--n_iters N_ITERS Number of iterations you want to
108+
run the test for. For instance,
109+
if n_iters = 100, this performance
110+
test script will run the create_keyring
111+
method 100 times and report the
112+
execution time of each of the calls.
113+
--output_file OUTPUT_FILE The output file for execution times
114+
for each function call,
115+
default='kms_keyring_create' in the
116+
results folder.
117+
118+
Decrypt
119+
-------
120+
121+
To run the performance test for execution time, please use the
122+
following commands in the performance_tests directory
123+
124+
.. code::
125+
126+
usage: python test/keyrings/test_aws_kms_keyring.py decrypt
127+
128+
optional arguments:
129+
-h, --help show this help message and exit.
130+
--ciphertext_data_filename CIPHERTEXT_DATA_FILENAME Filename containing ciphertext data
131+
you want to decrypt.
132+
default='test/resources/ciphertext/kms/ciphertext-data-medium.ct'.
133+
You can choose to use any other
134+
ciphertext file as well. Some example
135+
ciphertext data files are present in
136+
the 'test/resources' directory.
137+
--kms_key_id KMS_KEY_ID The KMS key ID you want to use.
138+
--n_iters N_ITERS Number of iterations you want to
139+
run the test for. For instance,
140+
if n_iters = 100, this performance
141+
test script will run the create_keyring
142+
method 100 times and report the
143+
execution time of each of the calls.
144+
--output_file OUTPUT_FILE The output file for execution times
145+
for each function call,
146+
default='kms_keyring_create' in the
147+
results folder.
148+
149+
Consolidate Time Results
150+
========================
151+
152+
In order to find the minimum, maximum, average, 99th percentile and bottom
153+
99th percentile trimmed average times from the n_iters runs, please use the
154+
following script from the performance_tests directory with the csv file
155+
containing times for each of the n_iters runs generated in the previous
156+
"Execution Time" section:
157+
158+
.. code::
159+
160+
usage: python consolidate_results.py results/kms_keyring_decrypt.csv
161+
162+
Memory Consumption
163+
==================
164+
165+
To get the memory consumption, simply replace 'python'
166+
with 'mprof run' in the previously mentioned commands.
167+
168+
For example, if you want to calculate the memory consumption
169+
of the encrypt function of a AWS KMS Keyring, simply write:
170+
171+
.. code::
172+
173+
usage: mprof run test/keyrings/test_aws_kms_keyring.py encrypt
174+
175+
176+
This should generate an mprofile log file in your current directory.
177+
This mprofile log file contains the total memory consumed by the program
178+
with respect to time elapsed.
179+
To plot the memory consumption with respect to time, please use the following
180+
command from the same directory
181+
182+
.. code::
183+
184+
usage: mprof plot
185+
186+
187+
This 'mprof plot' command will plot the most recent mprofile log file.
188+
189+
190+
Performance Graph
191+
=================
192+
193+
To generate a performance graph, please use the following command
194+
to generate the pstats log file by specifying the output pstats file
195+
path. Here, 'results/kms_keyring_create.pstats' is set as the default
196+
output file.
197+
198+
.. code::
199+
200+
usage: python -m cProfile -o results/kms_keyring_create.pstats test/keyrings/test_aws_kms_keyring.py create
201+
202+
203+
After generating the pstats file, please run the following command
204+
to generate the performance graph. The output performance graph will
205+
be a .png file that you specify. Here, 'results/kms_keyring_create.png'
206+
is set as the default output file.
207+
208+
.. code::
209+
210+
usage: gprof2dot -f pstats results/kms_keyring_create.pstats | dot -Tpng -o results/kms_keyring_create.png && eog results/kms_keyring_create.png
211+
212+
213+
Note: This project does not adhere to semantic versioning; as such it
214+
makes no guarantees that functionality will persist across major,
215+
minor, or patch versions.
216+
**DO NOT** take a standalone dependency on this library.
217+
218+
.. _AWS Encryption SDK Python: https://github.com/aws/aws-encryption-sdk-python/
219+
.. _AWS KMS Keyring: https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/use-kms-keyring.html

‎performance_tests/__init__.py

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
"""Stub module indicator to make linter configuration simpler."""

‎performance_tests/consolidate_results.py

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
"""Script for consolidating results for execution times"""
4+
5+
import argparse
6+
import csv
7+
8+
import numpy as np
9+
10+
11+
def calculate_statistics(_csv_file):
12+
"""Calculate average, trimmed average, minimum, maximum and p99 statistics for execution times in a CSV file."""
13+
with open(_csv_file, 'r', encoding='utf-8') as file:
14+
reader = csv.reader(file)
15+
data = [float(row[0]) for row in reader]
16+
17+
output_stats = {}
18+
19+
# Calculate statistics
20+
if data:
21+
data = np.sort(data)
22+
output_stats['total_entries'] = len(data)
23+
output_stats['average'] = np.mean(data)
24+
output_stats['trimmed_average_99_bottom'] = np.mean(data[0:int(0.99 * len(data))])
25+
output_stats['minimum'] = min(data)
26+
output_stats['maximum'] = max(data)
27+
output_stats['perc_99'] = np.percentile(data, 99)
28+
return output_stats
29+
30+
return None
31+
32+
33+
if __name__ == "__main__":
34+
parser = argparse.ArgumentParser()
35+
parser.add_argument('csv_file',
36+
help='csv file containing the outputs of execution times for n_iter iterations')
37+
args = parser.parse_args()
38+
39+
statistics = calculate_statistics(args.csv_file)
40+
if statistics:
41+
print("CSV File:", args.csv_file)
42+
print("Total Entries:", statistics['total_entries'])
43+
print("Average:", statistics['average'])
44+
print("Bottom 99th percentile trimmed average:", statistics['trimmed_average_99_bottom'])
45+
print("Minimum:", statistics['minimum'])
46+
print("Maximum:", statistics['maximum'])
47+
print("99th percentile:", statistics['perc_99'])
48+
else:
49+
print("No data found in the CSV file.")

0 commit comments

Comments
 (0)