From 10fdceb18d7cf04692d4c7cb6426157dc956d2f3 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Fri, 17 Jun 2022 10:51:33 +0200 Subject: [PATCH 1/2] fix(perf): retry tests in rare old CI machines --- poetry.lock | 53 ++++++-------------- pyproject.toml | 1 + tests/performance/conftest.py | 18 ------- tests/performance/test_high_level_imports.py | 7 +++ 4 files changed, 24 insertions(+), 55 deletions(-) delete mode 100644 tests/performance/conftest.py diff --git a/poetry.lock b/poetry.lock index f67f25b12ca..f6869d14306 100644 --- a/poetry.lock +++ b/poetry.lock @@ -867,6 +867,17 @@ pytest = ">=5.0" [package.extras] dev = ["pre-commit", "tox", "pytest-asyncio"] +[[package]] +name = "pytest-rerunfailures" +version = "10.2" +description = "pytest plugin to re-run tests to eliminate flaky failures" +category = "dev" +optional = false +python-versions = ">= 3.6" + +[package.dependencies] +pytest = ">=5.3" + [[package]] name = "python-dateutil" version = "2.8.2" @@ -1101,7 +1112,7 @@ pydantic = ["pydantic", "email-validator"] [metadata] lock-version = "1.1" python-versions = "^3.6.2" -content-hash = "e457c68bd754118733c7ad1c54d389f4aa3b06164d947fae5d682566e202b776" +content-hash = "896da945bd733721bcbce899620cbd7af3845d5bbebd68046c454af65b57e91d" [metadata.files] atomicwrites = [ @@ -1307,28 +1318,12 @@ markdown = [ {file = "Markdown-3.3.5.tar.gz", hash = "sha256:26e9546bfbcde5fcd072bd8f612c9c1b6e2677cb8aadbdf65206674f46dde069"}, ] markupsafe = [ - {file = "MarkupSafe-2.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d8446c54dc28c01e5a2dbac5a25f071f6653e6e40f3a8818e8b45d790fe6ef53"}, - {file = "MarkupSafe-2.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:36bc903cbb393720fad60fc28c10de6acf10dc6cc883f3e24ee4012371399a38"}, - {file = "MarkupSafe-2.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d7d807855b419fc2ed3e631034685db6079889a1f01d5d9dac950f764da3dad"}, - {file = "MarkupSafe-2.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:add36cb2dbb8b736611303cd3bfcee00afd96471b09cda130da3581cbdc56a6d"}, - {file = "MarkupSafe-2.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:168cd0a3642de83558a5153c8bd34f175a9a6e7f6dc6384b9655d2697312a646"}, - {file = "MarkupSafe-2.0.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4dc8f9fb58f7364b63fd9f85013b780ef83c11857ae79f2feda41e270468dd9b"}, - {file = "MarkupSafe-2.0.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:20dca64a3ef2d6e4d5d615a3fd418ad3bde77a47ec8a23d984a12b5b4c74491a"}, - {file = "MarkupSafe-2.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:cdfba22ea2f0029c9261a4bd07e830a8da012291fbe44dc794e488b6c9bb353a"}, - {file = "MarkupSafe-2.0.1-cp310-cp310-win32.whl", hash = "sha256:99df47edb6bda1249d3e80fdabb1dab8c08ef3975f69aed437cb69d0a5de1e28"}, - {file = "MarkupSafe-2.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:e0f138900af21926a02425cf736db95be9f4af72ba1bb21453432a07f6082134"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:0955295dd5eec6cb6cc2fe1698f4c6d84af2e92de33fbcac4111913cd100a6ff"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:0446679737af14f45767963a1a9ef7620189912317d095f2d9ffa183a4d25d2b"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:f826e31d18b516f653fe296d967d700fddad5901ae07c622bb3705955e1faa94"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:fa130dd50c57d53368c9d59395cb5526eda596d3ffe36666cd81a44d56e48872"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:905fec760bd2fa1388bb5b489ee8ee5f7291d692638ea5f67982d968366bef9f"}, - {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf5d821ffabf0ef3533c39c518f3357b171a1651c1ff6827325e4489b0e46c3c"}, - {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0d4b31cc67ab36e3392bbf3862cfbadac3db12bdd8b02a2731f509ed5b829724"}, - {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:baa1a4e8f868845af802979fcdbf0bb11f94f1cb7ced4c4b8a351bb60d108145"}, - {file = "MarkupSafe-2.0.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:deb993cacb280823246a026e3b2d81c493c53de6acfd5e6bfe31ab3402bb37dd"}, - {file = "MarkupSafe-2.0.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:63f3268ba69ace99cab4e3e3b5840b03340efed0948ab8f78d2fd87ee5442a4f"}, - {file = "MarkupSafe-2.0.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:8d206346619592c6200148b01a2142798c989edcb9c896f9ac9722a99d4e77e6"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-win32.whl", hash = "sha256:6c4ca60fa24e85fe25b912b01e62cb969d69a23a5d5867682dd3e80b5b02581d"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:b2f4bf27480f5e5e8ce285a8c8fd176c0b03e93dcc6646477d4630e83440c6a9"}, {file = "MarkupSafe-2.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0717a7390a68be14b8c793ba258e075c6f4ca819f15edfc2a3a027c823718567"}, @@ -1337,27 +1332,14 @@ markupsafe = [ {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:d7f9850398e85aba693bb640262d3611788b1f29a79f0c93c565694658f4071f"}, {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:6a7fae0dd14cf60ad5ff42baa2e95727c3d81ded453457771d02b7d2b3f9c0c2"}, {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:b7f2d075102dc8c794cbde1947378051c4e5180d52d276987b8d28a3bd58c17d"}, - {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e9936f0b261d4df76ad22f8fee3ae83b60d7c3e871292cd42f40b81b70afae85"}, - {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:2a7d351cbd8cfeb19ca00de495e224dea7e7d919659c2841bbb7f420ad03e2d6"}, - {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:60bf42e36abfaf9aff1f50f52644b336d4f0a3fd6d8a60ca0d054ac9f713a864"}, - {file = "MarkupSafe-2.0.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d6c7ebd4e944c85e2c3421e612a7057a2f48d478d79e61800d81468a8d842207"}, - {file = "MarkupSafe-2.0.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f0567c4dc99f264f49fe27da5f735f414c4e7e7dd850cfd8e69f0862d7c74ea9"}, - {file = "MarkupSafe-2.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:89c687013cb1cd489a0f0ac24febe8c7a666e6e221b783e53ac50ebf68e45d86"}, {file = "MarkupSafe-2.0.1-cp37-cp37m-win32.whl", hash = "sha256:a30e67a65b53ea0a5e62fe23682cfe22712e01f453b95233b25502f7c61cb415"}, {file = "MarkupSafe-2.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:611d1ad9a4288cf3e3c16014564df047fe08410e628f89805e475368bd304914"}, - {file = "MarkupSafe-2.0.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5bb28c636d87e840583ee3adeb78172efc47c8b26127267f54a9c0ec251d41a9"}, {file = "MarkupSafe-2.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:be98f628055368795d818ebf93da628541e10b75b41c559fdf36d104c5787066"}, {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:1d609f577dc6e1aa17d746f8bd3c31aa4d258f4070d61b2aa5c4166c1539de35"}, {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7d91275b0245b1da4d4cfa07e0faedd5b0812efc15b702576d103293e252af1b"}, {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:01a9b8ea66f1658938f65b93a85ebe8bc016e6769611be228d797c9d998dd298"}, {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:47ab1e7b91c098ab893b828deafa1203de86d0bc6ab587b160f78fe6c4011f75"}, {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:97383d78eb34da7e1fa37dd273c20ad4320929af65d156e35a5e2d89566d9dfb"}, - {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6fcf051089389abe060c9cd7caa212c707e58153afa2c649f00346ce6d260f1b"}, - {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:5855f8438a7d1d458206a2466bf82b0f104a3724bf96a1c781ab731e4201731a"}, - {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3dd007d54ee88b46be476e293f48c85048603f5f516008bee124ddd891398ed6"}, - {file = "MarkupSafe-2.0.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:aca6377c0cb8a8253e493c6b451565ac77e98c2951c45f913e0b52facdcff83f"}, - {file = "MarkupSafe-2.0.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:04635854b943835a6ea959e948d19dcd311762c5c0c6e1f0e16ee57022669194"}, - {file = "MarkupSafe-2.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6300b8454aa6930a24b9618fbb54b5a68135092bc666f7b06901f897fa5c2fee"}, {file = "MarkupSafe-2.0.1-cp38-cp38-win32.whl", hash = "sha256:023cb26ec21ece8dc3907c0e8320058b2e0cb3c55cf9564da612bc325bed5e64"}, {file = "MarkupSafe-2.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:984d76483eb32f1bcb536dc27e4ad56bba4baa70be32fa87152832cdd9db0833"}, {file = "MarkupSafe-2.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2ef54abee730b502252bcdf31b10dacb0a416229b72c18b19e24a4509f273d26"}, @@ -1367,12 +1349,6 @@ markupsafe = [ {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:4efca8f86c54b22348a5467704e3fec767b2db12fc39c6d963168ab1d3fc9135"}, {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:ab3ef638ace319fa26553db0624c4699e31a28bb2a835c5faca8f8acf6a5a902"}, {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:f8ba0e8349a38d3001fae7eadded3f6606f0da5d748ee53cc1dab1d6527b9509"}, - {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c47adbc92fc1bb2b3274c4b3a43ae0e4573d9fbff4f54cd484555edbf030baf1"}, - {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:37205cac2a79194e3750b0af2a5720d95f786a55ce7df90c3af697bfa100eaac"}, - {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1f2ade76b9903f39aa442b4aadd2177decb66525062db244b35d71d0ee8599b6"}, - {file = "MarkupSafe-2.0.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4296f2b1ce8c86a6aea78613c34bb1a672ea0e3de9c6ba08a960efe0b0a09047"}, - {file = "MarkupSafe-2.0.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f02365d4e99430a12647f09b6cc8bab61a6564363f313126f775eb4f6ef798e"}, - {file = "MarkupSafe-2.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5b6d930f030f8ed98e3e6c98ffa0652bdb82601e7a016ec2ab5d7ff23baa78d1"}, {file = "MarkupSafe-2.0.1-cp39-cp39-win32.whl", hash = "sha256:10f82115e21dc0dfec9ab5c0223652f7197feb168c940f3ef61563fc2d6beb74"}, {file = "MarkupSafe-2.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8"}, {file = "MarkupSafe-2.0.1.tar.gz", hash = "sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a"}, @@ -1462,7 +1438,6 @@ pbr = [ {file = "pbr-5.8.0.tar.gz", hash = "sha256:672d8ebee84921862110f23fcec2acea191ef58543d34dfe9ef3d9f13c31cddf"}, ] pdoc3 = [ - {file = "pdoc3-0.10.0-py3-none-any.whl", hash = "sha256:ba45d1ada1bd987427d2bf5cdec30b2631a3ff5fb01f6d0e77648a572ce6028b"}, {file = "pdoc3-0.10.0.tar.gz", hash = "sha256:5f22e7bcb969006738e1aa4219c75a32f34c2d62d46dc9d2fb2d3e0b0287e4b7"}, ] platformdirs = [ @@ -1550,6 +1525,10 @@ pytest-mock = [ {file = "pytest-mock-3.6.1.tar.gz", hash = "sha256:40217a058c52a63f1042f0784f62009e976ba824c418cced42e88d5f40ab0e62"}, {file = "pytest_mock-3.6.1-py3-none-any.whl", hash = "sha256:30c2f2cc9759e76eee674b81ea28c9f0b94f8f0445a1b87762cadf774f0df7e3"}, ] +pytest-rerunfailures = [ + {file = "pytest-rerunfailures-10.2.tar.gz", hash = "sha256:9e1e1bad51e07642c5bbab809fc1d4ec8eebcb7de86f90f1a26e6ef9de446697"}, + {file = "pytest_rerunfailures-10.2-py3-none-any.whl", hash = "sha256:d31d8e828dfd39363ad99cd390187bf506c7a433a89f15c3126c7d16ab723fe2"}, +] python-dateutil = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, diff --git a/pyproject.toml b/pyproject.toml index 53b73908361..d002346bd2a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,6 +59,7 @@ mypy-boto3-secretsmanager = "^1.24.0" mypy-boto3-ssm = "^1.24.0" mypy-boto3-appconfig = "^1.24.0" mypy-boto3-dynamodb = "^1.24.0" +pytest-rerunfailures = "^10.2" [tool.poetry.extras] diff --git a/tests/performance/conftest.py b/tests/performance/conftest.py deleted file mode 100644 index 30cb371ca87..00000000000 --- a/tests/performance/conftest.py +++ /dev/null @@ -1,18 +0,0 @@ -import time -from contextlib import contextmanager -from typing import Generator - - -@contextmanager -def timing() -> Generator: - """ "Generator to quickly time operations. It can add 5ms so take that into account in elapsed time - - Examples - -------- - - with timing() as t: - print("something") - elapsed = t() - """ - start = time.perf_counter() - yield lambda: time.perf_counter() - start # gen as lambda to calculate elapsed time diff --git a/tests/performance/test_high_level_imports.py b/tests/performance/test_high_level_imports.py index e3914b26f57..2bfcf59560f 100644 --- a/tests/performance/test_high_level_imports.py +++ b/tests/performance/test_high_level_imports.py @@ -1,4 +1,5 @@ import importlib +import os import time from contextlib import contextmanager from types import ModuleType @@ -11,6 +12,8 @@ TRACER_INIT_SLA: float = 0.5 IMPORT_INIT_SLA: float = 0.035 +IS_CI = os.getenv("CI", False) + @contextmanager def timing() -> Generator: @@ -37,6 +40,7 @@ def core_utilities() -> Tuple[ModuleType, ModuleType, ModuleType]: @pytest.mark.perf +@pytest.mark.flaky(reruns=1, condition=IS_CI) def test_import_times_ceiling(): # GIVEN Core utilities are imported # WHEN none are used @@ -52,6 +56,7 @@ def test_import_times_ceiling(): @pytest.mark.perf +@pytest.mark.flaky(reruns=1, reruns_delay=1, condition=IS_CI) def test_tracer_init(): # GIVEN Tracer is initialized # WHEN default options are used @@ -68,6 +73,7 @@ def test_tracer_init(): @pytest.mark.perf +@pytest.mark.flaky(reruns=1, reruns_delay=1, condition=IS_CI) def test_metrics_init(): # GIVEN Metrics is initialized # WHEN default options are used @@ -82,6 +88,7 @@ def test_metrics_init(): @pytest.mark.perf +@pytest.mark.flaky(reruns=1, reruns_delay=1, condition=IS_CI) def test_logger_init(): # GIVEN Logger is initialized # WHEN default options are used From f240dfa158e0e9283461cc93a5b85f3369596d3e Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Fri, 17 Jun 2022 12:13:59 +0200 Subject: [PATCH 2/2] fix(perf): use pytest-benchmark for more reliable timers --- poetry.lock | 49 ++++++--- pyproject.toml | 2 +- tests/performance/test_high_level_imports.py | 107 +++++++++---------- 3 files changed, 84 insertions(+), 74 deletions(-) diff --git a/poetry.lock b/poetry.lock index f6869d14306..538be837ee7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -740,6 +740,14 @@ category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "py-cpuinfo" +version = "8.0.0" +description = "Get CPU info with pure Python 2 & 3" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "pycodestyle" version = "2.8.0" @@ -838,6 +846,23 @@ pytest = ">=5.4.0" [package.extras] testing = ["coverage", "hypothesis (>=5.7.1)"] +[[package]] +name = "pytest-benchmark" +version = "3.4.1" +description = "A ``pytest`` fixture for benchmarking code. It will group the tests into rounds that are calibrated to the chosen timer." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[package.dependencies] +py-cpuinfo = "*" +pytest = ">=3.8" + +[package.extras] +aspect = ["aspectlib"] +elasticsearch = ["elasticsearch"] +histogram = ["pygal", "pygaljs"] + [[package]] name = "pytest-cov" version = "3.0.0" @@ -867,17 +892,6 @@ pytest = ">=5.0" [package.extras] dev = ["pre-commit", "tox", "pytest-asyncio"] -[[package]] -name = "pytest-rerunfailures" -version = "10.2" -description = "pytest plugin to re-run tests to eliminate flaky failures" -category = "dev" -optional = false -python-versions = ">= 3.6" - -[package.dependencies] -pytest = ">=5.3" - [[package]] name = "python-dateutil" version = "2.8.2" @@ -1112,7 +1126,7 @@ pydantic = ["pydantic", "email-validator"] [metadata] lock-version = "1.1" python-versions = "^3.6.2" -content-hash = "896da945bd733721bcbce899620cbd7af3845d5bbebd68046c454af65b57e91d" +content-hash = "449ddaa45a50ae48f9feacc55a1e70fef05cad5e0a6978ec024bb770ace9e28a" [metadata.files] atomicwrites = [ @@ -1452,6 +1466,9 @@ py = [ {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, ] +py-cpuinfo = [ + {file = "py-cpuinfo-8.0.0.tar.gz", hash = "sha256:5f269be0e08e33fd959de96b34cd4aeeeacac014dd8305f70eb28d06de2345c5"}, +] pycodestyle = [ {file = "pycodestyle-2.8.0-py2.py3-none-any.whl", hash = "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20"}, {file = "pycodestyle-2.8.0.tar.gz", hash = "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f"}, @@ -1517,6 +1534,10 @@ pytest-asyncio = [ {file = "pytest-asyncio-0.16.0.tar.gz", hash = "sha256:7496c5977ce88c34379df64a66459fe395cd05543f0a2f837016e7144391fcfb"}, {file = "pytest_asyncio-0.16.0-py3-none-any.whl", hash = "sha256:5f2a21273c47b331ae6aa5b36087047b4899e40f03f18397c0e65fa5cca54e9b"}, ] +pytest-benchmark = [ + {file = "pytest-benchmark-3.4.1.tar.gz", hash = "sha256:40e263f912de5a81d891619032983557d62a3d85843f9a9f30b98baea0cd7b47"}, + {file = "pytest_benchmark-3.4.1-py2.py3-none-any.whl", hash = "sha256:36d2b08c4882f6f997fd3126a3d6dfd70f3249cde178ed8bbc0b73db7c20f809"}, +] pytest-cov = [ {file = "pytest-cov-3.0.0.tar.gz", hash = "sha256:e7f0f5b1617d2210a2cabc266dfe2f4c75a8d32fb89eafb7ad9d06f6d076d470"}, {file = "pytest_cov-3.0.0-py3-none-any.whl", hash = "sha256:578d5d15ac4a25e5f961c938b85a05b09fdaae9deef3bb6de9a6e766622ca7a6"}, @@ -1525,10 +1546,6 @@ pytest-mock = [ {file = "pytest-mock-3.6.1.tar.gz", hash = "sha256:40217a058c52a63f1042f0784f62009e976ba824c418cced42e88d5f40ab0e62"}, {file = "pytest_mock-3.6.1-py3-none-any.whl", hash = "sha256:30c2f2cc9759e76eee674b81ea28c9f0b94f8f0445a1b87762cadf774f0df7e3"}, ] -pytest-rerunfailures = [ - {file = "pytest-rerunfailures-10.2.tar.gz", hash = "sha256:9e1e1bad51e07642c5bbab809fc1d4ec8eebcb7de86f90f1a26e6ef9de446697"}, - {file = "pytest_rerunfailures-10.2-py3-none-any.whl", hash = "sha256:d31d8e828dfd39363ad99cd390187bf506c7a433a89f15c3126c7d16ab723fe2"}, -] python-dateutil = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, diff --git a/pyproject.toml b/pyproject.toml index d002346bd2a..15ff4ad6b19 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,7 +59,7 @@ mypy-boto3-secretsmanager = "^1.24.0" mypy-boto3-ssm = "^1.24.0" mypy-boto3-appconfig = "^1.24.0" mypy-boto3-dynamodb = "^1.24.0" -pytest-rerunfailures = "^10.2" +pytest-benchmark = "^3.4.1" [tool.poetry.extras] diff --git a/tests/performance/test_high_level_imports.py b/tests/performance/test_high_level_imports.py index 2bfcf59560f..7639065dd83 100644 --- a/tests/performance/test_high_level_imports.py +++ b/tests/performance/test_high_level_imports.py @@ -1,9 +1,6 @@ import importlib -import os -import time -from contextlib import contextmanager from types import ModuleType -from typing import Generator, Tuple +from typing import Tuple import pytest @@ -11,92 +8,88 @@ METRICS_INIT_SLA: float = 0.005 TRACER_INIT_SLA: float = 0.5 IMPORT_INIT_SLA: float = 0.035 +PARENT_PACKAGE = "aws_lambda_powertools" +TRACING_PACKAGE = "aws_lambda_powertools.tracing" +LOGGING_PACKAGE = "aws_lambda_powertools.logging" +METRICS_PACKAGE = "aws_lambda_powertools.metrics" -IS_CI = os.getenv("CI", False) +def import_core_utilities() -> Tuple[ModuleType, ModuleType, ModuleType]: + """Dynamically imports and return Tracing, Logging, and Metrics modules""" + return ( + importlib.import_module(TRACING_PACKAGE), + importlib.import_module(LOGGING_PACKAGE), + importlib.import_module(METRICS_PACKAGE), + ) -@contextmanager -def timing() -> Generator: - """ "Generator to quickly time operations. It can add 5ms so take that into account in elapsed time - Examples - -------- +@pytest.fixture(autouse=True) +def clear_cache(): + importlib.invalidate_caches() - with timing() as t: - print("something") - elapsed = t() - """ - start = time.perf_counter() - yield lambda: time.perf_counter() - start # gen as lambda to calculate elapsed time +def import_init_tracer(): + tracing = importlib.import_module(TRACING_PACKAGE) + tracing.Tracer(disabled=True) -def core_utilities() -> Tuple[ModuleType, ModuleType, ModuleType]: - """Return Tracing, Logging, and Metrics module""" - tracing = importlib.import_module("aws_lambda_powertools.tracing") - logging = importlib.import_module("aws_lambda_powertools.logging") - metrics = importlib.import_module("aws_lambda_powertools.metrics") - return tracing, logging, metrics +def import_init_metrics(): + metrics = importlib.import_module(METRICS_PACKAGE) + metrics.Metrics() + + +def import_init_logger(): + logging = importlib.import_module(LOGGING_PACKAGE) + logging.Logger() @pytest.mark.perf -@pytest.mark.flaky(reruns=1, condition=IS_CI) -def test_import_times_ceiling(): +@pytest.mark.benchmark(group="core", disable_gc=True, warmup=False) +def test_import_times_ceiling(benchmark): # GIVEN Core utilities are imported # WHEN none are used # THEN import and any global initialization perf should be below 30ms # though we adjust to 35ms to take into account different CI machines, etc. # instead of re-running tests which can lead to false positives - with timing() as t: - core_utilities() - - elapsed = t() - if elapsed > IMPORT_INIT_SLA: - pytest.fail(f"High level imports should be below ${IMPORT_INIT_SLA}s: {elapsed}") + benchmark.pedantic(import_core_utilities) + stat = benchmark.stats.stats.max + if stat > IMPORT_INIT_SLA: + pytest.fail(f"High level imports should be below {IMPORT_INIT_SLA}s: {stat}") @pytest.mark.perf -@pytest.mark.flaky(reruns=1, reruns_delay=1, condition=IS_CI) -def test_tracer_init(): +@pytest.mark.benchmark(group="core", disable_gc=True, warmup=False) +def test_tracer_init(benchmark): # GIVEN Tracer is initialized # WHEN default options are used # THEN initialization X-Ray SDK perf should be below 450ms # though we adjust to 500ms to take into account different CI machines, etc. # instead of re-running tests which can lead to false positives - with timing() as t: - tracing, _, _ = core_utilities() - tracing.Tracer(disabled=True) # boto3 takes ~200ms, and remaining is X-Ray SDK init - - elapsed = t() - if elapsed > TRACER_INIT_SLA: - pytest.fail(f"High level imports should be below ${TRACER_INIT_SLA}s: {elapsed}") + benchmark.pedantic(import_init_tracer) + stat = benchmark.stats.stats.max + if stat > TRACER_INIT_SLA: + pytest.fail(f"High level imports should be below {TRACER_INIT_SLA}s: {stat}") @pytest.mark.perf -@pytest.mark.flaky(reruns=1, reruns_delay=1, condition=IS_CI) -def test_metrics_init(): +@pytest.mark.benchmark(group="core", disable_gc=True, warmup=False) +def test_metrics_init(benchmark): # GIVEN Metrics is initialized # WHEN default options are used # THEN initialization perf should be below 5ms - with timing() as t: - _, _, metrics = core_utilities() - metrics.Metrics() - - elapsed = t() - if elapsed > METRICS_INIT_SLA: - pytest.fail(f"High level imports should be below ${METRICS_INIT_SLA}s: {elapsed}") + benchmark.pedantic(import_init_metrics) + stat = benchmark.stats.stats.max + if stat > METRICS_INIT_SLA: + pytest.fail(f"High level imports should be below ${METRICS_INIT_SLA}s: {stat}") @pytest.mark.perf -@pytest.mark.flaky(reruns=1, reruns_delay=1, condition=IS_CI) -def test_logger_init(): +@pytest.mark.benchmark(group="core", disable_gc=True, warmup=False) +def test_logger_init(benchmark): # GIVEN Logger is initialized # WHEN default options are used # THEN initialization perf should be below 5ms - with timing() as t: - _, logging, _ = core_utilities() - logging.Logger() - - elapsed = t() - if elapsed > LOGGER_INIT_SLA: - pytest.fail(f"High level imports should be below ${LOGGER_INIT_SLA}s: {elapsed}") + benchmark.pedantic(import_init_logger) + stat = benchmark.stats.stats.max + if stat > LOGGER_INIT_SLA: + pytest.fail(f"High level imports should be below ${LOGGER_INIT_SLA}s: {stat}")