diff --git a/Makefile b/Makefile index 94f9fc975b8..3977ee8e7a7 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ unit-test: poetry run pytest tests/unit e2e-test: - poetry run pytest -rP -n 3 --dist loadscope --durations=0 --durations-min=1 tests/e2e + poetry run pytest -rP -n auto --dist loadfile -o log_cli=true tests/e2e coverage-html: poetry run pytest -m "not perf" --ignore tests/e2e --cov=aws_lambda_powertools --cov-report=html diff --git a/poetry.lock b/poetry.lock index 905c852476c..9b94cb96cfe 100644 --- a/poetry.lock +++ b/poetry.lock @@ -678,7 +678,7 @@ mkdocs = ">=0.17" [[package]] name = "mkdocs-material" -version = "8.4.0" +version = "8.4.1" description = "Documentation that simply works" category = "dev" optional = false @@ -721,11 +721,11 @@ reports = ["lxml"] [[package]] name = "mypy-boto3-appconfig" -version = "1.24.29" -description = "Type annotations for boto3.AppConfig 1.24.29 service generated with mypy-boto3-builder 7.7.3" +version = "1.24.36.post1" +description = "Type annotations for boto3.AppConfig 1.24.36 service generated with mypy-boto3-builder 7.10.0" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.dependencies] typing-extensions = ">=4.1.0" @@ -743,33 +743,33 @@ typing-extensions = ">=4.1.0" [[package]] name = "mypy-boto3-cloudwatch" -version = "1.24.35" -description = "Type annotations for boto3.CloudWatch 1.24.35 service generated with mypy-boto3-builder 7.9.2" +version = "1.24.55" +description = "Type annotations for boto3.CloudWatch 1.24.55 service generated with mypy-boto3-builder 7.11.6" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.dependencies] typing-extensions = ">=4.1.0" [[package]] name = "mypy-boto3-dynamodb" -version = "1.24.27" -description = "Type annotations for boto3.DynamoDB 1.24.27 service generated with mypy-boto3-builder 7.6.0" +version = "1.24.55.post1" +description = "Type annotations for boto3.DynamoDB 1.24.55 service generated with mypy-boto3-builder 7.11.7" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.dependencies] typing-extensions = ">=4.1.0" [[package]] name = "mypy-boto3-lambda" -version = "1.24.0" -description = "Type annotations for boto3.Lambda 1.24.0 service generated with mypy-boto3-builder 7.6.1" +version = "1.24.54" +description = "Type annotations for boto3.Lambda 1.24.54 service generated with mypy-boto3-builder 7.11.6" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.dependencies] typing-extensions = ">=4.1.0" @@ -798,33 +798,33 @@ typing-extensions = ">=4.1.0" [[package]] name = "mypy-boto3-secretsmanager" -version = "1.24.11.post3" -description = "Type annotations for boto3.SecretsManager 1.24.11 service generated with mypy-boto3-builder 7.7.1" +version = "1.24.54" +description = "Type annotations for boto3.SecretsManager 1.24.54 service generated with mypy-boto3-builder 7.11.6" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.dependencies] typing-extensions = ">=4.1.0" [[package]] name = "mypy-boto3-ssm" -version = "1.24.0" -description = "Type annotations for boto3.SSM 1.24.0 service generated with mypy-boto3-builder 7.6.1" +version = "1.24.39.post2" +description = "Type annotations for boto3.SSM 1.24.39 service generated with mypy-boto3-builder 7.10.1" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.dependencies] typing-extensions = ">=4.1.0" [[package]] name = "mypy-boto3-xray" -version = "1.24.0" -description = "Type annotations for boto3.XRay 1.24.0 service generated with mypy-boto3-builder 7.6.1" +version = "1.24.36.post1" +description = "Type annotations for boto3.XRay 1.24.36 service generated with mypy-boto3-builder 7.10.0" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.dependencies] typing-extensions = ">=4.1.0" @@ -1356,7 +1356,7 @@ pydantic = ["pydantic", "email-validator"] [metadata] lock-version = "1.1" python-versions = "^3.6.2" -content-hash = "77b3593db443d2972a854cf7eaf6643e33315d5da218933f360b33a2e3bb945d" +content-hash = "0a9e21c2f15825934ad6c786121da020c4a964c5a0dd138e0e8ae09c0865a055" [metadata.files] atomicwrites = [ @@ -1647,8 +1647,8 @@ mkdocs-git-revision-date-plugin = [ {file = "mkdocs_git_revision_date_plugin-0.3.2-py3-none-any.whl", hash = "sha256:2e67956cb01823dd2418e2833f3623dee8604cdf223bddd005fe36226a56f6ef"}, ] mkdocs-material = [ - {file = "mkdocs-material-8.4.0.tar.gz", hash = "sha256:6c0a6e6cda8b43956e0c562374588160af8110584a1444f422b1cfd91930f9c7"}, - {file = "mkdocs_material-8.4.0-py2.py3-none-any.whl", hash = "sha256:ef6641e1910d4f217873ac376b4594f3157dca3949901b88b4991ba8e5477577"}, + {file = "mkdocs-material-8.4.1.tar.gz", hash = "sha256:92c70f94b2e1f8a05d9e05eec1c7af9dffc516802d69222329db89503c97b4f3"}, + {file = "mkdocs_material-8.4.1-py2.py3-none-any.whl", hash = "sha256:319a6254819ce9d864ff79de48c43842fccfdebb43e4e6820eef75216f8cfb0a"}, ] mkdocs-material-extensions = [ {file = "mkdocs-material-extensions-1.0.3.tar.gz", hash = "sha256:bfd24dfdef7b41c312ede42648f9eb83476ea168ec163b613f9abd12bbfddba2"}, @@ -1680,24 +1680,24 @@ mypy = [ {file = "mypy-0.971.tar.gz", hash = "sha256:40b0f21484238269ae6a57200c807d80debc6459d444c0489a102d7c6a75fa56"}, ] mypy-boto3-appconfig = [ - {file = "mypy-boto3-appconfig-1.24.29.tar.gz", hash = "sha256:10583d309a9db99babfbe85d3b6467b49b3509a57e4f8771da239f6d5cb3731b"}, - {file = "mypy_boto3_appconfig-1.24.29-py3-none-any.whl", hash = "sha256:e9d9e2e25fdd82bffc6262dc184edf5d0d3d9fbb0ab35e597a1ea57ba13d4d80"}, + {file = "mypy-boto3-appconfig-1.24.36.post1.tar.gz", hash = "sha256:e1916b3754915cb411ef977083500e1f30f81f7b3aea6ff5eed1cec91944dea6"}, + {file = "mypy_boto3_appconfig-1.24.36.post1-py3-none-any.whl", hash = "sha256:a5dbe549dbebf4bc7a6cfcbfa9dff89ceb4983c042b785763ee656504bdb49f6"}, ] mypy-boto3-cloudformation = [ {file = "mypy-boto3-cloudformation-1.24.36.post1.tar.gz", hash = "sha256:ed7df9ae3a8390a145229122a1489d0a58bbf9986cb54f0d7a65ed54f12c8e63"}, {file = "mypy_boto3_cloudformation-1.24.36.post1-py3-none-any.whl", hash = "sha256:b39020c13a876bb18908aad22326478d0ac3faec0bdac0d2c11dc318c9dcf149"}, ] mypy-boto3-cloudwatch = [ - {file = "mypy-boto3-cloudwatch-1.24.35.tar.gz", hash = "sha256:92a818e2ea330f9afb5f8f9c15df47934736041e3ccfd696ffc0774bad14e0aa"}, - {file = "mypy_boto3_cloudwatch-1.24.35-py3-none-any.whl", hash = "sha256:28947763d70cdac24aca25779cd5b00cd995636f5815fac3d95009430ce02b72"}, + {file = "mypy-boto3-cloudwatch-1.24.55.tar.gz", hash = "sha256:f8950de7a93b3db890cd8524514a2245d9b5fd83ce2dd60a37047a2cd42d5dd6"}, + {file = "mypy_boto3_cloudwatch-1.24.55-py3-none-any.whl", hash = "sha256:23faf8fdfe928f9dcce453a60b03bda69177554eb88c2d7e5240ff91b5b14388"}, ] mypy-boto3-dynamodb = [ - {file = "mypy-boto3-dynamodb-1.24.27.tar.gz", hash = "sha256:c982d24f9b2525a70f408ad40eff69660d56928217597d88860b60436b25efbf"}, - {file = "mypy_boto3_dynamodb-1.24.27-py3-none-any.whl", hash = "sha256:63f7d9755fc5cf2e637edf8d33024050152a53013d1a102716ae0d534563ef07"}, + {file = "mypy-boto3-dynamodb-1.24.55.post1.tar.gz", hash = "sha256:c469223c15556d93d247d38c0c31ce3c08d8073ca4597158a27abc70b8d7fbee"}, + {file = "mypy_boto3_dynamodb-1.24.55.post1-py3-none-any.whl", hash = "sha256:c762975d023b356c573d58105c7bfc1b9e7ee62c1299f09784e9dede533179e1"}, ] mypy-boto3-lambda = [ - {file = "mypy-boto3-lambda-1.24.0.tar.gz", hash = "sha256:ab425f941d0d50a2b8a20cc13cebe03c3097b122259bf00e7b295d284814bd6f"}, - {file = "mypy_boto3_lambda-1.24.0-py3-none-any.whl", hash = "sha256:a286a464513adf50847bda8573f2dc7adc348234827d1ac0200e610ee9a09b80"}, + {file = "mypy-boto3-lambda-1.24.54.tar.gz", hash = "sha256:c76d28d84bdf94c8980acd85bc07f2747559ca11a990fd6785c9c2389e13aff1"}, + {file = "mypy_boto3_lambda-1.24.54-py3-none-any.whl", hash = "sha256:231b6aac22b107ebb7afa2ec6dc1311b769dbdd5bfae957cf60db3e8bc3133d7"}, ] mypy-boto3-logs = [ {file = "mypy-boto3-logs-1.24.36.post1.tar.gz", hash = "sha256:8b00c2d5328e72023b1d1acd65e7cea7854f07827d23ce21c78391ca74271290"}, @@ -1708,16 +1708,16 @@ mypy-boto3-s3 = [ {file = "mypy_boto3_s3-1.24.36.post1-py3-none-any.whl", hash = "sha256:30ae59b33c55f8b7b693170f9519ea5b91a2fbf31a73de79cdef57a27d784e5a"}, ] mypy-boto3-secretsmanager = [ - {file = "mypy-boto3-secretsmanager-1.24.11.post3.tar.gz", hash = "sha256:f153b3f5ff2c65664a906fb2c97a6598a57da9f1da77679dbaf541051dcff36e"}, - {file = "mypy_boto3_secretsmanager-1.24.11.post3-py3-none-any.whl", hash = "sha256:d9655d568f7fd8fe05265613b85fba55ab6e4dcd078989af1ef9f0ffe4b45019"}, + {file = "mypy-boto3-secretsmanager-1.24.54.tar.gz", hash = "sha256:a846b79f86e218a794dbc858c08290bb6aebffa180c80cf0a463c32a04621ff1"}, + {file = "mypy_boto3_secretsmanager-1.24.54-py3-none-any.whl", hash = "sha256:b89c9a0ff65a8ab2c4e4d3f6e721a0477b7d0fec246ffc08e4378420eb50b4d0"}, ] mypy-boto3-ssm = [ - {file = "mypy-boto3-ssm-1.24.0.tar.gz", hash = "sha256:bab58398947c3627a4e7610cd0f57b525c12fd1d0a6bb862400b6af0a4e684fc"}, - {file = "mypy_boto3_ssm-1.24.0-py3-none-any.whl", hash = "sha256:1f17055abb8d70f25e6ece2ef4c0dc74d585744c25a3a833c2985d74165ac0c6"}, + {file = "mypy-boto3-ssm-1.24.39.post2.tar.gz", hash = "sha256:2859bdcef110d9cc53007a7adba9c765e804b886f98d742a496bb8f7dac07308"}, + {file = "mypy_boto3_ssm-1.24.39.post2-py3-none-any.whl", hash = "sha256:bfdb434c513fbb1f3bc4b5c158ed4e7a46cb578e5eb01e818d45f4f38296ef2c"}, ] mypy-boto3-xray = [ - {file = "mypy-boto3-xray-1.24.0.tar.gz", hash = "sha256:fbe211b7601684a2d4defa2f959286f1441027c15044c0c0013257e22307778a"}, - {file = "mypy_boto3_xray-1.24.0-py3-none-any.whl", hash = "sha256:6b9bc96e7924215fe833fe0d732d5e3ce98f7739b373432b9735a9905f867171"}, + {file = "mypy-boto3-xray-1.24.36.post1.tar.gz", hash = "sha256:104f1ecf7f1f6278c582201e71a7ab64843d3a3fdc8f23295cf68788cc77e9bb"}, + {file = "mypy_boto3_xray-1.24.36.post1-py3-none-any.whl", hash = "sha256:97b9f0686c717c8be99ac06cb52febaf71712b4e4cd0b61ed2eb5ed012a9b5fd"}, ] mypy-extensions = [ {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, diff --git a/pyproject.toml b/pyproject.toml index ae6e1a5d56a..b12fdcc092a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,20 +52,20 @@ flake8-bugbear = "^22.7.1" mkdocs-git-revision-date-plugin = "^0.3.2" mike = "^0.6.0" mypy = "^0.971" -mypy-boto3-secretsmanager = "^1.24.11" -mypy-boto3-ssm = "^1.24.0" -mypy-boto3-appconfig = "^1.24.29" -mypy-boto3-dynamodb = "^1.24.27" retry = "^0.9.2" pytest-xdist = "^2.5.0" aws-cdk-lib = "^2.23.0" pytest-benchmark = "^3.4.1" -mypy-boto3-cloudwatch = "^1.24.35" -mypy-boto3-lambda = "^1.24.0" -mypy-boto3-xray = "^1.24.0" -mypy-boto3-s3 = { version = "^1.24.0", python = ">=3.7" } +mypy-boto3-appconfig = { version = "^1.24.29", python = ">=3.7" } mypy-boto3-cloudformation = { version = "^1.24.0", python = ">=3.7" } +mypy-boto3-cloudwatch = { version = "^1.24.35", python = ">=3.7" } +mypy-boto3-dynamodb = { version = "^1.24.27", python = ">=3.7" } +mypy-boto3-lambda = { version = "^1.24.0", python = ">=3.7" } mypy-boto3-logs = { version = "^1.24.0", python = ">=3.7" } +mypy-boto3-secretsmanager = { version = "^1.24.11", python = ">=3.7" } +mypy-boto3-ssm = { version = "^1.24.0", python = ">=3.7" } +mypy-boto3-s3 = { version = "^1.24.0", python = ">=3.7" } +mypy-boto3-xray = { version = "^1.24.0", python = ">=3.7" } types-requests = "^2.28.8" typing-extensions = { version = "^4.3.0", python = ">=3.7" } python-snappy = "^0.6.1" diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 3865be4d3e7..ac55d373e63 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -1,66 +1,35 @@ -import datetime -import sys -import uuid -from dataclasses import dataclass - -import boto3 - -from tests.e2e.utils import data_fetcher, infrastructure - -# We only need typing_extensions for python versions <3.8 -if sys.version_info >= (3, 8): - from typing import TypedDict -else: - from typing_extensions import TypedDict - -from typing import Dict, Generator, Optional - import pytest - -class LambdaConfig(TypedDict): - parameters: dict - environment_variables: Dict[str, str] - - -@dataclass -class InfrastructureOutput: - arns: Dict[str, str] - execution_time: datetime.datetime - - def get_lambda_arns(self) -> Dict[str, str]: - return self.arns - - def get_lambda_function_arn(self, cf_output_name: str) -> Optional[str]: - return self.arns.get(cf_output_name) - - def get_lambda_function_name(self, cf_output_name: str) -> Optional[str]: - lambda_arn = self.get_lambda_function_arn(cf_output_name=cf_output_name) - return lambda_arn.split(":")[-1] if lambda_arn else None - - def get_lambda_execution_time(self) -> datetime.datetime: - return self.execution_time - - def get_lambda_execution_time_timestamp(self) -> int: - return int(self.execution_time.timestamp() * 1000) - - -@pytest.fixture(scope="module") -def create_infrastructure(config, request) -> Generator[Dict[str, str], None, None]: - stack_name = f"test-lambda-{uuid.uuid4()}" - test_dir = request.fspath.dirname - handlers_dir = f"{test_dir}/handlers/" - - infra = infrastructure.Infrastructure(stack_name=stack_name, handlers_dir=handlers_dir, config=config) - yield infra.deploy(Stack=infrastructure.InfrastructureStack) - infra.delete() - - -@pytest.fixture(scope="module") -def execute_lambda(create_infrastructure) -> InfrastructureOutput: - execution_time = datetime.datetime.utcnow() - session = boto3.Session() - client = session.client("lambda") - for _, arn in create_infrastructure.items(): - data_fetcher.get_lambda_response(lambda_arn=arn, client=client) - return InfrastructureOutput(arns=create_infrastructure, execution_time=execution_time) +from tests.e2e.utils.infrastructure import LambdaLayerStack, deploy_once + + +@pytest.fixture(scope="session") +def lambda_layer_arn(lambda_layer_deployment): + yield lambda_layer_deployment.get("LayerArn") + + +@pytest.fixture(scope="session") +def lambda_layer_deployment(request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory, worker_id: str): + """Setup and teardown logic for E2E test infrastructure + + Parameters + ---------- + request : pytest.FixtureRequest + pytest request fixture to introspect absolute path to test being executed + tmp_path_factory : pytest.TempPathFactory + pytest temporary path factory to discover shared tmp when multiple CPU processes are spun up + worker_id : str + pytest-xdist worker identification to detect whether parallelization is enabled + + Yields + ------ + Dict[str, str] + CloudFormation Outputs from deployed infrastructure + """ + yield from deploy_once( + stack=LambdaLayerStack, + request=request, + tmp_path_factory=tmp_path_factory, + worker_id=worker_id, + layer_arn="", + ) diff --git a/tests/e2e/logger/conftest.py b/tests/e2e/logger/conftest.py index 201a5f7dca1..82a89314258 100644 --- a/tests/e2e/logger/conftest.py +++ b/tests/e2e/logger/conftest.py @@ -1,25 +1,28 @@ +from pathlib import Path + import pytest from tests.e2e.logger.infrastructure import LoggerStack -from tests.e2e.utils.infrastructure import deploy_once @pytest.fixture(autouse=True, scope="module") -def infrastructure(request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory, worker_id: str): +def infrastructure(request: pytest.FixtureRequest, lambda_layer_arn: str): """Setup and teardown logic for E2E test infrastructure Parameters ---------- request : pytest.FixtureRequest pytest request fixture to introspect absolute path to test being executed - tmp_path_factory : pytest.TempPathFactory - pytest temporary path factory to discover shared tmp when multiple CPU processes are spun up - worker_id : str - pytest-xdist worker identification to detect whether parallelization is enabled + lambda_layer_arn : str + Lambda Layer ARN Yields ------ Dict[str, str] CloudFormation Outputs from deployed infrastructure """ - yield from deploy_once(stack=LoggerStack, request=request, tmp_path_factory=tmp_path_factory, worker_id=worker_id) + stack = LoggerStack(handlers_dir=Path(f"{request.path.parent}/handlers"), layer_arn=lambda_layer_arn) + try: + yield stack.deploy() + finally: + stack.delete() diff --git a/tests/e2e/logger/infrastructure.py b/tests/e2e/logger/infrastructure.py index 76595908206..68aaa8eb38a 100644 --- a/tests/e2e/logger/infrastructure.py +++ b/tests/e2e/logger/infrastructure.py @@ -1,11 +1,13 @@ from pathlib import Path -from tests.e2e.utils.infrastructure import BaseInfrastructureV2 +from tests.e2e.utils.infrastructure import BaseInfrastructure -class LoggerStack(BaseInfrastructureV2): - def __init__(self, handlers_dir: Path, feature_name: str = "logger") -> None: - super().__init__(feature_name, handlers_dir) +class LoggerStack(BaseInfrastructure): + FEATURE_NAME = "logger" + + def __init__(self, handlers_dir: Path, feature_name: str = FEATURE_NAME, layer_arn: str = "") -> None: + super().__init__(feature_name, handlers_dir, layer_arn) def create_resources(self): self.create_lambda_functions() diff --git a/tests/e2e/metrics/conftest.py b/tests/e2e/metrics/conftest.py index 18f4564e714..663c8845be4 100644 --- a/tests/e2e/metrics/conftest.py +++ b/tests/e2e/metrics/conftest.py @@ -1,25 +1,28 @@ +from pathlib import Path + import pytest from tests.e2e.metrics.infrastructure import MetricsStack -from tests.e2e.utils.infrastructure import deploy_once @pytest.fixture(autouse=True, scope="module") -def infrastructure(request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory, worker_id: str): +def infrastructure(request: pytest.FixtureRequest, lambda_layer_arn: str): """Setup and teardown logic for E2E test infrastructure Parameters ---------- request : pytest.FixtureRequest pytest request fixture to introspect absolute path to test being executed - tmp_path_factory : pytest.TempPathFactory - pytest temporary path factory to discover shared tmp when multiple CPU processes are spun up - worker_id : str - pytest-xdist worker identification to detect whether parallelization is enabled + lambda_layer_arn : str + Lambda Layer ARN Yields ------ Dict[str, str] CloudFormation Outputs from deployed infrastructure """ - yield from deploy_once(stack=MetricsStack, request=request, tmp_path_factory=tmp_path_factory, worker_id=worker_id) + stack = MetricsStack(handlers_dir=Path(f"{request.path.parent}/handlers"), layer_arn=lambda_layer_arn) + try: + yield stack.deploy() + finally: + stack.delete() diff --git a/tests/e2e/metrics/infrastructure.py b/tests/e2e/metrics/infrastructure.py index e01fb16b02e..9afa59bb5cd 100644 --- a/tests/e2e/metrics/infrastructure.py +++ b/tests/e2e/metrics/infrastructure.py @@ -1,11 +1,13 @@ from pathlib import Path -from tests.e2e.utils.infrastructure import BaseInfrastructureV2 +from tests.e2e.utils.infrastructure import BaseInfrastructure -class MetricsStack(BaseInfrastructureV2): - def __init__(self, handlers_dir: Path, feature_name: str = "metrics") -> None: - super().__init__(feature_name, handlers_dir) +class MetricsStack(BaseInfrastructure): + FEATURE_NAME = "metrics" + + def __init__(self, handlers_dir: Path, feature_name: str = FEATURE_NAME, layer_arn: str = "") -> None: + super().__init__(feature_name, handlers_dir, layer_arn) def create_resources(self): self.create_lambda_functions() diff --git a/tests/e2e/tracer/conftest.py b/tests/e2e/tracer/conftest.py index 599d7ab4ca8..3b724bf1247 100644 --- a/tests/e2e/tracer/conftest.py +++ b/tests/e2e/tracer/conftest.py @@ -1,25 +1,28 @@ +from pathlib import Path + import pytest from tests.e2e.tracer.infrastructure import TracerStack -from tests.e2e.utils.infrastructure import deploy_once @pytest.fixture(autouse=True, scope="module") -def infrastructure(request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory, worker_id: str): +def infrastructure(request: pytest.FixtureRequest, lambda_layer_arn: str): """Setup and teardown logic for E2E test infrastructure Parameters ---------- request : pytest.FixtureRequest pytest request fixture to introspect absolute path to test being executed - tmp_path_factory : pytest.TempPathFactory - pytest temporary path factory to discover shared tmp when multiple CPU processes are spun up - worker_id : str - pytest-xdist worker identification to detect whether parallelization is enabled + lambda_layer_arn : str + Lambda Layer ARN Yields ------ Dict[str, str] CloudFormation Outputs from deployed infrastructure """ - yield from deploy_once(stack=TracerStack, request=request, tmp_path_factory=tmp_path_factory, worker_id=worker_id) + stack = TracerStack(handlers_dir=Path(f"{request.path.parent}/handlers"), layer_arn=lambda_layer_arn) + try: + yield stack.deploy() + finally: + stack.delete() diff --git a/tests/e2e/tracer/infrastructure.py b/tests/e2e/tracer/infrastructure.py index bd40fd2ca13..9b388558c0b 100644 --- a/tests/e2e/tracer/infrastructure.py +++ b/tests/e2e/tracer/infrastructure.py @@ -1,16 +1,17 @@ from pathlib import Path from tests.e2e.utils.data_builder import build_service_name -from tests.e2e.utils.infrastructure import BaseInfrastructureV2 +from tests.e2e.utils.infrastructure import BaseInfrastructure -class TracerStack(BaseInfrastructureV2): +class TracerStack(BaseInfrastructure): # Maintenance: Tracer doesn't support dynamic service injection (tracer.py L310) # we could move after handler response or adopt env vars usage in e2e tests SERVICE_NAME: str = build_service_name() + FEATURE_NAME = "tracer" - def __init__(self, handlers_dir: Path, feature_name: str = "tracer") -> None: - super().__init__(feature_name, handlers_dir) + def __init__(self, handlers_dir: Path, feature_name: str = FEATURE_NAME, layer_arn: str = "") -> None: + super().__init__(feature_name, handlers_dir, layer_arn) def create_resources(self) -> None: env_vars = {"POWERTOOLS_SERVICE_NAME": self.SERVICE_NAME} diff --git a/tests/e2e/utils/Dockerfile b/tests/e2e/utils/Dockerfile index eccfe2c6dfd..586847bb3fa 100644 --- a/tests/e2e/utils/Dockerfile +++ b/tests/e2e/utils/Dockerfile @@ -1,5 +1,5 @@ -# Image used by CDK's LayerVersion construct to create Lambda Layer with Powertools -# library code. +# Image used by CDK's LayerVersion construct to create Lambda Layer with Powertools +# library code. # The correct AWS SAM build image based on the runtime of the function will be # passed as build arg. The default allows to do `docker build .` when testing. ARG IMAGE=public.ecr.aws/sam/build-python3.7 @@ -9,8 +9,6 @@ ARG PIP_INDEX_URL ARG PIP_EXTRA_INDEX_URL ARG HTTPS_PROXY -# Upgrade pip (required by cryptography v3.4 and above, which is a dependency of poetry) RUN pip install --upgrade pip -RUN pip install pipenv poetry CMD [ "python" ] diff --git a/tests/e2e/utils/asset.py b/tests/e2e/utils/asset.py index 04d368a6ff4..db9e7299d1a 100644 --- a/tests/e2e/utils/asset.py +++ b/tests/e2e/utils/asset.py @@ -1,5 +1,6 @@ import io import json +import logging import zipfile from pathlib import Path from typing import Dict, List, Optional @@ -9,9 +10,7 @@ from mypy_boto3_s3 import S3Client from pydantic import BaseModel, Field -from aws_lambda_powertools import Logger - -logger = Logger(service="e2e-utils") +logger = logging.getLogger(__name__) class AssetManifest(BaseModel): @@ -113,6 +112,7 @@ def upload(self): We follow the same design cdk-assets: https://github.com/aws/aws-cdk-rfcs/blob/master/text/0092-asset-publishing.md. """ + logger.debug(f"Upload {len(self.assets)} assets") for asset in self.assets: if not asset.is_zip: logger.debug(f"Asset '{asset.object_key}' is not zip. Skipping upload.") @@ -138,7 +138,7 @@ def _find_assets_from_template(self) -> List[Asset]: def _compress_assets(self, asset: Asset) -> io.BytesIO: buf = io.BytesIO() asset_dir = f"{self.assets_location}/{asset.asset_path}" - asset_files = list(Path(asset_dir).iterdir()) + asset_files = list(Path(asset_dir).rglob("*")) with zipfile.ZipFile(buf, "w", compression=zipfile.ZIP_DEFLATED) as archive: for asset_file in asset_files: logger.debug(f"Adding file '{asset_file}' to the archive.") diff --git a/tests/e2e/utils/infrastructure.py b/tests/e2e/utils/infrastructure.py index ced6d70a1ad..7f232bb063f 100644 --- a/tests/e2e/utils/infrastructure.py +++ b/tests/e2e/utils/infrastructure.py @@ -1,12 +1,10 @@ -import io import json -import os +import logging import sys -import zipfile from abc import ABC, abstractmethod from enum import Enum from pathlib import Path -from typing import Dict, Generator, List, Optional, Tuple, Type +from typing import Dict, Generator, Optional, Tuple, Type from uuid import uuid4 import boto3 @@ -21,11 +19,7 @@ PYTHON_RUNTIME_VERSION = f"V{''.join(map(str, sys.version_info[:2]))}" - -class PythonVersion(Enum): - V37 = {"runtime": Runtime.PYTHON_3_7, "image": Runtime.PYTHON_3_7.bundling_image.image} - V38 = {"runtime": Runtime.PYTHON_3_8, "image": Runtime.PYTHON_3_8.bundling_image.image} - V39 = {"runtime": Runtime.PYTHON_3_9, "image": Runtime.PYTHON_3_9.bundling_image.image} +logger = logging.getLogger(__name__) class BaseInfrastructureStack(ABC): @@ -38,191 +32,22 @@ def __call__(self) -> Tuple[dict, str]: ... -class InfrastructureStack(BaseInfrastructureStack): - def __init__(self, handlers_dir: str, stack_name: str, config: dict) -> None: - self.stack_name = stack_name - self.handlers_dir = handlers_dir - self.config = config - - def _create_layer(self, stack: Stack): - output_dir = Path(str(AssetStaging.BUNDLING_OUTPUT_DIR), "python") - input_dir = Path(str(AssetStaging.BUNDLING_INPUT_DIR), "aws_lambda_powertools") - powertools_layer = LayerVersion( - stack, - "aws-lambda-powertools", - layer_version_name="aws-lambda-powertools", - compatible_runtimes=[PythonVersion[PYTHON_RUNTIME_VERSION].value["runtime"]], - code=Code.from_asset( - path=".", - bundling=BundlingOptions( - image=DockerImage.from_build( - str(Path(__file__).parent), - build_args={"IMAGE": PythonVersion[PYTHON_RUNTIME_VERSION].value["image"]}, - ), - command=[ - "bash", - "-c", - rf"poetry export --with-credentials --format requirements.txt --output /tmp/requirements.txt &&\ - pip install -r /tmp/requirements.txt -t {output_dir} &&\ - cp -R {input_dir} {output_dir}", - ], - ), - ), - ) - return powertools_layer - - def _find_handlers(self, directory: str) -> List: - for root, _, files in os.walk(directory): - return [os.path.join(root, filename) for filename in files if filename.endswith(".py")] - - def synthesize(self, handlers: List[str]) -> Tuple[dict, str, str]: - integration_test_app = App() - stack = Stack(integration_test_app, self.stack_name) - powertools_layer = self._create_layer(stack) - code = Code.from_asset(self.handlers_dir) - - for filename_path in handlers: - filename = Path(filename_path).stem - function_python = Function( - stack, - f"{filename}-lambda", - runtime=PythonVersion[PYTHON_RUNTIME_VERSION].value["runtime"], - code=code, - handler=f"{filename}.lambda_handler", - layers=[powertools_layer], - environment=self.config.get("environment_variables"), - tracing=Tracing.ACTIVE - if self.config.get("parameters", {}).get("tracing") == "ACTIVE" - else Tracing.DISABLED, - ) - - aws_logs.LogGroup( - stack, - f"{filename}-lg", - log_group_name=f"/aws/lambda/{function_python.function_name}", - retention=aws_logs.RetentionDays.ONE_DAY, - removal_policy=RemovalPolicy.DESTROY, - ) - CfnOutput(stack, f"{filename}_arn", value=function_python.function_arn) - cloud_assembly = integration_test_app.synth() - cf_template = cloud_assembly.get_stack_by_name(self.stack_name).template - cloud_assembly_directory = cloud_assembly.directory - cloud_assembly_assets_manifest_path = cloud_assembly.get_stack_by_name(self.stack_name).dependencies[0].file - - return (cf_template, cloud_assembly_directory, cloud_assembly_assets_manifest_path) - - def __call__(self) -> Tuple[dict, str]: - handlers = self._find_handlers(directory=self.handlers_dir) - return self.synthesize(handlers=handlers) - - -class Infrastructure: - def __init__(self, stack_name: str, handlers_dir: str, config: dict) -> None: - session = boto3.Session() - self.s3_client = session.client("s3") - self.lambda_client = session.client("lambda") - self.cfn = session.client("cloudformation") - self.s3_resource = session.resource("s3") - self.account_id = session.client("sts").get_caller_identity()["Account"] - self.region = session.region_name - self.stack_name = stack_name - self.handlers_dir = handlers_dir - self.config = config - - def deploy(self, Stack: Type[BaseInfrastructureStack]) -> Dict[str, str]: - - stack = Stack(handlers_dir=self.handlers_dir, stack_name=self.stack_name, config=self.config) - template, asset_root_dir, asset_manifest_file = stack() - self._upload_assets(asset_root_dir, asset_manifest_file) - - response = self._deploy_stack(self.stack_name, template) - - return self._transform_output(response["Stacks"][0]["Outputs"]) - - def delete(self): - self.cfn.delete_stack(StackName=self.stack_name) - - def _upload_assets(self, asset_root_dir: str, asset_manifest_file: str): - """ - This method is drop-in replacement for cdk-assets package s3 upload part. - https://www.npmjs.com/package/cdk-assets. - We use custom solution to avoid dependencies from nodejs ecosystem. - We follow the same design cdk-assets: - https://github.com/aws/aws-cdk-rfcs/blob/master/text/0092-asset-publishing.md. - """ - - assets = self._find_assets(asset_manifest_file, self.account_id, self.region) - - for s3_key, config in assets.items(): - print(config) - s3_bucket = self.s3_resource.Bucket(config["bucket_name"]) - - if config["asset_packaging"] != "zip": - print("Asset is not a zip file. Skipping upload") - continue - - if bool(list(s3_bucket.objects.filter(Prefix=s3_key))): - print("object exists, skipping") - continue - - buf = io.BytesIO() - asset_dir = f"{asset_root_dir}/{config['asset_path']}" - os.chdir(asset_dir) - asset_files = self._find_files(directory=".") - with zipfile.ZipFile(buf, "w", compression=zipfile.ZIP_DEFLATED) as zf: - for asset_file in asset_files: - zf.write(os.path.join(asset_file)) - buf.seek(0) - self.s3_client.upload_fileobj(Fileobj=buf, Bucket=config["bucket_name"], Key=s3_key) - - def _find_files(self, directory: str) -> List: - file_paths = [] - for root, _, files in os.walk(directory): - for filename in files: - file_paths.append(os.path.join(root, filename)) - return file_paths - - def _deploy_stack(self, stack_name: str, template: dict): - response = self.cfn.create_stack( - StackName=stack_name, - TemplateBody=yaml.dump(template), - TimeoutInMinutes=10, - OnFailure="ROLLBACK", - Capabilities=["CAPABILITY_IAM"], - ) - waiter = self.cfn.get_waiter("stack_create_complete") - waiter.wait(StackName=stack_name, WaiterConfig={"Delay": 10, "MaxAttempts": 50}) - response = self.cfn.describe_stacks(StackName=stack_name) - return response - - def _find_assets(self, asset_template: str, account_id: str, region: str): - assets = {} - with open(asset_template, mode="r") as template: - for _, config in json.loads(template.read())["files"].items(): - asset_path = config["source"]["path"] - asset_packaging = config["source"]["packaging"] - bucket_name = config["destinations"]["current_account-current_region"]["bucketName"] - object_key = config["destinations"]["current_account-current_region"]["objectKey"] - - assets[object_key] = { - "bucket_name": bucket_name.replace("${AWS::AccountId}", account_id).replace( - "${AWS::Region}", region - ), - "asset_path": asset_path, - "asset_packaging": asset_packaging, - } - - return assets - - def _transform_output(self, outputs: dict): - return {output["OutputKey"]: output["OutputValue"] for output in outputs if output["OutputKey"]} +class PythonVersion(Enum): + V37 = {"runtime": Runtime.PYTHON_3_7, "image": Runtime.PYTHON_3_7.bundling_image.image} + V38 = {"runtime": Runtime.PYTHON_3_8, "image": Runtime.PYTHON_3_8.bundling_image.image} + V39 = {"runtime": Runtime.PYTHON_3_9, "image": Runtime.PYTHON_3_9.bundling_image.image} -class BaseInfrastructureV2(ABC): - def __init__(self, feature_name: str, handlers_dir: Path) -> None: +class BaseInfrastructure(ABC): + def __init__(self, feature_name: str, handlers_dir: Path, layer_arn: str = "") -> None: + self.feature_name = feature_name self.stack_name = f"test-{feature_name}-{uuid4()}" self.handlers_dir = handlers_dir + self.layer_arn = layer_arn self.stack_outputs: Dict[str, str] = {} + + # NOTE: Investigate why cdk.Environment in Stack + # changes synthesized asset (no object_key in asset manifest) self.app = App() self.stack = Stack(self.app, self.stack_name) self.session = boto3.Session() @@ -242,7 +67,7 @@ def create_lambda_functions(self, function_props: Optional[Dict] = None): Parameters ---------- function_props: Optional[Dict] - CDK Lambda FunctionProps as dictionary to override defaults + Dictionary representing CDK Lambda FunctionProps to override defaults Examples -------- @@ -263,41 +88,41 @@ def create_lambda_functions(self, function_props: Optional[Dict] = None): """ handlers = list(self.handlers_dir.rglob("*.py")) source = Code.from_asset(f"{self.handlers_dir}") - props_override = function_props or {} + logger.debug(f"Creating functions for handlers: {handlers}") + if not self.layer_arn: + raise ValueError( + """Lambda Layer ARN cannot be empty when creating Lambda functions. + Make sure to inject `lambda_layer_arn` fixture and pass at the constructor level""" + ) + layer = LayerVersion.from_layer_version_arn(self.stack, "layer-arn", layer_version_arn=self.layer_arn) + function_settings_override = function_props or {} for fn in handlers: fn_name = fn.stem + fn_name_pascal_case = fn_name.title().replace("_", "") # basic_handler -> BasicHandler + logger.debug(f"Creating function: {fn_name_pascal_case}") function_settings = { "id": f"{fn_name}-lambda", "code": source, "handler": f"{fn_name}.lambda_handler", "tracing": Tracing.ACTIVE, "runtime": Runtime.PYTHON_3_9, - "layers": [ - LayerVersion.from_layer_version_arn( - self.stack, - f"{fn_name}-lambda-powertools", - f"arn:aws:lambda:{self.region}:017000801446:layer:AWSLambdaPowertoolsPython:29", - ) - ], - **props_override, + "layers": [layer], + **function_settings_override, } - function_python = Function(self.stack, **function_settings) + function = Function(self.stack, **function_settings) aws_logs.LogGroup( self.stack, id=f"{fn_name}-lg", - log_group_name=f"/aws/lambda/{function_python.function_name}", + log_group_name=f"/aws/lambda/{function.function_name}", retention=aws_logs.RetentionDays.ONE_DAY, removal_policy=RemovalPolicy.DESTROY, ) - # CFN Outputs only support hyphen - fn_name_pascal_case = fn_name.title().replace("_", "") # basic_handler -> BasicHandler - self._add_resource_output( - name=fn_name_pascal_case, value=function_python.function_name, arn=function_python.function_arn - ) + # CFN Outputs only support hyphen hence pascal case + self.add_cfn_output(name=fn_name_pascal_case, value=function.function_name, arn=function.function_arn) def deploy(self) -> Dict[str, str]: """Creates CloudFormation Stack and return stack outputs as dict @@ -310,10 +135,12 @@ def deploy(self) -> Dict[str, str]: template, asset_manifest_file = self._synthesize() assets = Assets(asset_manifest=asset_manifest_file, account_id=self.account_id, region=self.region) assets.upload() - return self._deploy_stack(self.stack_name, template) + self.stack_outputs = self._deploy_stack(self.stack_name, template) + return self.stack_outputs def delete(self) -> None: """Delete CloudFormation Stack""" + logger.debug(f"Deleting stack: {self.stack_name}") self.cfn.delete_stack(StackName=self.stack_name) @abstractmethod @@ -330,7 +157,7 @@ def created_resources(self): s3 = s3.Bucket(self.stack, "MyBucket") # This will create MyBucket and MyBucketArn CloudFormation Output - self._add_resource_output(name="MyBucket", value=s3.bucket_name, arn_value=bucket.bucket_arn) + self.add_cfn_output(name="MyBucket", value=s3.bucket_name, arn_value=bucket.bucket_arn) ``` Creating Lambda functions available in the handlers directory @@ -343,7 +170,9 @@ def created_resources(self): ... def _synthesize(self) -> Tuple[Dict, Path]: + logger.debug("Creating CDK Stack resources") self.create_resources() + logger.debug("Synthesizing CDK Stack into raw CloudFormation template") cloud_assembly = self.app.synth() cf_template: Dict = cloud_assembly.get_stack_by_name(self.stack_name).template cloud_assembly_assets_manifest_path: str = ( @@ -352,6 +181,7 @@ def _synthesize(self) -> Tuple[Dict, Path]: return cf_template, Path(cloud_assembly_assets_manifest_path) def _deploy_stack(self, stack_name: str, template: Dict) -> Dict[str, str]: + logger.debug(f"Creating CloudFormation Stack: {stack_name}") self.cfn.create_stack( StackName=stack_name, TemplateBody=yaml.dump(template), @@ -364,16 +194,10 @@ def _deploy_stack(self, stack_name: str, template: Dict) -> Dict[str, str]: stack_details = self.cfn.describe_stacks(StackName=stack_name) stack_outputs = stack_details["Stacks"][0]["Outputs"] - self.stack_outputs = { - output["OutputKey"]: output["OutputValue"] for output in stack_outputs if output["OutputKey"] - } - - return self.stack_outputs - - def _add_resource_output(self, name: str, value: str, arn: str): - """Add both resource value and ARN as Outputs to facilitate tests. + return {output["OutputKey"]: output["OutputValue"] for output in stack_outputs if output["OutputKey"]} - This will create two outputs: {Name} and {Name}Arn + def add_cfn_output(self, name: str, value: str, arn: str = ""): + """Create {Name} and optionally {Name}Arn CloudFormation Outputs. Parameters ---------- @@ -385,20 +209,22 @@ def _add_resource_output(self, name: str, value: str, arn: str): CloudFormation Output Value for ARN """ CfnOutput(self.stack, f"{name}", value=value) - CfnOutput(self.stack, f"{name}Arn", value=arn) + if arn: + CfnOutput(self.stack, f"{name}Arn", value=arn) def deploy_once( - stack: Type[BaseInfrastructureV2], + stack: Type[BaseInfrastructure], request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory, worker_id: str, + layer_arn: str, ) -> Generator[Dict[str, str], None, None]: """Deploys provided stack once whether CPU parallelization is enabled or not Parameters ---------- - stack : Type[BaseInfrastructureV2] + stack : Type[BaseInfrastructure] stack class to instantiate and deploy, for example MetricStack. Not to be confused with class instance (MetricStack()). request : pytest.FixtureRequest @@ -413,8 +239,8 @@ def deploy_once( Generator[Dict[str, str], None, None] stack CloudFormation outputs """ - handlers_dir = f"{request.path.parent}/handlers" - stack = stack(handlers_dir=Path(handlers_dir)) + handlers_dir = f"{request.node.path.parent}/handlers" + stack = stack(handlers_dir=Path(handlers_dir), layer_arn=layer_arn) try: if worker_id == "master": @@ -423,8 +249,8 @@ def deploy_once( else: # tmp dir shared by all workers root_tmp_dir = tmp_path_factory.getbasetemp().parent - cache = root_tmp_dir / "cache.json" + with FileLock(f"{cache}.lock"): # If cache exists, return stack outputs back # otherwise it's the first run by the main worker @@ -437,3 +263,38 @@ def deploy_once( yield stack_outputs finally: stack.delete() + + +class LambdaLayerStack(BaseInfrastructure): + FEATURE_NAME = "lambda-layer" + + def __init__(self, handlers_dir: Path, feature_name: str = FEATURE_NAME, layer_arn: str = "") -> None: + super().__init__(feature_name, handlers_dir, layer_arn) + + def create_resources(self): + layer = self._create_layer() + CfnOutput(self.stack, "LayerArn", value=layer) + + def _create_layer(self) -> str: + logger.debug("Creating Lambda Layer with latest source code available") + output_dir = Path(str(AssetStaging.BUNDLING_OUTPUT_DIR), "python") + input_dir = Path(str(AssetStaging.BUNDLING_INPUT_DIR), "aws_lambda_powertools") + + build_commands = [f"pip install .[pydantic] -t {output_dir}", f"cp -R {input_dir} {output_dir}"] + layer = LayerVersion( + self.stack, + "aws-lambda-powertools-e2e-test", + layer_version_name="aws-lambda-powertools-e2e-test", + compatible_runtimes=[PythonVersion[PYTHON_RUNTIME_VERSION].value["runtime"]], + code=Code.from_asset( + path=".", + bundling=BundlingOptions( + image=DockerImage.from_build( + str(Path(__file__).parent), + build_args={"IMAGE": PythonVersion[PYTHON_RUNTIME_VERSION].value["image"]}, + ), + command=["bash", "-c", " && ".join(build_commands)], + ), + ), + ) + return layer.layer_version_arn diff --git a/tests/utils.py b/tests/utils.py deleted file mode 100644 index e69de29bb2d..00000000000