Skip to content

chore(tests): build and deploy Lambda Layer stack once #1466

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Merged
Show file tree
Hide file tree
Changes from 12 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ unit-test:
poetry run pytest tests/unit

e2e-test:
poetry run pytest -rP -n 3 --dist loadscope --durations=0 --durations-min=1 tests/e2e
poetry run pytest -rP -n auto --dist loadfile -o log_cli=true tests/e2e

coverage-html:
poetry run pytest -m "not perf" --ignore tests/e2e --cov=aws_lambda_powertools --cov-report=html
Expand Down
97 changes: 33 additions & 64 deletions tests/e2e/conftest.py
Original file line number Diff line number Diff line change
@@ -1,66 +1,35 @@
import datetime
import sys
import uuid
from dataclasses import dataclass

import boto3

from tests.e2e.utils import data_fetcher, infrastructure

# We only need typing_extensions for python versions <3.8
if sys.version_info >= (3, 8):
from typing import TypedDict
else:
from typing_extensions import TypedDict

from typing import Dict, Generator, Optional

import pytest


class LambdaConfig(TypedDict):
parameters: dict
environment_variables: Dict[str, str]


@dataclass
class InfrastructureOutput:
arns: Dict[str, str]
execution_time: datetime.datetime

def get_lambda_arns(self) -> Dict[str, str]:
return self.arns

def get_lambda_function_arn(self, cf_output_name: str) -> Optional[str]:
return self.arns.get(cf_output_name)

def get_lambda_function_name(self, cf_output_name: str) -> Optional[str]:
lambda_arn = self.get_lambda_function_arn(cf_output_name=cf_output_name)
return lambda_arn.split(":")[-1] if lambda_arn else None

def get_lambda_execution_time(self) -> datetime.datetime:
return self.execution_time

def get_lambda_execution_time_timestamp(self) -> int:
return int(self.execution_time.timestamp() * 1000)


@pytest.fixture(scope="module")
def create_infrastructure(config, request) -> Generator[Dict[str, str], None, None]:
stack_name = f"test-lambda-{uuid.uuid4()}"
test_dir = request.fspath.dirname
handlers_dir = f"{test_dir}/handlers/"

infra = infrastructure.Infrastructure(stack_name=stack_name, handlers_dir=handlers_dir, config=config)
yield infra.deploy(Stack=infrastructure.InfrastructureStack)
infra.delete()


@pytest.fixture(scope="module")
def execute_lambda(create_infrastructure) -> InfrastructureOutput:
execution_time = datetime.datetime.utcnow()
session = boto3.Session()
client = session.client("lambda")
for _, arn in create_infrastructure.items():
data_fetcher.get_lambda_response(lambda_arn=arn, client=client)
return InfrastructureOutput(arns=create_infrastructure, execution_time=execution_time)
from tests.e2e.utils.infrastructure import LambdaLayerStack, deploy_once


@pytest.fixture(scope="session")
def lambda_layer_arn(lambda_layer_deployment):
yield lambda_layer_deployment.get("LayerArn")


@pytest.fixture(scope="session")
def lambda_layer_deployment(request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory, worker_id: str):
"""Setup and teardown logic for E2E test infrastructure

Parameters
----------
request : pytest.FixtureRequest
pytest request fixture to introspect absolute path to test being executed
tmp_path_factory : pytest.TempPathFactory
pytest temporary path factory to discover shared tmp when multiple CPU processes are spun up
worker_id : str
pytest-xdist worker identification to detect whether parallelization is enabled

Yields
------
Dict[str, str]
CloudFormation Outputs from deployed infrastructure
"""
yield from deploy_once(
stack=LambdaLayerStack,
request=request,
tmp_path_factory=tmp_path_factory,
worker_id=worker_id,
layer_arn="",
)
17 changes: 10 additions & 7 deletions tests/e2e/logger/conftest.py
Original file line number Diff line number Diff line change
@@ -1,25 +1,28 @@
from pathlib import Path

import pytest

from tests.e2e.logger.infrastructure import LoggerStack
from tests.e2e.utils.infrastructure import deploy_once


@pytest.fixture(autouse=True, scope="module")
def infrastructure(request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory, worker_id: str):
def infrastructure(request: pytest.FixtureRequest, lambda_layer_arn: str):
"""Setup and teardown logic for E2E test infrastructure

Parameters
----------
request : pytest.FixtureRequest
pytest request fixture to introspect absolute path to test being executed
tmp_path_factory : pytest.TempPathFactory
pytest temporary path factory to discover shared tmp when multiple CPU processes are spun up
worker_id : str
pytest-xdist worker identification to detect whether parallelization is enabled
lambda_layer_arn : str
Lambda Layer ARN

Yields
------
Dict[str, str]
CloudFormation Outputs from deployed infrastructure
"""
yield from deploy_once(stack=LoggerStack, request=request, tmp_path_factory=tmp_path_factory, worker_id=worker_id)
stack = LoggerStack(handlers_dir=Path(f"{request.path.parent}/handlers"), layer_arn=lambda_layer_arn)
try:
yield stack.deploy()
finally:
stack.delete()
6 changes: 4 additions & 2 deletions tests/e2e/logger/infrastructure.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@


class LoggerStack(BaseInfrastructureV2):
def __init__(self, handlers_dir: Path, feature_name: str = "logger") -> None:
super().__init__(feature_name, handlers_dir)
FEATURE_NAME = "logger"

def __init__(self, handlers_dir: Path, feature_name: str = FEATURE_NAME, layer_arn: str = "") -> None:
super().__init__(feature_name, handlers_dir, layer_arn)

def create_resources(self):
self.create_lambda_functions()
17 changes: 10 additions & 7 deletions tests/e2e/metrics/conftest.py
Original file line number Diff line number Diff line change
@@ -1,25 +1,28 @@
from pathlib import Path

import pytest

from tests.e2e.metrics.infrastructure import MetricsStack
from tests.e2e.utils.infrastructure import deploy_once


@pytest.fixture(autouse=True, scope="module")
def infrastructure(request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory, worker_id: str):
def infrastructure(request: pytest.FixtureRequest, lambda_layer_arn: str):
"""Setup and teardown logic for E2E test infrastructure

Parameters
----------
request : pytest.FixtureRequest
pytest request fixture to introspect absolute path to test being executed
tmp_path_factory : pytest.TempPathFactory
pytest temporary path factory to discover shared tmp when multiple CPU processes are spun up
worker_id : str
pytest-xdist worker identification to detect whether parallelization is enabled
lambda_layer_arn : str
Lambda Layer ARN

Yields
------
Dict[str, str]
CloudFormation Outputs from deployed infrastructure
"""
yield from deploy_once(stack=MetricsStack, request=request, tmp_path_factory=tmp_path_factory, worker_id=worker_id)
stack = MetricsStack(handlers_dir=Path(f"{request.path.parent}/handlers"), layer_arn=lambda_layer_arn)
try:
yield stack.deploy()
finally:
stack.delete()
6 changes: 4 additions & 2 deletions tests/e2e/metrics/infrastructure.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@


class MetricsStack(BaseInfrastructureV2):
def __init__(self, handlers_dir: Path, feature_name: str = "metrics") -> None:
super().__init__(feature_name, handlers_dir)
FEATURE_NAME = "metrics"

def __init__(self, handlers_dir: Path, feature_name: str = FEATURE_NAME, layer_arn: str = "") -> None:
super().__init__(feature_name, handlers_dir, layer_arn)

def create_resources(self):
self.create_lambda_functions()
17 changes: 10 additions & 7 deletions tests/e2e/tracer/conftest.py
Original file line number Diff line number Diff line change
@@ -1,25 +1,28 @@
from pathlib import Path

import pytest

from tests.e2e.tracer.infrastructure import TracerStack
from tests.e2e.utils.infrastructure import deploy_once


@pytest.fixture(autouse=True, scope="module")
def infrastructure(request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory, worker_id: str):
def infrastructure(request: pytest.FixtureRequest, lambda_layer_arn: str):
"""Setup and teardown logic for E2E test infrastructure

Parameters
----------
request : pytest.FixtureRequest
pytest request fixture to introspect absolute path to test being executed
tmp_path_factory : pytest.TempPathFactory
pytest temporary path factory to discover shared tmp when multiple CPU processes are spun up
worker_id : str
pytest-xdist worker identification to detect whether parallelization is enabled
lambda_layer_arn : str
Lambda Layer ARN

Yields
------
Dict[str, str]
CloudFormation Outputs from deployed infrastructure
"""
yield from deploy_once(stack=TracerStack, request=request, tmp_path_factory=tmp_path_factory, worker_id=worker_id)
stack = TracerStack(handlers_dir=Path(f"{request.path.parent}/handlers"), layer_arn=lambda_layer_arn)
try:
yield stack.deploy()
finally:
stack.delete()
5 changes: 3 additions & 2 deletions tests/e2e/tracer/infrastructure.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,10 @@ class TracerStack(BaseInfrastructureV2):
# Maintenance: Tracer doesn't support dynamic service injection (tracer.py L310)
# we could move after handler response or adopt env vars usage in e2e tests
SERVICE_NAME: str = build_service_name()
FEATURE_NAME = "tracer"

def __init__(self, handlers_dir: Path, feature_name: str = "tracer") -> None:
super().__init__(feature_name, handlers_dir)
def __init__(self, handlers_dir: Path, feature_name: str = FEATURE_NAME, layer_arn: str = "") -> None:
super().__init__(feature_name, handlers_dir, layer_arn)

def create_resources(self) -> None:
env_vars = {"POWERTOOLS_SERVICE_NAME": self.SERVICE_NAME}
Expand Down
6 changes: 2 additions & 4 deletions tests/e2e/utils/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Image used by CDK's LayerVersion construct to create Lambda Layer with Powertools
# library code.
# Image used by CDK's LayerVersion construct to create Lambda Layer with Powertools
# library code.
# The correct AWS SAM build image based on the runtime of the function will be
# passed as build arg. The default allows to do `docker build .` when testing.
ARG IMAGE=public.ecr.aws/sam/build-python3.7
Expand All @@ -9,8 +9,6 @@ ARG PIP_INDEX_URL
ARG PIP_EXTRA_INDEX_URL
ARG HTTPS_PROXY

# Upgrade pip (required by cryptography v3.4 and above, which is a dependency of poetry)
RUN pip install --upgrade pip
RUN pip install pipenv poetry

CMD [ "python" ]
8 changes: 4 additions & 4 deletions tests/e2e/utils/asset.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import io
import json
import logging
import zipfile
from pathlib import Path
from typing import Dict, List, Optional
Expand All @@ -9,9 +10,7 @@
from mypy_boto3_s3 import S3Client
from pydantic import BaseModel, Field

from aws_lambda_powertools import Logger

logger = Logger(service="e2e-utils")
logger = logging.getLogger(__name__)


class AssetManifest(BaseModel):
Expand Down Expand Up @@ -113,6 +112,7 @@ def upload(self):
We follow the same design cdk-assets:
https://github.com/aws/aws-cdk-rfcs/blob/master/text/0092-asset-publishing.md.
"""
logger.debug(f"Upload {len(self.assets)} assets")
for asset in self.assets:
if not asset.is_zip:
logger.debug(f"Asset '{asset.object_key}' is not zip. Skipping upload.")
Expand All @@ -138,7 +138,7 @@ def _find_assets_from_template(self) -> List[Asset]:
def _compress_assets(self, asset: Asset) -> io.BytesIO:
buf = io.BytesIO()
asset_dir = f"{self.assets_location}/{asset.asset_path}"
asset_files = list(Path(asset_dir).iterdir())
asset_files = list(Path(asset_dir).rglob("*"))
with zipfile.ZipFile(buf, "w", compression=zipfile.ZIP_DEFLATED) as archive:
for asset_file in asset_files:
logger.debug(f"Adding file '{asset_file}' to the archive.")
Expand Down
Loading