chore(ci): migrate E2E tests to CDK CLI and off Docker (aws-powertools#1501)

Co-authored-by: Rúben Fonseca <[email protected]>

Author: heitorlessa

.github/workflows/run-e2e-tests.yml

@@ -28,8 +28,8 @@ jobs:
     strategy:
       matrix:
         # Maintenance: disabled until we discover concurrency lock issue with multiple versions and tmp
-        # version: ["3.7", "3.8", "3.9"]
-        version: ["3.7"]
+        version: ["3.7", "3.8", "3.9"]
+        # version: ["3.7"]
     steps:
       - name: "Checkout"
         uses: actions/checkout@v3
@@ -41,6 +41,14 @@ jobs:
           python-version: ${{ matrix.version }}
           architecture: "x64"
           cache: "poetry"
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: "16.12"
+      - name: Install CDK CLI
+        run: |
+          npm install
+          cdk --version
       - name: Install dependencies
         run: make dev
       - name: Configure AWS credentials

.gitignore

@@ -310,3 +310,7 @@ site/
 
 !.github/workflows/lib
 examples/**/sam/.aws-sam
+
+cdk.out
+# NOTE: different accounts will be used for E2E thus creating unnecessary git clutter
+cdk.context.json

MAINTAINERS.md

@@ -300,16 +300,6 @@ def handler(event, context):
         @functools.wraps(lambda_handler)
         def decorate(event, context, **kwargs):
             with self.provider.in_subsegment(name=f"## {lambda_handler_name}") as subsegment:
-                global is_cold_start
-                logger.debug("Annotating cold start")
-                subsegment.put_annotation(key="ColdStart", value=is_cold_start)
-
-                if is_cold_start:
-                    is_cold_start = False
-
-                if self.service:
-                    subsegment.put_annotation(key="Service", value=self.service)
-
                 try:
                     logger.debug("Calling lambda handler")
                     response = lambda_handler(event, context, **kwargs)
@@ -325,7 +315,18 @@ def decorate(event, context, **kwargs):
                     self._add_full_exception_as_metadata(
                         method_name=lambda_handler_name, error=err, subsegment=subsegment, capture_error=capture_error
                     )
+
                     raise
+                finally:
+                    global is_cold_start
+                    logger.debug("Annotating cold start")
+                    subsegment.put_annotation(key="ColdStart", value=is_cold_start)
+
+                    if is_cold_start:
+                        is_cold_start = False
+
+                    if self.service:
+                        subsegment.put_annotation(key="Service", value=self.service)
 
                 return response
 
@@ -672,7 +673,7 @@ def _add_response_as_metadata(
         if data is None or not capture_response or subsegment is None:
             return
 
-        subsegment.put_metadata(key=f"{method_name} response", value=data, namespace=self._config["service"])
+        subsegment.put_metadata(key=f"{method_name} response", value=data, namespace=self.service)
 
     def _add_full_exception_as_metadata(
         self,
@@ -697,7 +698,7 @@ def _add_full_exception_as_metadata(
         if not capture_error:
             return
 
-        subsegment.put_metadata(key=f"{method_name} error", value=error, namespace=self._config["service"])
+        subsegment.put_metadata(key=f"{method_name} error", value=error, namespace=self.service)
 
     @staticmethod
     def _disable_tracer_provider():

aws_lambda_powertools/tracing/tracer.py

@@ -0,0 +1,7 @@
+{
+  "name": "aws-lambda-powertools-python-e2e",
+  "version": "1.0.0",
+  "devDependencies": {
+	"aws-cdk": "2.40.0"
+  }
+}

package-lock.json

@@ -8,7 +8,6 @@ def main():
     workers = len(list(features)) - 1
 
     command = f"poetry run pytest -n {workers} --dist loadfile -o log_cli=true tests/e2e"
-    print(f"Running E2E tests with: {command}")
     subprocess.run(command.split(), shell=False)
 
 

package.json

@@ -50,7 +50,7 @@ xenon = "^0.9.0"
 flake8-eradicate = "^1.2.1"
 flake8-bugbear = "^22.9.23"
 mkdocs-git-revision-date-plugin = "^0.3.2"
-mike = "^0.6.0"
+mike = "^1.1.2"
 mypy = "^0.971"
 retry = "^0.9.2"
 pytest-xdist = "^2.5.0"
@@ -73,6 +73,7 @@ types-requests = "^2.28.11"
 typing-extensions = "^4.4.0"
 mkdocs-material = "^8.5.4"
 filelock = "^3.8.0"
+checksumdir = "^1.2.0"
 
 [tool.poetry.extras]
 pydantic = ["pydantic", "email-validator"]

parallel_run_e2e.py

@@ -1,35 +1,29 @@
 import pytest
 
-from tests.e2e.utils.infrastructure import LambdaLayerStack, deploy_once
+from tests.e2e.utils.infrastructure import call_once
+from tests.e2e.utils.lambda_layer.powertools_layer import LocalLambdaPowertoolsLayer
 
 
-@pytest.fixture(scope="session")
-def lambda_layer_arn(lambda_layer_deployment):
-    yield lambda_layer_deployment.get("LayerArn")
-
-
-@pytest.fixture(scope="session")
-def lambda_layer_deployment(request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory, worker_id: str):
-    """Setup and teardown logic for E2E test infrastructure
+@pytest.fixture(scope="session", autouse=True)
+def lambda_layer_build(tmp_path_factory: pytest.TempPathFactory, worker_id: str) -> str:
+    """Build Lambda Layer once before stacks are created
 
     Parameters
     ----------
-    request : pytest.FixtureRequest
-        pytest request fixture to introspect absolute path to test being executed
     tmp_path_factory : pytest.TempPathFactory
         pytest temporary path factory to discover shared tmp when multiple CPU processes are spun up
     worker_id : str
         pytest-xdist worker identification to detect whether parallelization is enabled
 
     Yields
     ------
-    Dict[str, str]
-        CloudFormation Outputs from deployed infrastructure
+    str
+        Lambda Layer artefact location
     """
-    yield from deploy_once(
-        stack=LambdaLayerStack,
-        request=request,
+
+    layer = LocalLambdaPowertoolsLayer()
+    yield from call_once(
+        task=layer.build,
         tmp_path_factory=tmp_path_factory,
         worker_id=worker_id,
-        layer_arn="",
     )

poetry.lock

@@ -1,27 +1,18 @@
-from pathlib import Path
-
 import pytest
 
 from tests.e2e.event_handler.infrastructure import EventHandlerStack
 
 
 @pytest.fixture(autouse=True, scope="module")
-def infrastructure(request: pytest.FixtureRequest, lambda_layer_arn: str):
+def infrastructure():
     """Setup and teardown logic for E2E test infrastructure
 
-    Parameters
-    ----------
-    request : pytest.FixtureRequest
-        pytest request fixture to introspect absolute path to test being executed
-    lambda_layer_arn : str
-        Lambda Layer ARN
-
     Yields
     ------
     Dict[str, str]
         CloudFormation Outputs from deployed infrastructure
     """
-    stack = EventHandlerStack(handlers_dir=Path(f"{request.path.parent}/handlers"), layer_arn=lambda_layer_arn)
+    stack = EventHandlerStack()
     try:
         yield stack.deploy()
     finally:

pyproject.toml

@@ -1,4 +1,3 @@
-from pathlib import Path
 from typing import Dict, Optional
 
 from aws_cdk import CfnOutput
@@ -14,11 +13,6 @@
 
 
 class EventHandlerStack(BaseInfrastructure):
-    FEATURE_NAME = "event-handlers"
-
-    def __init__(self, handlers_dir: Path, feature_name: str = FEATURE_NAME, layer_arn: str = "") -> None:
-        super().__init__(feature_name, handlers_dir, layer_arn)
-
     def create_resources(self):
         functions = self.create_lambda_functions()
 
@@ -28,7 +22,12 @@ def create_resources(self):
         self._create_lambda_function_url(function=functions["LambdaFunctionUrlHandler"])
 
     def _create_alb(self, function: Function):
-        vpc = ec2.Vpc(self.stack, "EventHandlerVPC", max_azs=2)
+        vpc = ec2.Vpc.from_lookup(
+            self.stack,
+            "VPC",
+            is_default=True,
+            region=self.region,
+        )
 
         alb = elbv2.ApplicationLoadBalancer(self.stack, "ALB", vpc=vpc, internet_facing=True)
         CfnOutput(self.stack, "ALBDnsName", value=alb.load_balancer_dns_name)

tests/e2e/conftest.py

@@ -1,27 +1,18 @@
-from pathlib import Path
-
 import pytest
 
 from tests.e2e.logger.infrastructure import LoggerStack
 
 
 @pytest.fixture(autouse=True, scope="module")
-def infrastructure(request: pytest.FixtureRequest, lambda_layer_arn: str):
+def infrastructure(tmp_path_factory, worker_id):
     """Setup and teardown logic for E2E test infrastructure
 
-    Parameters
-    ----------
-    request : pytest.FixtureRequest
-        pytest request fixture to introspect absolute path to test being executed
-    lambda_layer_arn : str
-        Lambda Layer ARN
-
     Yields
     ------
     Dict[str, str]
         CloudFormation Outputs from deployed infrastructure
     """
-    stack = LoggerStack(handlers_dir=Path(f"{request.path.parent}/handlers"), layer_arn=lambda_layer_arn)
+    stack = LoggerStack()
     try:
         yield stack.deploy()
     finally:

tests/e2e/event_handler/conftest.py

@@ -1,13 +1,6 @@
-from pathlib import Path
-
 from tests.e2e.utils.infrastructure import BaseInfrastructure
 
 
 class LoggerStack(BaseInfrastructure):
-    FEATURE_NAME = "logger"
-
-    def __init__(self, handlers_dir: Path, feature_name: str = FEATURE_NAME, layer_arn: str = "") -> None:
-        super().__init__(feature_name, handlers_dir, layer_arn)
-
     def create_resources(self):
         self.create_lambda_functions()

tests/e2e/event_handler/infrastructure.py

@@ -1,27 +1,18 @@
-from pathlib import Path
-
 import pytest
 
 from tests.e2e.metrics.infrastructure import MetricsStack
 
 
 @pytest.fixture(autouse=True, scope="module")
-def infrastructure(request: pytest.FixtureRequest, lambda_layer_arn: str):
+def infrastructure(tmp_path_factory, worker_id):
     """Setup and teardown logic for E2E test infrastructure
 
-    Parameters
-    ----------
-    request : pytest.FixtureRequest
-        pytest request fixture to introspect absolute path to test being executed
-    lambda_layer_arn : str
-        Lambda Layer ARN
-
     Yields
     ------
     Dict[str, str]
         CloudFormation Outputs from deployed infrastructure
     """
-    stack = MetricsStack(handlers_dir=Path(f"{request.path.parent}/handlers"), layer_arn=lambda_layer_arn)
+    stack = MetricsStack()
     try:
         yield stack.deploy()
     finally:

tests/e2e/logger/conftest.py

@@ -1,13 +1,6 @@
-from pathlib import Path
-
 from tests.e2e.utils.infrastructure import BaseInfrastructure
 
 
 class MetricsStack(BaseInfrastructure):
-    FEATURE_NAME = "metrics"
-
-    def __init__(self, handlers_dir: Path, feature_name: str = FEATURE_NAME, layer_arn: str = "") -> None:
-        super().__init__(feature_name, handlers_dir, layer_arn)
-
     def create_resources(self):
         self.create_lambda_functions()

tests/e2e/logger/infrastructure.py

@@ -1,27 +1,19 @@
-from pathlib import Path
-
 import pytest
 
 from tests.e2e.tracer.infrastructure import TracerStack
 
 
 @pytest.fixture(autouse=True, scope="module")
-def infrastructure(request: pytest.FixtureRequest, lambda_layer_arn: str):
+def infrastructure():
     """Setup and teardown logic for E2E test infrastructure
 
-    Parameters
-    ----------
-    request : pytest.FixtureRequest
-        pytest request fixture to introspect absolute path to test being executed
-    lambda_layer_arn : str
-        Lambda Layer ARN
 
     Yields
     ------
     Dict[str, str]
         CloudFormation Outputs from deployed infrastructure
     """
-    stack = TracerStack(handlers_dir=Path(f"{request.path.parent}/handlers"), layer_arn=lambda_layer_arn)
+    stack = TracerStack()
     try:
         yield stack.deploy()
     finally:

tests/e2e/metrics/conftest.py

@@ -13,4 +13,5 @@ async def async_get_users():
 
 
 def lambda_handler(event: dict, context: LambdaContext):
+    tracer.service = event.get("service")
     return asyncio.run(async_get_users())

tests/e2e/metrics/infrastructure.py

@@ -13,4 +13,5 @@ def get_todos():
 
 @tracer.capture_lambda_handler
 def lambda_handler(event: dict, context: LambdaContext):
+    tracer.service = event.get("service")
     return get_todos()

tests/e2e/tracer/conftest.py

@@ -26,6 +26,8 @@ def get_all(self):
 
 
 def lambda_handler(event: dict, context: LambdaContext):
+    # Maintenance: create a public method to set these explicitly
+    tracer.service = event["service"]
 
     todos = Todos()
     comments = Comments()

tests/e2e/tracer/handlers/async_capture.py

@@ -1,18 +1,6 @@
-from pathlib import Path
-
-from tests.e2e.utils.data_builder import build_service_name
 from tests.e2e.utils.infrastructure import BaseInfrastructure
 
 
 class TracerStack(BaseInfrastructure):
-    # Maintenance: Tracer doesn't support dynamic service injection (tracer.py L310)
-    # we could move after handler response or adopt env vars usage in e2e tests
-    SERVICE_NAME: str = build_service_name()
-    FEATURE_NAME = "tracer"
-
-    def __init__(self, handlers_dir: Path, feature_name: str = FEATURE_NAME, layer_arn: str = "") -> None:
-        super().__init__(feature_name, handlers_dir, layer_arn)
-
     def create_resources(self) -> None:
-        env_vars = {"POWERTOOLS_SERVICE_NAME": self.SERVICE_NAME}
-        self.create_lambda_functions(function_props={"environment": env_vars})
+        self.create_lambda_functions()

tests/e2e/tracer/handlers/basic_handler.py

@@ -1,7 +1,8 @@
+import json
+
 import pytest
 
 from tests.e2e.tracer.handlers import async_capture, basic_handler
-from tests.e2e.tracer.infrastructure import TracerStack
 from tests.e2e.utils import data_builder, data_fetcher
 
 
@@ -37,6 +38,7 @@ def async_fn(infrastructure: dict) -> str:
 
 def test_lambda_handler_trace_is_visible(basic_handler_fn_arn: str, basic_handler_fn: str):
     # GIVEN
+    service = data_builder.build_service_name()
     handler_name = basic_handler.lambda_handler.__name__
     handler_subsegment = f"## {handler_name}"
     handler_metadata_key = f"{handler_name} response"
@@ -48,21 +50,23 @@ def test_lambda_handler_trace_is_visible(basic_handler_fn_arn: str, basic_handle
     trace_query = data_builder.build_trace_default_query(function_name=basic_handler_fn)
 
     # WHEN
-    _, execution_time = data_fetcher.get_lambda_response(lambda_arn=basic_handler_fn_arn)
-    data_fetcher.get_lambda_response(lambda_arn=basic_handler_fn_arn)
+    event = json.dumps({"service": service})
+    _, execution_time = data_fetcher.get_lambda_response(lambda_arn=basic_handler_fn_arn, payload=event)
+    data_fetcher.get_lambda_response(lambda_arn=basic_handler_fn_arn, payload=event)
 
     # THEN
     trace = data_fetcher.get_traces(start_date=execution_time, filter_expression=trace_query, minimum_traces=2)
 
     assert len(trace.get_annotation(key="ColdStart", value=True)) == 1
-    assert len(trace.get_metadata(key=handler_metadata_key, namespace=TracerStack.SERVICE_NAME)) == 2
-    assert len(trace.get_metadata(key=method_metadata_key, namespace=TracerStack.SERVICE_NAME)) == 2
+    assert len(trace.get_metadata(key=handler_metadata_key, namespace=service)) == 2
+    assert len(trace.get_metadata(key=method_metadata_key, namespace=service)) == 2
     assert len(trace.get_subsegment(name=handler_subsegment)) == 2
     assert len(trace.get_subsegment(name=method_subsegment)) == 2
 
 
 def test_lambda_handler_trace_multiple_functions_same_name(same_function_name_arn: str, same_function_name_fn: str):
     # GIVEN
+    service = data_builder.build_service_name()
     method_name_todos = "same_function_name.Todos.get_all"
     method_subsegment_todos = f"## {method_name_todos}"
     method_metadata_key_todos = f"{method_name_todos} response"
@@ -74,30 +78,33 @@ def test_lambda_handler_trace_multiple_functions_same_name(same_function_name_ar
     trace_query = data_builder.build_trace_default_query(function_name=same_function_name_fn)
 
     # WHEN
-    _, execution_time = data_fetcher.get_lambda_response(lambda_arn=same_function_name_arn)
+    event = json.dumps({"service": service})
+    _, execution_time = data_fetcher.get_lambda_response(lambda_arn=same_function_name_arn, payload=event)
 
     # THEN
     trace = data_fetcher.get_traces(start_date=execution_time, filter_expression=trace_query)
 
-    assert len(trace.get_metadata(key=method_metadata_key_todos, namespace=TracerStack.SERVICE_NAME)) == 1
-    assert len(trace.get_metadata(key=method_metadata_key_comments, namespace=TracerStack.SERVICE_NAME)) == 1
+    assert len(trace.get_metadata(key=method_metadata_key_todos, namespace=service)) == 1
+    assert len(trace.get_metadata(key=method_metadata_key_comments, namespace=service)) == 1
     assert len(trace.get_subsegment(name=method_subsegment_todos)) == 1
     assert len(trace.get_subsegment(name=method_subsegment_comments)) == 1
 
 
 def test_async_trace_is_visible(async_fn_arn: str, async_fn: str):
     # GIVEN
+    service = data_builder.build_service_name()
     async_fn_name = f"async_capture.{async_capture.async_get_users.__name__}"
     async_fn_name_subsegment = f"## {async_fn_name}"
     async_fn_name_metadata_key = f"{async_fn_name} response"
 
     trace_query = data_builder.build_trace_default_query(function_name=async_fn)
 
     # WHEN
-    _, execution_time = data_fetcher.get_lambda_response(lambda_arn=async_fn_arn)
+    event = json.dumps({"service": service})
+    _, execution_time = data_fetcher.get_lambda_response(lambda_arn=async_fn_arn, payload=event)
 
     # THEN
     trace = data_fetcher.get_traces(start_date=execution_time, filter_expression=trace_query)
 
     assert len(trace.get_subsegment(name=async_fn_name_subsegment)) == 1
-    assert len(trace.get_metadata(key=async_fn_name_metadata_key, namespace=TracerStack.SERVICE_NAME)) == 1
+    assert len(trace.get_metadata(key=async_fn_name_metadata_key, namespace=service)) == 1

tests/e2e/tracer/handlers/same_function_name.py

@@ -0,0 +1,20 @@
+from abc import ABC, abstractmethod
+from typing import Dict, Optional
+
+
+class InfrastructureProvider(ABC):
+    @abstractmethod
+    def create_lambda_functions(self, function_props: Optional[Dict] = None) -> Dict:
+        pass
+
+    @abstractmethod
+    def deploy(self) -> Dict[str, str]:
+        pass
+
+    @abstractmethod
+    def delete(self):
+        pass
+
+    @abstractmethod
+    def create_resources(self):
+        pass

tests/e2e/tracer/infrastructure.py

@@ -0,0 +1,8 @@
+import sys
+
+from aws_lambda_powertools import PACKAGE_PATH
+
+PYTHON_RUNTIME_VERSION = f"V{''.join(map(str, sys.version_info[:2]))}"
+SOURCE_CODE_ROOT_PATH = PACKAGE_PATH.parent
+CDK_OUT_PATH = SOURCE_CODE_ROOT_PATH / "cdk.out"
+LAYER_BUILD_PATH = CDK_OUT_PATH / "layer_build"

tests/e2e/tracer/test_tracer.py

@@ -0,0 +1,32 @@
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+
+class BaseLocalLambdaLayer(ABC):
+    def __init__(self, output_dir: Path):
+        self.output_dir = output_dir / "layer_build"
+        self.target_dir = f"{self.output_dir}/python"
+
+    @abstractmethod
+    def build(self) -> str:
+        """Builds a Lambda Layer locally
+
+        Returns
+        -------
+        build_path : str
+            Path where newly built Lambda Layer is
+        """
+        raise NotImplementedError()
+
+    def before_build(self):
+        """Any step to run before build process begins.
+
+        By default, it creates output dir and its parents if it doesn't exist.
+        """
+        if not self.output_dir.exists():
+            # Create missing parent directories if missing
+            self.output_dir.mkdir(parents=True, exist_ok=True)
+
+    def after_build(self):
+        """Any step after a build succeed"""
+        ...

tests/e2e/utils/Dockerfile

@@ -0,0 +1,48 @@
+import logging
+import subprocess
+from pathlib import Path
+
+from checksumdir import dirhash
+
+from aws_lambda_powertools import PACKAGE_PATH
+from tests.e2e.utils.constants import CDK_OUT_PATH, SOURCE_CODE_ROOT_PATH
+from tests.e2e.utils.lambda_layer.base import BaseLocalLambdaLayer
+
+logger = logging.getLogger(__name__)
+
+
+class LocalLambdaPowertoolsLayer(BaseLocalLambdaLayer):
+    IGNORE_EXTENSIONS = ["pyc"]
+
+    def __init__(self, output_dir: Path = CDK_OUT_PATH):
+        super().__init__(output_dir)
+        self.package = f"{SOURCE_CODE_ROOT_PATH}[pydantic]"
+        self.build_args = "--platform manylinux1_x86_64 --only-binary=:all: --upgrade"
+        self.build_command = f"python -m pip install {self.package} {self.build_args} --target {self.target_dir}"
+        self.source_diff_file: Path = CDK_OUT_PATH / "layer_build.diff"
+
+    def build(self) -> str:
+        self.before_build()
+
+        if self._has_source_changed():
+            subprocess.run(self.build_command, shell=True)
+
+        self.after_build()
+
+        return str(self.output_dir)
+
+    def _has_source_changed(self) -> bool:
+        """Hashes source code and
+
+        Returns
+        -------
+        change : bool
+            Whether source code hash has changed
+        """
+        diff = self.source_diff_file.read_text() if self.source_diff_file.exists() else ""
+        new_diff = dirhash(dirname=PACKAGE_PATH, excluded_extensions=self.IGNORE_EXTENSIONS)
+        if new_diff != diff or not self.output_dir.exists():
+            self.source_diff_file.write_text(new_diff)
+            return True
+
+        return False