chore: cleanup leftover code; add debug

Author: heitorlessa

tests/e2e/infrastructure.py

@@ -4,7 +4,9 @@
 
 
 class LoggerStack(BaseInfrastructureV2):
-    def __init__(self, handlers_dir: Path, feature_name: str = "logger", layer_arn: str = "") -> None:
+    FEATURE_NAME = "logger"
+
+    def __init__(self, handlers_dir: Path, feature_name: str = FEATURE_NAME, layer_arn: str = "") -> None:
         super().__init__(feature_name, handlers_dir, layer_arn)
 
     def create_resources(self):

tests/e2e/logger/infrastructure.py

@@ -4,7 +4,9 @@
 
 
 class MetricsStack(BaseInfrastructureV2):
-    def __init__(self, handlers_dir: Path, feature_name: str = "metrics", layer_arn: str = "") -> None:
+    FEATURE_NAME = "metrics"
+
+    def __init__(self, handlers_dir: Path, feature_name: str = FEATURE_NAME, layer_arn: str = "") -> None:
         super().__init__(feature_name, handlers_dir, layer_arn)
 
     def create_resources(self):

tests/e2e/metrics/infrastructure.py

@@ -8,8 +8,9 @@ class TracerStack(BaseInfrastructureV2):
     # Maintenance: Tracer doesn't support dynamic service injection (tracer.py L310)
     # we could move after handler response or adopt env vars usage in e2e tests
     SERVICE_NAME: str = build_service_name()
+    FEATURE_NAME = "tracer"
 
-    def __init__(self, handlers_dir: Path, feature_name: str = "tracer", layer_arn: str = "") -> None:
+    def __init__(self, handlers_dir: Path, feature_name: str = FEATURE_NAME, layer_arn: str = "") -> None:
         super().__init__(feature_name, handlers_dir, layer_arn)
 
     def create_resources(self) -> None:

tests/e2e/tracer/infrastructure.py

@@ -1,5 +1,6 @@
 import io
 import json
+import logging
 import zipfile
 from pathlib import Path
 from typing import Dict, List, Optional
@@ -9,9 +10,7 @@
 from mypy_boto3_s3 import S3Client
 from pydantic import BaseModel, Field
 
-from aws_lambda_powertools import Logger
-
-logger = Logger(service="e2e-utils")
+logger = logging.getLogger(__name__)
 
 
 class AssetManifest(BaseModel):
@@ -113,6 +112,7 @@ def upload(self):
         We follow the same design cdk-assets:
         https://github.com/aws/aws-cdk-rfcs/blob/master/text/0092-asset-publishing.md.
         """
+        logger.debug(f"Upload {len(self.assets)} assets")
         for asset in self.assets:
             if not asset.is_zip:
                 logger.debug(f"Asset '{asset.object_key}' is not zip. Skipping upload.")

tests/e2e/utils/asset.py

@@ -1,4 +1,5 @@
 import json
+import logging
 import sys
 from abc import ABC, abstractmethod
 from enum import Enum
@@ -18,6 +19,8 @@
 
 PYTHON_RUNTIME_VERSION = f"V{''.join(map(str, sys.version_info[:2]))}"
 
+logger = logging.getLogger(__name__)
+
 
 class BaseInfrastructureStack(ABC):
     @abstractmethod
@@ -36,19 +39,19 @@ class PythonVersion(Enum):
 
 
 class BaseInfrastructureV2(ABC):
-    STACKS_OUTPUT: dict = {}
-
     def __init__(self, feature_name: str, handlers_dir: Path, layer_arn: str = "") -> None:
         self.feature_name = feature_name
         self.stack_name = f"test-{feature_name}-{uuid4()}"
         self.handlers_dir = handlers_dir
+        self.layer_arn = layer_arn
         self.stack_outputs: Dict[str, str] = {}
+
+        # NOTE: Investigate why cdk.Environment in Stack
+        # changes synthesized asset (no object_key in asset manifest)
         self.app = App()
-        # NOTE: Investigate why env changes synthesized asset (no object_key in asset manifest)
         self.stack = Stack(self.app, self.stack_name)
         self.session = boto3.Session()
         self.cfn: CloudFormationClient = self.session.client("cloudformation")
-        self.layer_arn = layer_arn
 
         # NOTE: CDK stack account and region are tokens, we need to resolve earlier
         self.account_id = self.session.client("sts").get_caller_identity()["Account"]
@@ -86,6 +89,7 @@ def create_lambda_functions(self, function_props: Optional[Dict] = None):
         handlers = list(self.handlers_dir.rglob("*.py"))
         source = Code.from_asset(f"{self.handlers_dir}")
         props_override = function_props or {}
+        logger.debug(f"Creating functions for handlers: {handlers}")
         if not self.layer_arn:
             raise ValueError(
                 """Lambda Layer ARN cannot be empty when creating Lambda functions.
@@ -96,6 +100,8 @@ def create_lambda_functions(self, function_props: Optional[Dict] = None):
 
         for fn in handlers:
             fn_name = fn.stem
+            fn_name_pascal_case = fn_name.title().replace("_", "")  # basic_handler -> BasicHandler
+            logger.debug(f"Creating function: {fn_name_pascal_case}")
             function_settings = {
                 "id": f"{fn_name}-lambda",
                 "code": source,
@@ -116,9 +122,8 @@ def create_lambda_functions(self, function_props: Optional[Dict] = None):
                 removal_policy=RemovalPolicy.DESTROY,
             )
 
-            # CFN Outputs only support hyphen
-            fn_name_pascal_case = fn_name.title().replace("_", "")  # basic_handler -> BasicHandler
-            self._add_resource_output(
+            # CFN Outputs only support hyphen hence pascal case
+            self.add_cfn_output(
                 name=fn_name_pascal_case, value=function_python.function_name, arn=function_python.function_arn
             )
 
@@ -133,14 +138,12 @@ def deploy(self) -> Dict[str, str]:
         template, asset_manifest_file = self._synthesize()
         assets = Assets(asset_manifest=asset_manifest_file, account_id=self.account_id, region=self.region)
         assets.upload()
-        outputs = self._deploy_stack(self.stack_name, template)
-        # NOTE: hydrate map of stack resolved outputs for future access
-        self.STACKS_OUTPUT[self.feature_name] = outputs
-
-        return outputs
+        self.stack_outputs = self._deploy_stack(self.stack_name, template)
+        return self.stack_outputs
 
     def delete(self) -> None:
         """Delete CloudFormation Stack"""
+        logger.debug(f"Deleting stack: {self.stack_name}")
         self.cfn.delete_stack(StackName=self.stack_name)
 
     @abstractmethod
@@ -157,7 +160,7 @@ def created_resources(self):
             s3 = s3.Bucket(self.stack, "MyBucket")
 
             # This will create MyBucket and MyBucketArn CloudFormation Output
-            self._add_resource_output(name="MyBucket", value=s3.bucket_name, arn_value=bucket.bucket_arn)
+            self.add_cfn_output(name="MyBucket", value=s3.bucket_name, arn_value=bucket.bucket_arn)
         ```
 
         Creating Lambda functions available in the handlers directory
@@ -170,7 +173,9 @@ def created_resources(self):
         ...
 
     def _synthesize(self) -> Tuple[Dict, Path]:
+        logger.debug("Creating CDK Stack resources")
         self.create_resources()
+        logger.debug("Synthesizing CDK Stack into raw CloudFormation template")
         cloud_assembly = self.app.synth()
         cf_template: Dict = cloud_assembly.get_stack_by_name(self.stack_name).template
         cloud_assembly_assets_manifest_path: str = (
@@ -179,6 +184,7 @@ def _synthesize(self) -> Tuple[Dict, Path]:
         return cf_template, Path(cloud_assembly_assets_manifest_path)
 
     def _deploy_stack(self, stack_name: str, template: Dict) -> Dict[str, str]:
+        logger.debug(f"Creating CloudFormation Stack: {stack_name}")
         self.cfn.create_stack(
             StackName=stack_name,
             TemplateBody=yaml.dump(template),
@@ -191,16 +197,10 @@ def _deploy_stack(self, stack_name: str, template: Dict) -> Dict[str, str]:
 
         stack_details = self.cfn.describe_stacks(StackName=stack_name)
         stack_outputs = stack_details["Stacks"][0]["Outputs"]
-        self.stack_outputs = {
-            output["OutputKey"]: output["OutputValue"] for output in stack_outputs if output["OutputKey"]
-        }
-
-        return self.stack_outputs
-
-    def _add_resource_output(self, name: str, value: str, arn: str):
-        """Add both resource value and ARN as Outputs to facilitate tests.
+        return {output["OutputKey"]: output["OutputValue"] for output in stack_outputs if output["OutputKey"]}
 
-        This will create two outputs: {Name} and {Name}Arn
+    def add_cfn_output(self, name: str, value: str, arn: str = ""):
+        """Create {Name} and optionally {Name}Arn CloudFormation Outputs.
 
         Parameters
         ----------
@@ -212,7 +212,8 @@ def _add_resource_output(self, name: str, value: str, arn: str):
             CloudFormation Output Value for ARN
         """
         CfnOutput(self.stack, f"{name}", value=value)
-        CfnOutput(self.stack, f"{name}Arn", value=arn)
+        if arn:
+            CfnOutput(self.stack, f"{name}Arn", value=arn)
 
 
 def deploy_once(
@@ -241,13 +242,7 @@ def deploy_once(
     Generator[Dict[str, str], None, None]
         stack CloudFormation outputs
     """
-    try:
-        handlers_dir = f"{request.path.parent}/handlers"
-    except AttributeError:
-        # session fixture has a slightly different object
-        # luckily it only runs Lambda Layer Stack which doesn't deploy Lambda fns
-        handlers_dir = f"{request.node.path.parent}/handlers"
-
+    handlers_dir = f"{request.node.path.parent}/handlers"
     stack = stack(handlers_dir=Path(handlers_dir), layer_arn=layer_arn)
 
     try:
@@ -257,10 +252,6 @@ def deploy_once(
         else:
             # tmp dir shared by all workers
             root_tmp_dir = tmp_path_factory.getbasetemp().parent
-
-            # cache and lock must be unique per stack
-            # otherwise separate processes deploy the first stack collected only
-            # since the original lock was based on parallel workers cache tmp dir
             cache = root_tmp_dir / "cache.json"
 
             with FileLock(f"{cache}.lock"):
@@ -288,6 +279,7 @@ def create_resources(self):
         CfnOutput(self.stack, "LayerArn", value=layer)
 
     def _create_layer(self) -> str:
+        logger.debug("Creating Lambda Layer with latest source code available")
         output_dir = Path(str(AssetStaging.BUNDLING_OUTPUT_DIR), "python")
         input_dir = Path(str(AssetStaging.BUNDLING_INPUT_DIR), "aws_lambda_powertools")
         build_commands = [f"pip install . -t {output_dir}", f"cp -R {input_dir} {output_dir}"]