refactor: simplify custom formatter for minor changes (#417)

Author: heitorlessa

aws_lambda_powertools/logging/formatter.py

@@ -69,7 +69,7 @@ def __init__(
 
         The `log_record_order` kwarg is used to specify the order of the keys used in
         the structured json logs. By default the order is: "level", "location", "message", "timestamp",
-        "service" and "sampling_rate".
+        "service".
 
         Other kwargs are used to specify log field format strings.
 
@@ -113,6 +113,10 @@ def __init__(
         keys_combined = {**self._build_default_keys(), **kwargs}
         self.log_format.update(**keys_combined)
 
+    def serialize(self, log: Dict) -> str:
+        """Serialize structured log dict to JSON str"""
+        return self.json_serializer(log)
+
     def format(self, record: logging.LogRecord) -> str:  # noqa: A003
         """Format logging record as structured JSON str"""
         formatted_log = self._extract_log_keys(log_record=record)
@@ -121,7 +125,7 @@ def format(self, record: logging.LogRecord) -> str:  # noqa: A003
         formatted_log["xray_trace_id"] = self._get_latest_trace_id()
         formatted_log = self._strip_none_records(records=formatted_log)
 
-        return self.json_serializer(formatted_log)
+        return self.serialize(log=formatted_log)
 
     def formatTime(self, record: logging.LogRecord, datefmt: Optional[str] = None) -> str:
         record_ts = self.converter(record.created)

aws_lambda_powertools/logging/logger.py

@@ -4,7 +4,7 @@
 import os
 import random
 import sys
-from typing import Any, Callable, Dict, Iterable, Optional, Union
+from typing import Any, Callable, Dict, Iterable, Optional, TypeVar, Union
 
 import jmespath
 
@@ -19,6 +19,8 @@
 
 is_cold_start = True
 
+PowertoolsFormatter = TypeVar("PowertoolsFormatter", bound=BasePowertoolsFormatter)
+
 
 def _is_cold_start() -> bool:
     """Verifies whether is cold start
@@ -70,8 +72,8 @@ class Logger(logging.Logger):  # lgtm [py/missing-call-to-init]
         sample rate for debug calls within execution context defaults to 0.0
     stream: sys.stdout, optional
         valid output for a logging stream, by default sys.stdout
-    logger_formatter: BasePowertoolsFormatter, optional
-        custom logging formatter that implements BasePowertoolsFormatter
+    logger_formatter: PowertoolsFormatter, optional
+        custom logging formatter that implements PowertoolsFormatter
     logger_handler: logging.Handler, optional
         custom logging handler e.g. logging.FileHandler("file.log")
 
@@ -87,7 +89,7 @@ class Logger(logging.Logger):  # lgtm [py/missing-call-to-init]
     json_default : Callable, optional
         function to coerce unserializable values, by default `str()`
 
-        Only used when no custom JSON encoder is set
+        Only used when no custom formatter is set
     utc : bool, optional
         set logging timestamp to UTC, by default False to continue to use local time as per stdlib
     log_record_order : list, optional
@@ -170,7 +172,7 @@ def __init__(
         child: bool = False,
         sampling_rate: float = None,
         stream: sys.stdout = None,
-        logger_formatter: Optional[BasePowertoolsFormatter] = None,
+        logger_formatter: Optional[PowertoolsFormatter] = None,
         logger_handler: Optional[logging.Handler] = None,
         **kwargs,
     ):
@@ -198,7 +200,7 @@ def __getattr__(self, name):
         return getattr(self._logger, name)
 
     def _get_logger(self):
-        """ Returns a Logger named {self.service}, or {self.service.filename} for child loggers"""
+        """Returns a Logger named {self.service}, or {self.service.filename} for child loggers"""
         logger_name = self.service
         if self.child:
             logger_name = f"{self.service}.{self._get_caller_filename()}"
@@ -346,7 +348,7 @@ def registered_handler(self) -> logging.Handler:
         return handlers[0]
 
     @property
-    def registered_formatter(self) -> Optional[BasePowertoolsFormatter]:
+    def registered_formatter(self) -> Optional[PowertoolsFormatter]:
         """Convenience property to access logger formatter"""
         return self.registered_handler.formatter
 
@@ -384,7 +386,7 @@ def set_correlation_id(self, value: str):
 
     @staticmethod
     def _get_log_level(level: Union[str, int, None]) -> Union[str, int]:
-        """ Returns preferred log level set by the customer in upper case """
+        """Returns preferred log level set by the customer in upper case"""
         if isinstance(level, int):
             return level
 
@@ -396,7 +398,7 @@ def _get_log_level(level: Union[str, int, None]) -> Union[str, int]:
 
     @staticmethod
     def _get_caller_filename():
-        """ Return caller filename by finding the caller frame """
+        """Return caller filename by finding the caller frame"""
         # Current frame         => _get_logger()
         # Previous frame        => logger.py
         # Before previous frame => Caller

aws_lambda_powertools/tracing/tracer.py

@@ -720,7 +720,7 @@ def __build_config(
         patch_modules: Union[List, Tuple] = None,
         provider: BaseProvider = None,
     ):
-        """ Populates Tracer config for new and existing initializations """
+        """Populates Tracer config for new and existing initializations"""
         is_disabled = disabled if disabled is not None else self._is_tracer_disabled()
         is_service = resolve_env_var_choice(choice=service, env=os.getenv(constants.SERVICE_NAME_ENV))
 

aws_lambda_powertools/utilities/data_classes/api_gateway_proxy_event.py

@@ -164,7 +164,7 @@ def path(self) -> str:
 
     @property
     def stage(self) -> str:
-        """The deployment stage of the API request """
+        """The deployment stage of the API request"""
         return self["requestContext"]["stage"]
 
     @property
@@ -352,7 +352,7 @@ def authorizer(self) -> Optional[RequestContextV2Authorizer]:
 
     @property
     def domain_name(self) -> str:
-        """A domain name """
+        """A domain name"""
         return self["requestContext"]["domainName"]
 
     @property
@@ -375,7 +375,7 @@ def route_key(self) -> str:
 
     @property
     def stage(self) -> str:
-        """The deployment stage of the API request """
+        """The deployment stage of the API request"""
         return self["requestContext"]["stage"]
 
     @property

aws_lambda_powertools/utilities/data_classes/appsync_resolver_event.py

@@ -22,7 +22,7 @@ class AppSyncIdentityIAM(DictWrapper):
 
     @property
     def source_ip(self) -> List[str]:
-        """The source IP address of the caller received by AWS AppSync. """
+        """The source IP address of the caller received by AWS AppSync."""
         return self["sourceIp"]
 
     @property
@@ -67,7 +67,7 @@ class AppSyncIdentityCognito(DictWrapper):
 
     @property
     def source_ip(self) -> List[str]:
-        """The source IP address of the caller received by AWS AppSync. """
+        """The source IP address of the caller received by AWS AppSync."""
         return self["sourceIp"]
 
     @property

aws_lambda_powertools/utilities/data_classes/event_bridge_event.py

@@ -60,7 +60,7 @@ def detail_type(self) -> str:
 
     @property
     def detail(self) -> Dict[str, Any]:
-        """A JSON object, whose content is at the discretion of the service originating the event. """
+        """A JSON object, whose content is at the discretion of the service originating the event."""
         return self["detail"]
 
     @property

aws_lambda_powertools/utilities/data_classes/s3_object_event.py

@@ -53,7 +53,7 @@ def payload(self) -> str:
 
 
 class S3ObjectUserRequest(DictWrapper):
-    """ Information about the original call to S3 Object Lambda."""
+    """Information about the original call to S3 Object Lambda."""
 
     @property
     def url(self) -> str:

aws_lambda_powertools/utilities/data_classes/sns_event.py

@@ -46,7 +46,7 @@ def message_id(self) -> str:
 
     @property
     def message(self) -> str:
-        """A string that describes the message. """
+        """A string that describes the message."""
         return self["Sns"]["Message"]
 
     @property

aws_lambda_powertools/utilities/data_classes/sqs_event.py

@@ -70,7 +70,7 @@ def binary_value(self) -> Optional[str]:
 
     @property
     def data_type(self) -> str:
-        """ The message attribute data type. Supported types include `String`, `Number`, and `Binary`."""
+        """The message attribute data type. Supported types include `String`, `Number`, and `Binary`."""
         return self["dataType"]
 
 
@@ -120,7 +120,7 @@ def md5_of_body(self) -> str:
 
     @property
     def event_source(self) -> str:
-        """The AWS service from which the SQS record originated. For SQS, this is `aws:sqs` """
+        """The AWS service from which the SQS record originated. For SQS, this is `aws:sqs`"""
         return self["eventSource"]
 
     @property

aws_lambda_powertools/utilities/idempotency/persistence/base.py

@@ -110,7 +110,7 @@ class BasePersistenceLayer(ABC):
     """
 
     def __init__(self):
-        """Initialize the defaults """
+        """Initialize the defaults"""
         self.configured = False
         self.event_key_jmespath: Optional[str] = None
         self.event_key_compiled_jmespath = None

docs/core/logger.md

@@ -555,6 +555,32 @@ Sampling decision happens at the Logger initialization. This means sampling may
     }
     ```
 
+### LambdaPowertoolsFormatter
+
+Logger propagates a few formatting configurations to the built-in `LambdaPowertoolsFormatter` logging formatter.
+
+If you prefer configuring it separately, or you'd want to bring this JSON Formatter to another application, these are the supported settings:
+
+Parameter | Description | Default
+------------------------------------------------- | ------------------------------------------------- | -------------------------------------------------
+**`json_serializer`** | function to serialize `obj` to a JSON formatted `str` | `json.dumps`
+**`json_deserializer`** | function to deserialize `str`, `bytes`, `bytearray` containing a JSON document to a Python obj | `json.loads`
+**`json_default`** | function to coerce unserializable values, when no custom serializer/deserializer is set | `str`
+**`datefmt`** | string directives (strftime) to format log timestamp | `%Y-%m-%d %H:%M:%S,%F%z`, where `%F` is a custom ms directive
+**`utc`** | set logging timestamp to UTC | `False`
+**`log_record_order`** | set order of log keys when logging | `["level", "location", "message", "timestamp"]`
+**`kwargs`** | key-value to be included in log messages | `None`
+
+=== "LambdaPowertoolsFormatter.py"
+
+    ```python hl_lines="2 4-5"
+    from aws_lambda_powertools import Logger
+	from aws_lambda_powertools.logging.formatter import LambdaPowertoolsFormatter
+
+	formatter = LambdaPowertoolsFormatter(utc=True, log_record_order=["message"])
+    logger = Logger(service="example", logger_formatter=formatter)
+    ```
+
 ### Migrating from other Loggers
 
 If you're migrating from other Loggers, there are few key points to be aware of: [Service parameter](#the-service-parameter), [Inheriting Loggers](#inheriting-loggers), [Overriding Log records](#overriding-log-records), and [Logging exceptions](#logging-exceptions).
@@ -645,7 +671,6 @@ Logger allows you to either change the format or suppress the following keys alt
 	}
 	```
 
-
 #### Reordering log keys position
 
 You can change the order of [standard Logger keys](#standard-structured-keys) or any keys that will be appended later at runtime via the `log_record_order` parameter.
@@ -744,9 +769,30 @@ By default, Logger uses StreamHandler and logs to standard output. You can overr
 
 #### Bring your own formatter
 
-By default, Logger uses a custom Formatter that persists its custom structure between non-cold start invocations. There could be scenarios where the existing feature set isn't sufficient to your formatting needs.
+By default, Logger uses [LambdaPowertoolsFormatter](#lambdapowertoolsformatter) that persists its custom structure between non-cold start invocations. There could be scenarios where the existing feature set isn't sufficient to your formatting needs.
+
+For **minor changes like remapping keys** after all log record processing has completed, you can override `serialize` method from [LambdaPowertoolsFormatter](#lambdapowertoolsformatter):
+
+=== "custom_formatter.py"
+
+	```python
+	from aws_lambda_powertools import Logger
+	from aws_lambda_powertools.logging.formatter import LambdaPowertoolsFormatter
+
+	from typing import Dict
+
+	class CustomFormatter(LambdaPowertoolsFormatter):
+		def serialize(self, log: Dict) -> str:
+			"""Serialize final structured log dict to JSON str"""
+			log["event"] = log.pop("message")  # rename message key to event
+			return self.json_serializer(log)   # use configured json serializer
+
+	my_formatter = CustomFormatter()
+	logger = Logger(service="example", logger_formatter=my_formatter)
+	logger.info("hello")
+	```
 
-For this, you can subclass `BasePowertoolsFormatter`, implement `append_keys` method, and override `format` standard logging method. This ensures the current feature set of Logger like injecting Lambda context and sampling will continue to work.
+For **replacing the formatter entirely**, you can subclass `BasePowertoolsFormatter`, implement `append_keys` method, and override `format` standard logging method. This ensures the current feature set of Logger like [injecting Lambda context](#capturing-lambda-context-info) and [sampling](#sampling-debug-logs) will continue to work.
 
 !!! info
 	You might need to implement `remove_keys` method if you make use of the feature too.
@@ -758,7 +804,7 @@ For this, you can subclass `BasePowertoolsFormatter`, implement `append_keys` me
 	from aws_lambda_powertools.logging.formatter import BasePowertoolsFormatter
 
     class CustomFormatter(BasePowertoolsFormatter):
-        custom_format = {}  # will hold our structured keys
+        custom_format = {}  # arbitrary dict to hold our structured keys
 
         def append_keys(self, **additional_keys):
 			# also used by `inject_lambda_context` decorator

tests/functional/test_metrics.py

@@ -85,7 +85,7 @@ def a_hundred_metrics() -> List[Dict[str, str]]:
 def serialize_metrics(
     metrics: List[Dict], dimensions: List[Dict], namespace: str, metadatas: List[Dict] = None
 ) -> Dict:
-    """ Helper function to build EMF object from a list of metrics, dimensions """
+    """Helper function to build EMF object from a list of metrics, dimensions"""
     my_metrics = MetricManager(namespace=namespace)
     for dimension in dimensions:
         my_metrics.add_dimension(**dimension)
@@ -102,7 +102,7 @@ def serialize_metrics(
 
 
 def serialize_single_metric(metric: Dict, dimension: Dict, namespace: str, metadata: Dict = None) -> Dict:
-    """ Helper function to build EMF object from a given metric, dimension and namespace """
+    """Helper function to build EMF object from a given metric, dimension and namespace"""
     my_metrics = MetricManager(namespace=namespace)
     my_metrics.add_metric(**metric)
     my_metrics.add_dimension(**dimension)
@@ -114,7 +114,7 @@ def serialize_single_metric(metric: Dict, dimension: Dict, namespace: str, metad
 
 
 def remove_timestamp(metrics: List):
-    """ Helper function to remove Timestamp key from EMF objects as they're built at serialization """
+    """Helper function to remove Timestamp key from EMF objects as they're built at serialization"""
     for metric in metrics:
         del metric["_aws"]["Timestamp"]