diff --git a/aws_lambda_powertools/logging/formatter.py b/aws_lambda_powertools/logging/formatter.py index d1541305f1b..47418063732 100644 --- a/aws_lambda_powertools/logging/formatter.py +++ b/aws_lambda_powertools/logging/formatter.py @@ -69,7 +69,7 @@ def __init__( The `log_record_order` kwarg is used to specify the order of the keys used in the structured json logs. By default the order is: "level", "location", "message", "timestamp", - "service" and "sampling_rate". + "service". Other kwargs are used to specify log field format strings. @@ -113,6 +113,10 @@ def __init__( keys_combined = {**self._build_default_keys(), **kwargs} self.log_format.update(**keys_combined) + def serialize(self, log: Dict) -> str: + """Serialize structured log dict to JSON str""" + return self.json_serializer(log) + def format(self, record: logging.LogRecord) -> str: # noqa: A003 """Format logging record as structured JSON str""" formatted_log = self._extract_log_keys(log_record=record) @@ -121,7 +125,7 @@ def format(self, record: logging.LogRecord) -> str: # noqa: A003 formatted_log["xray_trace_id"] = self._get_latest_trace_id() formatted_log = self._strip_none_records(records=formatted_log) - return self.json_serializer(formatted_log) + return self.serialize(log=formatted_log) def formatTime(self, record: logging.LogRecord, datefmt: Optional[str] = None) -> str: record_ts = self.converter(record.created) diff --git a/aws_lambda_powertools/logging/logger.py b/aws_lambda_powertools/logging/logger.py index 77e0f3db059..3231f30eccd 100644 --- a/aws_lambda_powertools/logging/logger.py +++ b/aws_lambda_powertools/logging/logger.py @@ -4,7 +4,7 @@ import os import random import sys -from typing import Any, Callable, Dict, Iterable, Optional, Union +from typing import Any, Callable, Dict, Iterable, Optional, TypeVar, Union import jmespath @@ -19,6 +19,8 @@ is_cold_start = True +PowertoolsFormatter = TypeVar("PowertoolsFormatter", bound=BasePowertoolsFormatter) + def _is_cold_start() -> bool: """Verifies whether is cold start @@ -70,8 +72,8 @@ class Logger(logging.Logger): # lgtm [py/missing-call-to-init] sample rate for debug calls within execution context defaults to 0.0 stream: sys.stdout, optional valid output for a logging stream, by default sys.stdout - logger_formatter: BasePowertoolsFormatter, optional - custom logging formatter that implements BasePowertoolsFormatter + logger_formatter: PowertoolsFormatter, optional + custom logging formatter that implements PowertoolsFormatter logger_handler: logging.Handler, optional custom logging handler e.g. logging.FileHandler("file.log") @@ -87,7 +89,7 @@ class Logger(logging.Logger): # lgtm [py/missing-call-to-init] json_default : Callable, optional function to coerce unserializable values, by default `str()` - Only used when no custom JSON encoder is set + Only used when no custom formatter is set utc : bool, optional set logging timestamp to UTC, by default False to continue to use local time as per stdlib log_record_order : list, optional @@ -170,7 +172,7 @@ def __init__( child: bool = False, sampling_rate: float = None, stream: sys.stdout = None, - logger_formatter: Optional[BasePowertoolsFormatter] = None, + logger_formatter: Optional[PowertoolsFormatter] = None, logger_handler: Optional[logging.Handler] = None, **kwargs, ): @@ -198,7 +200,7 @@ def __getattr__(self, name): return getattr(self._logger, name) def _get_logger(self): - """ Returns a Logger named {self.service}, or {self.service.filename} for child loggers""" + """Returns a Logger named {self.service}, or {self.service.filename} for child loggers""" logger_name = self.service if self.child: logger_name = f"{self.service}.{self._get_caller_filename()}" @@ -346,7 +348,7 @@ def registered_handler(self) -> logging.Handler: return handlers[0] @property - def registered_formatter(self) -> Optional[BasePowertoolsFormatter]: + def registered_formatter(self) -> Optional[PowertoolsFormatter]: """Convenience property to access logger formatter""" return self.registered_handler.formatter @@ -384,7 +386,7 @@ def set_correlation_id(self, value: str): @staticmethod def _get_log_level(level: Union[str, int, None]) -> Union[str, int]: - """ Returns preferred log level set by the customer in upper case """ + """Returns preferred log level set by the customer in upper case""" if isinstance(level, int): return level @@ -396,7 +398,7 @@ def _get_log_level(level: Union[str, int, None]) -> Union[str, int]: @staticmethod def _get_caller_filename(): - """ Return caller filename by finding the caller frame """ + """Return caller filename by finding the caller frame""" # Current frame => _get_logger() # Previous frame => logger.py # Before previous frame => Caller diff --git a/aws_lambda_powertools/tracing/tracer.py b/aws_lambda_powertools/tracing/tracer.py index 5e2e545e356..47568802202 100644 --- a/aws_lambda_powertools/tracing/tracer.py +++ b/aws_lambda_powertools/tracing/tracer.py @@ -720,7 +720,7 @@ def __build_config( patch_modules: Union[List, Tuple] = None, provider: BaseProvider = None, ): - """ Populates Tracer config for new and existing initializations """ + """Populates Tracer config for new and existing initializations""" is_disabled = disabled if disabled is not None else self._is_tracer_disabled() is_service = resolve_env_var_choice(choice=service, env=os.getenv(constants.SERVICE_NAME_ENV)) diff --git a/aws_lambda_powertools/utilities/data_classes/api_gateway_proxy_event.py b/aws_lambda_powertools/utilities/data_classes/api_gateway_proxy_event.py index 20cbfa58fd2..1ce6a742125 100644 --- a/aws_lambda_powertools/utilities/data_classes/api_gateway_proxy_event.py +++ b/aws_lambda_powertools/utilities/data_classes/api_gateway_proxy_event.py @@ -164,7 +164,7 @@ def path(self) -> str: @property def stage(self) -> str: - """The deployment stage of the API request """ + """The deployment stage of the API request""" return self["requestContext"]["stage"] @property @@ -352,7 +352,7 @@ def authorizer(self) -> Optional[RequestContextV2Authorizer]: @property def domain_name(self) -> str: - """A domain name """ + """A domain name""" return self["requestContext"]["domainName"] @property @@ -375,7 +375,7 @@ def route_key(self) -> str: @property def stage(self) -> str: - """The deployment stage of the API request """ + """The deployment stage of the API request""" return self["requestContext"]["stage"] @property diff --git a/aws_lambda_powertools/utilities/data_classes/appsync_resolver_event.py b/aws_lambda_powertools/utilities/data_classes/appsync_resolver_event.py index dae09065568..56d37851631 100644 --- a/aws_lambda_powertools/utilities/data_classes/appsync_resolver_event.py +++ b/aws_lambda_powertools/utilities/data_classes/appsync_resolver_event.py @@ -22,7 +22,7 @@ class AppSyncIdentityIAM(DictWrapper): @property def source_ip(self) -> List[str]: - """The source IP address of the caller received by AWS AppSync. """ + """The source IP address of the caller received by AWS AppSync.""" return self["sourceIp"] @property @@ -67,7 +67,7 @@ class AppSyncIdentityCognito(DictWrapper): @property def source_ip(self) -> List[str]: - """The source IP address of the caller received by AWS AppSync. """ + """The source IP address of the caller received by AWS AppSync.""" return self["sourceIp"] @property diff --git a/aws_lambda_powertools/utilities/data_classes/event_bridge_event.py b/aws_lambda_powertools/utilities/data_classes/event_bridge_event.py index 9c00922069e..bdbf9d68afa 100644 --- a/aws_lambda_powertools/utilities/data_classes/event_bridge_event.py +++ b/aws_lambda_powertools/utilities/data_classes/event_bridge_event.py @@ -60,7 +60,7 @@ def detail_type(self) -> str: @property def detail(self) -> Dict[str, Any]: - """A JSON object, whose content is at the discretion of the service originating the event. """ + """A JSON object, whose content is at the discretion of the service originating the event.""" return self["detail"] @property diff --git a/aws_lambda_powertools/utilities/data_classes/s3_object_event.py b/aws_lambda_powertools/utilities/data_classes/s3_object_event.py index f653f7aca6e..b22434c68e3 100644 --- a/aws_lambda_powertools/utilities/data_classes/s3_object_event.py +++ b/aws_lambda_powertools/utilities/data_classes/s3_object_event.py @@ -53,7 +53,7 @@ def payload(self) -> str: class S3ObjectUserRequest(DictWrapper): - """ Information about the original call to S3 Object Lambda.""" + """Information about the original call to S3 Object Lambda.""" @property def url(self) -> str: diff --git a/aws_lambda_powertools/utilities/data_classes/sns_event.py b/aws_lambda_powertools/utilities/data_classes/sns_event.py index e96b096fe6b..84ee1c1ef0f 100644 --- a/aws_lambda_powertools/utilities/data_classes/sns_event.py +++ b/aws_lambda_powertools/utilities/data_classes/sns_event.py @@ -46,7 +46,7 @@ def message_id(self) -> str: @property def message(self) -> str: - """A string that describes the message. """ + """A string that describes the message.""" return self["Sns"]["Message"] @property diff --git a/aws_lambda_powertools/utilities/data_classes/sqs_event.py b/aws_lambda_powertools/utilities/data_classes/sqs_event.py index 778b8f56f36..0e70684cc3f 100644 --- a/aws_lambda_powertools/utilities/data_classes/sqs_event.py +++ b/aws_lambda_powertools/utilities/data_classes/sqs_event.py @@ -70,7 +70,7 @@ def binary_value(self) -> Optional[str]: @property def data_type(self) -> str: - """ The message attribute data type. Supported types include `String`, `Number`, and `Binary`.""" + """The message attribute data type. Supported types include `String`, `Number`, and `Binary`.""" return self["dataType"] @@ -120,7 +120,7 @@ def md5_of_body(self) -> str: @property def event_source(self) -> str: - """The AWS service from which the SQS record originated. For SQS, this is `aws:sqs` """ + """The AWS service from which the SQS record originated. For SQS, this is `aws:sqs`""" return self["eventSource"] @property diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/base.py b/aws_lambda_powertools/utilities/idempotency/persistence/base.py index 263414a9573..0cbd34213c1 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/base.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/base.py @@ -110,7 +110,7 @@ class BasePersistenceLayer(ABC): """ def __init__(self): - """Initialize the defaults """ + """Initialize the defaults""" self.configured = False self.event_key_jmespath: Optional[str] = None self.event_key_compiled_jmespath = None diff --git a/docs/core/logger.md b/docs/core/logger.md index e1969c5ab0f..f8e806aa6b4 100644 --- a/docs/core/logger.md +++ b/docs/core/logger.md @@ -555,6 +555,32 @@ Sampling decision happens at the Logger initialization. This means sampling may } ``` +### LambdaPowertoolsFormatter + +Logger propagates a few formatting configurations to the built-in `LambdaPowertoolsFormatter` logging formatter. + +If you prefer configuring it separately, or you'd want to bring this JSON Formatter to another application, these are the supported settings: + +Parameter | Description | Default +------------------------------------------------- | ------------------------------------------------- | ------------------------------------------------- +**`json_serializer`** | function to serialize `obj` to a JSON formatted `str` | `json.dumps` +**`json_deserializer`** | function to deserialize `str`, `bytes`, `bytearray` containing a JSON document to a Python obj | `json.loads` +**`json_default`** | function to coerce unserializable values, when no custom serializer/deserializer is set | `str` +**`datefmt`** | string directives (strftime) to format log timestamp | `%Y-%m-%d %H:%M:%S,%F%z`, where `%F` is a custom ms directive +**`utc`** | set logging timestamp to UTC | `False` +**`log_record_order`** | set order of log keys when logging | `["level", "location", "message", "timestamp"]` +**`kwargs`** | key-value to be included in log messages | `None` + +=== "LambdaPowertoolsFormatter.py" + + ```python hl_lines="2 4-5" + from aws_lambda_powertools import Logger + from aws_lambda_powertools.logging.formatter import LambdaPowertoolsFormatter + + formatter = LambdaPowertoolsFormatter(utc=True, log_record_order=["message"]) + logger = Logger(service="example", logger_formatter=formatter) + ``` + ### Migrating from other Loggers If you're migrating from other Loggers, there are few key points to be aware of: [Service parameter](#the-service-parameter), [Inheriting Loggers](#inheriting-loggers), [Overriding Log records](#overriding-log-records), and [Logging exceptions](#logging-exceptions). @@ -645,7 +671,6 @@ Logger allows you to either change the format or suppress the following keys alt } ``` - #### Reordering log keys position You can change the order of [standard Logger keys](#standard-structured-keys) or any keys that will be appended later at runtime via the `log_record_order` parameter. @@ -744,9 +769,30 @@ By default, Logger uses StreamHandler and logs to standard output. You can overr #### Bring your own formatter -By default, Logger uses a custom Formatter that persists its custom structure between non-cold start invocations. There could be scenarios where the existing feature set isn't sufficient to your formatting needs. +By default, Logger uses [LambdaPowertoolsFormatter](#lambdapowertoolsformatter) that persists its custom structure between non-cold start invocations. There could be scenarios where the existing feature set isn't sufficient to your formatting needs. + +For **minor changes like remapping keys** after all log record processing has completed, you can override `serialize` method from [LambdaPowertoolsFormatter](#lambdapowertoolsformatter): + +=== "custom_formatter.py" + + ```python + from aws_lambda_powertools import Logger + from aws_lambda_powertools.logging.formatter import LambdaPowertoolsFormatter + + from typing import Dict + + class CustomFormatter(LambdaPowertoolsFormatter): + def serialize(self, log: Dict) -> str: + """Serialize final structured log dict to JSON str""" + log["event"] = log.pop("message") # rename message key to event + return self.json_serializer(log) # use configured json serializer + + my_formatter = CustomFormatter() + logger = Logger(service="example", logger_formatter=my_formatter) + logger.info("hello") + ``` -For this, you can subclass `BasePowertoolsFormatter`, implement `append_keys` method, and override `format` standard logging method. This ensures the current feature set of Logger like injecting Lambda context and sampling will continue to work. +For **replacing the formatter entirely**, you can subclass `BasePowertoolsFormatter`, implement `append_keys` method, and override `format` standard logging method. This ensures the current feature set of Logger like [injecting Lambda context](#capturing-lambda-context-info) and [sampling](#sampling-debug-logs) will continue to work. !!! info You might need to implement `remove_keys` method if you make use of the feature too. @@ -758,7 +804,7 @@ For this, you can subclass `BasePowertoolsFormatter`, implement `append_keys` me from aws_lambda_powertools.logging.formatter import BasePowertoolsFormatter class CustomFormatter(BasePowertoolsFormatter): - custom_format = {} # will hold our structured keys + custom_format = {} # arbitrary dict to hold our structured keys def append_keys(self, **additional_keys): # also used by `inject_lambda_context` decorator diff --git a/tests/functional/test_metrics.py b/tests/functional/test_metrics.py index ee725da2699..ae160c65d87 100644 --- a/tests/functional/test_metrics.py +++ b/tests/functional/test_metrics.py @@ -85,7 +85,7 @@ def a_hundred_metrics() -> List[Dict[str, str]]: def serialize_metrics( metrics: List[Dict], dimensions: List[Dict], namespace: str, metadatas: List[Dict] = None ) -> Dict: - """ Helper function to build EMF object from a list of metrics, dimensions """ + """Helper function to build EMF object from a list of metrics, dimensions""" my_metrics = MetricManager(namespace=namespace) for dimension in dimensions: my_metrics.add_dimension(**dimension) @@ -102,7 +102,7 @@ def serialize_metrics( def serialize_single_metric(metric: Dict, dimension: Dict, namespace: str, metadata: Dict = None) -> Dict: - """ Helper function to build EMF object from a given metric, dimension and namespace """ + """Helper function to build EMF object from a given metric, dimension and namespace""" my_metrics = MetricManager(namespace=namespace) my_metrics.add_metric(**metric) my_metrics.add_dimension(**dimension) @@ -114,7 +114,7 @@ def serialize_single_metric(metric: Dict, dimension: Dict, namespace: str, metad def remove_timestamp(metrics: List): - """ Helper function to remove Timestamp key from EMF objects as they're built at serialization """ + """Helper function to remove Timestamp key from EMF objects as they're built at serialization""" for metric in metrics: del metric["_aws"]["Timestamp"]