feat(logging): suppress some log keys

Some keys like `location` and `timestamp` can be suppressed by setting
them to None.

Changes:
* logger.py - Convert some methods to static functions
* test_aws_lambda_logging - Add more tests and some which can be used
  for documentation

Author: michaelbrewer

aws_lambda_powertools/logging/formatter.py

@@ -27,7 +27,7 @@ def __init__(self, **kwargs):
 
         super(JsonFormatter, self).__init__(datefmt=datefmt)
         self.reserved_keys = ["timestamp", "level", "location"]
-        self.format_dict = dict.fromkeys(kwargs.pop("format_key", ["level", "location", "message", "timestamp"]))
+        self.format_dict = dict.fromkeys(kwargs.pop("format_keys", ["level", "location", "message", "timestamp"]))
         self.format_dict.update(
             {"level": "%(levelname)s", "location": "%(funcName)s:%(lineno)d", "timestamp": "%(asctime)s"}
         )
@@ -71,4 +71,7 @@ def format(self, record):  # noqa: A003
         if record.exc_text:
             log_dict["exception"] = record.exc_text
 
+        # Filter out top level key with values that are None
+        log_dict = {k: v for k, v in log_dict.items() if v is not None}
+
         return json.dumps(log_dict, default=self.default_json_formatter)

aws_lambda_powertools/logging/logger.py

@@ -123,7 +123,7 @@ def __init__(
     ):
         self.service = service or os.getenv("POWERTOOLS_SERVICE_NAME") or "service_undefined"
         self.sampling_rate = sampling_rate or os.getenv("POWERTOOLS_LOGGER_SAMPLE_RATE") or 0.0
-        self.log_level = self._get_log_level(level)
+        self.log_level = _get_log_level(level)
         self.child = child
         self._handler = logging.StreamHandler(stream) if stream is not None else logging.StreamHandler(sys.stdout)
         self._default_log_keys = {"service": self.service, "sampling_rate": self.sampling_rate}
@@ -136,35 +136,14 @@ def __getattr__(self, name):
         # https://github.com/awslabs/aws-lambda-powertools-python/issues/97
         return getattr(self._logger, name)
 
-    def _get_log_level(self, level: Union[str, int]) -> Union[str, int]:
-        """ Returns preferred log level set by the customer in upper case """
-        if isinstance(level, int):
-            return level
-
-        log_level: str = level or os.getenv("LOG_LEVEL")
-        log_level = log_level.upper() if log_level is not None else logging.INFO
-
-        return log_level
-
     def _get_logger(self):
         """ Returns a Logger named {self.service}, or {self.service.filename} for child loggers"""
         logger_name = self.service
         if self.child:
-            logger_name = f"{self.service}.{self._get_caller_filename()}"
+            logger_name = f"{self.service}.{_get_caller_filename()}"
 
         return logging.getLogger(logger_name)
 
-    def _get_caller_filename(self):
-        """ Return caller filename by finding the caller frame """
-        # Current frame         => _get_logger()
-        # Previous frame        => logger.py
-        # Before previous frame => Caller
-        frame = inspect.currentframe()
-        caller_frame = frame.f_back.f_back.f_back
-        filename = caller_frame.f_globals["__name__"]
-
-        return filename
-
     def _init_logger(self, **kwargs):
         """Configures new logger"""
 
@@ -207,6 +186,8 @@ def inject_lambda_context(self, lambda_handler: Callable[[Dict, Any], Any] = Non
 
         Parameters
         ----------
+        lambda_handler : Callable
+            Method to inject the lambda context
         log_event : bool, optional
             Instructs logger to log Lambda Event, by default False
 
@@ -292,6 +273,29 @@ def structure_logs(self, append: bool = False, **kwargs):
                 handler.setFormatter(JsonFormatter(**self._default_log_keys, **kwargs))
 
 
+def _get_log_level(level: Union[str, int]) -> Union[str, int]:
+    """ Returns preferred log level set by the customer in upper case """
+    if isinstance(level, int):
+        return level
+
+    log_level: str = level or os.getenv("LOG_LEVEL")
+    log_level = log_level.upper() if log_level is not None else logging.INFO
+
+    return log_level
+
+
+def _get_caller_filename():
+    """ Return caller filename by finding the caller frame """
+    # Current frame         => _get_logger()
+    # Previous frame        => logger.py
+    # Before previous frame => Caller
+    frame = inspect.currentframe()
+    caller_frame = frame.f_back.f_back.f_back
+    filename = caller_frame.f_globals["__name__"]
+
+    return filename
+
+
 def set_package_logger(
     level: Union[str, int] = logging.DEBUG, stream: sys.stdout = None, formatter: logging.Formatter = None
 ):

tests/functional/test_aws_lambda_logging.py

@@ -127,18 +127,44 @@ def test_log_dict_key_seq(stdout):
 
     log_dict: dict = json.loads(stdout.getvalue())
 
-    # THEN the key sequence should be `level,location,message,timestamp`
+    # THEN the beginning key sequence must be `level,location,message,timestamp`
     assert ",".join(list(log_dict.keys())[:4]) == "level,location,message,timestamp"
 
 
 def test_log_dict_key_custom_seq(stdout):
-    # GIVEN any logger configuration
-    logger = Logger(level="INFO", stream=stdout, another="xxx", format_key=["message"])
+    # GIVEN a logger configuration with format_keys set to ["message"]
+    logger = Logger(stream=stdout, format_keys=["message"])
 
     # WHEN logging a message
     logger.info("Message")
 
     log_dict: dict = json.loads(stdout.getvalue())
 
-    # THEN the key sequence should be `level,location,message,timestamp`
-    assert ",".join(list(log_dict.keys())[:4]) == "message,level,location,timestamp"
+    # THEN the first key should be "message"
+    assert list(log_dict.keys())[0] == "message"
+
+
+def test_log_custom_formatting(stdout):
+    # GIVEN a logger where we have a custom location format
+    logger = Logger(stream=stdout, location="[%(funcName)s] %(module)s")
+
+    # WHEN logging a message
+    logger.info("foo")
+
+    log_dict: dict = json.loads(stdout.getvalue())
+
+    # THEN the `location` match the formatting
+    assert log_dict["location"] == "[test_log_custom_formatting] test_aws_lambda_logging"
+
+
+def test_log_dict_key_strip_nones(stdout):
+    # GIVEN a logger confirmation where we set `location` and `timestamp` to None
+    logger = Logger(stream=stdout, location=None, timestamp=None)
+
+    # WHEN logging a message
+    logger.info("foo")
+
+    log_dict: dict = json.loads(stdout.getvalue())
+
+    # THEN the keys should only include `level`, `message`, `service`, `sampling_rate`
+    assert sorted(log_dict.keys()) == ["level", "message", "sampling_rate", "service"]