feat: add flag to disable log deduplication filter #262

Author: heitorlessa

aws_lambda_powertools/logging/logger.py

@@ -129,12 +129,14 @@ def __init__(
         self.sampling_rate = resolve_env_var_choice(
             choice=sampling_rate, env=os.getenv(constants.LOGGER_LOG_SAMPLING_RATE, 0.0)
         )
+        self._is_deduplication_disabled = resolve_truthy_env_var_choice(
+            env=os.getenv(constants.LOGGER_LOG_DEDUPLICATION_ENV, "false")
+        )
         self.log_level = self._get_log_level(level)
         self.child = child
         self._handler = logging.StreamHandler(stream) if stream is not None else logging.StreamHandler(sys.stdout)
         self._default_log_keys = {"service": self.service, "sampling_rate": self.sampling_rate}
         self._logger = self._get_logger()
-
         self._init_logger(**kwargs)
 
     def __getattr__(self, name):
@@ -167,12 +169,16 @@ def _init_logger(self, **kwargs):
         self._logger.addHandler(self._handler)
         self.structure_logs(**kwargs)
 
-        logger.debug("Adding filter in root logger to suppress child logger records to bubble up")
-        for handler in logging.root.handlers:
-            # It'll add a filter to suppress any child logger from self.service
-            # Where service is Order, it'll reject parent logger Order,
-            # and child loggers such as Order.checkout, Order.shared
-            handler.addFilter(SuppressFilter(self.service))
+        # Pytest Live Log feature duplicates log records for colored output
+        # but we explicitly add a filter for log deduplication.
+        # This flag disables this protection when you explicit want logs to be duplicated (#262)
+        if not self._is_deduplication_disabled:
+            logger.debug("Adding filter in root logger to suppress child logger records to bubble up")
+            for handler in logging.root.handlers:
+                # It'll add a filter to suppress any child logger from self.service
+                # Where service is Order, it'll reject parent logger Order,
+                # and child loggers such as Order.checkout, Order.shared
+                handler.addFilter(SuppressFilter(self.service))
 
         # as per bug in #249, we should not be pre-configuring an existing logger
         # therefore we set a custom attribute in the Logger that will be returned

aws_lambda_powertools/shared/constants.py

@@ -4,6 +4,7 @@
 
 LOGGER_LOG_SAMPLING_RATE: str = "POWERTOOLS_LOGGER_SAMPLE_RATE"
 LOGGER_LOG_EVENT_ENV: str = "POWERTOOLS_LOGGER_LOG_EVENT"
+LOGGER_LOG_DEDUPLICATION_ENV: str = "POWERTOOLS_LOG_DEDUPLICATION_DISABLED"
 
 MIDDLEWARE_FACTORY_TRACE_ENV: str = "POWERTOOLS_TRACE_MIDDLEWARES"
 

tests/functional/test_logger.py

@@ -11,6 +11,7 @@
 from aws_lambda_powertools import Logger, Tracer
 from aws_lambda_powertools.logging.exceptions import InvalidLoggerSamplingRateError
 from aws_lambda_powertools.logging.logger import set_package_logger
+from aws_lambda_powertools.shared import constants
 
 
 @pytest.fixture
@@ -376,6 +377,7 @@ def test_logger_do_not_log_twice_when_root_logger_is_setup(stdout, service_name)
     child_logger = Logger(service=service_name, child=True, stream=stdout)
     logger.info("PARENT")
     child_logger.info("CHILD")
+    root_logger.info("ROOT")
 
     # THEN it should only contain only two log entries
     # since child's log records propagated to root logger should be rejected
@@ -400,3 +402,23 @@ def test_logger_extra_kwargs(stdout, service_name):
 
     # THEN second log should not have request_id in the root structure
     assert "request_id" not in no_extra_fields_log
+
+
+def test_logger_log_twice_when_log_filter_isnt_present_and_root_logger_is_setup(monkeypatch, stdout, service_name):
+    # GIVEN Lambda configures the root logger with a handler
+    root_logger = logging.getLogger()
+    root_logger.addHandler(logging.StreamHandler(stream=stdout))
+
+    # WHEN we create a new Logger and child Logger
+    # and log deduplication filter for child messages are disabled
+    # see #262 for more details on why this is needed for Pytest Live Log feature
+    monkeypatch.setenv(constants.LOGGER_LOG_DEDUPLICATION_ENV, "true")
+    logger = Logger(service=service_name, stream=stdout)
+    child_logger = Logger(service=service_name, child=True, stream=stdout)
+    logger.info("PARENT")
+    child_logger.info("CHILD")
+
+    # THEN it should only contain only two log entries
+    # since child's log records propagated to root logger should be rejected
+    logs = list(stdout.getvalue().strip().split("\n"))
+    assert len(logs) == 4