feat(logger): custom function for unserializable values (JSON replacer) (#2739)

Co-authored-by: Andrea Amorosi <[email protected]>

Author: arnabrahman

Diff for: docs/core/logger.md

@@ -760,6 +760,26 @@ This is how the printed log would look:
 !!! tip "Custom Log formatter and Child loggers"
     It is not necessary to pass the `LogFormatter` each time a [child logger](#using-multiple-logger-instances-across-your-code) is created. The parent's LogFormatter will be inherited by the child logger.
 
+### Bring your own JSON serializer
+
+You can extend the default JSON serializer by passing a custom serializer function to the `Logger` constructor, using the `jsonReplacerFn` option. This is useful when you want to customize the serialization of specific values.
+
+=== "unserializableValues.ts"
+
+    ```typescript hl_lines="4-5 7"
+    --8<-- "examples/snippets/logger/unserializableValues.ts"
+    ```
+
+=== "unserializableValues.json"
+
+    ```json hl_lines="8"
+    --8<-- "examples/snippets/logger/unserializableValues.json"
+    ```
+
+By default, Logger uses `JSON.stringify()` to serialize log items and a [custom replacer function](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify#the_replacer_parameter) to serialize common unserializable values such as `BigInt`, circular references, and `Error` objects.
+
+When you extend the default JSON serializer, we will call your custom serializer function before the default one. This allows you to customize the serialization while still benefiting from the default behavior.
+
 ## Testing your code
 
 ### Inject Lambda Context

Diff for: examples/snippets/logger/unserializableValues.json

@@ -0,0 +1,9 @@
+{
+  "level": "INFO",
+  "message": "Serialize with custom serializer",
+  "sampling_rate": 0,
+  "service": "serverlessAirline",
+  "timestamp": "2024-07-07T09:52:14.212Z",
+  "xray_trace_id": "1-668a654d-396c646b760ee7d067f32f18",
+  "serializedValue": [1, 2, 3]
+}

Diff for: examples/snippets/logger/unserializableValues.ts

@@ -0,0 +1,13 @@
+import { Logger } from '@aws-lambda-powertools/logger';
+import type { CustomReplacerFn } from '@aws-lambda-powertools/logger/types';
+
+const jsonReplacerFn: CustomReplacerFn = (_: string, value: unknown) =>
+  value instanceof Set ? [...value] : value;
+
+const logger = new Logger({ serviceName: 'serverlessAirline', jsonReplacerFn });
+
+export const handler = async (): Promise<void> => {
+  logger.info('Serialize with custom serializer', {
+    serializedValue: new Set([1, 2, 3]),
+  });
+};

Diff for: packages/logger/src/Logger.ts

@@ -24,6 +24,7 @@ import type {
   LogItemMessage,
   LoggerInterface,
   PowertoolsLogData,
+  CustomJsonReplacerFn,
 } from './types/Logger.js';
 
 /**
@@ -200,6 +201,10 @@ class Logger extends Utility implements LoggerInterface {
    * We keep this value to be able to reset the log level to the initial value when the sample rate is refreshed.
    */
   #initialLogLevel = 12;
+  /**
+   * Replacer function used to serialize the log items.
+   */
+  #jsonReplacerFn?: CustomJsonReplacerFn;
 
   /**
    * Log level used by the current instance of Logger.
@@ -309,6 +314,7 @@ class Logger extends Utility implements LoggerInterface {
           environment: this.powertoolsLogData.environment,
           persistentLogAttributes: this.persistentLogAttributes,
           temporaryLogAttributes: this.temporaryLogAttributes,
+          jsonReplacerFn: this.#jsonReplacerFn,
         },
         options
       )
@@ -674,6 +680,42 @@ class Logger extends Utility implements LoggerInterface {
     return new Logger(options);
   }
 
+  /**
+   * A custom JSON replacer function that is used to serialize the log items.
+   *
+   * By default, we already extend the default serialization behavior to handle `BigInt` and `Error` objects, as well as remove circular references.
+   * When a custom JSON replacer function is passed to the Logger constructor, it will be called **before** our custom rules for each key-value pair in the object being stringified.
+   *
+   * This allows you to customize the serialization while still benefiting from the default behavior.
+   *
+   * @see {@link ConstructorOptions.jsonReplacerFn}
+   *
+   * @param key - The key of the value being stringified.
+   * @param value - The value being stringified.
+   */
+  protected getJsonReplacer(): (key: string, value: unknown) => void {
+    const references = new WeakSet();
+
+    return (key, value) => {
+      if (this.#jsonReplacerFn) value = this.#jsonReplacerFn?.(key, value);
+
+      if (value instanceof Error) {
+        value = this.getLogFormatter().formatError(value);
+      }
+      if (typeof value === 'bigint') {
+        return value.toString();
+      }
+      if (typeof value === 'object' && value !== null) {
+        if (references.has(value)) {
+          return;
+        }
+        references.add(value);
+      }
+
+      return value;
+    };
+  }
+
   /**
    * It stores information that is printed in all log items.
    *
@@ -835,40 +877,6 @@ class Logger extends Utility implements LoggerInterface {
     return this.powertoolsLogData;
   }
 
-  /**
-   * When the data added in the log item contains object references or BigInt values,
-   * `JSON.stringify()` can't handle them and instead throws errors:
-   * `TypeError: cyclic object value` or `TypeError: Do not know how to serialize a BigInt`.
-   * To mitigate these issues, this method will find and remove all cyclic references and convert BigInt values to strings.
-   *
-   * @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify#exceptions
-   * @private
-   */
-  private getReplacer(): (
-    key: string,
-    value: LogAttributes | Error | bigint
-  ) => void {
-    const references = new WeakSet();
-
-    return (key, value) => {
-      let item = value;
-      if (item instanceof Error) {
-        item = this.getLogFormatter().formatError(item);
-      }
-      if (typeof item === 'bigint') {
-        return item.toString();
-      }
-      if (typeof item === 'object' && value !== null) {
-        if (references.has(item)) {
-          return;
-        }
-        references.add(item);
-      }
-
-      return item;
-    };
-  }
-
   /**
    * It returns true and type guards the log level if a given log level is valid.
    *
@@ -920,7 +928,7 @@ class Logger extends Utility implements LoggerInterface {
     this.console[consoleMethod](
       JSON.stringify(
         log.getAttributes(),
-        this.getReplacer(),
+        this.getJsonReplacer(),
         this.logIndentation
       )
     );
@@ -1119,6 +1127,7 @@ class Logger extends Utility implements LoggerInterface {
       persistentKeys,
       persistentLogAttributes, // deprecated in favor of persistentKeys
       environment,
+      jsonReplacerFn,
     } = options;
 
     if (persistentLogAttributes && persistentKeys) {
@@ -1143,6 +1152,7 @@ class Logger extends Utility implements LoggerInterface {
     this.setLogFormatter(logFormatter);
     this.setConsole();
     this.setLogIndentation();
+    this.#jsonReplacerFn = jsonReplacerFn;
 
     return this;
   }

Diff for: packages/logger/src/types/Logger.ts

@@ -28,13 +28,35 @@ type InjectLambdaContextOptions = {
   resetKeys?: boolean;
 };
 
+/**
+ * A custom JSON replacer function that can be passed to the Logger constructor to extend the default serialization behavior.
+ *
+ * By default, we already extend the default serialization behavior to handle `BigInt` and `Error` objects, as well as remove circular references.
+ * When a custom JSON replacer function is passed to the Logger constructor, it will be called **before** our custom rules for each key-value pair in the object being stringified.
+ *
+ * This allows you to customize the serialization while still benefiting from the default behavior.
+ *
+ * @param key - The key of the value being stringified.
+ * @param value - The value being stringified.
+ */
+type CustomJsonReplacerFn = (key: string, value: unknown) => unknown;
+
 type BaseConstructorOptions = {
   logLevel?: LogLevel;
   serviceName?: string;
   sampleRateValue?: number;
   logFormatter?: LogFormatterInterface;
   customConfigService?: ConfigServiceInterface;
   environment?: Environment;
+  /**
+   * A custom JSON replacer function that can be passed to the Logger constructor to extend the default serialization behavior.
+   *
+   * By default, we already extend the default serialization behavior to handle `BigInt` and `Error` objects, as well as remove circular references.
+   * When a custom JSON replacer function is passed to the Logger constructor, it will be called **before** our custom rules for each key-value pair in the object being stringified.
+   *
+   * This allows you to customize the serialization while still benefiting from the default behavior.
+   */
+  jsonReplacerFn?: CustomJsonReplacerFn;
 };
 
 type PersistentKeysOption = {
@@ -139,4 +161,5 @@ export type {
   PowertoolsLogData,
   ConstructorOptions,
   InjectLambdaContextOptions,
+  CustomJsonReplacerFn,
 };

Diff for: packages/logger/src/types/index.ts

@@ -14,4 +14,5 @@ export type {
   PowertoolsLogData,
   ConstructorOptions,
   InjectLambdaContextOptions,
+  CustomJsonReplacerFn,
 } from './Logger.js';

Diff for: packages/logger/tests/unit/Logger.test.ts

@@ -10,9 +10,10 @@ import { ConfigServiceInterface } from '../../src/types/ConfigServiceInterface.j
 import { EnvironmentVariablesService } from '../../src/config/EnvironmentVariablesService.js';
 import { PowertoolsLogFormatter } from '../../src/formatter/PowertoolsLogFormatter.js';
 import { LogLevelThresholds, LogLevel } from '../../src/types/Log.js';
-import type {
-  LogFunction,
-  ConstructorOptions,
+import {
+  type LogFunction,
+  type ConstructorOptions,
+  type CustomJsonReplacerFn,
 } from '../../src/types/Logger.js';
 import { LogJsonIndent } from '../../src/constants.js';
 import type { Context } from 'aws-lambda';
@@ -1190,7 +1191,7 @@ describe('Class: Logger', () => {
         });
       });
 
-      describe('Feature: handle safely unexpected errors', () => {
+      describe('Feature: custom JSON replacer function', () => {
         test('when a logged item references itself, the logger ignores the keys that cause a circular reference', () => {
           // Prepare
           const logger = new Logger({
@@ -1312,6 +1313,100 @@ describe('Class: Logger', () => {
             })
           );
         });
+
+        it('should correctly serialize custom values using the provided jsonReplacerFn', () => {
+          // Prepare
+          const jsonReplacerFn: CustomJsonReplacerFn = (
+            _: string,
+            value: unknown
+          ) => (value instanceof Set ? [...value] : value);
+
+          const logger = new Logger({ jsonReplacerFn });
+          const consoleSpy = jest.spyOn(
+            logger['console'],
+            getConsoleMethod(methodOfLogger)
+          );
+          const message = `This is an ${methodOfLogger} log with Set value`;
+
+          const logItem = { value: new Set([1, 2]) };
+
+          // Act
+          logger[methodOfLogger](message, logItem);
+
+          // Assess
+          expect(consoleSpy).toHaveBeenCalledTimes(1);
+          expect(consoleSpy).toHaveBeenNthCalledWith(
+            1,
+            JSON.stringify({
+              level: methodOfLogger.toUpperCase(),
+              message: message,
+              sampling_rate: 0,
+              service: 'hello-world',
+              timestamp: '2016-06-20T12:08:10.000Z',
+              xray_trace_id: '1-5759e988-bd862e3fe1be46a994272793',
+              value: [1, 2],
+            })
+          );
+        });
+
+        it('should serialize using both the existing replacer and the customer-provided one', () => {
+          // Prepare
+          const jsonReplacerFn: CustomJsonReplacerFn = (
+            _: string,
+            value: unknown
+          ) => {
+            if (value instanceof Set || value instanceof Map) {
+              return [...value];
+            }
+
+            return value;
+          };
+
+          const logger = new Logger({ jsonReplacerFn });
+          const consoleSpy = jest.spyOn(
+            logger['console'],
+            getConsoleMethod(methodOfLogger)
+          );
+
+          const message = `This is an ${methodOfLogger} log with Set value`;
+          const logItem = { value: new Set([1, 2]), number: BigInt(42) };
+
+          // Act
+          logger[methodOfLogger](message, logItem);
+
+          // Assess
+          expect(consoleSpy).toHaveBeenCalledTimes(1);
+          expect(consoleSpy).toHaveBeenNthCalledWith(
+            1,
+            JSON.stringify({
+              level: methodOfLogger.toUpperCase(),
+              message: message,
+              sampling_rate: 0,
+              service: 'hello-world',
+              timestamp: '2016-06-20T12:08:10.000Z',
+              xray_trace_id: '1-5759e988-bd862e3fe1be46a994272793',
+              value: [1, 2],
+              number: '42',
+            })
+          );
+        });
+
+        it('should pass the JSON customer-provided replacer function to child loggers', () => {
+          // Prepare
+          const jsonReplacerFn: CustomJsonReplacerFn = (
+            _: string,
+            value: unknown
+          ) => (value instanceof Set ? [...value] : value);
+          const logger = new Logger({ jsonReplacerFn });
+
+          // Act
+          const childLogger = logger.createChild();
+
+          // Assess
+          expect(() =>
+            childLogger.info('foo', { foo: new Set([1, 2]) })
+          ).not.toThrow();
+        });
       });
     }
   );