From 2a6369a6cf1459e1ea2f9625b4bd650e4531b25b Mon Sep 17 00:00:00 2001 From: Andrea Amorosi Date: Mon, 11 Sep 2023 16:23:33 +0200 Subject: [PATCH 1/2] docs(batch): tracing batch processing --- docs/index.md | 31 ++++++------- .../batch/advancedTracingRecordHandler.ts | 43 +++++++++++++++++++ docs/snippets/batch/customPartialProcessor.ts | 7 +-- docs/stylesheets/extra.css | 41 +++++++++++++----- docs/utilities/batch.md | 12 +++++- mkdocs.yml | 9 ++-- 6 files changed, 108 insertions(+), 35 deletions(-) create mode 100644 docs/snippets/batch/advancedTracingRecordHandler.ts diff --git a/docs/index.md b/docs/index.md index e3f7350ef3..dc80473c9d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -294,20 +294,21 @@ Core utilities such as Tracing, Logging, and Metrics will be available across al ???+ info Explicit parameters take precedence over environment variables -| Environment variable | Description | Utility | Default | -| -------------------------------------------- | ------------------------------------------------------------------------------------------------------------- | ------------------------------------ | ------------------- | -| **POWERTOOLS_SERVICE_NAME** | Sets service name used for tracing namespace, metrics dimension and structured logging | All | `service_undefined` | -| **POWERTOOLS_METRICS_NAMESPACE** | Sets namespace used for metrics | [Metrics](./core/metrics) | `default_namespace` | -| **POWERTOOLS_TRACE_ENABLED** | Explicitly disables tracing | [Tracer](./core/tracer) | `true` | -| **POWERTOOLS_TRACER_CAPTURE_RESPONSE** | Captures Lambda or method return as metadata. | [Tracer](./core/tracer) | `true` | -| **POWERTOOLS_TRACER_CAPTURE_ERROR** | Captures Lambda or method exception as metadata. | [Tracer](./core/tracer) | `true` | -| **POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS** | Captures HTTP(s) requests as segments. | [Tracer](./core/tracer) | `true` | -| **POWERTOOLS_LOGGER_LOG_EVENT** | Logs incoming event | [Logger](./core/logger) | `false` | -| **POWERTOOLS_LOGGER_SAMPLE_RATE** | Debug log sampling | [Logger](./core/logger) | `0` | -| **POWERTOOLS_DEV** | Increase JSON indentation to ease debugging when running functions locally or in a non-production environment | [Logger](./core/logger) | `false` | -| **LOG_LEVEL** | Sets logging level | [Logger](./core/logger) | `INFO` | -| **POWERTOOLS_PARAMETERS_MAX_AGE** | Adjust how long values are kept in cache (in seconds) | [Parameters](./utilities/parameters) | `5` | -| **POWERTOOLS_PARAMETERS_SSM_DECRYPT** | Sets whether to decrypt or not values retrieved from AWS Systems Manager Parameters Store | [Parameters](./utilities/parameters) | `false` | +| Environment variable | Description | Utility | Default | +| -------------------------------------------- | ------------------------------------------------------------------------------------------------------------- | --------------------------------------- | ------------------- | +| **POWERTOOLS_SERVICE_NAME** | Set service name used for tracing namespace, metrics dimension and structured logging | All | `service_undefined` | +| **POWERTOOLS_METRICS_NAMESPACE** | Set namespace used for metrics | [Metrics](core/metrics.md) | `default_namespace` | +| **POWERTOOLS_TRACE_ENABLED** | Explicitly disables tracing | [Tracer](core/tracer.md) | `true` | +| **POWERTOOLS_TRACER_CAPTURE_RESPONSE** | Capture Lambda or method return as metadata. | [Tracer](core/tracer.md) | `true` | +| **POWERTOOLS_TRACER_CAPTURE_ERROR** | Capture Lambda or method exception as metadata. | [Tracer](core/tracer.md) | `true` | +| **POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS** | Capture HTTP(s) requests as segments. | [Tracer](core/tracer.md) | `true` | +| **POWERTOOLS_LOGGER_LOG_EVENT** | Log incoming event | [Logger](core/logger.md) | `false` | +| **POWERTOOLS_LOGGER_SAMPLE_RATE** | Debug log sampling | [Logger](core/logger.md) | `0` | +| **POWERTOOLS_DEV** | Increase JSON indentation to ease debugging when running functions locally or in a non-production environment | [Logger](core/logger.md) | `false` | +| **LOG_LEVEL** | Set logging level | [Logger](core/logger.md) | `INFO` | +| **POWERTOOLS_PARAMETERS_MAX_AGE** | Adjust how long values are kept in cache (in seconds) | [Parameters](utilities/parameters.md) | `5` | +| **POWERTOOLS_PARAMETERS_SSM_DECRYPT** | Set whether to decrypt or not values retrieved from AWS Systems Manager Parameters Store | [Parameters](utilities/parameters.md) | `false` | +| **POWERTOOLS_IDEMPOTENCY_DISABLED** | Disable the Idempotency logic without changing your code, useful for testing | [Idempotency](utilities/idempotency.md) | `false` | Each Utility page provides information on example values and allowed values. @@ -331,7 +332,7 @@ The following companies, among others, use Powertools: ### Sharing your work -Share what you did with Powertools for AWS Lambda (TypeScript) 💞💞. Blog post, workshops, presentation, sample apps and others. Check out what the community has already shared about Powertools for AWS Lambda (TypeScript) [here](./we_made_this). +Share what you did with Powertools for AWS Lambda (TypeScript) 💞💞. Blog post, workshops, presentation, sample apps and others. Check out what the community has already shared about Powertools for AWS Lambda (TypeScript) [here](we_made_this.md). ### Using Lambda Layer diff --git a/docs/snippets/batch/advancedTracingRecordHandler.ts b/docs/snippets/batch/advancedTracingRecordHandler.ts new file mode 100644 index 0000000000..1d5c788b6c --- /dev/null +++ b/docs/snippets/batch/advancedTracingRecordHandler.ts @@ -0,0 +1,43 @@ +import { + BatchProcessor, + EventType, + processPartialResponse, +} from '@aws-lambda-powertools/batch'; +import { Tracer, captureLambdaHandler } from '@aws-lambda-powertools/tracer'; +import middy from '@middy/core'; +import type { + SQSEvent, + SQSRecord, + Context, + SQSBatchResponse, +} from 'aws-lambda'; + +const processor = new BatchProcessor(EventType.SQS); +const tracer = new Tracer({ serviceName: 'serverlessAirline' }); + +const recordHandler = (record: SQSRecord): void => { + const subsegment = tracer.getSegment()?.addNewSubsegment('### recordHandler'); // (1)! + subsegment?.addAnnotation('messageId', record.messageId); // (2)! + + const payload = record.body; + if (payload) { + try { + const item = JSON.parse(payload); + // do something with the item + subsegment?.addMetadata('item', item); + } catch (error) { + subsegment?.addError(error); + throw error; + } + } + + subsegment?.close(); +}; + +export const handler = middy( + async (event: SQSEvent, context: Context): Promise => { + return processPartialResponse(event, recordHandler, processor, { + context, + }); + } +).use(captureLambdaHandler(tracer)); diff --git a/docs/snippets/batch/customPartialProcessor.ts b/docs/snippets/batch/customPartialProcessor.ts index 9a45c92f01..6ebd113282 100644 --- a/docs/snippets/batch/customPartialProcessor.ts +++ b/docs/snippets/batch/customPartialProcessor.ts @@ -5,7 +5,8 @@ import { } from '@aws-sdk/client-dynamodb'; import { marshall } from '@aws-sdk/util-dynamodb'; import { - BasePartialProcessor, + EventType, + BasePartialBatchProcessor, processPartialResponse, } from '@aws-lambda-powertools/batch'; import type { @@ -17,12 +18,12 @@ import type { SQSEvent, Context, SQSBatchResponse } from 'aws-lambda'; const tableName = process.env.TABLE_NAME || 'table-not-found'; -class MyPartialProcessor extends BasePartialProcessor { +class MyPartialProcessor extends BasePartialBatchProcessor { #tableName: string; #client?: DynamoDBClient; public constructor(tableName: string) { - super(); + super(EventType.SQS); this.#tableName = tableName; } diff --git a/docs/stylesheets/extra.css b/docs/stylesheets/extra.css index 1e7d781b9b..d04907563a 100644 --- a/docs/stylesheets/extra.css +++ b/docs/stylesheets/extra.css @@ -1,25 +1,45 @@ -@font-face { - font-size: 0.65rem; +.md-grid { + max-width: 90vw } -.md-grid { - max-width: 81vw; +.highlight .hll { + background-color: lavender; + + [data-md-color-scheme="slate"] { + background-color: rgb(69, 48, 164) + } +} + +.md-typeset table:not([class]) { + font-size: 0.75rem } .md-typeset a { - border-bottom: 0.1px dashed black; + border-bottom: 0.1px dashed black +} + +.md-typeset table:not([class]) { + font-size: 0.75rem +} + +.md-tabs__link { + font-size: 0.8rem +} + +.md-tabs__link--active { + border-bottom: 6px solid white } .md-nav__link--active { - font-weight: bold; + font-weight: bold } .md-typeset .admonition, .md-typeset details { - font-size: 0.70rem; + font-size: 0.70rem } [data-md-color-scheme="slate"] { - --md-typeset-a-color: rgb(28, 152, 152); + --md-typeset-a-color: rgb(28, 152, 152) } .copyMe { @@ -27,6 +47,7 @@ border-bottom: 0.1px dashed black; } -p code { - font-weight: bolder; +p > code, +li > code { + font-weight: bold } \ No newline at end of file diff --git a/docs/utilities/batch.md b/docs/utilities/batch.md index 6b02a2f538..1e359f0d27 100644 --- a/docs/utilities/batch.md +++ b/docs/utilities/batch.md @@ -409,7 +409,7 @@ For these scenarios, you can subclass `BatchProcessor` and quickly override `suc ???+ example Let's suppose you'd like to add a metric named `BatchRecordFailures` for each batch record that failed processing -```typescript hl_lines="5-6 17 21 25 31 35 50-52" title="Extending failure handling mechanism in BatchProcessor" +```typescript hl_lines="17 21 25 31 35" title="Extending failure handling mechanism in BatchProcessor" --8<-- "docs/snippets/batch/extendingFailure.ts" ``` @@ -448,10 +448,18 @@ classDiagram You can then use this class as a context manager, or pass it to `processPartialResponse` to process the records in your Lambda handler function. -```typescript hl_lines="8 12-14 20 29 40 61 72 83 93-95" title="Creating a custom batch processor" +```typescript hl_lines="21 30 41 62 73 84" title="Creating a custom batch processor" --8<-- "docs/snippets/batch/customPartialProcessor.ts" ``` +## Tracing with AWS X-Ray + +You can use Tracer to create subsegments for each batch record processed. When doing so however you need to make + +```ts +--8<-- "docs/snippets/batch/advancedTracingRecordHandler.ts" +``` + ## Testing your code As there is no external calls, you can unit test your code with `BatchProcessor` quite easily. diff --git a/mkdocs.yml b/mkdocs.yml index f5f85643d4..0fdc7b8484 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -10,11 +10,10 @@ nav: - Roadmap: roadmap.md - API reference: api/" target="_blank - We Made This (Community): we_made_this.md - - Core utilities: + - Features: - core/tracer.md - core/logger.md - core/metrics.md - - Utilities: - utilities/parameters.md - utilities/idempotency.md - utilities/batch.md @@ -39,15 +38,12 @@ theme: features: - header.autohide - navigation.sections - - navigation.expand - navigation.top - navigation.instant - navigation.indexes - navigation.tracking - content.code.annotate - content.code.copy - - toc.follow - - toc.integrate - announce.dismiss icon: repo: fontawesome/brands/github @@ -65,6 +61,7 @@ markdown_extensions: - pymdownx.snippets: base_path: "." check_paths: true + restrict_base_path: false - meta - toc: permalink: true @@ -79,6 +76,8 @@ markdown_extensions: - name: mermaid class: mermaid format: !!python/name:pymdownx.superfences.fence_code_format + - pymdownx.tasklist: + custom_checkbox: true copyright: Copyright © 2023 Amazon Web Services From 25a35f33634ac796ba6a2e26e57570e5d03e189f Mon Sep 17 00:00:00 2001 From: Andrea Amorosi Date: Wed, 13 Sep 2023 11:10:50 +0200 Subject: [PATCH 2/2] docs(batch): add guidance on how to trace batch processing --- docs/snippets/batch/advancedTracingRecordHandler.ts | 2 +- docs/utilities/batch.md | 6 +++++- docs/utilities/parameters.md | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/snippets/batch/advancedTracingRecordHandler.ts b/docs/snippets/batch/advancedTracingRecordHandler.ts index 1d5c788b6c..4de43b3198 100644 --- a/docs/snippets/batch/advancedTracingRecordHandler.ts +++ b/docs/snippets/batch/advancedTracingRecordHandler.ts @@ -31,7 +31,7 @@ const recordHandler = (record: SQSRecord): void => { } } - subsegment?.close(); + subsegment?.close(); // (3)! }; export const handler = middy( diff --git a/docs/utilities/batch.md b/docs/utilities/batch.md index 1e359f0d27..ac426546e5 100644 --- a/docs/utilities/batch.md +++ b/docs/utilities/batch.md @@ -454,12 +454,16 @@ You can then use this class as a context manager, or pass it to `processPartialR ## Tracing with AWS X-Ray -You can use Tracer to create subsegments for each batch record processed. When doing so however you need to make +You can use Tracer to create subsegments for each batch record processed. To do so, you can open a new subsegment for each record, and close it when you're done processing it. When adding annotations and metadata to the subsegment, you can do so directly without calling `tracer.setSegment(subsegment)`. This allows you to work with the subsegment directly and avoid having to either pass the parent subsegment around or have to restore the parent subsegment at the end of the record processing. ```ts --8<-- "docs/snippets/batch/advancedTracingRecordHandler.ts" ``` +1. Retrieve the current segment, then create a subsegment for the record being processed +2. You can add annotations and metadata to the subsegment directly without calling `tracer.setSegment(subsegment)` +3. Close the subsegment when you're done processing the record + ## Testing your code As there is no external calls, you can unit test your code with `BatchProcessor` quite easily. diff --git a/docs/utilities/parameters.md b/docs/utilities/parameters.md index 8f98b07c08..c2b844376e 100644 --- a/docs/utilities/parameters.md +++ b/docs/utilities/parameters.md @@ -392,7 +392,7 @@ You can use the `awsSdkV3Client` parameter via any of the available [Provider Cl | [DynamoDBProvider](#dynamodbprovider) | `new DynamoDBClient();` | ???+ question "When is this useful?" - Injecting a custom AWS SDK v3 client allows you to [apply tracing](/core/tracer/#patching-aws-sdk-clients) or make unit/snapshot testing easier, including SDK customizations. + Injecting a custom AWS SDK v3 client allows you to [apply tracing](../core/tracer.md#patching-aws-sdk-clients) or make unit/snapshot testing easier, including SDK customizations. === "SSMProvider" ```typescript hl_lines="5 7"