From dd11ab4e38b170f1045758bb26625a5294b95993 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 25 Jul 2023 14:39:50 +0200 Subject: [PATCH 01/15] fix(parameters): make cache aware of single vs multiple calls Signed-off-by: heitorlessa --- aws_lambda_powertools/utilities/parameters/base.py | 2 +- aws_lambda_powertools/utilities/parameters/types.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/aws_lambda_powertools/utilities/parameters/base.py b/aws_lambda_powertools/utilities/parameters/base.py index e4be9d33cdc..78bf865faf0 100644 --- a/aws_lambda_powertools/utilities/parameters/base.py +++ b/aws_lambda_powertools/utilities/parameters/base.py @@ -27,7 +27,7 @@ from aws_lambda_powertools.shared import constants, user_agent from aws_lambda_powertools.shared.functions import resolve_max_age -from aws_lambda_powertools.utilities.parameters.types import TransformOptions +from aws_lambda_powertools.utilities.parameters.types import RecursiveOptions, TransformOptions from .exceptions import GetParameterError, TransformParameterError diff --git a/aws_lambda_powertools/utilities/parameters/types.py b/aws_lambda_powertools/utilities/parameters/types.py index 6a15873c496..2dbf1593d72 100644 --- a/aws_lambda_powertools/utilities/parameters/types.py +++ b/aws_lambda_powertools/utilities/parameters/types.py @@ -1,3 +1,4 @@ from typing_extensions import Literal TransformOptions = Literal["json", "binary", "auto", None] +RecursiveOptions = Literal[True, False] From ef10c4a65bd53f52cf5194f3390459444b58a473 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Tue, 25 Jul 2023 15:16:51 +0200 Subject: [PATCH 02/15] chore: cleanup, add test for single and nested Signed-off-by: heitorlessa --- aws_lambda_powertools/utilities/parameters/base.py | 2 +- aws_lambda_powertools/utilities/parameters/types.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/aws_lambda_powertools/utilities/parameters/base.py b/aws_lambda_powertools/utilities/parameters/base.py index 78bf865faf0..e4be9d33cdc 100644 --- a/aws_lambda_powertools/utilities/parameters/base.py +++ b/aws_lambda_powertools/utilities/parameters/base.py @@ -27,7 +27,7 @@ from aws_lambda_powertools.shared import constants, user_agent from aws_lambda_powertools.shared.functions import resolve_max_age -from aws_lambda_powertools.utilities.parameters.types import RecursiveOptions, TransformOptions +from aws_lambda_powertools.utilities.parameters.types import TransformOptions from .exceptions import GetParameterError, TransformParameterError diff --git a/aws_lambda_powertools/utilities/parameters/types.py b/aws_lambda_powertools/utilities/parameters/types.py index 2dbf1593d72..6a15873c496 100644 --- a/aws_lambda_powertools/utilities/parameters/types.py +++ b/aws_lambda_powertools/utilities/parameters/types.py @@ -1,4 +1,3 @@ from typing_extensions import Literal TransformOptions = Literal["json", "binary", "auto", None] -RecursiveOptions = Literal[True, False] From ff76bddc50f9d862ce9e4abde1708b2938dfb132 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 2 Aug 2023 15:35:10 +0200 Subject: [PATCH 03/15] fix(docs): address all warnings for internal links --- docs/index.md | 22 +++++++++++----------- docs/maintainers.md | 2 +- docs/tutorial/index.md | 4 ++-- docs/upgrade.md | 2 +- docs/utilities/idempotency.md | 16 ++++++++-------- docs/utilities/middleware_factory.md | 2 +- docs/we_made_this.md | 2 +- 7 files changed, 25 insertions(+), 25 deletions(-) diff --git a/docs/index.md b/docs/index.md index d0269260711..f44f258cdff 100644 --- a/docs/index.md +++ b/docs/index.md @@ -705,18 +705,18 @@ Core utilities such as Tracing, Logging, Metrics, and Event Handler will be avai | Environment variable | Description | Utility | Default | | ----------------------------------------- | -------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------- | --------------------- | | **POWERTOOLS_SERVICE_NAME** | Sets service name used for tracing namespace, metrics dimension and structured logging | All | `"service_undefined"` | -| **POWERTOOLS_METRICS_NAMESPACE** | Sets namespace used for metrics | [Metrics](./core/metrics){target="_blank"} | `None` | -| **POWERTOOLS_TRACE_DISABLED** | Explicitly disables tracing | [Tracing](./core/tracer){target="_blank"} | `false` | -| **POWERTOOLS_TRACER_CAPTURE_RESPONSE** | Captures Lambda or method return as metadata. | [Tracing](./core/tracer){target="_blank"} | `true` | -| **POWERTOOLS_TRACER_CAPTURE_ERROR** | Captures Lambda or method exception as metadata. | [Tracing](./core/tracer){target="_blank"} | `true` | -| **POWERTOOLS_TRACE_MIDDLEWARES** | Creates sub-segment for each custom middleware | [Middleware factory](./utilities/middleware_factory){target="_blank"} | `false` | -| **POWERTOOLS_LOGGER_LOG_EVENT** | Logs incoming event | [Logging](./core/logger){target="_blank"} | `false` | -| **POWERTOOLS_LOGGER_SAMPLE_RATE** | Debug log sampling | [Logging](./core/logger){target="_blank"} | `0` | -| **POWERTOOLS_LOG_DEDUPLICATION_DISABLED** | Disables log deduplication filter protection to use Pytest Live Log feature | [Logging](./core/logger){target="_blank"} | `false` | -| **POWERTOOLS_PARAMETERS_MAX_AGE** | Adjust how long values are kept in cache (in seconds) | [Parameters](./utilities/parameters/#adjusting-cache-ttl){target="_blank"} | `5` | -| **POWERTOOLS_PARAMETERS_SSM_DECRYPT** | Sets whether to decrypt or not values retrieved from AWS SSM Parameters Store | [Parameters](./utilities/parameters/#ssmprovider){target="_blank"} | `false` | +| **POWERTOOLS_METRICS_NAMESPACE** | Sets namespace used for metrics | [Metrics](./core/metrics.md){target="_blank"} | `None` | +| **POWERTOOLS_TRACE_DISABLED** | Explicitly disables tracing | [Tracing](./core/tracer.md){target="_blank"} | `false` | +| **POWERTOOLS_TRACER_CAPTURE_RESPONSE** | Captures Lambda or method return as metadata. | [Tracing](./core/tracer.md){target="_blank"} | `true` | +| **POWERTOOLS_TRACER_CAPTURE_ERROR** | Captures Lambda or method exception as metadata. | [Tracing](./core/tracer.md){target="_blank"} | `true` | +| **POWERTOOLS_TRACE_MIDDLEWARES** | Creates sub-segment for each custom middleware | [Middleware factory](./utilities/middleware_factory.md){target="_blank"} | `false` | +| **POWERTOOLS_LOGGER_LOG_EVENT** | Logs incoming event | [Logging](./core/logger.md){target="_blank"} | `false` | +| **POWERTOOLS_LOGGER_SAMPLE_RATE** | Debug log sampling | [Logging](./core/logger.md){target="_blank"} | `0` | +| **POWERTOOLS_LOG_DEDUPLICATION_DISABLED** | Disables log deduplication filter protection to use Pytest Live Log feature | [Logging](./core/logger.md){target="_blank"} | `false` | +| **POWERTOOLS_PARAMETERS_MAX_AGE** | Adjust how long values are kept in cache (in seconds) | [Parameters](./utilities/parameters.md#adjusting-cache-ttl){target="_blank"} | `5` | +| **POWERTOOLS_PARAMETERS_SSM_DECRYPT** | Sets whether to decrypt or not values retrieved from AWS SSM Parameters Store | [Parameters](./utilities/parameters.md#ssmprovider){target="_blank"} | `false` | | **POWERTOOLS_DEV** | Increases verbosity across utilities | Multiple; see [POWERTOOLS_DEV effect below](#optimizing-for-non-production-environments) | `false` | -| **LOG_LEVEL** | Sets logging level | [Logging](./core/logger){target="_blank"} | `INFO` | +| **LOG_LEVEL** | Sets logging level | [Logging](./core/logger.md){target="_blank"} | `INFO` | ### Optimizing for non-production environments diff --git a/docs/maintainers.md b/docs/maintainers.md index 8f3a1980141..c1fbfbb64d1 100644 --- a/docs/maintainers.md +++ b/docs/maintainers.md @@ -251,7 +251,7 @@ These are some questions to keep in mind when drafting your first or future rele - Can customers understand at a high level what changed in this release? - Is there a link to the documentation where they can read more about each main change? -- Are there any graphics or [code snippets](carbon.now.sh/) that can enhance readability? +- Are there any graphics or [code snippets](https://carbon.now.sh/) that can enhance readability? - Are we calling out any key contributor(s) to this release? - All contributors are automatically credited, use this as an exceptional case to feature them diff --git a/docs/tutorial/index.md b/docs/tutorial/index.md index ae4fd4a7181..bf9afdaf70b 100644 --- a/docs/tutorial/index.md +++ b/docs/tutorial/index.md @@ -226,7 +226,7 @@ For this to work, we could create a new Lambda function to handle incoming reque We could group similar routes and intents, separate read and write operations resulting in fewer functions. It doesn't address the boilerplate routing code, but maybe it will be easier to add additional URLs. ???+ info "Info: You might be already asking yourself about mono vs micro-functions" - If you want a more detailed explanation of these two approaches, head over to the [trade-offs on each approach](../core/event_handler/api_gateway/#considerations){target="_blank"} later. + If you want a more detailed explanation of these two approaches, head over to the [trade-offs on each approach](../core/event_handler/api_gateway.md#considerations){target="_blank"} later. A first attempt at the routing logic might look similar to the following code snippet. @@ -773,7 +773,7 @@ Another subtle difference is that you can now run your Lambda functions and unit Powertools for AWS Lambda (Python) optimizes for Lambda compute environment. As such, we add these and other common approaches to accelerate your development, so you don't worry about implementing every cross-cutting concern. ???+ tip - You can [opt-out some of these behaviours](../core/tracer/#advanced){target="_blank"} like disabling response capturing, explicitly patching only X modules, etc. + You can [opt-out some of these behaviours](../core/tracer.md#advanced){target="_blank"} like disabling response capturing, explicitly patching only X modules, etc. Repeat the process of building, deploying, and invoking your application via the API endpoint. Within the [AWS X-Ray Console](https://console.aws.amazon.com/xray/home#/traces/){target="_blank"}, you should see a similar view: diff --git a/docs/upgrade.md b/docs/upgrade.md index 064c6a9657a..d9602da1a53 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -44,7 +44,7 @@ You can migrate to `BatchProcessor` with the following changes: 1. If you use **`sqs_batch_decorator`**, change to **`batch_processor`** decorator 2. If you use **`PartialSQSProcessor`**, change to **`BatchProcessor`** -3. [Enable **`ReportBatchItemFailures`** in your Lambda Event Source](../utilities/batch#required-resources){target="_blank"} +3. [Enable **`ReportBatchItemFailures`** in your Lambda Event Source](./utilities/batch.md#required-resources){target="_blank"} 4. Change your Lambda Handler to return the new response format === "[Before] Decorator" diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 1b2128cd7f0..6e5c47af6fc 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -496,14 +496,14 @@ When using DynamoDB as a persistence layer, you can alter the attribute names by Idempotent decorator can be further configured with **`IdempotencyConfig`** as seen in the previous example. These are the available options for further configuration -| Parameter | Default | Description | -| ------------------------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **event_key_jmespath** | `""` | JMESPath expression to extract the idempotency key from the event record using [built-in functions](/utilities/jmespath_functions){target="_blank"} | -| **payload_validation_jmespath** | `""` | JMESPath expression to validate whether certain parameters have changed in the event while the event payload | -| **raise_on_no_idempotency_key** | `False` | Raise exception if no idempotency key was found in the request | -| **expires_after_seconds** | 3600 | The number of seconds to wait before a record is expired | -| **use_local_cache** | `False` | Whether to locally cache idempotency results | -| **local_cache_max_items** | 256 | Max number of items to store in local cache | +| Parameter | Default | Description | +| ------------------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **event_key_jmespath** | `""` | JMESPath expression to extract the idempotency key from the event record using [built-in functions](./jmespath_functions.md#built-in-jmespath-functions){target="_blank"} | +| **payload_validation_jmespath** | `""` | JMESPath expression to validate whether certain parameters have changed in the event while the event payload | +| **raise_on_no_idempotency_key** | `False` | Raise exception if no idempotency key was found in the request | +| **expires_after_seconds** | 3600 | The number of seconds to wait before a record is expired | +| **use_local_cache** | `False` | Whether to locally cache idempotency results | +| **local_cache_max_items** | 256 | Max number of items to store in local cache | | **hash_function** | `md5` | Function to use for calculating hashes, as provided by [hashlib](https://docs.python.org/3/library/hashlib.html){target="_blank" rel="nofollow"} in the standard library. | ### Handling concurrent executions with the same payload diff --git a/docs/utilities/middleware_factory.md b/docs/utilities/middleware_factory.md index 41670025b1e..6b0847ddb47 100644 --- a/docs/utilities/middleware_factory.md +++ b/docs/utilities/middleware_factory.md @@ -92,7 +92,7 @@ If you are making use of [Tracer](../core/tracer.md){target="_blank"}, you can t This makes use of an existing Tracer instance that you may have initialized anywhere in your code. ???+ warning - You must [enable Active Tracing](../core/tracer/#permissions){target="_blank"} in your Lambda function when using this feature, otherwise Lambda cannot send traces to XRay. + You must [enable Active Tracing](../core/tracer.md#permissions){target="_blank"} in your Lambda function when using this feature, otherwise Lambda cannot send traces to XRay. === "getting_started_middleware_tracer_function.py" ```python hl_lines="8 14 15 36" diff --git a/docs/we_made_this.md b/docs/we_made_this.md index a2f3d0e8022..a1229ec99aa 100644 --- a/docs/we_made_this.md +++ b/docs/we_made_this.md @@ -82,7 +82,7 @@ This article walks through how CyberArk uses Powertools to implement Feature Fla ### Designing for Idempotency -> **Author: [Valentin Dreismann](linkedin.com/in/valentin-dreismann-69694b16a){target="_blank" rel="nofollow"}** :material-linkedin: +> **Author: [Valentin Dreismann](https://linkedin.com/in/valentin-dreismann-69694b16a){target="_blank" rel="nofollow"}** :material-linkedin: This article outlines the importance of idempotency, key considerations and trade-offs when implementing in your systems. From 5ccacde7eb9bd64ec1fa354c3011b10450969f20 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 2 Aug 2023 15:37:59 +0200 Subject: [PATCH 04/15] docs(nav): add roadmap link in home section --- mkdocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/mkdocs.yml b/mkdocs.yml index 77322a535be..0d8a40abeb4 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -11,6 +11,7 @@ nav: - API reference: api/" target="_blank - Upgrade guide: upgrade.md - We Made This (Community): we_made_this.md + - Roadmap: roadmap.md - Features: - core/tracer.md - core/logger.md From 5927dc4005fc154718fa2d25376b438f0b838089 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 2 Aug 2023 15:57:10 +0200 Subject: [PATCH 05/15] docs(config): enable task list Signed-off-by: heitorlessa --- mkdocs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mkdocs.yml b/mkdocs.yml index 0d8a40abeb4..49bf5a347e5 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -96,6 +96,8 @@ markdown_extensions: - name: mermaid class: mermaid format: !!python/name:pymdownx.superfences.fence_code_format + - pymdownx.tasklist: + custom_checkbox: true copyright: Copyright © 2023 Amazon Web Services From b3e94183929fda600deb171e3d04bbe174a27f48 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 2 Aug 2023 15:57:32 +0200 Subject: [PATCH 06/15] docs(roadmap): cleanup intro, update observability providers Signed-off-by: heitorlessa --- docs/roadmap.md | 49 ++++++++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/docs/roadmap.md b/docs/roadmap.md index 6b4807f6ddc..72f95fdc95f 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -2,15 +2,30 @@ ## Overview -This is our public roadmap that outlines the high level direction we are working towards, namely [Themes](#themes). We update this document when our priorities change: security and stability is our top priority. +Our public roadmap outlines the high level direction we are working towards, namely [Themes](#themes). We update this document when our priorities change: security and stability is our top priority. -[See our latest list of activities ยป](https://github.com/orgs/aws-powertools/projects/3?query=is%3Aopen+sort%3Aupdated-desc){target="_blank"} +!!! info "For most up-to-date information, see our [board of activities](https://github.com/orgs/aws-powertools/projects/3/views/2?query=is%3Aopen+sort%3Aupdated-desc){target="_blank"}." ## Themes -!!! info "Operational Excellence is priority number 1." +Operational Excellence is priority number 1. This means bug fixing, stability, security, customer's support, and governance will take precedence above all else. -Themes are key activities maintainers are focusing on, besides bug reports. These are updated periodically and you can find the latest [under Epics in our public board](https://github.com/orgs/aws-powertools/projects/3/views/11?query=is%3Aopen+sort%3Aupdated-desc){target="_blank"}. +**What are themes?** + +They are key activities maintainers are focusing on. These are updated periodically and you can find the latest [under Themes in our public board](https://github.com/orgs/aws-powertools/projects/3/views/11?query=is%3Aopen+sort%3Aupdated-desc){target="_blank"}. + +### Observability providers + +We want to extend Tracer, Metrics, and Logger to support any [AWS Lambda certified observability partner](https://go.aws/3HtU6CZ){target="_blank"}, along with OpenTelemetry. + +At launch, we will support the top [two most requested observability providers](https://github.com/aws-powertools/powertools-lambda-python/issues/1433). OpenTelemetry will be a fast follow-up as we need to decide on a stable solution to cold start penalty. + +**Major updates** + +- [x] [Document how customers can use any provider with Logger](https://docs.powertools.aws.dev/lambda/python/latest/core/logger/#observability-providers) +- [x] [Extend Metrics to add support for any Provider](https://github.com/aws-powertools/powertools-lambda-python/pull/2194) +- [ ] [Extend Tracer to add support for any Provider](https://github.com/aws-powertools/powertools-lambda-python/issues/2030) +- [ ] Investigate alternative solution to OpenTelemetry cold start performance ### Increased end-to-end coverage @@ -18,10 +33,6 @@ We continue to work on increasing end-to-end coverage for all features. Our main Some Lambda Event Sources require clusters (e.g., MSK) leading to additional delays of up to 30m in the end-to-end feedback loop. We need a RFC to start discussing viable options, and whether we should publish JSON Schemas from identified contracts. -### Observability providers - -We want to extend Tracer, Metrics, and Logger to support any [observability provider](https://github.com/aws-powertools/powertools-lambda-python/issues/1433){target="_blank"}. We need a RFC to define a contract and to identify two most requested observability providers that we can work with as an initial step. - ### Lambda Layer in release notes We want to publish a JSON with a map of region and Lambda Layer ARN as a GitHub Release Note asset. @@ -56,11 +67,11 @@ graph LR Within our [public board](https://github.com/orgs/aws-powertools/projects/3/views/1?query=is%3Aopen+sort%3Aupdated-desc){target="_blank"}, you'll see the following values in the `Status` column: -* **Ideas**. Incoming and existing feature requests that are not being actively considered yet. These will be reviewed when bandwidth permits. -* **Backlog**. Accepted feature requests or enhancements that we want to work on. -* **Working on it**. Features or enhancements we're currently either researching or implementing it. -* **Coming soon**. Any feature, enhancement, or bug fixes that have been merged and are coming in the next release. -* **Shipped**. Features or enhancements that are now available in the most recent release. +- **Ideas**. Incoming and existing feature requests that are not being actively considered yet. These will be reviewed when bandwidth permits. +- **Backlog**. Accepted feature requests or enhancements that we want to work on. +- **Working on it**. Features or enhancements we're currently either researching or implementing it. +- **Coming soon**. Any feature, enhancement, or bug fixes that have been merged and are coming in the next release. +- **Shipped**. Features or enhancements that are now available in the most recent release. > Tasks or issues with empty `Status` will be categorized in upcoming review cycles. @@ -82,12 +93,12 @@ graph LR Our end-to-end mechanism follows four major steps: -* **Feature Request**. Ideas start with a [feature request](https://github.com/aws-powertools/powertools-lambda-python/issues/new?assignees=&labels=feature-request%2Ctriage&template=feature_request.yml&title=Feature+request%3A+TITLE){target="_blank"} to outline their use case at a high level. For complex use cases, maintainers might ask for/write a RFC. - * Maintainers review requests based on [project tenets](index.md#tenets){target="_blank"}, customers reaction (๐Ÿ‘), and use cases. -* **Request-for-comments (RFC)**. Design proposals use our [RFC issue template](https://github.com/aws-powertools/powertools-lambda-python/issues/new?assignees=&labels=RFC%2Ctriage&template=rfc.yml&title=RFC%3A+TITLE){target="_blank"} to describe its implementation, challenges, developer experience, dependencies, and alternative solutions. - * This helps refine the initial idea with community feedback before a decision is made. -* **Decision**. After carefully reviewing and discussing them, maintainers make a final decision on whether to start implementation, defer or reject it, and update everyone with the next steps. -* **Implementation**. For approved features, maintainers give priority to the original authors for implementation unless it is a sensitive task that is best handled by maintainers. +- **Feature Request**. Ideas start with a [feature request](https://github.com/aws-powertools/powertools-lambda-python/issues/new?assignees=&labels=feature-request%2Ctriage&template=feature_request.yml&title=Feature+request%3A+TITLE){target="_blank"} to outline their use case at a high level. For complex use cases, maintainers might ask for/write a RFC. + - Maintainers review requests based on [project tenets](index.md#tenets){target="_blank"}, customers reaction (๐Ÿ‘), and use cases. +- **Request-for-comments (RFC)**. Design proposals use our [RFC issue template](https://github.com/aws-powertools/powertools-lambda-python/issues/new?assignees=&labels=RFC%2Ctriage&template=rfc.yml&title=RFC%3A+TITLE){target="_blank"} to describe its implementation, challenges, developer experience, dependencies, and alternative solutions. + - This helps refine the initial idea with community feedback before a decision is made. +- **Decision**. After carefully reviewing and discussing them, maintainers make a final decision on whether to start implementation, defer or reject it, and update everyone with the next steps. +- **Implementation**. For approved features, maintainers give priority to the original authors for implementation unless it is a sensitive task that is best handled by maintainers. ???+ info "See [Maintainers](https://github.com/aws-powertools/powertools-lambda-python/blob/develop/MAINTAINERS.md){target="_blank"} document to understand how we triage issues and pull requests, labels and governance." From f61ddc19d668b0c89fee93b22f7bfce93590efd5 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 2 Aug 2023 15:59:08 +0200 Subject: [PATCH 07/15] docs(roadmap): close e2e cov, strict typing Signed-off-by: heitorlessa --- docs/roadmap.md | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/docs/roadmap.md b/docs/roadmap.md index 72f95fdc95f..78b2dc959a9 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -27,12 +27,6 @@ At launch, we will support the top [two most requested observability providers]( - [ ] [Extend Tracer to add support for any Provider](https://github.com/aws-powertools/powertools-lambda-python/issues/2030) - [ ] Investigate alternative solution to OpenTelemetry cold start performance -### Increased end-to-end coverage - -We continue to work on increasing end-to-end coverage for all features. Our main challenge is testing contracts for Lambda Event Sources (Parser, Event Source Data Classes) due to the lack of an official JSON schema. - -Some Lambda Event Sources require clusters (e.g., MSK) leading to additional delays of up to 30m in the end-to-end feedback loop. We need a RFC to start discussing viable options, and whether we should publish JSON Schemas from identified contracts. - ### Lambda Layer in release notes We want to publish a JSON with a map of region and Lambda Layer ARN as a GitHub Release Note asset. @@ -41,13 +35,8 @@ As of V2, we prioritize Lambda Layers being available before release notes are o This means we have room to include a JSON map for Lambda Layers and facilitate automation for customers wanting the latest version as soon as it's available. -### Strict typing - -We want to enable MyPy strict mode against the code base. We need a RFC to identify most critical areas to start, and do so gradually as to not impact new features and enhancements in parallel. - -This also means bringing `typing-extensions` as a runtime dependency to ensure complete coverage across all Python versions. Future wise, we might be able to experiment with [MyPyC](https://github.com/mypyc/mypyc){target="_blank" rel="nofollow"} to compile less performing parts of the code base as a C-Extension. - ### New utilities + With V2 launched, we want to resume working on new utilities, specifically but not limited to the most commonly asked: **(1)** [Sensitive Data Masking](https://github.com/aws-powertools/powertools-lambda-python/issues/1173){target="_blank"}, **(2)** [Integration/End-to-end Testing](https://github.com/aws-powertools/powertools-lambda-python/issues/1169){target="_blank"}, and **(3)** [Event Bridge](https://github.com/aws-powertools/powertools-lambda-python/issues/1168){target="_blank"}. From 7f9480ddf44b469e2518c62e21e942ae0b902d19 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 2 Aug 2023 16:15:42 +0200 Subject: [PATCH 08/15] docs(roadmap): updates for Layer; remove new utilities Signed-off-by: heitorlessa --- docs/roadmap.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/roadmap.md b/docs/roadmap.md index 78b2dc959a9..24275117a9f 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -35,10 +35,10 @@ As of V2, we prioritize Lambda Layers being available before release notes are o This means we have room to include a JSON map for Lambda Layers and facilitate automation for customers wanting the latest version as soon as it's available. -### New utilities +**Major updates** - -With V2 launched, we want to resume working on new utilities, specifically but not limited to the most commonly asked: **(1)** [Sensitive Data Masking](https://github.com/aws-powertools/powertools-lambda-python/issues/1173){target="_blank"}, **(2)** [Integration/End-to-end Testing](https://github.com/aws-powertools/powertools-lambda-python/issues/1169){target="_blank"}, and **(3)** [Event Bridge](https://github.com/aws-powertools/powertools-lambda-python/issues/1168){target="_blank"}. +- [x] Create secure mechanism to upload signed assets to GitHub Release Notes +- [ ] Create feature request to agree on JSON structure and asset name ### Open iteration planning From acb3a0f6d5570587f3cbf68da54ed7fc0e2b06a4 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 2 Aug 2023 16:16:13 +0200 Subject: [PATCH 09/15] docs(roadmap): add sensitive data masking Signed-off-by: heitorlessa --- docs/roadmap.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/roadmap.md b/docs/roadmap.md index 24275117a9f..73a1424be5a 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -27,6 +27,16 @@ At launch, we will support the top [two most requested observability providers]( - [ ] [Extend Tracer to add support for any Provider](https://github.com/aws-powertools/powertools-lambda-python/issues/2030) - [ ] Investigate alternative solution to OpenTelemetry cold start performance +### Sensitive Data Masking + +Data Masking will be a new utility to mask/unmask sensitive data using encryption providers. It's the second most voted feature request (behind [Observability Providers](#observability-providers)). + +**Major updates** + +- [x] [RFC to agree on design and MVP](https://github.com/aws-powertools/powertools-lambda-python/issues/1858) +- [ ] [POC with AWS KMS as the default provider](https://github.com/aws-powertools/powertools-lambda-python/pull/2197) +- [ ] Documentation to guide customers how to bring their own provider (e.g., `ItsDangerous`) + ### Lambda Layer in release notes We want to publish a JSON with a map of region and Lambda Layer ARN as a GitHub Release Note asset. From c556de272f3ab0b56d824e46d93339592e0b79d8 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 2 Aug 2023 16:17:04 +0200 Subject: [PATCH 10/15] docs(roadmap): add authentication utility Signed-off-by: heitorlessa --- docs/roadmap.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/roadmap.md b/docs/roadmap.md index 73a1424be5a..8f81ecc15e5 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -54,6 +54,17 @@ This means we have room to include a JSON map for Lambda Layers and facilitate a We want to experiment running a bi-weekly audio channel on [Discord](https://discord.gg/B8zZKbbyET){target="_blank" rel="nofollow"} to help us prioritize backlog in real-time. Depending on attendance, we might switch to run an office hours instead. +### Authentication (SigV4) + +[During customers interview](https://github.com/aws-powertools/powertools-lambda-python#connect), we hear that signing requests using [AWS SigV4](https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html) could be easier. + +Since JWT is a close second, this new utility would cover higher level functions to sign and verify requests more easily. + +**Major updates** + +- [ ] RFC to outline challenges, alternative solutions and desired experience +- [ ] MVP based off RFC + ## Roadmap status definition
From 29f882a410d9b3a90d793632d73f578411135bae Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 2 Aug 2023 16:28:47 +0200 Subject: [PATCH 11/15] docs(roadmap): add event handler revamp Signed-off-by: heitorlessa --- docs/roadmap.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/roadmap.md b/docs/roadmap.md index 8f81ecc15e5..bc28147b465 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -37,6 +37,20 @@ Data Masking will be a new utility to mask/unmask sensitive data using encryptio - [ ] [POC with AWS KMS as the default provider](https://github.com/aws-powertools/powertools-lambda-python/pull/2197) - [ ] Documentation to guide customers how to bring their own provider (e.g., `ItsDangerous`) +### Revamp Event Handler + +Event Handler provides lightweight routing for both [**REST**: Amazon API Gateway, Amazon Elastic Load Balancer and AWS Lambda Function URL](./core/event_handler/api_gateway.md), and [**GraphQL**: AWS AppSync](./core/event_handler/appsync.md). + +Based on customers feedback, we want to provide middleware authoring support for cross-cutting concerns. For REST APIs, we are also looking into auto-generate OpenAPI Schemas and a SwaggerUI route. For GraphQL, we are working on supporting batch invocations (N+1 problem) along with partial failure support. + +**Major updates** + +- [x] [Agree on experience for middleware support](https://github.com/aws-powertools/powertools-lambda-python/issues/953#issuecomment-1450223155) +- [x] [RFC to outline initial thoughts on OpenAPI integration](https://github.com/aws-powertools/powertools-lambda-python/issues/2421) +- [ ] MVP for REST middleware +- [ ] MVP for OpenAPI and SwaggerUI +- [ ] [MVP for AppSync Batch invoke and partial failure support](https://github.com/aws-powertools/powertools-lambda-python/pull/1998) + ### Lambda Layer in release notes We want to publish a JSON with a map of region and Lambda Layer ARN as a GitHub Release Note asset. From 3d76fbbec02b3f35cb0629792bad08fb73654aa2 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 2 Aug 2023 17:10:50 +0200 Subject: [PATCH 12/15] docs(roadmap): add operational metrics Signed-off-by: heitorlessa --- docs/roadmap.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/docs/roadmap.md b/docs/roadmap.md index bc28147b465..16f88c8e8d8 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -70,7 +70,7 @@ We want to experiment running a bi-weekly audio channel on [Discord](https://dis ### Authentication (SigV4) -[During customers interview](https://github.com/aws-powertools/powertools-lambda-python#connect), we hear that signing requests using [AWS SigV4](https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html) could be easier. +[During customers interview](https://github.com/aws-powertools/powertools-lambda-python#connect){target="_blank"}, we hear that signing requests using [AWS SigV4](https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html){target="_blank"} could be easier. Since JWT is a close second, this new utility would cover higher level functions to sign and verify requests more easily. @@ -79,6 +79,21 @@ Since JWT is a close second, this new utility would cover higher level functions - [ ] RFC to outline challenges, alternative solutions and desired experience - [ ] MVP based off RFC +### Enhanced operational metrics + +[Through customers interview](https://github.com/aws-powertools/powertools-lambda-python#connect){target="_blank"}, [Discord](https://discord.gg/B8zZKbbyET){target="_blank" rel="nofollow"}, and [1:1 customer enablement](https://github.com/aws-powertools/powertools-lambda-python#connect){target="_blank"}, we noticed customers often create the same set of custom operational metrics. + +We want to make this easier by extending certain utilities to accept a `metrics` instance and metrics configuration (what metrics to create). It would be opt-in due to costs associated with creating metrics. + +!!! question "Got ideas for custom metrics? Open up a [feature request](https://github.com/aws-powertools/powertools-lambda-python/issues/new?assignees=&labels=feature-request%2Ctriage&projects=&template=feature_request.yml&title=Feature+request%3A+TITLE)" + +**Major updates** + +- [ ] RFC to outline metrics for Batch (_e.g., Failed items, Batch size_) +- [ ] RFC to outline metrics for Feature flags (_e.g., matched rules_) +- [ ] RFC to outline metrics for Event Handler (_e.g., validation errors_ ) +- [ ] RFC to outline metrics for Idempotency (_e.g., cache hit_) + ## Roadmap status definition
From 8643b294fdb6a9cadea0d3d7dfb3f5d8485496c7 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 2 Aug 2023 17:22:51 +0200 Subject: [PATCH 13/15] docs(roadmap): add office hours --- docs/roadmap.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/roadmap.md b/docs/roadmap.md index 16f88c8e8d8..f272fe80d68 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -64,9 +64,17 @@ This means we have room to include a JSON map for Lambda Layers and facilitate a - [x] Create secure mechanism to upload signed assets to GitHub Release Notes - [ ] Create feature request to agree on JSON structure and asset name -### Open iteration planning +### Office hours -We want to experiment running a bi-weekly audio channel on [Discord](https://discord.gg/B8zZKbbyET){target="_blank" rel="nofollow"} to help us prioritize backlog in real-time. Depending on attendance, we might switch to run an office hours instead. +We heard from [customers](https://github.com/aws-powertools/powertools-lambda-python#connect){target="_blank"} that Powertools for AWS Lambda and its community can move faster than they are able to catch up. While documentation and release notes take these into account, they notice they don't always know advanced tricks, or what other customers tend to do in similar situations. + +We want to run a monthly office hours to start addressing that, and learn from customers how they're using Powertools and whether or not they need a closer support. + +Timezones being tricky, we plan to experiment with an afternoon slot in Central European that would also cover Middle East, US east coast, and South America. Depending on attendance, we plan to A/B test an Asia friendly one too. + +**Major updates** + +- [ ] Decide whether to use Amazon Chime or Zoom (we had audio setup issues on Discord) ### Authentication (SigV4) From 4e74156e7380e22d52a80b9efd5fabfdbd6591d9 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 2 Aug 2023 18:20:30 +0200 Subject: [PATCH 14/15] docs(roadmap): add v3 Signed-off-by: heitorlessa --- docs/roadmap.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/roadmap.md b/docs/roadmap.md index f272fe80d68..60c479cdf4d 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -102,6 +102,27 @@ We want to make this easier by extending certain utilities to accept a `metrics` - [ ] RFC to outline metrics for Event Handler (_e.g., validation errors_ ) - [ ] RFC to outline metrics for Idempotency (_e.g., cache hit_) +### V3 + +With Python 3.7 reaching [end-of-life in AWS Lambda by the end of the year](https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html), we want to plan some breaking changes. As always, we plan on having ample notice, a detailed upgrade guide, and keep breaking changes to a minimum to ease transition (e.g., it took ~7 months from v2 to surpass v1 downloads). + +For example, these are on our mind but not settled yet until we have a public tracker to discuss what these means in detail. + +- **Parser**: Drop Pydantic v1 +- **Parser**: Deserialize Amazon DynamoDB data types automatically (like Event Source Data Classes) +- **Parameters**: Increase default `max_age` for `get_secret` +- **Event Source Data Classes**: Return sane defaults for any property that has `Optional[]` returns +- **Python 3.7 EOL**: Update PyPi and Layers to only support 3.8 +- **Upgrade tool**: Consider building a CST (Concrete Syntax Tree) tool to ease certain upgrade actions like `pyupgrade` and `django-upgrade` +- **Batch**: Stop at first error for Amazon DynamoDB Streams and Amazon Kinesis Data Streams (e.g., `stop_on_failure=True`) + +**Major updates** + +- [ ] Create an issue to track breaking changes we consider making +- [ ] Create a v3 branch to allow early experimentation +- [ ] Create workflows to allow pre-releases +- [ ] Create a mechanism to keep ideas for breaking change somewhere regardless of v3 + ## Roadmap status definition
From 3064dba3c3ee50ba0a0a9e4d922b6e56e221d9a8 Mon Sep 17 00:00:00 2001 From: heitorlessa Date: Wed, 2 Aug 2023 18:28:52 +0200 Subject: [PATCH 15/15] docs(roadmap): improve wording in observability provider launch --- docs/roadmap.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/roadmap.md b/docs/roadmap.md index 60c479cdf4d..fc8b4441754 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -18,7 +18,9 @@ They are key activities maintainers are focusing on. These are updated periodica We want to extend Tracer, Metrics, and Logger to support any [AWS Lambda certified observability partner](https://go.aws/3HtU6CZ){target="_blank"}, along with OpenTelemetry. -At launch, we will support the top [two most requested observability providers](https://github.com/aws-powertools/powertools-lambda-python/issues/1433). OpenTelemetry will be a fast follow-up as we need to decide on a stable solution to cold start penalty. +At launch, we will support Datadog since it's [most requested observability provider](https://github.com/aws-powertools/powertools-lambda-python/issues/1433). OpenTelemetry will be a fast follow-up as we need to decide on a stable solution to cold start penalty. + +!!! tip "Help us identify which observability providers we should integrate next. Open [feature request](https://github.com/aws-powertools/powertools-lambda-python/issues/new?assignees=&labels=feature-request%2Ctriage&projects=&template=feature_request.yml&title=Feature+request%3A+TITLE){target="_blank"} or by voting `+1` in existing issues" **Major updates** @@ -93,7 +95,7 @@ Since JWT is a close second, this new utility would cover higher level functions We want to make this easier by extending certain utilities to accept a `metrics` instance and metrics configuration (what metrics to create). It would be opt-in due to costs associated with creating metrics. -!!! question "Got ideas for custom metrics? Open up a [feature request](https://github.com/aws-powertools/powertools-lambda-python/issues/new?assignees=&labels=feature-request%2Ctriage&projects=&template=feature_request.yml&title=Feature+request%3A+TITLE)" +!!! question "Got ideas for custom metrics? Open up a [feature request](https://github.com/aws-powertools/powertools-lambda-python/issues/new?assignees=&labels=feature-request%2Ctriage&projects=&template=feature_request.yml&title=Feature+request%3A+TITLE)"{target="_blank"} **Major updates**