Skip to content

Commit bde72f7

Browse files
author
Ran Isenberg
committed
feat: RFC: Validate incoming and outgoing events utility #95 
1 parent 96b7b8f commit bde72f7

File tree

10 files changed

+592
-1
lines changed

10 files changed

+592
-1
lines changed

Diff for: ‎README.md

+1
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ A suite of utilities for AWS Lambda Functions that makes tracing with AWS X-Ray,
1414
* **[Metrics](https://awslabs.github.io/aws-lambda-powertools-python/core/metrics/)** - Custom Metrics created asynchronously via CloudWatch Embedded Metric Format (EMF)
1515
* **[Bring your own middleware](https://awslabs.github.io/aws-lambda-powertools-python/utilities/middleware_factory/)** - Decorator factory to create your own middleware to run logic before, and after each Lambda invocation
1616
* **[Parameters utility](https://awslabs.github.io/aws-lambda-powertools-python/utilities/parameters/)** - Retrieve and cache parameter values from Parameter Store, Secrets Manager, or DynamoDB
17+
* **[Validation](https://)** -
1718

1819
### Installation
1920

Diff for: ‎aws_lambda_powertools/validation/__init__.py

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
"""Validation utility
2+
"""

Diff for: ‎aws_lambda_powertools/validation/schemas/__init__.py

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
"""Validation schemas
2+
"""

Diff for: ‎aws_lambda_powertools/validation/schemas/dynamodb.py

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
from datetime import date
2+
from typing import Any, Dict, List, Optional
3+
4+
from pydantic import BaseModel, root_validator
5+
from typing_extensions import Literal
6+
7+
8+
class DynamoScheme(BaseModel):
9+
ApproximateCreationDateTime: date
10+
Keys: Dict[Literal["id"], Dict[Literal["S"], str]]
11+
NewImage: Optional[Dict[str, Any]] = {}
12+
OldImage: Optional[Dict[str, Any]] = {}
13+
SequenceNumber: str
14+
SizeBytes: int
15+
StreamViewType: Literal["NEW_AND_OLD_IMAGES", "KEYS_ONLY", "NEW_IMAGE", "OLD_IMAGE"]
16+
17+
@root_validator
18+
def check_one_image_exists(cls, values):
19+
newimg, oldimg = values.get("NewImage"), values.get("OldImage")
20+
stream_type = values.get("StreamViewType")
21+
if stream_type == "NEW_AND_OLD_IMAGES" and not newimg and not oldimg:
22+
raise TypeError("DynamoDB streams schema failed validation, missing both new & old stream images")
23+
return values
24+
25+
26+
class DynamoRecordSchema(BaseModel):
27+
eventID: str
28+
eventName: Literal["INSERT", "MODIFY", "REMOVE"]
29+
eventVersion: float
30+
eventSource: Literal["aws:dynamodb"]
31+
awsRegion: str
32+
eventSourceARN: str
33+
dynamodb: DynamoScheme
34+
35+
36+
class DynamoDBSchema(BaseModel):
37+
Records: List[DynamoRecordSchema]

Diff for: ‎aws_lambda_powertools/validation/schemas/eventbridge.py

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
from datetime import datetime
2+
from typing import Any, Dict, List
3+
4+
from pydantic import BaseModel
5+
6+
7+
class EventBridgeSchema(BaseModel):
8+
version: str
9+
id: str # noqa: A003,VNE003
10+
source: str
11+
account: int
12+
time: datetime
13+
region: str
14+
resources: List[str]
15+
detail: Dict[str, Any]

Diff for: ‎aws_lambda_powertools/validation/schemas/sqs.py

+60
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
import re
2+
from datetime import datetime
3+
from typing import Dict, List, Optional
4+
5+
from pydantic import BaseModel, root_validator, validator
6+
from typing_extensions import Literal
7+
8+
9+
class SqsAttributesSchema(BaseModel):
10+
ApproximateReceiveCount: str
11+
ApproximateFirstReceiveTimestamp: datetime
12+
MessageDeduplicationId: Optional[str]
13+
MessageGroupId: Optional[str]
14+
SenderId: str
15+
SentTimestamp: datetime
16+
SequenceNumber: Optional[str]
17+
18+
19+
class SqsMsgAttributeSchema(BaseModel):
20+
stringValue: Optional[str]
21+
binaryValue: Optional[str]
22+
stringListValues: List[str] = []
23+
binaryListValues: List[str] = []
24+
dataType: str
25+
26+
@validator("dataType")
27+
def valid_type(cls, v): # noqa: VNE001
28+
pattern = re.compile("Number.*|String.*|Binary.*")
29+
if not pattern.match(v):
30+
raise TypeError("data type is invalid")
31+
return v
32+
33+
@root_validator
34+
def check_str_and_binary_values(cls, values):
35+
binary_val, str_val = values.get("binaryValue", ""), values.get("stringValue", "")
36+
dataType = values.get("dataType")
37+
if not str_val and not binary_val:
38+
raise TypeError("both binaryValue and stringValue are missing")
39+
if dataType.startswith("Binary") and not binary_val:
40+
raise TypeError("binaryValue is missing")
41+
if (dataType.startswith("String") or dataType.startswith("Number")) and not str_val:
42+
raise TypeError("stringValue is missing")
43+
return values
44+
45+
46+
class SqsRecordSchema(BaseModel):
47+
messageId: str
48+
receiptHandle: str
49+
body: str
50+
attributes: SqsAttributesSchema
51+
messageAttributes: Dict[str, SqsMsgAttributeSchema]
52+
md5OfBody: str
53+
md5OfMessageAttributes: str
54+
eventSource: Literal["aws:sqs"]
55+
eventSourceARN: str
56+
awsRegion: str
57+
58+
59+
class SqsSchema(BaseModel):
60+
Records: List[SqsRecordSchema]

Diff for: ‎aws_lambda_powertools/validation/validator.py

+146
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
import logging
2+
from abc import ABC, abstractmethod
3+
from typing import Any, Callable, Dict
4+
5+
from pydantic import BaseModel, ValidationError
6+
7+
from aws_lambda_powertools.middleware_factory import lambda_handler_decorator
8+
from aws_lambda_powertools.validation.schemas.dynamodb import DynamoDBSchema
9+
from aws_lambda_powertools.validation.schemas.eventbridge import EventBridgeSchema
10+
from aws_lambda_powertools.validation.schemas.sqs import SqsSchema
11+
12+
logger = logging.getLogger(__name__)
13+
14+
15+
class Envelope(ABC):
16+
def _parse_user_dict_schema(self, user_event: Dict[str, Any], inbound_schema_model: BaseModel) -> Any:
17+
logger.debug("parsing user dictionary schema")
18+
try:
19+
return inbound_schema_model(**user_event)
20+
except (ValidationError, TypeError):
21+
logger.exception("Valdation exception while extracting user custom schema")
22+
raise
23+
24+
def _parse_user_json_string_schema(self, user_event: str, inbound_schema_model: BaseModel) -> Any:
25+
logger.debug("parsing user dictionary schema")
26+
if inbound_schema_model == str:
27+
logger.debug("input is string, returning")
28+
return user_event
29+
logger.debug("trying to parse as json encoded string")
30+
try:
31+
return inbound_schema_model.parse_raw(user_event)
32+
except (ValidationError, TypeError):
33+
logger.exception("Validation exception while extracting user custom schema")
34+
raise
35+
36+
@abstractmethod
37+
def parse(self, event: Dict[str, Any], inbound_schema_model: BaseModel) -> Any:
38+
return NotImplemented
39+
40+
41+
class UserEnvelope(Envelope):
42+
def parse(self, event: Dict[str, Any], inbound_schema_model: BaseModel) -> Any:
43+
try:
44+
return inbound_schema_model(**event)
45+
except (ValidationError, TypeError):
46+
logger.exception("Validation exception received from input user custom envelope event")
47+
raise
48+
49+
50+
class EventBridgeEnvelope(Envelope):
51+
def parse(self, event: Dict[str, Any], inbound_schema_model: BaseModel) -> Any:
52+
try:
53+
parsed_envelope = EventBridgeSchema(**event)
54+
except (ValidationError, TypeError):
55+
logger.exception("Validation exception received from input eventbridge event")
56+
raise
57+
return self._parse_user_dict_schema(parsed_envelope.detail, inbound_schema_model)
58+
59+
60+
class SqsEnvelope(Envelope):
61+
def parse(self, event: Dict[str, Any], inbound_schema_model: BaseModel) -> Any:
62+
try:
63+
parsed_envelope = SqsSchema(**event)
64+
except (ValidationError, TypeError):
65+
logger.exception("Validation exception received from input sqs event")
66+
raise
67+
output = []
68+
for record in parsed_envelope.Records:
69+
parsed_msg = self._parse_user_json_string_schema(record.body, inbound_schema_model)
70+
output.append({"body": parsed_msg, "attributes": record.messageAttributes})
71+
return output
72+
73+
74+
class DynamoDBEnvelope(Envelope):
75+
def parse(self, event: Dict[str, Any], inbound_schema_model: BaseModel) -> Any:
76+
try:
77+
parsed_envelope = DynamoDBSchema(**event)
78+
except (ValidationError, TypeError):
79+
logger.exception("Validation exception received from input dynamodb stream event")
80+
raise
81+
output = []
82+
for record in parsed_envelope.Records:
83+
parsed_new_image = (
84+
{}
85+
if not record.dynamodb.NewImage
86+
else self._parse_user_dict_schema(record.dynamodb.NewImage, inbound_schema_model)
87+
) # noqa: E501
88+
parsed_old_image = (
89+
{}
90+
if not record.dynamodb.OldImage
91+
else self._parse_user_dict_schema(record.dynamodb.OldImage, inbound_schema_model)
92+
) # noqa: E501
93+
output.append({"new": parsed_new_image, "old": parsed_old_image})
94+
return output
95+
96+
97+
@lambda_handler_decorator
98+
def validator(
99+
handler: Callable[[Dict, Any], Any],
100+
event: Dict[str, Any],
101+
context: Dict[str, Any],
102+
inbound_schema_model: BaseModel,
103+
outbound_schema_model: BaseModel,
104+
envelope: Envelope,
105+
) -> Any:
106+
"""Decorator to create validation for lambda handlers events - both inbound and outbound
107+
108+
As Lambda follows (event, context) signature we can remove some of the boilerplate
109+
and also capture any exception any Lambda function throws or its response as metadata
110+
111+
Example
112+
-------
113+
**Lambda function using validation decorator**
114+
115+
@validator(inbound=inbound_schema_model, outbound=outbound_schema_model)
116+
def handler(parsed_event_model, context):
117+
...
118+
119+
Parameters
120+
----------
121+
todo add
122+
123+
Raises
124+
------
125+
err
126+
TypeError or pydantic.ValidationError or any exception raised by the lambda handler itself
127+
"""
128+
lambda_handler_name = handler.__name__
129+
logger.debug("Validating inbound schema")
130+
parsed_event_model = envelope.parse(event, inbound_schema_model)
131+
try:
132+
logger.debug(f"Calling handler {lambda_handler_name}")
133+
response = handler({"orig": event, "custom": parsed_event_model}, context)
134+
logger.debug("Received lambda handler response successfully")
135+
logger.debug(response)
136+
except Exception:
137+
logger.exception(f"Exception received from {lambda_handler_name}")
138+
raise
139+
140+
try:
141+
logger.debug("Validating outbound response schema")
142+
outbound_schema_model(**response)
143+
except (ValidationError, TypeError):
144+
logger.exception(f"Validation exception received from {lambda_handler_name} response event")
145+
raise
146+
return response

Diff for: ‎poetry.lock

+57-1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Diff for: ‎pyproject.toml

+2
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ python = "^3.6"
2222
aws-xray-sdk = "^2.5.0"
2323
fastjsonschema = "~=2.14.4"
2424
boto3 = "^1.12"
25+
pydantic = "^1.6.1"
2526

2627
[tool.poetry.dev-dependencies]
2728
coverage = {extras = ["toml"], version = "^5.0.3"}
@@ -48,6 +49,7 @@ xenon = "^0.7.0"
4849
flake8-eradicate = "^0.3.0"
4950
dataclasses = {version = "*", python = "~3.6"}
5051
flake8-bugbear = "^20.1.4"
52+
aws_lambda_context = "^1.1.0"
5153

5254
[tool.coverage.run]
5355
source = ["aws_lambda_powertools"]

0 commit comments

Comments
 (0)