From f6ccaa3be6750558beeef4d4aba952e3743d4cc6 Mon Sep 17 00:00:00 2001 From: Ran Isenberg Date: Sun, 15 Nov 2020 16:09:17 +0200 Subject: [PATCH 1/2] Feature: Add sns notification support to Parser utility #206 --- .../utilities/parser/envelopes/__init__.py | 3 +- .../utilities/parser/envelopes/sns.py | 42 +++++++++ .../utilities/parser/models/__init__.py | 4 + .../utilities/parser/models/sns.py | 36 +++++++ tests/events/snsEvent.json | 6 +- tests/functional/parser/schemas.py | 20 ++++ tests/functional/parser/test_sns.py | 93 +++++++++++++++++++ .../functional/test_lambda_trigger_events.py | 4 +- tests/functional/validator/conftest.py | 4 +- 9 files changed, 204 insertions(+), 8 deletions(-) create mode 100644 aws_lambda_powertools/utilities/parser/envelopes/sns.py create mode 100644 aws_lambda_powertools/utilities/parser/models/sns.py create mode 100644 tests/functional/parser/test_sns.py diff --git a/aws_lambda_powertools/utilities/parser/envelopes/__init__.py b/aws_lambda_powertools/utilities/parser/envelopes/__init__.py index 2398840a756..4be73363b0f 100644 --- a/aws_lambda_powertools/utilities/parser/envelopes/__init__.py +++ b/aws_lambda_powertools/utilities/parser/envelopes/__init__.py @@ -1,6 +1,7 @@ from .base import BaseEnvelope from .dynamodb import DynamoDBStreamEnvelope from .event_bridge import EventBridgeEnvelope +from .sns import SnsEnvelope from .sqs import SqsEnvelope -__all__ = ["DynamoDBStreamEnvelope", "EventBridgeEnvelope", "SqsEnvelope", "BaseEnvelope"] +__all__ = ["DynamoDBStreamEnvelope", "EventBridgeEnvelope", "SnsEnvelope", "SqsEnvelope", "BaseEnvelope"] diff --git a/aws_lambda_powertools/utilities/parser/envelopes/sns.py b/aws_lambda_powertools/utilities/parser/envelopes/sns.py new file mode 100644 index 00000000000..f703bb46c63 --- /dev/null +++ b/aws_lambda_powertools/utilities/parser/envelopes/sns.py @@ -0,0 +1,42 @@ +import logging +from typing import Any, Dict, List, Optional, Union + +from ..models import SnsModel +from ..types import Model +from .base import BaseEnvelope + +logger = logging.getLogger(__name__) + + +class SnsEnvelope(BaseEnvelope): + """SNS Envelope to extract array of Records + + The record's body parameter is a string, though it can also be a JSON encoded string. + Regardless of its type it'll be parsed into a BaseModel object. + + Note: Records will be parsed the same way so if model is str, + all items in the list will be parsed as str and npt as JSON (and vice versa) + """ + + def parse(self, data: Optional[Union[Dict[str, Any], Any]], model: Model) -> List[Optional[Model]]: + """Parses records found with model provided + + Parameters + ---------- + data : Dict + Lambda event to be parsed + model : Model + Data model provided to parse after extracting data using envelope + + Returns + ------- + List + List of records parsed with model provided + """ + logger.debug(f"Parsing incoming data with SNS model {SnsModel}") + parsed_envelope = SnsModel.parse_obj(data) + output = [] + logger.debug(f"Parsing SNS records in `body` with {model}") + for record in parsed_envelope.Records: + output.append(self._parse(data=record.Sns.Message, model=model)) + return output diff --git a/aws_lambda_powertools/utilities/parser/models/__init__.py b/aws_lambda_powertools/utilities/parser/models/__init__.py index e58a678e959..e9daded27ca 100644 --- a/aws_lambda_powertools/utilities/parser/models/__init__.py +++ b/aws_lambda_powertools/utilities/parser/models/__init__.py @@ -1,5 +1,6 @@ from .dynamodb import DynamoDBStreamChangedRecordModel, DynamoDBStreamModel, DynamoDBStreamRecordModel from .event_bridge import EventBridgeModel +from .sns import SnsModel, SnsNotificationModel, SnsRecordModel from .sqs import SqsModel, SqsRecordModel __all__ = [ @@ -7,6 +8,9 @@ "EventBridgeModel", "DynamoDBStreamChangedRecordModel", "DynamoDBStreamRecordModel", + "SnsModel", + "SnsNotificationModel", + "SnsRecordModel", "SqsModel", "SqsRecordModel", ] diff --git a/aws_lambda_powertools/utilities/parser/models/sns.py b/aws_lambda_powertools/utilities/parser/models/sns.py new file mode 100644 index 00000000000..5c8fb005a47 --- /dev/null +++ b/aws_lambda_powertools/utilities/parser/models/sns.py @@ -0,0 +1,36 @@ +from datetime import datetime +from typing import Dict, List, Optional + +from pydantic import BaseModel +from pydantic.networks import HttpUrl +from typing_extensions import Literal + + +class SqsMsgAttributeModel(BaseModel): + Type: str + Value: str + + +class SnsNotificationModel(BaseModel): + Subject: Optional[str] + TopicArn: str + UnsubscribeUrl: HttpUrl + Type: Literal["Notification"] + MessageAttributes: Dict[str, SqsMsgAttributeModel] + Message: str + MessageId: str + SigningCertUrl: HttpUrl + Signature: str + Timestamp: datetime + SignatureVersion: str + + +class SnsRecordModel(BaseModel): + EventSource: Literal["aws:sns"] + EventVersion: str + EventSubscriptionArn: str + Sns: SnsNotificationModel + + +class SnsModel(BaseModel): + Records: List[SnsRecordModel] diff --git a/tests/events/snsEvent.json b/tests/events/snsEvent.json index b351dfd1418..3d8a8ed443c 100644 --- a/tests/events/snsEvent.json +++ b/tests/events/snsEvent.json @@ -8,7 +8,7 @@ "SignatureVersion": "1", "Timestamp": "2019-01-02T12:45:07.000Z", "Signature": "tcc6faL2yUC6dgZdmrwh1Y4cGa/ebXEkAi6RibDsvpi+tE/1+82j...65r==", - "SigningCertUrl": "https://sns.us-east-2.amazonaws.com/SimpleNotificat ...", + "SigningCertUrl": "https://sns.us-east-2.amazonaws.com/SimpleNotification", "MessageId": "95df01b4-ee98-5cb9-9903-4c221d41eb5e", "Message": "Hello from SNS!", "MessageAttributes": { @@ -22,10 +22,10 @@ } }, "Type": "Notification", - "UnsubscribeUrl": "https://sns.us-east-2.amazonaws.com/?Action=Unsubscri ...", + "UnsubscribeUrl": "https://sns.us-east-2.amazonaws.com/?Action=Unsubscribe", "TopicArn": "arn:aws:sns:us-east-2:123456789012:sns-lambda", "Subject": "TestInvoke" } } ] -} +} \ No newline at end of file diff --git a/tests/functional/parser/schemas.py b/tests/functional/parser/schemas.py index 47614cb95d8..bfc601e3537 100644 --- a/tests/functional/parser/schemas.py +++ b/tests/functional/parser/schemas.py @@ -8,6 +8,9 @@ DynamoDBStreamModel, DynamoDBStreamRecordModel, EventBridgeModel, + SnsModel, + SnsNotificationModel, + SnsRecordModel, SqsModel, SqsRecordModel, ) @@ -51,3 +54,20 @@ class MyAdvancedSqsRecordModel(SqsRecordModel): class MyAdvancedSqsBusiness(SqsModel): Records: List[MyAdvancedSqsRecordModel] + + +class MySnsBusiness(BaseModel): + message: str + username: str + + +class MySnsNotificationModel(SnsNotificationModel): + Message: str + + +class MyAdvancedSnsRecordModel(SnsRecordModel): + Sns: MySnsNotificationModel + + +class MyAdvancedSnsBusiness(SnsModel): + Records: List[MyAdvancedSnsRecordModel] diff --git a/tests/functional/parser/test_sns.py b/tests/functional/parser/test_sns.py new file mode 100644 index 00000000000..0dd1e6b506f --- /dev/null +++ b/tests/functional/parser/test_sns.py @@ -0,0 +1,93 @@ +from typing import Any, List + +import pytest + +from aws_lambda_powertools.utilities.parser import ValidationError, envelopes, event_parser +from aws_lambda_powertools.utilities.typing import LambdaContext +from tests.functional.parser.schemas import MyAdvancedSnsBusiness, MySnsBusiness +from tests.functional.parser.utils import load_event +from tests.functional.validator.conftest import sns_event # noqa: F401 + + +@event_parser(model=MySnsBusiness, envelope=envelopes.SnsEnvelope) +def handle_sns_json_body(event: List[MySnsBusiness], _: LambdaContext): + assert len(event) == 1 + assert event[0].message == "hello world" + assert event[0].username == "lessa" + + +def test_handle_sns_trigger_event_json_body(sns_event): # noqa: F811 + handle_sns_json_body(sns_event, LambdaContext()) + + +def test_validate_event_does_not_conform_with_model(): + event: Any = {"invalid": "event"} + + with pytest.raises(ValidationError): + handle_sns_json_body(event, LambdaContext()) + + +def test_validate_event_does_not_conform_user_json_string_with_model(): + event: Any = { + "Records": [ + { + "EventVersion": "1.0", + "EventSubscriptionArn": "arn:aws:sns:us-east-2:123456789012:sns-la ...", + "EventSource": "aws:sns", + "Sns": { + "SignatureVersion": "1", + "Timestamp": "2019-01-02T12:45:07.000Z", + "Signature": "tcc6faL2yUC6dgZdmrwh1Y4cGa/ebXEkAi6RibDsvpi+tE/1+82j...65r==", + "SigningCertUrl": "https://sns.us-east-2.amazonaws.com/SimpleNotificat ...", + "MessageId": "95df01b4-ee98-5cb9-9903-4c221d41eb5e", + "Message": "not a valid JSON!", + "MessageAttributes": {"Test": {"Type": "String", "Value": "TestString"}}, + "Type": "Notification", + "UnsubscribeUrl": "https://sns.us-east-2.amazonaws.com/?Action=Unsubscri ...", + "TopicArn": "arn:aws:sns:us-east-2:123456789012:sns-lambda", + "Subject": "TestInvoke", + }, + } + ] + } + + with pytest.raises(ValidationError): + handle_sns_json_body(event, LambdaContext()) + + +@event_parser(model=MyAdvancedSnsBusiness) +def handle_sns_no_envelope(event: MyAdvancedSnsBusiness, _: LambdaContext): + records = event.Records + record = records[0] + + assert len(records) == 1 + assert record.EventVersion == "1.0" + assert record.EventSubscriptionArn == "arn:aws:sns:us-east-2:123456789012:sns-la ..." + assert record.EventSource == "aws:sns" + assert record.Sns.Type == "Notification" + assert record.Sns.UnsubscribeUrl.scheme == "https" + assert record.Sns.UnsubscribeUrl.host == "sns.us-east-2.amazonaws.com" + assert record.Sns.UnsubscribeUrl.query == "Action=Unsubscribe" + assert record.Sns.TopicArn == "arn:aws:sns:us-east-2:123456789012:sns-lambda" + assert record.Sns.Subject == "TestInvoke" + assert record.Sns.SignatureVersion == "1" + convert_time = int(round(record.Sns.Timestamp.timestamp() * 1000)) + assert convert_time == 1546433107000 + assert record.Sns.Signature == "tcc6faL2yUC6dgZdmrwh1Y4cGa/ebXEkAi6RibDsvpi+tE/1+82j...65r==" + assert record.Sns.SigningCertUrl.host == "sns.us-east-2.amazonaws.com" + assert record.Sns.SigningCertUrl.scheme == "https" + assert record.Sns.SigningCertUrl.host == "sns.us-east-2.amazonaws.com" + assert record.Sns.SigningCertUrl.path == "/SimpleNotification" + assert record.Sns.MessageId == "95df01b4-ee98-5cb9-9903-4c221d41eb5e" + assert record.Sns.Message == "Hello from SNS!" + attrib_dict = record.Sns.MessageAttributes + assert len(attrib_dict) == 2 + assert attrib_dict["Test"].Type == "String" + assert attrib_dict["Test"].Value == "TestString" + assert attrib_dict["TestBinary"].Type == "Binary" + assert attrib_dict["TestBinary"].Value == "TestBinary" + + +def test_handle_sns_trigger_event_no_envelope(): + event_dict = load_event("snsEvent.json") + handle_sns_no_envelope(event_dict, LambdaContext()) diff --git a/tests/functional/test_lambda_trigger_events.py b/tests/functional/test_lambda_trigger_events.py index 6cfdbc765b1..59a3687e0fe 100644 --- a/tests/functional/test_lambda_trigger_events.py +++ b/tests/functional/test_lambda_trigger_events.py @@ -487,7 +487,7 @@ def test_sns_trigger_event(): assert sns.signature_version == "1" assert sns.timestamp == "2019-01-02T12:45:07.000Z" assert sns.signature == "tcc6faL2yUC6dgZdmrwh1Y4cGa/ebXEkAi6RibDsvpi+tE/1+82j...65r==" - assert sns.signing_cert_url == "https://sns.us-east-2.amazonaws.com/SimpleNotificat ..." + assert sns.signing_cert_url == "https://sns.us-east-2.amazonaws.com/SimpleNotification" assert sns.message_id == "95df01b4-ee98-5cb9-9903-4c221d41eb5e" assert sns.message == "Hello from SNS!" message_attributes = sns.message_attributes @@ -495,7 +495,7 @@ def test_sns_trigger_event(): assert test_message_attribute.get_type == "String" assert test_message_attribute.value == "TestString" assert sns.get_type == "Notification" - assert sns.unsubscribe_url == "https://sns.us-east-2.amazonaws.com/?Action=Unsubscri ..." + assert sns.unsubscribe_url == "https://sns.us-east-2.amazonaws.com/?Action=Unsubscribe" assert sns.topic_arn == "arn:aws:sns:us-east-2:123456789012:sns-lambda" assert sns.subject == "TestInvoke" assert event.record._data == event["Records"][0] diff --git a/tests/functional/validator/conftest.py b/tests/functional/validator/conftest.py index 5c154b5aab4..740355db70b 100644 --- a/tests/functional/validator/conftest.py +++ b/tests/functional/validator/conftest.py @@ -195,8 +195,8 @@ def sns_event(): "Timestamp": "1970-01-01T00:00:00.000Z", "SignatureVersion": "1", "Signature": "EXAMPLE", - "SigningCertUrl": "EXAMPLE", - "UnsubscribeUrl": "EXAMPLE", + "SigningCertUrl": "https://www.example.com", + "UnsubscribeUrl": "https://www.example.com", "MessageAttributes": { "Test": {"Type": "String", "Value": "TestString"}, "TestBinary": {"Type": "Binary", "Value": "TestBinary"}, From b470a31043f7815a91523fef731b030cbb5ff6d6 Mon Sep 17 00:00:00 2001 From: Ran Isenberg Date: Wed, 18 Nov 2020 09:20:25 +0200 Subject: [PATCH 2/2] fix naming to sns --- aws_lambda_powertools/utilities/parser/models/sns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_lambda_powertools/utilities/parser/models/sns.py b/aws_lambda_powertools/utilities/parser/models/sns.py index 5c8fb005a47..4462bc4f130 100644 --- a/aws_lambda_powertools/utilities/parser/models/sns.py +++ b/aws_lambda_powertools/utilities/parser/models/sns.py @@ -6,7 +6,7 @@ from typing_extensions import Literal -class SqsMsgAttributeModel(BaseModel): +class SnsMsgAttributeModel(BaseModel): Type: str Value: str @@ -16,7 +16,7 @@ class SnsNotificationModel(BaseModel): TopicArn: str UnsubscribeUrl: HttpUrl Type: Literal["Notification"] - MessageAttributes: Dict[str, SqsMsgAttributeModel] + MessageAttributes: Dict[str, SnsMsgAttributeModel] Message: str MessageId: str SigningCertUrl: HttpUrl