Merge pull request #1710 from aws/normj/handle-milli-epochs

Handle when Lambda events are sent unix epoch dates in milliseconds

Author: normj

.github/workflows/update-Dockerfiles.yml

@@ -118,7 +118,7 @@ jobs:
           git add "**/*Dockerfile"
           git commit -m "chore: ASP.NET Core version update in Dockerfiles"
           git push origin $branch
-          echo "::set-output name=BRANCH::$branch"
+          echo "BRANCH=$branch" >> $GITHUB_OUTPUT
 
       # Create a Pull Request from the pushed branch
       - name: Pull Request

Libraries/src/Amazon.Lambda.Serialization.Json/Amazon.Lambda.Serialization.Json.csproj

@@ -9,7 +9,7 @@
     <AssemblyName>Amazon.Lambda.Serialization.Json</AssemblyName>
     <PackageId>Amazon.Lambda.Serialization.Json</PackageId>
     <PackageTags>AWS;Amazon;Lambda</PackageTags>
-    <VersionPrefix>2.2.0</VersionPrefix>
+    <VersionPrefix>2.2.1</VersionPrefix>
   </PropertyGroup>
 
   <ItemGroup>

Libraries/src/Amazon.Lambda.Serialization.Json/JsonNumberToDateTimeDataConverter.cs

@@ -12,6 +12,8 @@ namespace Amazon.Lambda.Serialization.Json
     /// </summary>
     internal class JsonNumberToDateTimeDataConverter : JsonConverter
     {
+        // The number of seconds from DateTime.MinValue to year 5000.
+        private const long YEAR_5000_IN_SECONDS = 157753180800;
         private static readonly TypeInfo DATETIME_TYPEINFO = typeof(DateTime).GetTypeInfo();
         private static readonly DateTime EPOCH_DATETIME = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
 
@@ -38,7 +40,20 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
                     break;
             }
 
-            var result = EPOCH_DATETIME.AddSeconds(seconds);
+            object result;
+
+            // If the time is in seconds is greater then the year 5000 it is safe to assume
+            // this is the special case of Kinesis sending the data which actually sends the time in milliseconds.
+            // https://github.com/aws/aws-lambda-dotnet/issues/839
+            if (seconds > YEAR_5000_IN_SECONDS)
+            {
+                result = EPOCH_DATETIME.AddMilliseconds(seconds);
+            }
+            else
+            {
+                result = EPOCH_DATETIME.AddSeconds(seconds);
+            }
+
             return result;
         }
 

Libraries/src/Amazon.Lambda.Serialization.SystemTextJson/Amazon.Lambda.Serialization.SystemTextJson.csproj

@@ -9,7 +9,7 @@
         <AssemblyName>Amazon.Lambda.Serialization.SystemTextJson</AssemblyName>
         <PackageId>Amazon.Lambda.Serialization.SystemTextJson</PackageId>
         <PackageTags>AWS;Amazon;Lambda</PackageTags>
-        <VersionPrefix>2.4.1</VersionPrefix>
+        <VersionPrefix>2.4.2</VersionPrefix>
 		<PackageReadmeFile>README.md</PackageReadmeFile>
     </PropertyGroup>
 	<ItemGroup>

Libraries/src/Amazon.Lambda.Serialization.SystemTextJson/Converters/DateTimeConverter.cs

@@ -9,6 +9,9 @@ namespace Amazon.Lambda.Serialization.SystemTextJson.Converters
     /// </summary>
     public class DateTimeConverter : JsonConverter<DateTime>
     {
+        // The number of seconds from DateTime.MinValue to year 5000.
+        private const long YEAR_5000_IN_SECONDS = 157753180800;
+
         /// <summary>
         /// Converts the value to a DateTime. If the JSON type is a number then it assumes the time is represented as 
         /// an epoch time.
@@ -28,7 +31,17 @@ public override DateTime Read(ref Utf8JsonReader reader, Type typeToConvert, Jso
             {
                 if (reader.TryGetInt64(out var intSeconds))
                 {
-                    return DateTime.UnixEpoch.AddSeconds(intSeconds);
+                    // If the time is in seconds is greater then the year 5000 it is safe to assume
+                    // this is the special case of Kinesis sending the data which actually sends the time in milliseconds.
+                    // https://github.com/aws/aws-lambda-dotnet/issues/839
+                    if (intSeconds > YEAR_5000_IN_SECONDS)
+                    {
+                        return DateTime.UnixEpoch.AddMilliseconds(intSeconds);
+                    }
+                    else
+                    {
+                        return DateTime.UnixEpoch.AddSeconds(intSeconds);
+                    }
                 }
                 if (reader.TryGetDouble(out var doubleSeconds))
                 {

Libraries/test/EventsTests.Shared/EventTests.cs

@@ -520,6 +520,79 @@ public void DynamoDbUpdateTest(Type serializerType)
             Handle(dynamodbEvent);
         }
 
+        [Theory]
+        [InlineData(typeof(JsonSerializer))]
+#if NETCOREAPP3_1_OR_GREATER
+        [InlineData(typeof(Amazon.Lambda.Serialization.SystemTextJson.LambdaJsonSerializer))]
+        [InlineData(typeof(Amazon.Lambda.Serialization.SystemTextJson.DefaultLambdaJsonSerializer))]
+#endif
+        public void DynamoDbWithMillisecondsTest(Type serializerType)
+        {
+            var serializer = Activator.CreateInstance(serializerType) as ILambdaSerializer;
+            Stream json = LoadJsonTestFile("dynamodb-with-ms-event.json");
+            var dynamodbEvent = serializer.Deserialize<DynamoDBEvent>(json);
+            Assert.Equal(dynamodbEvent.Records.Count, 2);
+
+            var record = dynamodbEvent.Records[0];
+            Assert.Equal(record.EventID, "f07f8ca4b0b26cb9c4e5e77e69f274ee");
+            Assert.Equal(record.EventVersion, "1.1");
+            Assert.Equal(record.Dynamodb.Keys.Count, 2);
+            Assert.Equal(record.Dynamodb.Keys["key"].S, "binary");
+            Assert.Equal(record.Dynamodb.Keys["val"].S, "data");
+            Assert.Null(record.UserIdentity);
+            Assert.Null(record.Dynamodb.OldImage);
+            Assert.Equal(record.Dynamodb.NewImage["val"].S, "data");
+            Assert.Equal(record.Dynamodb.NewImage["key"].S, "binary");
+            Assert.Null(record.Dynamodb.NewImage["key"].BOOL);
+            Assert.Null(record.Dynamodb.NewImage["key"].L);
+            Assert.Null(record.Dynamodb.NewImage["key"].M);
+            Assert.Null(record.Dynamodb.NewImage["key"].N);
+            Assert.Null(record.Dynamodb.NewImage["key"].NS);
+            Assert.Null(record.Dynamodb.NewImage["key"].NULL);
+            Assert.Null(record.Dynamodb.NewImage["key"].SS);
+            Assert.Equal(MemoryStreamToBase64String(record.Dynamodb.NewImage["asdf1"].B), "AAEqQQ==");
+            Assert.Equal(record.Dynamodb.NewImage["asdf2"].BS.Count, 2);
+            Assert.Equal(MemoryStreamToBase64String(record.Dynamodb.NewImage["asdf2"].BS[0]), "AAEqQQ==");
+            Assert.Equal(MemoryStreamToBase64String(record.Dynamodb.NewImage["asdf2"].BS[1]), "QSoBAA==");
+            Assert.Equal(record.Dynamodb.StreamViewType, "NEW_AND_OLD_IMAGES");
+            Assert.Equal(record.Dynamodb.SequenceNumber, "1405400000000002063282832");
+            Assert.Equal(record.Dynamodb.SizeBytes, 54);
+            Assert.Equal(record.AwsRegion, "us-east-1");
+            Assert.Equal(record.EventName, "INSERT");
+            Assert.Equal(record.EventSourceArn, "arn:aws:dynamodb:us-east-1:123456789012:table/Example-Table/stream/2016-12-01T00:00:00.000");
+            Assert.Equal(record.EventSource, "aws:dynamodb");
+            var recordDateTime = record.Dynamodb.ApproximateCreationDateTime;
+            Assert.Equal(recordDateTime.Ticks, 636162388200000000);
+
+            var topLevelList = record.Dynamodb.NewImage["misc1"].L;
+            Assert.Equal(0, topLevelList.Count);
+
+            var nestedMap = record.Dynamodb.NewImage["misc2"].M;
+            Assert.NotNull(nestedMap);
+            Assert.Equal(0, nestedMap["ItemsEmpty"].L.Count);
+            Assert.Equal(3, nestedMap["ItemsNonEmpty"].L.Count);
+            Assert.False(nestedMap["ItemBoolean"].BOOL);
+            Assert.True(nestedMap["ItemNull"].NULL);
+            Assert.Equal(3, nestedMap["ItemNumberSet"].NS.Count);
+            Assert.Equal(2, nestedMap["ItemStringSet"].SS.Count);
+
+            var secondRecord = dynamodbEvent.Records[1];
+            Assert.NotNull(secondRecord.UserIdentity);
+            Assert.Equal("dynamodb.amazonaws.com", secondRecord.UserIdentity.PrincipalId);
+            Assert.Equal("Service", secondRecord.UserIdentity.Type);
+            Assert.Null(secondRecord.Dynamodb.NewImage);
+            Assert.NotNull(secondRecord.Dynamodb.OldImage["asdf1"].B);
+            Assert.Null(secondRecord.Dynamodb.OldImage["asdf1"].S);
+            Assert.Null(secondRecord.Dynamodb.OldImage["asdf1"].L);
+            Assert.Null(secondRecord.Dynamodb.OldImage["asdf1"].M);
+            Assert.Null(secondRecord.Dynamodb.OldImage["asdf1"].N);
+            Assert.Null(secondRecord.Dynamodb.OldImage["asdf1"].NS);
+            Assert.Null(secondRecord.Dynamodb.OldImage["asdf1"].NULL);
+            Assert.Null(secondRecord.Dynamodb.OldImage["asdf1"].SS);
+
+            Handle(dynamodbEvent);
+        }
+
         [Theory]
         [InlineData(typeof(JsonSerializer))]
 #if NETCOREAPP3_1_OR_GREATER

Libraries/test/EventsTests.Shared/EventsTests.Shared.projitems

@@ -30,6 +30,7 @@
     <Content Include="$(MSBuildThisFileDirectory)cognito-presignup-event.json" />
     <Content Include="$(MSBuildThisFileDirectory)cognito-event.json" />
     <Content Include="$(MSBuildThisFileDirectory)config-event.json" />
+    <Content Include="$(MSBuildThisFileDirectory)dynamodb-with-ms-event.json" />
     <Content Include="$(MSBuildThisFileDirectory)kinesis-batchitemfailures-response.json" />
     <Content Include="$(MSBuildThisFileDirectory)dynamodb-batchitemfailures-response.json" />
     <Content Include="$(MSBuildThisFileDirectory)dynamodb-event.json" />

Libraries/test/EventsTests.Shared/dynamodb-with-ms-event.json

@@ -0,0 +1,100 @@
+{
+   "Records":[
+      {
+         "eventID":"f07f8ca4b0b26cb9c4e5e77e69f274ee",
+         "eventName":"INSERT",
+         "eventVersion":"1.1",
+         "eventSource":"aws:dynamodb",
+         "awsRegion":"us-east-1",
+         "dynamodb":{
+            "ApproximateCreationDateTime":1480642020000,
+            "Keys":{
+               "val":{
+                  "S":"data"
+               },
+               "key":{
+                  "S":"binary"
+               }
+            },
+            "NewImage":{
+               "val":{
+                  "S":"data"
+               },
+               "asdf1":{
+                  "B":"AAEqQQ=="
+               },
+               "asdf2":{
+                  "BS":[
+                     "AAEqQQ==",
+                     "QSoBAA=="
+                  ]
+               },
+               "misc1": {
+                  "L": []
+               },
+               "misc2": {
+                 "M": {
+                   "ItemsEmpty": { "L": [] },
+                   "ItemsNonEmpty": { "L": [{"S": "Cookies"} , {"S": "Coffee"}, {"N": "3.14159"}] },
+                   "ItemBoolean": { "BOOL": false },
+                   "ItemNumberSet": { "NS": ["0", "50", "150"] },
+                   "ItemNull": { "NULL": true },
+                   "ItemStringSet": { "SS": ["Hippo", "Zebra"] }
+                 }
+               },
+               "key":{
+                  "S":"binary"
+               }
+            },
+            "SequenceNumber":"1405400000000002063282832",
+            "SizeBytes":54,
+            "StreamViewType":"NEW_AND_OLD_IMAGES"
+         },
+         "eventSourceARN":"arn:aws:dynamodb:us-east-1:123456789012:table/Example-Table/stream/2016-12-01T00:00:00.000"
+      },
+      {
+         "eventID":"f07f8ca4b0b26cb9c4e5e77e42f274ee",
+         "eventName":"REMOVE",
+         "eventVersion":"1.1",
+         "eventSource":"aws:dynamodb",
+         "awsRegion":"us-east-1",
+         "userIdentity": {
+            "type": "Service",
+            "principalId": "dynamodb.amazonaws.com"
+         }, 
+         "dynamodb":{
+            "ApproximateCreationDateTime":1480642020,
+            "Keys":{
+               "val":{
+                  "S":"data"
+               },
+               "key":{
+                  "S":"binary"
+               }
+            },
+            "OldImage": {
+              "val": {
+                "S": "data"
+              },
+              "asdf1": {
+                "B": "AAEqQQ=="
+              },
+              "asdf2": {
+                "BS": [
+                  "AAEqQQ==",
+                  "QSoBAA==",
+                  "AAEqQQ=="
+                ]
+              },
+              "key": {
+                "S": "binary"
+              }
+            },
+            "SequenceNumber":"1405400000000002063282832",
+            "SizeBytes":54,
+            "StreamViewType":"OLD_IMAGE"
+         },
+         "eventSourceARN":"arn:aws:dynamodb:us-east-1:123456789012:table/Example-Table/stream/2016-12-01T00:00:00.000"
+      }
+   ]
+}