Skip to content

Commit b65a233

Browse files
authored
Merge pull request #1710 from aws/normj/handle-milli-epochs
Handle when Lambda events are sent unix epoch dates in milliseconds
2 parents 45aba15 + f04e80d commit b65a233

File tree

8 files changed

+207
-5
lines changed

8 files changed

+207
-5
lines changed

.github/workflows/update-Dockerfiles.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ jobs:
118118
git add "**/*Dockerfile"
119119
git commit -m "chore: ASP.NET Core version update in Dockerfiles"
120120
git push origin $branch
121-
echo "::set-output name=BRANCH::$branch"
121+
echo "BRANCH=$branch" >> $GITHUB_OUTPUT
122122
123123
# Create a Pull Request from the pushed branch
124124
- name: Pull Request

Libraries/src/Amazon.Lambda.Serialization.Json/Amazon.Lambda.Serialization.Json.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
<AssemblyName>Amazon.Lambda.Serialization.Json</AssemblyName>
1010
<PackageId>Amazon.Lambda.Serialization.Json</PackageId>
1111
<PackageTags>AWS;Amazon;Lambda</PackageTags>
12-
<VersionPrefix>2.2.0</VersionPrefix>
12+
<VersionPrefix>2.2.1</VersionPrefix>
1313
</PropertyGroup>
1414

1515
<ItemGroup>

Libraries/src/Amazon.Lambda.Serialization.Json/JsonNumberToDateTimeDataConverter.cs

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ namespace Amazon.Lambda.Serialization.Json
1212
/// </summary>
1313
internal class JsonNumberToDateTimeDataConverter : JsonConverter
1414
{
15+
// The number of seconds from DateTime.MinValue to year 5000.
16+
private const long YEAR_5000_IN_SECONDS = 157753180800;
1517
private static readonly TypeInfo DATETIME_TYPEINFO = typeof(DateTime).GetTypeInfo();
1618
private static readonly DateTime EPOCH_DATETIME = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
1719

@@ -38,7 +40,20 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
3840
break;
3941
}
4042

41-
var result = EPOCH_DATETIME.AddSeconds(seconds);
43+
object result;
44+
45+
// If the time is in seconds is greater then the year 5000 it is safe to assume
46+
// this is the special case of Kinesis sending the data which actually sends the time in milliseconds.
47+
// https://github.com/aws/aws-lambda-dotnet/issues/839
48+
if (seconds > YEAR_5000_IN_SECONDS)
49+
{
50+
result = EPOCH_DATETIME.AddMilliseconds(seconds);
51+
}
52+
else
53+
{
54+
result = EPOCH_DATETIME.AddSeconds(seconds);
55+
}
56+
4257
return result;
4358
}
4459

Libraries/src/Amazon.Lambda.Serialization.SystemTextJson/Amazon.Lambda.Serialization.SystemTextJson.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
<AssemblyName>Amazon.Lambda.Serialization.SystemTextJson</AssemblyName>
1010
<PackageId>Amazon.Lambda.Serialization.SystemTextJson</PackageId>
1111
<PackageTags>AWS;Amazon;Lambda</PackageTags>
12-
<VersionPrefix>2.4.1</VersionPrefix>
12+
<VersionPrefix>2.4.2</VersionPrefix>
1313
<PackageReadmeFile>README.md</PackageReadmeFile>
1414
</PropertyGroup>
1515
<ItemGroup>

Libraries/src/Amazon.Lambda.Serialization.SystemTextJson/Converters/DateTimeConverter.cs

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@ namespace Amazon.Lambda.Serialization.SystemTextJson.Converters
99
/// </summary>
1010
public class DateTimeConverter : JsonConverter<DateTime>
1111
{
12+
// The number of seconds from DateTime.MinValue to year 5000.
13+
private const long YEAR_5000_IN_SECONDS = 157753180800;
14+
1215
/// <summary>
1316
/// Converts the value to a DateTime. If the JSON type is a number then it assumes the time is represented as
1417
/// an epoch time.
@@ -28,7 +31,17 @@ public override DateTime Read(ref Utf8JsonReader reader, Type typeToConvert, Jso
2831
{
2932
if (reader.TryGetInt64(out var intSeconds))
3033
{
31-
return DateTime.UnixEpoch.AddSeconds(intSeconds);
34+
// If the time is in seconds is greater then the year 5000 it is safe to assume
35+
// this is the special case of Kinesis sending the data which actually sends the time in milliseconds.
36+
// https://github.com/aws/aws-lambda-dotnet/issues/839
37+
if (intSeconds > YEAR_5000_IN_SECONDS)
38+
{
39+
return DateTime.UnixEpoch.AddMilliseconds(intSeconds);
40+
}
41+
else
42+
{
43+
return DateTime.UnixEpoch.AddSeconds(intSeconds);
44+
}
3245
}
3346
if (reader.TryGetDouble(out var doubleSeconds))
3447
{

Libraries/test/EventsTests.Shared/EventTests.cs

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -520,6 +520,79 @@ public void DynamoDbUpdateTest(Type serializerType)
520520
Handle(dynamodbEvent);
521521
}
522522

523+
[Theory]
524+
[InlineData(typeof(JsonSerializer))]
525+
#if NETCOREAPP3_1_OR_GREATER
526+
[InlineData(typeof(Amazon.Lambda.Serialization.SystemTextJson.LambdaJsonSerializer))]
527+
[InlineData(typeof(Amazon.Lambda.Serialization.SystemTextJson.DefaultLambdaJsonSerializer))]
528+
#endif
529+
public void DynamoDbWithMillisecondsTest(Type serializerType)
530+
{
531+
var serializer = Activator.CreateInstance(serializerType) as ILambdaSerializer;
532+
Stream json = LoadJsonTestFile("dynamodb-with-ms-event.json");
533+
var dynamodbEvent = serializer.Deserialize<DynamoDBEvent>(json);
534+
Assert.Equal(dynamodbEvent.Records.Count, 2);
535+
536+
var record = dynamodbEvent.Records[0];
537+
Assert.Equal(record.EventID, "f07f8ca4b0b26cb9c4e5e77e69f274ee");
538+
Assert.Equal(record.EventVersion, "1.1");
539+
Assert.Equal(record.Dynamodb.Keys.Count, 2);
540+
Assert.Equal(record.Dynamodb.Keys["key"].S, "binary");
541+
Assert.Equal(record.Dynamodb.Keys["val"].S, "data");
542+
Assert.Null(record.UserIdentity);
543+
Assert.Null(record.Dynamodb.OldImage);
544+
Assert.Equal(record.Dynamodb.NewImage["val"].S, "data");
545+
Assert.Equal(record.Dynamodb.NewImage["key"].S, "binary");
546+
Assert.Null(record.Dynamodb.NewImage["key"].BOOL);
547+
Assert.Null(record.Dynamodb.NewImage["key"].L);
548+
Assert.Null(record.Dynamodb.NewImage["key"].M);
549+
Assert.Null(record.Dynamodb.NewImage["key"].N);
550+
Assert.Null(record.Dynamodb.NewImage["key"].NS);
551+
Assert.Null(record.Dynamodb.NewImage["key"].NULL);
552+
Assert.Null(record.Dynamodb.NewImage["key"].SS);
553+
Assert.Equal(MemoryStreamToBase64String(record.Dynamodb.NewImage["asdf1"].B), "AAEqQQ==");
554+
Assert.Equal(record.Dynamodb.NewImage["asdf2"].BS.Count, 2);
555+
Assert.Equal(MemoryStreamToBase64String(record.Dynamodb.NewImage["asdf2"].BS[0]), "AAEqQQ==");
556+
Assert.Equal(MemoryStreamToBase64String(record.Dynamodb.NewImage["asdf2"].BS[1]), "QSoBAA==");
557+
Assert.Equal(record.Dynamodb.StreamViewType, "NEW_AND_OLD_IMAGES");
558+
Assert.Equal(record.Dynamodb.SequenceNumber, "1405400000000002063282832");
559+
Assert.Equal(record.Dynamodb.SizeBytes, 54);
560+
Assert.Equal(record.AwsRegion, "us-east-1");
561+
Assert.Equal(record.EventName, "INSERT");
562+
Assert.Equal(record.EventSourceArn, "arn:aws:dynamodb:us-east-1:123456789012:table/Example-Table/stream/2016-12-01T00:00:00.000");
563+
Assert.Equal(record.EventSource, "aws:dynamodb");
564+
var recordDateTime = record.Dynamodb.ApproximateCreationDateTime;
565+
Assert.Equal(recordDateTime.Ticks, 636162388200000000);
566+
567+
var topLevelList = record.Dynamodb.NewImage["misc1"].L;
568+
Assert.Equal(0, topLevelList.Count);
569+
570+
var nestedMap = record.Dynamodb.NewImage["misc2"].M;
571+
Assert.NotNull(nestedMap);
572+
Assert.Equal(0, nestedMap["ItemsEmpty"].L.Count);
573+
Assert.Equal(3, nestedMap["ItemsNonEmpty"].L.Count);
574+
Assert.False(nestedMap["ItemBoolean"].BOOL);
575+
Assert.True(nestedMap["ItemNull"].NULL);
576+
Assert.Equal(3, nestedMap["ItemNumberSet"].NS.Count);
577+
Assert.Equal(2, nestedMap["ItemStringSet"].SS.Count);
578+
579+
var secondRecord = dynamodbEvent.Records[1];
580+
Assert.NotNull(secondRecord.UserIdentity);
581+
Assert.Equal("dynamodb.amazonaws.com", secondRecord.UserIdentity.PrincipalId);
582+
Assert.Equal("Service", secondRecord.UserIdentity.Type);
583+
Assert.Null(secondRecord.Dynamodb.NewImage);
584+
Assert.NotNull(secondRecord.Dynamodb.OldImage["asdf1"].B);
585+
Assert.Null(secondRecord.Dynamodb.OldImage["asdf1"].S);
586+
Assert.Null(secondRecord.Dynamodb.OldImage["asdf1"].L);
587+
Assert.Null(secondRecord.Dynamodb.OldImage["asdf1"].M);
588+
Assert.Null(secondRecord.Dynamodb.OldImage["asdf1"].N);
589+
Assert.Null(secondRecord.Dynamodb.OldImage["asdf1"].NS);
590+
Assert.Null(secondRecord.Dynamodb.OldImage["asdf1"].NULL);
591+
Assert.Null(secondRecord.Dynamodb.OldImage["asdf1"].SS);
592+
593+
Handle(dynamodbEvent);
594+
}
595+
523596
[Theory]
524597
[InlineData(typeof(JsonSerializer))]
525598
#if NETCOREAPP3_1_OR_GREATER

Libraries/test/EventsTests.Shared/EventsTests.Shared.projitems

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
<Content Include="$(MSBuildThisFileDirectory)cognito-presignup-event.json" />
3131
<Content Include="$(MSBuildThisFileDirectory)cognito-event.json" />
3232
<Content Include="$(MSBuildThisFileDirectory)config-event.json" />
33+
<Content Include="$(MSBuildThisFileDirectory)dynamodb-with-ms-event.json" />
3334
<Content Include="$(MSBuildThisFileDirectory)kinesis-batchitemfailures-response.json" />
3435
<Content Include="$(MSBuildThisFileDirectory)dynamodb-batchitemfailures-response.json" />
3536
<Content Include="$(MSBuildThisFileDirectory)dynamodb-event.json" />
Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
{
2+
"Records":[
3+
{
4+
"eventID":"f07f8ca4b0b26cb9c4e5e77e69f274ee",
5+
"eventName":"INSERT",
6+
"eventVersion":"1.1",
7+
"eventSource":"aws:dynamodb",
8+
"awsRegion":"us-east-1",
9+
"dynamodb":{
10+
"ApproximateCreationDateTime":1480642020000,
11+
"Keys":{
12+
"val":{
13+
"S":"data"
14+
},
15+
"key":{
16+
"S":"binary"
17+
}
18+
},
19+
"NewImage":{
20+
"val":{
21+
"S":"data"
22+
},
23+
"asdf1":{
24+
"B":"AAEqQQ=="
25+
},
26+
"asdf2":{
27+
"BS":[
28+
"AAEqQQ==",
29+
"QSoBAA=="
30+
]
31+
},
32+
"misc1": {
33+
"L": []
34+
},
35+
"misc2": {
36+
"M": {
37+
"ItemsEmpty": { "L": [] },
38+
"ItemsNonEmpty": { "L": [{"S": "Cookies"} , {"S": "Coffee"}, {"N": "3.14159"}] },
39+
"ItemBoolean": { "BOOL": false },
40+
"ItemNumberSet": { "NS": ["0", "50", "150"] },
41+
"ItemNull": { "NULL": true },
42+
"ItemStringSet": { "SS": ["Hippo", "Zebra"] }
43+
}
44+
},
45+
"key":{
46+
"S":"binary"
47+
}
48+
},
49+
"SequenceNumber":"1405400000000002063282832",
50+
"SizeBytes":54,
51+
"StreamViewType":"NEW_AND_OLD_IMAGES"
52+
},
53+
"eventSourceARN":"arn:aws:dynamodb:us-east-1:123456789012:table/Example-Table/stream/2016-12-01T00:00:00.000"
54+
},
55+
{
56+
"eventID":"f07f8ca4b0b26cb9c4e5e77e42f274ee",
57+
"eventName":"REMOVE",
58+
"eventVersion":"1.1",
59+
"eventSource":"aws:dynamodb",
60+
"awsRegion":"us-east-1",
61+
"userIdentity": {
62+
"type": "Service",
63+
"principalId": "dynamodb.amazonaws.com"
64+
},
65+
"dynamodb":{
66+
"ApproximateCreationDateTime":1480642020,
67+
"Keys":{
68+
"val":{
69+
"S":"data"
70+
},
71+
"key":{
72+
"S":"binary"
73+
}
74+
},
75+
"OldImage": {
76+
"val": {
77+
"S": "data"
78+
},
79+
"asdf1": {
80+
"B": "AAEqQQ=="
81+
},
82+
"asdf2": {
83+
"BS": [
84+
"AAEqQQ==",
85+
"QSoBAA==",
86+
"AAEqQQ=="
87+
]
88+
},
89+
"key": {
90+
"S": "binary"
91+
}
92+
},
93+
"SequenceNumber":"1405400000000002063282832",
94+
"SizeBytes":54,
95+
"StreamViewType":"OLD_IMAGE"
96+
},
97+
"eventSourceARN":"arn:aws:dynamodb:us-east-1:123456789012:table/Example-Table/stream/2016-12-01T00:00:00.000"
98+
}
99+
]
100+
}

0 commit comments

Comments
 (0)