diff --git a/src/Amazon.CloudWatch.EMF/Constants.cs b/src/Amazon.CloudWatch.EMF/Constants.cs index 4c82964..040b052 100644 --- a/src/Amazon.CloudWatch.EMF/Constants.cs +++ b/src/Amazon.CloudWatch.EMF/Constants.cs @@ -8,6 +8,8 @@ public class Constants public const int MAX_METRICS_PER_EVENT = 100; + public const int MAX_DATAPOINTS_PER_METRIC = 100; + public const string DEFAULT_NAMESPACE = "aws-embedded-metrics"; } } \ No newline at end of file diff --git a/src/Amazon.CloudWatch.EMF/Model/MetricsContext.cs b/src/Amazon.CloudWatch.EMF/Model/MetricsContext.cs index b1d0ae8..2e0fe70 100644 --- a/src/Amazon.CloudWatch.EMF/Model/MetricsContext.cs +++ b/src/Amazon.CloudWatch.EMF/Model/MetricsContext.cs @@ -206,20 +206,48 @@ public void PutMetadata(string key, object value) public List Serialize() { var nodes = new List(); - if (_rootNode.AWS.MetricDirective.Metrics.Count <= Constants.MAX_METRICS_PER_EVENT) + + if (_rootNode.AWS.MetricDirective.Metrics.Count <= Constants.MAX_METRICS_PER_EVENT && NoMetricWithTooManyDataPoints(_rootNode)) { nodes.Add(_rootNode); } else { - // split the root nodes into multiple and serialize each - var count = 0; - while (count < _rootNode.AWS.MetricDirective.Metrics.Count) + Dictionary metrics = new Dictionary(); + Queue metricDefinitions = + new Queue(_rootNode.AWS.MetricDirective.Metrics); + while (metricDefinitions.Count > 0) { - var metrics = _rootNode.AWS.MetricDirective.Metrics.Skip(count).Take(Constants.MAX_METRICS_PER_EVENT).ToList(); - var node = _rootNode.DeepCloneWithNewMetrics(metrics); + MetricDefinition metric = metricDefinitions.Dequeue(); + + if (metrics.Count == Constants.MAX_METRICS_PER_EVENT || metrics.ContainsKey(metric.Name)) + { + var node = _rootNode.DeepCloneWithNewMetrics(metrics.Values.ToList()); + nodes.Add(node); + metrics = new Dictionary(); + } + + if (metric.Values.Count <= Constants.MAX_DATAPOINTS_PER_METRIC) + { + metrics.Add(metric.Name, metric); + } + else + { + metrics.Add( + metric.Name, + new MetricDefinition(metric.Name, metric.Unit, metric.Values.Take(Constants.MAX_DATAPOINTS_PER_METRIC).ToList())); + metricDefinitions.Enqueue( + new MetricDefinition( + metric.Name, + metric.Unit, + metric.Values.Skip(Constants.MAX_DATAPOINTS_PER_METRIC).Take(metric.Values.Count).ToList())); + } + } + + if (metrics.Count > 0) + { + var node = _rootNode.DeepCloneWithNewMetrics(metrics.Values.ToList()); nodes.Add(node); - count += Constants.MAX_METRICS_PER_EVENT; } } @@ -231,5 +259,10 @@ public List Serialize() return results; } + + private bool NoMetricWithTooManyDataPoints(RootNode node) + { + return node.AWS.MetricDirective.Metrics.All(metric => metric.Values.Count <= Constants.MAX_DATAPOINTS_PER_METRIC); + } } } \ No newline at end of file diff --git a/src/Amazon.CloudWatch.EMF/Model/RootNode.cs b/src/Amazon.CloudWatch.EMF/Model/RootNode.cs index 0f76ee5..1d22ec6 100644 --- a/src/Amazon.CloudWatch.EMF/Model/RootNode.cs +++ b/src/Amazon.CloudWatch.EMF/Model/RootNode.cs @@ -95,6 +95,11 @@ internal RootNode DeepCloneWithNewMetrics(List metrics) { var clone = new RootNode(); clone.AWS = AWS.DeepCloneWithNewMetrics(metrics); + foreach (var property in GetProperties()) + { + clone.PutProperty(property.Key, property.Value); + } + return clone; } diff --git a/tests/Amazon.CloudWatch.EMF.Tests/Model/RootNodeTests.cs b/tests/Amazon.CloudWatch.EMF.Tests/Model/RootNodeTests.cs index 1504a9b..507b406 100644 --- a/tests/Amazon.CloudWatch.EMF.Tests/Model/RootNodeTests.cs +++ b/tests/Amazon.CloudWatch.EMF.Tests/Model/RootNodeTests.cs @@ -1,4 +1,6 @@ +using System; using System.Collections.Generic; +using System.Linq; using Amazon.CloudWatch.EMF.Model; using Newtonsoft.Json; using Xunit; @@ -13,7 +15,7 @@ public void PutProperty_SavesKeyValue() RootNode rootNode = new RootNode(); rootNode.PutProperty("Property", "Value"); - Assert.Equal( "Value",rootNode.GetProperties()["Property"]); + Assert.Equal("Value", rootNode.GetProperties()["Property"]); } [Fact] @@ -28,14 +30,14 @@ public void Serialize_Returns_ValidValues() mc.Namespace = "test-namespace"; List emfLogs = mc.Serialize(); - + var emfMap = JsonConvert.DeserializeObject>(emfLogs[0]); - Assert.Equal( "DefaultDimValue", emfMap["DefaultDim"]); - Assert.Equal( "us-east-1", emfMap["Region"]); + Assert.Equal("DefaultDimValue", emfMap["DefaultDim"]); + Assert.Equal("us-east-1", emfMap["Region"]); - Assert.Equal( 10.0, emfMap["Count"]); - Assert.Equal( "PropertyValue", emfMap["Property"]); + Assert.Equal(10.0, emfMap["Count"]); + Assert.Equal("PropertyValue", emfMap["Property"]); var metadata = JsonConvert.DeserializeObject>(emfMap["_aws"].ToString()); Assert.True(metadata.ContainsKey("Timestamp")); Assert.True(metadata.ContainsKey("CloudWatchMetrics")); @@ -69,5 +71,97 @@ public void MetadataNode_IsSerialized_IfMetricsArePresent() // assert Assert.Contains("_aws", json); } + + [Fact] + public void Serialize_MoreThan100DataPoints() + { + MetricsContext mc = new MetricsContext(); + const int expectedEmfLogs = 2; + const int dataPointCount = 102; + + mc.DefaultDimensions.AddDimension("DefaultDim", "DefaultDimValue"); + mc.PutDimension("Region", "us-east-1"); + for (var i = 0; i < dataPointCount; i++) + { + mc.PutMetric("Count", i); + } + + mc.PutProperty("Property", "PropertyValue"); + mc.Namespace = "test-namespace"; + + List emfLogs = mc.Serialize(); + Assert.Equal(expectedEmfLogs, emfLogs.Count); + + List> allMetricValues = new List>(); + foreach (var emfLog in emfLogs) + { + var emfMap = JsonConvert.DeserializeObject>(emfLog); + var metricValues = JsonConvert.DeserializeObject>(emfMap["Count"].ToString()); + allMetricValues.Add(metricValues); + + Assert.Equal("DefaultDimValue", emfMap["DefaultDim"]); + Assert.Equal("us-east-1", emfMap["Region"]); + Assert.Equal("PropertyValue", emfMap["Property"]); + + var metadata = JsonConvert.DeserializeObject>(emfMap["_aws"].ToString()); + Assert.True(metadata.ContainsKey("Timestamp")); + Assert.True(metadata.ContainsKey("CloudWatchMetrics")); + } + + List expectedValues = new List(); + for (int i = 0; i < dataPointCount; i++) + { + expectedValues.Add(i); + } + + Assert.Equal(expectedValues.Take(Constants.MAX_DATAPOINTS_PER_METRIC), allMetricValues[0] as List); + Assert.Equal(expectedValues.Skip(Constants.MAX_DATAPOINTS_PER_METRIC), allMetricValues[1] as List); + } + + [Fact] + public void Serialize_MoreThan100Metrics() + { + MetricsContext mc = new MetricsContext(); + const int expectedEmfLogs = 2; + const int metricCount = 101; + + mc.DefaultDimensions.AddDimension("DefaultDim", "DefaultDimValue"); + mc.PutDimension("Region", "us-east-1"); + for (var i = 0; i < metricCount; i++) + { + mc.PutMetric("Count" + i, i); + } + + mc.PutProperty("Property", "PropertyValue"); + mc.Namespace = "test-namespace"; + + List emfLogs = mc.Serialize(); + Assert.Equal(expectedEmfLogs, emfLogs.Count); + + List> allMetrics = new List>(); + for (int i = 0; i < expectedEmfLogs; i++) + { + var emfMap = JsonConvert.DeserializeObject>(emfLogs[i]); + + allMetrics.Add(emfMap.Where(pair => pair.Key.Contains("Count")).Select(pair => pair.Key).ToList()); + + Assert.Equal("DefaultDimValue", emfMap["DefaultDim"]); + Assert.Equal("us-east-1", emfMap["Region"]); + Assert.Equal("PropertyValue", emfMap["Property"]); + + var metadata = JsonConvert.DeserializeObject>(emfMap["_aws"].ToString()); + Assert.True(metadata.ContainsKey("Timestamp")); + Assert.True(metadata.ContainsKey("CloudWatchMetrics")); + } + + Assert.Equal(Constants.MAX_METRICS_PER_EVENT, allMetrics[0].Count); + for (var i = 0; i < Constants.MAX_METRICS_PER_EVENT; i++) + { + Assert.Contains("Count" + i, allMetrics[0]); + } + + Assert.Single(allMetrics[1]); + Assert.Contains("Count100", allMetrics[1]); + } } } \ No newline at end of file