Handle more than 100 metric values

src/Amazon.CloudWatch.EMF/Constants.cs

@@ -8,6 +8,8 @@ public class Constants
 
         public const int MAX_METRICS_PER_EVENT = 100;
 
+        public const int MAX_DATAPOINTS_PER_METRIC = 100;
+
         public const string DEFAULT_NAMESPACE = "aws-embedded-metrics";
     }
 }

src/Amazon.CloudWatch.EMF/Model/MetricsContext.cs

@@ -206,20 +206,48 @@ public void PutMetadata(string key, object value)
         public List<string> Serialize()
         {
             var nodes = new List<RootNode>();
-            if (_rootNode.AWS.MetricDirective.Metrics.Count <= Constants.MAX_METRICS_PER_EVENT)
+
+            if (_rootNode.AWS.MetricDirective.Metrics.Count <= Constants.MAX_METRICS_PER_EVENT && NoMetricWithTooManyDataPoints(_rootNode))
             {
                 nodes.Add(_rootNode);
             }
             else
             {
-                // split the root nodes into multiple and serialize each
-                var count = 0;
-                while (count < _rootNode.AWS.MetricDirective.Metrics.Count)
+                Dictionary<String, MetricDefinition> metrics = new Dictionary<string, MetricDefinition>();
+                Queue<MetricDefinition> metricDefinitions =
+                    new Queue<MetricDefinition>(_rootNode.AWS.MetricDirective.Metrics);
+                while (metricDefinitions.Count > 0)
                 {
-                    var metrics = _rootNode.AWS.MetricDirective.Metrics.Skip(count).Take(Constants.MAX_METRICS_PER_EVENT).ToList();
-                    var node = _rootNode.DeepCloneWithNewMetrics(metrics);
+                    MetricDefinition metric = metricDefinitions.Dequeue();
+
+                    if (metrics.Count == Constants.MAX_METRICS_PER_EVENT || metrics.ContainsKey(metric.Name))
+                    {
+                        var node = _rootNode.DeepCloneWithNewMetrics(metrics.Values.ToList());
+                        nodes.Add(node);
+                        metrics = new Dictionary<string, MetricDefinition>();
+                    }
+
+                    if (metric.Values.Count <= Constants.MAX_DATAPOINTS_PER_METRIC)
+                    {
+                        metrics.Add(metric.Name, metric);
+                    }
+                    else
+                    {
+                        metrics.Add(
+                            metric.Name,
+                            new MetricDefinition(metric.Name, metric.Unit, metric.Values.Take(Constants.MAX_DATAPOINTS_PER_METRIC).ToList()));
+                        metricDefinitions.Enqueue(
+                            new MetricDefinition(
+                                metric.Name,
+                                metric.Unit,
+                                metric.Values.Skip(Constants.MAX_DATAPOINTS_PER_METRIC).Take(metric.Values.Count).ToList()));
+                    }
+                }
+
+                if (metrics.Count > 0)
+                {
+                    var node = _rootNode.DeepCloneWithNewMetrics(metrics.Values.ToList());
                     nodes.Add(node);
-                    count += Constants.MAX_METRICS_PER_EVENT;
                 }
             }
 
@@ -231,5 +259,10 @@ public List<string> Serialize()
 
             return results;
         }
+
+        private bool NoMetricWithTooManyDataPoints(RootNode node)
+        {
+            return node.AWS.MetricDirective.Metrics.All(metric => metric.Values.Count <= Constants.MAX_DATAPOINTS_PER_METRIC);
+        }
     }
 }

src/Amazon.CloudWatch.EMF/Model/RootNode.cs

@@ -95,6 +95,11 @@ internal RootNode DeepCloneWithNewMetrics(List<MetricDefinition> metrics)
         {
             var clone = new RootNode();
             clone.AWS = AWS.DeepCloneWithNewMetrics(metrics);
+            foreach (var property in GetProperties())
+            {
+                clone.PutProperty(property.Key, property.Value);
+            }
+
             return clone;
         }
 

tests/Amazon.CloudWatch.EMF.Tests/Model/RootNodeTests.cs

@@ -1,4 +1,6 @@
+using System;
 using System.Collections.Generic;
+using System.Linq;
 using Amazon.CloudWatch.EMF.Model;
 using Newtonsoft.Json;
 using Xunit;
@@ -13,7 +15,7 @@ public void PutProperty_SavesKeyValue()
             RootNode rootNode = new RootNode();
             rootNode.PutProperty("Property", "Value");
 
-            Assert.Equal( "Value",rootNode.GetProperties()["Property"]);
+            Assert.Equal("Value", rootNode.GetProperties()["Property"]);
         }
 
         [Fact]
@@ -28,14 +30,14 @@ public void Serialize_Returns_ValidValues()
             mc.Namespace = "test-namespace";
 
             List<string> emfLogs = mc.Serialize();
-            
+
             var emfMap = JsonConvert.DeserializeObject<Dictionary<string, object>>(emfLogs[0]);
 
-            Assert.Equal( "DefaultDimValue", emfMap["DefaultDim"]);
-            Assert.Equal( "us-east-1", emfMap["Region"]);
+            Assert.Equal("DefaultDimValue", emfMap["DefaultDim"]);
+            Assert.Equal("us-east-1", emfMap["Region"]);
 
-            Assert.Equal( 10.0, emfMap["Count"]);
-            Assert.Equal( "PropertyValue", emfMap["Property"]);
+            Assert.Equal(10.0, emfMap["Count"]);
+            Assert.Equal("PropertyValue", emfMap["Property"]);
             var metadata = JsonConvert.DeserializeObject<Dictionary<string, object>>(emfMap["_aws"].ToString());
             Assert.True(metadata.ContainsKey("Timestamp"));
             Assert.True(metadata.ContainsKey("CloudWatchMetrics"));
@@ -69,5 +71,97 @@ public void MetadataNode_IsSerialized_IfMetricsArePresent()
             // assert
             Assert.Contains("_aws", json);
         }
+
+        [Fact]
+        public void Serialize_MoreThan100DataPoints()
+        {
+            MetricsContext mc = new MetricsContext();
+            const int expectedEmfLogs = 2;
+            const int dataPointCount = 102;
+
+            mc.DefaultDimensions.AddDimension("DefaultDim", "DefaultDimValue");
+            mc.PutDimension("Region", "us-east-1");
+            for (var i = 0; i < dataPointCount; i++)
+            {
+                mc.PutMetric("Count", i);
+            }
+
+            mc.PutProperty("Property", "PropertyValue");
+            mc.Namespace = "test-namespace";
+
+            List<string> emfLogs = mc.Serialize();
+            Assert.Equal(expectedEmfLogs, emfLogs.Count);
+
+            List<IList<Double>> allMetricValues = new List<IList<Double>>();
+            foreach (var emfLog in emfLogs)
+            {
+                var emfMap = JsonConvert.DeserializeObject<Dictionary<string, object>>(emfLog);
+                var metricValues = JsonConvert.DeserializeObject<List<Double>>(emfMap["Count"].ToString());
+                allMetricValues.Add(metricValues);
+
+                Assert.Equal("DefaultDimValue", emfMap["DefaultDim"]);
+                Assert.Equal("us-east-1", emfMap["Region"]);
+                Assert.Equal("PropertyValue", emfMap["Property"]);
+
+                var metadata = JsonConvert.DeserializeObject<Dictionary<string, object>>(emfMap["_aws"].ToString());
+                Assert.True(metadata.ContainsKey("Timestamp"));
+                Assert.True(metadata.ContainsKey("CloudWatchMetrics"));
+            }
+
+            List<Double> expectedValues = new List<Double>();
+            for (int i = 0; i < dataPointCount; i++)
+            {
+                expectedValues.Add(i);
+            }
+
+            Assert.Equal(expectedValues.Take(Constants.MAX_DATAPOINTS_PER_METRIC), allMetricValues[0] as List<Double>);
+            Assert.Equal(expectedValues.Skip(Constants.MAX_DATAPOINTS_PER_METRIC), allMetricValues[1] as List<Double>);
+        }
+
+        [Fact]
+        public void Serialize_MoreThan100Metrics()
+        {
+            MetricsContext mc = new MetricsContext();
+            const int expectedEmfLogs = 2;
+            const int metricCount = 101;
+
+            mc.DefaultDimensions.AddDimension("DefaultDim", "DefaultDimValue");
+            mc.PutDimension("Region", "us-east-1");
+            for (var i = 0; i < metricCount; i++)
+            {
+                mc.PutMetric("Count" + i, i);
+            }
+
+            mc.PutProperty("Property", "PropertyValue");
+            mc.Namespace = "test-namespace";
+
+            List<string> emfLogs = mc.Serialize();
+            Assert.Equal(expectedEmfLogs, emfLogs.Count);
+
+            List<List<string>> allMetrics = new List<List<string>>();
+            for (int i = 0; i < expectedEmfLogs; i++)
+            {
+                var emfMap = JsonConvert.DeserializeObject<Dictionary<string, object>>(emfLogs[i]);
+
+                allMetrics.Add(emfMap.Where(pair => pair.Key.Contains("Count")).Select(pair => pair.Key).ToList());
+
+                Assert.Equal("DefaultDimValue", emfMap["DefaultDim"]);
+                Assert.Equal("us-east-1", emfMap["Region"]);
+                Assert.Equal("PropertyValue", emfMap["Property"]);
+
+                var metadata = JsonConvert.DeserializeObject<Dictionary<string, object>>(emfMap["_aws"].ToString());
+                Assert.True(metadata.ContainsKey("Timestamp"));
+                Assert.True(metadata.ContainsKey("CloudWatchMetrics"));
+            }
+
+            Assert.Equal(Constants.MAX_METRICS_PER_EVENT, allMetrics[0].Count);
+            for (var i = 0; i < Constants.MAX_METRICS_PER_EVENT; i++)
+            {
+                Assert.Contains("Count" + i, allMetrics[0]);
+            }
+
+            Assert.Single(allMetrics[1]);
+            Assert.Contains("Count100", allMetrics[1]);
+        }
     }
 }