Skip to content

Commit c3816d5

Browse files
yaozhaoyyaozhaoy
authored
Split a metric when there're more than 100 data points (#74)
* Split a metric when there're more than 100 data points
1 parent 43840bf commit c3816d5

File tree

3 files changed

+112
-15
lines changed

3 files changed

+112
-15
lines changed

‎src/main/java/software/amazon/cloudwatchlogs/emf/Constants.java

+2
Original file line numberDiff line numberDiff line change
@@ -22,4 +22,6 @@ public class Constants {
2222
public static final String UNKNOWN = "Unknown";
2323

2424
public static final int MAX_METRICS_PER_EVENT = 100;
25+
26+
public static final int MAX_DATAPOINTS_PER_METRIC = 100;
2527
}

‎src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricsContext.java

+48-13
Original file line numberDiff line numberDiff line change
@@ -203,36 +203,71 @@ public MetricsContext createCopyWithContext() {
203203
* metrics in one log event. If there're more than 100 metrics, we split the metrics into
204204
* multiple log events.
205205
*
206+
* <p>If a metric has more than 100 data points, we also split the metric.
207+
*
206208
* @return the serialized strings.
207209
* @throws JsonProcessingException if there's any object that cannot be serialized
208210
*/
209211
public List<String> serialize() throws JsonProcessingException {
210-
if (rootNode.metrics().size() <= Constants.MAX_METRICS_PER_EVENT) {
212+
if (rootNode.metrics().size() <= Constants.MAX_METRICS_PER_EVENT
213+
&& !anyMetricWithTooManyDataPoints(rootNode)) {
211214
return Arrays.asList(this.rootNode.serialize());
212215
} else {
213216
List<RootNode> nodes = new ArrayList<>();
214217
Map<String, MetricDefinition> metrics = new HashMap<>();
215-
int count = 0;
216-
for (MetricDefinition metric : rootNode.metrics().values()) {
217-
metrics.put(metric.getName(), metric);
218-
count++;
218+
Queue<MetricDefinition> metricDefinitions =
219+
new LinkedList<>(rootNode.metrics().values());
220+
while (metricDefinitions.size() > 0) {
221+
MetricDefinition metric = metricDefinitions.poll();
222+
219223
if (metrics.size() == Constants.MAX_METRICS_PER_EVENT
220-
|| count == rootNode.metrics().size()) {
221-
Metadata metadata = rootNode.getAws();
222-
MetricDirective metricDirective = metadata.getCloudWatchMetrics().get(0);
223-
Metadata clonedMetadata =
224-
metadata.withCloudWatchMetrics(
225-
Arrays.asList(metricDirective.withMetrics(metrics)));
226-
nodes.add(rootNode.withAws(clonedMetadata));
224+
|| metrics.containsKey(metric.getName())) {
225+
nodes.add(buildRootNode(metrics));
227226
metrics = new HashMap<>();
228227
}
229-
}
230228

229+
if (metric.getValues().size() <= Constants.MAX_DATAPOINTS_PER_METRIC) {
230+
metrics.put(metric.getName(), metric);
231+
} else {
232+
metrics.put(
233+
metric.getName(),
234+
new MetricDefinition(
235+
metric.getName(),
236+
metric.getUnit(),
237+
metric.getValues()
238+
.subList(0, Constants.MAX_DATAPOINTS_PER_METRIC)));
239+
metricDefinitions.offer(
240+
new MetricDefinition(
241+
metric.getName(),
242+
metric.getUnit(),
243+
metric.getValues()
244+
.subList(
245+
Constants.MAX_DATAPOINTS_PER_METRIC,
246+
metric.getValues().size())));
247+
}
248+
}
249+
if (!metrics.isEmpty()) {
250+
nodes.add(buildRootNode(metrics));
251+
}
231252
List<String> strings = new ArrayList<>();
232253
for (RootNode node : nodes) {
233254
strings.add(node.serialize());
234255
}
235256
return strings;
236257
}
237258
}
259+
260+
private RootNode buildRootNode(Map<String, MetricDefinition> metrics) {
261+
Metadata metadata = rootNode.getAws();
262+
MetricDirective metricDirective = metadata.getCloudWatchMetrics().get(0);
263+
Metadata clonedMetadata =
264+
metadata.withCloudWatchMetrics(Arrays.asList(metricDirective.withMetrics(metrics)));
265+
return rootNode.withAws(clonedMetadata);
266+
}
267+
268+
private boolean anyMetricWithTooManyDataPoints(RootNode node) {
269+
return node.metrics().values().stream()
270+
.anyMatch(
271+
metric -> metric.getValues().size() > Constants.MAX_DATAPOINTS_PER_METRIC);
272+
}
238273
}

‎src/test/java/software/amazon/cloudwatchlogs/emf/model/MetricsContextTest.java

+62-2
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,17 @@
1818

1919
import static org.junit.Assert.assertEquals;
2020
import static org.junit.Assert.assertFalse;
21+
import static org.junit.Assert.assertTrue;
2122

2223
import com.fasterxml.jackson.core.JsonProcessingException;
2324
import com.fasterxml.jackson.core.type.TypeReference;
2425
import com.fasterxml.jackson.databind.json.JsonMapper;
2526
import java.util.ArrayList;
27+
import java.util.Arrays;
2628
import java.util.List;
2729
import java.util.Map;
2830
import org.junit.Test;
31+
import software.amazon.cloudwatchlogs.emf.Constants;
2932

3033
public class MetricsContextTest {
3134

@@ -75,6 +78,59 @@ public void testSerializeMoreThen100Metrics() throws JsonProcessingException {
7578
}
7679
}
7780

81+
@Test
82+
public void testSerializeAMetricWith101DataPoints() throws JsonProcessingException {
83+
MetricsContext mc = new MetricsContext();
84+
int dataPointCount = 101;
85+
int expectedEventCount = 2;
86+
String metricName = "metric";
87+
for (int i = 0; i < dataPointCount; i++) {
88+
mc.putMetric(metricName, i);
89+
}
90+
91+
List<String> events = mc.serialize();
92+
assertEquals(expectedEventCount, events.size());
93+
List<MetricDefinition> allMetrics = new ArrayList<>();
94+
for (String event : events) {
95+
allMetrics.addAll(parseMetrics(event));
96+
}
97+
List<Double> expectedValues = new ArrayList<>();
98+
for (int i = 0; i < Constants.MAX_DATAPOINTS_PER_METRIC; i++) {
99+
expectedValues.add((double) i);
100+
}
101+
assertEquals(expectedValues, allMetrics.get(0).getValues());
102+
assertTrue(allMetrics.get(1).getValues().equals(Arrays.asList(100.0)));
103+
}
104+
105+
@Test
106+
public void testSerializeMetricsWith101DataPoints() throws JsonProcessingException {
107+
MetricsContext mc = new MetricsContext();
108+
int dataPointCount = 101;
109+
int expectedEventCount = 2;
110+
String metricName = "metric1";
111+
for (int i = 0; i < dataPointCount; i++) {
112+
mc.putMetric(metricName, i);
113+
}
114+
mc.putMetric("metric2", 2);
115+
116+
List<String> events = mc.serialize();
117+
assertEquals(expectedEventCount, events.size());
118+
119+
List<MetricDefinition> metricsFromEvent1 = parseMetrics(events.get(0));
120+
List<MetricDefinition> metricsFromEvent2 = parseMetrics(events.get(1));
121+
122+
assertEquals(2, metricsFromEvent1.size());
123+
List<Double> expectedValues = new ArrayList<>();
124+
for (int i = 0; i < Constants.MAX_DATAPOINTS_PER_METRIC; i++) {
125+
expectedValues.add((double) i);
126+
}
127+
assertEquals(expectedValues, metricsFromEvent1.get(0).getValues());
128+
assertEquals(Arrays.asList(2.0), metricsFromEvent1.get(1).getValues());
129+
130+
assertEquals(1, metricsFromEvent2.size());
131+
assertEquals(Arrays.asList(100.0), metricsFromEvent2.get(0).getValues());
132+
}
133+
78134
@Test
79135
public void testSerializeZeroMetric() throws JsonProcessingException {
80136
MetricsContext mc = new MetricsContext();
@@ -106,8 +162,12 @@ private ArrayList<MetricDefinition> parseMetrics(String event) throws JsonProces
106162
for (Map<String, String> metric : metrics) {
107163
String name = metric.get("Name");
108164
Unit unit = Unit.fromValue(metric.get("Unit"));
109-
double value = (double) metadata_map.get(name);
110-
metricDefinitions.add(new MetricDefinition(name, unit, value));
165+
Object value = metadata_map.get(name);
166+
if (value instanceof ArrayList) {
167+
metricDefinitions.add(new MetricDefinition(name, unit, (ArrayList) value));
168+
} else {
169+
metricDefinitions.add(new MetricDefinition(name, unit, (double) value));
170+
}
111171
}
112172
return metricDefinitions;
113173
}

0 commit comments

Comments
 (0)