From dc47697d0a860c46dbbc06669558452ffa28e117 Mon Sep 17 00:00:00 2001 From: Yao Zhao Date: Fri, 18 Sep 2020 21:56:37 -0700 Subject: [PATCH 1/2] Handle log context with more than 100 metrics --- .../amazon/cloudwatchlogs/emf/Constants.java | 2 + .../cloudwatchlogs/emf/model/Metadata.java | 4 + .../emf/model/MetricDefinition.java | 23 ++++- .../emf/model/MetricDirective.java | 49 ++++++---- .../emf/model/MetricsContext.java | 42 ++++++-- .../cloudwatchlogs/emf/model/RootNode.java | 41 ++++---- .../emf/serializers/InstantSerializer.java | 1 - .../cloudwatchlogs/emf/sinks/AgentSink.java | 4 +- .../cloudwatchlogs/emf/sinks/ConsoleSink.java | 4 +- .../emf/model/MetricDefinitionTest.java | 12 ++- .../emf/model/MetricDirectiveTest.java | 28 +++++- .../emf/model/MetricsContextTest.java | 97 +++++++++++++++++++ .../emf/model/RootNodeTest.java | 23 +---- 13 files changed, 257 insertions(+), 73 deletions(-) create mode 100644 src/test/java/software/amazon/cloudwatchlogs/emf/model/MetricsContextTest.java diff --git a/src/main/java/software/amazon/cloudwatchlogs/emf/Constants.java b/src/main/java/software/amazon/cloudwatchlogs/emf/Constants.java index 443e651d..81e62c39 100644 --- a/src/main/java/software/amazon/cloudwatchlogs/emf/Constants.java +++ b/src/main/java/software/amazon/cloudwatchlogs/emf/Constants.java @@ -20,4 +20,6 @@ public class Constants { public static final int DEFAULT_AGENT_PORT = 25888; public static final String UNKNOWN = "Unknown"; + + public static final int MAX_METRICS_PER_EVENT = 100; } diff --git a/src/main/java/software/amazon/cloudwatchlogs/emf/model/Metadata.java b/src/main/java/software/amazon/cloudwatchlogs/emf/model/Metadata.java index d3f6c8ec..623eeabb 100644 --- a/src/main/java/software/amazon/cloudwatchlogs/emf/model/Metadata.java +++ b/src/main/java/software/amazon/cloudwatchlogs/emf/model/Metadata.java @@ -26,12 +26,15 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import lombok.AllArgsConstructor; import lombok.Getter; import lombok.Setter; +import lombok.With; import software.amazon.cloudwatchlogs.emf.serializers.InstantDeserializer; import software.amazon.cloudwatchlogs.emf.serializers.InstantSerializer; /** Represents the MetaData part of the EMF schema. */ +@AllArgsConstructor class Metadata { @Getter @@ -44,6 +47,7 @@ class Metadata { @Getter @Setter + @With @JsonProperty("CloudWatchMetrics") private List cloudWatchMetrics; diff --git a/src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricDefinition.java b/src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricDefinition.java index b4992acf..be9b9150 100644 --- a/src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricDefinition.java +++ b/src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricDefinition.java @@ -16,13 +16,16 @@ package software.amazon.cloudwatchlogs.emf.model; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NonNull; -import lombok.Setter; import software.amazon.cloudwatchlogs.emf.serializers.UnitDeserializer; import software.amazon.cloudwatchlogs.emf.serializers.UnitSerializer; @@ -30,19 +33,31 @@ @AllArgsConstructor class MetricDefinition { @NonNull - @Setter @Getter @JsonProperty("Name") private String name; - @Setter @Getter @JsonProperty("Unit") @JsonSerialize(using = UnitSerializer.class) @JsonDeserialize(using = UnitDeserializer.class) private Unit unit; + @JsonIgnore @NonNull @Getter private List values; + MetricDefinition(String name) { - this(name, Unit.NONE); + this(name, Unit.NONE, new ArrayList<>()); + } + + MetricDefinition(String name, double value) { + this(name, Unit.NONE, value); + } + + MetricDefinition(String name, Unit unit, double value) { + this(name, unit, new ArrayList<>(Arrays.asList(value))); + } + + void addValue(double value) { + values.add(value); } } diff --git a/src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricDirective.java b/src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricDirective.java index a43d6b3a..cc92d753 100644 --- a/src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricDirective.java +++ b/src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricDirective.java @@ -16,43 +16,58 @@ package software.amazon.cloudwatchlogs.emf.model; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; -import lombok.AccessLevel; -import lombok.Getter; -import lombok.Setter; +import lombok.*; /** Represents the MetricDirective part of the EMF schema. */ +@AllArgsConstructor class MetricDirective { @Setter @Getter @JsonProperty("Namespace") - private String namespace = "aws-embedded-metrics"; + private String namespace; - @Setter - @Getter - @JsonProperty("Metrics") - private List metrics = new ArrayList<>(); + @JsonIgnore @Setter @Getter @With private Map metrics; @Getter(AccessLevel.PROTECTED) - private List dimensions = new ArrayList<>(); + private List dimensions; @Setter @Getter(AccessLevel.PROTECTED) - private DimensionSet defaultDimensions = new DimensionSet(); + private DimensionSet defaultDimensions; - private boolean shouldUseDefaultDimension = true; + private boolean shouldUseDefaultDimension; + + MetricDirective() { + namespace = "aws-embedded-metrics"; + metrics = new HashMap<>(); + dimensions = new ArrayList<>(); + defaultDimensions = new DimensionSet(); + shouldUseDefaultDimension = true; + } void putDimensionSet(DimensionSet dimensionSet) { dimensions.add(dimensionSet); } - void putMetric(MetricDefinition metric) { - metrics.add(metric); + void putMetric(String key, double value) { + putMetric(key, value, Unit.NONE); + } + + void putMetric(String key, double value, Unit unit) { + if (metrics.containsKey(key)) { + metrics.get(key).addValue(value); + } else { + metrics.put(key, new MetricDefinition(key, unit, value)); + } + } + + @JsonProperty("Metrics") + Collection getAllMetrics() { + return metrics.values(); } @JsonProperty("Dimensions") diff --git a/src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricsContext.java b/src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricsContext.java index 1779d0f0..95b5dad7 100644 --- a/src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricsContext.java +++ b/src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricsContext.java @@ -17,10 +17,9 @@ package software.amazon.cloudwatchlogs.emf.model; import com.fasterxml.jackson.core.JsonProcessingException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; +import java.util.*; import lombok.Getter; +import software.amazon.cloudwatchlogs.emf.Constants; /** Stores metrics and their associated properties and dimensions. */ public class MetricsContext { @@ -101,8 +100,7 @@ public boolean hasDefaultDimensions() { * @param unit The unit of the metric */ public void putMetric(String key, double value, Unit unit) { - metricDirective.putMetric(new MetricDefinition(key, unit)); - rootNode.putMetric(key, value); + metricDirective.putMetric(key, value, unit); } /** @@ -201,12 +199,38 @@ public MetricsContext createCopyWithContext() { } /** - * Serialize the metrics in this context to a string. + * Serialize the metrics in this context to strings. * - * @return the serialized string + * @return the serialized strings. * @throws JsonProcessingException if there's any object that cannot be serialized */ - public String serialize() throws JsonProcessingException { - return this.rootNode.serialize(); + public List serialize() throws JsonProcessingException { + if (rootNode.metrics().size() <= Constants.MAX_METRICS_PER_EVENT) { + return Arrays.asList(this.rootNode.serialize()); + } else { + List nodes = new ArrayList<>(); + Map metrics = new HashMap<>(); + int count = 0; + for (MetricDefinition metric : rootNode.metrics().values()) { + metrics.put(metric.getName(), metric); + count++; + if (metrics.size() == Constants.MAX_METRICS_PER_EVENT + || count == rootNode.metrics().size()) { + Metadata metadata = rootNode.getAws(); + MetricDirective metricDirective = metadata.getCloudWatchMetrics().get(0); + Metadata clonedMetadata = + metadata.withCloudWatchMetrics( + Arrays.asList(metricDirective.withMetrics(metrics))); + nodes.add(rootNode.withAws(clonedMetadata)); + metrics = new HashMap<>(); + } + } + + List strings = new ArrayList<>(); + for (RootNode node : nodes) { + strings.add(node.serialize()); + } + return strings; + } } } diff --git a/src/main/java/software/amazon/cloudwatchlogs/emf/model/RootNode.java b/src/main/java/software/amazon/cloudwatchlogs/emf/model/RootNode.java index c9d33885..4049edf1 100644 --- a/src/main/java/software/amazon/cloudwatchlogs/emf/model/RootNode.java +++ b/src/main/java/software/amazon/cloudwatchlogs/emf/model/RootNode.java @@ -22,26 +22,32 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; +import lombok.AllArgsConstructor; import lombok.Getter; +import lombok.With; /** Represents the root of the EMF schema. */ +@AllArgsConstructor @JsonFilter("emptyMetricFilter") class RootNode { @Getter + @With @JsonProperty("_aws") - private Metadata aws = new Metadata(); + private Metadata aws; - private Map properties = new HashMap<>(); - private Map> metrics = new HashMap<>(); - private ObjectMapper objectMapper = new ObjectMapper(); + private Map properties; + private ObjectMapper objectMapper; RootNode() { final SimpleFilterProvider filterProvider = new SimpleFilterProvider().addFilter("emptyMetricFilter", new EmptyMetricsFilter()); + aws = new Metadata(); + properties = new HashMap<>(); + objectMapper = new ObjectMapper(); objectMapper.setFilterProvider(filterProvider); } @@ -49,17 +55,6 @@ public void putProperty(String key, Object value) { properties.put(key, value); } - /** - * Add a metric measurement. Multiple calls using the same key will be stored as an array of - * scalar values - */ - void putMetric(String key, double value) { - if (!metrics.containsKey(key)) { - metrics.put(key, new ArrayList<>()); - } - metrics.get(key).add(value); - } - Map getProperties() { return properties; } @@ -70,9 +65,13 @@ Map getTargetMembers() { Map targetMembers = new HashMap<>(); targetMembers.putAll(properties); targetMembers.putAll(getDimensions()); - for (Map.Entry> entry : metrics.entrySet()) { - List values = entry.getValue(); - targetMembers.put(entry.getKey(), values.size() == 1 ? values.get(0) : values); + List metrics = + aws.getCloudWatchMetrics().stream() + .flatMap(metricDirective -> metricDirective.getMetrics().values().stream()) + .collect(Collectors.toList()); + for (MetricDefinition metric : metrics) { + List values = metric.getValues(); + targetMembers.put(metric.getName(), values.size() == 1 ? values.get(0) : values); } return targetMembers; } @@ -88,6 +87,10 @@ Map getDimensions() { return dimensions; } + Map metrics() { + return aws.getCloudWatchMetrics().get(0).getMetrics(); + } + String serialize() throws JsonProcessingException { return objectMapper.writeValueAsString(this); } diff --git a/src/main/java/software/amazon/cloudwatchlogs/emf/serializers/InstantSerializer.java b/src/main/java/software/amazon/cloudwatchlogs/emf/serializers/InstantSerializer.java index 4f02f5dc..854a07c6 100644 --- a/src/main/java/software/amazon/cloudwatchlogs/emf/serializers/InstantSerializer.java +++ b/src/main/java/software/amazon/cloudwatchlogs/emf/serializers/InstantSerializer.java @@ -37,7 +37,6 @@ public class InstantSerializer extends StdSerializer { public void serialize(Instant value, JsonGenerator jgen, SerializerProvider provider) throws IOException, JsonProcessingException { - // Just serialize dimensions as an array. jgen.writeNumber(value.toEpochMilli()); } } diff --git a/src/main/java/software/amazon/cloudwatchlogs/emf/sinks/AgentSink.java b/src/main/java/software/amazon/cloudwatchlogs/emf/sinks/AgentSink.java index 4572492a..eea4e28e 100644 --- a/src/main/java/software/amazon/cloudwatchlogs/emf/sinks/AgentSink.java +++ b/src/main/java/software/amazon/cloudwatchlogs/emf/sinks/AgentSink.java @@ -48,7 +48,9 @@ public void accept(MetricsContext context) { } try { - client.sendMessage(context.serialize() + "\n"); + for (String event : context.serialize()) { + client.sendMessage(event + "\n"); + } } catch (JsonProcessingException e) { log.error("Failed to serialize the metrics with the exception: ", e); } diff --git a/src/main/java/software/amazon/cloudwatchlogs/emf/sinks/ConsoleSink.java b/src/main/java/software/amazon/cloudwatchlogs/emf/sinks/ConsoleSink.java index 9b5a4e9d..3501a000 100644 --- a/src/main/java/software/amazon/cloudwatchlogs/emf/sinks/ConsoleSink.java +++ b/src/main/java/software/amazon/cloudwatchlogs/emf/sinks/ConsoleSink.java @@ -33,7 +33,9 @@ public void accept(MetricsContext context) { try { // CHECKSTYLE OFF - System.out.println(context.serialize()); + for (String event : context.serialize()) { + System.out.println(event); + } // CHECKSTYLE ON } catch (JsonProcessingException e) { log.error("Failed to serialize a MetricsContext: ", e); diff --git a/src/test/java/software/amazon/cloudwatchlogs/emf/model/MetricDefinitionTest.java b/src/test/java/software/amazon/cloudwatchlogs/emf/model/MetricDefinitionTest.java index a131fc4f..2524eab4 100644 --- a/src/test/java/software/amazon/cloudwatchlogs/emf/model/MetricDefinitionTest.java +++ b/src/test/java/software/amazon/cloudwatchlogs/emf/model/MetricDefinitionTest.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Arrays; import org.junit.Test; public class MetricDefinitionTest { @@ -41,9 +42,18 @@ public void testSerializeMetricDefinitionWithoutUnit() throws JsonProcessingExce @Test public void testSerializeMetricDefinition() throws JsonProcessingException { ObjectMapper objectMapper = new ObjectMapper(); - MetricDefinition metricDefinition = new MetricDefinition("Time", Unit.MILLISECONDS); + MetricDefinition metricDefinition = new MetricDefinition("Time", Unit.MILLISECONDS, 10); String metricString = objectMapper.writeValueAsString(metricDefinition); assertEquals(metricString, "{\"Name\":\"Time\",\"Unit\":\"Milliseconds\"}"); } + + @Test + public void testAddValue() { + MetricDefinition md = new MetricDefinition("Time", Unit.MICROSECONDS, 10); + assertEquals(Arrays.asList(10d), md.getValues()); + + md.addValue(20); + assertEquals(Arrays.asList(10d, 20d), md.getValues()); + } } diff --git a/src/test/java/software/amazon/cloudwatchlogs/emf/model/MetricDirectiveTest.java b/src/test/java/software/amazon/cloudwatchlogs/emf/model/MetricDirectiveTest.java index d8480d88..43b8f085 100644 --- a/src/test/java/software/amazon/cloudwatchlogs/emf/model/MetricDirectiveTest.java +++ b/src/test/java/software/amazon/cloudwatchlogs/emf/model/MetricDirectiveTest.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Arrays; import org.junit.Test; public class MetricDirectiveTest { @@ -50,7 +51,7 @@ public void testSetNamespace() throws JsonProcessingException { @Test public void testPutMetric() throws JsonProcessingException { MetricDirective metricDirective = new MetricDirective(); - metricDirective.putMetric(new MetricDefinition("Time")); + metricDirective.putMetric("Time", 10); String serializedMetricDirective = objectMapper.writeValueAsString(metricDirective); @@ -59,6 +60,31 @@ public void testPutMetric() throws JsonProcessingException { "{\"Namespace\":\"aws-embedded-metrics\",\"Metrics\":[{\"Name\":\"Time\",\"Unit\":\"None\"}],\"Dimensions\":[[]]}"); } + @Test + public void testPutSameMetricMultipleTimes() { + MetricDirective metricDirective = new MetricDirective(); + metricDirective.putMetric("Time", 10); + metricDirective.putMetric("Time", 20); + + assertEquals(1, metricDirective.getAllMetrics().size()); + MetricDefinition[] mds = metricDirective.getAllMetrics().toArray(new MetricDefinition[0]); + assertEquals(mds[0].getValues(), Arrays.asList(10d, 20d)); + } + + @Test + public void testPutMetricWithoutUnit() { + MetricDirective metricDirective = new MetricDirective(); + metricDirective.putMetric("Time", 10); + assertEquals(metricDirective.getMetrics().get("Time").getUnit(), Unit.NONE); + } + + @Test + public void testPutMetricWithUnit() { + MetricDirective metricDirective = new MetricDirective(); + metricDirective.putMetric("Time", 10, Unit.MILLISECONDS); + assertEquals(metricDirective.getMetrics().get("Time").getUnit(), Unit.MILLISECONDS); + } + @Test public void testPutDimensions() throws JsonProcessingException { MetricDirective metricDirective = new MetricDirective(); diff --git a/src/test/java/software/amazon/cloudwatchlogs/emf/model/MetricsContextTest.java b/src/test/java/software/amazon/cloudwatchlogs/emf/model/MetricsContextTest.java new file mode 100644 index 00000000..d9c27f03 --- /dev/null +++ b/src/test/java/software/amazon/cloudwatchlogs/emf/model/MetricsContextTest.java @@ -0,0 +1,97 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package software.amazon.cloudwatchlogs.emf.model; + +import static org.junit.Assert.assertEquals; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.json.JsonMapper; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.junit.Test; + +public class MetricsContextTest { + + @Test + public void testSerializeLessThan100Metrics() throws JsonProcessingException { + MetricsContext mc = new MetricsContext(); + int metricCount = 10; + for (int i = 0; i < metricCount; i++) { + String key = "Metric-" + i; + mc.putMetric(key, i); + } + + List events = mc.serialize(); + assertEquals(1, events.size()); + + List metrics = parseMetrics(events.get(0)); + assertEquals(metrics.size(), metricCount); + for (MetricDefinition metric : metrics) { + MetricDefinition originalMetric = mc.getRootNode().metrics().get(metric.getName()); + assertEquals(originalMetric.getName(), metric.getName()); + assertEquals(originalMetric.getUnit(), metric.getUnit()); + } + } + + @Test + public void testSerializeMoreThen100Metrics() throws JsonProcessingException { + MetricsContext mc = new MetricsContext(); + int metricCount = 253; + int expectedEventCount = 3; + for (int i = 0; i < metricCount; i++) { + String key = "Metric-" + i; + mc.putMetric(key, i); + } + + List events = mc.serialize(); + assertEquals(expectedEventCount, events.size()); + + List allMetrics = new ArrayList<>(); + for (String event : events) { + allMetrics.addAll(parseMetrics(event)); + } + assertEquals(metricCount, allMetrics.size()); + for (MetricDefinition metric : allMetrics) { + MetricDefinition originalMetric = mc.getRootNode().metrics().get(metric.getName()); + assertEquals(originalMetric.getName(), metric.getName()); + assertEquals(originalMetric.getUnit(), metric.getUnit()); + } + } + + @SuppressWarnings("unchecked") + private ArrayList parseMetrics(String event) throws JsonProcessingException { + JsonMapper objectMapper = new JsonMapper(); + Map metadata_map = + objectMapper.readValue(event, new TypeReference>() {}); + Map metadata = (Map) metadata_map.get("_aws"); + ArrayList> metricDirectives = + (ArrayList>) metadata.get("CloudWatchMetrics"); + ArrayList> metrics = + (ArrayList>) metricDirectives.get(0).get("Metrics"); + + ArrayList metricDefinitions = new ArrayList<>(); + for (Map metric : metrics) { + String name = metric.get("Name"); + Unit unit = Unit.fromValue(metric.get("Unit")); + double value = (double) metadata_map.get(name); + metricDefinitions.add(new MetricDefinition(name, unit, value)); + } + return metricDefinitions; + } +} diff --git a/src/test/java/software/amazon/cloudwatchlogs/emf/model/RootNodeTest.java b/src/test/java/software/amazon/cloudwatchlogs/emf/model/RootNodeTest.java index 98ca9736..0e91f544 100644 --- a/src/test/java/software/amazon/cloudwatchlogs/emf/model/RootNodeTest.java +++ b/src/test/java/software/amazon/cloudwatchlogs/emf/model/RootNodeTest.java @@ -23,28 +23,12 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import java.util.Arrays; +import java.util.List; import java.util.Map; import org.junit.Test; public class RootNodeTest { - @Test - public void testPutMetric() { - RootNode rootNode = new RootNode(); - rootNode.putMetric("Time", 10.0); - - assertEquals(rootNode.getTargetMembers().get("Time"), 10.0); - } - - @Test - public void testPutSameMetricMultipleTimes() { - RootNode rootNode = new RootNode(); - rootNode.putMetric("Time", 10.0); - rootNode.putMetric("Time", 20.0); - - assertEquals(rootNode.getTargetMembers().get("Time"), Arrays.asList(10.0, 20.0)); - } - @Test public void testPutProperty() { RootNode rootNode = new RootNode(); @@ -103,9 +87,10 @@ public void testSerializeRootNode() throws JsonProcessingException { mc.putProperty("Property", "PropertyValue"); ObjectMapper objectMapper = new ObjectMapper(); - String emf_log = mc.serialize(); + List emf_logs = mc.serialize(); Map emf_map = - objectMapper.readValue(emf_log, new TypeReference>() {}); + objectMapper.readValue( + emf_logs.get(0), new TypeReference>() {}); assertEquals(emf_map.keySet().size(), 5); assertEquals(emf_map.get("Region"), "us-east-1"); From 3565e1b76f5d4e4708bd53cac50788488c9f3438 Mon Sep 17 00:00:00 2001 From: Yao Zhao Date: Mon, 21 Sep 2020 10:13:33 -0700 Subject: [PATCH 2/2] Addressed comments --- .../emf/model/MetricsContext.java | 4 +++- .../cloudwatchlogs/emf/model/RootNode.java | 13 +++++-------- .../emf/model/MetricsContextTest.java | 17 +++++++++++++++++ 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricsContext.java b/src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricsContext.java index 95b5dad7..b745e7b1 100644 --- a/src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricsContext.java +++ b/src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricsContext.java @@ -199,7 +199,9 @@ public MetricsContext createCopyWithContext() { } /** - * Serialize the metrics in this context to strings. + * Serialize the metrics in this context to strings. The EMF backend requires no more than 100 + * metrics in one log event. If there're more than 100 metrics, we split the metrics into + * multiple log events. * * @return the serialized strings. * @throws JsonProcessingException if there's any object that cannot be serialized diff --git a/src/main/java/software/amazon/cloudwatchlogs/emf/model/RootNode.java b/src/main/java/software/amazon/cloudwatchlogs/emf/model/RootNode.java index 4049edf1..a3ee234d 100644 --- a/src/main/java/software/amazon/cloudwatchlogs/emf/model/RootNode.java +++ b/src/main/java/software/amazon/cloudwatchlogs/emf/model/RootNode.java @@ -25,7 +25,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.With; @@ -65,13 +64,11 @@ Map getTargetMembers() { Map targetMembers = new HashMap<>(); targetMembers.putAll(properties); targetMembers.putAll(getDimensions()); - List metrics = - aws.getCloudWatchMetrics().stream() - .flatMap(metricDirective -> metricDirective.getMetrics().values().stream()) - .collect(Collectors.toList()); - for (MetricDefinition metric : metrics) { - List values = metric.getValues(); - targetMembers.put(metric.getName(), values.size() == 1 ? values.get(0) : values); + for (MetricDirective metricDirective : aws.getCloudWatchMetrics()) { + for (MetricDefinition metric : metricDirective.getMetrics().values()) { + List values = metric.getValues(); + targetMembers.put(metric.getName(), values.size() == 1 ? values.get(0) : values); + } } return targetMembers; } diff --git a/src/test/java/software/amazon/cloudwatchlogs/emf/model/MetricsContextTest.java b/src/test/java/software/amazon/cloudwatchlogs/emf/model/MetricsContextTest.java index d9c27f03..367de33d 100644 --- a/src/test/java/software/amazon/cloudwatchlogs/emf/model/MetricsContextTest.java +++ b/src/test/java/software/amazon/cloudwatchlogs/emf/model/MetricsContextTest.java @@ -17,6 +17,7 @@ package software.amazon.cloudwatchlogs.emf.model; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; @@ -74,6 +75,22 @@ public void testSerializeMoreThen100Metrics() throws JsonProcessingException { } } + @Test + public void testSerializeZeroMetric() throws JsonProcessingException { + MetricsContext mc = new MetricsContext(); + mc.putDimension(DimensionSet.of("Region", "IAD")); + List events = mc.serialize(); + + int expectedEventCount = 1; + assertEquals(expectedEventCount, events.size()); + + JsonMapper objectMapper = new JsonMapper(); + Map metadata_map = + objectMapper.readValue(events.get(0), new TypeReference>() {}); + // If there's no metric added, the _aws would be filtered out from the log event + assertFalse(metadata_map.containsKey("_aws")); + } + @SuppressWarnings("unchecked") private ArrayList parseMetrics(String event) throws JsonProcessingException { JsonMapper objectMapper = new JsonMapper();