Skip to content

Commit c404b63

Browse files
meshwa19meshwa19
authored
High Resolution Metrics Support (#129)
1 parent a8a9765 commit c404b63

File tree

17 files changed

+376
-20
lines changed

17 files changed

+376
-20
lines changed

‎README.md

+13-2
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,8 @@ class Example {
3838

3939
try {
4040
metrics.putDimensions(DimensionSet.of("Service", "Aggregator"));
41-
metrics.putMetric("ProcessingLatency", 100, Unit.MILLISECONDS);
41+
metrics.putMetric("ProcessingLatency", 100, Unit.MILLISECONDS, StorageResolution.STANDARD);
42+
metrics.putMetric("Memory.HeapUsed", 1600424.0, Unit.BYTES, StorageResolution.HIGH);
4243
} catch (InvalidDimensionException | InvalidMetricException e) {
4344
log.error(e);
4445
}
@@ -87,10 +88,12 @@ environment.getSink().shutdown().orTimeout(10_000L, TimeUnit.MILLISECONDS);
8788

8889
The `MetricsLogger` is the interface you will use to publish embedded metrics.
8990

91+
- MetricsLogger **putMetric**(String key, double value, Unit unit, StorageResolution storageResolution)
92+
- MetricsLogger **putMetric**(String key, double value, StorageResolution storageResolution)
9093
- MetricsLogger **putMetric**(String key, double value, Unit unit)
9194
- MetricsLogger **putMetric**(String key, double value)
9295

93-
Adds a new metric to the current logger context. Multiple metrics using the same key will be appended to an array of values. The Embedded Metric Format supports a maximum of 100 values per key.
96+
Adds a new metric to the current logger context. Multiple metrics using the same key will be appended to an array of values. Multiple metrics cannot have the same key and different storage resolutions. The Embedded Metric Format supports a maximum of 100 values per key.
9497

9598
Requirements:
9699

@@ -99,10 +102,18 @@ Requirements:
99102
- Values must be in the range of 8.515920e-109 to 1.174271e+108. In addition, special values (for example, NaN, +Infinity, -Infinity) are not supported.
100103
- Metrics must meet CloudWatch Metrics requirements, otherwise a `InvalidMetricException` will be thrown. See [MetricDatum](https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_MetricDatum.html) for valid values.
101104

105+
- ##### Storage Resolution
106+
An OPTIONAL value representing the storage resolution for the corresponding metric. Setting this to `High` specifies this metric as a high-resolution metric, so that CloudWatch stores the metric with sub-minute resolution down to one second. Setting this to `Standard` specifies this metric as a standard-resolution metric, which CloudWatch stores at 1-minute resolution. If a value is not provided, then a default value of `Standard` is assumed. See [Cloud Watch High-Resolution metrics](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/publishingMetrics.html#high-resolution-metrics)
107+
102108
Examples:
103109

104110
```java
111+
// Standard Resolution example
105112
putMetric("Latency", 200, Unit.MILLISECONDS)
113+
putMetric("Latency", 201, Unit.MILLISECONDS, StorageResolution.STANDARD)
114+
115+
// High Resolution example
116+
putMetric("Memory.HeapUsed", 1600424.0, Unit.BYTES, StorageResolution.HIGH);
106117
```
107118

108119
- MetricsLogger **putProperty**(String key, Object value )

‎canarytests/agent/src/main/java/emf/canary/ECSRunnable.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import software.amazon.cloudwatchlogs.emf.exception.InvalidNamespaceException;
99
import software.amazon.cloudwatchlogs.emf.logger.MetricsLogger;
1010
import software.amazon.cloudwatchlogs.emf.model.DimensionSet;
11+
import software.amazon.cloudwatchlogs.emf.model.StorageResolution;
1112
import software.amazon.cloudwatchlogs.emf.model.Unit;
1213

1314
import java.lang.management.ManagementFactory;
@@ -46,7 +47,7 @@ public void run() {
4647
try {
4748
logger.putMetric("Invoke", 1, Unit.COUNT);
4849
logger.putMetric("Memory.HeapTotal", heapTotal, Unit.COUNT);
49-
logger.putMetric("Memory.HeapUsed", heapUsed, Unit.COUNT);
50+
logger.putMetric("Memory.HeapUsed", heapUsed, Unit.COUNT, StorageResolution.HIGH);
5051
logger.putMetric("Memory.JVMUsedTotal", heapUsed + nonHeapUsed, Unit.COUNT);
5152
} catch (InvalidMetricException e) {
5253
System.out.println(e);

‎examples/agent/src/main/java/agent/App.java

+2
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import software.amazon.cloudwatchlogs.emf.logger.MetricsLogger;
1010
import software.amazon.cloudwatchlogs.emf.model.DimensionSet;
1111
import software.amazon.cloudwatchlogs.emf.model.Unit;
12+
import software.amazon.cloudwatchlogs.emf.model.StorageResolution;
1213

1314
import java.util.concurrent.TimeUnit;
1415

@@ -35,6 +36,7 @@ private static void emitMetric(Environment environment)
3536
MetricsLogger logger = new MetricsLogger(environment);
3637
logger.setDimensions(DimensionSet.of("Operation", "Agent"));
3738
logger.putMetric("ExampleMetric", 100, Unit.MILLISECONDS);
39+
logger.putMetric("ExampleHighResolutionMetric", 10, Unit.MILLISECONDS, StorageResolution.HIGH);
3840
logger.putProperty("RequestId", "422b1569-16f6-4a03-b8f0-fe3fd9b100f8");
3941
logger.flush();
4042
}

‎examples/ecs-firelens/src/main/java/App.java

+2
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import software.amazon.cloudwatchlogs.emf.environment.EnvironmentProvider;
2424
import software.amazon.cloudwatchlogs.emf.exception.InvalidMetricException;
2525
import software.amazon.cloudwatchlogs.emf.logger.MetricsLogger;
26+
import software.amazon.cloudwatchlogs.emf.model.StorageResolution;
2627
import software.amazon.cloudwatchlogs.emf.model.Unit;
2728
import sun.misc.Signal;
2829
import java.io.IOException;
@@ -39,6 +40,7 @@ public static void main(String[] args) throws Exception {
3940
MetricsLogger logger = new MetricsLogger();
4041
logger.setNamespace("FargateEMF");
4142
logger.putMetric("Latency", 63, Unit.MILLISECONDS);
43+
logger.putMetric("CPU Utilization", 87, Unit.PERCENT, StorageResolution.HIGH);
4244
logger.flush();
4345
HttpServer server = HttpServer.create(new InetSocketAddress(8000), 0);
4446
int portNumber = 8000;

‎examples/lambda/src/main/java/Handler.java

+2
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import software.amazon.cloudwatchlogs.emf.exception.InvalidMetricException;
66
import software.amazon.cloudwatchlogs.emf.logger.MetricsLogger;
77
import software.amazon.cloudwatchlogs.emf.model.DimensionSet;
8+
import software.amazon.cloudwatchlogs.emf.model.StorageResolution;
89
import software.amazon.cloudwatchlogs.emf.model.Unit;
910

1011
import java.util.HashMap;
@@ -20,6 +21,7 @@ public String handleRequest(Map<String, String> event, Context context) {
2021
try {
2122
logger.putDimensions(DimensionSet.of("Service", "Aggregator"));
2223
logger.putMetric("ProcessingLatency", 100, Unit.MILLISECONDS);
24+
logger.putMetric("CPU Utilization", 87, Unit.PERCENT, StorageResolution.HIGH);
2325
} catch (InvalidDimensionException | InvalidMetricException | DimensionSetExceededException e) {
2426
System.out.println(e);
2527
}

‎src/main/java/software/amazon/cloudwatchlogs/emf/logger/MetricsLogger.java

+47-4
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import software.amazon.cloudwatchlogs.emf.exception.InvalidTimestampException;
3333
import software.amazon.cloudwatchlogs.emf.model.DimensionSet;
3434
import software.amazon.cloudwatchlogs.emf.model.MetricsContext;
35+
import software.amazon.cloudwatchlogs.emf.model.StorageResolution;
3536
import software.amazon.cloudwatchlogs.emf.model.Unit;
3637
import software.amazon.cloudwatchlogs.emf.sinks.ISink;
3738

@@ -98,7 +99,7 @@ public void flush() {
9899
}
99100

100101
/**
101-
* Set a property on the published metrics. This is stored in the emitted log data and you are
102+
* Set a property on the published metrics. This is stored in the emitted log data, and you are
102103
* not charged for this data by CloudWatch Metrics. These values can be values that are useful
103104
* for searching on, but have too high cardinality to emit as dimensions to CloudWatch Metrics.
104105
*
@@ -189,20 +190,62 @@ public MetricsLogger resetDimensions(boolean useDefault) {
189190
* @param key is the name of the metric
190191
* @param value is the value of the metric
191192
* @param unit is the unit of the metric value
193+
* @param storageResolution is the resolution of the metric
194+
* @see <a
195+
* href="https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/publishingMetrics.html#high-resolution-metrics">CloudWatch
196+
* High Resolution Metrics</a>
192197
* @return the current logger
193198
* @throws InvalidMetricException if the metric is invalid
194199
*/
195-
public MetricsLogger putMetric(String key, double value, Unit unit)
200+
public MetricsLogger putMetric(
201+
String key, double value, Unit unit, StorageResolution storageResolution)
196202
throws InvalidMetricException {
197203
rwl.readLock().lock();
198204
try {
199-
this.context.putMetric(key, value, unit);
205+
this.context.putMetric(key, value, unit, storageResolution);
200206
return this;
201207
} finally {
202208
rwl.readLock().unlock();
203209
}
204210
}
205211

212+
/**
213+
* Put a metric value. This value will be emitted to CloudWatch Metrics asynchronously and does
214+
* not contribute to your account TPS limits. The value will also be available in your
215+
* CloudWatch Logs
216+
*
217+
* @param key is the name of the metric
218+
* @param value is the value of the metric
219+
* @param storageResolution is the resolution of the metric
220+
* @see <a
221+
* href="https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/publishingMetrics.html#high-resolution-metrics">CloudWatch
222+
* High Resolution Metrics</a>
223+
* @return the current logger
224+
* @throws InvalidMetricException if the metric is invalid
225+
*/
226+
public MetricsLogger putMetric(String key, double value, StorageResolution storageResolution)
227+
throws InvalidMetricException {
228+
this.putMetric(key, value, Unit.NONE, storageResolution);
229+
return this;
230+
}
231+
232+
/**
233+
* Put a metric value. This value will be emitted to CloudWatch Metrics asynchronously and does
234+
* not contribute to your account TPS limits. The value will also be available in your
235+
* CloudWatch Logs
236+
*
237+
* @param key is the name of the metric
238+
* @param value is the value of the metric
239+
* @param unit is the unit of the metric value
240+
* @return the current logger
241+
* @throws InvalidMetricException if the metric is invalid
242+
*/
243+
public MetricsLogger putMetric(String key, double value, Unit unit)
244+
throws InvalidMetricException {
245+
this.putMetric(key, value, unit, StorageResolution.STANDARD);
246+
return this;
247+
}
248+
206249
/**
207250
* Put a metric value. This value will be emitted to CloudWatch Metrics asynchronously and does
208251
* not contribute to your account TPS limits. The value will also be available in your
@@ -214,7 +257,7 @@ public MetricsLogger putMetric(String key, double value, Unit unit)
214257
* @throws InvalidMetricException if the metric is invalid
215258
*/
216259
public MetricsLogger putMetric(String key, double value) throws InvalidMetricException {
217-
this.putMetric(key, value, Unit.NONE);
260+
this.putMetric(key, value, Unit.NONE, StorageResolution.STANDARD);
218261
return this;
219262
}
220263

‎src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricDefinition.java

+25-3
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
package software.amazon.cloudwatchlogs.emf.model;
1818

1919
import com.fasterxml.jackson.annotation.JsonIgnore;
20+
import com.fasterxml.jackson.annotation.JsonInclude;
2021
import com.fasterxml.jackson.annotation.JsonProperty;
2122
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
2223
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
@@ -26,6 +27,9 @@
2627
import lombok.AllArgsConstructor;
2728
import lombok.Getter;
2829
import lombok.NonNull;
30+
import lombok.Setter;
31+
import software.amazon.cloudwatchlogs.emf.serializers.StorageResolutionFilter;
32+
import software.amazon.cloudwatchlogs.emf.serializers.StorageResolutionSerializer;
2933
import software.amazon.cloudwatchlogs.emf.serializers.UnitDeserializer;
3034
import software.amazon.cloudwatchlogs.emf.serializers.UnitSerializer;
3135

@@ -43,18 +47,36 @@ class MetricDefinition {
4347
@JsonDeserialize(using = UnitDeserializer.class)
4448
private Unit unit;
4549

50+
@Getter
51+
@Setter
52+
@JsonProperty("StorageResolution")
53+
@JsonInclude(
54+
value = JsonInclude.Include.CUSTOM,
55+
valueFilter =
56+
StorageResolutionFilter.class) // Do not serialize when valueFilter is true
57+
@JsonSerialize(using = StorageResolutionSerializer.class)
58+
public StorageResolution storageResolution;
59+
4660
@JsonIgnore @NonNull @Getter private List<Double> values;
4761

4862
MetricDefinition(String name) {
49-
this(name, Unit.NONE, new ArrayList<>());
63+
this(name, Unit.NONE, StorageResolution.STANDARD, new ArrayList<>());
5064
}
5165

5266
MetricDefinition(String name, double value) {
53-
this(name, Unit.NONE, value);
67+
this(name, Unit.NONE, StorageResolution.STANDARD, value);
5468
}
5569

5670
MetricDefinition(String name, Unit unit, double value) {
57-
this(name, unit, new ArrayList<>(Arrays.asList(value)));
71+
this(name, unit, StorageResolution.STANDARD, new ArrayList<>(Arrays.asList(value)));
72+
}
73+
74+
MetricDefinition(String name, StorageResolution storageResolution, double value) {
75+
this(name, Unit.NONE, storageResolution, new ArrayList<>(Arrays.asList(value)));
76+
}
77+
78+
MetricDefinition(String name, Unit unit, StorageResolution storageResolution, double value) {
79+
this(name, unit, storageResolution, new ArrayList<>(Arrays.asList(value)));
5880
}
5981

6082
void addValue(double value) {

‎src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricDirective.java

+13-2
Original file line numberDiff line numberDiff line change
@@ -66,15 +66,26 @@ void putDimensionSet(DimensionSet dimensionSet) {
6666
dimensions.add(dimensionSet);
6767
}
6868

69+
// Helper method for testing putMetric()
6970
void putMetric(String key, double value) {
70-
putMetric(key, value, Unit.NONE);
71+
putMetric(key, value, Unit.NONE, StorageResolution.STANDARD);
7172
}
7273

74+
// Helper method for testing putMetric()
7375
void putMetric(String key, double value, Unit unit) {
76+
putMetric(key, value, unit, StorageResolution.STANDARD);
77+
}
78+
79+
// Helper method for testing serialization
80+
void putMetric(String key, double value, StorageResolution storageResolution) {
81+
putMetric(key, value, Unit.NONE, storageResolution);
82+
}
83+
84+
void putMetric(String key, double value, Unit unit, StorageResolution storageResolution) {
7485
metrics.compute(
7586
key,
7687
(k, v) -> {
77-
if (v == null) return new MetricDefinition(key, unit, value);
88+
if (v == null) return new MetricDefinition(key, unit, storageResolution, value);
7889
else {
7990
v.addValue(value);
8091
return v;

‎src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricsContext.java

+43-3
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ public class MetricsContext {
3434
@Getter private final RootNode rootNode;
3535

3636
private MetricDirective metricDirective;
37+
private final Map<String, StorageResolution> metricNameAndResolutionMap = new HashMap<>();
3738

3839
public MetricsContext() {
3940
this(new RootNode());
@@ -108,6 +109,44 @@ public boolean hasDefaultDimensions() {
108109
* an array of scalar values.
109110
*
110111
* <pre>{@code
112+
* metricContext.putMetric("Latency", 100, Unit.MILLISECONDS, StorageResolution.HIGH)
113+
* }</pre>
114+
*
115+
* @param key Name of the metric
116+
* @param value Value of the metric
117+
* @param unit The unit of the metric
118+
* @param storageResolution The resolution of the metric
119+
* @throws InvalidMetricException if the metric is invalid
120+
*/
121+
public void putMetric(String key, double value, Unit unit, StorageResolution storageResolution)
122+
throws InvalidMetricException {
123+
Validator.validateMetric(key, value, unit, storageResolution, metricNameAndResolutionMap);
124+
metricDirective.putMetric(key, value, unit, storageResolution);
125+
metricNameAndResolutionMap.put(key, storageResolution);
126+
}
127+
/**
128+
* Add a metric measurement to the context with a storage resolution but without a unit.
129+
* Multiple calls using the same key will be stored as an array of scalar values.
130+
*
131+
* <pre>{@code
132+
* metricContext.putMetric("Latency", 100, StorageResolution.HIGH)
133+
* }</pre>
134+
*
135+
* @param key Name of the metric
136+
* @param value Value of the metric
137+
* @param storageResolution The resolution of the metric
138+
* @throws InvalidMetricException if the metric is invalid
139+
*/
140+
public void putMetric(String key, double value, StorageResolution storageResolution)
141+
throws InvalidMetricException {
142+
putMetric(key, value, Unit.NONE, storageResolution);
143+
}
144+
145+
/**
146+
* Add a metric measurement to the context without a storage resolution. Multiple calls using
147+
* the same key will be stored as an array of scalar values.
148+
*
149+
* <pre>{@code
111150
* metricContext.putMetric("Latency", 100, Unit.MILLISECONDS)
112151
* }</pre>
113152
*
@@ -117,8 +156,7 @@ public boolean hasDefaultDimensions() {
117156
* @throws InvalidMetricException if the metric is invalid
118157
*/
119158
public void putMetric(String key, double value, Unit unit) throws InvalidMetricException {
120-
Validator.validateMetric(key, value, unit);
121-
metricDirective.putMetric(key, value, unit);
159+
putMetric(key, value, unit, StorageResolution.STANDARD);
122160
}
123161

124162
/**
@@ -134,7 +172,7 @@ public void putMetric(String key, double value, Unit unit) throws InvalidMetricE
134172
* @throws InvalidMetricException if the metric is invalid
135173
*/
136174
public void putMetric(String key, double value) throws InvalidMetricException {
137-
putMetric(key, value, Unit.NONE);
175+
putMetric(key, value, Unit.NONE, StorageResolution.STANDARD);
138176
}
139177

140178
/**
@@ -297,12 +335,14 @@ public List<String> serialize() throws JsonProcessingException {
297335
new MetricDefinition(
298336
metric.getName(),
299337
metric.getUnit(),
338+
metric.getStorageResolution(),
300339
metric.getValues()
301340
.subList(0, Constants.MAX_DATAPOINTS_PER_METRIC)));
302341
metricDefinitions.offer(
303342
new MetricDefinition(
304343
metric.getName(),
305344
metric.getUnit(),
345+
metric.getStorageResolution(),
306346
metric.getValues()
307347
.subList(
308348
Constants.MAX_DATAPOINTS_PER_METRIC,

0 commit comments

Comments
 (0)