Skip to content

Commit 6ef1216

Browse files
lksfrnzlksfrnz
committed
Added Histogram Metric Type
1 parent d8ae46b commit 6ef1216

File tree

13 files changed

+592
-28
lines changed

13 files changed

+592
-28
lines changed

‎src/main/java/software/amazon/cloudwatchlogs/emf/model/AggregationType.java

+1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
public enum AggregationType {
2020
LIST(0),
2121
STATISTIC_SET(1),
22+
HISTOGRAM(2),
2223
UNKNOWN_TO_SDK_VERSION(-1);
2324

2425
private final int value;

‎src/main/java/software/amazon/cloudwatchlogs/emf/model/Histogram.java

+160
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
/* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
*
3+
* Licensed under the Apache License, Version 2.0 (the "License").
4+
* You may not use this file except in compliance with the License.
5+
* You may obtain a copy of the License at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software
10+
* distributed under the License is distributed on an "AS IS" BASIS,
11+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
* See the License for the specific language governing permissions and
13+
* limitations under the License.
14+
*/
15+
16+
package software.amazon.cloudwatchlogs.emf.model;
17+
18+
import com.fasterxml.jackson.annotation.JsonIgnore;
19+
import com.fasterxml.jackson.annotation.JsonProperty;
20+
import java.util.ArrayList;
21+
import java.util.Collections;
22+
import java.util.HashMap;
23+
import java.util.List;
24+
import java.util.Map;
25+
import software.amazon.cloudwatchlogs.emf.Constants;
26+
import software.amazon.cloudwatchlogs.emf.exception.InvalidMetricException;
27+
28+
/** Histogram metric type */
29+
class Histogram extends Statistics {
30+
Histogram(List<Double> values, List<Integer> counts) throws IllegalArgumentException {
31+
if (counts.size() != values.size()) {
32+
throw new IllegalArgumentException("Counts and values must have the same size");
33+
}
34+
35+
if (values.stream().anyMatch(n -> n == null) || counts.stream().anyMatch(n -> n == null)) {
36+
throw new IllegalArgumentException("Values and counts cannot contain null values");
37+
}
38+
39+
if (!validSize(counts.size())) {
40+
throw new IllegalArgumentException(
41+
String.format(
42+
"Histogram provided with %d bins but CloudWatch will drop Histograms with more than %d bins",
43+
counts.size(), Constants.MAX_DATAPOINTS_PER_METRIC));
44+
}
45+
46+
this.max = Collections.max(values);
47+
this.min = Collections.min(values);
48+
this.count = counts.stream().mapToInt(Integer::intValue).sum();
49+
this.sum = 0d;
50+
for (int i = 0; i < counts.size(); i++) {
51+
this.sum += values.get(i) * counts.get(i);
52+
}
53+
this.counts = counts;
54+
this.values = values;
55+
}
56+
57+
Histogram() {
58+
count = 0;
59+
sum = 0.;
60+
values = new ArrayList<>();
61+
counts = new ArrayList<>();
62+
}
63+
64+
@JsonProperty("Values")
65+
public List<Double> values;
66+
67+
@JsonProperty("Counts")
68+
public List<Integer> counts;
69+
70+
@JsonIgnore private boolean reduced = false;
71+
72+
@JsonIgnore private static final double EPSILON = 0.1;
73+
@JsonIgnore private static final double BIN_SIZE = Math.log(1 + EPSILON);
74+
@JsonIgnore private final Map<Double, Integer> buckets = new HashMap<>();
75+
76+
/**
77+
* @param value the value to add to the histogram
78+
* @throws InvalidMetricException if adding this value would increase the number of bins in the
79+
* histogram to more than {@value Constants#MAX_DATAPOINTS_PER_METRIC}
80+
* @see Constants#MAX_DATAPOINTS_PER_METRIC
81+
*/
82+
@Override
83+
void addValue(double value) throws InvalidMetricException {
84+
reduced = false;
85+
super.addValue(value);
86+
87+
double bucket = getBucket(value);
88+
if (!buckets.containsKey(bucket) && !validSize(counts.size() + 1)) {
89+
throw new InvalidMetricException(
90+
String.format(
91+
"Adding this value increases the number of bins in this histogram to %d"
92+
+ ", CloudWatch will drop any Histogram metrics with more than %d bins",
93+
buckets.size() + 1, Constants.MAX_DATAPOINTS_PER_METRIC));
94+
}
95+
// Add the value to the appropriate bucket (or create a new bucket if necessary)
96+
buckets.compute(
97+
bucket,
98+
(k, v) -> {
99+
if (v == null) {
100+
return 1;
101+
} else {
102+
return v + 1;
103+
}
104+
});
105+
}
106+
107+
/**
108+
* Updates the Values and Counts lists to represent the buckets of this histogram.
109+
*
110+
* @return the reduced histogram
111+
*/
112+
Histogram reduce() {
113+
if (reduced) {
114+
return this;
115+
}
116+
117+
this.values = new ArrayList<>(buckets.size());
118+
this.counts = new ArrayList<>(buckets.size());
119+
120+
for (Map.Entry<Double, Integer> entry : buckets.entrySet()) {
121+
this.values.add(entry.getKey());
122+
this.counts.add(entry.getValue());
123+
}
124+
125+
reduced = true;
126+
return this;
127+
}
128+
129+
/**
130+
* Gets the value of the bucket for the given value.
131+
*
132+
* @param value the value to find the closest bucket for
133+
* @return the value of the bucket the given value goes in
134+
*/
135+
private static double getBucket(double value) {
136+
short index = (short) Math.floor(Math.log(value) / BIN_SIZE);
137+
return Math.exp((index + 0.5) * BIN_SIZE);
138+
}
139+
140+
private boolean validSize(int size) {
141+
return size <= Constants.MAX_DATAPOINTS_PER_METRIC;
142+
}
143+
144+
@Override
145+
public boolean equals(Object o) {
146+
if (this == o) return true;
147+
if (o == null || getClass() != o.getClass()) return false;
148+
Histogram that = (Histogram) o;
149+
return count == that.count
150+
&& that.sum.equals(sum)
151+
&& that.max.equals(max)
152+
&& that.min.equals(min)
153+
&& buckets.equals(that.buckets);
154+
}
155+
156+
@Override
157+
public int hashCode() {
158+
return super.hashCode() + buckets.hashCode();
159+
}
160+
}

‎src/main/java/software/amazon/cloudwatchlogs/emf/model/HistogramMetric.java

+116
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package software.amazon.cloudwatchlogs.emf.model;
18+
19+
import java.util.LinkedList;
20+
import java.util.List;
21+
import java.util.Queue;
22+
import software.amazon.cloudwatchlogs.emf.Constants;
23+
import software.amazon.cloudwatchlogs.emf.exception.InvalidMetricException;
24+
25+
/** Represents the Histogram of the EMF schema. */
26+
public class HistogramMetric extends Metric<Histogram> {
27+
28+
HistogramMetric(
29+
Unit unit,
30+
StorageResolution storageResolution,
31+
List<Double> values,
32+
List<Integer> counts)
33+
throws IllegalArgumentException {
34+
this(unit, storageResolution, new Histogram(values, counts));
35+
}
36+
37+
protected HistogramMetric(
38+
String name, Unit unit, StorageResolution storageResolution, Histogram histogram) {
39+
this.unit = unit;
40+
this.storageResolution = storageResolution;
41+
this.values = histogram;
42+
this.name = name;
43+
}
44+
45+
HistogramMetric(Unit unit, StorageResolution storageResolution, Histogram histogram) {
46+
this.unit = unit;
47+
this.storageResolution = storageResolution;
48+
this.values = histogram;
49+
}
50+
51+
@Override
52+
protected Queue<Metric<Histogram>> serialize() throws InvalidMetricException {
53+
// Histograms will be rejected from CWL if they have more than
54+
// Constants.MAX_DATAPOINTS_PER_METRIC number of bins. Unlike MetricDefinition histograms
55+
// cannot be broken into multiple messages therefore an error is raised to let users know
56+
// their message won't be sent otherwise only this metric will be sent
57+
if (isOversized()) {
58+
throw new InvalidMetricException(
59+
String.format(
60+
"Histogram metric, %s, has %d values which exceeds the maximum amount "
61+
+ "of bins allowed, %d, and Histograms cannot be broken into "
62+
+ "multiple metrics therefore it will not be published",
63+
name, values.values.size(), Constants.MAX_DATAPOINTS_PER_METRIC));
64+
}
65+
Queue<Metric<Histogram>> metrics = new LinkedList<>();
66+
metrics.offer(this);
67+
return metrics;
68+
}
69+
70+
@Override
71+
protected boolean isOversized() {
72+
return values.values.size() > Constants.MAX_DATAPOINTS_PER_METRIC;
73+
}
74+
75+
@Override
76+
public boolean hasValidValues() {
77+
return values != null && values.count > 0 && !isOversized();
78+
}
79+
80+
public static HistogramMetricBuilder builder() {
81+
return new HistogramMetricBuilder();
82+
}
83+
84+
public static class HistogramMetricBuilder
85+
extends Metric.MetricBuilder<Histogram, HistogramMetricBuilder> {
86+
87+
@Override
88+
protected HistogramMetricBuilder getThis() {
89+
return this;
90+
}
91+
92+
public HistogramMetricBuilder() {
93+
this.values = new Histogram();
94+
}
95+
96+
@Override
97+
public Histogram getValues() {
98+
return values.reduce();
99+
}
100+
101+
@Override
102+
public HistogramMetricBuilder addValue(double value) {
103+
this.values.addValue(value);
104+
return this;
105+
}
106+
107+
@Override
108+
public HistogramMetric build() {
109+
values.reduce();
110+
if (name == null) {
111+
return new HistogramMetric(unit, storageResolution, values);
112+
}
113+
return new HistogramMetric(name, unit, storageResolution, values);
114+
}
115+
}
116+
}

‎src/main/java/software/amazon/cloudwatchlogs/emf/model/Metric.java

+6-4
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
import com.fasterxml.jackson.annotation.JsonProperty;
2222
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
2323
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
24-
import java.util.LinkedList;
24+
import java.util.Queue;
2525
import lombok.AccessLevel;
2626
import lombok.Getter;
2727
import lombok.NonNull;
@@ -52,7 +52,9 @@ public abstract class Metric<V> {
5252
@JsonSerialize(using = StorageResolutionSerializer.class)
5353
protected StorageResolution storageResolution = StorageResolution.STANDARD;
5454

55-
@JsonIgnore @Getter protected V values;
55+
@JsonIgnore
56+
@Getter(AccessLevel.PROTECTED)
57+
protected V values;
5658

5759
/** @return the values of this metric formatted to be flushed */
5860
protected Object getFormattedValues() {
@@ -72,7 +74,7 @@ protected Object getFormattedValues() {
7274
* @return a list of metrics based off of the values of this metric that aren't too large for
7375
* CWL
7476
*/
75-
protected abstract LinkedList<Metric> serialize();
77+
protected abstract Queue<Metric<V>> serialize();
7678

7779
public abstract static class MetricBuilder<V, T extends MetricBuilder<V, T>> extends Metric<V> {
7880

@@ -113,7 +115,7 @@ public boolean hasValidValues() {
113115
}
114116

115117
@Override
116-
protected LinkedList<Metric> serialize() {
118+
protected Queue<Metric<V>> serialize() {
117119
return build().serialize();
118120
}
119121

‎src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricDefinition.java

+5-3
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919
import java.util.ArrayList;
2020
import java.util.LinkedList;
2121
import java.util.List;
22+
import java.util.Queue;
23+
2224
import lombok.NonNull;
2325
import software.amazon.cloudwatchlogs.emf.Constants;
2426

@@ -43,8 +45,8 @@ private MetricDefinition(
4345
}
4446

4547
@Override
46-
protected LinkedList<Metric> serialize() {
47-
LinkedList<Metric> metrics = new LinkedList<>();
48+
protected Queue<Metric<List<Double>>> serialize() {
49+
Queue<Metric<List<Double>>> metrics = new LinkedList<>();
4850
MetricDefinition metric = this;
4951
while (metric != null) {
5052
metrics.add(metric.getFirstMetricBatch(Constants.MAX_DATAPOINTS_PER_METRIC));
@@ -67,7 +69,7 @@ private MetricDefinition getFirstMetricBatch(int batchSize) {
6769
}
6870

6971
private MetricDefinition getRemainingMetricBatch(int batchSize) {
70-
if (batchSize > values.size()) {
72+
if (batchSize >= values.size()) {
7173
return null;
7274
}
7375
List<Double> subList = values.subList(batchSize, values.size());

0 commit comments

Comments
 (0)