@@ -20,30 +20,34 @@ customConfig:
2020 type: vector
2121 version: "2"
2222 transforms:
23+ validEvents:
24+ type: filter
25+ inputs: [vector]
26+ condition: is_null(.errors)
2327
2428 # SparkHistoryServer spark-history
2529
2630 filteredSparkHistoryAutomaticLogConfigSparkHistory:
2731 type: filter
28- inputs: [vector ]
32+ inputs: [validEvents ]
2933 condition: >-
3034 .pod == "spark-history-node-automatic-log-config-0" &&
3135 .container == "spark-history"
3236 filteredSparkHistoryAutomaticLogConfigVector:
3337 type: filter
34- inputs: [vector ]
38+ inputs: [validEvents ]
3539 condition: >-
3640 .pod == "spark-history-node-automatic-log-config-0" &&
3741 .container == "vector"
3842 filteredSparkHistoryCustomLogConfigSparkHistory:
3943 type: filter
40- inputs: [vector ]
44+ inputs: [validEvents ]
4145 condition: >-
4246 .pod == "spark-history-node-custom-log-config-0" &&
4347 .container == "spark-history"
4448 filteredSparkHistoryCustomLogConfigVector:
4549 type: filter
46- inputs: [vector ]
50+ inputs: [validEvents ]
4751 condition: >-
4852 .pod == "spark-history-node-custom-log-config-0" &&
4953 .container == "vector"
@@ -52,42 +56,42 @@ customConfig:
5256
5357 filteredSparkAutomaticLogConfigDriverSpark:
5458 type: filter
55- inputs: [vector ]
59+ inputs: [validEvents ]
5660 condition: >-
5761 .cluster == "spark-automatic-log-config" &&
5862 ends_with(string!(.pod), "-driver") &&
5963 .container == "spark"
6064 filteredSparkAutomaticLogConfigDriverJob:
6165 type: filter
62- inputs: [vector ]
66+ inputs: [validEvents ]
6367 condition: >-
6468 .cluster == "spark-automatic-log-config" &&
6569 ends_with(string!(.pod), "-driver") &&
6670 .container == "job"
6771 filteredSparkAutomaticLogConfigDriverVector:
6872 type: filter
69- inputs: [vector ]
73+ inputs: [validEvents ]
7074 condition: >-
7175 .cluster == "spark-automatic-log-config" &&
7276 ends_with(string!(.pod), "-driver") &&
7377 .container == "vector"
7478 filteredSparkAutomaticLogConfigExecutorSpark:
7579 type: filter
76- inputs: [vector ]
80+ inputs: [validEvents ]
7781 condition: >-
7882 .cluster == "spark-automatic-log-config" &&
7983 ends_with(string!(.pod), "-exec-1") &&
8084 .container == "spark"
8185 filteredSparkAutomaticLogConfigExecutorJob:
8286 type: filter
83- inputs: [vector ]
87+ inputs: [validEvents ]
8488 condition: >-
8589 .cluster == "spark-automatic-log-config" &&
8690 ends_with(string!(.pod), "-exec-1") &&
8791 .container == "job"
8892 filteredSparkAutomaticLogConfigExecutorVector:
8993 type: filter
90- inputs: [vector ]
94+ inputs: [validEvents ]
9195 condition: >-
9296 .cluster == "spark-automatic-log-config" &&
9397 ends_with(string!(.pod), "-exec-1") &&
@@ -97,42 +101,42 @@ customConfig:
97101
98102 filteredSparkCustomLogConfigDriverSpark:
99103 type: filter
100- inputs: [vector ]
104+ inputs: [validEvents ]
101105 condition: >-
102106 .cluster == "spark-custom-log-config" &&
103107 ends_with(string!(.pod), "-driver") &&
104108 .container == "spark"
105109 filteredSparkCustomLogConfigDriverJob:
106110 type: filter
107- inputs: [vector ]
111+ inputs: [validEvents ]
108112 condition: >-
109113 .cluster == "spark-custom-log-config" &&
110114 ends_with(string!(.pod), "-driver") &&
111115 .container == "job"
112116 filteredSparkCustomLogConfigDriverVector:
113117 type: filter
114- inputs: [vector ]
118+ inputs: [validEvents ]
115119 condition: >-
116120 .cluster == "spark-custom-log-config" &&
117121 ends_with(string!(.pod), "-driver") &&
118122 .container == "vector"
119123 filteredSparkCustomLogConfigExecutorSpark:
120124 type: filter
121- inputs: [vector ]
125+ inputs: [validEvents ]
122126 condition: >-
123127 .cluster == "spark-custom-log-config" &&
124128 ends_with(string!(.pod), "-exec-1") &&
125129 .container == "spark"
126130 filteredSparkCustomLogConfigExecutorJob:
127131 type: filter
128- inputs: [vector ]
132+ inputs: [validEvents ]
129133 condition: >-
130134 .cluster == "spark-custom-log-config" &&
131135 ends_with(string!(.pod), "-exec-1") &&
132136 .container == "job"
133137 filteredSparkCustomLogConfigExecutorVector:
134138 type: filter
135- inputs: [vector ]
139+ inputs: [validEvents ]
136140 condition: >-
137141 .cluster == "spark-custom-log-config" &&
138142 ends_with(string!(.pod), "-exec-1") &&
@@ -142,42 +146,42 @@ customConfig:
142146
143147 filteredPysparkAutomaticLogConfigDriverSpark:
144148 type: filter
145- inputs: [vector ]
149+ inputs: [validEvents ]
146150 condition: >-
147151 .cluster == "pyspark-automatic-log-config" &&
148152 ends_with(string!(.pod), "-driver") &&
149153 .container == "spark"
150154 filteredPysparkAutomaticLogConfigDriverRequirements:
151155 type: filter
152- inputs: [vector ]
156+ inputs: [validEvents ]
153157 condition: >-
154158 .cluster == "pyspark-automatic-log-config" &&
155159 ends_with(string!(.pod), "-driver") &&
156160 .container == "requirements"
157161 filteredPysparkAutomaticLogConfigDriverVector:
158162 type: filter
159- inputs: [vector ]
163+ inputs: [validEvents ]
160164 condition: >-
161165 .cluster == "pyspark-automatic-log-config" &&
162166 ends_with(string!(.pod), "-driver") &&
163167 .container == "vector"
164168 filteredPysparkAutomaticLogConfigExecutorSpark:
165169 type: filter
166- inputs: [vector ]
170+ inputs: [validEvents ]
167171 condition: >-
168172 .cluster == "pyspark-automatic-log-config" &&
169173 ends_with(string!(.pod), "-exec-1") &&
170174 .container == "spark"
171175 filteredPysparkAutomaticLogConfigExecutorRequirements:
172176 type: filter
173- inputs: [vector ]
177+ inputs: [validEvents ]
174178 condition: >-
175179 .cluster == "pyspark-automatic-log-config" &&
176180 ends_with(string!(.pod), "-exec-1") &&
177181 .container == "requirements"
178182 filteredPysparkAutomaticLogConfigExecutorVector:
179183 type: filter
180- inputs: [vector ]
184+ inputs: [validEvents ]
181185 condition: >-
182186 .cluster == "pyspark-automatic-log-config" &&
183187 ends_with(string!(.pod), "-exec-1") &&
@@ -187,42 +191,42 @@ customConfig:
187191
188192 filteredPysparkCustomLogConfigDriverSpark:
189193 type: filter
190- inputs: [vector ]
194+ inputs: [validEvents ]
191195 condition: >-
192196 .cluster == "pyspark-custom-log-config" &&
193197 ends_with(string!(.pod), "-driver") &&
194198 .container == "spark"
195199 filteredPysparkCustomLogConfigDriverRequirements:
196200 type: filter
197- inputs: [vector ]
201+ inputs: [validEvents ]
198202 condition: >-
199203 .cluster == "pyspark-custom-log-config" &&
200204 ends_with(string!(.pod), "-driver") &&
201205 .container == "requirements"
202206 filteredPysparkCustomLogConfigDriverVector:
203207 type: filter
204- inputs: [vector ]
208+ inputs: [validEvents ]
205209 condition: >-
206210 .cluster == "pyspark-custom-log-config" &&
207211 ends_with(string!(.pod), "-driver") &&
208212 .container == "vector"
209213 filteredPysparkCustomLogConfigExecutorSpark:
210214 type: filter
211- inputs: [vector ]
215+ inputs: [validEvents ]
212216 condition: >-
213217 .cluster == "pyspark-custom-log-config" &&
214218 ends_with(string!(.pod), "-exec-1") &&
215219 .container == "spark"
216220 filteredPysparkCustomLogConfigExecutorRequirements:
217221 type: filter
218- inputs: [vector ]
222+ inputs: [validEvents ]
219223 condition: >-
220224 .cluster == "pyspark-custom-log-config" &&
221225 ends_with(string!(.pod), "-exec-1") &&
222226 .container == "requirements"
223227 filteredPysparkCustomLogConfigExecutorVector:
224228 type: filter
225- inputs: [vector ]
229+ inputs: [validEvents ]
226230 condition: >-
227231 .cluster == "pyspark-custom-log-config" &&
228232 ends_with(string!(.pod), "-exec-1") &&
@@ -239,15 +243,16 @@ customConfig:
239243 is_null(.logger) ||
240244 is_null(.message)
241245 sinks:
242- out :
246+ test :
243247 inputs: [filtered*]
248+ type: blackhole
244249{% if lookup ('env' , 'VECTOR_AGGREGATOR' ) %}
250+ aggregator:
251+ inputs: [vector]
245252 type: vector
246253 address: {{ lookup('env', 'VECTOR_AGGREGATOR') }}
247254 buffer:
248255 # Avoid back pressure from VECTOR_AGGREGATOR. The test should
249256 # not fail if the aggregator is not available.
250257 when_full: drop_newest
251- {% else %}
252- type: blackhole
253258{% endif %}
0 commit comments