Skip to content

Commit 2366f67

Browse files
sothawosothawo
authored
Enable scripted fields and runtime fields of collection type.
Original Pull Request #3080 Closes #3076 Signed-off-by: Peter-Josef Meisch <[email protected]>
1 parent 6f42431 commit 2366f67

File tree

5 files changed

+169
-33
lines changed

5 files changed

+169
-33
lines changed

‎src/main/antora/modules/ROOT/pages/elasticsearch/elasticsearch-new.adoc

+1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
* Upgrade to Elasticsearch 8.17.2.
88
* Add support for the `@SearchTemplateQuery` annotation on repository methods.
9+
* Scripted field properties of type collection can be populated from scripts returning arrays.
910

1011
[[new-features.5-4-0]]
1112
== New in Spring Data Elasticsearch 5.4

‎src/main/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverter.java

+18-4
Original file line numberDiff line numberDiff line change
@@ -394,7 +394,7 @@ private <R> R readEntity(ElasticsearchPersistentEntity<?> entity, Map<String, Ob
394394
}
395395

396396
if (source instanceof SearchDocument searchDocument) {
397-
populateScriptFields(targetEntity, result, searchDocument);
397+
populateScriptedFields(targetEntity, result, searchDocument);
398398
}
399399
return result;
400400
} catch (ConversionException e) {
@@ -652,7 +652,16 @@ private Object convertFromCollectionToObject(Object value, Class<?> target) {
652652
return conversionService.convert(value, target);
653653
}
654654

655-
private <T> void populateScriptFields(ElasticsearchPersistentEntity<?> entity, T result,
655+
/**
656+
* Checks if any of the properties of the entity is annotated with
657+
*
658+
* @{@link ScriptedField}. If so, the value of this property is set from the returned fields in the document.
659+
* @param entity the entity to defining the persistent property
660+
* @param result the rsult to populate
661+
* @param searchDocument the search result caontaining the fields
662+
* @param <T> the result type
663+
*/
664+
private <T> void populateScriptedFields(ElasticsearchPersistentEntity<?> entity, T result,
656665
SearchDocument searchDocument) {
657666
Map<String, List<Object>> fields = searchDocument.getFields();
658667
entity.doWithProperties((SimplePropertyHandler) property -> {
@@ -661,8 +670,13 @@ private <T> void populateScriptFields(ElasticsearchPersistentEntity<?> entity, T
661670
// noinspection ConstantConditions
662671
String name = scriptedField.name().isEmpty() ? property.getName() : scriptedField.name();
663672
if (fields.containsKey(name)) {
664-
Object value = searchDocument.getFieldValue(name);
665-
entity.getPropertyAccessor(result).setProperty(property, value);
673+
if (property.isCollectionLike()) {
674+
List<Object> values = searchDocument.getFieldValues(name);
675+
entity.getPropertyAccessor(result).setProperty(property, values);
676+
} else {
677+
Object value = searchDocument.getFieldValue(name);
678+
entity.getPropertyAccessor(result).setProperty(property, value);
679+
}
666680
}
667681
}
668682
});

‎src/main/java/org/springframework/data/elasticsearch/core/document/SearchDocument.java

+14
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,20 @@ default <V> V getFieldValue(final String name) {
5757
return (V) values.get(0);
5858
}
5959

60+
/**
61+
* @param name the field name
62+
* @param <V> the type of elements
63+
* @return the values of the given field.
64+
*/
65+
@Nullable
66+
default <V> List<V> getFieldValues(final String name) {
67+
List<Object> values = getFields().get(name);
68+
if (values == null) {
69+
return null;
70+
}
71+
return (List<V>) values;
72+
}
73+
6074
/**
6175
* @return the sort values for the search hit
6276
*/

‎src/test/java/org/springframework/data/elasticsearch/core/query/scriptedandruntimefields/ScriptedAndRuntimeFieldsIntegrationTests.java

+129-29
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,8 @@ void shouldUseRuntimeFieldFromQueryInSearch() {
9999
@DisplayName("should use runtime-field without script")
100100
void shouldUseRuntimeFieldWithoutScript() {
101101

102+
// a runtime field without a script can be used to redefine the type of a field for the search,
103+
// here we change the type from text to double
102104
insert("1", "11", 10);
103105
Query query = new CriteriaQuery(new Criteria("description").matches(11.0));
104106
RuntimeField runtimeField = new RuntimeField("description", "double");
@@ -133,6 +135,25 @@ void shouldReturnValueFromRuntimeFieldDefinedInMapping() {
133135
assertThat(foundPerson.getBirthDate()).isEqualTo(birthDate);
134136
}
135137

138+
@Test // #3076
139+
@DisplayName("should return scripted fields that are lists")
140+
void shouldReturnScriptedFieldsThatAreLists() {
141+
var person = new Person();
142+
person.setFirstName("John");
143+
person.setLastName("Doe");
144+
operations.save(person);
145+
var query = Query.findAll();
146+
query.addFields("allNames");
147+
query.addSourceFilter(new FetchSourceFilterBuilder().withIncludes("*").build());
148+
149+
var searchHits = operations.search(query, Person.class);
150+
151+
assertThat(searchHits.getTotalHits()).isEqualTo(1);
152+
var foundPerson = searchHits.getSearchHit(0).getContent();
153+
// the painless script seems to return the data sorted no matter in which order the values are emitted
154+
assertThat(foundPerson.getAllNames()).containsExactlyInAnyOrderElementsOf(List.of("John", "Doe"));
155+
}
156+
136157
@Test // #2035
137158
@DisplayName("should use repository method with ScriptedField parameters")
138159
void shouldUseRepositoryMethodWithScriptedFieldParameters() {
@@ -143,9 +164,11 @@ void shouldUseRepositoryMethodWithScriptedFieldParameters() {
143164

144165
repository.save(entity);
145166

146-
org.springframework.data.elasticsearch.core.query.ScriptedField scriptedField1 = getScriptedField("scriptedValue1",
167+
org.springframework.data.elasticsearch.core.query.ScriptedField scriptedField1 = buildScriptedField(
168+
"scriptedValue1",
147169
2);
148-
org.springframework.data.elasticsearch.core.query.ScriptedField scriptedField2 = getScriptedField("scriptedValue2",
170+
org.springframework.data.elasticsearch.core.query.ScriptedField scriptedField2 = buildScriptedField(
171+
"scriptedValue2",
149172
3);
150173

151174
var searchHits = repository.findByValue(3, scriptedField1, scriptedField2);
@@ -157,17 +180,6 @@ void shouldUseRepositoryMethodWithScriptedFieldParameters() {
157180
assertThat(foundEntity.getScriptedValue2()).isEqualTo(9);
158181
}
159182

160-
@NotNull
161-
private static org.springframework.data.elasticsearch.core.query.ScriptedField getScriptedField(String fieldName,
162-
int factor) {
163-
return org.springframework.data.elasticsearch.core.query.ScriptedField.of(
164-
fieldName,
165-
ScriptData.of(b -> b
166-
.withType(ScriptType.INLINE)
167-
.withScript("doc['value'].size() > 0 ? doc['value'].value * params['factor'] : 0")
168-
.withParams(Map.of("factor", factor))));
169-
}
170-
171183
@Test // #2035
172184
@DisplayName("should use repository string query method with ScriptedField parameters")
173185
void shouldUseRepositoryStringQueryMethodWithScriptedFieldParameters() {
@@ -178,9 +190,11 @@ void shouldUseRepositoryStringQueryMethodWithScriptedFieldParameters() {
178190

179191
repository.save(entity);
180192

181-
org.springframework.data.elasticsearch.core.query.ScriptedField scriptedField1 = getScriptedField("scriptedValue1",
193+
org.springframework.data.elasticsearch.core.query.ScriptedField scriptedField1 = buildScriptedField(
194+
"scriptedValue1",
182195
2);
183-
org.springframework.data.elasticsearch.core.query.ScriptedField scriptedField2 = getScriptedField("scriptedValue2",
196+
org.springframework.data.elasticsearch.core.query.ScriptedField scriptedField2 = buildScriptedField(
197+
"scriptedValue2",
184198
3);
185199

186200
var searchHits = repository.findWithScriptedFields(3, scriptedField1, scriptedField2);
@@ -202,8 +216,8 @@ void shouldUseRepositoryMethodWithRuntimeFieldParameters() {
202216

203217
repository.save(entity);
204218

205-
var runtimeField1 = getRuntimeField("scriptedValue1", 3);
206-
var runtimeField2 = getRuntimeField("scriptedValue2", 4);
219+
var runtimeField1 = buildRuntimeField("scriptedValue1", 3);
220+
var runtimeField2 = buildRuntimeField("scriptedValue2", 4);
207221

208222
var searchHits = repository.findByValue(3, runtimeField1, runtimeField2);
209223

@@ -214,14 +228,6 @@ void shouldUseRepositoryMethodWithRuntimeFieldParameters() {
214228
assertThat(foundEntity.getScriptedValue2()).isEqualTo(12);
215229
}
216230

217-
@NotNull
218-
private static RuntimeField getRuntimeField(String fieldName, int factor) {
219-
return new RuntimeField(
220-
fieldName,
221-
"long",
222-
String.format("emit(doc['value'].size() > 0 ? doc['value'].value * %d : 0)", factor));
223-
}
224-
225231
@Test // #2035
226232
@DisplayName("should use repository string query method with RuntimeField parameters")
227233
void shouldUseRepositoryStringQueryMethodWithRuntimeFieldParameters() {
@@ -232,8 +238,8 @@ void shouldUseRepositoryStringQueryMethodWithRuntimeFieldParameters() {
232238

233239
repository.save(entity);
234240

235-
var runtimeField1 = getRuntimeField("scriptedValue1", 3);
236-
var runtimeField2 = getRuntimeField("scriptedValue2", 4);
241+
var runtimeField1 = buildRuntimeField("scriptedValue1", 3);
242+
var runtimeField2 = buildRuntimeField("scriptedValue2", 4);
237243

238244
var searchHits = repository.findWithRuntimeFields(3, runtimeField1, runtimeField2);
239245

@@ -263,8 +269,7 @@ void shouldUseParametersForRuntimeFieldsInSearchQueries() {
263269
"priceWithTax",
264270
"double",
265271
"emit(doc['price'].value * params.tax)",
266-
Map.of("tax", 1.19)
267-
);
272+
Map.of("tax", 1.19));
268273
var query = CriteriaQuery.builder(
269274
Criteria.where("priceWithTax").greaterThan(100.0))
270275
.withRuntimeFields(List.of(runtimeField))
@@ -275,6 +280,56 @@ void shouldUseParametersForRuntimeFieldsInSearchQueries() {
275280
assertThat(searchHits).hasSize(1);
276281
}
277282

283+
@Test // #3076
284+
@DisplayName("should use runtime fields in queries returning lists")
285+
void shouldUseRuntimeFieldsInQueriesReturningLists() {
286+
287+
insert("1", "item 1", 80.0);
288+
289+
var runtimeField = new RuntimeField(
290+
"someStrings",
291+
"keyword",
292+
"emit('foo'); emit('bar');",
293+
null);
294+
295+
var query = Query.findAll();
296+
query.addRuntimeField(runtimeField);
297+
query.addFields("someStrings");
298+
query.addSourceFilter(new FetchSourceFilterBuilder().withIncludes("*").build());
299+
300+
var searchHits = operations.search(query, SomethingToBuy.class);
301+
302+
assertThat(searchHits).hasSize(1);
303+
var somethingToBuy = searchHits.getSearchHit(0).getContent();
304+
assertThat(somethingToBuy.someStrings).containsExactlyInAnyOrder("foo", "bar");
305+
}
306+
307+
/**
308+
* build a {@link org.springframework.data.elasticsearch.core.query.ScriptedField} to return the product of the
309+
* document's value property and the given factor
310+
*/
311+
@NotNull
312+
private static org.springframework.data.elasticsearch.core.query.ScriptedField buildScriptedField(String fieldName,
313+
int factor) {
314+
return org.springframework.data.elasticsearch.core.query.ScriptedField.of(
315+
fieldName,
316+
ScriptData.of(b -> b
317+
.withType(ScriptType.INLINE)
318+
.withScript("doc['value'].size() > 0 ? doc['value'].value * params['factor'] : 0")
319+
.withParams(Map.of("factor", factor))));
320+
}
321+
322+
/**
323+
* build a {@link RuntimeField} to return the product of the document's value property and the given factor
324+
*/
325+
@NotNull
326+
private static RuntimeField buildRuntimeField(String fieldName, int factor) {
327+
return new RuntimeField(
328+
fieldName,
329+
"long",
330+
String.format("emit(doc['value'].size() > 0 ? doc['value'].value * %d : 0)", factor));
331+
}
332+
278333
@SuppressWarnings("unused")
279334
@Document(indexName = "#{@indexNameProvider.indexName()}-something-to-by")
280335
private static class SomethingToBuy {
@@ -286,6 +341,9 @@ private static class SomethingToBuy {
286341
@Nullable
287342
@Field(type = FieldType.Double) private Double price;
288343

344+
@Nullable
345+
@ScriptedField private List<String> someStrings;
346+
289347
@Nullable
290348
public String getId() {
291349
return id;
@@ -312,6 +370,15 @@ public Double getPrice() {
312370
public void setPrice(@Nullable Double price) {
313371
this.price = price;
314372
}
373+
374+
@Nullable
375+
public List<String> getSomeStrings() {
376+
return someStrings;
377+
}
378+
379+
public void setSomeStrings(@Nullable List<String> someStrings) {
380+
this.someStrings = someStrings;
381+
}
315382
}
316383

317384
@SuppressWarnings("unused")
@@ -320,6 +387,13 @@ public void setPrice(@Nullable Double price) {
320387
public static class Person {
321388
@Nullable private String id;
322389

390+
// need keywords as we are using them in the script
391+
@Nullable
392+
@Field(type = FieldType.Keyword) private String firstName;
393+
@Nullable
394+
@Field(type = FieldType.Keyword) private String lastName;
395+
@ScriptedField private List<String> allNames = List.of();
396+
323397
@Field(type = FieldType.Date, format = DateFormat.basic_date)
324398
@Nullable private LocalDate birthDate;
325399

@@ -335,6 +409,24 @@ public void setId(@Nullable String id) {
335409
this.id = id;
336410
}
337411

412+
@Nullable
413+
public String getFirstName() {
414+
return firstName;
415+
}
416+
417+
public void setFirstName(@Nullable String firstName) {
418+
this.firstName = firstName;
419+
}
420+
421+
@Nullable
422+
public String getLastName() {
423+
return lastName;
424+
}
425+
426+
public void setLastName(@Nullable String lastName) {
427+
this.lastName = lastName;
428+
}
429+
338430
@Nullable
339431
public LocalDate getBirthDate() {
340432
return birthDate;
@@ -352,6 +444,14 @@ public Integer getAge() {
352444
public void setAge(@Nullable Integer age) {
353445
this.age = age;
354446
}
447+
448+
public List<String> getAllNames() {
449+
return allNames;
450+
}
451+
452+
public void setAllNames(List<String> allNames) {
453+
this.allNames = allNames;
454+
}
355455
}
356456

357457
@SuppressWarnings("unused")

‎src/test/resources/runtime-fields-person.json

+7
Original file line numberDiff line numberDiff line change
@@ -5,5 +5,12 @@
55
"lang": "painless",
66
"source": "Instant currentDate = Instant.ofEpochMilli(new Date().getTime()); Instant startDate = doc['birthDate'].value.toInstant(); emit(ChronoUnit.DAYS.between(startDate, currentDate) / 365);"
77
}
8+
},
9+
"allNames": {
10+
"type": "keyword",
11+
"script": {
12+
"lang": "painless",
13+
"source": "emit(doc['firstName'].value);emit(doc['lastName'].value);"
14+
}
815
}
916
}

0 commit comments

Comments
 (0)