From d4df1f806518ce85184f8553a14d4b7043792e6b Mon Sep 17 00:00:00 2001 From: Peter-Josef Meisch Date: Tue, 18 Mar 2025 20:22:34 +0100 Subject: [PATCH] Enable scripted fields and runtime fields of collection type. Closes #3076 Signed-off-by: Peter-Josef Meisch --- .../elasticsearch/elasticsearch-new.adoc | 1 + .../MappingElasticsearchConverter.java | 22 ++- .../core/document/SearchDocument.java | 14 ++ ...iptedAndRuntimeFieldsIntegrationTests.java | 158 ++++++++++++++---- src/test/resources/runtime-fields-person.json | 7 + 5 files changed, 169 insertions(+), 33 deletions(-) diff --git a/src/main/antora/modules/ROOT/pages/elasticsearch/elasticsearch-new.adoc b/src/main/antora/modules/ROOT/pages/elasticsearch/elasticsearch-new.adoc index 5e11d4f928..887dc49de6 100644 --- a/src/main/antora/modules/ROOT/pages/elasticsearch/elasticsearch-new.adoc +++ b/src/main/antora/modules/ROOT/pages/elasticsearch/elasticsearch-new.adoc @@ -6,6 +6,7 @@ * Upgrade to Elasticsearch 8.17.2. * Add support for the `@SearchTemplateQuery` annotation on repository methods. +* Scripted field properties of type collection can be populated from scripts returning arrays. [[new-features.5-4-0]] == New in Spring Data Elasticsearch 5.4 diff --git a/src/main/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverter.java b/src/main/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverter.java index d33cef5c34..ff564f7b7e 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverter.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverter.java @@ -394,7 +394,7 @@ private R readEntity(ElasticsearchPersistentEntity entity, Map target) { return conversionService.convert(value, target); } - private void populateScriptFields(ElasticsearchPersistentEntity entity, T result, + /** + * Checks if any of the properties of the entity is annotated with + * + * @{@link ScriptedField}. If so, the value of this property is set from the returned fields in the document. + * @param entity the entity to defining the persistent property + * @param result the rsult to populate + * @param searchDocument the search result caontaining the fields + * @param the result type + */ + private void populateScriptedFields(ElasticsearchPersistentEntity entity, T result, SearchDocument searchDocument) { Map> fields = searchDocument.getFields(); entity.doWithProperties((SimplePropertyHandler) property -> { @@ -661,8 +670,13 @@ private void populateScriptFields(ElasticsearchPersistentEntity entity, T // noinspection ConstantConditions String name = scriptedField.name().isEmpty() ? property.getName() : scriptedField.name(); if (fields.containsKey(name)) { - Object value = searchDocument.getFieldValue(name); - entity.getPropertyAccessor(result).setProperty(property, value); + if (property.isCollectionLike()) { + List values = searchDocument.getFieldValues(name); + entity.getPropertyAccessor(result).setProperty(property, values); + } else { + Object value = searchDocument.getFieldValue(name); + entity.getPropertyAccessor(result).setProperty(property, value); + } } } }); diff --git a/src/main/java/org/springframework/data/elasticsearch/core/document/SearchDocument.java b/src/main/java/org/springframework/data/elasticsearch/core/document/SearchDocument.java index d98c68524d..2fb158f29a 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/document/SearchDocument.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/document/SearchDocument.java @@ -57,6 +57,20 @@ default V getFieldValue(final String name) { return (V) values.get(0); } + /** + * @param name the field name + * @param the type of elements + * @return the values of the given field. + */ + @Nullable + default List getFieldValues(final String name) { + List values = getFields().get(name); + if (values == null) { + return null; + } + return (List) values; + } + /** * @return the sort values for the search hit */ diff --git a/src/test/java/org/springframework/data/elasticsearch/core/query/scriptedandruntimefields/ScriptedAndRuntimeFieldsIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/query/scriptedandruntimefields/ScriptedAndRuntimeFieldsIntegrationTests.java index 9d05a240c3..14cedf76b8 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/query/scriptedandruntimefields/ScriptedAndRuntimeFieldsIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/query/scriptedandruntimefields/ScriptedAndRuntimeFieldsIntegrationTests.java @@ -99,6 +99,8 @@ void shouldUseRuntimeFieldFromQueryInSearch() { @DisplayName("should use runtime-field without script") void shouldUseRuntimeFieldWithoutScript() { + // a runtime field without a script can be used to redefine the type of a field for the search, + // here we change the type from text to double insert("1", "11", 10); Query query = new CriteriaQuery(new Criteria("description").matches(11.0)); RuntimeField runtimeField = new RuntimeField("description", "double"); @@ -133,6 +135,25 @@ void shouldReturnValueFromRuntimeFieldDefinedInMapping() { assertThat(foundPerson.getBirthDate()).isEqualTo(birthDate); } + @Test // #3076 + @DisplayName("should return scripted fields that are lists") + void shouldReturnScriptedFieldsThatAreLists() { + var person = new Person(); + person.setFirstName("John"); + person.setLastName("Doe"); + operations.save(person); + var query = Query.findAll(); + query.addFields("allNames"); + query.addSourceFilter(new FetchSourceFilterBuilder().withIncludes("*").build()); + + var searchHits = operations.search(query, Person.class); + + assertThat(searchHits.getTotalHits()).isEqualTo(1); + var foundPerson = searchHits.getSearchHit(0).getContent(); + // the painless script seems to return the data sorted no matter in which order the values are emitted + assertThat(foundPerson.getAllNames()).containsExactlyInAnyOrderElementsOf(List.of("John", "Doe")); + } + @Test // #2035 @DisplayName("should use repository method with ScriptedField parameters") void shouldUseRepositoryMethodWithScriptedFieldParameters() { @@ -143,9 +164,11 @@ void shouldUseRepositoryMethodWithScriptedFieldParameters() { repository.save(entity); - org.springframework.data.elasticsearch.core.query.ScriptedField scriptedField1 = getScriptedField("scriptedValue1", + org.springframework.data.elasticsearch.core.query.ScriptedField scriptedField1 = buildScriptedField( + "scriptedValue1", 2); - org.springframework.data.elasticsearch.core.query.ScriptedField scriptedField2 = getScriptedField("scriptedValue2", + org.springframework.data.elasticsearch.core.query.ScriptedField scriptedField2 = buildScriptedField( + "scriptedValue2", 3); var searchHits = repository.findByValue(3, scriptedField1, scriptedField2); @@ -157,17 +180,6 @@ void shouldUseRepositoryMethodWithScriptedFieldParameters() { assertThat(foundEntity.getScriptedValue2()).isEqualTo(9); } - @NotNull - private static org.springframework.data.elasticsearch.core.query.ScriptedField getScriptedField(String fieldName, - int factor) { - return org.springframework.data.elasticsearch.core.query.ScriptedField.of( - fieldName, - ScriptData.of(b -> b - .withType(ScriptType.INLINE) - .withScript("doc['value'].size() > 0 ? doc['value'].value * params['factor'] : 0") - .withParams(Map.of("factor", factor)))); - } - @Test // #2035 @DisplayName("should use repository string query method with ScriptedField parameters") void shouldUseRepositoryStringQueryMethodWithScriptedFieldParameters() { @@ -178,9 +190,11 @@ void shouldUseRepositoryStringQueryMethodWithScriptedFieldParameters() { repository.save(entity); - org.springframework.data.elasticsearch.core.query.ScriptedField scriptedField1 = getScriptedField("scriptedValue1", + org.springframework.data.elasticsearch.core.query.ScriptedField scriptedField1 = buildScriptedField( + "scriptedValue1", 2); - org.springframework.data.elasticsearch.core.query.ScriptedField scriptedField2 = getScriptedField("scriptedValue2", + org.springframework.data.elasticsearch.core.query.ScriptedField scriptedField2 = buildScriptedField( + "scriptedValue2", 3); var searchHits = repository.findWithScriptedFields(3, scriptedField1, scriptedField2); @@ -202,8 +216,8 @@ void shouldUseRepositoryMethodWithRuntimeFieldParameters() { repository.save(entity); - var runtimeField1 = getRuntimeField("scriptedValue1", 3); - var runtimeField2 = getRuntimeField("scriptedValue2", 4); + var runtimeField1 = buildRuntimeField("scriptedValue1", 3); + var runtimeField2 = buildRuntimeField("scriptedValue2", 4); var searchHits = repository.findByValue(3, runtimeField1, runtimeField2); @@ -214,14 +228,6 @@ void shouldUseRepositoryMethodWithRuntimeFieldParameters() { assertThat(foundEntity.getScriptedValue2()).isEqualTo(12); } - @NotNull - private static RuntimeField getRuntimeField(String fieldName, int factor) { - return new RuntimeField( - fieldName, - "long", - String.format("emit(doc['value'].size() > 0 ? doc['value'].value * %d : 0)", factor)); - } - @Test // #2035 @DisplayName("should use repository string query method with RuntimeField parameters") void shouldUseRepositoryStringQueryMethodWithRuntimeFieldParameters() { @@ -232,8 +238,8 @@ void shouldUseRepositoryStringQueryMethodWithRuntimeFieldParameters() { repository.save(entity); - var runtimeField1 = getRuntimeField("scriptedValue1", 3); - var runtimeField2 = getRuntimeField("scriptedValue2", 4); + var runtimeField1 = buildRuntimeField("scriptedValue1", 3); + var runtimeField2 = buildRuntimeField("scriptedValue2", 4); var searchHits = repository.findWithRuntimeFields(3, runtimeField1, runtimeField2); @@ -263,8 +269,7 @@ void shouldUseParametersForRuntimeFieldsInSearchQueries() { "priceWithTax", "double", "emit(doc['price'].value * params.tax)", - Map.of("tax", 1.19) - ); + Map.of("tax", 1.19)); var query = CriteriaQuery.builder( Criteria.where("priceWithTax").greaterThan(100.0)) .withRuntimeFields(List.of(runtimeField)) @@ -275,6 +280,56 @@ void shouldUseParametersForRuntimeFieldsInSearchQueries() { assertThat(searchHits).hasSize(1); } + @Test // #3076 + @DisplayName("should use runtime fields in queries returning lists") + void shouldUseRuntimeFieldsInQueriesReturningLists() { + + insert("1", "item 1", 80.0); + + var runtimeField = new RuntimeField( + "someStrings", + "keyword", + "emit('foo'); emit('bar');", + null); + + var query = Query.findAll(); + query.addRuntimeField(runtimeField); + query.addFields("someStrings"); + query.addSourceFilter(new FetchSourceFilterBuilder().withIncludes("*").build()); + + var searchHits = operations.search(query, SomethingToBuy.class); + + assertThat(searchHits).hasSize(1); + var somethingToBuy = searchHits.getSearchHit(0).getContent(); + assertThat(somethingToBuy.someStrings).containsExactlyInAnyOrder("foo", "bar"); + } + + /** + * build a {@link org.springframework.data.elasticsearch.core.query.ScriptedField} to return the product of the + * document's value property and the given factor + */ + @NotNull + private static org.springframework.data.elasticsearch.core.query.ScriptedField buildScriptedField(String fieldName, + int factor) { + return org.springframework.data.elasticsearch.core.query.ScriptedField.of( + fieldName, + ScriptData.of(b -> b + .withType(ScriptType.INLINE) + .withScript("doc['value'].size() > 0 ? doc['value'].value * params['factor'] : 0") + .withParams(Map.of("factor", factor)))); + } + + /** + * build a {@link RuntimeField} to return the product of the document's value property and the given factor + */ + @NotNull + private static RuntimeField buildRuntimeField(String fieldName, int factor) { + return new RuntimeField( + fieldName, + "long", + String.format("emit(doc['value'].size() > 0 ? doc['value'].value * %d : 0)", factor)); + } + @SuppressWarnings("unused") @Document(indexName = "#{@indexNameProvider.indexName()}-something-to-by") private static class SomethingToBuy { @@ -286,6 +341,9 @@ private static class SomethingToBuy { @Nullable @Field(type = FieldType.Double) private Double price; + @Nullable + @ScriptedField private List someStrings; + @Nullable public String getId() { return id; @@ -312,6 +370,15 @@ public Double getPrice() { public void setPrice(@Nullable Double price) { this.price = price; } + + @Nullable + public List getSomeStrings() { + return someStrings; + } + + public void setSomeStrings(@Nullable List someStrings) { + this.someStrings = someStrings; + } } @SuppressWarnings("unused") @@ -320,6 +387,13 @@ public void setPrice(@Nullable Double price) { public static class Person { @Nullable private String id; + // need keywords as we are using them in the script + @Nullable + @Field(type = FieldType.Keyword) private String firstName; + @Nullable + @Field(type = FieldType.Keyword) private String lastName; + @ScriptedField private List allNames = List.of(); + @Field(type = FieldType.Date, format = DateFormat.basic_date) @Nullable private LocalDate birthDate; @@ -335,6 +409,24 @@ public void setId(@Nullable String id) { this.id = id; } + @Nullable + public String getFirstName() { + return firstName; + } + + public void setFirstName(@Nullable String firstName) { + this.firstName = firstName; + } + + @Nullable + public String getLastName() { + return lastName; + } + + public void setLastName(@Nullable String lastName) { + this.lastName = lastName; + } + @Nullable public LocalDate getBirthDate() { return birthDate; @@ -352,6 +444,14 @@ public Integer getAge() { public void setAge(@Nullable Integer age) { this.age = age; } + + public List getAllNames() { + return allNames; + } + + public void setAllNames(List allNames) { + this.allNames = allNames; + } } @SuppressWarnings("unused") diff --git a/src/test/resources/runtime-fields-person.json b/src/test/resources/runtime-fields-person.json index 85f24d9649..f225e02a09 100644 --- a/src/test/resources/runtime-fields-person.json +++ b/src/test/resources/runtime-fields-person.json @@ -5,5 +5,12 @@ "lang": "painless", "source": "Instant currentDate = Instant.ofEpochMilli(new Date().getTime()); Instant startDate = doc['birthDate'].value.toInstant(); emit(ChronoUnit.DAYS.between(startDate, currentDate) / 365);" } + }, + "allNames": { + "type": "keyword", + "script": { + "lang": "painless", + "source": "emit(doc['firstName'].value);emit(doc['lastName'].value);" + } } }