Enable scripted fields and runtime fields of collection type.

src/main/antora/modules/ROOT/pages/elasticsearch/elasticsearch-new.adoc

@@ -6,6 +6,7 @@
 
 * Upgrade to Elasticsearch 8.17.2.
 * Add support for the `@SearchTemplateQuery` annotation on repository methods.
+* Scripted field properties of type collection can be populated from scripts returning arrays.
 
 [[new-features.5-4-0]]
 == New in Spring Data Elasticsearch 5.4

src/main/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverter.java

@@ -394,7 +394,7 @@ private <R> R readEntity(ElasticsearchPersistentEntity<?> entity, Map<String, Ob
 				}
 
 				if (source instanceof SearchDocument searchDocument) {
-					populateScriptFields(targetEntity, result, searchDocument);
+					populateScriptedFields(targetEntity, result, searchDocument);
 				}
 				return result;
 			} catch (ConversionException e) {
@@ -652,7 +652,16 @@ private Object convertFromCollectionToObject(Object value, Class<?> target) {
 			return conversionService.convert(value, target);
 		}
 
-		private <T> void populateScriptFields(ElasticsearchPersistentEntity<?> entity, T result,
+		/**
+		 * Checks if any of the properties of the entity is annotated with
+		 *
+		 * @{@link ScriptedField}. If so, the value of this property is set from the returned fields in the document.
+		 * @param entity the entity to defining the persistent property
+		 * @param result the rsult to populate
+		 * @param searchDocument the search result caontaining the fields
+		 * @param <T> the result type
+		 */
+		private <T> void populateScriptedFields(ElasticsearchPersistentEntity<?> entity, T result,
 				SearchDocument searchDocument) {
 			Map<String, List<Object>> fields = searchDocument.getFields();
 			entity.doWithProperties((SimplePropertyHandler) property -> {
@@ -661,8 +670,13 @@ private <T> void populateScriptFields(ElasticsearchPersistentEntity<?> entity, T
 					// noinspection ConstantConditions
 					String name = scriptedField.name().isEmpty() ? property.getName() : scriptedField.name();
 					if (fields.containsKey(name)) {
-						Object value = searchDocument.getFieldValue(name);
-						entity.getPropertyAccessor(result).setProperty(property, value);
+						if (property.isCollectionLike()) {
+							List<Object> values = searchDocument.getFieldValues(name);
+							entity.getPropertyAccessor(result).setProperty(property, values);
+						} else {
+							Object value = searchDocument.getFieldValue(name);
+							entity.getPropertyAccessor(result).setProperty(property, value);
+						}
 					}
 				}
 			});

src/main/java/org/springframework/data/elasticsearch/core/document/SearchDocument.java

@@ -57,6 +57,20 @@ default <V> V getFieldValue(final String name) {
 		return (V) values.get(0);
 	}
 
+	/**
+	 * @param name the field name
+	 * @param <V> the type of elements
+	 * @return the values of the given field.
+	 */
+	@Nullable
+	default <V> List<V> getFieldValues(final String name) {
+		List<Object> values = getFields().get(name);
+		if (values == null) {
+			return null;
+		}
+		return (List<V>) values;
+	}
+
 	/**
 	 * @return the sort values for the search hit
 	 */

src/test/java/org/springframework/data/elasticsearch/core/query/scriptedandruntimefields/ScriptedAndRuntimeFieldsIntegrationTests.java

@@ -99,6 +99,8 @@ void shouldUseRuntimeFieldFromQueryInSearch() {
 	@DisplayName("should use runtime-field without script")
 	void shouldUseRuntimeFieldWithoutScript() {
 
+		// a runtime field without a script can be used to redefine the type of a field for the search,
+		// here we change the type from text to double
 		insert("1", "11", 10);
 		Query query = new CriteriaQuery(new Criteria("description").matches(11.0));
 		RuntimeField runtimeField = new RuntimeField("description", "double");
@@ -133,6 +135,25 @@ void shouldReturnValueFromRuntimeFieldDefinedInMapping() {
 		assertThat(foundPerson.getBirthDate()).isEqualTo(birthDate);
 	}
 
+	@Test // #3076
+	@DisplayName("should return scripted fields that are lists")
+	void shouldReturnScriptedFieldsThatAreLists() {
+		var person = new Person();
+		person.setFirstName("John");
+		person.setLastName("Doe");
+		operations.save(person);
+		var query = Query.findAll();
+		query.addFields("allNames");
+		query.addSourceFilter(new FetchSourceFilterBuilder().withIncludes("*").build());
+
+		var searchHits = operations.search(query, Person.class);
+
+		assertThat(searchHits.getTotalHits()).isEqualTo(1);
+		var foundPerson = searchHits.getSearchHit(0).getContent();
+		// the painless script seems to return the data sorted no matter in which order the values are emitted
+		assertThat(foundPerson.getAllNames()).containsExactlyInAnyOrderElementsOf(List.of("John", "Doe"));
+	}
+
 	@Test // #2035
 	@DisplayName("should use repository method with ScriptedField parameters")
 	void shouldUseRepositoryMethodWithScriptedFieldParameters() {
@@ -143,9 +164,11 @@ void shouldUseRepositoryMethodWithScriptedFieldParameters() {
 
 		repository.save(entity);
 
-		org.springframework.data.elasticsearch.core.query.ScriptedField scriptedField1 = getScriptedField("scriptedValue1",
+		org.springframework.data.elasticsearch.core.query.ScriptedField scriptedField1 = buildScriptedField(
+				"scriptedValue1",
 				2);
-		org.springframework.data.elasticsearch.core.query.ScriptedField scriptedField2 = getScriptedField("scriptedValue2",
+		org.springframework.data.elasticsearch.core.query.ScriptedField scriptedField2 = buildScriptedField(
+				"scriptedValue2",
 				3);
 
 		var searchHits = repository.findByValue(3, scriptedField1, scriptedField2);
@@ -157,17 +180,6 @@ void shouldUseRepositoryMethodWithScriptedFieldParameters() {
 		assertThat(foundEntity.getScriptedValue2()).isEqualTo(9);
 	}
 
-	@NotNull
-	private static org.springframework.data.elasticsearch.core.query.ScriptedField getScriptedField(String fieldName,
-			int factor) {
-		return org.springframework.data.elasticsearch.core.query.ScriptedField.of(
-				fieldName,
-				ScriptData.of(b -> b
-						.withType(ScriptType.INLINE)
-						.withScript("doc['value'].size() > 0 ? doc['value'].value * params['factor'] : 0")
-						.withParams(Map.of("factor", factor))));
-	}
-
 	@Test // #2035
 	@DisplayName("should use repository string query method with ScriptedField parameters")
 	void shouldUseRepositoryStringQueryMethodWithScriptedFieldParameters() {
@@ -178,9 +190,11 @@ void shouldUseRepositoryStringQueryMethodWithScriptedFieldParameters() {
 
 		repository.save(entity);
 
-		org.springframework.data.elasticsearch.core.query.ScriptedField scriptedField1 = getScriptedField("scriptedValue1",
+		org.springframework.data.elasticsearch.core.query.ScriptedField scriptedField1 = buildScriptedField(
+				"scriptedValue1",
 				2);
-		org.springframework.data.elasticsearch.core.query.ScriptedField scriptedField2 = getScriptedField("scriptedValue2",
+		org.springframework.data.elasticsearch.core.query.ScriptedField scriptedField2 = buildScriptedField(
+				"scriptedValue2",
 				3);
 
 		var searchHits = repository.findWithScriptedFields(3, scriptedField1, scriptedField2);
@@ -202,8 +216,8 @@ void shouldUseRepositoryMethodWithRuntimeFieldParameters() {
 
 		repository.save(entity);
 
-		var runtimeField1 = getRuntimeField("scriptedValue1", 3);
-		var runtimeField2 = getRuntimeField("scriptedValue2", 4);
+		var runtimeField1 = buildRuntimeField("scriptedValue1", 3);
+		var runtimeField2 = buildRuntimeField("scriptedValue2", 4);
 
 		var searchHits = repository.findByValue(3, runtimeField1, runtimeField2);
 
@@ -214,14 +228,6 @@ void shouldUseRepositoryMethodWithRuntimeFieldParameters() {
 		assertThat(foundEntity.getScriptedValue2()).isEqualTo(12);
 	}
 
-	@NotNull
-	private static RuntimeField getRuntimeField(String fieldName, int factor) {
-		return new RuntimeField(
-				fieldName,
-				"long",
-				String.format("emit(doc['value'].size() > 0 ? doc['value'].value * %d : 0)", factor));
-	}
-
 	@Test // #2035
 	@DisplayName("should use repository string query method with RuntimeField parameters")
 	void shouldUseRepositoryStringQueryMethodWithRuntimeFieldParameters() {
@@ -232,8 +238,8 @@ void shouldUseRepositoryStringQueryMethodWithRuntimeFieldParameters() {
 
 		repository.save(entity);
 
-		var runtimeField1 = getRuntimeField("scriptedValue1", 3);
-		var runtimeField2 = getRuntimeField("scriptedValue2", 4);
+		var runtimeField1 = buildRuntimeField("scriptedValue1", 3);
+		var runtimeField2 = buildRuntimeField("scriptedValue2", 4);
 
 		var searchHits = repository.findWithRuntimeFields(3, runtimeField1, runtimeField2);
 
@@ -263,8 +269,7 @@ void shouldUseParametersForRuntimeFieldsInSearchQueries() {
 				"priceWithTax",
 				"double",
 				"emit(doc['price'].value * params.tax)",
-				Map.of("tax", 1.19)
-		);
+				Map.of("tax", 1.19));
 		var query = CriteriaQuery.builder(
 				Criteria.where("priceWithTax").greaterThan(100.0))
 				.withRuntimeFields(List.of(runtimeField))
@@ -275,6 +280,56 @@ void shouldUseParametersForRuntimeFieldsInSearchQueries() {
 		assertThat(searchHits).hasSize(1);
 	}
 
+	@Test // #3076
+	@DisplayName("should use runtime fields in queries returning lists")
+	void shouldUseRuntimeFieldsInQueriesReturningLists() {
+
+		insert("1", "item 1", 80.0);
+
+		var runtimeField = new RuntimeField(
+				"someStrings",
+				"keyword",
+				"emit('foo'); emit('bar');",
+				null);
+
+		var query = Query.findAll();
+		query.addRuntimeField(runtimeField);
+		query.addFields("someStrings");
+		query.addSourceFilter(new FetchSourceFilterBuilder().withIncludes("*").build());
+
+		var searchHits = operations.search(query, SomethingToBuy.class);
+
+		assertThat(searchHits).hasSize(1);
+		var somethingToBuy = searchHits.getSearchHit(0).getContent();
+		assertThat(somethingToBuy.someStrings).containsExactlyInAnyOrder("foo", "bar");
+	}
+
+	/**
+	 * build a {@link org.springframework.data.elasticsearch.core.query.ScriptedField} to return the product of the
+	 * document's value property and the given factor
+	 */
+	@NotNull
+	private static org.springframework.data.elasticsearch.core.query.ScriptedField buildScriptedField(String fieldName,
+			int factor) {
+		return org.springframework.data.elasticsearch.core.query.ScriptedField.of(
+				fieldName,
+				ScriptData.of(b -> b
+						.withType(ScriptType.INLINE)
+						.withScript("doc['value'].size() > 0 ? doc['value'].value * params['factor'] : 0")
+						.withParams(Map.of("factor", factor))));
+	}
+
+	/**
+	 * build a {@link RuntimeField} to return the product of the document's value property and the given factor
+	 */
+	@NotNull
+	private static RuntimeField buildRuntimeField(String fieldName, int factor) {
+		return new RuntimeField(
+				fieldName,
+				"long",
+				String.format("emit(doc['value'].size() > 0 ? doc['value'].value * %d : 0)", factor));
+	}
+
 	@SuppressWarnings("unused")
 	@Document(indexName = "#{@indexNameProvider.indexName()}-something-to-by")
 	private static class SomethingToBuy {
@@ -286,6 +341,9 @@ private static class SomethingToBuy {
 		@Nullable
 		@Field(type = FieldType.Double) private Double price;
 
+		@Nullable
+		@ScriptedField private List<String> someStrings;
+
 		@Nullable
 		public String getId() {
 			return id;
@@ -312,6 +370,15 @@ public Double getPrice() {
 		public void setPrice(@Nullable Double price) {
 			this.price = price;
 		}
+
+		@Nullable
+		public List<String> getSomeStrings() {
+			return someStrings;
+		}
+
+		public void setSomeStrings(@Nullable List<String> someStrings) {
+			this.someStrings = someStrings;
+		}
 	}
 
 	@SuppressWarnings("unused")
@@ -320,6 +387,13 @@ public void setPrice(@Nullable Double price) {
 	public static class Person {
 		@Nullable private String id;
 
+		// need keywords as we are using them in the script
+		@Nullable
+		@Field(type = FieldType.Keyword) private String firstName;
+		@Nullable
+		@Field(type = FieldType.Keyword) private String lastName;
+		@ScriptedField private List<String> allNames = List.of();
+
 		@Field(type = FieldType.Date, format = DateFormat.basic_date)
 		@Nullable private LocalDate birthDate;
 
@@ -335,6 +409,24 @@ public void setId(@Nullable String id) {
 			this.id = id;
 		}
 
+		@Nullable
+		public String getFirstName() {
+			return firstName;
+		}
+
+		public void setFirstName(@Nullable String firstName) {
+			this.firstName = firstName;
+		}
+
+		@Nullable
+		public String getLastName() {
+			return lastName;
+		}
+
+		public void setLastName(@Nullable String lastName) {
+			this.lastName = lastName;
+		}
+
 		@Nullable
 		public LocalDate getBirthDate() {
 			return birthDate;
@@ -352,6 +444,14 @@ public Integer getAge() {
 		public void setAge(@Nullable Integer age) {
 			this.age = age;
 		}
+
+		public List<String> getAllNames() {
+			return allNames;
+		}
+
+		public void setAllNames(List<String> allNames) {
+			this.allNames = allNames;
+		}
 	}
 
 	@SuppressWarnings("unused")

src/test/resources/runtime-fields-person.json

@@ -5,5 +5,12 @@
             "lang": "painless",
             "source": "Instant currentDate = Instant.ofEpochMilli(new Date().getTime()); Instant startDate = doc['birthDate'].value.toInstant(); emit(ChronoUnit.DAYS.between(startDate, currentDate) / 365);"
         }
+    },
+    "allNames": {
+        "type": "keyword",
+        "script": {
+            "lang": "painless",
+            "source": "emit(doc['firstName'].value);emit(doc['lastName'].value);"
+        }
     }
 }