From 399325c794c3592509ad84c82c338ff13c5f89ad Mon Sep 17 00:00:00 2001 From: Peter-Josef Meisch Date: Wed, 12 May 2021 21:52:58 +0200 Subject: [PATCH] datatype detection support in mapping. --- .../reference/elasticsearch-misc.adoc | 10 ++ .../annotations/DynamicTemplates.java | 10 +- .../elasticsearch/annotations/Mapping.java | 25 ++++- .../core/index/MappingBuilder.java | 18 ++- .../index/MappingBuilderIntegrationTests.java | 19 ++++ .../core/index/MappingBuilderUnitTests.java | 104 ++++++++++++++++++ 6 files changed, 178 insertions(+), 8 deletions(-) diff --git a/src/main/asciidoc/reference/elasticsearch-misc.adoc b/src/main/asciidoc/reference/elasticsearch-misc.adoc index 042759e7c..372dd04d3 100644 --- a/src/main/asciidoc/reference/elasticsearch-misc.adoc +++ b/src/main/asciidoc/reference/elasticsearch-misc.adoc @@ -46,6 +46,16 @@ class Entity { <.> `sortModes`, `sortOrders` and `sortMissingValues` are optional, but if they are set, the number of entries must match the number of `sortFields` elements ==== +[[elasticsearch.misc.mappings]] +== Index Mapping + +When Spring Data Elasticsearch creates the index mapping with the `IndexOperations.createMapping()` methods, it uses the annotations described in <>, especially the `@Field` annotation. In addition to that it is possible to add the `@Mapping` annotation to a class. This annotation has the following properties: + +* `mappingPath` a classpath resource in JSON format which is used as the mapping, no other mapping processing is done. +* `enabled` when set to false, this flag is written to the mapping and no further processing is done. +* `dateDetection` and `numericDetection` set the corresponding properties in the mapping when not set to `DEFAULT`. +* `dynamicDateFormats` when this String array is not empty, it defines the date formats used for automatic date detection. + [[elasticsearch.misc.filter]] == Filter Builder diff --git a/src/main/java/org/springframework/data/elasticsearch/annotations/DynamicTemplates.java b/src/main/java/org/springframework/data/elasticsearch/annotations/DynamicTemplates.java index d2cea3f39..107ee94be 100644 --- a/src/main/java/org/springframework/data/elasticsearch/annotations/DynamicTemplates.java +++ b/src/main/java/org/springframework/data/elasticsearch/annotations/DynamicTemplates.java @@ -9,19 +9,17 @@ import org.springframework.data.annotation.Persistent; /** - * Elasticsearch dynamic templates mapping. - * This annotation is handy if you prefer apply dynamic templates on fields with annotation e.g. {@link Field} - * with type = FieldType.Object etc. instead of static mapping on Document via {@link Mapping} annotation. - * DynamicTemplates annotation is omitted if {@link Mapping} annotation is used. + * Elasticsearch dynamic templates mapping. This annotation is handy if you prefer apply dynamic templates on fields + * with annotation e.g. {@link Field} with type = FieldType.Object etc. instead of static mapping on Document via + * {@link Mapping} annotation. DynamicTemplates annotation is omitted if {@link Mapping} annotation is used. * * @author Petr Kukral */ @Persistent @Inherited @Retention(RetentionPolicy.RUNTIME) -@Target({ElementType.TYPE}) +@Target({ ElementType.TYPE }) public @interface DynamicTemplates { String mappingPath() default ""; - } diff --git a/src/main/java/org/springframework/data/elasticsearch/annotations/Mapping.java b/src/main/java/org/springframework/data/elasticsearch/annotations/Mapping.java index b334b46f1..c41e1300e 100644 --- a/src/main/java/org/springframework/data/elasticsearch/annotations/Mapping.java +++ b/src/main/java/org/springframework/data/elasticsearch/annotations/Mapping.java @@ -38,8 +38,31 @@ /** * whether mappings are enabled - * + * * @since 4.2 */ boolean enabled() default true; + /** + * whether date_detection is enabled + * + * @since 4.3 + */ + Detection dateDetection() default Detection.DEFAULT; + + /** + * whether numeric_detection is enabled + * + * @since 4.3 + */ + Detection numericDetection() default Detection.DEFAULT; + + /** + * custom dynamic date formats + * @since 4.3 + */ + String[] dynamicDateFormats() default {}; + + enum Detection { + DEFAULT, TRUE, FALSE; + } } diff --git a/src/main/java/org/springframework/data/elasticsearch/core/index/MappingBuilder.java b/src/main/java/org/springframework/data/elasticsearch/core/index/MappingBuilder.java index 2cc1df4df..43b4bb4c5 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/index/MappingBuilder.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/index/MappingBuilder.java @@ -92,6 +92,9 @@ public class MappingBuilder { private static final String JOIN_TYPE_RELATIONS = "relations"; private static final String MAPPING_ENABLED = "enabled"; + private static final String DATE_DETECTION = "date_detection"; + private static final String NUMERIC_DETECTION = "numeric_detection"; + private static final String DYNAMIC_DATE_FORMATS = "dynamic_date_formats"; private final ElasticsearchConverter elasticsearchConverter; @@ -147,11 +150,24 @@ private void mapEntity(XContentBuilder builder, @Nullable ElasticsearchPersisten @Nullable Field parentFieldAnnotation, @Nullable DynamicMapping dynamicMapping) throws IOException { if (entity != null && entity.isAnnotationPresent(Mapping.class)) { + Mapping mappingAnnotation = entity.getRequiredAnnotation(Mapping.class); - if (!entity.getRequiredAnnotation(Mapping.class).enabled()) { + if (!mappingAnnotation.enabled()) { builder.field(MAPPING_ENABLED, false); return; } + + if (mappingAnnotation.dateDetection() != Mapping.Detection.DEFAULT) { + builder.field(DATE_DETECTION, Boolean.parseBoolean(mappingAnnotation.dateDetection().name())); + } + + if (mappingAnnotation.numericDetection() != Mapping.Detection.DEFAULT) { + builder.field(NUMERIC_DETECTION, Boolean.parseBoolean(mappingAnnotation.numericDetection().name())); + } + + if (mappingAnnotation.dynamicDateFormats().length > 0) { + builder.field(DYNAMIC_DATE_FORMATS, mappingAnnotation.dynamicDateFormats()); + } } boolean writeNestedProperties = !isRootObject && (isAnyPropertyAnnotatedWithField(entity) || nestedOrObjectField); diff --git a/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderIntegrationTests.java b/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderIntegrationTests.java index e03e9d395..de0e9e4bb 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderIntegrationTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderIntegrationTests.java @@ -306,6 +306,17 @@ void shouldWriteDynamicMappingEntries() { indexOps.delete(); } + @Test // #638 + @DisplayName("should write dynamic detection values") + void shouldWriteDynamicDetectionValues() { + + IndexOperations indexOps = operations.indexOps(DynamicDetectionMapping.class); + indexOps.create(); + indexOps.putMapping(); + indexOps.delete(); + } + + // region entities @Document(indexName = "ignore-above-index") static class IgnoreAboveEntity { @Nullable @Id private String id; @@ -1113,4 +1124,12 @@ public void setAuthor(Author author) { } } + @Document(indexName = "dynamic-detection-mapping-true") + @Mapping(dateDetection = Mapping.Detection.TRUE, numericDetection = Mapping.Detection.TRUE, + dynamicDateFormats = { "MM/dd/yyyy" }) + private static class DynamicDetectionMapping { + @Id @Nullable private String id; + } + // endregion + } diff --git a/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderUnitTests.java b/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderUnitTests.java index 39b61e899..7fbaca7ff 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderUnitTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderUnitTests.java @@ -774,6 +774,87 @@ void shouldNotWriteTypeHintsWhenContextIsConfiguredToDoSoButEntityShouldNot() th assertEquals(expected, mapping, true); } + @Test // #638 + @DisplayName("should not write dynamic detection mapping entries in default setting") + void shouldNotWriteDynamicDetectionMappingEntriesInDefaultSetting() throws JSONException { + + String expected = "{\n" + // + " \"properties\": {\n" + // + " \"_class\": {\n" + // + " \"type\": \"keyword\",\n" + // + " \"index\": false,\n" + // + " \"doc_values\": false\n" + // + " }\n" + // + " }\n" + // + "}"; // + + String mapping = getMappingBuilder().buildPropertyMapping(DynamicDetectionMappingDefault.class); + + assertEquals(expected, mapping, true); + } + + @Test // #638 + @DisplayName("should write dynamic detection mapping entries when set to false") + void shouldWriteDynamicDetectionMappingEntriesWhenSetToFalse() throws JSONException { + + String expected = "{\n" + // + " \"date_detection\": false," + // + " \"numeric_detection\": false," + // + " \"properties\": {\n" + // + " \"_class\": {\n" + // + " \"type\": \"keyword\",\n" + // + " \"index\": false,\n" + // + " \"doc_values\": false\n" + // + " }\n" + // + " }\n" + // + "}"; // + + String mapping = getMappingBuilder().buildPropertyMapping(DynamicDetectionMappingFalse.class); + + assertEquals(expected, mapping, true); + } + + @Test // #638 + @DisplayName("should write dynamic detection mapping entries when set to true") + void shouldWriteDynamicDetectionMappingEntriesWhenSetToTrue() throws JSONException { + + String expected = "{\n" + // + " \"date_detection\": true," + // + " \"numeric_detection\": true," + // + " \"properties\": {\n" + // + " \"_class\": {\n" + // + " \"type\": \"keyword\",\n" + // + " \"index\": false,\n" + // + " \"doc_values\": false\n" + // + " }\n" + // + " }\n" + // + "}"; // + + String mapping = getMappingBuilder().buildPropertyMapping(DynamicDetectionMappingTrue.class); + + assertEquals(expected, mapping, true); + } + + @Test // #638 + @DisplayName("should write dynamic date formats") + void shouldWriteDynamicDateFormats() throws JSONException { + + String expected = "{\n" + // + " \"dynamic_date_formats\": [\"date1\",\"date2\"]," + // + " \"properties\": {\n" + // + " \"_class\": {\n" + // + " \"type\": \"keyword\",\n" + // + " \"index\": false,\n" + // + " \"doc_values\": false\n" + // + " }\n" + // + " }\n" + // + "}"; // + + String mapping = getMappingBuilder().buildPropertyMapping(DynamicDateFormatsMapping.class); + + assertEquals(expected, mapping, true); + } + // region entities @Document(indexName = "ignore-above-index") static class IgnoreAboveEntity { @@ -1690,5 +1771,28 @@ private static class MagazineWithTypeHints { @Field(type = Text) @Nullable private String title; @Field(type = Nested) @Nullable private List authors; } + + @Document(indexName = "dynamic-field-mapping-default") + private static class DynamicDetectionMappingDefault { + @Id @Nullable private String id; + } + + @Document(indexName = "dynamic-dateformats-mapping") + @Mapping(dynamicDateFormats = {"date1", "date2"}) + private static class DynamicDateFormatsMapping { + @Id @Nullable private String id; + } + + @Document(indexName = "dynamic-detection-mapping-true") + @Mapping(dateDetection = Mapping.Detection.TRUE, numericDetection = Mapping.Detection.TRUE) + private static class DynamicDetectionMappingTrue { + @Id @Nullable private String id; + } + + @Document(indexName = "dynamic-detection-mapping-false") + @Mapping(dateDetection = Mapping.Detection.FALSE, numericDetection = Mapping.Detection.FALSE) + private static class DynamicDetectionMappingFalse { + @Id @Nullable private String id; + } // endregion }