Skip to content

datatype detection support in mapping. #1810

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions src/main/asciidoc/reference/elasticsearch-misc.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,16 @@ class Entity {
<.> `sortModes`, `sortOrders` and `sortMissingValues` are optional, but if they are set, the number of entries must match the number of `sortFields` elements
====

[[elasticsearch.misc.mappings]]
== Index Mapping

When Spring Data Elasticsearch creates the index mapping with the `IndexOperations.createMapping()` methods, it uses the annotations described in <<elasticsearch.mapping.meta-model.annotations>>, especially the `@Field` annotation. In addition to that it is possible to add the `@Mapping` annotation to a class. This annotation has the following properties:

* `mappingPath` a classpath resource in JSON format which is used as the mapping, no other mapping processing is done.
* `enabled` when set to false, this flag is written to the mapping and no further processing is done.
* `dateDetection` and `numericDetection` set the corresponding properties in the mapping when not set to `DEFAULT`.
* `dynamicDateFormats` when this String array is not empty, it defines the date formats used for automatic date detection.

[[elasticsearch.misc.filter]]
== Filter Builder

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,19 +9,17 @@
import org.springframework.data.annotation.Persistent;

/**
* Elasticsearch dynamic templates mapping.
* This annotation is handy if you prefer apply dynamic templates on fields with annotation e.g. {@link Field}
* with type = FieldType.Object etc. instead of static mapping on Document via {@link Mapping} annotation.
* DynamicTemplates annotation is omitted if {@link Mapping} annotation is used.
* Elasticsearch dynamic templates mapping. This annotation is handy if you prefer apply dynamic templates on fields
* with annotation e.g. {@link Field} with type = FieldType.Object etc. instead of static mapping on Document via
* {@link Mapping} annotation. DynamicTemplates annotation is omitted if {@link Mapping} annotation is used.
*
* @author Petr Kukral
*/
@Persistent
@Inherited
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.TYPE})
@Target({ ElementType.TYPE })
public @interface DynamicTemplates {

String mappingPath() default "";

}
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,31 @@

/**
* whether mappings are enabled
*
*
* @since 4.2
*/
boolean enabled() default true;
/**
* whether date_detection is enabled
*
* @since 4.3
*/
Detection dateDetection() default Detection.DEFAULT;

/**
* whether numeric_detection is enabled
*
* @since 4.3
*/
Detection numericDetection() default Detection.DEFAULT;

/**
* custom dynamic date formats
* @since 4.3
*/
String[] dynamicDateFormats() default {};

enum Detection {
DEFAULT, TRUE, FALSE;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,9 @@ public class MappingBuilder {
private static final String JOIN_TYPE_RELATIONS = "relations";

private static final String MAPPING_ENABLED = "enabled";
private static final String DATE_DETECTION = "date_detection";
private static final String NUMERIC_DETECTION = "numeric_detection";
private static final String DYNAMIC_DATE_FORMATS = "dynamic_date_formats";

private final ElasticsearchConverter elasticsearchConverter;

Expand Down Expand Up @@ -147,11 +150,24 @@ private void mapEntity(XContentBuilder builder, @Nullable ElasticsearchPersisten
@Nullable Field parentFieldAnnotation, @Nullable DynamicMapping dynamicMapping) throws IOException {

if (entity != null && entity.isAnnotationPresent(Mapping.class)) {
Mapping mappingAnnotation = entity.getRequiredAnnotation(Mapping.class);

if (!entity.getRequiredAnnotation(Mapping.class).enabled()) {
if (!mappingAnnotation.enabled()) {
builder.field(MAPPING_ENABLED, false);
return;
}

if (mappingAnnotation.dateDetection() != Mapping.Detection.DEFAULT) {
builder.field(DATE_DETECTION, Boolean.parseBoolean(mappingAnnotation.dateDetection().name()));
}

if (mappingAnnotation.numericDetection() != Mapping.Detection.DEFAULT) {
builder.field(NUMERIC_DETECTION, Boolean.parseBoolean(mappingAnnotation.numericDetection().name()));
}

if (mappingAnnotation.dynamicDateFormats().length > 0) {
builder.field(DYNAMIC_DATE_FORMATS, mappingAnnotation.dynamicDateFormats());
}
}

boolean writeNestedProperties = !isRootObject && (isAnyPropertyAnnotatedWithField(entity) || nestedOrObjectField);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -306,6 +306,17 @@ void shouldWriteDynamicMappingEntries() {
indexOps.delete();
}

@Test // #638
@DisplayName("should write dynamic detection values")
void shouldWriteDynamicDetectionValues() {

IndexOperations indexOps = operations.indexOps(DynamicDetectionMapping.class);
indexOps.create();
indexOps.putMapping();
indexOps.delete();
}

// region entities
@Document(indexName = "ignore-above-index")
static class IgnoreAboveEntity {
@Nullable @Id private String id;
Expand Down Expand Up @@ -1113,4 +1124,12 @@ public void setAuthor(Author author) {
}
}

@Document(indexName = "dynamic-detection-mapping-true")
@Mapping(dateDetection = Mapping.Detection.TRUE, numericDetection = Mapping.Detection.TRUE,
dynamicDateFormats = { "MM/dd/yyyy" })
private static class DynamicDetectionMapping {
@Id @Nullable private String id;
}
// endregion

}
Original file line number Diff line number Diff line change
Expand Up @@ -774,6 +774,87 @@ void shouldNotWriteTypeHintsWhenContextIsConfiguredToDoSoButEntityShouldNot() th
assertEquals(expected, mapping, true);
}

@Test // #638
@DisplayName("should not write dynamic detection mapping entries in default setting")
void shouldNotWriteDynamicDetectionMappingEntriesInDefaultSetting() throws JSONException {

String expected = "{\n" + //
" \"properties\": {\n" + //
" \"_class\": {\n" + //
" \"type\": \"keyword\",\n" + //
" \"index\": false,\n" + //
" \"doc_values\": false\n" + //
" }\n" + //
" }\n" + //
"}"; //

String mapping = getMappingBuilder().buildPropertyMapping(DynamicDetectionMappingDefault.class);

assertEquals(expected, mapping, true);
}

@Test // #638
@DisplayName("should write dynamic detection mapping entries when set to false")
void shouldWriteDynamicDetectionMappingEntriesWhenSetToFalse() throws JSONException {

String expected = "{\n" + //
" \"date_detection\": false," + //
" \"numeric_detection\": false," + //
" \"properties\": {\n" + //
" \"_class\": {\n" + //
" \"type\": \"keyword\",\n" + //
" \"index\": false,\n" + //
" \"doc_values\": false\n" + //
" }\n" + //
" }\n" + //
"}"; //

String mapping = getMappingBuilder().buildPropertyMapping(DynamicDetectionMappingFalse.class);

assertEquals(expected, mapping, true);
}

@Test // #638
@DisplayName("should write dynamic detection mapping entries when set to true")
void shouldWriteDynamicDetectionMappingEntriesWhenSetToTrue() throws JSONException {

String expected = "{\n" + //
" \"date_detection\": true," + //
" \"numeric_detection\": true," + //
" \"properties\": {\n" + //
" \"_class\": {\n" + //
" \"type\": \"keyword\",\n" + //
" \"index\": false,\n" + //
" \"doc_values\": false\n" + //
" }\n" + //
" }\n" + //
"}"; //

String mapping = getMappingBuilder().buildPropertyMapping(DynamicDetectionMappingTrue.class);

assertEquals(expected, mapping, true);
}

@Test // #638
@DisplayName("should write dynamic date formats")
void shouldWriteDynamicDateFormats() throws JSONException {

String expected = "{\n" + //
" \"dynamic_date_formats\": [\"date1\",\"date2\"]," + //
" \"properties\": {\n" + //
" \"_class\": {\n" + //
" \"type\": \"keyword\",\n" + //
" \"index\": false,\n" + //
" \"doc_values\": false\n" + //
" }\n" + //
" }\n" + //
"}"; //

String mapping = getMappingBuilder().buildPropertyMapping(DynamicDateFormatsMapping.class);

assertEquals(expected, mapping, true);
}

// region entities
@Document(indexName = "ignore-above-index")
static class IgnoreAboveEntity {
Expand Down Expand Up @@ -1690,5 +1771,28 @@ private static class MagazineWithTypeHints {
@Field(type = Text) @Nullable private String title;
@Field(type = Nested) @Nullable private List<Author> authors;
}

@Document(indexName = "dynamic-field-mapping-default")
private static class DynamicDetectionMappingDefault {
@Id @Nullable private String id;
}

@Document(indexName = "dynamic-dateformats-mapping")
@Mapping(dynamicDateFormats = {"date1", "date2"})
private static class DynamicDateFormatsMapping {
@Id @Nullable private String id;
}

@Document(indexName = "dynamic-detection-mapping-true")
@Mapping(dateDetection = Mapping.Detection.TRUE, numericDetection = Mapping.Detection.TRUE)
private static class DynamicDetectionMappingTrue {
@Id @Nullable private String id;
}

@Document(indexName = "dynamic-detection-mapping-false")
@Mapping(dateDetection = Mapping.Detection.FALSE, numericDetection = Mapping.Detection.FALSE)
private static class DynamicDetectionMappingFalse {
@Id @Nullable private String id;
}
// endregion
}