[Backport 8.4] Add documentation for plugin-defined custom variants

docs/api-conventions/variant-types.asciidoc

@@ -57,4 +57,50 @@ include-tagged::{doc-tests-src}/api_conventions/ApiConventionsTest.java[variant-
 <2> Test a larger set of variant kinds.
 <3> Get the kind and value held by the variant object.
 
+[discrete]
+[[variant-types-custom]]
+==== Custom extensions provided by {es} plugins
+
+{es} accepts plugins that can extend the available variants for a number of types. This includes queries, aggregations, text analyzers and tokenizers, ingest processors, etc.
+
+The {java-client} classes for these types accept a `_custom` variant in addition to the builtin ones. This allows you to use these plugin-defined extensions by providing arbitrary JSON in requests, and also receive arbitrary JSON produced by the plugins in responses.
+
+In the examples below we use a hypothetical plugin that adds a `sphere-distance` aggregation that groups documents containing 3D coordinates according to their distance to a reference location.
+
+To create a custom aggregation, use the `_custom()` aggregation type and provide its identifier, defined by the plugin, and parameters. The parameters can be any object or value that can be serialized to JSON. In the example below we use a simple map:
+
+["source","java"]
+--------------------------------------------------
+include-tagged::{doc-tests-src}/api_conventions/ApiConventionsTest.java[custom-variant-creation]
+--------------------------------------------------
+<1> Parameters for the custom aggregation.
+<2> Create a custom aggregation named `neighbors` of kind `sphere-distance` with its parameters.
+
+The results of custom variants are returned as raw JSON represented by a `JsonData` object. You can then traverse the JSON tree to get the data. Since this is not always convenient, you can also define classes that represent that JSON data and deserialize them from the raw JSON.
+
+Traversing the JSON tree:
+
+["source","java"]
+--------------------------------------------------
+include-tagged::{doc-tests-src}/api_conventions/ApiConventionsTest.java[custom-variant-navigation-json]
+--------------------------------------------------
+<1> Use `Void` if you're only interested in aggregation results, not search hits (see also <<aggregations>>).
+<2> Get the `neighbors` aggregation result as custom JSON result.
+<3> Traverse the JSON tree to extract the result data.
+
+Using a class that represents the custom aggregation results:
+
+["source","java"]
+--------------------------------------------------
+include-tagged::{doc-tests-src}/api_conventions/ApiConventionsTest.java[custom-variant-navigation-typed]
+--------------------------------------------------
+<1> Deserialize the custom JSON to a dedicated `SphereDistanceAggregate` class.
+
+Where `SphereDistanceAggregate` can be defined as follows:
+["source","java"]
+--------------------------------------------------
+include-tagged::{doc-tests-src}/api_conventions/ApiConventionsTest.java[custom-variant-types]
+--------------------------------------------------
+
+
 {doc-tests-blurb}

java-client/src/test/java/co/elastic/clients/documentation/DocTestsTransport.java

@@ -24,6 +24,8 @@
 import co.elastic.clients.transport.ElasticsearchTransport;
 import co.elastic.clients.transport.Endpoint;
 import co.elastic.clients.transport.TransportOptions;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.PropertyNamingStrategies;
 
 import javax.annotation.Nullable;
 import java.io.IOException;
@@ -40,7 +42,9 @@
  */
 public class DocTestsTransport implements ElasticsearchTransport {
 
-    private final JsonpMapper mapper = new JacksonJsonpMapper();
+    private final JsonpMapper mapper = new JacksonJsonpMapper(
+        new ObjectMapper().setPropertyNamingStrategy(PropertyNamingStrategies.SNAKE_CASE)
+    );
 
     private final ThreadLocal<Object> result = new ThreadLocal<>();
 

java-client/src/test/java/co/elastic/clients/documentation/api_conventions/ApiConventionsTest.java

@@ -31,13 +31,21 @@
 import co.elastic.clients.elasticsearch.indices.Alias;
 import co.elastic.clients.elasticsearch.indices.CreateIndexRequest;
 import co.elastic.clients.elasticsearch.indices.CreateIndexResponse;
-import co.elastic.clients.transport.ElasticsearchTransport;
+import co.elastic.clients.json.JsonData;
 import co.elastic.clients.util.ApiTypeHelper;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.PropertyNamingStrategies;
+import jakarta.json.JsonArray;
+import jakarta.json.JsonObject;
+import jakarta.json.JsonValue;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.StringReader;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
@@ -47,7 +55,7 @@ public class ApiConventionsTest extends Assertions {
 
     private static class SomeApplicationData {}
 
-    private ElasticsearchTransport transport = new DocTestsTransport();
+    private DocTestsTransport transport = new DocTestsTransport();
     Logger logger = LoggerFactory.getLogger(this.getClass());
 
     public void blockingAndAsync() throws Exception {
@@ -121,6 +129,8 @@ public void builderLambdasShort() throws Exception {
     }
 
     public void builderIntervals() throws Exception {
+        ObjectMapper mapper = new ObjectMapper();
+        mapper.setPropertyNamingStrategy(PropertyNamingStrategies.SNAKE_CASE);
         ElasticsearchClient client = new ElasticsearchClient(transport);
 
         //tag::builder-intervals
@@ -193,6 +203,104 @@ public void variantCreation() {
         //end::variant-kind
     }
 
+    //tag::custom-variant-types
+    public static class SphereDistanceAggregate {
+        private final List<Bucket> buckets;
+        @JsonCreator
+        public SphereDistanceAggregate(
+            @JsonProperty("buckets") List<Bucket> buckets
+        ) {
+            this.buckets = buckets;
+        }
+        public List<Bucket> buckets() {
+            return buckets;
+        };
+    }
+
+    public static class Bucket {
+        private final double key;
+        private final double docCount;
+        @JsonCreator
+        public Bucket(
+            @JsonProperty("key") double key,
+            @JsonProperty("doc_count") double docCount) {
+            this.key = key;
+            this.docCount = docCount;
+        }
+        public double key() {
+            return key;
+        }
+        public double docCount() {
+            return docCount;
+        }
+    }
+    //end::custom-variant-types
+
+    @Test
+    public void customVariants() throws Exception {
+
+        ElasticsearchClient esClient = new ElasticsearchClient(transport);
+
+        String json = "{\"took\":1,\"timed_out\":false,\"_shards\":{\"failed\":0.0,\"successful\":1.0,\"total\":1.0},\n" +
+            "\"hits\":{\"total\":{\"relation\":\"eq\",\"value\":0},\"hits\":[]},\n" +
+            "\"aggregations\":{\"sphere-distance#neighbors\":{\"buckets\":[{\"key\": 1.0,\"doc_count\":1}]}}}";
+
+        transport.setResult(SearchResponse.of(b -> b.withJson(
+            transport.jsonpMapper().jsonProvider().createParser(new StringReader(json)),
+            transport.jsonpMapper())
+        ));
+
+        //tag::custom-variant-creation
+        Map<String, Object> params = new HashMap<>(); // <1>
+        params.put("interval", 10);
+        params.put("scale", "log");
+        params.put("origin", new Double[]{145.0, 12.5, 1649.0});
+
+        SearchRequest request = SearchRequest.of(r -> r
+            .index("stars")
+            .aggregations("neighbors", agg -> agg
+                ._custom("sphere-distance", params) // <2>
+            )
+        );
+        //end::custom-variant-creation
+
+        {
+            //tag::custom-variant-navigation-json
+            SearchResponse<Void> response = esClient.search(request, Void.class); // <1>
+
+            JsonData neighbors = response
+                .aggregations().get("neighbors")
+                ._custom(); // <2>
+
+            JsonArray buckets = neighbors.toJson() // <3>
+                .asJsonObject()
+                .getJsonArray("buckets");
+
+            for (JsonValue item : buckets) {
+                JsonObject bucket = item.asJsonObject();
+                double key = bucket.getJsonNumber("key").doubleValue();
+                double docCount = bucket.getJsonNumber("doc_count").longValue();
+                doSomething(key, docCount);
+            }
+            //end::custom-variant-navigation-json
+        }
+
+        {
+            //tag::custom-variant-navigation-typed
+            SearchResponse<Void> response = esClient.search(request, Void.class);
+
+            SphereDistanceAggregate neighbors = response
+                .aggregations().get("neighbors")
+                ._custom()
+                .to(SphereDistanceAggregate.class); // <1>
+
+            for (Bucket bucket : neighbors.buckets()) {
+                doSomething(bucket.key(), bucket.docCount());
+            }
+            //end::custom-variant-navigation-typed
+        }
+    }
+
     @Test
     public void collections() {
         //tag::collections-list