Add Creating a custom Serializer section

Author: stevejgordon

docs/client-concepts/serialization/custom-serialization.asciidoc

@@ -110,13 +110,14 @@ When serializing this class, rather than include a string value representing the
 include::{doc-tests-src}/ClientConcepts/Serialization/CustomSerializationTests.cs[tag=converter-usings]
 include::{doc-tests-src}/ClientConcepts/Serialization/CustomSerializationTests.cs[tag=customer-converter]
 ----
-<1> When reading, this converter will read the `isStandard` boolean and translate this to the correct `CustomerType` enum value.
-<2> When writing, this converter will translate the `CustomerType` enum value to an `isStandard` boolean property.
+<1> When reading, this converter reads the `isStandard` boolean and translate this to the correct `CustomerType` enum value.
+<2> When writing, this converter translates the `CustomerType` enum value to an `isStandard` boolean property.
 
 We can then index a customer document into {es}.
 
 [source,csharp]
 ----
+include::{doc-tests-src}/ClientConcepts/Serialization/CustomSerializationTests.cs[tag=usings]
 include::{doc-tests-src}/ClientConcepts/Serialization/CustomSerializationTests.cs[tag=index-customer-with-converter]
 ----
 
@@ -130,9 +131,100 @@ The `Customer` instance is serialized using the custom converter, creating the f
 }
 ----
 
-[[injecting-custom-serializer]]
-===== Injecting a custom serializer
+[[creating-custom-system-text-json-serializer]]
+===== Creating a custom `SystemTextJsonSerializer`
+
+The built-in `DefaultSourceSerializer` includes the registration of `JsonConverter` instances which apply during source serialization. In most cases, these provide the proper behavior for serializing source documents, including those which use `Elastic.Clients.Elasticsearch` types on their properties.
+
+An example of a situation where you may require more control over the converter registration order is for serializing `enum` types. The `DefaultSourceSerializer` registers the `System.Text.Json.Serialization.JsonStringEnumConverter`, so enum values are serialized using their string representation. Generally, this is the preferred option for types used to index documents to {es}.
+
+In some scenarios, you may need to control the string value sent for an enumeration value. That is not directly supported in `System.Text.Json` but can be achieved by creating a custom `JsonConverter` for the `enum` type you wish to customize. In this situation, it is not sufficient to use the `JsonConverterAttribute` on the `enum` type to register the converter. `System.Text.Json` will prefer the converters added to the `Converters` collection on the `JsonSerializerOptions` over an attribute applied to an `enum` type. It is, therefore, necessary to either remove the `JsonStringEnumConverter` from the `Converters` collection or register a specialized converter for your `enum` type before the `JsonStringEnumConverter`.
+
+The latter is possible via several techniques. When using the {es} .NET library, we can achieve this by deriving from the abstract `SystemTextJsonSerializer` class.
+
+Here we have a POCO which uses the `CustomerType` enum as the type for a property.
+
+[source,csharp]
+----
+include::{doc-tests-src}/ClientConcepts/Serialization/CustomSerializationTests.cs[tag=usings-serialization]
+include::{doc-tests-src}/ClientConcepts/Serialization/CustomSerializationTests.cs[tag=customer-without-jsonconverter-attribute]
+----
+
+To customize the strings that are used during serialization of the `CustomerType` we defined a custom `JsonConverter` specific to our `enum` type.
+
+[source,csharp]
+----
+include::{doc-tests-src}/ClientConcepts/Serialization/CustomSerializationTests.cs[tag=usings-serialization]
+include::{doc-tests-src}/ClientConcepts/Serialization/CustomSerializationTests.cs[tag=customer-type-converter]
+----
+<1> When reading, this converter translates the string used in the JSON to the matching enum value.
+<2> When writing, this converter translates the `CustomerType` enum value to an custom string value written to the JSON.
+
+We can create a serializer which derives from `SystemTextJsonSerializer` to give us full control of converter registration.
+
+[source,csharp]
+----
+include::{doc-tests-src}/ClientConcepts/Serialization/CustomSerializationTests.cs[tag=derived-converter-usings]
+include::{doc-tests-src}/ClientConcepts/Serialization/CustomSerializationTests.cs[tag=my-custom-serializer]
+----
+<1> Inherit from `SystemTextJsonSerializer`.
+<2> In the constructor, use the factory method `DefaultSourceSerializer.CreateDefaultJsonSerializerOptions` to create default options for serialization. By passing `false`, no default converters are registered at this stage.
+<3> Register our `CustomerTypeConverter` as the first converter.
+<4> To apply any default converters call the `DefaultSourceSerializer.AddDefaultConverters` helper method, passing the options to modify.
+<5> Implement the `CreateJsonSerializerOptions` method returning the stored `JsonSerializerOptions`.
+
+Because we have registered our `CustomerTypeConverter` before the default converters (which include the `JsonStringEnumConverter`), our converter takes precedence when serializing `CustomerType` instances on source documents.
+
+The base `SystemTextJsonSerializer` class handles the implementation details of binding, required to ensure that the built-in converters can access the `IElasticsearchClientSettings` where needed.
+
+We can then index a customer document into {es}.
+
+[source,csharp]
+----
+include::{doc-tests-src}/ClientConcepts/Serialization/CustomSerializationTests.cs[tag=usings]
+include::{doc-tests-src}/ClientConcepts/Serialization/CustomSerializationTests.cs[tag=index-customer-without-jsonconverter-attribute]
+----
+
+The `Customer` instance is serialized using the custom `enum` converter, creating the following JSON document.
+
+[source,javascript]
+----
+{
+  "customerName": "Customer Ltd",
+  "customerType": "premium" // <1>
+}
+----
+<1> The string value applied during serialization is provided by our custom converter.
+
+[[creating-custom-serializers]]
+===== Creating a custom `Serializer`
+
+If you prefer to use an alternative JSON serialization library for your source types, you can inject a serializer that is isolated to only be called for the serialization of `_source`, `_fields`, or wherever a user provided value is expected to be written and returned.
+
+Implementing `Elastic.Transport.Serializer` is technically enough to inject your own source serializer.
+
+[source,csharp]
+----
+include::{doc-tests-src}/ClientConcepts/Serialization/CustomSerializationTests.cs[tag=vanilla-serializer-using-directives]
+include::{doc-tests-src}/ClientConcepts/Serialization/CustomSerializationTests.cs[tag=vanilla-serializer]
+----
+
+Registering up the serializer is performed in the `ConnectionSettings` constructor
+
+[source,csharp]
+----
+include::{doc-tests-src}/ClientConcepts/Serialization/CustomSerializationTests.cs[tag=usings]
+include::{doc-tests-src}/ClientConcepts/Serialization/CustomSerializationTests.cs[tag=register-vanilla-serializer]
+----
+<1> If implementing `Serializer` is enough, why do we need to provide an instance wrapped in a factory `Func`?
+
+There are various cases where you might have a POCO type that contains an `Elastic.Clients.Elasticsearch` type as one of its properties. The `SourceSerializerFactory` delegate provides access to the default built-in serializer so that you can access it when necessary. For example, consider if you want to use percolation; you need to store {es} queries as part of the `_source` of your document, which means you need to have a POCO that looks  like this.
+
+[source,csharp]
+----
+include::{doc-tests-src}/ClientConcepts/Serialization/CustomSerializationTests.cs[tag=querydsl-using-directives]
+include::{doc-tests-src}/ClientConcepts/Serialization/CustomSerializationTests.cs[tag=percolation-document]
+----
 
-TODO
-- Deriving from SystemTextJsonSerializer for enum Converter
-- Deriving from Serializer for different library
+A custom serializer would not know how to serialize `Query` or other `Elastic.Clients.Elasticsearch` types that could appear as part of
+the `_source` of a document. Therefore, your custom `Serializer` would need to store a reference to our built-in serializer, and  delegate serialization of Elastic types back to it.

src/Elastic.Clients.Elasticsearch/Serialization/DefaultSourceSerializer.cs

@@ -5,7 +5,6 @@
 using System;
 using System.Text.Json;
 using System.Text.Json.Serialization;
-using Elastic.Clients.Elasticsearch.QueryDsl;
 
 namespace Elastic.Clients.Elasticsearch.Serialization;
 
@@ -21,8 +20,7 @@ public class DefaultSourceSerializer : SystemTextJsonSerializer
 	/// </summary>
 	public static JsonConverter[] DefaultBuiltInConverters => new JsonConverter[]
 	{
-		new JsonStringEnumConverter(),
-		new QueryConverter()
+		new JsonStringEnumConverter()
 	};
 
 	private readonly JsonSerializerOptions _jsonSerializerOptions;

tests/Tests/Documentation/ClientConcepts/Serialization/CustomSerializationTests.cs

@@ -2,13 +2,16 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information.
 
+// **********************************
 // IMPORTANT: These tests have a secondary use as code snippets used in documentation.
 // We disable formatting in sections of this file to ensure the correct indentation when tagged regions are
 // included in the asciidocs. While hard to read, this formatting should be left as-is for docs generation.
 // We also include using directives that are not required due to global using directives, but remain here
 // so that can appear in the documentation.
+// **********************************
 
-#pragma warning disable IDE0005
+#pragma warning disable CS0105 // Using directive appeared previously in this namespace
+#pragma warning disable IDE0005 // Remove unnecessary using directives
 //tag::usings[]
 //tag::converter-usings[]
 using System;
@@ -22,10 +25,26 @@
 using Elastic.Clients.Elasticsearch;
 using Elastic.Clients.Elasticsearch.Serialization;
 //end::usings[]
+//tag::derived-converter-usings[]
+using System.Text.Json;
+using Elastic.Clients.Elasticsearch.Serialization;
+//end::derived-converter-usings[]
+//tag::vanilla-serializer-using-directives[]
+using System;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+using Elastic.Transport;
+//end::vanilla-serializer-using-directives[]
 using System.Text;
 using VerifyXunit;
 using System.IO;
-#pragma warning restore IDE0005
+using System.Threading;
+//tag::querydsl-using-directives[]
+using Elastic.Clients.Elasticsearch.QueryDsl;
+//end::querydsl-using-directives[]
+#pragma warning restore IDE0005 // Remove unnecessary using directives
+#pragma warning restore CS0105 // Using directive appeared previously in this namespace
 
 namespace Tests.Documentation.Serialization;
 
@@ -67,7 +86,8 @@ static void ConfigureOptions(JsonSerializerOptions o) => // <1>
     #pragma warning disable format
     //tag::index-person[]
     var person = new Person { FirstName = "Steve" };
-    var indexResponse = await client.IndexAsync(person, "my-index-name"); //end::index-person[]
+    var indexResponse = await client.IndexAsync(person, "my-index-name");
+    //end::index-person[]
     #pragma warning restore format
 
         var requestJson = Encoding.UTF8.GetString(indexResponse.ApiCallDetails.RequestBodyInBytes);
@@ -297,77 +317,143 @@ public async Task DerivingFromSystemTextJsonSerializer_ToRegisterACustomEnumConv
             .DisableDirectStreaming();
         client = new ElasticsearchClient(settings);
 
-        var customer = new Customer
-        {
-            CustomerName = "Customer Ltd",
-            Type = CustomerType.Enhanced
-        };
+#pragma warning disable format
+//tag::index-customer-without-jsonconverter-attribute[]
+var customer = new Customer
+{
+    CustomerName = "Customer Ltd",
+    CustomerType = CustomerType.Enhanced
+};
 
-        var indexResponse = await client.IndexAsync(customer, "my-index-name");
+var indexResponse = await client.IndexAsync(customer, "my-index-name");
+//end::index-customer-without-jsonconverter-attribute[]
+#pragma warning restore format
 
         var requestJson = Encoding.UTF8.GetString(indexResponse.ApiCallDetails.RequestBodyInBytes);
         await Verifier.Verify(requestJson);
 
         var ms = new MemoryStream(indexResponse.ApiCallDetails.RequestBodyInBytes);
         var deserializedCustomer = client.SourceSerializer.Deserialize<Customer>(ms);
         deserializedCustomer.CustomerName.Should().Be("Customer Ltd");
-        deserializedCustomer.Type.Should().Be(CustomerType.Enhanced);
+        deserializedCustomer.CustomerType.Should().Be(CustomerType.Enhanced);
     }
 
-    public class Customer
-    {
-        public string CustomerName { get; set; }
-        public CustomerType Type { get; set; }
-    }
+#pragma warning disable format
+//tag::customer-without-jsonconverter-attribute[]
+public class Customer
+{
+    public string CustomerName { get; set; }
+    public CustomerType CustomerType { get; set; }
+}
 
-    public enum CustomerType
-    {
-        Standard,
-        Enhanced
-    }
+public enum CustomerType
+{
+    Standard,
+    Enhanced
+}
+//end::customer-without-jsonconverter-attribute[]
+#pragma warning restore format
+
+#pragma warning disable format
+//tag::my-custom-serializer[]
+public class MyCustomSerializer : SystemTextJsonSerializer // <1>
+{
+    private readonly JsonSerializerOptions _options;
 
-    public class MyCustomSerializer : SystemTextJsonSerializer
+    public MyCustomSerializer(IElasticsearchClientSettings settings) : base(settings)
     {
-        private readonly JsonSerializerOptions _options;
+        var options = DefaultSourceSerializer.CreateDefaultJsonSerializerOptions(false); // <2>
 
-        public MyCustomSerializer(IElasticsearchClientSettings settings) : base(settings)
-        {
-            var options = DefaultSourceSerializer.CreateDefaultJsonSerializerOptions(false);
+        options.Converters.Add(new CustomerTypeConverter()); // <3>
 
-            options.Converters.Add(new CustomerTypeConverter());
+        _options = DefaultSourceSerializer.AddDefaultConverters(options); // <4>
+    }
 
-            _options = DefaultSourceSerializer.AddDefaultConverters(options);
-        }
+    protected override JsonSerializerOptions CreateJsonSerializerOptions() => _options; // <5>
+}
+//end::my-custom-serializer[]
+#pragma warning restore format
 
-        protected override JsonSerializerOptions CreateJsonSerializerOptions() => _options;
+#pragma warning disable format
+//tag::customer-type-converter[]
+public class CustomerTypeConverter : JsonConverter<CustomerType>
+{
+    public override CustomerType Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+    {
+        return reader.GetString() switch // <1>
+        {
+            "basic" => CustomerType.Standard,
+            "premium" => CustomerType.Enhanced,
+            _ => throw new JsonException(
+                $"Unknown value read when deserializing {nameof(CustomerType)}."),
+        };
     }
 
-    public class CustomerTypeConverter : JsonConverter<CustomerType>
+    public override void Write(Utf8JsonWriter writer, CustomerType value, JsonSerializerOptions options)
     {
-        public override CustomerType Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+        switch (value) // <2>
         {
-            return reader.GetString() switch
-            {
-                "basic" => CustomerType.Standard,
-                "premium" => CustomerType.Enhanced,
-                _ => throw new JsonException(
-                    $"Unknown value read when deserializing {nameof(CustomerType)}."),
-            };
+            case CustomerType.Standard:
+                writer.WriteStringValue("basic");
+                return;
+            case CustomerType.Enhanced:
+                writer.WriteStringValue("premium");
+                return;
         }
 
-        public override void Write(Utf8JsonWriter writer, CustomerType value, JsonSerializerOptions options)
-        {
-            switch (value)
-            {
-                case CustomerType.Standard:
-                    writer.WriteStringValue("basic");
-                    return;
-                case CustomerType.Enhanced:
-                    writer.WriteStringValue("premium");
-                    return;
-            }
-
-            writer.WriteNullValue();
-        }
+        writer.WriteNullValue();
     }
 }
+//end::customer-type-converter[]
+#pragma warning restore format
+
+public void RegisterVanillaSerializer()
+{
+#pragma warning disable format
+//tag::register-vanilla-serializer[]
+var nodePool = new SingleNodePool(new Uri("http://localhost:9200"));
+var settings = new ElasticsearchClientSettings(
+    nodePool,
+    sourceSerializer: (defaultSerializer, settings) =>
+        new VanillaSerializer()); // <1>
+var client = new ElasticsearchClient(settings);
+//end::register-vanilla-serializer[]
+#pragma warning restore format
+}
+
+#pragma warning disable format
+//tag::vanilla-serializer[]
+public class VanillaSerializer : Serializer
+{
+    public override object Deserialize(Type type, Stream stream) =>
+        throw new NotImplementedException();
+
+    public override T Deserialize<T>(Stream stream) =>
+        throw new NotImplementedException();
+
+    public override ValueTask<object> DeserializeAsync(Type type, Stream stream, CancellationToken cancellationToken = default) =>
+        throw new NotImplementedException();
+
+    public override ValueTask<T> DeserializeAsync<T>(Stream stream, CancellationToken cancellationToken = default) =>
+        throw new NotImplementedException();
+
+    public override void Serialize<T>(T data, Stream stream, SerializationFormatting formatting = SerializationFormatting.None) =>
+        throw new NotImplementedException();
+
+    public override Task SerializeAsync<T>(T data, Stream stream, 
+        SerializationFormatting formatting = SerializationFormatting.None, CancellationToken cancellationToken = default) =>
+            throw new NotImplementedException();
+}
+//end::vanilla-serializer[]
+#pragma warning restore format
+
+#pragma warning disable format
+//tag::percolation-document[]
+public class MyPercolationDocument
+{
+    public Query Query { get; set; }
+    public string Category { get; set; }
+}
+//end::percolation-document[]
+#pragma warning restore format
+}