Adds more write overloads to Utf8Json

src/Elasticsearch.Net/Serialization/DiagnosticsSerializerProxy.cs

@@ -32,12 +32,16 @@ public SerializerRegistrationInformation(Type type, string purpose)
 		public override string ToString() => _stringRepresentation;
 	}
 
+	//TODO this no longer needs to be IInternalSerializerWithFormatter
+	// Previous we checked that the serializer we are wrapping implements the interface and if so set the formatter
+	// However its totally fine for formatter to be null. `IJsonFormatter` solves this better.
 	/// <summary>
 	/// Wraps configured serializer so that we can emit diagnostics per configured serializer.
 	/// </summary>
-	internal class DiagnosticsSerializerProxy : IElasticsearchSerializer, IInternalSerializerWithFormatter
+	internal class DiagnosticsSerializerProxy : IElasticsearchSerializer, IInternalSerializerWithFormatter, IInternalSerializer
 	{
 		private readonly IElasticsearchSerializer _serializer;
+		private readonly bool _wrapsUtf8JsonSerializer;
 		private readonly SerializerRegistrationInformation _state;
 		private readonly IJsonFormatterResolver _formatterResolver;
 		private static DiagnosticSource DiagnosticSource { get; } = new DiagnosticListener(DiagnosticSources.Serializer.SourceName);
@@ -47,11 +51,25 @@ public DiagnosticsSerializerProxy(IElasticsearchSerializer serializer, string pu
 			_serializer = serializer;
 			_state = new SerializerRegistrationInformation(serializer.GetType(), purpose);
 			if (serializer is IInternalSerializerWithFormatter withFormatter)
+			{
 				_formatterResolver = withFormatter.FormatterResolver;
+				_wrapsUtf8JsonSerializer = true;
+			}
+			else
+			{
+				_wrapsUtf8JsonSerializer = false;
+				_formatterResolver = null;
+			}
 		}
 
 		public IJsonFormatterResolver FormatterResolver => _formatterResolver;
 
+		public bool TryGetJsonFormatter(out IJsonFormatterResolver formatterResolver)
+		{
+			formatterResolver = _formatterResolver;
+			return _wrapsUtf8JsonSerializer;
+		}
+
 		public object Deserialize(Type type, Stream stream)
 		{
 			using (DiagnosticSource.Diagnose(DiagnosticSources.Serializer.Deserialize, _state))

src/Elasticsearch.Net/Serialization/ElasticsearchSerializerExtensions.cs

@@ -1,9 +1,23 @@
 using Elasticsearch.Net.Extensions;
+using Elasticsearch.Net.Utf8Json;
 
 namespace Elasticsearch.Net
 {
 	public static class ElasticsearchSerializerExtensions
 	{
+		internal static void SerializeUsingWriter<T>(this IElasticsearchSerializer serializer, ref JsonWriter writer, T body, IConnectionConfigurationValues settings, SerializationFormatting formatting)
+		{
+			if (serializer is IInternalSerializer s && s.TryGetJsonFormatter(out var formatterResolver))
+			{
+				JsonSerializer.Serialize(ref writer, body, formatterResolver);
+				return;
+			}
+
+			var memoryStreamFactory = settings.MemoryStreamFactory;
+			var bodyBytes = serializer.SerializeToBytes(body, memoryStreamFactory, formatting);
+			writer.WriteRaw(bodyBytes);
+		}
+
 		/// <summary>
 		/// Extension method that serializes an instance of <typeparamref name="T"/> to a byte array.
 		/// </summary>
@@ -65,5 +79,6 @@ public static string SerializeToString<T>(
 				return ms.Utf8String();
 			}
 		}
+
 	}
 }

src/Elasticsearch.Net/Serialization/IInternalSerializerWithFormatter.cs

@@ -1,9 +1,14 @@
 using Elasticsearch.Net.Utf8Json;
 
-namespace Elasticsearch.Net 
+namespace Elasticsearch.Net
 {
+	internal interface IInternalSerializer
+	{
+		bool TryGetJsonFormatter(out IJsonFormatterResolver formatterResolver);
+	}
+
 	internal interface IInternalSerializerWithFormatter
 	{
 		IJsonFormatterResolver FormatterResolver { get; }
 	}
-}
+}

src/Elasticsearch.Net/Serialization/LowLevelRequestResponseSerializer.cs

@@ -7,29 +7,23 @@
 
 namespace Elasticsearch.Net
 {
-	public class LowLevelRequestResponseSerializer : IElasticsearchSerializer
+	public class LowLevelRequestResponseSerializer : IElasticsearchSerializer, IInternalSerializerWithFormatter
 	{
+		IJsonFormatterResolver IInternalSerializerWithFormatter.FormatterResolver => null;
+
 		public static readonly LowLevelRequestResponseSerializer Instance = new LowLevelRequestResponseSerializer();
 
-		public object Deserialize(Type type, Stream stream)
-		{
-			return JsonSerializer.NonGeneric.Deserialize(type, stream, ElasticsearchNetFormatterResolver.Instance);
-		}
-
-		public T Deserialize<T>(Stream stream)
-		{
-			return JsonSerializer.Deserialize<T>(stream, ElasticsearchNetFormatterResolver.Instance);
-		}
-
-		public Task<object> DeserializeAsync(Type type, Stream stream, CancellationToken cancellationToken = default)
-		{
-			return JsonSerializer.NonGeneric.DeserializeAsync(type, stream, ElasticsearchNetFormatterResolver.Instance);
-		}
-
-		public Task<T> DeserializeAsync<T>(Stream stream, CancellationToken cancellationToken = default)
-		{
-			return JsonSerializer.DeserializeAsync<T>(stream, ElasticsearchNetFormatterResolver.Instance);
-		}
+		public object Deserialize(Type type, Stream stream) =>
+			JsonSerializer.NonGeneric.Deserialize(type, stream, ElasticsearchNetFormatterResolver.Instance);
+
+		public T Deserialize<T>(Stream stream) =>
+			JsonSerializer.Deserialize<T>(stream, ElasticsearchNetFormatterResolver.Instance);
+
+		public Task<object> DeserializeAsync(Type type, Stream stream, CancellationToken cancellationToken = default) =>
+			JsonSerializer.NonGeneric.DeserializeAsync(type, stream, ElasticsearchNetFormatterResolver.Instance);
+
+		public Task<T> DeserializeAsync<T>(Stream stream, CancellationToken cancellationToken = default) =>
+			JsonSerializer.DeserializeAsync<T>(stream, ElasticsearchNetFormatterResolver.Instance);
 
 		public void Serialize<T>(T data, Stream writableStream, SerializationFormatting formatting = SerializationFormatting.None) =>
 			JsonSerializer.Serialize(writableStream, data, ElasticsearchNetFormatterResolver.Instance);

src/Elasticsearch.Net/Utf8Json/IJsonFormatterResolver.cs

@@ -23,6 +23,7 @@
 #endregion
 
 using System;
+using System.Collections.Concurrent;
 using System.Reflection;
 
 namespace Elasticsearch.Net.Utf8Json
@@ -60,11 +61,10 @@ public static IJsonFormatter<T> GetFormatterWithVerify<T>(this IJsonFormatterRes
             return formatter;
         }
 
+		private static readonly MethodInfo _getFormatterMethod = typeof(IJsonFormatterResolver).GetRuntimeMethod("GetFormatter", Type.EmptyTypes);
         public static object GetFormatterDynamic(this IJsonFormatterResolver resolver, Type type)
         {
-            var methodInfo = typeof(IJsonFormatterResolver).GetRuntimeMethod("GetFormatter", Type.EmptyTypes);
-
-            var formatter = methodInfo.MakeGenericMethod(type).Invoke(resolver, null);
+            var formatter = _getFormatterMethod.MakeGenericMethod(type).Invoke(resolver, null);
             return formatter;
         }
     }

src/Elasticsearch.Net/Utf8Json/Internal/UnsafeMemory.Low.cs

@@ -35,9 +35,10 @@ internal static class UnsafeMemory
     {
         public static readonly bool Is32Bit = (IntPtr.Size == 4);
 
-        public static void WriteRaw(ref JsonWriter writer, byte[] src)
+        public static void WriteRaw(ref JsonWriter writer, byte[] src) => WriteRaw(ref writer, src, src.Length);
+        public static void WriteRaw(ref JsonWriter writer, byte[] src, int length)
         {
-            switch (src.Length)
+            switch (length)
             {
                 case 0: break;
                 case 1: if (Is32Bit) { UnsafeMemory32.WriteRaw1(ref writer, src); } else { UnsafeMemory64.WriteRaw1(ref writer, src); } break;
@@ -77,19 +78,20 @@ public static void WriteRaw(ref JsonWriter writer, byte[] src)
             }
         }
 
-        public static unsafe void MemoryCopy(ref JsonWriter writer, byte[] src)
+        public static void MemoryCopy(ref JsonWriter writer, byte[] src) => MemoryCopy(ref writer, src, src.Length);
+        public static unsafe void MemoryCopy(ref JsonWriter writer, byte[] src, int length)
         {
-            BinaryUtil.EnsureCapacity(ref writer.buffer, writer.offset, src.Length);
+            BinaryUtil.EnsureCapacity(ref writer.buffer, writer.offset, length);
 #if !NET45
             fixed (void* dstP = &writer.buffer[writer.offset])
             fixed (void* srcP = &src[0])
             {
-                Buffer.MemoryCopy(srcP, dstP, writer.buffer.Length - writer.offset, src.Length);
+                Buffer.MemoryCopy(srcP, dstP, writer.buffer.Length - writer.offset, length);
             }
 #else
-            Buffer.BlockCopy(src, 0, writer.buffer, writer.offset, src.Length);
+            Buffer.BlockCopy(src, 0, writer.buffer, writer.offset, length);
 #endif
-            writer.offset += src.Length;
+            writer.offset += length;
         }
     }
 

src/Elasticsearch.Net/Utf8Json/JsonWriter.cs

@@ -23,6 +23,7 @@
 #endregion
 
 using System;
+using System.IO;
 using System.Runtime.CompilerServices;
 using System.Text;
 using Elasticsearch.Net.Extensions;
@@ -147,6 +148,27 @@ public void WriteRaw(byte[] rawValue)
             offset += rawValue.Length;
 #endif
         }
+        public void WriteRaw(byte[] rawValue, int length)
+        {
+            UnsafeMemory.WriteRaw(ref this, rawValue, length);
+        }
+        public void WriteRaw(MemoryStream ms)
+		{
+			if (ms.TryGetBuffer(out var b) && !(b.Array is null) && b.Offset == 0)
+				WriteRaw(b.Array, b.Count);
+			else
+			{
+				var bytes = ms.ToArray();
+				this.WriteRaw(bytes);
+			}
+		}
+
+		public void WriteSerialized<T>(T value, IElasticsearchSerializer serializer, IConnectionConfigurationValues settings, SerializationFormatting formatting = SerializationFormatting.None)
+		{
+			using var ms = settings.MemoryStreamFactory.Create();
+			serializer.Serialize(value, ms, formatting);
+			WriteRaw(ms);
+		}
 
 #if NETSTANDARD
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -483,5 +505,6 @@ private static void ToUnicode(char c, ref int offset, byte[] buffer)
 			buffer[offset++] = (byte)CharUtils.HexDigit((c >> 4) & '\x000f');
 			buffer[offset++] = (byte)CharUtils.HexDigit(c & '\x000f');
 		}
+
 	}
 }

src/Elasticsearch.Net/Utf8Json/Resolvers/CompositeResolver.cs

@@ -157,7 +157,7 @@ internal abstract class DynamicCompositeResolver : IJsonFormatterResolver
     {
 		private static readonly string ModuleName =  $"{ResolverConfig.Namespace}.DynamicCompositeResolver";
 
-        static readonly DynamicAssembly assembly;
+		static readonly DynamicAssembly assembly;
 
         static DynamicCompositeResolver()
         {

src/Nest/CommonAbstractions/Extensions/TypeExtensions.cs

@@ -5,6 +5,7 @@
 using System.Linq.Expressions;
 using System.Reflection;
 using System.Text;
+using Elasticsearch.Net.CrossPlatform;
 
 namespace Nest
 {
@@ -150,5 +151,9 @@ private static bool IsHidingMember(PropertyInfo propertyInfo)
 		}
 
 		internal delegate T ObjectActivator<out T>(params object[] args);
+
+		private static readonly Assembly NestAssembly = typeof(TypeExtensions).Assembly();
+
+		public static bool IsNestType(this Type type) => type.Assembly() == NestAssembly;
 	}
 }

src/Nest/CommonAbstractions/SerializationBehavior/DefaultHighLevelSerializer.cs

@@ -14,25 +14,17 @@ internal class DefaultHighLevelSerializer : IElasticsearchSerializer, IInternalS
 
 		public IJsonFormatterResolver FormatterResolver { get; }
 
-		public T Deserialize<T>(Stream stream)
-		{
-			return JsonSerializer.Deserialize<T>(stream, FormatterResolver);
-		}
-
-		public object Deserialize(Type type, Stream stream)
-		{
-			return JsonSerializer.NonGeneric.Deserialize(type, stream, FormatterResolver);
-		}
-
-		public Task<T> DeserializeAsync<T>(Stream stream, CancellationToken cancellationToken = default)
-		{
-			return JsonSerializer.DeserializeAsync<T>(stream, FormatterResolver);
-		}
-
-		public Task<object> DeserializeAsync(Type type, Stream stream, CancellationToken cancellationToken = default)
-		{
-			return JsonSerializer.NonGeneric.DeserializeAsync(type, stream, FormatterResolver);
-		}
+		public T Deserialize<T>(Stream stream) =>
+			JsonSerializer.Deserialize<T>(stream, FormatterResolver);
+
+		public object Deserialize(Type type, Stream stream) =>
+			JsonSerializer.NonGeneric.Deserialize(type, stream, FormatterResolver);
+
+		public Task<T> DeserializeAsync<T>(Stream stream, CancellationToken cancellationToken = default) =>
+			JsonSerializer.DeserializeAsync<T>(stream, FormatterResolver);
+
+		public Task<object> DeserializeAsync(Type type, Stream stream, CancellationToken cancellationToken = default) =>
+			JsonSerializer.NonGeneric.DeserializeAsync(type, stream, FormatterResolver);
 
 		public virtual void Serialize<T>(T data, Stream writableStream, SerializationFormatting formatting = SerializationFormatting.None) =>
 			JsonSerializer.Serialize(writableStream, data, FormatterResolver);

src/Nest/CommonAbstractions/SerializationBehavior/JsonFormatters/ProxyRequestFormatterBase.cs

@@ -37,8 +37,7 @@ public void Serialize(ref JsonWriter writer, TRequestInterface value, IJsonForma
 			using (var ms = settings.MemoryStreamFactory.Create())
 			{
 				untypedDocumentRequest.WriteJson(serializer, ms, SerializationFormatting.None);
-				var v = ms.ToArray();
-				writer.WriteRaw(v);
+				writer.WriteRaw(ms);
 			}
 		}
 	}

src/Nest/CommonAbstractions/SerializationBehavior/JsonFormatters/SourceFormatter.cs

@@ -17,36 +17,41 @@ public T Deserialize(ref JsonReader reader, IJsonFormatterResolver formatterReso
 			var settings = formatterResolver.GetConnectionSettings();
 
 			// avoid deserialization through stream when not using custom source serializer
-			if (ReferenceEquals(settings.SourceSerializer, settings.RequestResponseSerializer))
+			if (AttemptFastPath(settings.SourceSerializer))
 				return formatterResolver.GetFormatter<T>().Deserialize(ref reader, formatterResolver);
 
 			var arraySegment = reader.ReadNextBlockSegment();
 			using (var ms = settings.MemoryStreamFactory.Create(arraySegment.Array, arraySegment.Offset, arraySegment.Count))
 				return settings.SourceSerializer.Deserialize<T>(ms);
 		}
 
+		/// <summary>
+		/// Avoid serialization to bytes when not using a custom source serializer.
+		/// This used to check for reference of the source serializer and the request response serializer
+		/// However each now gets wrapped in a new `DiagnosticsSerializerProxy` so this check no longer works
+		/// therefor we now check if the SourceSerializer is internal with a formatter.
+		/// DiagnosticsSerializerProxy implements this interface, it simply proxies to whatever it wraps so
+		/// we need to assert the resolver is not actually null here since it can wrap something that is not
+		/// `IInternalSerializerWithFormatter`
+		/// </summary>
+		private static bool AttemptFastPath(IElasticsearchSerializer serializer) =>
+			serializer is IInternalSerializer s && s.TryGetJsonFormatter(out var _);
+
+
 		public virtual void Serialize(ref JsonWriter writer, T value, IJsonFormatterResolver formatterResolver)
 		{
 			var settings = formatterResolver.GetConnectionSettings();
 
-			// avoid serialization to bytes when not using custom source serializer
-			if (ReferenceEquals(settings.SourceSerializer, settings.RequestResponseSerializer))
+			if (AttemptFastPath(settings.SourceSerializer))
 			{
 				formatterResolver.GetFormatter<T>().Serialize(ref writer, value, formatterResolver);
 				return;
 			}
 
 			var sourceSerializer = settings.SourceSerializer;
 			var f = ForceFormatting ?? SerializationFormatting.None;
-			byte[] bytes;
-			using (var ms = settings.MemoryStreamFactory.Create())
-			{
-				sourceSerializer.Serialize(value, ms, f);
-				// TODO: read each byte instead of creating and allocating an array
-				bytes = ms.ToArray();
-			}
 
-			writer.WriteRaw(bytes);
+			writer.WriteSerialized(value, sourceSerializer, settings, f);
 		}
 	}
 }

src/Nest/CommonAbstractions/SerializationBehavior/SourceValueWriteConverter.cs

@@ -1,4 +1,3 @@
-using Elasticsearch.Net.CrossPlatform;
 using Elasticsearch.Net.Utf8Json;
 
 namespace Nest
@@ -13,8 +12,7 @@ public override void Serialize(ref JsonWriter writer, T value, IJsonFormatterRes
 				return;
 			}
 
-			var nestType = value.GetType().Assembly() == typeof(SourceWriteFormatter<>).Assembly();
-			if (nestType)
+			if (value.GetType().IsNestType())
 				formatterResolver.GetFormatter<T>().Serialize(ref writer, value, formatterResolver);
 			else
 				base.Serialize(ref writer, value, formatterResolver);