Skip to content

Adds more write overloads to Utf8Json #4209

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 5 commits into from
Closed
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions src/Elasticsearch.Net/Utf8Json/IJsonFormatterResolver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
#endregion

using System;
using System.Collections.Concurrent;
using System.Reflection;

namespace Elasticsearch.Net.Utf8Json
Expand Down Expand Up @@ -60,11 +61,10 @@ public static IJsonFormatter<T> GetFormatterWithVerify<T>(this IJsonFormatterRes
return formatter;
}

private static readonly MethodInfo _getFormatterMethod = typeof(IJsonFormatterResolver).GetRuntimeMethod("GetFormatter", Type.EmptyTypes);
public static object GetFormatterDynamic(this IJsonFormatterResolver resolver, Type type)
{
var methodInfo = typeof(IJsonFormatterResolver).GetRuntimeMethod("GetFormatter", Type.EmptyTypes);

var formatter = methodInfo.MakeGenericMethod(type).Invoke(resolver, null);
var formatter = _getFormatterMethod.MakeGenericMethod(type).Invoke(resolver, null);
return formatter;
}
}
Expand Down
16 changes: 9 additions & 7 deletions src/Elasticsearch.Net/Utf8Json/Internal/UnsafeMemory.Low.cs
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,10 @@ internal static class UnsafeMemory
{
public static readonly bool Is32Bit = (IntPtr.Size == 4);

public static void WriteRaw(ref JsonWriter writer, byte[] src)
public static void WriteRaw(ref JsonWriter writer, byte[] src) => WriteRaw(ref writer, src, src.Length);
public static void WriteRaw(ref JsonWriter writer, byte[] src, int length)
{
switch (src.Length)
switch (length)
{
case 0: break;
case 1: if (Is32Bit) { UnsafeMemory32.WriteRaw1(ref writer, src); } else { UnsafeMemory64.WriteRaw1(ref writer, src); } break;
Expand Down Expand Up @@ -77,19 +78,20 @@ public static void WriteRaw(ref JsonWriter writer, byte[] src)
}
}

public static unsafe void MemoryCopy(ref JsonWriter writer, byte[] src)
public static void MemoryCopy(ref JsonWriter writer, byte[] src) => MemoryCopy(ref writer, src, src.Length);
public static unsafe void MemoryCopy(ref JsonWriter writer, byte[] src, int length)
{
BinaryUtil.EnsureCapacity(ref writer.buffer, writer.offset, src.Length);
BinaryUtil.EnsureCapacity(ref writer.buffer, writer.offset, length);
#if !NET45
fixed (void* dstP = &writer.buffer[writer.offset])
fixed (void* srcP = &src[0])
{
Buffer.MemoryCopy(srcP, dstP, writer.buffer.Length - writer.offset, src.Length);
Buffer.MemoryCopy(srcP, dstP, writer.buffer.Length - writer.offset, length);
}
#else
Buffer.BlockCopy(src, 0, writer.buffer, writer.offset, src.Length);
Buffer.BlockCopy(src, 0, writer.buffer, writer.offset, length);
#endif
writer.offset += src.Length;
writer.offset += length;
}
}

Expand Down
23 changes: 23 additions & 0 deletions src/Elasticsearch.Net/Utf8Json/JsonWriter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
#endregion

using System;
using System.IO;
using System.Runtime.CompilerServices;
using System.Text;
using Elasticsearch.Net.Extensions;
Expand Down Expand Up @@ -147,6 +148,27 @@ public void WriteRaw(byte[] rawValue)
offset += rawValue.Length;
#endif
}
public void WriteRaw(byte[] rawValue, int length)
{
UnsafeMemory.WriteRaw(ref this, rawValue, length);
}
public void WriteRaw(MemoryStream ms)
{
if (ms.TryGetBuffer(out var b) && !(b.Array is null) && b.Offset == 0)
WriteRaw(b.Array, b.Count);
else
{
var bytes = ms.ToArray();
this.WriteRaw(bytes);
}
}

public void WriteSerialized<T>(T value, IElasticsearchSerializer serializer, IConnectionConfigurationValues settings, SerializationFormatting formatting = SerializationFormatting.None)
{
using var ms = settings.MemoryStreamFactory.Create();
serializer.Serialize(value, ms, formatting);
WriteRaw(ms);
}

#if NETSTANDARD
[MethodImpl(MethodImplOptions.AggressiveInlining)]
Expand Down Expand Up @@ -483,5 +505,6 @@ private static void ToUnicode(char c, ref int offset, byte[] buffer)
buffer[offset++] = (byte)CharUtils.HexDigit((c >> 4) & '\x000f');
buffer[offset++] = (byte)CharUtils.HexDigit(c & '\x000f');
}

}
}
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ internal abstract class DynamicCompositeResolver : IJsonFormatterResolver
{
private static readonly string ModuleName = $"{ResolverConfig.Namespace}.DynamicCompositeResolver";

static readonly DynamicAssembly assembly;
static readonly DynamicAssembly assembly;

static DynamicCompositeResolver()
{
Expand Down
5 changes: 5 additions & 0 deletions src/Nest/CommonAbstractions/Extensions/TypeExtensions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System.Linq.Expressions;
using System.Reflection;
using System.Text;
using Elasticsearch.Net.CrossPlatform;

namespace Nest
{
Expand Down Expand Up @@ -150,5 +151,9 @@ private static bool IsHidingMember(PropertyInfo propertyInfo)
}

internal delegate T ObjectActivator<out T>(params object[] args);

private static readonly Assembly NestAssembly = typeof(TypeExtensions).Assembly();

public static bool IsNestType(this Type type) => type.Assembly() == NestAssembly;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,25 +14,17 @@ internal class DefaultHighLevelSerializer : IElasticsearchSerializer, IInternalS

public IJsonFormatterResolver FormatterResolver { get; }

public T Deserialize<T>(Stream stream)
{
return JsonSerializer.Deserialize<T>(stream, FormatterResolver);
}

public object Deserialize(Type type, Stream stream)
{
return JsonSerializer.NonGeneric.Deserialize(type, stream, FormatterResolver);
}

public Task<T> DeserializeAsync<T>(Stream stream, CancellationToken cancellationToken = default)
{
return JsonSerializer.DeserializeAsync<T>(stream, FormatterResolver);
}

public Task<object> DeserializeAsync(Type type, Stream stream, CancellationToken cancellationToken = default)
{
return JsonSerializer.NonGeneric.DeserializeAsync(type, stream, FormatterResolver);
}
public T Deserialize<T>(Stream stream) =>
JsonSerializer.Deserialize<T>(stream, FormatterResolver);

public object Deserialize(Type type, Stream stream) =>
JsonSerializer.NonGeneric.Deserialize(type, stream, FormatterResolver);

public Task<T> DeserializeAsync<T>(Stream stream, CancellationToken cancellationToken = default) =>
JsonSerializer.DeserializeAsync<T>(stream, FormatterResolver);

public Task<object> DeserializeAsync(Type type, Stream stream, CancellationToken cancellationToken = default) =>
JsonSerializer.NonGeneric.DeserializeAsync(type, stream, FormatterResolver);

public virtual void Serialize<T>(T data, Stream writableStream, SerializationFormatting formatting = SerializationFormatting.None) =>
JsonSerializer.Serialize(writableStream, data, FormatterResolver);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,7 @@ public void Serialize(ref JsonWriter writer, TRequestInterface value, IJsonForma
using (var ms = settings.MemoryStreamFactory.Create())
{
untypedDocumentRequest.WriteJson(serializer, ms, SerializationFormatting.None);
var v = ms.ToArray();
writer.WriteRaw(v);
writer.WriteRaw(ms);
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,23 +30,17 @@ public virtual void Serialize(ref JsonWriter writer, T value, IJsonFormatterReso
var settings = formatterResolver.GetConnectionSettings();

// avoid serialization to bytes when not using custom source serializer
if (ReferenceEquals(settings.SourceSerializer, settings.RequestResponseSerializer))
if (ReferenceEquals(settings.SourceSerializer, settings.RequestResponseSerializer)
|| settings.SourceSerializer is IInternalSerializerWithFormatter s && s.FormatterResolver != null)
{
formatterResolver.GetFormatter<T>().Serialize(ref writer, value, formatterResolver);
return;
}

var sourceSerializer = settings.SourceSerializer;
var f = ForceFormatting ?? SerializationFormatting.None;
byte[] bytes;
using (var ms = settings.MemoryStreamFactory.Create())
{
sourceSerializer.Serialize(value, ms, f);
// TODO: read each byte instead of creating and allocating an array
bytes = ms.ToArray();
}

writer.WriteRaw(bytes);
writer.WriteSerialized(value, sourceSerializer, settings, f);
}
}
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
using Elasticsearch.Net.CrossPlatform;
using Elasticsearch.Net.Utf8Json;

namespace Nest
Expand All @@ -13,8 +12,7 @@ public override void Serialize(ref JsonWriter writer, T value, IJsonFormatterRes
return;
}

var nestType = value.GetType().Assembly() == typeof(SourceWriteFormatter<>).Assembly();
if (nestType)
if (value.GetType().IsNestType())
formatterResolver.GetFormatter<T>().Serialize(ref writer, value, formatterResolver);
else
base.Serialize(ref writer, value, formatterResolver);
Expand Down
14 changes: 4 additions & 10 deletions src/Nest/Document/Multiple/Bulk/BulkRequestFormatter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ internal class BulkRequestFormatter : IJsonFormatter<IBulkRequest>
{
private const byte Newline = (byte)'\n';

private static SourceWriteFormatter<object> SourceWriter { get; } = new SourceWriteFormatter<object>();

public IBulkRequest Deserialize(ref JsonReader reader, IJsonFormatterResolver formatterResolver) =>
throw new NotSupportedException();

Expand All @@ -17,16 +19,13 @@ public void Serialize(ref JsonWriter writer, IBulkRequest value, IJsonFormatterR
return;

var settings = formatterResolver.GetConnectionSettings();
var memoryStreamFactory = settings.MemoryStreamFactory;
var requestResponseSerializer = settings.RequestResponseSerializer;
var sourceSerializer = settings.SourceSerializer;
var inferrer = settings.Inferrer;
var formatter = formatterResolver.GetFormatter<object>();

for (var index = 0; index < value.Operations.Count; index++)
{
var op = value.Operations[index];
op.Index = op.Index ?? value.Index ?? op.ClrType;
op.Index ??= value.Index ?? op.ClrType;
if (op.Index.Equals(value.Index)) op.Index = null;
op.Id = op.GetIdForOperation(inferrer);
op.Routing = op.GetRoutingForOperation(inferrer);
Expand All @@ -42,12 +41,7 @@ public void Serialize(ref JsonWriter writer, IBulkRequest value, IJsonFormatterR
if (body == null)
continue;

var bodySerializer = op.Operation == "update" || body is ILazyDocument
? requestResponseSerializer
: sourceSerializer;

var bodyBytes = bodySerializer.SerializeToBytes(body, memoryStreamFactory, SerializationFormatting.None);
writer.WriteRaw(bodyBytes);
SourceWriter.Serialize(ref writer, body, formatterResolver);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This looks like a behavioural change?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Because SerializationFormatting.None is not specified explicitly?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The behavioural change looks like the SourceWriter is now always used.

Before, the requestResponseSerializer was used when the operation was an update or the body was an ILazyDocument

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ahh you are right, will address!

writer.WriteRaw(Newline);
}
}
Expand Down
6 changes: 2 additions & 4 deletions src/Nest/Search/MultiSearch/MultiSearchFormatter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,9 @@ public void Serialize(ref JsonWriter writer, IMultiSearchRequest value, IJsonFor
ignore_unavailable = GetString("ignore_unavailable")
};

var headerBytes = serializer.SerializeToBytes(header, memoryStreamFactory, SerializationFormatting.None);
writer.WriteRaw(headerBytes);
writer.WriteSerialized(header, serializer, settings, SerializationFormatting.None);
writer.WriteRaw(Newline);
var bodyBytes = serializer.SerializeToBytes(operation, memoryStreamFactory, SerializationFormatting.None);
writer.WriteRaw(bodyBytes);
writer.WriteSerialized(operation, serializer, settings, SerializationFormatting.None);
writer.WriteRaw(Newline);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,9 @@ string GetString(string key)
ignore_unavailable = GetString("ignore_unavailable")
};

var headerBytes = serializer.SerializeToBytes(header, memoryStreamFactory, SerializationFormatting.None);
writer.WriteRaw(headerBytes);
writer.WriteSerialized(header, serializer, settings, SerializationFormatting.None);
writer.WriteRaw(Newline);
var bodyBytes = serializer.SerializeToBytes(operation, memoryStreamFactory, SerializationFormatting.None);
writer.WriteRaw(bodyBytes);
writer.WriteSerialized(operation, serializer, settings, SerializationFormatting.None);
writer.WriteRaw(Newline);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,14 +63,9 @@ public void Serialize(ref JsonWriter writer, IPostJobDataRequest value, IJsonFor

var settings = formatterResolver.GetConnectionSettings();
var sourceSerializer = settings.SourceSerializer;
var memoryStreamFactory = settings.MemoryStreamFactory;

foreach (var data in value.Data)
{
var bodyJson = sourceSerializer.SerializeToBytes(data, memoryStreamFactory, SerializationFormatting.None);
writer.WriteRaw(bodyJson);
writer.WriteRaw(Newline);
}
writer.WriteSerialized(data, sourceSerializer, settings, SerializationFormatting.None);
}
}
}