Skip to content

[WIP] Remove ToArray calls on MemoryStream #3797

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions src/Elasticsearch.Net/Extensions/UtilExtensions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,16 @@ internal static string Utf8String(this MemoryStream ms)

return Encoding.UTF8.GetString(buffer.Array, buffer.Offset, buffer.Count);
}
/// <summary> Attempts to return to underlying bytes buffer (no-copy) if possible </summary>
internal static byte[] ToArrayOrBuffer(this MemoryStream ms)
{
if (ms is null) return null;

if (!ms.TryGetBuffer(out var buffer) || buffer.Array is null)
return ms.ToArray();

return buffer.Array;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it safe to call TryGetBuffer on a RecyclableMemoryStream? It looks like this method is not overloaded.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's safe in the sense that it works but see my comment on the PR, updating to a later version that actually implements it yields some problems under heavy usage.

I have a feeling it sometimes writes at an offset which trygetbuffer does not take into account, not been able to trap it yesterday though

}

internal static byte[] Utf8Bytes(this string s) => s.IsNullOrEmpty() ? null : Encoding.UTF8.GetBytes(s);

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
using Elasticsearch.Net.Extensions;
using Elasticsearch.Net.Utf8Json;

namespace Elasticsearch.Net
{
Expand All @@ -16,7 +15,7 @@ public static byte[] SerializeToBytes<T>(
using (var ms = memoryStreamFactory.Create())
{
serializer.Serialize(data, ms, formatting);
return ms.ToArray();
return ms.ToArrayOrBuffer();
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,7 @@ private static bool NeedsToEagerReadStream<TResponse>()

private static byte[] SwapStreams(ref Stream responseStream, ref MemoryStream ms)
{
var bytes = ms.ToArray();
var bytes = ms.ToArrayOrBuffer();
responseStream.Dispose();
responseStream = ms;
responseStream.Position = 0;
Expand Down
4 changes: 2 additions & 2 deletions src/Elasticsearch.Net/Transport/PostData.cs
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ public override void Write(Stream writableStream, IConnectionConfigurationValues
ms.CopyTo(writableStream, BufferSize);
}
if (Type != 0)
WrittenBytes = ms?.ToArray();
WrittenBytes = ms?.ToArrayOrBuffer();
}

public override async Task WriteAsync(Stream writableStream, IConnectionConfigurationValues settings, CancellationToken cancellationToken)
Expand Down Expand Up @@ -169,7 +169,7 @@ await settings.RequestResponseSerializer.SerializeAsync(o, stream, Serialization
await ms.CopyToAsync(writableStream, BufferSize, cancellationToken).ConfigureAwait(false);
}
if (Type != 0)
WrittenBytes = ms?.ToArray();
WrittenBytes = ms?.ToArrayOrBuffer();
}
}
}
5 changes: 3 additions & 2 deletions src/Elasticsearch.Net/Transport/SerializableData.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using Elasticsearch.Net.Extensions;

namespace Elasticsearch.Net
{
Expand Down Expand Up @@ -31,7 +32,7 @@ public override void Write(Stream writableStream, IConnectionConfigurationValues
ms.CopyTo(writableStream, BufferSize);
}
if (Type != 0)
WrittenBytes = ms?.ToArray();
WrittenBytes = ms?.ToArrayOrBuffer();
}

public static implicit operator SerializableData<T>(T serializableData) => new SerializableData<T>(serializableData);
Expand All @@ -53,7 +54,7 @@ public override async Task WriteAsync(Stream writableStream, IConnectionConfigur
await ms.CopyToAsync(writableStream, BufferSize, cancellationToken).ConfigureAwait(false);
}
if (Type != 0)
WrittenBytes = ms?.ToArray();
WrittenBytes = ms?.ToArrayOrBuffer();
}
}
}
3 changes: 1 addition & 2 deletions src/Elasticsearch.Net/Utf8Json/Internal/BinaryUtil.cs
Original file line number Diff line number Diff line change
Expand Up @@ -112,9 +112,8 @@ public static
byte[] FastCloneWithResize(byte[] src, int newSize)
{
if (newSize < 0) throw new ArgumentOutOfRangeException("newSize");
if (src.Length < newSize) throw new ArgumentException("length < newSize");

if (src == null) return new byte[newSize];
if (src.Length < newSize) throw new ArgumentException("length < newSize");

byte[] dst = new byte[newSize];

Expand Down
5 changes: 3 additions & 2 deletions src/Nest/Cat/CatHelpResponseBuilder.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using System.Threading;
using System.Threading.Tasks;
using Elasticsearch.Net;
using Elasticsearch.Net.Extensions;

namespace Nest
{
Expand All @@ -20,7 +21,7 @@ public override object DeserializeResponse(IElasticsearchSerializer builtInSeria
using (var ms = response.ConnectionConfiguration.MemoryStreamFactory.Create())
{
stream.CopyTo(ms);
var body = ms.ToArray().Utf8String();
var body = ms.Utf8String();
Parse(catResponse, body);
}

Expand All @@ -46,7 +47,7 @@ public override async Task<object> DeserializeResponseAsync(IElasticsearchSerial
using (var ms = response.ConnectionConfiguration.MemoryStreamFactory.Create())
{
await stream.CopyToAsync(ms, 81920, ctx);
var body = ms.ToArray().Utf8String();
var body = ms.Utf8String();
Parse(catResponse, body);
}

Expand Down
4 changes: 2 additions & 2 deletions src/Nest/CommonAbstractions/Extensions/Extensions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Runtime.Serialization;
Expand Down Expand Up @@ -96,7 +97,6 @@ internal static string ToEnumValue<T>(this T enumValue) where T : struct

return null;
}

internal static string Utf8String(this ref ArraySegment<byte> segment) =>
StringEncoding.UTF8.GetString(segment.Array, segment.Offset, segment.Count);

Expand Down Expand Up @@ -161,7 +161,7 @@ internal static bool IsEmpty<T>(this IEnumerable<T> list)
return !enumerable.Any() || enumerable.All(t => t == null);
}

internal static void ThrowIfNull<T>(this T value, string name, string message = null)
internal static void ThrowIfNull<T>(this T value, string name, string message = null)
{
if (value == null && message.IsNullOrEmpty()) throw new ArgumentNullException(name);
else if (value == null) throw new ArgumentNullException(name, "Argument can not be null when " + message);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
using System.Reflection;
using Elasticsearch.Net;
using Elasticsearch.Net.Extensions;
using Elasticsearch.Net.Utf8Json;

namespace Nest
Expand Down Expand Up @@ -41,7 +42,7 @@ public void Serialize(ref JsonWriter writer, TRequestInterface value, IJsonForma
using (var ms = settings.MemoryStreamFactory.Create())
{
untypedDocumentRequest.WriteJson(serializer, ms, SerializationFormatting.None);
var v = ms.ToArray();
var v = ms.ToArrayOrBuffer();
writer.WriteRaw(v);
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using Elasticsearch.Net;
using Elasticsearch.Net.Extensions;
using Elasticsearch.Net.Utf8Json;

namespace Nest
Expand Down Expand Up @@ -43,7 +44,7 @@ public virtual void Serialize(ref JsonWriter writer, T value, IJsonFormatterReso
{
sourceSerializer.Serialize(value, ms, f);
// TODO: read each byte instead of creating and allocating an array
bytes = ms.ToArray();
bytes = ms.ToArrayOrBuffer();
}

writer.WriteRaw(bytes);
Expand Down