Skip to content

Commit ed46998

Browse files
stevejgordonstevejgordon
authored
[Backport 8.0] Source serialization always sends fractional format for double and floats (#7476)
* Bump version to 8.0.8 * Add custom converters for double and float source serialization (#7467) This ensures we maintain at least one fractional decimal which STJ would otherwise loose.
1 parent 2176f75 commit ed46998

9 files changed

+338
-5
lines changed

‎.github/auto-label.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"rules": {
33
"Infra": [".ci/**/*", ".github/**/*","build/**/*"],
4-
"v8.0.7": "**/*"
4+
"v8.0.8": "**/*"
55
}
66
}

‎exclusion.dic

+4-1
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,7 @@ json
33
async
44
inferrer
55
elasticsearch
6-
asciidocs
6+
asciidocs
7+
yyyy
8+
enum
9+
trippable

‎global.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
"rollForward": "minor",
55
"allowPrerelease": false
66
},
7-
"version": "8.0.7",
7+
"version": "8.0.8",
88
"doc_current": "main",
99
"doc_branch": "main"
1010
}

‎src/Elastic.Clients.Elasticsearch/Serialization/DefaultSourceSerializer.cs

+4-2
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,9 @@ public class DefaultSourceSerializer : SystemTextJsonSerializer
2020
/// </summary>
2121
public static JsonConverter[] DefaultBuiltInConverters => new JsonConverter[]
2222
{
23-
new JsonStringEnumConverter()
23+
new JsonStringEnumConverter(),
24+
new DoubleWithFractionalPortionConverter(),
25+
new FloatWithFractionalPortionConverter()
2426
};
2527

2628
private readonly JsonSerializerOptions _jsonSerializerOptions;
@@ -63,7 +65,7 @@ public static JsonSerializerOptions CreateDefaultJsonSerializerOptions(bool incl
6365
{
6466
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
6567
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
66-
NumberHandling = JsonNumberHandling.AllowReadingFromString | JsonNumberHandling.AllowNamedFloatingPointLiterals
68+
NumberHandling = JsonNumberHandling.AllowReadingFromString // For numerically mapped fields, it is possible for values in the source to be returned as strings, if they were indexed as such.
6769
};
6870

6971
if (includeDefaultConverters)

‎src/Elastic.Clients.Elasticsearch/Serialization/DoubleWithFractionalPortionConverter.cs

+89
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information.
4+
5+
#pragma warning disable IDE0005
6+
using System;
7+
using System.Buffers.Text;
8+
using System.Globalization;
9+
using System.Text;
10+
using System.Text.Json;
11+
using System.Text.Json.Serialization;
12+
using static Elastic.Clients.Elasticsearch.Serialization.JsonConstants;
13+
#pragma warning restore IDE0005
14+
15+
namespace Elastic.Clients.Elasticsearch.Serialization;
16+
17+
internal sealed class DoubleWithFractionalPortionConverter : JsonConverter<double>
18+
{
19+
// We don't handle floating point literals (NaN, etc.) because for source serialization because Elasticsearch only support finite values for numeric fields.
20+
// We must handle the possibility of numbers as strings in the source however.
21+
22+
public override double Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
23+
{
24+
if (reader.TokenType == JsonTokenType.String && options.NumberHandling.HasFlag(JsonNumberHandling.AllowReadingFromString))
25+
{
26+
var value = reader.GetString();
27+
28+
if (!double.TryParse(value, out var parsedValue))
29+
ThrowHelper.ThrowJsonException($"Unable to parse '{value}' as a double.");
30+
31+
return parsedValue;
32+
}
33+
34+
return reader.GetDouble();
35+
}
36+
37+
public override void Write(Utf8JsonWriter writer, double value, JsonSerializerOptions options)
38+
{
39+
Span<byte> utf8bytes = stackalloc byte[128]; // This is the size used in STJ for future proofing. https://github.com/dotnet/runtime/blob/dae6c2472b699b7cff2efeb5ce06b75c9551bc40/src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs#L79
40+
41+
// NOTE: This code is based on https://github.com/dotnet/runtime/blob/dae6c2472b699b7cff2efeb5ce06b75c9551bc40/src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs#L79
42+
43+
// Frameworks that are not .NET Core 3.0 or higher do not produce round-trippable strings by
44+
// default. Further, the Utf8Formatter on older frameworks does not support taking a precision
45+
// specifier for 'G' nor does it represent other formats such as 'R'. As such, we duplicate
46+
// the .NET Core 3.0 logic of forwarding to the UTF16 formatter and transcoding it back to UTF8,
47+
// with some additional changes to remove dependencies on Span APIs which don't exist downlevel.
48+
49+
// PERFORMANCE: This code could be benchmarked and tweaked to make it faster.
50+
51+
#if NETCOREAPP
52+
if (Utf8Formatter.TryFormat(value, utf8bytes, out var bytesWritten))
53+
{
54+
if (utf8bytes.IndexOfAny(NonIntegerBytes) == -1)
55+
{
56+
utf8bytes[bytesWritten++] = (byte)'.';
57+
utf8bytes[bytesWritten++] = (byte)'0';
58+
}
59+
60+
#pragma warning disable IDE0057 // Use range operator
61+
writer.WriteRawValue(utf8bytes.Slice(0, bytesWritten), skipInputValidation: true);
62+
#pragma warning restore IDE0057 // Use range operator
63+
64+
return;
65+
}
66+
#else
67+
var utf16Text = value.ToString("G17", CultureInfo.InvariantCulture);
68+
69+
if (utf16Text.Length < utf8bytes.Length)
70+
{
71+
try
72+
{
73+
var bytes = Encoding.UTF8.GetBytes(utf16Text);
74+
75+
if (bytes.Length < utf8bytes.Length)
76+
{
77+
bytes.CopyTo(utf8bytes);
78+
}
79+
}
80+
catch
81+
{
82+
// Swallow this and fall through to our general exception.
83+
}
84+
}
85+
#endif
86+
87+
ThrowHelper.ThrowJsonException($"Unable to serialize double value.");
88+
}
89+
}

‎src/Elastic.Clients.Elasticsearch/Serialization/FloatWithFractionalPortionConverter.cs

+89
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information.
4+
5+
#pragma warning disable IDE0005
6+
using System;
7+
using System.Buffers.Text;
8+
using System.Globalization;
9+
using System.Text;
10+
using System.Text.Json;
11+
using System.Text.Json.Serialization;
12+
using static Elastic.Clients.Elasticsearch.Serialization.JsonConstants;
13+
#pragma warning restore IDE0005
14+
15+
namespace Elastic.Clients.Elasticsearch.Serialization;
16+
17+
internal sealed class FloatWithFractionalPortionConverter : JsonConverter<float>
18+
{
19+
// We don't handle floating point literals (NaN, etc.) because for source serialization because Elasticsearch only supports finite values for numeric fields.
20+
// We must handle the possibility of numbers as strings in the source however.
21+
22+
public override float Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
23+
{
24+
if (reader.TokenType == JsonTokenType.String && options.NumberHandling.HasFlag(JsonNumberHandling.AllowReadingFromString))
25+
{
26+
var value = reader.GetString();
27+
28+
if (!float.TryParse(value, out var parsedValue))
29+
ThrowHelper.ThrowJsonException($"Unable to parse '{value}' as a float.");
30+
31+
return parsedValue;
32+
}
33+
34+
return reader.GetSingle();
35+
}
36+
37+
public override void Write(Utf8JsonWriter writer, float value, JsonSerializerOptions options)
38+
{
39+
Span<byte> utf8bytes = stackalloc byte[128]; // This is the size used in STJ for future proofing. https://github.com/dotnet/runtime/blob/dae6c2472b699b7cff2efeb5ce06b75c9551bc40/src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs#L79
40+
41+
// NOTE: This code is based on https://github.com/dotnet/runtime/blob/dae6c2472b699b7cff2efeb5ce06b75c9551bc40/src/libraries/System.Text.Json/src/System/Text/Json/JsonConstants.cs#L79
42+
43+
// Frameworks that are not .NET Core 3.0 or higher do not produce round-trippable strings by
44+
// default. Further, the Utf8Formatter on older frameworks does not support taking a precision
45+
// specifier for 'G' nor does it represent other formats such as 'R'. As such, we duplicate
46+
// the .NET Core 3.0 logic of forwarding to the UTF16 formatter and transcoding it back to UTF8,
47+
// with some additional changes to remove dependencies on Span APIs which don't exist downlevel.
48+
49+
// PERFORMANCE: This code could be benchmarked and tweaked to make it faster.
50+
51+
#if NETCOREAPP
52+
if (Utf8Formatter.TryFormat(value, utf8bytes, out var bytesWritten))
53+
{
54+
if (utf8bytes.IndexOfAny(NonIntegerBytes) == -1)
55+
{
56+
utf8bytes[bytesWritten++] = (byte)'.';
57+
utf8bytes[bytesWritten++] = (byte)'0';
58+
}
59+
60+
#pragma warning disable IDE0057 // Use range operator
61+
writer.WriteRawValue(utf8bytes.Slice(0, bytesWritten), skipInputValidation: true);
62+
#pragma warning restore IDE0057 // Use range operator
63+
64+
return;
65+
}
66+
#else
67+
var utf16Text = value.ToString("G9", CultureInfo.InvariantCulture);
68+
69+
if (utf16Text.Length < utf8bytes.Length)
70+
{
71+
try
72+
{
73+
var bytes = Encoding.UTF8.GetBytes(utf16Text);
74+
75+
if (bytes.Length < utf8bytes.Length)
76+
{
77+
bytes.CopyTo(utf8bytes);
78+
}
79+
}
80+
catch
81+
{
82+
// Swallow this and fall through to our general exception.
83+
}
84+
}
85+
#endif
86+
87+
ThrowHelper.ThrowJsonException($"Unable to serialize float value.");
88+
}
89+
}

‎src/Elastic.Clients.Elasticsearch/Serialization/JsonConstants.cs

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information.
4+
5+
using System;
6+
7+
namespace Elastic.Clients.Elasticsearch.Serialization;
8+
9+
internal static class JsonConstants
10+
{
11+
#pragma warning disable IDE0230 // Use UTF-8 string literal
12+
public static ReadOnlySpan<byte> NonIntegerBytes => new[] { (byte)'E', (byte)'.' }; // In the future, when we move to the .NET 7 SDK, it would be nice to use u8 literals e.g. "E."u8
13+
#pragma warning restore IDE0230 // Use UTF-8 string literal
14+
}

‎tests/Tests/Serialization/SerializerTestBase.cs

+17
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,7 @@ protected string SerializeAndGetJsonString<T>(T data, ElasticsearchClientSetting
115115
public abstract class SerializerTestBase
116116
{
117117
protected static readonly Serializer _requestResponseSerializer;
118+
protected static readonly Serializer _sourceSerializer;
118119
protected static readonly IElasticsearchClientSettings _settings;
119120

120121
static SerializerTestBase()
@@ -125,6 +126,7 @@ static SerializerTestBase()
125126
var client = new ElasticsearchClient(settings);
126127

127128
_requestResponseSerializer = client.RequestResponseSerializer;
129+
_sourceSerializer = client.SourceSerializer;
128130
_settings = client.ElasticsearchClientSettings;
129131
}
130132

@@ -163,12 +165,27 @@ protected static string SerializeAndGetJsonString<T>(T data)
163165
return reader.ReadToEnd();
164166
}
165167

168+
protected static string SourceSerializeAndGetJsonString<T>(T data)
169+
{
170+
var stream = new MemoryStream();
171+
_sourceSerializer.Serialize(data, stream);
172+
stream.Position = 0;
173+
var reader = new StreamReader(stream);
174+
return reader.ReadToEnd();
175+
}
176+
166177
protected static T DeserializeJsonString<T>(string json)
167178
{
168179
var stream = WrapInStream(json);
169180
return _requestResponseSerializer.Deserialize<T>(stream);
170181
}
171182

183+
protected static T SourceDeserializeJsonString<T>(string json)
184+
{
185+
var stream = WrapInStream(json);
186+
return _sourceSerializer.Deserialize<T>(stream);
187+
}
188+
172189
/// <summary>
173190
/// Serialises the <paramref name="data"/> using the sync and async request/response serializer methods, comparing the results.
174191
/// </summary>

0 commit comments

Comments
 (0)