Skip to content

Commit 6da2b93

Browse files
Mpdreamzrusscam
Mpdreamz
authored and
russcam
committed
Adds versioned benchmarking (#4206)
* Adds versioned benchmarking Allows the benchmarking project to reference Nest.v7 and so makes it easier for us to benchmark proposed changes. * fix compilation errors and temporarily disable dependency on #4202 (cherry picked from commit 16d8a08)
1 parent 1902aca commit 6da2b93

File tree

13 files changed

+174
-50
lines changed

13 files changed

+174
-50
lines changed

‎nuget.config

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
<packageSources>
44
<add key="nuget.org" value="https://www.nuget.org/api/v2/" />
55
<add key="Elastic Abstractions CI" value="https://ci.appveyor.com/nuget/elasticsearch-net-abstractions" />
6+
<add key="Versioned Clients" value="https://ci.appveyor.com/nuget/elasticsearch-net" />
67
<add key="AssemblyRewriter" value="https://ci.appveyor.com/nuget/assemblyrewriter" />
78
<add key="AssemblyDiffer" value="https://ci.appveyor.com/nuget/assemblydiffer" />
89
</packageSources>

‎src/Elasticsearch.Net/Providers/MemoryStreamFactory.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ namespace Elasticsearch.Net
88
/// </summary>
99
public class MemoryStreamFactory : IMemoryStreamFactory
1010
{
11+
public static MemoryStreamFactory Default { get; } = new MemoryStreamFactory();
12+
1113
/// <inheritdoc />
1214
public MemoryStream Create() => new MemoryStream();
1315

‎src/Tests/Tests.Benchmarking/BenchmarkProgram.cs

Lines changed: 18 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
using System.IO;
44
using System.Linq;
55
using System.Reflection;
6+
using System.Runtime.InteropServices;
67
using BenchmarkDotNet.Attributes;
78
using BenchmarkDotNet.Configs;
89
using BenchmarkDotNet.Diagnosers;
@@ -30,21 +31,21 @@ static Program()
3031
if (!Directory.Exists(Path.Combine(dirInfo.FullName, ".git"))) Environment.Exit(2);
3132

3233
Console.WriteLine(dirInfo.FullName);
33-
using (var repos = new Repository(dirInfo.FullName))
34-
{
35-
Commit = repos.Head.Tip.Sha;
36-
CommitMessage = repos.Head.Tip.Message?.Trim(' ', '\t', '\r', '\n');
37-
Branch = repos.Head.FriendlyName;
38-
var remoteName = repos.Head.RemoteName;
39-
Repository =
40-
repos.Network.Remotes.FirstOrDefault(r => r.Name == remoteName)?.Url
41-
?? repos.Network.Remotes.FirstOrDefault()?.Url;
42-
}
34+
// using (var repos = new Repository(dirInfo.FullName))
35+
// {
36+
// Commit = repos.Head.Tip.Sha;
37+
// CommitMessage = repos.Head.Tip.Message?.Trim(' ', '\t', '\r', '\n');
38+
// Branch = repos.Head.FriendlyName;
39+
// var remoteName = repos.Head.RemoteName;
40+
// Repository =
41+
// repos.Network.Remotes.FirstOrDefault(r => r.Name == remoteName)?.Url
42+
// ?? repos.Network.Remotes.FirstOrDefault()?.Url;
43+
// }
4344
}
4445

4546
public static int Main(string[] arguments)
4647
{
47-
Console.WriteLine($"Tests.Benchmarking: [{Branch}]@({Commit}) on {Repository} : {CommitMessage} - ");
48+
//Console.WriteLine($"Tests.Benchmarking: [{Branch}]@({Commit}) on {Repository} : {CommitMessage} - ");
4849
var config = CreateDefaultConfig();
4950
if (arguments.Any() && arguments[0].Equals("--all", StringComparison.OrdinalIgnoreCase))
5051
{
@@ -61,14 +62,15 @@ public static int Main(string[] arguments)
6162

6263
private static IConfig CreateDefaultConfig()
6364
{
64-
var jobs = new[]
65+
var jobs = new List<Job>
6566
{
66-
Job.ShortRun.With(Runtime.Core).With(Jit.RyuJit),
67-
Job.ShortRun.With(Runtime.Clr).With(Jit.RyuJit),
68-
Job.ShortRun.With(Runtime.Clr).With(Jit.LegacyJit),
67+
Job.MediumRun.With(CoreRuntime.Core30).With(Jit.RyuJit),
6968
};
69+
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
70+
jobs.Add(Job.MediumRun.With(ClrRuntime.Net472).With(Jit.LegacyJit));
71+
7072
var config = DefaultConfig.Instance
71-
.With(jobs)
73+
.With(jobs.ToArray())
7274
.With(MemoryDiagnoser.Default);
7375
return config;
7476
}

‎src/Tests/Tests.Benchmarking/BulkBenchmarkTests.cs

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
using System.Collections.Generic;
2+
using System.Linq;
3+
using BenchmarkDotNet.Attributes;
4+
using Elasticsearch.Net;
5+
using Nest;
6+
using Tests.Benchmarking.Framework;
7+
using Tests.Core.Client;
8+
using Tests.Domain;
9+
10+
namespace Tests.Benchmarking
11+
{
12+
[BenchmarkConfig(5)]
13+
public class BulkBenchmarkTests
14+
{
15+
private static readonly IList<Project> Projects = Project.Generator.Clone().Generate(10000);
16+
private static readonly byte[] Response = TestClient.DefaultInMemoryClient.ConnectionSettings.RequestResponseSerializer.SerializeToBytes(ReturnBulkResponse(Projects));
17+
18+
private static readonly IElasticClient Client =
19+
new ElasticClient(new ConnectionSettings(new InMemoryConnection(Response, 200, null, null))
20+
.DefaultIndex("index")
21+
.EnableHttpCompression(false)
22+
);
23+
24+
//TODO can uncomment when https://github.com/elastic/elasticsearch-net/pull/4202 lands
25+
// private static readonly IElasticClient ClientNoRecyclableMemory =
26+
// new ElasticClient(new ConnectionSettings(new InMemoryConnection(Response, 200, null, null))
27+
// .DefaultIndex("index")
28+
// .EnableHttpCompression(false)
29+
// .MemoryStreamFactory(MemoryStreamFactory.Default)
30+
// );
31+
32+
private static readonly Nest7.IElasticClient ClientV7 =
33+
new Nest7.ElasticClient(new Nest7.ConnectionSettings(
34+
new Elasticsearch.Net7.InMemoryConnection(Response, 200, null, null))
35+
.DefaultIndex("index")
36+
.EnableHttpCompression(false)
37+
);
38+
39+
[GlobalSetup]
40+
public void Setup() { }
41+
42+
[Benchmark(Description = "PR")]
43+
public BulkResponse NestUpdatedBulk() => Client.Bulk(b => b.IndexMany(Projects));
44+
45+
// [Benchmark(Description = "PR no recyclable")]
46+
// public BulkResponse NoRecyclableMemory() => ClientNoRecyclableMemory.Bulk(b => b.IndexMany(Projects));
47+
48+
[Benchmark(Description = "7.x")]
49+
public Nest7.BulkResponse NestCurrentBulk() => ClientV7.Bulk(b => b.IndexMany(Projects));
50+
51+
private static object BulkItemResponse(Project project) => new
52+
{
53+
index = new
54+
{
55+
_index = "nest-52cfd7aa",
56+
_id = project.Name,
57+
_version = 1,
58+
_shards = new
59+
{
60+
total = 2,
61+
successful = 1,
62+
failed = 0
63+
},
64+
created = true,
65+
status = 201
66+
}
67+
};
68+
69+
private static object ReturnBulkResponse(IList<Project> projects) => new
70+
{
71+
took = 276,
72+
errors = false,
73+
items = projects
74+
.Select(p => BulkItemResponse(p))
75+
.ToArray()
76+
};
77+
}
78+
}

‎src/Tests/Tests.Benchmarking/Tests.Benchmarking.csproj

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
<PropertyGroup>
44
<TargetFrameworks>netcoreapp3.0;net472</TargetFrameworks>
55
<OutputType>Exe</OutputType>
6+
<TieredCompilation>false</TieredCompilation>
67
</PropertyGroup>
78
<PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">
89
<Optimize>true</Optimize>
@@ -11,8 +12,9 @@
1112
<ProjectReference Include="..\Tests.Core\Tests.Core.csproj" />
1213
</ItemGroup>
1314
<ItemGroup>
14-
<PackageReference Include="BenchmarkDotNet" Version="0.11.5" />
15+
<PackageReference Include="BenchmarkDotNet" Version="0.12.0" />
1516
<PackageReference Include="Elastic.BenchmarkDotNetExporter" Version="0.1.0-ci20191017T152836" />
1617
<PackageReference Include="LibGit2Sharp" Version="0.26.0-preview-0062" />
18+
<PackageReference Include="NEST.v7" Version="7.5.0-ci20191031T180108" />
1719
</ItemGroup>
1820
</Project>

‎src/Tests/Tests.Core/Client/Settings/AlwaysInMemoryConnectionSettings.cs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ public class AlwaysInMemoryConnectionSettings : TestConnectionSettings
1313
{
1414
public AlwaysInMemoryConnectionSettings() : base(forceInMemory: true) { }
1515

16+
public AlwaysInMemoryConnectionSettings(byte[] bytes) : base(forceInMemory: true, response: bytes) { }
17+
1618
public AlwaysInMemoryConnectionSettings(
1719
Func<ICollection<Uri>, IConnectionPool> createPool = null,
1820
SourceSerializerFactory sourceSerializerFactory = null,
@@ -23,7 +25,7 @@ public AlwaysInMemoryConnectionSettings(
2325
createPool,
2426
sourceSerializerFactory,
2527
propertyMappingProvider,
26-
true,
28+
forceInMemory: true,
2729
port
2830
) { }
2931
}

‎src/Tests/Tests.Core/Client/Settings/TestConnectionSettings.cs

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,12 @@ public TestConnectionSettings(
2121
SourceSerializerFactory sourceSerializerFactory = null,
2222
IPropertyMappingProvider propertyMappingProvider = null,
2323
bool forceInMemory = false,
24-
int port = 9200
24+
int port = 9200,
25+
byte[] response = null
2526
)
2627
: base(
2728
CreatePool(createPool, port),
28-
TestConfiguration.Instance.CreateConnection(forceInMemory),
29+
TestConfiguration.Instance.CreateConnection(forceInMemory, response),
2930
CreateSerializerFactory(sourceSerializerFactory),
3031
propertyMappingProvider
3132
) =>
@@ -40,7 +41,7 @@ public TestConnectionSettings(
4041

4142
private static string LocalHost => "localhost";
4243

43-
private void ApplyTestSettings() =>
44+
private void ApplyTestSettings() =>
4445
RerouteToProxyIfNeeded()
4546
.EnableDebugMode()
4647
.EnableHttpCompression(TestConfiguration.Instance.Random.HttpCompression)

‎src/Tests/Tests.Core/Client/TestClient.cs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,12 @@ public static class TestClient
1111
public static readonly TestConfigurationBase Configuration = TestConfiguration.Instance;
1212
public static readonly IElasticClient Default = new ElasticClient(new TestConnectionSettings().ApplyDomainSettings());
1313
public static readonly IElasticClient DefaultInMemoryClient = new ElasticClient(new AlwaysInMemoryConnectionSettings().ApplyDomainSettings());
14+
public static IElasticClient FixedInMemoryClient(byte[] response) => new ElasticClient(
15+
new AlwaysInMemoryConnectionSettings(response)
16+
.ApplyDomainSettings()
17+
.DisableDirectStreaming()
18+
.EnableHttpCompression(false)
19+
);
1420

1521
public static readonly IElasticClient DisabledStreaming =
1622
new ElasticClient(new TestConnectionSettings().ApplyDomainSettings().DisableDirectStreaming());

‎src/Tests/Tests.Core/Extensions/TestConfigurationExtensions.cs

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,12 @@ namespace Tests.Core.Extensions
77
{
88
public static class TestConfigurationExtensions
99
{
10-
public static IConnection CreateConnection(this TestConfigurationBase configuration, bool forceInMemory = false) =>
11-
configuration.RunIntegrationTests && !forceInMemory ? (IConnection)new HttpConnection() : new InMemoryConnection();
10+
public static IConnection CreateConnection(this TestConfigurationBase configuration, bool forceInMemory = false, byte[] response = null) =>
11+
forceInMemory
12+
? new InMemoryConnection(response)
13+
: configuration.RunIntegrationTests
14+
? (IConnection)new HttpConnection()
15+
: new InMemoryConnection(response);
1216

1317
public static bool InRange(this TestConfigurationBase configuration, string range) =>
1418
configuration.ElasticsearchVersion.InRange(range);

‎src/Tests/Tests.Domain/Project.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ public class Project
7878
.RuleFor(p => p.DateString, (p, d) => d.StartedOn.ToString("yyyy-MM-ddTHH\\:mm\\:ss.fffffffzzz"))
7979
.RuleFor(p => p.LastActivity, p => p.Date.Recent())
8080
.RuleFor(p => p.LeadDeveloper, p => Developer.Developers[Gimme.Random.Number(0, Developer.Developers.Count - 1)])
81-
.RuleFor(p => p.Tags, f => Tag.Generator.Generate(Gimme.Random.Number(2, 50)))
81+
.RuleFor(p => p.Tags, f => Tag.Generator.Generate(Gimme.Random.Number(2, 10)))
8282
.RuleFor(p => p.CuratedTags, f => Tag.Generator.Generate(Gimme.Random.Number(1, 5)))
8383
.RuleFor(p => p.LocationPoint, f => SimpleGeoPoint.Generator.Generate())
8484
.RuleFor(p => p.LocationShape, f => new PointGeoShape(new GeoCoordinate(f.Address.Latitude(), f.Address.Latitude())))

‎src/Tests/Tests.ScratchPad/Program.cs

Lines changed: 49 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,13 @@
33
using System.Diagnostics;
44
using System.Linq;
55
using System.Threading.Tasks;
6+
using BenchmarkDotNet.Attributes;
67
using BenchmarkDotNet.Running;
78
using Elasticsearch.Net;
89
using Elasticsearch.Net.Diagnostics;
910
using Nest;
11+
using Tests.Core.Client;
12+
using Tests.Domain;
1013
using Xunit.Sdk;
1114

1215
namespace Tests.ScratchPad
@@ -46,33 +49,60 @@ void WriteToConsole<T>(string eventName, T data)
4649
}
4750
}
4851

49-
private static async Task Main(string[] args)
50-
{
51-
DiagnosticListener.AllListeners.Subscribe(new ListenerObserver());
52-
53-
using (var node = new Elastic.Managed.Ephemeral.EphemeralCluster("7.0.0"))
54-
{
55-
node.Start();
52+
private static readonly IList<Project> Projects = Project.Generator.Clone().Generate(10000);
53+
private static readonly byte[] Response = TestClient.DefaultInMemoryClient.ConnectionSettings.RequestResponseSerializer.SerializeToBytes(ReturnBulkResponse(Projects));
5654

57-
var settings = new ConnectionSettings(new StaticConnectionPool(new[] { node.NodesUris("ipv4.fiddler").First() }))
58-
.EnableHttpCompression()
59-
.Proxy(new Uri("http://127.0.0.1:8080"), (string)null, (string)null)
60-
;
61-
var client = new ElasticClient(settings);
55+
private static readonly IElasticClient Client =
56+
new ElasticClient(new ConnectionSettings(new InMemoryConnection(Response, 200, null, null))
57+
.DefaultIndex("index")
58+
.EnableHttpCompression(false)
59+
);
6260

63-
var x = client.Search<object>(s=>s.AllIndices());
6461

65-
await Task.Delay(TimeSpan.FromSeconds(7));
66-
67-
Console.WriteLine(new string('-', Console.WindowWidth - 1));
62+
private static async Task Main(string[] args)
63+
{
64+
Console.Write($"Warmup...");
65+
var response = Client.Bulk(b => b.IndexMany(Projects));
66+
Console.WriteLine("\rWarmed up kicking off in 2 seconds!");
6867

69-
var y = client.Search<object>(s=>s.Index("does-not-exist"));
68+
await Task.Delay(TimeSpan.FromSeconds(2));
69+
Console.WriteLine($"Kicking off");
7070

71-
await Task.Delay(TimeSpan.FromSeconds(7));
71+
for (var i = 0; i < 10_000; i++)
72+
{
73+
var r = Client.Bulk(b => b.IndexMany(Projects));
74+
Console.Write($"\r{i}: {r.IsValid} {r.Items.Count}");
75+
}
76+
}
7277

7378

79+
private static object BulkItemResponse(Project project) => new
80+
{
81+
index = new
82+
{
83+
_index = "nest-52cfd7aa",
84+
_type = "_doc",
85+
_id = project.Name,
86+
_version = 1,
87+
_shards = new
88+
{
89+
total = 2,
90+
successful = 1,
91+
failed = 0
92+
},
93+
created = true,
94+
status = 201
7495
}
75-
}
96+
};
97+
98+
private static object ReturnBulkResponse(IList<Project> projects) => new
99+
{
100+
took = 276,
101+
errors = false,
102+
items = projects
103+
.Select(p => BulkItemResponse(p))
104+
.ToArray()
105+
};
76106

77107
private static void Bench<TBenchmark>() where TBenchmark : RunBase => BenchmarkRunner.Run<TBenchmark>();
78108

‎src/Tests/Tests.ScratchPad/Tests.ScratchPad.csproj

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,12 @@
33
<PropertyGroup>
44
<OutputType>Exe</OutputType>
55
<TargetFramework>netcoreapp3.0</TargetFramework>
6-
</PropertyGroup>
7-
<PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">
86
<Optimize>true</Optimize>
7+
<DebugSymbols>true</DebugSymbols>
98
</PropertyGroup>
109
<ItemGroup>
1110
<PackageReference Include="System.Diagnostics.DiagnosticSource" Version="4.5.1" />
1211
<ProjectReference Include="..\Tests.Core\Tests.Core.csproj" />
1312
<PackageReference Include="BenchMarkDotNet" Version="0.11.5" />
1413
</ItemGroup>
15-
</Project>
14+
</Project>

‎src/Tests/Tests/ClientConcepts/Connection/HttpConnectionTests.cs

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -160,10 +160,7 @@ public class TestableHttpConnection : HttpConnection
160160
public int ClientCount => Clients.Count;
161161
public HttpClientHandler LastHttpClientHandler => (HttpClientHandler)_handler.InnerHandler;
162162

163-
public TestableHttpConnection(Action<HttpResponseMessage> response)
164-
{
165-
_response = response;
166-
}
163+
public TestableHttpConnection(Action<HttpResponseMessage> response) => _response = response;
167164

168165
public TestableHttpConnection()
169166
{

0 commit comments

Comments
 (0)