Skip to content

Commit 6443536

Browse files
Stuart Camcodebrain
Stuart Cam
authored and
codebrain
committed
Benchmarking implementation (#2813)
Added benchmarking capabilities; - Control over run counts - Collects memory diagnostics - Store results in an ES instance (cherry picked from commit eca4335)
1 parent 98ae32b commit 6443536

17 files changed

+1102
-328
lines changed

‎build/scripts/Benchmarking.fsx

Lines changed: 217 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
1-
#I @"../../packages/build/FAKE/tools"
1+
#r "../../packages/build/NEST/lib/net45/Nest.dll"
2+
#r "../../packages/build/Elasticsearch.Net/lib/net45/Elasticsearch.Net.dll"
3+
#r "../../packages/build/Newtonsoft.Json/lib/net45/Newtonsoft.Json.dll"
4+
#r "../../packages/build/FSharp.Data/lib/net40/FSharp.Data.dll"
5+
#I @"../../packages/build/FAKE/tools"
26
#r @"FakeLib.dll"
7+
#nowarn "0044" //TODO sort out FAKE 5
8+
39
open Fake
410

511
#load @"Paths.fsx"
@@ -10,26 +16,131 @@ open System.Linq
1016
open System.Diagnostics
1117
open Paths
1218

19+
open FSharp.Data
20+
21+
open Nest
22+
open Elasticsearch.Net
23+
open Newtonsoft.Json
24+
open Git.Branches
25+
1326
module Benchmarker =
14-
let private testsProjectDirectory = Path.GetFullPath(Paths.Source("Tests"))
15-
let private benchmarkOutput = Path.GetFullPath(Paths.Output("benchmarks")) |> directoryInfo
1627

17-
let private copyToOutput file = CopyFile benchmarkOutput.FullName file
28+
let pipelineName = "benchmark-pipeline"
29+
let indexName = IndexName.op_Implicit("benchmark-reports")
30+
let typeName = TypeName.op_Implicit("benchmarkreport")
31+
32+
type Memory(gen0Collections:int, gen1Collections: int, gen2Collections: int, totalOperations:int64, bytesAllocatedPerOperation:int64) =
33+
member val Gen0Collections=gen0Collections with get, set
34+
member val Gen1Collections=gen1Collections with get, set
35+
member val Gen2Collections=gen2Collections with get, set
36+
member val TotalOperations=totalOperations with get, set
37+
member val BytesAllocatedPerOperation=bytesAllocatedPerOperation with get, set
38+
39+
type ChronometerFrequency(hertz:double) =
40+
member val Hertz=hertz with get, set
41+
42+
type HostEnvironmentInfo(benchmarkDotNetCaption:string, benchmarkDotNetVersion:string, osVersion: string, processorName:string,
43+
processorCount:int, runtimeVersion:string, architecture:string, hasAttachedDebugger:bool, hasRyuJit:bool,
44+
configuration:string, jitModules:string, dotnetCliVersion:string, chronometerFrequency:ChronometerFrequency,
45+
hardwareTimerKind:string) =
46+
member val BenchmarkDotNetCaption=benchmarkDotNetCaption with get, set
47+
member val BenchmarkDotNetVersion=benchmarkDotNetVersion with get, set
48+
member val OsVersion=osVersion with get, set
49+
member val ProcessorName=processorName with get, set
50+
member val ProcessorCount=processorCount with get, set
51+
member val RuntimeVersion=runtimeVersion with get, set
52+
member val Architecture=architecture with get, set
53+
member val HasAttachedDebugger=hasAttachedDebugger with get, set
54+
member val HasRyuJit=hasRyuJit with get, set
55+
member val Configuration=configuration with get, set
56+
member val JitModules=jitModules with get, set
57+
member val DotNetCliVersion=dotnetCliVersion with get, set
58+
member val ChronometerFrequency=chronometerFrequency with get, set
59+
member val HardwareTimerKind=hardwareTimerKind with get, set
60+
61+
type ConfidenceInterval(n:int, mean: double, standardError:double, level:int, margin:double, lower:double, upper:double) =
62+
member val N=n with get, set
63+
member val Mean=mean with get, set
64+
member val StandardError=standardError with get, set
65+
member val Level=level with get, set
66+
member val Margin=margin with get, set
67+
member val Lower=lower with get, set
68+
member val Upper=upper with get, set
69+
70+
type Percentiles (p0:double, p25:double, p50:double, p67:double, p80:double, p85:double, p90:double, p95:double, p100:double) =
71+
member val P0=p0 with get, set
72+
member val P25=p25 with get, set
73+
member val P50=p50 with get, set
74+
member val P67=p67 with get, set
75+
member val P80=p80 with get, set
76+
member val P85=p85 with get, set
77+
member val P90=p90 with get, set
78+
member val P95=p95 with get, set
79+
member val P100=p100 with get, set
80+
81+
type Statistics(n:int, min:double, lowerFence:double, q1:double, median:double, mean:double, q3:double, upperFence:double, max:double,
82+
interquartileRange:double, outliers:double list, standardError:double, variance:double, standardDeviation:double,
83+
skewness:double, kurtosis:double, confidenceInterval:ConfidenceInterval, percentiles:Percentiles) =
84+
member val N=n with get, set
85+
member val Min=min with get, set
86+
member val LowerFence=lowerFence with get, set
87+
member val Q1=q1 with get, set
88+
member val Median=median with get, set
89+
member val Mean=mean with get, set
90+
member val Q3=q3 with get, set
91+
member val UpperFence=upperFence with get, set
92+
member val Max=max with get, set
93+
member val InterquartileRange=interquartileRange with get, set
94+
member val Outliers=outliers with get, set
95+
member val StandardError=standardError with get, set
96+
member val Variance=variance with get, set
97+
member val StandardDeviation=standardDeviation with get, set
98+
member val Skewness=skewness with get, set
99+
member val Kurtosis=kurtosis with get, set
100+
member val ConfidenceInterval=confidenceInterval with get, set
101+
member val Percentiles=percentiles with get, set
102+
103+
type Benchmark(displayInfo:string, namespyce:string, tipe:string, method:string, methodTitle:string, parameters:string,
104+
statistics:Statistics, memory:Memory) =
105+
member val DisplayInfo=displayInfo with get, set
106+
member val Namespace=namespyce with get, set
107+
member val Type=tipe with get, set
108+
member val Method=method with get, set
109+
member val MethodTitle=methodTitle with get, set
110+
member val Parameters=parameters with get, set
111+
member val Statistics=statistics with get, set
112+
member val Memory=memory with get, set
113+
114+
type BenchmarkReport(title: string, totalTime:TimeSpan, date:DateTime, commit:string, host:HostEnvironmentInfo, benchmarks:Benchmark list) =
115+
member val Title = title with get, set
116+
member val TotalTime = totalTime with get, set
117+
member val Date = date with get, set
118+
member val Commit = commit with get, set
119+
member val HostEnvironmentInfo = host with get, set
120+
member val Benchmarks = benchmarks with get, set
121+
122+
let private testsProjectDirectory = Path.GetFullPath(Paths.Source("Tests"))
123+
let private benchmarkOutput = Path.GetFullPath(Paths.Output("benchmarks")) |> directoryInfo
124+
let private copyToOutput file = CopyFile benchmarkOutput.FullName file
125+
126+
let Run(runInteractive:bool) =
18127

19-
let Run() =
20128
ensureDirExists benchmarkOutput
129+
21130
let projectJson = testsProjectDirectory @@ "project.json"
131+
22132
// running benchmarks can timeout so clean up any generated benchmark files
23133
try
24-
DotNetCli.Restore(fun p ->
25-
{ p with
26-
Project = projectJson
27-
})
28-
29-
DotNetCli.RunCommand(fun p ->
30-
{ p with
31-
WorkingDir = testsProjectDirectory
32-
}) "run Benchmark"
134+
if runInteractive then
135+
DotNetCli.RunCommand(fun p ->
136+
{ p with
137+
WorkingDir = testsProjectDirectory
138+
}) "run -f net46 -c Release Benchmark"
139+
else
140+
DotNetCli.RunCommand(fun p ->
141+
{ p with
142+
WorkingDir = testsProjectDirectory
143+
}) "run -f net46 -c Release Benchmark non-interactive"
33144
finally
34145
let benchmarkOutputFiles =
35146
let output = combinePaths testsProjectDirectory "BenchmarkDotNet.Artifacts"
@@ -38,3 +149,95 @@ module Benchmarker =
38149

39150
for file in benchmarkOutputFiles do copyToOutput file
40151
DeleteFiles benchmarkOutputFiles
152+
153+
let IndexResult (client:ElasticClient, file:string, date:DateTime, commit:string, indexName, typeName) =
154+
155+
trace (sprintf "Indexing report %s into Elasticsearch" file)
156+
157+
let document = JsonConvert.DeserializeObject<BenchmarkReport>(File.ReadAllText(file))
158+
document.Date <- date
159+
document.Commit <- commit
160+
161+
let indexRequest = new IndexRequest<BenchmarkReport>(indexName, typeName)
162+
indexRequest.Document <- document
163+
indexRequest.Pipeline <- pipelineName
164+
165+
let indexResponse = client.Index(indexRequest)
166+
167+
if indexResponse.IsValid = false then
168+
raise (Exception("Unable to index report into Elasticsearch: " + indexResponse.ServerError.Error.ToString()))
169+
170+
let IndexResults (url, username, password) =
171+
if (String.IsNullOrEmpty url = false) then
172+
trace "Indexing benchmark reports into Elasticsearch"
173+
174+
let date = DateTime.UtcNow
175+
let commit = getSHA1 "." "HEAD"
176+
177+
let benchmarkJsonFiles =
178+
Directory.EnumerateFiles(benchmarkOutput.FullName, "*-custom.json", SearchOption.AllDirectories)
179+
|> Seq.toList
180+
181+
let uri = new Uri(url)
182+
let connectionSettings = new ConnectionSettings(uri);
183+
184+
if (String.IsNullOrEmpty username = false && String.IsNullOrEmpty password = false) then
185+
connectionSettings.BasicAuthentication(username, password) |> ignore
186+
187+
let client = new ElasticClient(connectionSettings)
188+
189+
let indexTemplateExists = client.IndexTemplateExists(Name.op_Implicit("benchmarks")).Exists
190+
191+
if indexTemplateExists |> not then
192+
193+
let typeMapping = new TypeMappingDescriptor<BenchmarkReport>()
194+
typeMapping.AutoMap() |> ignore
195+
typeMapping.Properties(fun p ->
196+
p.Nested<Benchmark>(fun n ->
197+
n.AutoMap().Name(PropertyName.op_Implicit("benchmarks")) :> INestedProperty
198+
) :> IPromise<IProperties>
199+
) |> ignore
200+
201+
let mappings = new Mappings()
202+
mappings.Add(typeName, typeMapping :> ITypeMapping)
203+
204+
let indexSettings = new IndexSettings()
205+
indexSettings.NumberOfShards <- Nullable 1
206+
207+
let putIndexTemplateRequest = new PutIndexTemplateRequest(Name.op_Implicit("benchmarks"))
208+
putIndexTemplateRequest.Template <- "benchmark-reports-*"
209+
putIndexTemplateRequest.Mappings <- mappings
210+
putIndexTemplateRequest.Settings <- indexSettings
211+
212+
let putIndexTemplateResponse = client.PutIndexTemplate(putIndexTemplateRequest)
213+
214+
if putIndexTemplateResponse.IsValid = false then
215+
raise (Exception("Unable to create index template into Elasticsearch"))
216+
217+
let processor = new GrokProcessor();
218+
processor.Field <- new Field("_ingest._value.displayInfo")
219+
processor.Patterns <- ["%{WORD:_ingest._value.class}.%{DATA:_ingest._value.method}: Job-%{WORD:_ingest._value.jobName}\\(Jit=%{WORD:_ingest._value.jit}, Runtime=%{WORD:_ingest._value.clr}, LaunchCount=%{NUMBER:_ingest._value.launchCount}, RunStrategy=%{WORD:_ingest._value.runStrategy}, TargetCount=%{NUMBER:_ingest._value.targetCount}, UnrollFactor=%{NUMBER:_ingest._value.unrollFactor}, WarmupCount=%{NUMBER:_ingest._value.warmupCount}\\)"]
220+
221+
let forEachProcessor = new ForeachProcessor()
222+
forEachProcessor.Field <- new Field("benchmarks")
223+
forEachProcessor.Processor <- processor
224+
225+
let dateIndexProcessor = new DateIndexNameProcessor();
226+
dateIndexProcessor.Field <- new Field("date")
227+
dateIndexProcessor.IndexNamePrefix <- "benchmark-reports-"
228+
dateIndexProcessor.DateRounding <- DateRounding.Month
229+
dateIndexProcessor.DateFormats <- ["yyyy-MM-dd'T'HH:mm:ss.SSSSSSSZ"]
230+
231+
let request = new PutPipelineRequest(Id.op_Implicit(pipelineName))
232+
request.Description <- "Benchmark settings pipeline"
233+
request.Processors <- [dateIndexProcessor; forEachProcessor]
234+
235+
let createPipeline = client.PutPipeline(request)
236+
237+
if createPipeline.IsValid = false then
238+
raise (Exception("Unable to create pipeline"))
239+
240+
for file in benchmarkJsonFiles
241+
do IndexResult (client, file, date, commit, indexName, typeName)
242+
243+
trace "Indexed benchmark reports into Elasticsearch"

‎build/scripts/Building.fsx

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -71,12 +71,8 @@ module Build =
7171
compileCore incremental
7272

7373
let Clean() =
74-
match (quickBuild, getBuildParam "target" = "clean") with
75-
| (false, _)
76-
| (_, true) ->
77-
tracefn "Cleaning known output folders"
78-
CleanDir Paths.BuildOutput
79-
DotNetCli.RunCommand (fun p -> { p with TimeOut = TimeSpan.FromMinutes(3.) }) "clean src/Elasticsearch.sln -c Release" |> ignore
80-
DotNetProject.All |> Seq.iter(fun p -> CleanDir(Paths.BinFolder p.Name))
81-
| (_, _) ->
82-
tracefn "Skipping clean target only run when calling 'release', 'canary', 'clean' as targets directly"
74+
tracefn "Cleaning known output folders"
75+
CleanDir Paths.BuildOutput
76+
DotNetCli.RunCommand (fun p -> { p with TimeOut = TimeSpan.FromMinutes(3.) }) "clean src/Elasticsearch.sln -c Release" |> ignore
77+
DotNetProject.All |> Seq.iter(fun p -> CleanDir(Paths.BinFolder p.Name))
78+

‎build/scripts/Commandline.fsx

Lines changed: 31 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -62,21 +62,48 @@ module Commandline =
6262
match filteredArgs with
6363
| _ :: tail -> target :: tail
6464
| [] -> [target]
65+
66+
let private (|IsUrl|_|) (candidate:string) =
67+
match Uri.TryCreate(candidate, UriKind.RelativeOrAbsolute) with
68+
| true, _ -> Some candidate
69+
| _ -> None
6570

6671
let parse () =
6772
setEnvironVar "FAKEBUILD" "1"
6873
printfn "%A" arguments
6974
match arguments with
70-
| [] | ["build"] | ["test"] | ["clean"] -> ignore()
75+
| [] | ["build"] | ["test"] | ["clean"] | ["benchmark"] | ["profile"] -> ignore()
7176
| ["release"; version] -> setBuildParam "version" version
7277

7378
| ["test"; testFilter] -> setBuildParam "testfilter" testFilter
7479

75-
| ["profile"; esVersions] -> setBuildParam "esversions" esVersions
76-
| ["profile"; esVersions; testFilter] ->
80+
| ["benchmark"; IsUrl elasticsearch; username; password] ->
81+
setBuildParam "elasticsearch" elasticsearch
82+
setBuildParam "nonInteractive" "0"
83+
setBuildParam "username" username
84+
setBuildParam "password" password
85+
86+
| ["benchmark"; IsUrl elasticsearch] ->
87+
setBuildParam "elasticsearch" elasticsearch
88+
setBuildParam "nonInteractive" "0"
89+
90+
| ["benchmark"; IsUrl elasticsearch; "non-interactive"] ->
91+
setBuildParam "elasticsearch" elasticsearch
92+
setBuildParam "nonInteractive" "1"
93+
94+
| ["benchmark"; "non-interactive"] ->
95+
setBuildParam "nonInteractive" "1"
96+
97+
| ["profile"; IsUrl elasticsearch] ->
98+
setBuildParam "elasticsearch" elasticsearch
99+
100+
| ["profile"; esVersions] ->
77101
setBuildParam "esversions" esVersions
78-
setBuildParam "testfilter" testFilter
79102

103+
| ["profile"; esVersions; testFilter] ->
104+
setBuildParam "esversions" esVersions
105+
setBuildParam "testfilter" testFilter
106+
80107
| ["integrate"; esVersions] -> setBuildParam "esversions" esVersions
81108
| ["integrate"; esVersions; clusterFilter] ->
82109
setBuildParam "esversions" esVersions

‎build/scripts/Profiling.fsx

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@ open Tooling
3131

3232
module Profiler =
3333

34+
let indexName = IndexName.op_Implicit("profiling-reports")
35+
let typeName = TypeName.op_Implicit("report")
36+
3437
type Profile = XmlProvider<"../../build/profiling/profile-example.xml">
3538

3639
type Function(id:string, fqn:string, totalTime:int, ownTime:int, calls:int, instances:int) =
@@ -47,8 +50,8 @@ module Profiler =
4750
member val Commit = commit with get, set
4851
member val Functions = functions with get, set
4952

50-
let private project = "Tests"
51-
let private profiledApp = sprintf "%s/%s/%s.exe" (Paths.Output("v4.6")) project project
53+
let private project = PrivateProject(Tests).Name
54+
let private profiledApp = sprintf "%s/net46/%s.exe" (Paths.Output("Tests")) project
5255
let private snapShotOutput = Paths.Output("ProfilingSnapshot.dtp")
5356
let private snapShotStatsOutput = Paths.Output("ProfilingSnapshotStats.html")
5457
let private profileOutput = Paths.Output("ProfilingReport.xml")
@@ -62,7 +65,8 @@ module Profiler =
6265

6366
let Run() =
6467
let date = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffz")
65-
Tooling.execProcessWithTimeout profiledApp ["Profile";"Class";getBuildParam "testfilter"] (TimeSpan.FromMinutes 30.) |> ignore
68+
trace profiledApp
69+
Tooling.execProcessWithTimeout profiledApp ["Profile";"Class";getBuildParam "testfilter"] (TimeSpan.FromMinutes 30.) "." |> ignore
6670
trace "Profiling finished."
6771

6872
let performanceOutput = Paths.Output("profiling/performance") |> directoryInfo
@@ -121,8 +125,6 @@ module Profiler =
121125

122126
let reportDoc = new Report(report.Name, report.Date, report.Commit, functions)
123127

124-
let indexName = IndexName.op_Implicit("reports")
125-
let typeName = TypeName.op_Implicit("report")
126128
let indexExists = client.IndexExists(Indices.op_Implicit(indexName)).Exists
127129

128130
if indexExists = false then
@@ -142,7 +144,6 @@ module Profiler =
142144
if createIndex.IsValid = false then
143145
raise (Exception("Unable to create index into Elasticsearch"))
144146

145-
146147
let indexRequest = new IndexRequest<Report>(indexName, typeName)
147148
indexRequest.Document <- reportDoc
148149

‎build/scripts/Targets.fsx

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,13 @@ Target "Profile" <| fun _ ->
4747

4848
Target "Integrate" <| Tests.RunIntegrationTests
4949

50-
Target "Benchmark" Benchmarker.Run
50+
Target "Benchmark" <| fun _ ->
51+
let runInteractive = ((getBuildParam "nonInteractive") <> "1")
52+
Benchmarker.Run(runInteractive)
53+
let url = getBuildParam "elasticsearch"
54+
let username = getBuildParam "username"
55+
let password = getBuildParam "password"
56+
Benchmarker.IndexResults (url, username, password)
5157

5258
Target "InheritDoc" InheritDoc.PatchInheritDocs
5359

0 commit comments

Comments
 (0)