Skip to content

Commit f608764

Browse files
codebraincodebrain
committed
Intermediate checkin. Benchmarking results are now imported into Elasticsearch.
1 parent 494582b commit f608764

File tree

12 files changed

+464
-289
lines changed

12 files changed

+464
-289
lines changed

‎build/scripts/Benchmarking.fsx

Lines changed: 179 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
1-
#I @"../../packages/build/FAKE/tools"
1+
#r "../../packages/build/NEST/lib/net45/Nest.dll"
2+
#r "../../packages/build/Elasticsearch.Net/lib/net45/Elasticsearch.Net.dll"
3+
#r "../../packages/build/Newtonsoft.Json/lib/net45/Newtonsoft.Json.dll"
4+
#r "../../packages/build/FSharp.Data/lib/net40/FSharp.Data.dll"
5+
#I @"../../packages/build/FAKE/tools"
26
#r @"FakeLib.dll"
7+
#nowarn "0044" //TODO sort out FAKE 5
8+
39
open Fake
410

511
#load @"Paths.fsx"
@@ -10,20 +16,113 @@ open System.Linq
1016
open System.Diagnostics
1117
open Paths
1218

19+
open FSharp.Data
20+
21+
open Nest
22+
open Elasticsearch.Net
23+
open Newtonsoft.Json
24+
open Git.Branches
25+
1326
module Benchmarker =
14-
let private testsProjectDirectory = Path.GetFullPath(Paths.Source("Tests"))
15-
let private benchmarkOutput = Path.GetFullPath(Paths.Output("benchmarks")) |> directoryInfo
1627

17-
let private copyToOutput file = CopyFile benchmarkOutput.FullName file
28+
type ProcessorName(isValueCreated:bool, value:string) =
29+
member val IsValueCreated=isValueCreated with get, set
30+
member val Value=value with get, set
31+
32+
type ChronometerFrequency(hertz:int) =
33+
member val Hertz=hertz with get, set
34+
35+
type HostEnvironmentInfo(benchmarkDotNetCaption:string, benchmarkDotNetVersion:string, osVersion: string, processorName:ProcessorName,
36+
processorCount:int, runtimeVersion:string, architecture:string, hasAttachedDebugger:bool, hasRyuJit:bool,
37+
configuration:string, jitModules:string, dotnetCliVersion:string, chronometerFrequency:ChronometerFrequency,
38+
hardwareTimerKind:string) =
39+
member val BenchmarkDotNetCaption=benchmarkDotNetCaption with get, set
40+
member val BenchmarkDotNetVersion=benchmarkDotNetVersion with get, set
41+
member val OsVersion=osVersion with get, set
42+
member val ProcessorName=processorName with get, set
43+
member val ProcessorCount=processorCount with get, set
44+
member val RuntimeVersion=runtimeVersion with get, set
45+
member val Architecture=architecture with get, set
46+
member val HasAttachedDebugger=hasAttachedDebugger with get, set
47+
member val HasRyuJit=hasRyuJit with get, set
48+
member val Configuration=configuration with get, set
49+
member val JitModules=jitModules with get, set
50+
member val DotNetCliVersion=dotnetCliVersion with get, set
51+
member val ChronometerFrequency=chronometerFrequency with get, set
52+
member val HardwareTimerKind=hardwareTimerKind with get, set
53+
54+
type ConfidenceInterval(mean: double, error:double, level:int, margin:double, lower:double, upper:double) =
55+
member val Mean=mean with get, set
56+
member val Error=error with get, set
57+
member val Level=level with get, set
58+
member val Margin=margin with get, set
59+
member val Lower=lower with get, set
60+
member val Upper=upper with get, set
61+
62+
type Percentiles (p0:double, p25:double, p50:double, p67:double, p80:double, p85:double, p90:double, p95:double, p100:double) =
63+
member val P0=p0 with get, set
64+
member val P25=p25 with get, set
65+
member val P50=p50 with get, set
66+
member val P67=p67 with get, set
67+
member val P80=p80 with get, set
68+
member val P85=p85 with get, set
69+
member val P90=p90 with get, set
70+
member val P95=p95 with get, set
71+
member val P100=p100 with get, set
72+
73+
type Statistics(n:int, min:double, lowerFence:double, q1:double, median:double, mean:double, q3:double, upperFence:double, max:double,
74+
interquartileRange:double, outliers:double list, standardError:double, variance:double, standardDeviation:double,
75+
skewness:double, kurtosis:double, confidenceInterval:ConfidenceInterval, percentiles:Percentiles) =
76+
member val N=n with get, set
77+
member val Min=min with get, set
78+
member val LowerFence=lowerFence with get, set
79+
member val Q1=q1 with get, set
80+
member val Median=median with get, set
81+
member val Mean=mean with get, set
82+
member val Q3=q3 with get, set
83+
member val UpperFence=upperFence with get, set
84+
member val Max=max with get, set
85+
member val InterquartileRange=interquartileRange with get, set
86+
member val Outliers=outliers with get, set
87+
member val StandardError=standardError with get, set
88+
member val Variance=variance with get, set
89+
member val StandardDeviation=standardDeviation with get, set
90+
member val Skewness=skewness with get, set
91+
member val Kurtosis=kurtosis with get, set
92+
member val ConfidenceInterval=confidenceInterval with get, set
93+
member val Percentiles=percentiles with get, set
94+
95+
type Benchmark(displayInfo:string, namespyce:string, tipe:string, method:string, methodTitle:string, parameters:string, statistics:Statistics) =
96+
member val DisplayInfo=displayInfo with get, set
97+
member val Namespace=namespyce with get, set
98+
member val Type=tipe with get, set
99+
member val Method=method with get, set
100+
member val MethodTitle=methodTitle with get, set
101+
member val Parameters=parameters with get, set
102+
member val Statistics=statistics with get, set
103+
104+
type Report(title: string, date:DateTime, commit:string, host:HostEnvironmentInfo, benchmarks:Benchmark list) =
105+
member val Title = title with get, set
106+
member val Date = date with get, set
107+
member val Commit = commit with get, set
108+
member val HostEnvironmentInfo = host with get, set
109+
member val Benchmarks = benchmarks with get, set
110+
111+
let private testsProjectDirectory = Path.GetFullPath(Paths.Source("Tests"))
112+
let private benchmarkOutput = Path.GetFullPath(Paths.Output("benchmarks")) |> directoryInfo
113+
let private copyToOutput file = CopyFile benchmarkOutput.FullName file
114+
115+
let Run() =
18116

19-
let Run() =
20117
ensureDirExists benchmarkOutput
118+
21119
let projectJson = testsProjectDirectory @@ "project.json"
120+
22121
// running benchmarks can timeout so clean up any generated benchmark files
23122
try
24123
DotNetCli.RunCommand(fun p ->
25124
{ p with
26-
WorkingDir = testsProjectDirectory
125+
WorkingDir = testsProjectDirectory
27126
}) "run -f net46 -c Release Benchmark"
28127
finally
29128
let benchmarkOutputFiles =
@@ -33,3 +132,77 @@ module Benchmarker =
33132

34133
for file in benchmarkOutputFiles do copyToOutput file
35134
DeleteFiles benchmarkOutputFiles
135+
136+
let IndexResult (client:ElasticClient, file:string, date:DateTime, commit:string, indexName, typeName) =
137+
138+
let document = JsonConvert.DeserializeObject<Report>(File.ReadAllText(file))
139+
document.Date <- date
140+
document.Commit <- commit
141+
142+
let indexRequest = new IndexRequest<Report>(indexName, typeName)
143+
indexRequest.Document <- document
144+
indexRequest.Pipeline <- "benchmark-pipeline"
145+
146+
let indexResponse = client.Index(indexRequest)
147+
148+
if indexResponse.IsValid = false then
149+
raise (Exception("Unable to index report into Elasticsearch: " + indexResponse.ServerError.Error.ToString()))
150+
151+
let IndexResults url =
152+
if (String.IsNullOrEmpty url = false) then
153+
trace "Indexing benchmark results into Elasticsearch"
154+
155+
let date = DateTime.UtcNow
156+
let commit = getSHA1 "." "HEAD"
157+
158+
let benchmarkJsonFiles =
159+
Directory.EnumerateFiles(benchmarkOutput.FullName, "*-brief.json", SearchOption.AllDirectories)
160+
|> Seq.toList
161+
162+
let uri = new Uri(url)
163+
let client = new ElasticClient(uri)
164+
165+
let indexName = IndexName.op_Implicit("reports")
166+
let typeName = TypeName.op_Implicit("report")
167+
let indexExists = client.IndexExists(Indices.op_Implicit(indexName)).Exists
168+
169+
if indexExists = false then
170+
let createIndex = client.CreateIndex(indexName, fun c ->
171+
c.Mappings(fun m ->
172+
m.Map<Report>(fun mm ->
173+
mm.AutoMap()
174+
.Properties(fun p ->
175+
p.Nested<Benchmark>(fun n ->
176+
n.AutoMap().Name(PropertyName.op_Implicit("benchmarks")) :> INestedProperty
177+
) :> IPromise<IProperties>
178+
) :> ITypeMapping
179+
) :> IPromise<IMappings>
180+
) :> ICreateIndexRequest
181+
)
182+
183+
if createIndex.IsValid = false then
184+
raise (Exception("Unable to create index into Elasticsearch"))
185+
186+
//let pipelineExists = client.GetPipeline(new GetPipelineRequest(Id.op_Implicit("benchmark-pipeline")))
187+
188+
//if pipelineExists.IsValid = false then
189+
let forEachProcessor = new ForeachProcessor()
190+
let processor = new GrokProcessor();
191+
processor.Field <- new Field("_ingest._value.displayInfo")
192+
processor.Patterns <- ["%{WORD:_ingest._value.class}.%{WORD:_ingest._value.method}: Job-%{WORD:_ingest._value.jobName}\\(Jit=%{WORD:_ingest._value.jit}, Runtime=%{WORD:_ingest._value.clr}, LaunchCount=%{NUMBER:_ingest._value.launchCount}, RunStrategy=%{WORD:_ingest._value.runStrategy}, TargetCount=%{NUMBER:_ingest._value.targetCount}, UnrollFactor=%{NUMBER:_ingest._value.unrollFactor}, WarmupCount=%{NUMBER:_ingest._value.warmupCount}\\)"]
193+
194+
forEachProcessor.Field <- new Field("benchmarks")
195+
forEachProcessor.Processor <- processor
196+
197+
let request = new PutPipelineRequest(Id.op_Implicit("benchmark-pipeline"))
198+
request.Description <- "Grok benchmark settings"
199+
request.Processors <- [forEachProcessor]
200+
201+
let createPipeline = client.PutPipeline(request)
202+
203+
if createPipeline.IsValid = false then
204+
raise (Exception("Unable to create pipeline"))
205+
206+
for file in benchmarkJsonFiles do IndexResult (client, file, date, commit, indexName, typeName)
207+
208+
trace "Indexed benchmark results into Elasticsearch"

‎build/scripts/Commandline.fsx

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,11 @@ module Commandline =
6262
match filteredArgs with
6363
| _ :: tail -> target :: tail
6464
| [] -> [target]
65+
66+
let private (|IsUrl|_|) (candidate:string) =
67+
match Uri.TryCreate(candidate, UriKind.RelativeOrAbsolute) with
68+
| true, _ -> Some candidate
69+
| _ -> None
6570

6671
let parse () =
6772
setEnvironVar "FAKEBUILD" "1"
@@ -72,6 +77,9 @@ module Commandline =
7277

7378
| ["test"; testFilter] -> setBuildParam "testfilter" testFilter
7479

80+
| ["benchmark"; IsUrl elasticsearch] ->
81+
setBuildParam "elasticsearch" elasticsearch
82+
7583
| ["profile"; esVersions] ->
7684
setBuildParam "esversions" esVersions
7785
| ["profile"; esVersions; testFilter] ->

‎build/scripts/Profiling.fsx

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,6 @@ module Profiler =
144144
if createIndex.IsValid = false then
145145
raise (Exception("Unable to create index into Elasticsearch"))
146146

147-
148147
let indexRequest = new IndexRequest<Report>(indexName, typeName)
149148
indexRequest.Document <- reportDoc
150149

‎build/scripts/Targets.fsx

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,10 @@ Target "Profile" <| fun _ ->
4848

4949
Target "Integrate" <| Tests.RunIntegrationTests
5050

51-
Target "Benchmark" Benchmarker.Run
51+
Target "Benchmark" <| fun _ ->
52+
Benchmarker.Run()
53+
let url = getBuildParam "elasticsearch"
54+
Benchmarker.IndexResults url
5255

5356
Target "InheritDoc" InheritDoc.PatchInheritDocs
5457

0 commit comments

Comments
 (0)