Benchmarking implementation

build/scripts/Benchmarking.fsx

@@ -1,5 +1,11 @@
-#I @"../../packages/build/FAKE/tools"
+#r "../../packages/build/NEST/lib/net45/Nest.dll"
+#r "../../packages/build/Elasticsearch.Net/lib/net45/Elasticsearch.Net.dll"
+#r "../../packages/build/Newtonsoft.Json/lib/net45/Newtonsoft.Json.dll"
+#r "../../packages/build/FSharp.Data/lib/net40/FSharp.Data.dll"
+#I @"../../packages/build/FAKE/tools"
 #r @"FakeLib.dll"
+#nowarn "0044" //TODO sort out FAKE 5
+
 open Fake
 
 #load @"Paths.fsx"
@@ -10,26 +16,131 @@ open System.Linq
 open System.Diagnostics
 open Paths
 
+open FSharp.Data
+
+open Nest
+open Elasticsearch.Net
+open Newtonsoft.Json
+open Git.Branches
+
 module Benchmarker =
-   let private testsProjectDirectory = Path.GetFullPath(Paths.Source("Tests"))
-   let private benchmarkOutput = Path.GetFullPath(Paths.Output("benchmarks")) |> directoryInfo
 
-   let private copyToOutput file = CopyFile benchmarkOutput.FullName file
+    let pipelineName = "benchmark-pipeline"
+    let indexName = IndexName.op_Implicit("benchmark-reports")
+    let typeName = TypeName.op_Implicit("benchmarkreport")
+
+    type Memory(gen0Collections:int, gen1Collections: int, gen2Collections: int, totalOperations:int64, bytesAllocatedPerOperation:int64) =
+        member val Gen0Collections=gen0Collections with get, set
+        member val Gen1Collections=gen1Collections with get, set
+        member val Gen2Collections=gen2Collections with get, set
+        member val TotalOperations=totalOperations with get, set
+        member val BytesAllocatedPerOperation=bytesAllocatedPerOperation with get, set
+
+    type ChronometerFrequency(hertz:double) =
+        member val Hertz=hertz with get, set
+
+    type HostEnvironmentInfo(benchmarkDotNetCaption:string, benchmarkDotNetVersion:string, osVersion: string, processorName:string,
+                             processorCount:int, runtimeVersion:string, architecture:string, hasAttachedDebugger:bool, hasRyuJit:bool,
+                             configuration:string, jitModules:string, dotnetCliVersion:string, chronometerFrequency:ChronometerFrequency,
+                             hardwareTimerKind:string) =
+        member val BenchmarkDotNetCaption=benchmarkDotNetCaption with get, set
+        member val BenchmarkDotNetVersion=benchmarkDotNetVersion with get, set
+        member val OsVersion=osVersion with get, set
+        member val ProcessorName=processorName with get, set
+        member val ProcessorCount=processorCount with get, set
+        member val RuntimeVersion=runtimeVersion with get, set
+        member val Architecture=architecture with get, set
+        member val HasAttachedDebugger=hasAttachedDebugger with get, set
+        member val HasRyuJit=hasRyuJit with get, set
+        member val Configuration=configuration with get, set
+        member val JitModules=jitModules with get, set
+        member val DotNetCliVersion=dotnetCliVersion with get, set
+        member val ChronometerFrequency=chronometerFrequency with get, set
+        member val HardwareTimerKind=hardwareTimerKind with get, set
+
+    type ConfidenceInterval(n:int, mean: double, standardError:double, level:int, margin:double, lower:double, upper:double) =
+        member val N=n with get, set
+        member val Mean=mean with get, set
+        member val StandardError=standardError with get, set
+        member val Level=level with get, set
+        member val Margin=margin with get, set
+        member val Lower=lower with get, set
+        member val Upper=upper with get, set
+
+    type Percentiles (p0:double, p25:double, p50:double, p67:double, p80:double, p85:double, p90:double, p95:double, p100:double) =
+        member val P0=p0 with get, set
+        member val P25=p25 with get, set
+        member val P50=p50 with get, set
+        member val P67=p67 with get, set
+        member val P80=p80 with get, set
+        member val P85=p85 with get, set
+        member val P90=p90 with get, set
+        member val P95=p95 with get, set
+        member val P100=p100 with get, set
+
+    type Statistics(n:int, min:double, lowerFence:double, q1:double, median:double, mean:double, q3:double, upperFence:double, max:double,
+                    interquartileRange:double, outliers:double list, standardError:double, variance:double, standardDeviation:double,
+                    skewness:double, kurtosis:double, confidenceInterval:ConfidenceInterval, percentiles:Percentiles) =
+        member val N=n with get, set
+        member val Min=min with get, set
+        member val LowerFence=lowerFence with get, set
+        member val Q1=q1 with get, set
+        member val Median=median with get, set
+        member val Mean=mean with get, set
+        member val Q3=q3 with get, set
+        member val UpperFence=upperFence with get, set
+        member val Max=max with get, set
+        member val InterquartileRange=interquartileRange with get, set
+        member val Outliers=outliers with get, set
+        member val StandardError=standardError with get, set
+        member val Variance=variance with get, set
+        member val StandardDeviation=standardDeviation with get, set
+        member val Skewness=skewness with get, set
+        member val Kurtosis=kurtosis with get, set
+        member val ConfidenceInterval=confidenceInterval with get, set
+        member val Percentiles=percentiles with get, set
+
+    type Benchmark(displayInfo:string, namespyce:string, tipe:string, method:string, methodTitle:string, parameters:string,
+                   statistics:Statistics, memory:Memory) = 
+        member val DisplayInfo=displayInfo with get, set
+        member val Namespace=namespyce with get, set
+        member val Type=tipe with get, set
+        member val Method=method with get, set
+        member val MethodTitle=methodTitle with get, set
+        member val Parameters=parameters with get, set
+        member val Statistics=statistics with get, set
+        member val Memory=memory with get, set
+
+    type BenchmarkReport(title: string, totalTime:TimeSpan, date:DateTime, commit:string, host:HostEnvironmentInfo, benchmarks:Benchmark list) =
+        member val Title = title with get, set
+        member val TotalTime = totalTime with get, set
+        member val Date = date with get, set
+        member val Commit = commit with get, set
+        member val HostEnvironmentInfo = host with get, set
+        member val Benchmarks = benchmarks with get, set
+
+    let private testsProjectDirectory = Path.GetFullPath(Paths.Source("Tests"))
+    let private benchmarkOutput = Path.GetFullPath(Paths.Output("benchmarks")) |> directoryInfo
+    let private copyToOutput file = CopyFile benchmarkOutput.FullName file
+
+    let Run(runInteractive:bool) =
 
-   let Run() =
         ensureDirExists benchmarkOutput
+
         let projectJson = testsProjectDirectory @@ "project.json"
+
         // running benchmarks can timeout so clean up any generated benchmark files
         try
-            DotNetCli.Restore(fun p ->
-                { p with
-                    Project = projectJson
-                })
-
-            DotNetCli.RunCommand(fun p ->
-                { p with
-                    WorkingDir = testsProjectDirectory
-                }) "run Benchmark"
+            if runInteractive then
+                DotNetCli.RunCommand(fun p ->
+                    { p with
+                        WorkingDir = testsProjectDirectory
+                    }) "run -f net46 -c Release Benchmark"
+             else
+                DotNetCli.RunCommand(fun p ->
+                    { p with
+                        WorkingDir = testsProjectDirectory
+                    }) "run -f net46 -c Release Benchmark non-interactive"
         finally
             let benchmarkOutputFiles =
                 let output = combinePaths testsProjectDirectory "BenchmarkDotNet.Artifacts"
@@ -38,3 +149,95 @@ module Benchmarker =
 
             for file in benchmarkOutputFiles do copyToOutput file
             DeleteFiles benchmarkOutputFiles
+
+    let IndexResult (client:ElasticClient, file:string, date:DateTime, commit:string, indexName, typeName) =
+
+        trace (sprintf "Indexing report %s into Elasticsearch" file)
+
+        let document = JsonConvert.DeserializeObject<BenchmarkReport>(File.ReadAllText(file))
+        document.Date <- date
+        document.Commit <- commit
+
+        let indexRequest = new IndexRequest<BenchmarkReport>(indexName, typeName)
+        indexRequest.Document <- document
+        indexRequest.Pipeline <- pipelineName
+
+        let indexResponse = client.Index(indexRequest)
+
+        if indexResponse.IsValid = false then
+                raise (Exception("Unable to index report into Elasticsearch: " + indexResponse.ServerError.Error.ToString()))
+
+    let IndexResults (url, username, password) =
+        if (String.IsNullOrEmpty url = false) then
+            trace "Indexing benchmark reports into Elasticsearch"
+
+            let date = DateTime.UtcNow
+            let commit = getSHA1 "." "HEAD"
+
+            let benchmarkJsonFiles =
+                Directory.EnumerateFiles(benchmarkOutput.FullName, "*-custom.json", SearchOption.AllDirectories)
+                |> Seq.toList
+
+            let uri = new Uri(url)
+            let connectionSettings = new ConnectionSettings(uri);
+
+            if (String.IsNullOrEmpty username = false && String.IsNullOrEmpty password = false) then
+                connectionSettings.BasicAuthentication(username, password) |> ignore
+
+            let client = new ElasticClient(connectionSettings)
+
+            let indexTemplateExists = client.IndexTemplateExists(Name.op_Implicit("benchmarks")).Exists
+
+            if indexTemplateExists |> not then
+
+                let typeMapping = new TypeMappingDescriptor<BenchmarkReport>()
+                typeMapping.AutoMap() |> ignore
+                typeMapping.Properties(fun p -> 
+                    p.Nested<Benchmark>(fun n ->
+                        n.AutoMap().Name(PropertyName.op_Implicit("benchmarks")) :> INestedProperty
+                    ) :> IPromise<IProperties>
+                ) |> ignore
+
+                let mappings = new Mappings()
+                mappings.Add(typeName, typeMapping :> ITypeMapping)
+
+                let indexSettings = new IndexSettings()
+                indexSettings.NumberOfShards <- Nullable 1
+
+                let putIndexTemplateRequest = new PutIndexTemplateRequest(Name.op_Implicit("benchmarks"))
+                putIndexTemplateRequest.Template <- "benchmark-reports-*"
+                putIndexTemplateRequest.Mappings <- mappings
+                putIndexTemplateRequest.Settings <- indexSettings
+
+                let putIndexTemplateResponse = client.PutIndexTemplate(putIndexTemplateRequest)
+
+                if putIndexTemplateResponse.IsValid = false then
+                    raise (Exception("Unable to create index template into Elasticsearch"))
+
+            let processor = new GrokProcessor();
+            processor.Field <- new Field("_ingest._value.displayInfo")
+            processor.Patterns <- ["%{WORD:_ingest._value.class}.%{DATA:_ingest._value.method}: Job-%{WORD:_ingest._value.jobName}\\(Jit=%{WORD:_ingest._value.jit}, Runtime=%{WORD:_ingest._value.clr}, LaunchCount=%{NUMBER:_ingest._value.launchCount}, RunStrategy=%{WORD:_ingest._value.runStrategy}, TargetCount=%{NUMBER:_ingest._value.targetCount}, UnrollFactor=%{NUMBER:_ingest._value.unrollFactor}, WarmupCount=%{NUMBER:_ingest._value.warmupCount}\\)"]
+
+            let forEachProcessor = new ForeachProcessor()
+            forEachProcessor.Field <- new Field("benchmarks")
+            forEachProcessor.Processor <- processor
+
+            let dateIndexProcessor = new DateIndexNameProcessor();
+            dateIndexProcessor.Field <- new Field("date")
+            dateIndexProcessor.IndexNamePrefix <- "benchmark-reports-"
+            dateIndexProcessor.DateRounding <- DateRounding.Month
+            dateIndexProcessor.DateFormats <- ["yyyy-MM-dd'T'HH:mm:ss.SSSSSSSZ"]
+
+            let request = new PutPipelineRequest(Id.op_Implicit(pipelineName))
+            request.Description <- "Benchmark settings pipeline"
+            request.Processors <- [dateIndexProcessor; forEachProcessor]
+
+            let createPipeline = client.PutPipeline(request)
+
+            if createPipeline.IsValid = false then
+                raise (Exception("Unable to create pipeline"))
+
+            for file in benchmarkJsonFiles
+                do IndexResult (client, file, date, commit, indexName, typeName)
+
+            trace "Indexed benchmark reports into Elasticsearch"

build/scripts/Building.fsx

@@ -72,12 +72,8 @@ module Build =
         compileCore incremental
 
     let Clean() =
-        match (quickBuild, getBuildParam "target" = "clean") with
-        | (false, _) 
-        | (_, true) -> 
-            tracefn "Cleaning known output folders"
-            CleanDir Paths.BuildOutput
-            DotNetCli.RunCommand (fun p -> { p with TimeOut = TimeSpan.FromMinutes(3.) }) "clean src/Elasticsearch.sln -c Release" |> ignore
-            DotNetProject.All |> Seq.iter(fun p -> CleanDir(Paths.BinFolder p.Name))
-        | (_, _) -> 
-            tracefn "Skipping clean target only run when calling 'release', 'canary', 'clean' as targets directly"
+        tracefn "Cleaning known output folders"
+        CleanDir Paths.BuildOutput
+        DotNetCli.RunCommand (fun p -> { p with TimeOut = TimeSpan.FromMinutes(3.) }) "clean src/Elasticsearch.sln -c Release" |> ignore
+        DotNetProject.All |> Seq.iter(fun p -> CleanDir(Paths.BinFolder p.Name))
+

build/scripts/Commandline.fsx

@@ -62,21 +62,48 @@ module Commandline =
         match filteredArgs with
         | _ :: tail -> target :: tail
         | [] -> [target]
+
+    let private (|IsUrl|_|) (candidate:string) =
+        match Uri.TryCreate(candidate, UriKind.RelativeOrAbsolute) with
+        | true, _ -> Some candidate
+        | _ -> None
 
     let parse () =
         setEnvironVar "FAKEBUILD" "1"
         printfn "%A" arguments
         match arguments with
-        | [] | ["build"] | ["test"] | ["clean"] -> ignore()
+        | [] | ["build"] | ["test"] | ["clean"] | ["benchmark"] | ["profile"] -> ignore()
         | ["release"; version] -> setBuildParam "version" version
 
         | ["test"; testFilter] -> setBuildParam "testfilter" testFilter
 
-        | ["profile"; esVersions] -> setBuildParam "esversions" esVersions
-        | ["profile"; esVersions; testFilter] ->
+        | ["benchmark"; IsUrl elasticsearch; username; password] ->
+            setBuildParam "elasticsearch" elasticsearch
+            setBuildParam "nonInteractive" "0"
+            setBuildParam "username" username
+            setBuildParam "password" password
+
+        | ["benchmark"; IsUrl elasticsearch] ->
+            setBuildParam "elasticsearch" elasticsearch
+            setBuildParam "nonInteractive" "0"
+
+        | ["benchmark"; IsUrl elasticsearch; "non-interactive"] ->
+            setBuildParam "elasticsearch" elasticsearch
+            setBuildParam "nonInteractive" "1"
+
+        | ["benchmark"; "non-interactive"] ->
+            setBuildParam "nonInteractive" "1"
+
+        | ["profile"; IsUrl elasticsearch] ->
+            setBuildParam "elasticsearch" elasticsearch
+
+        | ["profile"; esVersions] -> 
             setBuildParam "esversions" esVersions
-            setBuildParam "testfilter" testFilter
 
+        | ["profile"; esVersions; testFilter] ->
+            setBuildParam "esversions" esVersions
+            setBuildParam "testfilter" testFilter        
+
         | ["integrate"; esVersions] -> setBuildParam "esversions" esVersions
         | ["integrate"; esVersions; clusterFilter] ->
             setBuildParam "esversions" esVersions

build/scripts/Profiling.fsx

@@ -32,6 +32,9 @@ open Tooling
 
 module Profiler =
 
+    let indexName = IndexName.op_Implicit("profiling-reports")
+    let typeName = TypeName.op_Implicit("report")
+
     type Profile = XmlProvider<"../../build/profiling/profile-example.xml">
 
     type Function(id:string, fqn:string, totalTime:int, ownTime:int, calls:int, instances:int) =
@@ -48,8 +51,8 @@ module Profiler =
         member val Commit = commit with get, set
         member val Functions = functions with get, set
 
-    let private project = "Tests"
-    let private profiledApp = sprintf "%s/%s/%s.exe" (Paths.Output("v4.6")) project project
+    let private project = PrivateProject(Tests).Name
+    let private profiledApp = sprintf "%s/net46/%s.exe" (Paths.Output("Tests")) project
     let private snapShotOutput = Paths.Output("ProfilingSnapshot.dtp")
     let private snapShotStatsOutput = Paths.Output("ProfilingSnapshotStats.html")
     let private profileOutput = Paths.Output("ProfilingReport.xml")
@@ -63,7 +66,8 @@ module Profiler =
 
     let Run() = 
         let date = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffz")
-        Tooling.execProcessWithTimeout profiledApp ["Profile";"Class";getBuildParam "testfilter"] (TimeSpan.FromMinutes 30.) |> ignore
+        trace profiledApp
+        Tooling.execProcessWithTimeout profiledApp ["Profile";"Class";getBuildParam "testfilter"] (TimeSpan.FromMinutes 30.) "." |> ignore
         trace "Profiling finished."
 
         let performanceOutput = Paths.Output("profiling/performance") |> directoryInfo
@@ -122,8 +126,6 @@ module Profiler =
 
                 let reportDoc = new Report(report.Name, report.Date, report.Commit, functions)
 
-                let indexName = IndexName.op_Implicit("reports")
-                let typeName = TypeName.op_Implicit("report")
                 let indexExists = client.IndexExists(Indices.op_Implicit(indexName)).Exists
 
                 if indexExists = false then
@@ -143,7 +145,6 @@ module Profiler =
                     if createIndex.IsValid = false then
                         raise (Exception("Unable to create index into Elasticsearch"))
 
-
                 let indexRequest = new IndexRequest<Report>(indexName, typeName)
                 indexRequest.Document <- reportDoc
 

build/scripts/Targets.fsx

@@ -48,7 +48,13 @@ Target "Profile" <| fun _ ->
 
 Target "Integrate" <| Tests.RunIntegrationTests
 
-Target "Benchmark" Benchmarker.Run
+Target "Benchmark" <| fun _ ->
+    let runInteractive = ((getBuildParam "nonInteractive") <> "1")
+    Benchmarker.Run(runInteractive)
+    let url = getBuildParam "elasticsearch"
+    let username = getBuildParam "username"
+    let password = getBuildParam "password"
+    Benchmarker.IndexResults (url, username, password)
 
 Target "InheritDoc"  InheritDoc.PatchInheritDocs