Skip to content

Commit 479f316

Browse files
committed
Port: improve benchmarking of multi-threaded compilation
scala/scala#6200
1 parent f790b8b commit 479f316

File tree

5 files changed

+246
-265
lines changed

5 files changed

+246
-265
lines changed

compiler/src/dotty/tools/dotc/Run.scala

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,8 @@ class Run(comp: Compiler, ictx: Context) {
122122

123123
def runPhases(implicit ctx: Context) = {
124124
var lastPrintedTree: PrintedTree = NoPrintedTree
125-
val profiler = Profiler()
125+
val profiler = ctx.profiler
126+
126127
for (phase <- ctx.allPhases)
127128
if (phase.isRunnable)
128129
Stats.trackTime(s"$phase ms ") {
@@ -146,6 +147,7 @@ class Run(comp: Compiler, ictx: Context) {
146147
}
147148

148149
val runCtx = ctx.fresh
150+
runCtx.setProfiler(Profiler())
149151
ctx.phases.foreach(_.initContext(runCtx))
150152
runPhases(runCtx)
151153
if (!ctx.reporter.hasErrors) Rewrites.writeBack()

compiler/src/dotty/tools/dotc/core/Contexts.scala

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ import Run.RunInfo
1818
import util.Positions._
1919
import ast.Trees._
2020
import ast.untpd
21-
import util.{FreshNameCreator, SimpleIdentityMap, SourceFile, NoSource}
21+
import util.{FreshNameCreator, NoSource, SimpleIdentityMap, SourceFile}
2222
import typer.{Implicits, ImportInfo, Inliner, NamerContextOps, SearchHistory, TypeAssigner, Typer}
2323
import Implicits.ContextualImplicits
2424
import config.Settings._
@@ -28,9 +28,11 @@ import reporting.diagnostic.Message
2828
import collection.mutable
2929
import collection.immutable.BitSet
3030
import printing._
31-
import config.{Settings, ScalaSettings, Platform, JavaPlatform}
31+
import config.{JavaPlatform, Platform, ScalaSettings, Settings}
32+
3233
import language.implicitConversions
3334
import DenotTransformers.DenotTransformer
35+
import dotty.tools.dotc.profile.Profiler
3436
import util.Property.Key
3537
import util.Store
3638
import xsbti.AnalysisCallback
@@ -44,7 +46,8 @@ object Contexts {
4446
private val (freshNamesLoc, store5) = store4.newLocation[FreshNameCreator](new FreshNameCreator.Default)
4547
private val (compilationUnitLoc, store6) = store5.newLocation[CompilationUnit]()
4648
private val (runInfoLoc, store7) = store6.newLocation[RunInfo]()
47-
private val initialStore = store7
49+
private val (profilerLoc, store8) = store7.newLocation[Profiler]()
50+
private val initialStore = store8
4851

4952
/** A context is passed basically everywhere in dotc.
5053
* This is convenient but carries the risk of captured contexts in
@@ -197,6 +200,9 @@ object Contexts {
197200
/** The current compiler-run specific Info */
198201
def runInfo: RunInfo = store(runInfoLoc)
199202

203+
/** The current compiler-run profiler */
204+
def profiler: Profiler = store(profilerLoc)
205+
200206
/** The new implicit references that are introduced by this scope */
201207
protected var implicitsCache: ContextualImplicits = null
202208
def implicits: ContextualImplicits = {
@@ -461,6 +467,7 @@ object Contexts {
461467
def setSettings(settingsState: SettingsState): this.type = updateStore(settingsStateLoc, settingsState)
462468
def setCompilationUnit(compilationUnit: CompilationUnit): this.type = updateStore(compilationUnitLoc, compilationUnit)
463469
def setRunInfo(runInfo: RunInfo): this.type = updateStore(runInfoLoc, runInfo)
470+
def setProfiler(profiler: Profiler): this.type = updateStore(profilerLoc, profiler)
464471
def setFreshNames(freshNames: FreshNameCreator): this.type = updateStore(freshNamesLoc, freshNames)
465472

466473
def setProperty[T](key: Key[T], value: T): this.type =
Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
package dotty.tools.dotc.profile
2+
3+
import java.util.concurrent.ThreadPoolExecutor.AbortPolicy
4+
import java.util.concurrent._
5+
import java.util.concurrent.atomic.AtomicInteger
6+
7+
import dotty.tools.dotc.core.Phases.Phase
8+
import dotty.tools.dotc.core.Contexts.Context
9+
10+
sealed trait AsyncHelper {
11+
12+
def newUnboundedQueueFixedThreadPool
13+
(nThreads: Int,
14+
shortId: String, priority : Int = Thread.NORM_PRIORITY) : ThreadPoolExecutor
15+
def newBoundedQueueFixedThreadPool
16+
(nThreads: Int, maxQueueSize: Int, rejectHandler: RejectedExecutionHandler,
17+
shortId: String, priority : Int = Thread.NORM_PRIORITY) : ThreadPoolExecutor
18+
19+
}
20+
21+
object AsyncHelper {
22+
def apply(phase: Phase)(implicit ctx: Context): AsyncHelper = ctx.profiler match {
23+
case NoOpProfiler => new BasicAsyncHelper(phase)
24+
case r: RealProfiler => new ProfilingAsyncHelper(phase, r)
25+
}
26+
27+
private abstract class BaseAsyncHelper(phase: Phase)(implicit ctx: Context) extends AsyncHelper {
28+
val baseGroup = new ThreadGroup(s"scalac-${phase.phaseName}")
29+
private def childGroup(name: String) = new ThreadGroup(baseGroup, name)
30+
31+
protected def wrapRunnable(r: Runnable): Runnable
32+
33+
protected class CommonThreadFactory(shortId: String,
34+
daemon: Boolean = true,
35+
priority: Int) extends ThreadFactory {
36+
private val group: ThreadGroup = childGroup(shortId)
37+
private val threadNumber: AtomicInteger = new AtomicInteger(1)
38+
private val namePrefix = s"${baseGroup.getName}-$shortId-"
39+
40+
override def newThread(r: Runnable): Thread = {
41+
val wrapped = wrapRunnable(r)
42+
val t: Thread = new Thread(group, wrapped, namePrefix + threadNumber.getAndIncrement, 0)
43+
if (t.isDaemon != daemon) t.setDaemon(daemon)
44+
if (t.getPriority != priority) t.setPriority(priority)
45+
t
46+
}
47+
}
48+
}
49+
50+
private final class BasicAsyncHelper(phase: Phase)(implicit ctx: Context) extends BaseAsyncHelper(phase) {
51+
52+
override def newUnboundedQueueFixedThreadPool(nThreads: Int, shortId: String, priority: Int): ThreadPoolExecutor = {
53+
val threadFactory = new CommonThreadFactory(shortId, priority = priority)
54+
//like Executors.newFixedThreadPool
55+
new ThreadPoolExecutor(nThreads, nThreads, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue[Runnable], threadFactory)
56+
}
57+
58+
override def newBoundedQueueFixedThreadPool(nThreads: Int, maxQueueSize: Int, rejectHandler: RejectedExecutionHandler, shortId: String, priority: Int): ThreadPoolExecutor = {
59+
val threadFactory = new CommonThreadFactory(shortId, priority = priority)
60+
//like Executors.newFixedThreadPool
61+
new ThreadPoolExecutor(nThreads, nThreads, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue[Runnable](maxQueueSize), threadFactory, rejectHandler)
62+
}
63+
64+
override protected def wrapRunnable(r: Runnable): Runnable = r
65+
}
66+
67+
private class ProfilingAsyncHelper(phase: Phase, private val profiler: RealProfiler)(implicit ctx: Context) extends BaseAsyncHelper(phase) {
68+
69+
override def newUnboundedQueueFixedThreadPool(nThreads: Int, shortId: String, priority: Int): ThreadPoolExecutor = {
70+
val threadFactory = new CommonThreadFactory(shortId, priority = priority)
71+
//like Executors.newFixedThreadPool
72+
new SinglePhaseInstrumentedThreadPoolExecutor(nThreads, nThreads, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue[Runnable], threadFactory, new AbortPolicy)
73+
}
74+
75+
override def newBoundedQueueFixedThreadPool(nThreads: Int, maxQueueSize: Int, rejectHandler: RejectedExecutionHandler, shortId: String, priority: Int): ThreadPoolExecutor = {
76+
val threadFactory = new CommonThreadFactory(shortId, priority = priority)
77+
//like Executors.newFixedThreadPool
78+
new SinglePhaseInstrumentedThreadPoolExecutor(nThreads, nThreads, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue[Runnable](maxQueueSize), threadFactory, rejectHandler)
79+
}
80+
81+
override protected def wrapRunnable(r: Runnable): Runnable = () => {
82+
val data = new ThreadProfileData
83+
localData.set(data)
84+
85+
val profileStart = Profiler.emptySnap
86+
try r.run finally {
87+
val snap = profiler.snapThread()
88+
val threadRange = ProfileRange(profileStart, snap, phase, 0, "", Thread.currentThread())
89+
profiler.completeBackground(threadRange)
90+
}
91+
}
92+
93+
/**
94+
* data for thread run. Not threadsafe, only written from a single thread
95+
*/
96+
final class ThreadProfileData {
97+
var firstStartNs = 0L
98+
var taskCount = 0
99+
100+
var idleNs = 0L
101+
var runningNs = 0L
102+
103+
var lastStartNs = 0L
104+
var lastEndNs = 0L
105+
}
106+
107+
val localData = new ThreadLocal[ThreadProfileData]
108+
109+
private class SinglePhaseInstrumentedThreadPoolExecutor
110+
( corePoolSize: Int, maximumPoolSize: Int, keepAliveTime: Long, unit: TimeUnit,
111+
workQueue: BlockingQueue[Runnable], threadFactory: ThreadFactory, handler: RejectedExecutionHandler
112+
) extends ThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, handler) {
113+
114+
override def beforeExecute(t: Thread, r: Runnable): Unit = {
115+
val data = localData.get
116+
data.taskCount += 1
117+
val now = System.nanoTime()
118+
119+
if (data.firstStartNs == 0) data.firstStartNs = now
120+
else data.idleNs += now - data.lastEndNs
121+
122+
data.lastStartNs = now
123+
124+
super.beforeExecute(t, r)
125+
}
126+
127+
override def afterExecute(r: Runnable, t: Throwable): Unit = {
128+
val now = System.nanoTime()
129+
val data = localData.get
130+
131+
data.lastEndNs = now
132+
data.runningNs += now - data.lastStartNs
133+
134+
super.afterExecute(r, t)
135+
}
136+
137+
}
138+
}
139+
}

compiler/src/dotty/tools/dotc/profile/InPhase.scala

Lines changed: 0 additions & 86 deletions
This file was deleted.

0 commit comments

Comments
 (0)