diff --git a/compiler/src/dotty/tools/dotc/CompilationUnit.scala b/compiler/src/dotty/tools/dotc/CompilationUnit.scala index 2358739ebd74..c1a9e17c20f7 100644 --- a/compiler/src/dotty/tools/dotc/CompilationUnit.scala +++ b/compiler/src/dotty/tools/dotc/CompilationUnit.scala @@ -18,6 +18,8 @@ import StdNames.nme import scala.annotation.internal.sharable import scala.util.control.NoStackTrace import transform.MacroAnnotations +import dotty.tools.dotc.interfaces.AbstractFile +import dotty.tools.io.NoAbstractFile class CompilationUnit protected (val source: SourceFile, val info: CompilationUnitInfo | Null) { @@ -45,6 +47,7 @@ class CompilationUnit protected (val source: SourceFile, val info: CompilationUn /** Pickled TASTY binaries, indexed by class. */ var pickled: Map[ClassSymbol, () => Array[Byte]] = Map() + var outlinePickled: Map[ClassSymbol, () => Array[Byte]] = Map() /** The fresh name creator for the current unit. * FIXME(#7661): This is not fine-grained enough to enable reproducible builds, @@ -97,12 +100,17 @@ class CompilationUnit protected (val source: SourceFile, val info: CompilationUn // when this unit is unsuspended. depRecorder.clear() if !suspended then - if (ctx.settings.XprintSuspension.value) - report.echo(i"suspended: $this") - suspended = true - ctx.run.nn.suspendedUnits += this - if ctx.phase == Phases.inliningPhase then - suspendedAtInliningPhase = true + if ctx.settings.YnoSuspendedUnits.value then + report.error(i"Compilation unit suspended $this (-Yno-suspended-units is set)") + else + if (ctx.settings.XprintSuspension.value) + report.echo(i"suspended: $this") + suspended = true + ctx.run.nn.suspendedUnits += this + if ctx.phase == Phases.inliningPhase then + suspendedAtInliningPhase = true + else if ctx.settings.YearlyTastyOutput.value != NoAbstractFile then + report.error(i"Compilation units may not be suspended before inlining with -Ypickle-write") throw CompilationUnit.SuspendException() private var myAssignmentSpans: Map[Int, List[Span]] | Null = null diff --git a/compiler/src/dotty/tools/dotc/Compiler.scala b/compiler/src/dotty/tools/dotc/Compiler.scala index 290df761d117..6fa115b83f5b 100644 --- a/compiler/src/dotty/tools/dotc/Compiler.scala +++ b/compiler/src/dotty/tools/dotc/Compiler.scala @@ -40,20 +40,21 @@ class Compiler { List(new sbt.ExtractDependencies) :: // Sends information on classes' dependencies to sbt via callbacks List(new semanticdb.ExtractSemanticDB.ExtractSemanticInfo) :: // Extract info into .semanticdb files List(new PostTyper) :: // Additional checks and cleanups after type checking + // List(new sbt.ExtractAPI.Outline) :: // [runs in outline] Sends a representation of the API of classes to sbt via callbacks List(new sjs.PrepJSInterop) :: // Additional checks and transformations for Scala.js (Scala.js only) - List(new sbt.ExtractAPI) :: // Sends a representation of the API of classes to sbt via callbacks List(new SetRootTree) :: // Set the `rootTreeOrProvider` on class symbols Nil /** Phases dealing with TASTY tree pickling and unpickling */ protected def picklerPhases: List[List[Phase]] = - List(new Pickler) :: // Generate TASTY info - List(new Inlining) :: // Inline and execute macros - List(new PostInlining) :: // Add mirror support for inlined code - List(new CheckUnused.PostInlining) :: // Check for unused elements - List(new Staging) :: // Check staging levels and heal staged types - List(new Splicing) :: // Replace level 1 splices with holes - List(new PickleQuotes) :: // Turn quoted trees into explicit run-time data structures + List(new Pickler) :: // Generate TASTY info + List(new sbt.ExtractAPI) :: // [runs when not outline] Sends a representation of the API of classes to sbt via callbacks + List(new Inlining) :: // Inline and execute macros + List(new PostInlining) :: // Add mirror support for inlined code + List(new CheckUnused.PostInlining) :: // Check for unused elements + List(new Staging) :: // Check staging levels and heal staged types + List(new Splicing) :: // Replace level 1 splices with holes + List(new PickleQuotes) :: // Turn quoted trees into explicit run-time data structures Nil /** Phases dealing with the transformation from pickled trees to backend trees */ diff --git a/compiler/src/dotty/tools/dotc/Driver.scala b/compiler/src/dotty/tools/dotc/Driver.scala index ae2219a4f049..760cc27a61d6 100644 --- a/compiler/src/dotty/tools/dotc/Driver.scala +++ b/compiler/src/dotty/tools/dotc/Driver.scala @@ -13,6 +13,28 @@ import config.Feature import scala.util.control.NonFatal import fromtasty.{TASTYCompiler, TastyFileUtil} +import dotty.tools.io.NoAbstractFile +import dotty.tools.io.{VirtualFile, VirtualDirectory} + +import java.nio.file.Path as JPath +import scala.concurrent.* +import scala.annotation.internal.sharable +import scala.concurrent.duration.* +import scala.util.{Success, Failure} +import scala.annotation.targetName +import dotty.tools.dotc.classpath.FileUtils.hasScalaExtension +import dotty.tools.dotc.core.Symbols + +object Driver { + @sharable lazy val executor = + // TODO: systemParallelism may change over time - is it possible to update the pool size? + val pool = java.util.concurrent.Executors.newFixedThreadPool(systemParallelism()).nn + sys.addShutdownHook(pool.shutdown()) + ExecutionContext.fromExecutor(pool) + + /** 1 less than the system's own processor count (minimum 1) */ + def systemParallelism() = math.max(1, Runtime.getRuntime().nn.availableProcessors() - 1) +} /** Run the Dotty compiler. * @@ -28,6 +50,183 @@ class Driver { protected def emptyReporter: Reporter = new StoreReporter(null) + protected def doCompile(files: List[AbstractFile])(using ictx: Context): Reporter = + val isOutline = ictx.settings.Youtline.value(using ictx) + + if !isOutline then inContext(ictx): + report.echo(s"basic compilation enabled on files ${files.headOption.map(f => s"$f...").getOrElse("[]")}")(using ictx) + doCompile(newCompiler, files) // standard compilation + else + report.echo(s"Outline compilation enabled on files ${files.headOption.map(f => s"$f...").getOrElse("[]")}")(using ictx) + val maxParallelism = ictx.settings.YmaxParallelism.valueIn(ictx.settingsState) + val absParallelism = math.abs(maxParallelism) + val isParallel = maxParallelism >= 0 + val parallelism = + val ceiling = Driver.systemParallelism() + if absParallelism > 0 then math.min(absParallelism, ceiling) + else ceiling + + // NOTE: sbt will delete this potentially as soon as you call `apiPhaseCompleted` + val pickleWriteOutput = ictx.settings.YearlyTastyOutput.valueIn(ictx.settingsState) + val profileDestination = ictx.settings.YprofileDestination.valueIn(ictx.settingsState) + + if pickleWriteOutput == NoAbstractFile then + report.error("Requested outline compilation with `-Yexperimental-outline` " + + "but did not provide output directory for TASTY files (missing `-Yearly-tasty-output` flag).")(using ictx) + return ictx.reporter + + val pickleWriteSource = + pickleWriteOutput.underlyingSource match + case Some(source) => + source.file.asInstanceOf[java.io.File | Null] match + case f: java.io.File => Some(source) + case null => + report.warning(s"Could not resolve file of ${source} (of class ${source.getClass.getName})") + None + case None => + if pickleWriteOutput.isInstanceOf[dotty.tools.io.JarArchive] then + report.warning(s"Could not resolve underlying source of jar ${pickleWriteOutput} (of class ${pickleWriteOutput.getClass.getName})") + None + else + report.warning(s"Could not resolve underlying source of ${pickleWriteOutput} (of class ${pickleWriteOutput.getClass.getName})") + Some(pickleWriteOutput) + + val outlineOutput = new VirtualDirectory("") { + override def underlyingSource: Option[AbstractFile] = pickleWriteSource + } + + val firstPassCtx = ictx.fresh + .setSetting(ictx.settings.YoutlineClasspath, outlineOutput) + inContext(firstPassCtx): + doCompile(newCompiler, files) + + def secondPassCtx(id: Int, group: List[AbstractFile], promise: scala.concurrent.Promise[Unit]): Context = + val profileDestination0 = + if profileDestination.nonEmpty then + val ext = dotty.tools.io.Path.fileExtension(profileDestination) + val filename = dotty.tools.io.Path.fileName(profileDestination) + s"$filename-worker-$id${if ext.isEmpty then "" else s".$ext"}" + else profileDestination + + val baseCtx = initCtx.fresh + .setSettings(ictx.settingsState) // copy over the classpath arguments also + .setSetting(ictx.settings.YsecondPass, true) + .setSetting(ictx.settings.YoutlineClasspath, outlineOutput) + .setCallbacks(ictx.store) + .setDepsFinishPromise(promise) + .setReporter(if isParallel then new StoreReporter(ictx.reporter) else ictx.reporter) + + if profileDestination0.nonEmpty then + baseCtx.setSetting(ictx.settings.YprofileDestination, profileDestination0) + + // if ictx.settings.YoutlineClasspath.valueIn(ictx.settingsState).isEmpty then + // baseCtx.setSetting(baseCtx.settings.YoutlineClasspath, pickleWriteAsClasspath) + val fileNames: Array[String] = + if sourcesRequired then group.map(_.toString).toArray else Array.empty + setup(fileNames, baseCtx) match + case Some((_, ctx)) => + assert(ctx.incCallback != null, s"cannot run outline without incremental callback") + assert(ctx.depsFinishPromiseOpt.isDefined, s"cannot run outline without dependencies promise") + ctx + case None => baseCtx + end secondPassCtx + + val scalaFiles = files.filter(_.hasScalaExtension) + + // 516 units, 8 cores => maxGroupSize = 65, unitGroups = 8, compilers = 8 + if !firstPassCtx.reporter.hasErrors && scalaFiles.nonEmpty then + val maxGroupSize = Math.ceil(scalaFiles.length.toDouble / parallelism).toInt + val fileGroups = scalaFiles.grouped(maxGroupSize).toList + val compilers = fileGroups.length + + + + def userRequestedSingleGroup = maxParallelism == 1 + + // TODO: probably not good to warn here because maybe compile is incremental + // if compilers == 1 && !userRequestedSingleGroup then + // val knownParallelism = maxParallelism > 0 + // val requestedParallelism = s"Requested parallelism with `-Ymax-parallelism` was ${maxParallelism}" + // val computedAddedum = + // if knownParallelism then "." + // else s""", + // | therefore operating with computed parallelism of ${parallelism}.""".stripMargin + // val message = + // s"""Outline compilation second pass will run with a single compile group. + // | ${requestedParallelism}$computedAddedum + // | With ${scalaUnits.length} units to compile I can only batch them into a single group. + // | This will increase build times. + // | Perhaps consider turning off -Youtline for this project.""".stripMargin + // report.warning(message)(using firstPassCtx) + + val promises = fileGroups.map(_ => scala.concurrent.Promise[Unit]()) + + locally: + import scala.concurrent.ExecutionContext.Implicits.global + Future.sequence(promises.map(_.future)).andThen { + case Success(_) => + ictx.withIncCallback(_.dependencyPhaseCompleted()) + case Failure(ex) => + ex.printStackTrace() + report.error(s"Exception during parallel compilation: ${ex.getMessage}")(using firstPassCtx) + } + + report.echo(s"Compiling $compilers groups of files ${if isParallel then "in parallel" else "sequentially"}")(using firstPassCtx) + + def compileEager( + id: Int, + promise: Promise[Unit], + fileGroup: List[AbstractFile] + ): Reporter = { + if ctx.settings.verbose.value then + report.echo("#Compiling: " + fileGroup.take(3).mkString("", ", ", "...")) + val secondCtx = secondPassCtx(id, fileGroup, promise) + val reporter = inContext(secondCtx): + doCompile(newCompiler, fileGroup) // second pass + if !secondCtx.reporter.hasErrors then + assert(promise.isCompleted, s"promise was not completed") + if ctx.settings.verbose.value then + report.echo("#Done: " + fileGroup.mkString(" ")) + reporter + } + + def compileFuture( + id: Int, + promise: Promise[Unit], + fileGroup: List[AbstractFile] + )(using ExecutionContext): Future[Reporter] = + Future { + // println("#Compiling: " + fileGroup.mkString(" ")) + val secondCtx = secondPassCtx(id, fileGroup, promise) + val reporter = inContext(secondCtx): + doCompile(newCompiler, fileGroup) // second pass + // println("#Done: " + fileGroup.mkString(" ")) + reporter + } + + def fileGroupIds = LazyList.iterate(0)(_ + 1).take(compilers) + def taggedGroups = fileGroupIds.lazyZip(promises).lazyZip(fileGroups) + + if isParallel then + // val executor = java.util.concurrent.Executors.newFixedThreadPool(compilers).nn + given ec: ExecutionContext = Driver.executor // ExecutionContext.fromExecutor(executor) + val futureReporters = Future.sequence(taggedGroups.map(compileFuture)).andThen { + case Success(reporters) => + reporters.foreach(_.flush()(using firstPassCtx)) + case Failure(ex) => + ex.printStackTrace + report.error(s"Exception during parallel compilation: ${ex.getMessage}")(using firstPassCtx) + } + Await.ready(futureReporters, Duration.Inf) + // executor.shutdown() + else + taggedGroups.map(compileEager) + firstPassCtx.reporter + else + ictx.withIncCallback(_.dependencyPhaseCompleted()) // may be just java files compiled + firstPassCtx.reporter + end doCompile + protected def doCompile(compiler: Compiler, files: List[AbstractFile])(using Context): Reporter = if files.nonEmpty then var runOrNull = ctx.run @@ -193,7 +392,7 @@ class Driver { def process(args: Array[String], rootCtx: Context): Reporter = { setup(args, rootCtx) match case Some((files, compileCtx)) => - doCompile(newCompiler(using compileCtx), files)(using compileCtx) + doCompile(files)(using compileCtx) case None => rootCtx.reporter } diff --git a/compiler/src/dotty/tools/dotc/Run.scala b/compiler/src/dotty/tools/dotc/Run.scala index d18a2ddc7db0..e80a1db760be 100644 --- a/compiler/src/dotty/tools/dotc/Run.scala +++ b/compiler/src/dotty/tools/dotc/Run.scala @@ -287,9 +287,10 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint then ActiveProfile(ctx.settings.VprofileDetails.value.max(0).min(1000)) else NoProfile - // If testing pickler, make sure to stop after pickling phase: + // If testing pickler, of outline first pass, make sure to stop after pickling phase: val stopAfter = if (ctx.settings.YtestPickler.value) List("pickler") + else if (ctx.isOutlineFirstPass) List("sbt-api") else ctx.settings.YstopAfter.value val pluginPlan = ctx.base.addPluginPhases(ctx.base.phasePlan) diff --git a/compiler/src/dotty/tools/dotc/ast/tpd.scala b/compiler/src/dotty/tools/dotc/ast/tpd.scala index 71b85d97a187..20adbe335f4f 100644 --- a/compiler/src/dotty/tools/dotc/ast/tpd.scala +++ b/compiler/src/dotty/tools/dotc/ast/tpd.scala @@ -1381,6 +1381,17 @@ object tpd extends Trees.Instance[Type] with TypedTreeInfo { def runtimeCall(name: TermName, args: List[Tree])(using Context): Tree = Ident(defn.ScalaRuntimeModule.requiredMethod(name).termRef).appliedToTermArgs(args) + object ElidedTree: + def from(tree: Tree)(using Context): Tree = + Typed(tree, TypeTree(AnnotatedType(tree.tpe, Annotation(defn.ElidedTreeAnnot, tree.span)))) + + def isElided(tree: Tree)(using Context): Boolean = unapply(tree) + + def unapply(tree: Tree)(using Context): Boolean = tree match + case Typed(_, tpt) => tpt.tpe.hasAnnotation(defn.ElidedTreeAnnot) + case _ => false + + /** An extractor that pulls out type arguments */ object MaybePoly: def unapply(tree: Tree): Option[(Tree, List[Tree])] = tree match diff --git a/compiler/src/dotty/tools/dotc/classpath/FileUtils.scala b/compiler/src/dotty/tools/dotc/classpath/FileUtils.scala index 030b0b61044a..c2f04597a0f0 100644 --- a/compiler/src/dotty/tools/dotc/classpath/FileUtils.scala +++ b/compiler/src/dotty/tools/dotc/classpath/FileUtils.scala @@ -23,6 +23,8 @@ object FileUtils { def hasTastyExtension: Boolean = file.ext.isTasty + def hasScalaExtension: Boolean = file.ext.isScala + def isTasty: Boolean = !file.isDirectory && hasTastyExtension def isScalaBinary: Boolean = file.isClass || file.isTasty diff --git a/compiler/src/dotty/tools/dotc/config/PathResolver.scala b/compiler/src/dotty/tools/dotc/config/PathResolver.scala index 29e6e35855c8..15f1d606d255 100644 --- a/compiler/src/dotty/tools/dotc/config/PathResolver.scala +++ b/compiler/src/dotty/tools/dotc/config/PathResolver.scala @@ -12,6 +12,8 @@ import PartialFunction.condOpt import core.Contexts.* import Settings.* import dotty.tools.io.File +import dotty.tools.io.NoAbstractFile +import dotty.tools.dotc.classpath.FileUtils.isJarOrZip object PathResolver { @@ -208,6 +210,12 @@ class PathResolver(using c: Context) { if (!settings.classpath.isDefault) settings.classpath.value else sys.env.getOrElse("CLASSPATH", ".") + def outlineClasspath: List[dotty.tools.io.AbstractFile] = + if c.isOutlineFirstPass then Nil + else c.settings.YoutlineClasspath.value match + case NoAbstractFile => Nil + case file => file :: Nil + import classPathFactory.* // Assemble the elements! @@ -215,14 +223,15 @@ class PathResolver(using c: Context) { val release = Option(ctx.settings.javaOutputVersion.value).filter(_.nonEmpty) List( - JrtClassPath(release), // 1. The Java 9+ classpath (backed by the jrt:/ virtual system, if available) - classesInPath(javaBootClassPath), // 2. The Java bootstrap class path. - contentsOfDirsInPath(javaExtDirs), // 3. The Java extension class path. - classesInExpandedPath(javaUserClassPath), // 4. The Java application class path. - classesInPath(scalaBootClassPath), // 5. The Scala boot class path. - contentsOfDirsInPath(scalaExtDirs), // 6. The Scala extension class path. - classesInExpandedPath(userClassPath), // 7. The Scala application class path. - sourcesInPath(sourcePath) // 8. The Scala source path. + outlineClasspath.map(newClassPath), // 1. The outline classpath (backed by the -Youtline-classpath option) + JrtClassPath(release), // 2. The Java 9+ classpath (backed by the jrt:/ virtual system, if available) + classesInPath(javaBootClassPath), // 3. The Java bootstrap class path. + contentsOfDirsInPath(javaExtDirs), // 4. The Java extension class path. + classesInExpandedPath(javaUserClassPath), // 5. The Java application class path. + classesInPath(scalaBootClassPath), // 6. The Scala boot class path. + contentsOfDirsInPath(scalaExtDirs), // 7. The Scala extension class path. + classesInExpandedPath(userClassPath), // 8. The Scala application class path. + sourcesInPath(sourcePath) // 9. The Scala source path. ) lazy val containers: List[ClassPath] = basis.flatten.distinct @@ -238,12 +247,13 @@ class PathResolver(using c: Context) { | scalaExtDirs = %s | userClassPath = %s | sourcePath = %s + | outlineClasspath = %s |}""".trim.stripMargin.format( scalaHome, ppcp(javaBootClassPath), ppcp(javaExtDirs), ppcp(javaUserClassPath), useJavaClassPath, ppcp(scalaBootClassPath), ppcp(scalaExtDirs), ppcp(userClassPath), - ppcp(sourcePath) + ppcp(sourcePath), outlineClasspath.map("\n" + _).mkString ) } diff --git a/compiler/src/dotty/tools/dotc/config/Properties.scala b/compiler/src/dotty/tools/dotc/config/Properties.scala index 3392882057e7..f0edfb9407fe 100644 --- a/compiler/src/dotty/tools/dotc/config/Properties.scala +++ b/compiler/src/dotty/tools/dotc/config/Properties.scala @@ -9,6 +9,7 @@ import scala.annotation.internal.sharable import java.io.IOException import java.util.jar.Attributes.{ Name => AttributeName } import java.nio.charset.StandardCharsets +import scala.util.control.NonFatal /** Loads `library.properties` from the jar. */ object Properties extends PropertiesTrait { @@ -30,9 +31,26 @@ trait PropertiesTrait { /** The loaded properties */ @sharable protected lazy val scalaProps: java.util.Properties = { val props = new java.util.Properties - val stream = pickJarBasedOn getResourceAsStream propFilename - if (stream ne null) - quietlyDispose(props load stream, stream.close) + val file = pickJarBasedOn.getResource(propFilename) + if file != null then + scala.util.Using.Manager { use => + val stream = use(file.openStream()) + val buffered = use(new java.io.BufferedInputStream(stream)) + props.load(buffered) + }.fold(ex => (), identity) // swallow errors + // catch + // case ioe: IOException => + // println("swallowing " + ioe) + // ioe.printStackTrace() + // return props + // quietlyDispose({ + // try props.load(stream) + // catch + // case ioe: IOException => + // println("swallowing " + ioe) + // ioe.printStackTrace() + // props + // }, stream.close) props } @@ -70,13 +88,19 @@ trait PropertiesTrait { * or `"(unknown)"` if it cannot be determined. */ val simpleVersionString: String = { - val v = scalaPropOrElse("version.number", "(unknown)") - v + ( - if (v.contains("SNAPSHOT") || v.contains("NIGHTLY")) - "-git-" + scalaPropOrElse("git.hash", "(unknown)") - else - "" - ) + try + val v = scalaPropOrElse("version.number", "(unknown)") + v + ( + if (v.contains("SNAPSHOT") || v.contains("NIGHTLY")) + "-git-" + scalaPropOrElse("git.hash", "(unknown)") + else + "" + ) + catch + case NonFatal(t) => + println("swallowing " + t) + t.printStackTrace() + "(unknown)" } /** The version number of the jar this was loaded from plus `"version "` prefix, diff --git a/compiler/src/dotty/tools/dotc/config/ScalaSettings.scala b/compiler/src/dotty/tools/dotc/config/ScalaSettings.scala index 687adfe05ca7..fef05625b756 100644 --- a/compiler/src/dotty/tools/dotc/config/ScalaSettings.scala +++ b/compiler/src/dotty/tools/dotc/config/ScalaSettings.scala @@ -31,7 +31,7 @@ object ScalaSettings extends ScalaSettings // Kept as seperate type to avoid breaking backward compatibility abstract class ScalaSettings extends SettingGroup, AllScalaSettings: - val settingsByCategory: Map[SettingCategory, List[Setting[_]]] = + val settingsByCategory: Map[SettingCategory, List[Setting[_]]] = allSettings.groupBy(_.category) .view.mapValues(_.toList).toMap .withDefaultValue(Nil) @@ -43,7 +43,7 @@ abstract class ScalaSettings extends SettingGroup, AllScalaSettings: val verboseSettings: List[Setting[_]] = settingsByCategory(VerboseSetting).sortBy(_.name) val settingsByAliases: Map[String, Setting[_]] = allSettings.flatMap(s => s.aliases.map(_ -> s)).toMap - + trait AllScalaSettings extends CommonScalaSettings, PluginSettings, VerboseSettings, WarningSettings, XSettings, YSettings: self: SettingGroup => @@ -380,6 +380,7 @@ private sealed trait YSettings: val YprintPos: Setting[Boolean] = BooleanSetting(ForkSetting, "Yprint-pos", "Show tree positions.") val YprintPosSyms: Setting[Boolean] = BooleanSetting(ForkSetting, "Yprint-pos-syms", "Show symbol definitions positions.") val YnoDeepSubtypes: Setting[Boolean] = BooleanSetting(ForkSetting, "Yno-deep-subtypes", "Throw an exception on deep subtyping call stacks.") + val YnoSuspendedUnits: Setting[Boolean] = BooleanSetting(ForkSetting, "Yno-suspended-units", "Do not suspend units, e.g. when calling a macro defined in the same run. This will error instead of suspending.") val YnoPatmatOpt: Setting[Boolean] = BooleanSetting(ForkSetting, "Yno-patmat-opt", "Disable all pattern matching optimizations.") val YplainPrinter: Setting[Boolean] = BooleanSetting(ForkSetting, "Yplain-printer", "Pretty-print using a plain printer.") val YprintSyms: Setting[Boolean] = BooleanSetting(ForkSetting, "Yprint-syms", "When printing trees print info in symbols instead of corresponding info in trees.") @@ -439,7 +440,11 @@ private sealed trait YSettings: val YdebugMacros: Setting[Boolean] = BooleanSetting(ForkSetting, "Ydebug-macros", "Show debug info when quote pattern match fails") // Pipeline compilation options - val YjavaTasty: Setting[Boolean] = BooleanSetting(ForkSetting, "Yjava-tasty", "Pickler phase should compute pickles for .java defined symbols for use by build tools") - val YjavaTastyOutput: Setting[AbstractFile] = OutputSetting(ForkSetting, "Yjava-tasty-output", "directory|jar", "(Internal use only!) destination for generated .tasty files containing Java type signatures.", NoAbstractFile) + val YjavaTasty: Setting[Boolean] = BooleanSetting(ForkSetting, "Yjava-tasty", "Pickler phase should compute TASTy for .java defined symbols for use by build tools", aliases = List("-Ypickle-java"), preferPrevious = true) + val YearlyTastyOutput: Setting[AbstractFile] = OutputSetting(ForkSetting, "Yearly-tasty-output", "directory|jar", "Destination to write generated .tasty files to for use in pipelined compilation.", NoAbstractFile, aliases = List("-Ypickle-write"), preferPrevious = true) val YallowOutlineFromTasty: Setting[Boolean] = BooleanSetting(ForkSetting, "Yallow-outline-from-tasty", "Allow outline TASTy to be loaded with the -from-tasty option.") + val Youtline: Setting[Boolean] = BooleanSetting(ForkSetting, "Yexperimental-outline", "Run typechecking in two passes, a prior outline pass, and a subsequent full pass. This may be useful in combination with pipelining.") + val YsecondPass: Setting[Boolean] = BooleanSetting(ForkSetting, "Ysecond-pass", "Internal use only, signal that the compiler is in the second pass after outlining.") + val YoutlineClasspath: Setting[AbstractFile] = OutputSetting(ForkSetting, "Youtline-classpath", "directory|jar", "Destination for outline classfiles.", NoAbstractFile) + val YmaxParallelism: Setting[Int] = IntSetting(ForkSetting, "Ymax-parallelism", "When combined with -Yexperimental-outline, maximum number of parallel threads to use, 0 means all available processors", 0) end YSettings diff --git a/compiler/src/dotty/tools/dotc/config/Settings.scala b/compiler/src/dotty/tools/dotc/config/Settings.scala index 816d85e6c6fd..12918c4bb213 100644 --- a/compiler/src/dotty/tools/dotc/config/Settings.scala +++ b/compiler/src/dotty/tools/dotc/config/Settings.scala @@ -14,6 +14,7 @@ import collection.mutable import reflect.ClassTag import scala.util.{Success, Failure} import dotty.tools.dotc.config.Settings.Setting.ChoiceWithHelp +import org.scalajs.ir.Trees.JSBinaryOp.in object Settings: @@ -24,6 +25,7 @@ object Settings: val VersionTag: ClassTag[ScalaVersion] = ClassTag(classOf[ScalaVersion]) val OptionTag: ClassTag[Option[?]] = ClassTag(classOf[Option[?]]) val OutputTag: ClassTag[AbstractFile] = ClassTag(classOf[AbstractFile]) + val ClasspathTag: ClassTag[IArray[AbstractFile]] = ClassTag(classOf[Array[AbstractFile]]) trait SettingCategory: def prefixLetter: String @@ -79,6 +81,7 @@ object Settings: aliases: List[String] = Nil, depends: List[(Setting[?], Any)] = Nil, ignoreInvalidArgs: Boolean = false, + preferPrevious: Boolean = false, propertyClass: Option[Class[?]] = None, deprecationMsg: Option[String] = None, // kept only for -Ykind-projector option compatibility @@ -125,11 +128,16 @@ object Settings: valueList.filter(current.contains).foreach(s => dangers :+= s"Setting $name set to $s redundantly") current ++ valueList else - if sstate.wasChanged(idx) then dangers :+= s"Flag $name set repeatedly" + if sstate.wasChanged(idx) then + assert(!preferPrevious, "should have shortcutted with ignoreValue, side-effect may be present!") + dangers :+= s"Flag $name set repeatedly" value ArgsSummary(updateIn(sstate, valueNew), args, errors, dangers) end update + def ignoreValue(args: List[String]): ArgsSummary = + ArgsSummary(sstate, args, errors, warnings) + def fail(msg: String, args: List[String]) = ArgsSummary(sstate, args, errors :+ msg, warnings) @@ -196,7 +204,8 @@ object Settings: def doSet(argRest: String) = ((summon[ClassTag[T]], args): @unchecked) match { case (BooleanTag, _) => - setBoolean(argRest, args) + if sstate.wasChanged(idx) && preferPrevious then ignoreValue(args) + else setBoolean(argRest, args) case (OptionTag, _) => update(Some(propertyClass.get.getConstructor().newInstance()), args) case (ct, args) => @@ -216,7 +225,10 @@ object Settings: case StringTag => setString(arg, argsLeft) case OutputTag => - setOutput(arg, argsLeft) + if sstate.wasChanged(idx) && preferPrevious then + ignoreValue(argsLeft) // do not risk side effects e.g. overwriting a jar + else + setOutput(arg, argsLeft) case IntTag => setInt(arg, argsLeft) case VersionTag => @@ -333,8 +345,8 @@ object Settings: assert(!name.startsWith("-"), s"Setting $name cannot start with -") "-" + name - def BooleanSetting(category: SettingCategory, name: String, descr: String, initialValue: Boolean = false, aliases: List[String] = Nil): Setting[Boolean] = - publish(Setting(category, prependName(name), descr, initialValue, aliases = aliases)) + def BooleanSetting(category: SettingCategory, name: String, descr: String, initialValue: Boolean = false, aliases: List[String] = Nil, preferPrevious: Boolean = false): Setting[Boolean] = + publish(Setting(category, prependName(name), descr, initialValue, aliases = aliases, preferPrevious = preferPrevious)) def StringSetting(category: SettingCategory, name: String, helpArg: String, descr: String, default: String, aliases: List[String] = Nil): Setting[String] = publish(Setting(category, prependName(name), descr, default, helpArg, aliases = aliases)) @@ -357,8 +369,8 @@ object Settings: def MultiStringSetting(category: SettingCategory, name: String, helpArg: String, descr: String, default: List[String] = Nil, aliases: List[String] = Nil): Setting[List[String]] = publish(Setting(category, prependName(name), descr, default, helpArg, aliases = aliases)) - def OutputSetting(category: SettingCategory, name: String, helpArg: String, descr: String, default: AbstractFile): Setting[AbstractFile] = - publish(Setting(category, prependName(name), descr, default, helpArg)) + def OutputSetting(category: SettingCategory, name: String, helpArg: String, descr: String, default: AbstractFile, aliases: List[String] = Nil, preferPrevious: Boolean = false): Setting[AbstractFile] = + publish(Setting(category, prependName(name), descr, default, helpArg, aliases = aliases, preferPrevious = preferPrevious)) def PathSetting(category: SettingCategory, name: String, descr: String, default: String, aliases: List[String] = Nil): Setting[String] = publish(Setting(category, prependName(name), descr, default, aliases = aliases)) diff --git a/compiler/src/dotty/tools/dotc/core/Contexts.scala b/compiler/src/dotty/tools/dotc/core/Contexts.scala index ae21c6fb8763..82f1da9a88cd 100644 --- a/compiler/src/dotty/tools/dotc/core/Contexts.scala +++ b/compiler/src/dotty/tools/dotc/core/Contexts.scala @@ -54,8 +54,9 @@ object Contexts { private val (importInfoLoc, store9) = store8.newLocation[ImportInfo | Null]() private val (typeAssignerLoc, store10) = store9.newLocation[TypeAssigner](TypeAssigner) private val (progressCallbackLoc, store11) = store10.newLocation[ProgressCallback | Null]() + private val (depsFinishPromiseLoc, store12) = store11.newLocation[scala.concurrent.Promise[Unit]]() - private val initialStore = store11 + private val initialStore = store12 /** The current context */ inline def ctx(using ctx: Context): Context = ctx @@ -173,6 +174,16 @@ object Contexts { val local = incCallback if local != null then op(local) + def isOutline(using Context): Boolean = settings.Youtline.value + + def isOutlineFirstPass(using Context): Boolean = + isOutline && !settings.YsecondPass.value + + def isOutlineSecondPass(using Context): Boolean = + isOutline && settings.YsecondPass.value + + def depsFinishPromiseOpt: Option[scala.concurrent.Promise[Unit]] = Option(store(depsFinishPromiseLoc)) + def runZincPhases: Boolean = def forceRun = settings.YdumpSbtInc.value || settings.YforceSbtPhases.value val local = incCallback @@ -667,6 +678,12 @@ object Contexts { this._source = source this + def setCallbacks(store: Store): this.type = + this + .updateStore(compilerCallbackLoc, store(compilerCallbackLoc)) + .updateStore(incCallbackLoc, store(incCallbackLoc)) + .updateStore(progressCallbackLoc, store(progressCallbackLoc)) + private def setMoreProperties(moreProperties: Map[Key[Any], Any]): this.type = util.Stats.record("Context.setMoreProperties") this._moreProperties = moreProperties @@ -684,6 +701,8 @@ object Contexts { def setCompilerCallback(callback: CompilerCallback): this.type = updateStore(compilerCallbackLoc, callback) def setIncCallback(callback: IncrementalCallback): this.type = updateStore(incCallbackLoc, callback) + def setDepsFinishPromise(promise: scala.concurrent.Promise[Unit]): this.type = + updateStore(depsFinishPromiseLoc, promise) def setProgressCallback(callback: ProgressCallback): this.type = updateStore(progressCallbackLoc, callback) def setPrinterFn(printer: Context => Printer): this.type = updateStore(printerFnLoc, printer) def setSettings(settingsState: SettingsState): this.type = updateStore(settingsStateLoc, settingsState) diff --git a/compiler/src/dotty/tools/dotc/core/Definitions.scala b/compiler/src/dotty/tools/dotc/core/Definitions.scala index 789e744fbfc9..b1b8bc5ea171 100644 --- a/compiler/src/dotty/tools/dotc/core/Definitions.scala +++ b/compiler/src/dotty/tools/dotc/core/Definitions.scala @@ -486,6 +486,12 @@ class Definitions { @tu lazy val Predef_undefined: Symbol = ScalaPredefModule.requiredMethod(nme.???) @tu lazy val ScalaPredefModuleClass: ClassSymbol = ScalaPredefModule.moduleClass.asClass + // val Predef_undefinedElidedTree = new PerRun[ast.tpd.Tree]({ + // import dotty.tools.dotc.ast.tpd.TreeOps + // val base = ast.tpd.ref(defn.Predef_undefined) + // ast.tpd.ElidedTree.from(base) + // }) + @tu lazy val SubTypeClass: ClassSymbol = requiredClass("scala.<:<") @tu lazy val SubType_refl: Symbol = SubTypeClass.companionModule.requiredMethod(nme.refl) @@ -1018,6 +1024,7 @@ class Definitions { @tu lazy val IntoAnnot: ClassSymbol = requiredClass("scala.annotation.into") @tu lazy val IntoParamAnnot: ClassSymbol = requiredClass("scala.annotation.internal.$into") @tu lazy val ErasedParamAnnot: ClassSymbol = requiredClass("scala.annotation.internal.ErasedParam") + @tu lazy val ElidedTreeAnnot: ClassSymbol = requiredClass("scala.annotation.internal.ElidedTree") @tu lazy val MainAnnot: ClassSymbol = requiredClass("scala.main") @tu lazy val MappedAlternativeAnnot: ClassSymbol = requiredClass("scala.annotation.internal.MappedAlternative") @tu lazy val MigrationAnnot: ClassSymbol = requiredClass("scala.annotation.migration") @@ -1065,6 +1072,7 @@ class Definitions { @tu lazy val PublicInBinaryAnnot: ClassSymbol = requiredClass("scala.annotation.publicInBinary") @tu lazy val JavaRepeatableAnnot: ClassSymbol = requiredClass("java.lang.annotation.Repeatable") + @tu lazy val JavaAnnotationAnnot: ClassSymbol = requiredClass("java.lang.annotation.Annotation") // Initialization annotations @tu lazy val InitModule: Symbol = requiredModule("scala.annotation.init") diff --git a/compiler/src/dotty/tools/dotc/core/Mode.scala b/compiler/src/dotty/tools/dotc/core/Mode.scala index 71b49394ae14..402c2eef7dd1 100644 --- a/compiler/src/dotty/tools/dotc/core/Mode.scala +++ b/compiler/src/dotty/tools/dotc/core/Mode.scala @@ -130,8 +130,11 @@ object Mode { /** We are in the IDE */ val Interactive: Mode = newMode(20, "Interactive") - /** We are typing the body of an inline method */ - val InlineableBody: Mode = newMode(21, "InlineableBody") + // /** We are typing the body of an inline method */ + // val InlineableBody: Mode = newMode(21, "InlineableBody") // TODO unused? + + /** We are in the rhs of an inline definition */ + val InlineRHS = newMode(21, "InlineRHS") /** We are synthesizing the receiver of an extension method */ val SynthesizeExtMethodReceiver: Mode = newMode(23, "SynthesizeExtMethodReceiver") diff --git a/compiler/src/dotty/tools/dotc/core/Phases.scala b/compiler/src/dotty/tools/dotc/core/Phases.scala index c704846a82da..59736447af3c 100644 --- a/compiler/src/dotty/tools/dotc/core/Phases.scala +++ b/compiler/src/dotty/tools/dotc/core/Phases.scala @@ -210,6 +210,7 @@ object Phases { private var myTyperPhase: Phase = uninitialized private var myPostTyperPhase: Phase = uninitialized private var mySbtExtractDependenciesPhase: Phase = uninitialized + private var mySbtExtractAPIPhase: Phase = uninitialized private var myPicklerPhase: Phase = uninitialized private var myInliningPhase: Phase = uninitialized private var myStagingPhase: Phase = uninitialized @@ -235,6 +236,7 @@ object Phases { final def typerPhase: Phase = myTyperPhase final def postTyperPhase: Phase = myPostTyperPhase final def sbtExtractDependenciesPhase: Phase = mySbtExtractDependenciesPhase + final def sbtExtractAPIPhase: Phase = mySbtExtractAPIPhase final def picklerPhase: Phase = myPicklerPhase final def inliningPhase: Phase = myInliningPhase final def stagingPhase: Phase = myStagingPhase @@ -263,6 +265,7 @@ object Phases { myTyperPhase = phaseOfClass(classOf[TyperPhase]) myPostTyperPhase = phaseOfClass(classOf[PostTyper]) mySbtExtractDependenciesPhase = phaseOfClass(classOf[sbt.ExtractDependencies]) + mySbtExtractAPIPhase = phaseOfClass(classOf[sbt.ExtractAPI]) myPicklerPhase = phaseOfClass(classOf[Pickler]) myInliningPhase = phaseOfClass(classOf[Inlining]) myStagingPhase = phaseOfClass(classOf[Staging]) @@ -336,19 +339,29 @@ object Phases { /** skip the phase for a Java compilation unit, may depend on -Yjava-tasty */ def skipIfJava(using Context): Boolean = true + final def isAfterLastJavaPhase(using Context): Boolean = + // With `-Yjava-tasty` nominally the final phase is expected be ExtractAPI, + // otherwise drop Java sources at the end of TyperPhase. + // Checks if the last Java phase is before this phase, + // which always fails if the terminal phase is before lastJavaPhase. + val lastJavaPhase = if ctx.settings.YjavaTasty.value then sbtExtractAPIPhase else typerPhase + lastJavaPhase <= this + /** @pre `isRunnable` returns true */ def run(using Context): Unit /** @pre `isRunnable` returns true */ def runOn(units: List[CompilationUnit])(using runCtx: Context): List[CompilationUnit] = val buf = List.newBuilder[CompilationUnit] - // factor out typedAsJava check when not needed - val doSkipJava = ctx.settings.YjavaTasty.value && this <= picklerPhase && skipIfJava + + // Test that we are in a state where we need to check if the phase should be skipped for a java file, + // this prevents checking the expensive `unit.typedAsJava` unnecessarily. + val doCheckJava = skipIfJava && !isAfterLastJavaPhase for unit <- units do given unitCtx: Context = runCtx.fresh.setPhase(this.start).setCompilationUnit(unit).withRootImports if ctx.run.enterUnit(unit) then try - if doSkipJava && unit.typedAsJava then + if doCheckJava && unit.typedAsJava then () else run @@ -503,6 +516,7 @@ object Phases { def typerPhase(using Context): Phase = ctx.base.typerPhase def postTyperPhase(using Context): Phase = ctx.base.postTyperPhase def sbtExtractDependenciesPhase(using Context): Phase = ctx.base.sbtExtractDependenciesPhase + def sbtExtractAPIPhase(using Context): Phase = ctx.base.sbtExtractAPIPhase def picklerPhase(using Context): Phase = ctx.base.picklerPhase def inliningPhase(using Context): Phase = ctx.base.inliningPhase def stagingPhase(using Context): Phase = ctx.base.stagingPhase diff --git a/compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala b/compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala index 8b5a7ddfa65c..dac2ba6b33a9 100644 --- a/compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala +++ b/compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala @@ -204,7 +204,7 @@ object SymbolLoaders { } def needCompile(bin: AbstractFile, src: AbstractFile): Boolean = - src.lastModified >= bin.lastModified + src.lastModified >= bin.lastModified && !bin.isVirtual private def nameOf(classRep: ClassRepresentation)(using Context): TermName = classRep.fileName.sliceToTermName(0, classRep.nameLength) @@ -456,7 +456,8 @@ class TastyLoader(val tastyFile: AbstractFile) extends SymbolLoader { val tastyUUID = unpickler.unpickler.header.uuid new ClassfileTastyUUIDParser(classfile)(ctx).checkTastyUUID(tastyUUID) else - // This will be the case in any of our tests that compile with `-Youtput-only-tasty` + // This will be the case in any of our tests that compile with `-Youtput-only-tasty`, or when + // tasty file compiled by `-Yearly-tasty-output-write` comes from an early output jar. report.inform(s"No classfiles found for $tastyFile when checking TASTy UUID") private def mayLoadTreesFromTasty(using Context): Boolean = diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala index 7d2d95aa9601..3f28554a3c60 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala @@ -345,10 +345,12 @@ class TreePickler(pickler: TastyPickler, attributes: Attributes) { case _: Template | _: Hole => pickleTree(tpt) case _ if tpt.isType => pickleTpt(tpt) } - if isOutlinePickle && sym.isTerm && isJavaPickle then - // TODO: if we introduce outline typing for Scala definitions - // then we will need to update the check here - pickleElidedUnlessEmpty(rhs, tpt.tpe) + if isOutlinePickle then + if sym.isTerm && isJavaPickle then + pickleElidedUnlessEmpty(rhs, tpt.tpe) + else rhs match + case ElidedTree() => pickleElidedUnlessEmpty(rhs, tpt.tpe) + case _ => pickleTreeUnlessEmpty(rhs) else pickleTreeUnlessEmpty(rhs) pickleModifiers(sym, mdef) diff --git a/compiler/src/dotty/tools/dotc/inlines/Inliner.scala b/compiler/src/dotty/tools/dotc/inlines/Inliner.scala index 8bd89a71fa50..1b4d985c7c4c 100644 --- a/compiler/src/dotty/tools/dotc/inlines/Inliner.scala +++ b/compiler/src/dotty/tools/dotc/inlines/Inliner.scala @@ -1042,6 +1042,9 @@ class Inliner(val call: tpd.Tree)(using Context): for sym <- dependencies do if ctx.compilationUnit.source.file == sym.associatedFile then report.error(em"Cannot call macro $sym defined in the same source file", call.srcPos) + else if ctx.settings.YnoSuspendedUnits.value then + val addendum = ", suspension prevented by -Yno-suspended-units" + report.error(em"Cannot call macro $sym defined in the same compilation run$addendum", call.srcPos) if (suspendable && ctx.settings.XprintSuspension.value) report.echo(i"suspension triggered by macro call to ${sym.showLocated} in ${sym.associatedFile}", call.srcPos) if suspendable then diff --git a/compiler/src/dotty/tools/dotc/sbt/ExtractAPI.scala b/compiler/src/dotty/tools/dotc/sbt/ExtractAPI.scala index 079687ac3122..843d3140519d 100644 --- a/compiler/src/dotty/tools/dotc/sbt/ExtractAPI.scala +++ b/compiler/src/dotty/tools/dotc/sbt/ExtractAPI.scala @@ -19,6 +19,7 @@ import StdNames.str import NameOps.* import inlines.Inlines import transform.ValueClasses +import transform.Pickler import dotty.tools.io.{File, FileExtension, JarArchive} import util.{Property, SourceFile} import java.io.PrintWriter @@ -50,29 +51,36 @@ class ExtractAPI extends Phase { override def description: String = ExtractAPI.description - override def isRunnable(using Context): Boolean = { - super.isRunnable && ctx.runZincPhases - } - // Check no needed. Does not transform trees override def isCheckable: Boolean = false - // when `-Yjava-tasty` is set we actually want to run this phase on Java sources - override def skipIfJava(using Context): Boolean = false - // SuperAccessors need to be part of the API (see the scripted test // `trait-super` for an example where this matters), this is only the case // after `PostTyper` (unlike `ExtractDependencies`, the simplication to trees // done by `PostTyper` do not affect this phase because it only cares about // definitions, and `PostTyper` does not change definitions). - override def runsAfter: Set[String] = Set(transform.PostTyper.name) + override def runsAfter: Set[String] = Set(transform.Pickler.name) // why pickler - because we need the tasty + override def isRunnable(using Context): Boolean = + super.isRunnable && (ctx.runZincPhases || ctx.settings.YjavaTasty.value) && !ctx.isOutlineSecondPass + + // when `-Yjava-tasty` is set we actually want to run this phase on Java sources + override def skipIfJava(using Context): Boolean = false override def runOn(units: List[CompilationUnit])(using Context): List[CompilationUnit] = val nonLocalClassSymbols = new mutable.HashSet[Symbol] val ctx0 = ctx.withProperty(NonLocalClassSymbolsInCurrentUnits, Some(nonLocalClassSymbols)) - val units0 = super.runOn(units)(using ctx0) - ctx.withIncCallback(recordNonLocalClasses(nonLocalClassSymbols, _)) - units0 + inContext(ctx0) { + val units0 = super.runOn(units) + val scalaUnits = + if ctx.settings.YjavaTasty.value then + units0.filterNot(_.typedAsJava) // remove java sources, this is the terminal phase when `-Ypickle-java` is set + else + units0 // still run the phase for the side effects (writing TASTy files to -Yearly-tasty-output) + ctx.withIncCallback(recordNonLocalClasses(nonLocalClassSymbols, _)) + Pickler.writeEarlyTasty(units0) + ExtractAPI.signalPipelineCompleted() + scalaUnits + } end runOn private def recordNonLocalClasses(nonLocalClassSymbols: mutable.HashSet[Symbol], cb: interfaces.IncrementalCallback)(using Context): Unit = @@ -80,8 +88,6 @@ class ExtractAPI extends Phase { val sourceFile = cls.source if sourceFile.exists && cls.isDefinedInCurrentRun then recordNonLocalClass(cls, sourceFile, cb) - cb.apiPhaseCompleted() - cb.dependencyPhaseCompleted() private def recordNonLocalClass(cls: Symbol, sourceFile: SourceFile, cb: interfaces.IncrementalCallback)(using Context): Unit = def registerProductNames(fullClassName: String, binaryClassName: String) = @@ -146,6 +152,28 @@ object ExtractAPI: val name: String = "sbt-api" val description: String = "sends a representation of the API of classes to sbt" + def signalPipelineCompleted()(using Context): Unit = + + ctx.withIncCallback: cb => + ctx.depsFinishPromiseOpt match + case Some(promise) => + assert(ctx.isOutlineSecondPass, "promise should only be set in the second pass") + // suspended units could cause this to be completed twice + // TODO: instead of promise, we have a stream? + // the problem is that suspension is not always going to be the same, + // so each time all promises succeed, you would then install listeners in the compilers + // with suspended units. Anyway this along with Pickler.writeEarlyTasty need to + // agree upon reentrancy of writing early-tasty/signalling to sbt, because waiting for + // suspended units kills the purpose, so do we warn if it occurs, or expect the user to + // turn on -Yno-suspended-units? (or we forbid supension in typer phase with -Ypickle-write) + promise.success(()) + case _ => + assert(!ctx.isOutlineSecondPass, s"in -Youtline second pass must have deps finish promise set") + cb.apiPhaseCompleted() + if !ctx.isOutline then + cb.dependencyPhaseCompleted() // in outlining, wait until the second pass + end signalPipelineCompleted + private val NonLocalClassSymbolsInCurrentUnits: Property.Key[mutable.HashSet[Symbol]] = Property.Key() /** Extracts full (including private members) API representation out of Symbols and Types. diff --git a/compiler/src/dotty/tools/dotc/sbt/ExtractDependencies.scala b/compiler/src/dotty/tools/dotc/sbt/ExtractDependencies.scala index 352636f681c3..d54f9ac9608f 100644 --- a/compiler/src/dotty/tools/dotc/sbt/ExtractDependencies.scala +++ b/compiler/src/dotty/tools/dotc/sbt/ExtractDependencies.scala @@ -18,7 +18,7 @@ import dotty.tools.dotc.core.Phases.* import dotty.tools.dotc.core.Symbols.* import dotty.tools.dotc.core.Denotations.StaleSymbol import dotty.tools.dotc.core.Types.* - +import dotty.tools.dotc.transform.Pickler import dotty.tools.dotc.util.{SrcPos, NoSourcePosition} import dotty.tools.io import dotty.tools.io.{AbstractFile, PlainFile, ZipArchive, NoAbstractFile, FileExtension} @@ -30,6 +30,7 @@ import scala.jdk.CollectionConverters.* import scala.collection.{Set, mutable} import scala.compiletime.uninitialized +import dotty.tools.io.VirtualFile /** This phase sends information on classes' dependencies to sbt via callbacks. * @@ -67,6 +68,12 @@ class ExtractDependencies extends Phase { // when `-Yjava-tasty` is set we actually want to run this phase on Java sources override def skipIfJava(using Context): Boolean = false + override def runOn(units: List[CompilationUnit])(using runCtx: Context): List[CompilationUnit] = + val units0 = super.runOn(units) + if ctx.isOutlineSecondPass then + ExtractAPI.signalPipelineCompleted() + units0 + // This phase should be run directly after `Frontend`, if it is run after // `PostTyper`, some dependencies will be lost because trees get simplified. // See the scripted test `constants` for an example where this matters. @@ -495,24 +502,27 @@ class DependencyRecorder { if depFile != null then { // Cannot ignore inheritance relationship coming from the same source (see sbt/zinc#417) def allowLocal = depCtx == DependencyByInheritance || depCtx == LocalDependencyByInheritance - val isTastyOrSig = depFile.hasTastyExtension + val isTasty = depFile.hasTastyExtension + val isZipEntry = depFile.isInstanceOf[ZipArchive#Entry] + val isPlainFile = depFile.isInstanceOf[PlainFile] + val isExternal = isZipEntry || isPlainFile def processExternalDependency() = { val binaryClassName = depClass.binaryClassName - depFile match { - case ze: ZipArchive#Entry => // The dependency comes from a JAR - ze.underlyingSource match - case Some(zip) if zip.jpath != null => - binaryDependency(zip.jpath, binaryClassName) - case _ => - case pf: PlainFile => // The dependency comes from a class file, Zinc handles JRT filesystem - binaryDependency(if isTastyOrSig then cachedSiblingClass(pf) else pf.jpath, binaryClassName) - case _ => - internalError(s"Ignoring dependency $depFile of unknown class ${depFile.getClass}}", fromClass.srcPos) + if isZipEntry then { + depFile.underlyingSource match + case Some(zip) if zip.jpath != null => + binaryDependency(zip.jpath, binaryClassName) + case _ => } + else if isPlainFile then + val pf = depFile.asInstanceOf[PlainFile] + binaryDependency(if isTasty then cachedSiblingClass(pf) else pf.jpath, binaryClassName) + else + internalError(s"Ignoring dependency $depFile of unknown class ${depFile.getClass}}", fromClass.srcPos) } - if isTastyOrSig || depFile.hasClassExtension then + if (isTasty && isExternal) || depFile.hasClassExtension then processExternalDependency() else if allowLocal || depFile != sourceFile.file then // We cannot ignore dependencies coming from the same source file because diff --git a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala index 77eef4564bbf..f9c8775a9c70 100644 --- a/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala +++ b/compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala @@ -56,7 +56,7 @@ class ExtractSemanticDB private (phaseMode: ExtractSemanticDB.PhaseMode) extends override def isRunnable(using Context) = import ExtractSemanticDB.{semanticdbTarget, outputDirectory} def writesToOutputJar = semanticdbTarget.isEmpty && outputDirectory.isInstanceOf[JarArchive] - super.isRunnable && ctx.settings.Xsemanticdb.value && !writesToOutputJar + super.isRunnable && ctx.settings.Xsemanticdb.value && !writesToOutputJar && !ctx.isOutlineFirstPass // Check not needed since it does not transform trees override def isCheckable: Boolean = false diff --git a/compiler/src/dotty/tools/dotc/transform/CheckShadowing.scala b/compiler/src/dotty/tools/dotc/transform/CheckShadowing.scala index 87d652bd9133..384f1114916c 100644 --- a/compiler/src/dotty/tools/dotc/transform/CheckShadowing.scala +++ b/compiler/src/dotty/tools/dotc/transform/CheckShadowing.scala @@ -52,7 +52,8 @@ class CheckShadowing extends MiniPhase: override def isRunnable(using Context): Boolean = super.isRunnable && ctx.settings.Wshadow.value.nonEmpty && - !ctx.isJava + !ctx.isJava && + !ctx.isOutlineFirstPass // Setup before the traversal override def prepareForUnit(tree: tpd.Tree)(using Context): Context = diff --git a/compiler/src/dotty/tools/dotc/transform/CheckUnused.scala b/compiler/src/dotty/tools/dotc/transform/CheckUnused.scala index 7cff6fa5f1f0..a0b236eb9a4b 100644 --- a/compiler/src/dotty/tools/dotc/transform/CheckUnused.scala +++ b/compiler/src/dotty/tools/dotc/transform/CheckUnused.scala @@ -50,7 +50,8 @@ class CheckUnused private (phaseMode: CheckUnused.PhaseMode, suffix: String, _ke override def isRunnable(using Context): Boolean = super.isRunnable && ctx.settings.Wunused.value.nonEmpty && - !ctx.isJava + !ctx.isJava && + !ctx.isOutlineFirstPass // ========== SETUP ============ diff --git a/compiler/src/dotty/tools/dotc/transform/Pickler.scala b/compiler/src/dotty/tools/dotc/transform/Pickler.scala index b0aed580e824..d9ce2823781e 100644 --- a/compiler/src/dotty/tools/dotc/transform/Pickler.scala +++ b/compiler/src/dotty/tools/dotc/transform/Pickler.scala @@ -14,6 +14,7 @@ import StdNames.{str, nme} import Periods.* import Phases.* import Symbols.* +import StdNames.str import Flags.Module import reporting.{ThrowingReporter, Profile, Message} import collection.mutable @@ -33,7 +34,7 @@ object Pickler { */ inline val ParallelPickling = true - class EarlyFileWriter private (writer: TastyWriter, origin: AbstractFile): + class EarlyFileWriter private (writer: TastyWriter, val origin: AbstractFile): def this(dest: AbstractFile)(using @constructorOnly ctx: Context) = this(TastyWriter(dest), dest) export writer.writeTasty @@ -44,6 +45,74 @@ object Pickler { case jar: JarArchive => jar.close() // also close the file system case _ => } + + def writeEarlyTasty(units: List[CompilationUnit])(using Context): Unit = + val earlyOut = ctx.settings.YearlyTastyOutput.value + val base = + if earlyOut.isDirectory && earlyOut.exists then + Pickler.EarlyFileWriter(earlyOut) :: Nil + else + Nil + + val writers = + if ctx.isOutlineFirstPass then + val extra = Pickler.EarlyFileWriter(ctx.settings.YoutlineClasspath.value) + extra +: base + else + base + if writers.nonEmpty then + writeSigFiles(units, writers) + + // Why we only write to early output in the first run? + // =================================================== + // TL;DR the point of pipeline compilation is to start downstream projects early, + // so we don't want to wait for suspended units to be compiled. + // + // But why is it safe to ignore suspended units? + // If this project contains a transparent macro that is called in the same project, + // the compilation unit of that call will be suspended (if the macro implementation + // is also in this project), causing a second run. + // However before we do that run, we will have already requested sbt to begin + // early downstream compilation. This means that the suspended definitions will not + // be visible in *early* downstream compilation. + // + // However, sbt will by default prevent downstream compilation happening in this scenario, + // due to the existence of macro definitions. So we are protected from failure if user tries + // to use the suspended definitions. + // + // Additionally, it is recommended for the user to move macro implementations to another project + // if they want to force early output. In this scenario the suspensions will no longer occur, so now + // they will become visible in the early-output. + // + // See `sbt-test/pipelining/pipelining-scala-macro` and `sbt-test/pipelining/pipelining-scala-macro-force` + // for examples of this in action. + // + // Therefore we only need to write to early output in the first run. We also provide the option + // to diagnose suspensions with the `-Yno-suspended-units` flag. + private def writeSigFiles(units: List[CompilationUnit], writers: Seq[Pickler.EarlyFileWriter])(using Context): Unit = { + try + for + unit <- units + (cls, pickled) <- locally { + if ctx.isOutlineFirstPass then + unit.outlinePickled + else + assert(!ctx.isOutline) + unit.pickled + } + if cls.isDefinedInCurrentRun + do + val internalName = + if cls.is(Module) then cls.binaryClassName.stripSuffix(str.MODULE_SUFFIX).nn + else cls.binaryClassName + val bytes = pickled() + writers.foreach(_.writeTasty(internalName, bytes)) + finally + writers.foreach(_.close()) + if ctx.settings.verbose.value then + report.echo(s"[sig files written to ${writers.map(_.origin).mkString(", ")}]") + end try + } } /** This phase pickles trees */ @@ -56,7 +125,7 @@ class Pickler extends Phase { // No need to repickle trees coming from TASTY override def isRunnable(using Context): Boolean = - super.isRunnable && (!ctx.settings.fromTasty.value || ctx.settings.YjavaTasty.value) + super.isRunnable && !ctx.settings.fromTasty.value // when `-Yjava-tasty` is set we actually want to run this phase on Java sources override def skipIfJava(using Context): Boolean = false @@ -92,11 +161,9 @@ class Pickler extends Phase { body(scratch) } - private val executor = Executor[Array[Byte]]() + private var executor = Executor[Array[Byte]]() - private def useExecutor(using Context) = - Pickler.ParallelPickling && !ctx.settings.YtestPickler.value && - !ctx.settings.YjavaTasty.value // disable parallel pickling when `-Yjava-tasty` is set (internal testing only) + private def useExecutor(using Context) = Pickler.ParallelPickling && !ctx.settings.YtestPickler.value private def printerContext(isOutline: Boolean)(using Context): Context = if isOutline then ctx.fresh.setPrinterFn(OutlinePrinter(_)) @@ -120,7 +187,7 @@ class Pickler extends Phase { if isJavaAttr then // assert that Java sources didn't reach Pickler without `-Yjava-tasty`. assert(ctx.settings.YjavaTasty.value, "unexpected Java source file without -Yjava-tasty") - val isOutline = isJavaAttr // TODO: later we may want outline for Scala sources too + val isOutline = isJavaAttr || ctx.isOutlineFirstPass val attributes = Attributes( sourceFile = sourceRelativePath, scala2StandardLibrary = ctx.settings.YcompileScala2Library.value, @@ -191,27 +258,21 @@ class Pickler extends Phase { printedTasty(cls) = TastyPrinter.showContents(pickled, noColor = true, testPickler = true) () => pickled - unit.pickled += (cls -> demandPickled) + if ctx.isOutlineFirstPass then + unit.outlinePickled += (cls -> demandPickled) + else // either non-outline, or second pass + unit.pickled += (cls -> demandPickled) end for } override def runOn(units: List[CompilationUnit])(using Context): List[CompilationUnit] = { - val sigWriter: Option[Pickler.EarlyFileWriter] = ctx.settings.YjavaTastyOutput.value match - case jar: JarArchive if jar.exists => - Some(Pickler.EarlyFileWriter(jar)) - case _ => - None val units0 = - if ctx.settings.fromTasty.value then - // we still run the phase for the side effect of writing the pipeline tasty files - units + if useExecutor then + executor.start() + try super.runOn(units) + finally executor.close() // NOTE: each run will create a new Pickler phase, so safe to close here. else - if useExecutor then - executor.start() - try super.runOn(units) - finally executor.close() - else - super.runOn(units) + super.runOn(units) if ctx.settings.YtestPickler.value then val ctx2 = ctx.fresh .setSetting(ctx.settings.YreadComments, true) @@ -222,34 +283,7 @@ class Pickler extends Phase { .setReporter(new ThrowingReporter(ctx.reporter)) .addMode(Mode.ReadPositions) ) - val result = - if ctx.settings.YjavaTasty.value then - sigWriter.foreach(writeJavaSigFiles(units0, _)) - units0.filterNot(_.typedAsJava) // remove java sources, this is the terminal phase when `-Yjava-tasty` is set - else - units0 - result - } - - private def writeJavaSigFiles(units: List[CompilationUnit], writer: Pickler.EarlyFileWriter)(using Context): Unit = { - var count = 0 - try - for - unit <- units if unit.typedAsJava - (cls, pickled) <- unit.pickled - if cls.isDefinedInCurrentRun - do - val binaryClassName = cls.binaryClassName - val internalName = - if (cls.is(Module)) binaryClassName.stripSuffix(str.MODULE_SUFFIX).nn - else binaryClassName - val _ = writer.writeTasty(internalName, pickled()) - count += 1 - finally - writer.close() - if ctx.settings.verbose.value then - report.echo(s"[$count java sig files written]") - end try + units0 } private def testUnpickler(using Context): Unit = diff --git a/compiler/src/dotty/tools/dotc/transform/PostTyper.scala b/compiler/src/dotty/tools/dotc/transform/PostTyper.scala index 4ce41ca21bfe..a70daa89fd86 100644 --- a/compiler/src/dotty/tools/dotc/transform/PostTyper.scala +++ b/compiler/src/dotty/tools/dotc/transform/PostTyper.scala @@ -390,7 +390,8 @@ class PostTyper extends MacroTransform with InfoTransformer { thisPhase => Checking.checkPolyFunctionType(tree.tpt) val tree1 = cpy.ValDef(tree)(rhs = normalizeErasedRhs(tree.rhs, tree.symbol)) if tree1.removeAttachment(desugar.UntupledParam).isDefined then - checkStableSelection(tree.rhs) + if !(ctx.isOutlineFirstPass && ElidedTree.isElided(tree.rhs)) then + checkStableSelection(tree.rhs) processValOrDefDef(super.transform(tree1)) case tree: DefDef => annotateExperimental(tree.symbol) diff --git a/compiler/src/dotty/tools/dotc/typer/Namer.scala b/compiler/src/dotty/tools/dotc/typer/Namer.scala index 24721f1cd758..584013456357 100644 --- a/compiler/src/dotty/tools/dotc/typer/Namer.scala +++ b/compiler/src/dotty/tools/dotc/typer/Namer.scala @@ -1937,7 +1937,8 @@ class Namer { typer: Typer => approxTp var rhsCtx = ctx.fresh.addMode(Mode.InferringReturnType) - if sym.isInlineMethod then rhsCtx = rhsCtx.addMode(Mode.InlineableBody) + // if sym.isInlineMethod then rhsCtx = rhsCtx.addMode(Mode.InlineableBody) // TODO: remove because it's unused + if sym.is(Inline) then rhsCtx = rhsCtx.addMode(Mode.InlineRHS) if sym.is(ExtensionMethod) then rhsCtx = rhsCtx.addMode(Mode.InExtensionMethod) rhsCtx = prepareRhsCtx(rhsCtx, paramss) diff --git a/compiler/src/dotty/tools/dotc/typer/Typer.scala b/compiler/src/dotty/tools/dotc/typer/Typer.scala index 4729902c45bb..792f9403f60b 100644 --- a/compiler/src/dotty/tools/dotc/typer/Typer.scala +++ b/compiler/src/dotty/tools/dotc/typer/Typer.scala @@ -2550,6 +2550,36 @@ class Typer(@constructorOnly nestingLevel: Int = 0) extends Namer if filters == List(MessageFilter.None) then sup.markUsed() ctx.run.nn.suppressions.addSuppression(sup) + + /** Can the body of this method be dropped and replaced by `_` without + * breaking separate compilation ? This is used to generate tasty outlines. */ + private def canDropBody(definition: untpd.ValOrDefDef, sym: Symbol)(using Context): Boolean = + def mayNeedSuperAccessor = + val inTrait = sym.enclosingClass.is(Trait) + val acc = new untpd.UntypedTreeAccumulator[Boolean]: + override def apply(x: Boolean, tree: untpd.Tree)(using Context) = x || (tree match + case Super(qual, mix) => + // Super accessors are needed for all super calls that either + // appear in a trait or have as a target a member of some outer class, + // this is an approximation since the super call is untyped at this point. + inTrait || !mix.name.isEmpty + case _ => + foldOver(x, tree) + ) + acc(false, definition.rhs) + end mayNeedSuperAccessor + val bodyNeededFlags = definition match + case _: untpd.ValDef => Inline | Final | Erased + case _ => Inline | Erased + !(ctx.settings.scalajs.value || definition.rhs.isEmpty || + // Lambdas cannot be skipped, because typechecking them may constrain type variables. + definition.name == nme.ANON_FUN || + // The body of inline defs, and inline/final vals are part of the public API. + sym.isOneOf(bodyNeededFlags) || ctx.mode.is(Mode.InlineRHS) || + // Super accessors are part of the public API (subclasses need to implement them). + mayNeedSuperAccessor) + end canDropBody + def typedValDef(vdef: untpd.ValDef, sym: Symbol)(using Context): Tree = { val ValDef(name, tpt, _) = vdef checkNonRootName(vdef.name, vdef.nameSpan) @@ -2559,7 +2589,14 @@ class Typer(@constructorOnly nestingLevel: Int = 0) extends Namer val tpt1 = checkSimpleKinded(typedType(tpt)) val rhs1 = vdef.rhs match { case rhs @ Ident(nme.WILDCARD) => rhs withType tpt1.tpe - case rhs => typedExpr(rhs, tpt1.tpe.widenExpr) + case rhs => + if (ctx.isOutlineFirstPass && canDropBody(vdef, sym)) + if ctx.mode.is(Mode.InlineRHS) then + report.error(i"unexpected elided body in rhs of $sym", rhs.srcPos) + // defn.Predef_undefinedElidedTree() + ElidedTree.from(cpy.Ident(rhs)(nme.WILDCARD).withType(tpt1.tpe.widenExpr)) + else + typedExpr(rhs, tpt1.tpe.widenExpr) } val vdef1 = assignType(cpy.ValDef(vdef)(name, tpt1, rhs1), sym) postProcessInfo(vdef1, sym) @@ -2594,7 +2631,7 @@ class Typer(@constructorOnly nestingLevel: Int = 0) extends Namer if (sym.isOneOf(GivenOrImplicit)) checkImplicitConversionDefOK(sym) val tpt1 = checkSimpleKinded(typedType(tpt)) - val rhsCtx = ctx.fresh + val rhsCtx: FreshContext = ctx.fresh // assert fresh val tparamss = paramss1.collect { case untpd.TypeDefs(tparams) => tparams } @@ -2618,11 +2655,19 @@ class Typer(@constructorOnly nestingLevel: Int = 0) extends Namer else if !sym.isPrimaryConstructor then linkConstructorParams(sym, tparamSyms, rhsCtx) - if sym.isInlineMethod then rhsCtx.addMode(Mode.InlineableBody) + // if sym.isInlineMethod then rhsCtx.addMode(Mode.InlineableBody) // TODO: remove because it's unused + if sym.is(Inline) then rhsCtx.addMode(Mode.InlineRHS) if sym.is(ExtensionMethod) then rhsCtx.addMode(Mode.InExtensionMethod) val rhs1 = PrepareInlineable.dropInlineIfError(sym, if sym.isScala2Macro then typedScala2MacroBody(ddef.rhs)(using rhsCtx) - else typedExpr(ddef.rhs, tpt1.tpe.widenExpr)(using rhsCtx)) + else if ctx.isOutlineFirstPass && canDropBody(ddef, sym) then + if ctx.mode.is(Mode.InlineRHS) then + report.error(i"unexpected elided body in rhs of $sym", ddef.rhs.srcPos) + // defn.Predef_undefinedElidedTree() // span needed? + ElidedTree.from(cpy.Ident(ddef.rhs)(nme.WILDCARD).withType(tpt1.tpe)) + else + typedExpr(ddef.rhs, tpt1.tpe.widenExpr)(using rhsCtx) + ) if sym.isInlineMethod then if StagingLevel.level > 0 then @@ -3346,9 +3391,19 @@ class Typer(@constructorOnly nestingLevel: Int = 0) extends Namer val xtree = stat.removeAttachment(ExpandedTree).get traverse(xtree :: rest) case stat :: rest => - val stat1 = typed(stat)(using ctx.exprContext(stat, exprOwner)) - if !Linter.warnOnInterestingResultInStatement(stat1) then checkStatementPurity(stat1)(stat, exprOwner) - buf += stat1 + def isOutlinedTemplateStat = ctx.isOutlineFirstPass && exprOwner.isLocalDummy + + val stat1 = + if !isOutlinedTemplateStat || ctx.mode.is(Mode.InlineRHS) then + val stat1 = typed(stat)(using ctx.exprContext(stat, exprOwner)) + if !Linter.warnOnInterestingResultInStatement(stat1) then checkStatementPurity(stat1)(stat, exprOwner) + buf += stat1 + stat1 + else + // With -Ypickle-write, we skip the statements in a class that are not definitions. + if ctx.mode.is(Mode.InlineRHS) then + report.error(i"unexpected elided statement of ${exprOwner.enclosingMethodOrClass}", stat.srcPos) + EmptyTree traverse(rest)(using stat1.nullableContext) case nil => (buf.toList, ctx) diff --git a/compiler/src/dotty/tools/io/AbstractFile.scala b/compiler/src/dotty/tools/io/AbstractFile.scala index 233b1ca8fb62..1896f6028c19 100644 --- a/compiler/src/dotty/tools/io/AbstractFile.scala +++ b/compiler/src/dotty/tools/io/AbstractFile.scala @@ -277,7 +277,11 @@ abstract class AbstractFile extends Iterable[AbstractFile] { // a race condition in creating the entry after the failed lookup may throw val path = jpath.resolve(name) try - if (isDir) Files.createDirectory(path) + // We intentionally use `Files.createDirectories` instead of + // `Files.createDirectory` here because the latter throws an exception if + // the directory already exists, which can happen when two threads race to + // create the same directory. + if (isDir) Files.createDirectories(path) else Files.createFile(path) catch case _: FileAlreadyExistsException => () new PlainFile(new File(path)) diff --git a/compiler/src/dotty/tools/io/VirtualDirectory.scala b/compiler/src/dotty/tools/io/VirtualDirectory.scala index 157f63a2ac1a..a47ed9d89de6 100644 --- a/compiler/src/dotty/tools/io/VirtualDirectory.scala +++ b/compiler/src/dotty/tools/io/VirtualDirectory.scala @@ -16,7 +16,7 @@ import java.io.{InputStream, OutputStream} * ''Note: This library is considered experimental and should not be used unless you know what you are doing.'' */ class VirtualDirectory(val name: String, maybeContainer: Option[VirtualDirectory] = None) -extends AbstractFile { +extends AbstractFile { outer => def path: String = maybeContainer match { case None => name @@ -56,14 +56,18 @@ extends AbstractFile { override def fileNamed(name: String): AbstractFile = Option(lookupName(name, directory = false)) getOrElse { - val newFile = new VirtualFile(name, s"$path/$name") + val newFile = new VirtualFile(name, s"$path/$name", Some(this)) { + override val underlyingSource: Option[AbstractFile] = outer.underlyingSource + } files(name) = newFile newFile } override def subdirectoryNamed(name: String): AbstractFile = Option(lookupName(name, directory = true)) getOrElse { - val dir = new VirtualDirectory(name, Some(this)) + val dir = new VirtualDirectory(name, Some(this)) { + override val underlyingSource: Option[AbstractFile] = outer.underlyingSource + } files(name) = dir dir } diff --git a/compiler/src/dotty/tools/io/VirtualFile.scala b/compiler/src/dotty/tools/io/VirtualFile.scala index 9d290a9b0e6a..f2393a6c8c87 100644 --- a/compiler/src/dotty/tools/io/VirtualFile.scala +++ b/compiler/src/dotty/tools/io/VirtualFile.scala @@ -16,7 +16,9 @@ import java.io.{ ByteArrayInputStream, ByteArrayOutputStream, InputStream, Outpu * * ''Note: This library is considered experimental and should not be used unless you know what you are doing.'' */ -class VirtualFile(val name: String, override val path: String) extends AbstractFile { +class VirtualFile(val name: String, override val path: String, val enclosingDirectory: Option[VirtualDirectory]) extends AbstractFile { + + def this(name: String, path: String) = this(name, path, None) /** * Initializes this instance with the specified name and an @@ -27,6 +29,7 @@ class VirtualFile(val name: String, override val path: String) extends AbstractF */ def this(name: String) = this(name, name) + /** * Initializes this instance with the specified path * and a name taken from the last path element. @@ -60,7 +63,7 @@ class VirtualFile(val name: String, override val path: String) extends AbstractF } } - def container: AbstractFile = NoAbstractFile + def container: AbstractFile = enclosingDirectory.getOrElse(NoAbstractFile) /** Is this abstract file a directory? */ def isDirectory: Boolean = false diff --git a/compiler/test/dotty/tools/dotc/sbt/ProgressCallbackTest.scala b/compiler/test/dotty/tools/dotc/sbt/ProgressCallbackTest.scala index 489dc0f1759c..49fd3ee68d5f 100644 --- a/compiler/test/dotty/tools/dotc/sbt/ProgressCallbackTest.scala +++ b/compiler/test/dotty/tools/dotc/sbt/ProgressCallbackTest.scala @@ -97,7 +97,11 @@ final class ProgressCallbackTest extends DottyTest: locally: // (4) assert that the final progress recorded is at the target phase, // and progress is equal to the number of phases before the target. - val (befores, target +: next +: _) = runnableSubPhases.span(_ != targetPhase): @unchecked + // + // (4.1) extract the real befores by looking at the runnable phases + val (befores, target +: _) = runnableSubPhases.span(_ != targetPhase): @unchecked + // (4.2) extract the predicted next phase by looking at all phases + val (_, `target` +: next +: _) = allSubPhases.span(_ != targetPhase): @unchecked // (4.1) we expect cancellation to occur *as we enter* the target phase, // so no units should be visited in this phase. Therefore progress // should be equal to the number of phases before the target. (as we have 1 unit) diff --git a/library/src/scala/annotation/internal/ElidedTree.scala b/library/src/scala/annotation/internal/ElidedTree.scala new file mode 100644 index 000000000000..211f439155d3 --- /dev/null +++ b/library/src/scala/annotation/internal/ElidedTree.scala @@ -0,0 +1,17 @@ +package scala.annotation.internal + +import scala.annotation.Annotation + +/** An annotation produced by outline typing to indicate that a + * tree was elided. + * + * e.g. + * ``` + * val foo: Int = {...} + * ``` + * will be transformed to + * ``` + * val foo: Int = (_ : @ElidedTree) + * ``` + */ +final class ElidedTree() extends Annotation diff --git a/project/Build.scala b/project/Build.scala index fb94a7da68fa..aa9ab4096784 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -43,7 +43,7 @@ object DottyJSPlugin extends AutoPlugin { override def requires: Plugins = ScalaJSPlugin override def projectSettings: Seq[Setting[_]] = Def.settings( - commonBootstrappedSettings, + // commonBootstrappedSettings, // instead apply bootstrapped at each use site /* #11709 Remove the dependency on scala3-library that ScalaJSPlugin adds. * Instead, in this build, we use `.dependsOn` relationships to depend on @@ -872,6 +872,7 @@ object Build { lazy val nonBootstrappedDottyCompilerSettings = commonDottyCompilerSettings ++ Seq( // packageAll packages all and then returns a map with the abs location + exportPipelining := false, // classes are needed for scala3-sbt-bridge packageAll := Def.taskDyn { // Use a dynamic task to avoid loops when loading the settings Def.task { Map( @@ -1040,7 +1041,6 @@ object Build { lazy val `scala2-library-bootstrapped` = project.in(file("scala2-library-bootstrapped")). withCommonSettings(Bootstrapped). dependsOn(dottyCompiler(Bootstrapped) % "provided; compile->runtime; test->test"). - settings(commonBootstrappedSettings). settings(scala2LibraryBootstrappedSettings). settings(moduleName := "scala2-library") @@ -1051,7 +1051,6 @@ object Build { lazy val `scala2-library-cc` = project.in(file("scala2-library-cc")). withCommonSettings(Bootstrapped). dependsOn(dottyCompiler(Bootstrapped) % "provided; compile->runtime; test->test"). - settings(commonBootstrappedSettings). settings(scala2LibraryBootstrappedSettings). settings( moduleName := "scala2-library-cc", @@ -1258,6 +1257,9 @@ object Build { settings( description := "sbt compiler bridge for Dotty", + // Compile / scalacOptions += "-Ylog-classpath", + exportPipelining := false, // classes are needed for compiling the bootstrapped projects + Test / sources := Seq(), Compile / scalaSource := baseDirectory.value, Compile / javaSource := baseDirectory.value, @@ -1275,7 +1277,7 @@ object Build { lazy val `scala3-sbt-bridge-tests` = project.in(file("sbt-bridge/test")). dependsOn(dottyCompiler(Bootstrapped) % Test). dependsOn(`scala3-sbt-bridge`). - settings(commonBootstrappedSettings). + withCommonSettings(Bootstrapped). settings( Compile / sources := Seq(), Test / scalaSource := baseDirectory.value, @@ -1364,7 +1366,7 @@ object Build { lazy val `scala3-language-server` = project.in(file("language-server")). dependsOn(dottyCompiler(Bootstrapped)). - settings(commonBootstrappedSettings). + withCommonSettings(Bootstrapped). settings( libraryDependencies ++= Seq( "org.eclipse.lsp4j" % "org.eclipse.lsp4j" % "0.6.0", @@ -1407,6 +1409,7 @@ object Build { * useful, as that would not provide the linker and JS runners. */ lazy val sjsSandbox = project.in(file("sandbox/scalajs")). + withCommonSettings(Bootstrapped). enablePlugins(DottyJSPlugin). dependsOn(`scala3-library-bootstrappedJS`). settings( @@ -1424,6 +1427,7 @@ object Build { * It will grow in the future, as more stuff is confirmed to be supported. */ lazy val sjsJUnitTests = project.in(file("tests/sjs-junit")). + withCommonSettings(Bootstrapped). enablePlugins(DottyJSPlugin). dependsOn(`scala3-library-bootstrappedJS`). settings( @@ -1652,7 +1656,7 @@ object Build { lazy val `scaladoc-testcases` = project.in(file("scaladoc-testcases")). dependsOn(`scala3-compiler-bootstrapped`). - settings(commonBootstrappedSettings) + withCommonSettings(Bootstrapped) /** @@ -1663,11 +1667,13 @@ object Build { * Made as an indepented project to be scaladoc-agnostic. */ lazy val `scaladoc-js-common` = project.in(file("scaladoc-js/common")). + withCommonSettings(Bootstrapped). enablePlugins(DottyJSPlugin). dependsOn(`scala3-library-bootstrappedJS`). settings(libraryDependencies += ("org.scala-js" %%% "scalajs-dom" % "2.8.0")) lazy val `scaladoc-js-main` = project.in(file("scaladoc-js/main")). + withCommonSettings(Bootstrapped). enablePlugins(DottyJSPlugin). dependsOn(`scaladoc-js-common`). settings( @@ -1676,6 +1682,7 @@ object Build { ) lazy val `scaladoc-js-contributors` = project.in(file("scaladoc-js/contributors")). + withCommonSettings(Bootstrapped). enablePlugins(DottyJSPlugin). dependsOn(`scaladoc-js-common`). settings( @@ -1726,7 +1733,7 @@ object Build { lazy val scaladoc = project.in(file("scaladoc")). configs(SourceLinksIntegrationTest). - settings(commonBootstrappedSettings). + withCommonSettings(Bootstrapped). dependsOn(`scala3-compiler-bootstrapped`). dependsOn(`scala3-tasty-inspector`). settings(inConfig(SourceLinksIntegrationTest)(Defaults.testSettings)). @@ -1947,7 +1954,7 @@ object Build { lazy val `community-build` = project.in(file("community-build")). dependsOn(dottyLibrary(Bootstrapped)). - settings(commonBootstrappedSettings). + withCommonSettings(Bootstrapped). settings( prepareCommunityBuild := { (`scala3-sbt-bridge` / publishLocal).value diff --git a/sbt-bridge/src/dotty/tools/xsbt/CompilerBridgeDriver.java b/sbt-bridge/src/dotty/tools/xsbt/CompilerBridgeDriver.java index 20256d9e17cc..918b6d730b14 100644 --- a/sbt-bridge/src/dotty/tools/xsbt/CompilerBridgeDriver.java +++ b/sbt-bridge/src/dotty/tools/xsbt/CompilerBridgeDriver.java @@ -17,6 +17,7 @@ import dotty.tools.io.Streamable; import scala.collection.mutable.ListBuffer; import scala.jdk.javaapi.CollectionConverters; +import java.util.concurrent.ConcurrentHashMap; import scala.io.Codec; import xsbti.Problem; import xsbti.*; @@ -28,6 +29,7 @@ import java.io.OutputStream; import java.util.Comparator; import java.util.Collections; +import java.util.stream.Collectors; import java.util.HashMap; import java.util.Map; import java.util.Arrays; @@ -62,7 +64,8 @@ public boolean sourcesRequired() { } private static VirtualFile asVirtualFile(SourceFile sourceFile, DelegatingReporter reporter, - HashMap lookup) { + ConcurrentHashMap lookup) { + // !!!! MUST BE CONCURRENT HASH MAP FOR PARALLEL OUTLINE SECOND PASS !!!! return lookup.computeIfAbsent(sourceFile.file(), path -> { reportMissingFile(reporter, sourceFile); if (sourceFile.file().jpath() != null) @@ -90,7 +93,9 @@ synchronized public void run( Arrays.sort(sortedSources, (x0, x1) -> x0.id().compareTo(x1.id())); ListBuffer sourcesBuffer = new ListBuffer<>(); - HashMap lookup = new HashMap<>(sources.length, 0.25f); + + // !!!! MUST BE CONCURRENT HASH MAP FOR PARALLEL OUTLINE SECOND PASS !!!! + ConcurrentHashMap lookup = new ConcurrentHashMap<>(sources.length, 0.25f); for (int i = 0; i < sources.length; i++) { VirtualFile source = sortedSources[i]; @@ -136,9 +141,8 @@ synchronized public void run( if (!delegate.hasErrors()) { log.debug(this::prettyPrintCompilationArguments); - Compiler compiler = newCompiler(context); - doCompile(compiler, sourcesBuffer.toList(), context); + doCompile(sourcesBuffer.toList(), context); for (xsbti.Problem problem: delegate.problems()) { try { @@ -202,7 +206,16 @@ public InputStream inputStream() { private String infoOnCachedCompiler() { String compilerId = Integer.toHexString(hashCode()); - String compilerVersion = Properties.versionString(); + String compilerVersion; + try { + compilerVersion = Properties.versionString(); + } catch (Throwable t) { + if (scala.util.control.NonFatal.apply(t)) { + compilerVersion = "unknown"; + } else { + throw t; + } + }; return String.format("[zinc] Running cached compiler %s for Scala Compiler %s", compilerId, compilerVersion); } diff --git a/sbt-test/pipelining/Yearly-tasty-outline/a/src/main/scala/a/A.scala b/sbt-test/pipelining/Yearly-tasty-outline/a/src/main/scala/a/A.scala new file mode 100644 index 000000000000..4b10db3eb385 --- /dev/null +++ b/sbt-test/pipelining/Yearly-tasty-outline/a/src/main/scala/a/A.scala @@ -0,0 +1,5 @@ +package a + +object A { + val foo: (1,2,3) = (1,2,3) +} diff --git a/sbt-test/pipelining/Yearly-tasty-outline/b/src/main/scala/b/B.scala b/sbt-test/pipelining/Yearly-tasty-outline/b/src/main/scala/b/B.scala new file mode 100644 index 000000000000..0b84e390cbce --- /dev/null +++ b/sbt-test/pipelining/Yearly-tasty-outline/b/src/main/scala/b/B.scala @@ -0,0 +1,8 @@ +package b + +import a.A + +object B { + val f: 2 = A.foo(1) + val g: (1,2,3) = A.foo +} diff --git a/sbt-test/pipelining/Yearly-tasty-outline/build.sbt b/sbt-test/pipelining/Yearly-tasty-outline/build.sbt new file mode 100644 index 000000000000..44e2084163b9 --- /dev/null +++ b/sbt-test/pipelining/Yearly-tasty-outline/build.sbt @@ -0,0 +1,19 @@ +// early out is a jar +lazy val a = project.in(file("a")) + .settings( + scalacOptions ++= Seq( + "-Yexperimental-outline", "-Ymax-parallelism:1", + // test of manually setting the outline-classpath (usually automatically done in the second pass) + "-Youtline-classpath", ((ThisBuild / baseDirectory).value / "a-early.jar").toString, + "-Yearly-tasty-output", ((ThisBuild / baseDirectory).value / "a-early.jar").toString, + "-Ycheck:all", + "-Ystop-after:sbt-api-outline", + ) + ) + +// reads classpaths from early tasty outputs. No need for extra flags as the full tasty is available. +lazy val b = project.in(file("b")) + .settings( + Compile / unmanagedClasspath += Attributed.blank((ThisBuild / baseDirectory).value / "a-early.jar"), + scalacOptions += "-Ycheck:all", + ) diff --git a/sbt-test/pipelining/Yearly-tasty-outline/project/DottyInjectedPlugin.scala b/sbt-test/pipelining/Yearly-tasty-outline/project/DottyInjectedPlugin.scala new file mode 100644 index 000000000000..69f15d168bfc --- /dev/null +++ b/sbt-test/pipelining/Yearly-tasty-outline/project/DottyInjectedPlugin.scala @@ -0,0 +1,12 @@ +import sbt._ +import Keys._ + +object DottyInjectedPlugin extends AutoPlugin { + override def requires = plugins.JvmPlugin + override def trigger = allRequirements + + override val projectSettings = Seq( + scalaVersion := sys.props("plugin.scalaVersion"), + scalacOptions += "-source:3.0-migration" + ) +} diff --git a/sbt-test/pipelining/Yearly-tasty-outline/test b/sbt-test/pipelining/Yearly-tasty-outline/test new file mode 100644 index 000000000000..05bcb0e04ee1 --- /dev/null +++ b/sbt-test/pipelining/Yearly-tasty-outline/test @@ -0,0 +1,5 @@ +# create this file so that it is visible when +# manually setting -Youtline-classpath from sbt +$ touch a-early.jar +> a/compile +> b/compile diff --git a/sbt-test/pipelining/Yearly-tasty-output-inline/a/src/main/scala/a/A.scala b/sbt-test/pipelining/Yearly-tasty-output-inline/a/src/main/scala/a/A.scala new file mode 100644 index 000000000000..930e0ee78eb9 --- /dev/null +++ b/sbt-test/pipelining/Yearly-tasty-output-inline/a/src/main/scala/a/A.scala @@ -0,0 +1,10 @@ +package a + +import scala.quoted.* + +object A { + inline def power(x: Double, inline n: Int): Double = + inline if (n == 0) 1.0 + else inline if (n % 2 == 1) x * power(x, n - 1) + else power(x * x, n / 2) +} diff --git a/sbt-test/pipelining/Yearly-tasty-output-inline/b/src/main/scala/b/B.scala b/sbt-test/pipelining/Yearly-tasty-output-inline/b/src/main/scala/b/B.scala new file mode 100644 index 000000000000..7055d6d2d006 --- /dev/null +++ b/sbt-test/pipelining/Yearly-tasty-output-inline/b/src/main/scala/b/B.scala @@ -0,0 +1,10 @@ +package b + +import a.A + +object B { + @main def run = + assert(A.power(2.0, 2) == 4.0) + assert(A.power(2.0, 3) == 8.0) + assert(A.power(2.0, 4) == 16.0) +} diff --git a/sbt-test/pipelining/Yearly-tasty-output-inline/build.sbt b/sbt-test/pipelining/Yearly-tasty-output-inline/build.sbt new file mode 100644 index 000000000000..c0c726ce6a02 --- /dev/null +++ b/sbt-test/pipelining/Yearly-tasty-output-inline/build.sbt @@ -0,0 +1,14 @@ +// defines a inline method +lazy val a = project.in(file("a")) + .settings( + scalacOptions ++= Seq("-Yearly-tasty-output", ((ThisBuild / baseDirectory).value / "a-early.jar").toString), + scalacOptions += "-Ystop-after:firstTransform", + scalacOptions += "-Ycheck:all", + ) + +// uses the inline method, this is fine as there is no macro classloader involved +lazy val b = project.in(file("b")) + .settings( + Compile / unmanagedClasspath += Attributed.blank((ThisBuild / baseDirectory).value / "a-early.jar"), + scalacOptions += "-Ycheck:all", + ) diff --git a/sbt-test/pipelining/Yearly-tasty-output-inline/project/DottyInjectedPlugin.scala b/sbt-test/pipelining/Yearly-tasty-output-inline/project/DottyInjectedPlugin.scala new file mode 100644 index 000000000000..69f15d168bfc --- /dev/null +++ b/sbt-test/pipelining/Yearly-tasty-output-inline/project/DottyInjectedPlugin.scala @@ -0,0 +1,12 @@ +import sbt._ +import Keys._ + +object DottyInjectedPlugin extends AutoPlugin { + override def requires = plugins.JvmPlugin + override def trigger = allRequirements + + override val projectSettings = Seq( + scalaVersion := sys.props("plugin.scalaVersion"), + scalacOptions += "-source:3.0-migration" + ) +} diff --git a/sbt-test/pipelining/Yearly-tasty-output-inline/test b/sbt-test/pipelining/Yearly-tasty-output-inline/test new file mode 100644 index 000000000000..9779d91ce131 --- /dev/null +++ b/sbt-test/pipelining/Yearly-tasty-output-inline/test @@ -0,0 +1,3 @@ +> a/compile +# uses the early output jar of a +> b/run diff --git a/sbt-test/pipelining/Yearly-tasty-output/a/src/main/scala/a/A.scala b/sbt-test/pipelining/Yearly-tasty-output/a/src/main/scala/a/A.scala new file mode 100644 index 000000000000..4b10db3eb385 --- /dev/null +++ b/sbt-test/pipelining/Yearly-tasty-output/a/src/main/scala/a/A.scala @@ -0,0 +1,5 @@ +package a + +object A { + val foo: (1,2,3) = (1,2,3) +} diff --git a/sbt-test/pipelining/Yearly-tasty-output/b-early-out/.keep b/sbt-test/pipelining/Yearly-tasty-output/b-early-out/.keep new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/sbt-test/pipelining/Yearly-tasty-output/b/src/main/scala/b/B.scala b/sbt-test/pipelining/Yearly-tasty-output/b/src/main/scala/b/B.scala new file mode 100644 index 000000000000..5e6fa369e309 --- /dev/null +++ b/sbt-test/pipelining/Yearly-tasty-output/b/src/main/scala/b/B.scala @@ -0,0 +1,5 @@ +package b + +object B { + val bar: (1,2,3) = (1,2,3) +} diff --git a/sbt-test/pipelining/Yearly-tasty-output/build.sbt b/sbt-test/pipelining/Yearly-tasty-output/build.sbt new file mode 100644 index 000000000000..62990c616071 --- /dev/null +++ b/sbt-test/pipelining/Yearly-tasty-output/build.sbt @@ -0,0 +1,23 @@ +// early out is a jar +lazy val a = project.in(file("a")) + .settings( + scalacOptions ++= Seq("-Yearly-tasty-output", ((ThisBuild / baseDirectory).value / "a-early.jar").toString), + scalacOptions += "-Ystop-after:firstTransform", + scalacOptions += "-Ycheck:all", + ) + +// early out is a directory +lazy val b = project.in(file("b")) + .settings( + scalacOptions ++= Seq("-Yearly-tasty-output", ((ThisBuild / baseDirectory).value / "b-early-out").toString), + scalacOptions += "-Ystop-after:firstTransform", + scalacOptions += "-Ycheck:all", + ) + +// reads classpaths from early tasty outputs. No need for extra flags as the full tasty is available. +lazy val c = project.in(file("c")) + .settings( + Compile / unmanagedClasspath += Attributed.blank((ThisBuild / baseDirectory).value / "a-early.jar"), + Compile / unmanagedClasspath += Attributed.blank((ThisBuild / baseDirectory).value / "b-early-out"), + scalacOptions += "-Ycheck:all", + ) diff --git a/sbt-test/pipelining/Yearly-tasty-output/c/src/main/scala/c/C.scala b/sbt-test/pipelining/Yearly-tasty-output/c/src/main/scala/c/C.scala new file mode 100644 index 000000000000..fd1876088778 --- /dev/null +++ b/sbt-test/pipelining/Yearly-tasty-output/c/src/main/scala/c/C.scala @@ -0,0 +1,9 @@ +package c + +import a.A +import b.B + +object C { + val f: 2 = A.foo(1) + val g: 3 = B.bar(2) +} diff --git a/sbt-test/pipelining/Yearly-tasty-output/project/DottyInjectedPlugin.scala b/sbt-test/pipelining/Yearly-tasty-output/project/DottyInjectedPlugin.scala new file mode 100644 index 000000000000..69f15d168bfc --- /dev/null +++ b/sbt-test/pipelining/Yearly-tasty-output/project/DottyInjectedPlugin.scala @@ -0,0 +1,12 @@ +import sbt._ +import Keys._ + +object DottyInjectedPlugin extends AutoPlugin { + override def requires = plugins.JvmPlugin + override def trigger = allRequirements + + override val projectSettings = Seq( + scalaVersion := sys.props("plugin.scalaVersion"), + scalacOptions += "-source:3.0-migration" + ) +} diff --git a/sbt-test/pipelining/Yearly-tasty-output/test b/sbt-test/pipelining/Yearly-tasty-output/test new file mode 100644 index 000000000000..52d60facc75b --- /dev/null +++ b/sbt-test/pipelining/Yearly-tasty-output/test @@ -0,0 +1,5 @@ +> a/compile +# same as a but with a directory output +> b/compile +# c uses the early output jar of a and b +> c/compile diff --git a/sbt-test/pipelining/Yjava-tasty-annotation/build.sbt b/sbt-test/pipelining/Yjava-tasty-annotation/build.sbt index 18f6b8224968..20a13d7d4ba0 100644 --- a/sbt-test/pipelining/Yjava-tasty-annotation/build.sbt +++ b/sbt-test/pipelining/Yjava-tasty-annotation/build.sbt @@ -1,7 +1,7 @@ lazy val a = project.in(file("a")) .settings( scalacOptions += "-Yjava-tasty", // enable pickling of java signatures - scalacOptions ++= Seq("-Yjava-tasty-output", ((ThisBuild / baseDirectory).value / "a-annotation-java-tasty.jar").toString), + scalacOptions ++= Seq("-Yearly-tasty-output", ((ThisBuild / baseDirectory).value / "a-annotation-java-tasty.jar").toString), scalacOptions += "-Ycheck:all", Compile / classDirectory := ((ThisBuild / baseDirectory).value / "a-annotation-classes"), // send classfiles to a different directory ) diff --git a/sbt-test/pipelining/Yjava-tasty-enum/build.sbt b/sbt-test/pipelining/Yjava-tasty-enum/build.sbt index aca2391987e9..2083003d9ebe 100644 --- a/sbt-test/pipelining/Yjava-tasty-enum/build.sbt +++ b/sbt-test/pipelining/Yjava-tasty-enum/build.sbt @@ -2,7 +2,7 @@ lazy val a = project.in(file("a")) .settings( compileOrder := CompileOrder.Mixed, // ensure we send java sources to Scala compiler scalacOptions += "-Yjava-tasty", // enable pickling of java signatures - scalacOptions ++= Seq("-Yjava-tasty-output", ((ThisBuild / baseDirectory).value / "a-enum-java-tasty.jar").toString), + scalacOptions ++= Seq("-Yearly-tasty-output", ((ThisBuild / baseDirectory).value / "a-enum-java-tasty.jar").toString), scalacOptions += "-Ycheck:all", Compile / classDirectory := ((ThisBuild / baseDirectory).value / "a-enum-classes"), // send classfiles to a different directory ) diff --git a/sbt-test/pipelining/Yjava-tasty-from-tasty/build.sbt b/sbt-test/pipelining/Yjava-tasty-from-tasty/build.sbt index e4b15d3d9c7e..040c3bf6eac8 100644 --- a/sbt-test/pipelining/Yjava-tasty-from-tasty/build.sbt +++ b/sbt-test/pipelining/Yjava-tasty-from-tasty/build.sbt @@ -3,7 +3,7 @@ lazy val a = project.in(file("a")) .settings( compileOrder := CompileOrder.Mixed, // ensure we send java sources to Scala compiler scalacOptions += "-Yjava-tasty", // enable pickling of java signatures - scalacOptions ++= Seq("-Yjava-tasty-output", ((ThisBuild / baseDirectory).value / "a-pre-java-tasty.jar").toString), + scalacOptions ++= Seq("-Yearly-tasty-output", ((ThisBuild / baseDirectory).value / "a-pre-java-tasty.jar").toString), scalacOptions += "-Ycheck:all", Compile / classDirectory := ((ThisBuild / baseDirectory).value / "a-pre-classes"), // send classfiles to a different directory ) @@ -17,7 +17,7 @@ lazy val a_from_tasty = project.in(file("a_from_tasty")) scalacOptions += "-from-tasty", // read the jar file tasties as the source files scalacOptions += "-Yjava-tasty", scalacOptions += "-Yallow-outline-from-tasty", // allow outline signatures to be read with -from-tasty - scalacOptions ++= Seq("-Yjava-tasty-output", ((ThisBuild / baseDirectory).value / "a_from_tasty-java-tasty.jar").toString), + scalacOptions ++= Seq("-Yearly-tasty-output", ((ThisBuild / baseDirectory).value / "a_from_tasty-java-tasty.jar").toString), scalacOptions += "-Ycheck:all", Compile / classDirectory := ((ThisBuild / baseDirectory).value / "a_from_tasty-classes"), // send classfiles to a different directory ) diff --git a/sbt-test/pipelining/Yjava-tasty-fromjavaobject/build.sbt b/sbt-test/pipelining/Yjava-tasty-fromjavaobject/build.sbt index 6738db3016fa..9013490f1f54 100644 --- a/sbt-test/pipelining/Yjava-tasty-fromjavaobject/build.sbt +++ b/sbt-test/pipelining/Yjava-tasty-fromjavaobject/build.sbt @@ -2,7 +2,7 @@ lazy val a = project.in(file("a")) .settings( compileOrder := CompileOrder.Mixed, // ensure we send java sources to Scala compiler scalacOptions += "-Yjava-tasty", // enable pickling of java signatures - scalacOptions ++= Seq("-Yjava-tasty-output", ((ThisBuild / baseDirectory).value / "a-enum-java-tasty.jar").toString), + scalacOptions ++= Seq("-Yearly-tasty-output", ((ThisBuild / baseDirectory).value / "a-enum-java-tasty.jar").toString), scalacOptions += "-Ycheck:all", Compile / classDirectory := ((ThisBuild / baseDirectory).value / "a-enum-classes"), // send classfiles to a different directory ) @@ -14,7 +14,7 @@ lazy val aCheck = project.in(file("a-check")) Compile / sources := (a / Compile / sources).value, // use the same sources as a compileOrder := CompileOrder.Mixed, // ensure we send java sources to Scala compiler scalacOptions += "-Yjava-tasty", // enable pickling of java signatures - scalacOptions ++= Seq("-Yjava-tasty-output", ((ThisBuild / baseDirectory).value / "a-enum-java-tasty-2.jar").toString), + scalacOptions ++= Seq("-Yearly-tasty-output", ((ThisBuild / baseDirectory).value / "a-enum-java-tasty-2.jar").toString), Compile / classDirectory := ((ThisBuild / baseDirectory).value / "a-enum-classes-2"), // send classfiles to a different directory ) diff --git a/sbt-test/pipelining/Yjava-tasty-generic/a/src/main/scala/a/A.java b/sbt-test/pipelining/Yjava-tasty-generic/a/src/main/scala/a/A.java index 1fcb7e78ae3d..c6e7431f0bbe 100644 --- a/sbt-test/pipelining/Yjava-tasty-generic/a/src/main/scala/a/A.java +++ b/sbt-test/pipelining/Yjava-tasty-generic/a/src/main/scala/a/A.java @@ -1,6 +1,8 @@ // this test ensures that it is possible to read a generic java class from TASTy. package a; +import java.lang.Object; + public abstract class A { private final int _value; @@ -11,4 +13,8 @@ protected A(final int value) { public int value() { return _value; } + + public int hash(Object any) { + return any.hashCode(); + } } diff --git a/sbt-test/pipelining/Yjava-tasty-generic/b/src/main/scala/b/B.scala b/sbt-test/pipelining/Yjava-tasty-generic/b/src/main/scala/b/B.scala index f132e012a5fc..62e58aa72f94 100644 --- a/sbt-test/pipelining/Yjava-tasty-generic/b/src/main/scala/b/B.scala +++ b/sbt-test/pipelining/Yjava-tasty-generic/b/src/main/scala/b/B.scala @@ -7,9 +7,15 @@ class B[T] { } object B { + + val someAny: Any = 23 + + val inner = (new B[Int]).inner + @main def test = { - val derived: Int = (new B[Int]).inner.value + val derived: Int = inner.value assert(derived == 23, s"actually was $derived") + assert(inner.hash(someAny) == someAny.hashCode, s"actually was ${inner.hash(someAny)}") } } diff --git a/sbt-test/pipelining/Yjava-tasty-generic/build.sbt b/sbt-test/pipelining/Yjava-tasty-generic/build.sbt index 07e2ea56fbaa..9e2796600333 100644 --- a/sbt-test/pipelining/Yjava-tasty-generic/build.sbt +++ b/sbt-test/pipelining/Yjava-tasty-generic/build.sbt @@ -1,7 +1,7 @@ lazy val a = project.in(file("a")) .settings( scalacOptions += "-Yjava-tasty", // enable pickling of java signatures - scalacOptions ++= Seq("-Yjava-tasty-output", ((ThisBuild / baseDirectory).value / "a-generic-java-tasty.jar").toString), + scalacOptions ++= Seq("-Yearly-tasty-output", ((ThisBuild / baseDirectory).value / "a-generic-java-tasty.jar").toString), scalacOptions += "-Ycheck:all", Compile / classDirectory := ((ThisBuild / baseDirectory).value / "a-generic-classes"), // send classfiles to a different directory ) diff --git a/sbt-test/pipelining/Yjava-tasty-paths/build.sbt b/sbt-test/pipelining/Yjava-tasty-paths/build.sbt index d63d1f9a3f7e..49487fccb57e 100644 --- a/sbt-test/pipelining/Yjava-tasty-paths/build.sbt +++ b/sbt-test/pipelining/Yjava-tasty-paths/build.sbt @@ -1,7 +1,7 @@ lazy val a = project.in(file("a")) .settings( scalacOptions += "-Yjava-tasty", // enable pickling of java signatures - scalacOptions ++= Seq("-Yjava-tasty-output", ((ThisBuild / baseDirectory).value / "a-paths-java-tasty.jar").toString), + scalacOptions ++= Seq("-Yearly-tasty-output", ((ThisBuild / baseDirectory).value / "a-paths-java-tasty.jar").toString), scalacOptions += "-Ycheck:all", Compile / classDirectory := ((ThisBuild / baseDirectory).value / "a-paths-classes"), // send classfiles to a different directory ) diff --git a/sbt-test/pipelining/Yjava-tasty-result-types/build.sbt b/sbt-test/pipelining/Yjava-tasty-result-types/build.sbt index 512344f0635b..80bcf71b3365 100644 --- a/sbt-test/pipelining/Yjava-tasty-result-types/build.sbt +++ b/sbt-test/pipelining/Yjava-tasty-result-types/build.sbt @@ -1,7 +1,7 @@ lazy val a = project.in(file("a")) .settings( scalacOptions += "-Yjava-tasty", // enable pickling of java signatures - scalacOptions ++= Seq("-Yjava-tasty-output", ((ThisBuild / baseDirectory).value / "a-result-types-java-tasty.jar").toString), + scalacOptions ++= Seq("-Yearly-tasty-output", ((ThisBuild / baseDirectory).value / "a-result-types-java-tasty.jar").toString), scalacOptions += "-Ycheck:all", Compile / classDirectory := ((ThisBuild / baseDirectory).value / "a-result-types-classes"), // send classfiles to a different directory ) diff --git a/sbt-test/pipelining/pipelining-changes/build.sbt b/sbt-test/pipelining/pipelining-changes/build.sbt new file mode 100644 index 000000000000..630bd4be5b3e --- /dev/null +++ b/sbt-test/pipelining/pipelining-changes/build.sbt @@ -0,0 +1,27 @@ +import sbt.internal.inc.Analysis +import complete.DefaultParsers._ + +ThisBuild / usePipelining := true + +// Reset compiler iterations, necessary because tests run in batch mode +val recordPreviousIterations = taskKey[Unit]("Record previous iterations.") +recordPreviousIterations := { + val log = streams.value.log + CompileState.previousIterations = { + val previousAnalysis = (previousCompile in Compile).value.analysis.asScala + previousAnalysis match { + case None => + log.info("No previous analysis detected") + 0 + case Some(a: Analysis) => a.compilations.allCompilations.size + } + } +} + +val checkIterations = inputKey[Unit]("Verifies the accumulated number of iterations of incremental compilation.") + +checkIterations := { + val expected: Int = (Space ~> NatBasic).parsed + val actual: Int = ((compile in Compile).value match { case a: Analysis => a.compilations.allCompilations.size }) - CompileState.previousIterations + assert(expected == actual, s"Expected $expected compilations, got $actual (previous: ${CompileState.previousIterations})") +} diff --git a/sbt-test/pipelining/pipelining-changes/changes/A1.scala b/sbt-test/pipelining/pipelining-changes/changes/A1.scala new file mode 100644 index 000000000000..db5605e419d1 --- /dev/null +++ b/sbt-test/pipelining/pipelining-changes/changes/A1.scala @@ -0,0 +1,5 @@ +package a + +enum A { + case A, B +} diff --git a/sbt-test/pipelining/pipelining-changes/project/CompileState.scala b/sbt-test/pipelining/pipelining-changes/project/CompileState.scala new file mode 100644 index 000000000000..078db9c7bf56 --- /dev/null +++ b/sbt-test/pipelining/pipelining-changes/project/CompileState.scala @@ -0,0 +1,4 @@ +// This is necessary because tests are run in batch mode +object CompileState { + @volatile var previousIterations: Int = -1 +} diff --git a/sbt-test/pipelining/pipelining-changes/project/DottyInjectedPlugin.scala b/sbt-test/pipelining/pipelining-changes/project/DottyInjectedPlugin.scala new file mode 100644 index 000000000000..1c6c00400f04 --- /dev/null +++ b/sbt-test/pipelining/pipelining-changes/project/DottyInjectedPlugin.scala @@ -0,0 +1,11 @@ +import sbt._ +import Keys._ + +object DottyInjectedPlugin extends AutoPlugin { + override def requires = plugins.JvmPlugin + override def trigger = allRequirements + + override val projectSettings = Seq( + scalaVersion := sys.props("plugin.scalaVersion"), + ) +} diff --git a/sbt-test/pipelining/pipelining-changes/src/main/scala/a/A.scala b/sbt-test/pipelining/pipelining-changes/src/main/scala/a/A.scala new file mode 100644 index 000000000000..4a0eec46ec7e --- /dev/null +++ b/sbt-test/pipelining/pipelining-changes/src/main/scala/a/A.scala @@ -0,0 +1,5 @@ +package a + +enum A { + case A +} diff --git a/sbt-test/pipelining/pipelining-changes/src/main/scala/a/App.scala b/sbt-test/pipelining/pipelining-changes/src/main/scala/a/App.scala new file mode 100644 index 000000000000..a9862cea9dc4 --- /dev/null +++ b/sbt-test/pipelining/pipelining-changes/src/main/scala/a/App.scala @@ -0,0 +1,11 @@ +package a + +import scala.deriving.Mirror + +object App { + val m = summon[Mirror.SumOf[a.A]] + def size = compiletime.constValue[Tuple.Size[m.MirroredElemTypes]] + + @main def test = + assert(size == 2, s"Expected size 2, got $size") +} diff --git a/sbt-test/pipelining/pipelining-changes/test b/sbt-test/pipelining/pipelining-changes/test new file mode 100644 index 000000000000..e6fb01d57f5a --- /dev/null +++ b/sbt-test/pipelining/pipelining-changes/test @@ -0,0 +1,7 @@ +# test the interaction of incremental compilation and pipelining +> compile +> recordPreviousIterations +$ copy-file changes/A1.scala src/main/scala/a/A.scala +# A recompilation should trigger recompilation of App.scala, otherwise test assert will fail +> run +> checkIterations 2 diff --git a/sbt-test/pipelining/pipelining-scala-inline/a/src/main/scala/a/A.scala b/sbt-test/pipelining/pipelining-scala-inline/a/src/main/scala/a/A.scala new file mode 100644 index 000000000000..c2dfb3e2c886 --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-inline/a/src/main/scala/a/A.scala @@ -0,0 +1,8 @@ +package a + +object A { + inline def power(x: Double, inline n: Int): Double = + inline if (n == 0) 1.0 + else inline if (n % 2 == 1) x * power(x, n - 1) + else power(x * x, n / 2) +} diff --git a/sbt-test/pipelining/pipelining-scala-inline/b/src/main/scala/b/B.scala b/sbt-test/pipelining/pipelining-scala-inline/b/src/main/scala/b/B.scala new file mode 100644 index 000000000000..7055d6d2d006 --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-inline/b/src/main/scala/b/B.scala @@ -0,0 +1,10 @@ +package b + +import a.A + +object B { + @main def run = + assert(A.power(2.0, 2) == 4.0) + assert(A.power(2.0, 3) == 8.0) + assert(A.power(2.0, 4) == 16.0) +} diff --git a/sbt-test/pipelining/pipelining-scala-inline/build.sbt b/sbt-test/pipelining/pipelining-scala-inline/build.sbt new file mode 100644 index 000000000000..cd2a0c4eef07 --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-inline/build.sbt @@ -0,0 +1,35 @@ +ThisBuild / usePipelining := true + +// defines a purely inline function, and we always force the early output, this should not be needed in practice +// because pure inline methods do not have a Macro flag. +lazy val a = project.in(file("a")) + .settings( + scalacOptions += "-Ycheck:all", + Compile / incOptions := { + val old = (Compile / incOptions).value + val hooks = old.externalHooks + val newHooks = hooks.withExternalLookup( + new sbt.internal.inc.NoopExternalLookup { + // assert that the analysis contains the class `a.A` and that it does not have a macro. + override def shouldDoEarlyOutput(analysis: xsbti.compile.CompileAnalysis): Boolean = { + val internalClasses = analysis.asInstanceOf[sbt.internal.inc.Analysis].apis.internal + val a_A = internalClasses.get("a.A") + assert(a_A.exists(cls => !cls.hasMacro), "`a.A` wasn't found, or it had a macro.") + + // returning true will force the early output ping and activate downstream pipelining, + // this is fine for inline methods, but see `sbt-test/pipelining/pipelining-scala-macro-fail` for how + // we can force a failure by returning true here. + true + } + } + ) + old.withExternalHooks(newHooks) + }, + ) + +// uses the purely inline function +lazy val b = project.in(file("b")) + .dependsOn(a) + .settings( + scalacOptions += "-Ycheck:all", + ) diff --git a/sbt-test/pipelining/pipelining-scala-inline/project/DottyInjectedPlugin.scala b/sbt-test/pipelining/pipelining-scala-inline/project/DottyInjectedPlugin.scala new file mode 100644 index 000000000000..69f15d168bfc --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-inline/project/DottyInjectedPlugin.scala @@ -0,0 +1,12 @@ +import sbt._ +import Keys._ + +object DottyInjectedPlugin extends AutoPlugin { + override def requires = plugins.JvmPlugin + override def trigger = allRequirements + + override val projectSettings = Seq( + scalaVersion := sys.props("plugin.scalaVersion"), + scalacOptions += "-source:3.0-migration" + ) +} diff --git a/sbt-test/pipelining/pipelining-scala-inline/test b/sbt-test/pipelining/pipelining-scala-inline/test new file mode 100644 index 000000000000..48a2443830b5 --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-inline/test @@ -0,0 +1 @@ +> b/run diff --git a/sbt-test/pipelining/pipelining-scala-java-basic/a/src/main/scala/a/A.scala b/sbt-test/pipelining/pipelining-scala-java-basic/a/src/main/scala/a/A.scala new file mode 100644 index 000000000000..4b10db3eb385 --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-java-basic/a/src/main/scala/a/A.scala @@ -0,0 +1,5 @@ +package a + +object A { + val foo: (1,2,3) = (1,2,3) +} diff --git a/sbt-test/pipelining/pipelining-scala-java-basic/b/src/main/scala/b/B.java b/sbt-test/pipelining/pipelining-scala-java-basic/b/src/main/scala/b/B.java new file mode 100644 index 000000000000..7cac88d3cd46 --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-java-basic/b/src/main/scala/b/B.java @@ -0,0 +1,5 @@ +package b; + +public class B { + public static final String VALUE = "B"; +} diff --git a/sbt-test/pipelining/pipelining-scala-java-basic/build.sbt b/sbt-test/pipelining/pipelining-scala-java-basic/build.sbt new file mode 100644 index 000000000000..2b49443ae8f0 --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-java-basic/build.sbt @@ -0,0 +1,17 @@ +ThisBuild / usePipelining := true + +lazy val a = project.in(file("a")) + .settings( + scalacOptions += "-Ycheck:all", + ) + +lazy val b = project.in(file("b")) + .settings( + scalacOptions += "-Ycheck:all", + ) + +lazy val c = project.in(file("c")) + .dependsOn(a, b) + .settings( + scalacOptions += "-Ycheck:all", + ) diff --git a/sbt-test/pipelining/pipelining-scala-java-basic/c/src/main/scala/c/C.scala b/sbt-test/pipelining/pipelining-scala-java-basic/c/src/main/scala/c/C.scala new file mode 100644 index 000000000000..b8e23e0b5920 --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-java-basic/c/src/main/scala/c/C.scala @@ -0,0 +1,15 @@ +package c + +import a.A +import b.B + +object C { + val c_1: 2 = A.foo(1) + val c_2: "B" = B.VALUE + + @main def run = + assert(A.foo(0) == 1) + assert(A.foo(1) == 2) + assert(A.foo(2) == 3) + assert(B.VALUE == "B") +} diff --git a/sbt-test/pipelining/pipelining-scala-java-basic/project/DottyInjectedPlugin.scala b/sbt-test/pipelining/pipelining-scala-java-basic/project/DottyInjectedPlugin.scala new file mode 100644 index 000000000000..69f15d168bfc --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-java-basic/project/DottyInjectedPlugin.scala @@ -0,0 +1,12 @@ +import sbt._ +import Keys._ + +object DottyInjectedPlugin extends AutoPlugin { + override def requires = plugins.JvmPlugin + override def trigger = allRequirements + + override val projectSettings = Seq( + scalaVersion := sys.props("plugin.scalaVersion"), + scalacOptions += "-source:3.0-migration" + ) +} diff --git a/sbt-test/pipelining/pipelining-scala-java-basic/test b/sbt-test/pipelining/pipelining-scala-java-basic/test new file mode 100644 index 000000000000..77f2017c835f --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-java-basic/test @@ -0,0 +1 @@ +> c/run diff --git a/sbt-test/pipelining/pipelining-scala-macro-fail/a/src/main/scala/a/A.scala b/sbt-test/pipelining/pipelining-scala-macro-fail/a/src/main/scala/a/A.scala new file mode 100644 index 000000000000..d98a9d2c1159 --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-macro-fail/a/src/main/scala/a/A.scala @@ -0,0 +1,18 @@ +package a + +import scala.quoted.* + +object A { + + inline def power(x: Double, inline n: Int): Double = + ${ powerCode('x, 'n) } + + def powerCode(x: Expr[Double], n: Expr[Int])(using Quotes): Expr[Double] = { + def impl(x: Double, n: Int): Double = + if (n == 0) 1.0 + else if (n % 2 == 1) x * impl(x, n - 1) + else impl(x * x, n / 2) + + Expr(impl(x.valueOrError, n.valueOrError)) + } +} diff --git a/sbt-test/pipelining/pipelining-scala-macro-fail/b/src/main/scala/b/B.scala b/sbt-test/pipelining/pipelining-scala-macro-fail/b/src/main/scala/b/B.scala new file mode 100644 index 000000000000..7055d6d2d006 --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-macro-fail/b/src/main/scala/b/B.scala @@ -0,0 +1,10 @@ +package b + +import a.A + +object B { + @main def run = + assert(A.power(2.0, 2) == 4.0) + assert(A.power(2.0, 3) == 8.0) + assert(A.power(2.0, 4) == 16.0) +} diff --git a/sbt-test/pipelining/pipelining-scala-macro-fail/build.sbt b/sbt-test/pipelining/pipelining-scala-macro-fail/build.sbt new file mode 100644 index 000000000000..c98e664af507 --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-macro-fail/build.sbt @@ -0,0 +1,28 @@ +ThisBuild / usePipelining := true + +// defines a macro, normally this would cause sbt not to write the early output jar, but we force it +// this will cause b to fail to compile due to the missing macro class, +// see `sbt-test/pipelining/pipelining-scala-macro` for how by default sbt does the right thing +lazy val a = project.in(file("a")) + .settings( + scalacOptions += "-Ycheck:all", + Compile / incOptions := { + val old = (Compile / incOptions).value + val hooks = old.externalHooks + val newHooks = hooks.withExternalLookup( + new sbt.internal.inc.NoopExternalLookup { + // force early output, this is safe in projects where the macro implementation is not in the same project, + // however in this build, b will now fail as it will not find the macro implementation class. + override def shouldDoEarlyOutput(analysis: xsbti.compile.CompileAnalysis): Boolean = true + } + ) + old.withExternalHooks(newHooks) + }, + ) + +// uses the macro, this will fail because we forced early output ping, causing the missing macro implementation class +lazy val b = project.in(file("b")) + .dependsOn(a) + .settings( + scalacOptions += "-Ycheck:all", + ) diff --git a/sbt-test/pipelining/pipelining-scala-macro-fail/project/DottyInjectedPlugin.scala b/sbt-test/pipelining/pipelining-scala-macro-fail/project/DottyInjectedPlugin.scala new file mode 100644 index 000000000000..69f15d168bfc --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-macro-fail/project/DottyInjectedPlugin.scala @@ -0,0 +1,12 @@ +import sbt._ +import Keys._ + +object DottyInjectedPlugin extends AutoPlugin { + override def requires = plugins.JvmPlugin + override def trigger = allRequirements + + override val projectSettings = Seq( + scalaVersion := sys.props("plugin.scalaVersion"), + scalacOptions += "-source:3.0-migration" + ) +} diff --git a/sbt-test/pipelining/pipelining-scala-macro-fail/test b/sbt-test/pipelining/pipelining-scala-macro-fail/test new file mode 100644 index 000000000000..13daffd6dfa0 --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-macro-fail/test @@ -0,0 +1,2 @@ +> a/compile +-> b/compile diff --git a/sbt-test/pipelining/pipelining-scala-macro-force/a/src/main/scala/a/A.scala b/sbt-test/pipelining/pipelining-scala-macro-force/a/src/main/scala/a/A.scala new file mode 100644 index 000000000000..520aec03482a --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-macro-force/a/src/main/scala/a/A.scala @@ -0,0 +1,13 @@ +package a + +import scala.quoted.* + +object A { + + transparent inline def transparentPower(x: Double, inline n: Int): Double = + ${ macros.MacroImpl.powerCode('x, 'n) } + + inline def power(x: Double, inline n: Int): Double = + ${ macros.MacroImpl.powerCode('x, 'n) } + +} diff --git a/sbt-test/pipelining/pipelining-scala-macro-force/a/src/main/scala/a/AConsume.scala b/sbt-test/pipelining/pipelining-scala-macro-force/a/src/main/scala/a/AConsume.scala new file mode 100644 index 000000000000..1a4b0c234910 --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-macro-force/a/src/main/scala/a/AConsume.scala @@ -0,0 +1,5 @@ +package a + +object AConsume { + def sixtyFour: Double = A.power(2.0, 6) // cause a suspension in inlining +} diff --git a/sbt-test/pipelining/pipelining-scala-macro-force/a/src/main/scala/a/AConsumeTransparent.scala b/sbt-test/pipelining/pipelining-scala-macro-force/a/src/main/scala/a/AConsumeTransparent.scala new file mode 100644 index 000000000000..cbd356047c4d --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-macro-force/a/src/main/scala/a/AConsumeTransparent.scala @@ -0,0 +1,5 @@ +package a + +object AConsumeTransparent { + def thirtyTwo: Double = A.transparentPower(2.0, 5) // cause a suspension in typer +} diff --git a/sbt-test/pipelining/pipelining-scala-macro-force/b/src/main/scala/b/B.scala b/sbt-test/pipelining/pipelining-scala-macro-force/b/src/main/scala/b/B.scala new file mode 100644 index 000000000000..7955b1d7cfbb --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-macro-force/b/src/main/scala/b/B.scala @@ -0,0 +1,14 @@ +package b + +import a.A +import a.AConsumeTransparent +import a.AConsume + +object B { + @main def run = + assert(A.power(2.0, 2) == 4.0) + assert(A.power(2.0, 3) == 8.0) + assert(A.power(2.0, 4) == 16.0) + assert(AConsumeTransparent.thirtyTwo == 32.0) // these are not actually suspended in this project + assert(AConsume.sixtyFour == 64.0) // check that suspended definition is still available +} diff --git a/sbt-test/pipelining/pipelining-scala-macro-force/build.sbt b/sbt-test/pipelining/pipelining-scala-macro-force/build.sbt new file mode 100644 index 000000000000..57d3898aa683 --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-macro-force/build.sbt @@ -0,0 +1,48 @@ +ThisBuild / usePipelining := true + +// defines just the macro implementations +lazy val macros = project.in(file("macros")) + .settings( + scalacOptions += "-Ycheck:all", + Compile / exportPipelining := false // downstream waits until classfiles are available + ) + +// defines a macro, we need to force sbt to produce the early output jar +// because it will detect macros in the analysis. +// However the classes for the implementation are provided by `macros` +lazy val a = project.in(file("a")) + .dependsOn(macros) + .settings( + scalacOptions += "-Youtline", + scalacOptions += "-Ymax-parallelism:1", + scalacOptions += "-Ycheck:all", + scalacOptions += "-Yno-suspended-units", + scalacOptions += "-Xprint-suspension", + Compile / incOptions := { + val old = (Compile / incOptions).value + val hooks = old.externalHooks + val newHooks = hooks.withExternalLookup( + new sbt.internal.inc.NoopExternalLookup { + // force early output, this is safe because the macro class from `macros` will be available. + override def shouldDoEarlyOutput(analysis: xsbti.compile.CompileAnalysis): Boolean = { + val internalClasses = analysis.asInstanceOf[sbt.internal.inc.Analysis].apis.internal + val a_A = internalClasses.get("a.A") + val a_AConsume = internalClasses.get("a.AConsume") + val a_AConsumeTransparent = internalClasses.get("a.AConsumeTransparent") + assert(a_A.exists(cls => cls.hasMacro), s"`a.A` wasn't found, or it didn't have a macro.") + assert(a_AConsume.isDefined, s"`a.AConsume` wasn't found.") + assert(a_AConsumeTransparent.isDefined, s"`a.AConsumeTransparent` wasn't found.") + true // because `a.A` has macros, normally this would be false + } + } + ) + old.withExternalHooks(newHooks) + }, + ) + +// uses the macro, will still succeed as the macro implementation class is available +lazy val b = project.in(file("b")) + .dependsOn(a) + .settings( + scalacOptions += "-Ycheck:all", + ) diff --git a/sbt-test/pipelining/pipelining-scala-macro-force/macros/src/main/scala/macros/MacroImpl.scala b/sbt-test/pipelining/pipelining-scala-macro-force/macros/src/main/scala/macros/MacroImpl.scala new file mode 100644 index 000000000000..d7c03aaf0ae0 --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-macro-force/macros/src/main/scala/macros/MacroImpl.scala @@ -0,0 +1,15 @@ +package macros + +import scala.quoted.* + +object MacroImpl { + + def powerCode(x: Expr[Double], n: Expr[Int])(using Quotes): Expr[Double] = { + def impl(x: Double, n: Int): Double = + if (n == 0) 1.0 + else if (n % 2 == 1) x * impl(x, n - 1) + else impl(x * x, n / 2) + + Expr(impl(x.valueOrError, n.valueOrError)) + } +} diff --git a/sbt-test/pipelining/pipelining-scala-macro-force/project/DottyInjectedPlugin.scala b/sbt-test/pipelining/pipelining-scala-macro-force/project/DottyInjectedPlugin.scala new file mode 100644 index 000000000000..69f15d168bfc --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-macro-force/project/DottyInjectedPlugin.scala @@ -0,0 +1,12 @@ +import sbt._ +import Keys._ + +object DottyInjectedPlugin extends AutoPlugin { + override def requires = plugins.JvmPlugin + override def trigger = allRequirements + + override val projectSettings = Seq( + scalaVersion := sys.props("plugin.scalaVersion"), + scalacOptions += "-source:3.0-migration" + ) +} diff --git a/sbt-test/pipelining/pipelining-scala-macro-force/test b/sbt-test/pipelining/pipelining-scala-macro-force/test new file mode 100644 index 000000000000..48a2443830b5 --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-macro-force/test @@ -0,0 +1 @@ +> b/run diff --git a/sbt-test/pipelining/pipelining-scala-macro/a/src/main/scala/a/A.scala b/sbt-test/pipelining/pipelining-scala-macro/a/src/main/scala/a/A.scala new file mode 100644 index 000000000000..9077f0a2e849 --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-macro/a/src/main/scala/a/A.scala @@ -0,0 +1,21 @@ +package a + +import scala.quoted.* + +object A { + + transparent inline def transparentPower(x: Double, inline n: Int): Double = + ${ powerCode('x, 'n) } + + inline def power(x: Double, inline n: Int): Double = + ${ powerCode('x, 'n) } + + def powerCode(x: Expr[Double], n: Expr[Int])(using Quotes): Expr[Double] = { + def impl(x: Double, n: Int): Double = + if (n == 0) 1.0 + else if (n % 2 == 1) x * impl(x, n - 1) + else impl(x * x, n / 2) + + Expr(impl(x.valueOrError, n.valueOrError)) + } +} diff --git a/sbt-test/pipelining/pipelining-scala-macro/a/src/main/scala/a/ASuspendInlining.scala b/sbt-test/pipelining/pipelining-scala-macro/a/src/main/scala/a/ASuspendInlining.scala new file mode 100644 index 000000000000..0fa449601d31 --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-macro/a/src/main/scala/a/ASuspendInlining.scala @@ -0,0 +1,5 @@ +package a + +object ASuspendInlining { + def sixtyFour: Double = A.power(2.0, 6) // cause a suspension in inlining +} diff --git a/sbt-test/pipelining/pipelining-scala-macro/a/src/main/scala/a/ASuspendTyper.scala b/sbt-test/pipelining/pipelining-scala-macro/a/src/main/scala/a/ASuspendTyper.scala new file mode 100644 index 000000000000..2af5139b30bc --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-macro/a/src/main/scala/a/ASuspendTyper.scala @@ -0,0 +1,5 @@ +package a + +object ASuspendTyper { + def thirtyTwo: Double = A.transparentPower(2.0, 5) // cause a suspension in typer +} diff --git a/sbt-test/pipelining/pipelining-scala-macro/b/src/main/scala/b/B.scala b/sbt-test/pipelining/pipelining-scala-macro/b/src/main/scala/b/B.scala new file mode 100644 index 000000000000..17f72ddf1644 --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-macro/b/src/main/scala/b/B.scala @@ -0,0 +1,14 @@ +package b + +import a.A +import a.ASuspendTyper +import a.ASuspendInlining + +object B { + @main def run = + assert(A.power(2.0, 2) == 4.0) + assert(A.power(2.0, 3) == 8.0) + assert(A.power(2.0, 4) == 16.0) + assert(ASuspendTyper.thirtyTwo == 32.0) // check that suspended definition is still available + assert(ASuspendInlining.sixtyFour == 64.0) // check that suspended definition is still available +} diff --git a/sbt-test/pipelining/pipelining-scala-macro/build.sbt b/sbt-test/pipelining/pipelining-scala-macro/build.sbt new file mode 100644 index 000000000000..58e513bdf163 --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-macro/build.sbt @@ -0,0 +1,57 @@ +ThisBuild / usePipelining := true + +// defines a macro, sbt will not force the early output +// because it will detect macros in the analysis, so b will compile fine, +// see `sbt-test/pipelining/pipelining-scala-macro-fail` for how we can +// force a failure by always forcing early output. +lazy val a = project.in(file("a")) + .settings( + scalacOptions += "-Ycheck:all", + scalacOptions += "-Xprint-suspension", + exportPipelining := false, + Compile / incOptions := { + val old = (Compile / incOptions).value + val hooks = old.externalHooks + val newHooks = hooks.withExternalLookup( + new sbt.internal.inc.NoopExternalLookup { + @volatile var knownSuspension = false + + def didFindMacros(analysis: xsbti.compile.CompileAnalysis) = { + val foundMacros = analysis.asInstanceOf[sbt.internal.inc.Analysis].apis.internal.values.exists(_.hasMacro) + assert(foundMacros, "expected macros to be found in the analysis.") + foundMacros + } + + // force early output, this is safe because the macro class from `macros` will be available. + override def shouldDoEarlyOutput(analysis: xsbti.compile.CompileAnalysis): Boolean = { + val internalClasses = analysis.asInstanceOf[sbt.internal.inc.Analysis].apis.internal + val a_A = internalClasses.get("a.A") + val a_ASuspendTyper = internalClasses.get("a.ASuspendTyper") + val a_ASuspendInlining = internalClasses.get("a.ASuspendInlining") + assert(a_A.isDefined, s"`a.A` wasn't found.") + + if (!knownSuspension) { + // this callback is called multiple times, so we only want to assert the first time, + // in subsequent runs the suspended definition will be "resumed", so a.ASuspendTyper be found. + knownSuspension = true + assert(a_ASuspendTyper.isEmpty, s"`a.ASuspendTyper` should have been suspended initially.") + } + + assert(a_ASuspendInlining.isDefined, s"`a.ASuspendInlining` wasn't found.") + + // do what sbt does typically, + // it will not force early output because macros are found + !didFindMacros(analysis) + } + } + ) + old.withExternalHooks(newHooks) + }, + ) + +// uses the macro, sbt is smart enough to not use pipelining flags when upstream compilation has macros +lazy val b = project.in(file("b")) + .dependsOn(a) + .settings( + scalacOptions += "-Ycheck:all", + ) diff --git a/sbt-test/pipelining/pipelining-scala-macro/project/DottyInjectedPlugin.scala b/sbt-test/pipelining/pipelining-scala-macro/project/DottyInjectedPlugin.scala new file mode 100644 index 000000000000..69f15d168bfc --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-macro/project/DottyInjectedPlugin.scala @@ -0,0 +1,12 @@ +import sbt._ +import Keys._ + +object DottyInjectedPlugin extends AutoPlugin { + override def requires = plugins.JvmPlugin + override def trigger = allRequirements + + override val projectSettings = Seq( + scalaVersion := sys.props("plugin.scalaVersion"), + scalacOptions += "-source:3.0-migration" + ) +} diff --git a/sbt-test/pipelining/pipelining-scala-macro/test b/sbt-test/pipelining/pipelining-scala-macro/test new file mode 100644 index 000000000000..48a2443830b5 --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-macro/test @@ -0,0 +1 @@ +> b/run diff --git a/sbt-test/pipelining/pipelining-scala-only-outline/a/src/main/scala/a/A.scala b/sbt-test/pipelining/pipelining-scala-only-outline/a/src/main/scala/a/A.scala new file mode 100644 index 000000000000..4b10db3eb385 --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-only-outline/a/src/main/scala/a/A.scala @@ -0,0 +1,5 @@ +package a + +object A { + val foo: (1,2,3) = (1,2,3) +} diff --git a/sbt-test/pipelining/pipelining-scala-only-outline/b/src/main/scala/b/B.scala b/sbt-test/pipelining/pipelining-scala-only-outline/b/src/main/scala/b/B.scala new file mode 100644 index 000000000000..971d07d5656d --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-only-outline/b/src/main/scala/b/B.scala @@ -0,0 +1,12 @@ +package b + +import a.A + +object B { + val b: 2 = A.foo(1) + + @main def run = + assert(A.foo(0) == 1) + assert(A.foo(1) == 2) + assert(A.foo(2) == 3) +} diff --git a/sbt-test/pipelining/pipelining-scala-only-outline/build.sbt b/sbt-test/pipelining/pipelining-scala-only-outline/build.sbt new file mode 100644 index 000000000000..26fa3eaa07f8 --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-only-outline/build.sbt @@ -0,0 +1,16 @@ +ThisBuild / usePipelining := true + +lazy val a = project.in(file("a")) + .settings( + scalacOptions += "-Yexperimental-outline", + scalacOptions += "-Ymax-parallelism:1", + scalacOptions += "-Ycheck:all", + ) + +lazy val b = project.in(file("b")) + .dependsOn(a) + .settings( + scalacOptions += "-Yexperimental-outline", + scalacOptions += "-Ymax-parallelism:1", + scalacOptions += "-Ycheck:all", + ) diff --git a/sbt-test/pipelining/pipelining-scala-only-outline/project/DottyInjectedPlugin.scala b/sbt-test/pipelining/pipelining-scala-only-outline/project/DottyInjectedPlugin.scala new file mode 100644 index 000000000000..69f15d168bfc --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-only-outline/project/DottyInjectedPlugin.scala @@ -0,0 +1,12 @@ +import sbt._ +import Keys._ + +object DottyInjectedPlugin extends AutoPlugin { + override def requires = plugins.JvmPlugin + override def trigger = allRequirements + + override val projectSettings = Seq( + scalaVersion := sys.props("plugin.scalaVersion"), + scalacOptions += "-source:3.0-migration" + ) +} diff --git a/sbt-test/pipelining/pipelining-scala-only-outline/test b/sbt-test/pipelining/pipelining-scala-only-outline/test new file mode 100644 index 000000000000..48a2443830b5 --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-only-outline/test @@ -0,0 +1 @@ +> b/run diff --git a/sbt-test/pipelining/pipelining-scala-only/a/src/main/scala/a/A.scala b/sbt-test/pipelining/pipelining-scala-only/a/src/main/scala/a/A.scala new file mode 100644 index 000000000000..4b10db3eb385 --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-only/a/src/main/scala/a/A.scala @@ -0,0 +1,5 @@ +package a + +object A { + val foo: (1,2,3) = (1,2,3) +} diff --git a/sbt-test/pipelining/pipelining-scala-only/b/src/main/scala/b/B.scala b/sbt-test/pipelining/pipelining-scala-only/b/src/main/scala/b/B.scala new file mode 100644 index 000000000000..971d07d5656d --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-only/b/src/main/scala/b/B.scala @@ -0,0 +1,12 @@ +package b + +import a.A + +object B { + val b: 2 = A.foo(1) + + @main def run = + assert(A.foo(0) == 1) + assert(A.foo(1) == 2) + assert(A.foo(2) == 3) +} diff --git a/sbt-test/pipelining/pipelining-scala-only/build.sbt b/sbt-test/pipelining/pipelining-scala-only/build.sbt new file mode 100644 index 000000000000..16e182e48801 --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-only/build.sbt @@ -0,0 +1,12 @@ +ThisBuild / usePipelining := true + +lazy val a = project.in(file("a")) + .settings( + scalacOptions += "-Ycheck:all", + ) + +lazy val b = project.in(file("b")) + .dependsOn(a) + .settings( + scalacOptions += "-Ycheck:all", + ) diff --git a/sbt-test/pipelining/pipelining-scala-only/project/DottyInjectedPlugin.scala b/sbt-test/pipelining/pipelining-scala-only/project/DottyInjectedPlugin.scala new file mode 100644 index 000000000000..69f15d168bfc --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-only/project/DottyInjectedPlugin.scala @@ -0,0 +1,12 @@ +import sbt._ +import Keys._ + +object DottyInjectedPlugin extends AutoPlugin { + override def requires = plugins.JvmPlugin + override def trigger = allRequirements + + override val projectSettings = Seq( + scalaVersion := sys.props("plugin.scalaVersion"), + scalacOptions += "-source:3.0-migration" + ) +} diff --git a/sbt-test/pipelining/pipelining-scala-only/test b/sbt-test/pipelining/pipelining-scala-only/test new file mode 100644 index 000000000000..48a2443830b5 --- /dev/null +++ b/sbt-test/pipelining/pipelining-scala-only/test @@ -0,0 +1 @@ +> b/run diff --git a/sbt-test/pipelining/pipelining-test/a/src/main/scala/a/A.scala b/sbt-test/pipelining/pipelining-test/a/src/main/scala/a/A.scala new file mode 100644 index 000000000000..4b10db3eb385 --- /dev/null +++ b/sbt-test/pipelining/pipelining-test/a/src/main/scala/a/A.scala @@ -0,0 +1,5 @@ +package a + +object A { + val foo: (1,2,3) = (1,2,3) +} diff --git a/sbt-test/pipelining/pipelining-test/a/src/test/scala/a/Hello.scala b/sbt-test/pipelining/pipelining-test/a/src/test/scala/a/Hello.scala new file mode 100644 index 000000000000..1cfa3424bd98 --- /dev/null +++ b/sbt-test/pipelining/pipelining-test/a/src/test/scala/a/Hello.scala @@ -0,0 +1,12 @@ +package a + +import a.A + +import org.junit.Test + +class Hello { + + @Test def test(): Unit = { + assert(A.foo == (1,2,3)) + } +} diff --git a/sbt-test/pipelining/pipelining-test/build.sbt b/sbt-test/pipelining/pipelining-test/build.sbt new file mode 100644 index 000000000000..576ecc793ac6 --- /dev/null +++ b/sbt-test/pipelining/pipelining-test/build.sbt @@ -0,0 +1,7 @@ +ThisBuild / usePipelining := true + +lazy val a = project.in(file("a")) + .settings( + scalacOptions += "-Ycheck:all", + libraryDependencies += "com.novocode" % "junit-interface" % "0.11" % "test", + ) diff --git a/sbt-test/pipelining/pipelining-test/project/DottyInjectedPlugin.scala b/sbt-test/pipelining/pipelining-test/project/DottyInjectedPlugin.scala new file mode 100644 index 000000000000..69f15d168bfc --- /dev/null +++ b/sbt-test/pipelining/pipelining-test/project/DottyInjectedPlugin.scala @@ -0,0 +1,12 @@ +import sbt._ +import Keys._ + +object DottyInjectedPlugin extends AutoPlugin { + override def requires = plugins.JvmPlugin + override def trigger = allRequirements + + override val projectSettings = Seq( + scalaVersion := sys.props("plugin.scalaVersion"), + scalacOptions += "-source:3.0-migration" + ) +} diff --git a/sbt-test/pipelining/pipelining-test/test b/sbt-test/pipelining/pipelining-test/test new file mode 100644 index 000000000000..e2b8e39082b2 --- /dev/null +++ b/sbt-test/pipelining/pipelining-test/test @@ -0,0 +1,12 @@ +# run the tests on a project with pipelining +# exercises the fact that -Ypickle-java and -Ypickle-write +# flags are set twice. +# steps: +# - Compile scope is compiled with flags `-Ypickle-java -Ypickle-write early/a-early-7423784.jar` +# - sbt copies `early/a-early-7423784.jar` to `early/a-early.jar` +# - Test scope is compiled with flags `-Ypickle-java -Ypickle-write early-test/a-early-963232.jar -Ypickle-java -Ypickle-write early/a-early.jar -classpath early/a-early.jar` +# e.g. for some reason the classpath has the same `a-early.jar` that +# is passed with `Ypickle-write`. +# Therefore we MUST avoid even reading the second `-Ypickle-write` setting, +# otherwise we will zero-out `a-early.jar`, causing type errors because its contents are blank. +> a/test