From 341e8891a0ed2571ce467f3b031329017c93ba98 Mon Sep 17 00:00:00 2001 From: Guillaume Martres Date: Tue, 26 Mar 2019 15:41:49 +0100 Subject: [PATCH 1/7] CompilationTests.tastyBootstrap: Avoid hardcoding path The hardcoded scala-2.12 won't be valid once we bootstrap. --- compiler/test/dotty/Properties.scala | 5 +++++ compiler/test/dotty/tools/TestSources.scala | 2 +- compiler/test/dotty/tools/dotc/CompilationTests.scala | 10 ++++------ project/Build.scala | 1 + 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/compiler/test/dotty/Properties.scala b/compiler/test/dotty/Properties.scala index d293a11dea39..f7cde1f95634 100644 --- a/compiler/test/dotty/Properties.scala +++ b/compiler/test/dotty/Properties.scala @@ -1,5 +1,7 @@ package dotty +import java.nio.file._ + /** Runtime properties from defines or environmnent */ object Properties { @@ -36,6 +38,9 @@ object Properties { */ val testsSafeMode: Boolean = sys.props.isDefinedAt("dotty.tests.safemode") + /** Extra directory containing sources for the compiler */ + def dottyCompilerManagedSources: Path = Paths.get(sys.props("dotty.tests.dottyCompilerManagedSources")) + /** dotty-interfaces jar */ def dottyInterfaces: String = sys.props("dotty.tests.classes.dottyInterfaces") diff --git a/compiler/test/dotty/tools/TestSources.scala b/compiler/test/dotty/tools/TestSources.scala index 32712763f61c..587db0a4efb2 100644 --- a/compiler/test/dotty/tools/TestSources.scala +++ b/compiler/test/dotty/tools/TestSources.scala @@ -62,7 +62,7 @@ object TestSources { (fileName.endsWith(".scala") || fileName.endsWith(".java")) && !excludedFiles.contains(fileName) } - assert(Files.isDirectory(path)) + assert(Files.isDirectory(path), s"Not a directory: $path") val files = if (shallow) Files.list(path) else Files.walk(path) try { val sources = files diff --git a/compiler/test/dotty/tools/dotc/CompilationTests.scala b/compiler/test/dotty/tools/dotc/CompilationTests.scala index 9eb7f58303c8..ece4f316ce65 100644 --- a/compiler/test/dotty/tools/dotc/CompilationTests.scala +++ b/compiler/test/dotty/tools/dotc/CompilationTests.scala @@ -239,12 +239,10 @@ class CompilationTests extends ParallelTesting { defaultOptions.and("-Ycheck-reentrant", "-strict", "-priorityclasspath", defaultOutputDir))(libGroup) val compilerSources = sources(Paths.get("compiler/src")) + val compilerManagedSources = sources(Properties.dottyCompilerManagedSources) - val scalaJSIRDir = Paths.get("compiler/target/scala-2.12/src_managed/main/scalajs-ir-src/org/scalajs/ir") - val scalaJSIRSources = sources(scalaJSIRDir, shallow = true) - - val dotty1 = compileList("dotty", compilerSources ++ scalaJSIRSources, opt)(dotty1Group) - val dotty2 = compileList("dotty", compilerSources ++ scalaJSIRSources, opt)(dotty2Group) + val dotty1 = compileList("dotty", compilerSources ++ compilerManagedSources, opt)(dotty1Group) + val dotty2 = compileList("dotty", compilerSources ++ compilerManagedSources, opt)(dotty2Group) val tests = { lib.keepOutput :: dotty1.keepOutput :: { @@ -262,7 +260,7 @@ class CompilationTests extends ParallelTesting { compileShallowFilesInDir("compiler/src/dotty/tools/dotc/util", opt) + compileShallowFilesInDir("compiler/src/dotty/tools/backend", opt) + compileShallowFilesInDir("compiler/src/dotty/tools/backend/jvm", opt) + - compileList("shallow-scalajs-ir", scalaJSIRSources, opt) + compileList("managed-sources", compilerManagedSources, opt) }.keepOutput :: Nil }.map(_.checkCompile()) diff --git a/project/Build.scala b/project/Build.scala index 1b57c7bd2ee8..e7eeff2babc8 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -461,6 +461,7 @@ object Build { else List() val jarOpts = List( + "-Ddotty.tests.dottyCompilerManagedSources=" + (sourceManaged in Compile).value, "-Ddotty.tests.classes.dottyInterfaces=" + jars("dotty-interfaces"), "-Ddotty.tests.classes.dottyLibrary=" + jars("dotty-library"), "-Ddotty.tests.classes.dottyCompiler=" + jars("dotty-compiler"), From 78408e5a132960c562f5592aefce02c3fde155d9 Mon Sep 17 00:00:00 2001 From: Guillaume Martres Date: Tue, 26 Mar 2019 16:28:12 +0100 Subject: [PATCH 2/7] Avoid initializing Product and Serializable too early This lead to crashes in the full bootstrap and doesn't seem necessary in any case. --- compiler/src/dotty/tools/dotc/core/Definitions.scala | 7 ------- compiler/src/dotty/tools/dotc/core/StdNames.scala | 1 + .../tools/dotc/core/unpickleScala2/Scala2Unpickler.scala | 8 ++++++++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/core/Definitions.scala b/compiler/src/dotty/tools/dotc/core/Definitions.scala index 9f9aeedf02e1..9b0f5c0f69d7 100644 --- a/compiler/src/dotty/tools/dotc/core/Definitions.scala +++ b/compiler/src/dotty/tools/dotc/core/Definitions.scala @@ -1391,13 +1391,6 @@ class Definitions { for (m <- ScalaShadowingPackageClass.info.decls) ScalaPackageClass.enter(m) - // Temporary measure, as long as we do not read these classes from Tasty. - // Scala-2 classes don't have NoInits set even if they are pure. We override this - // for Product and Serializable so that case classes can be pure. A full solution - // requires that we read all Scala code from Tasty. - ProductClass.setFlag(NoInits) - SerializableClass.setFlag(NoInits) - // force initialization of every symbol that is synthesized or hijacked by the compiler val forced = syntheticCoreClasses ++ syntheticCoreMethods ++ ScalaValueClasses() diff --git a/compiler/src/dotty/tools/dotc/core/StdNames.scala b/compiler/src/dotty/tools/dotc/core/StdNames.scala index f9471b5b487c..553f53dd9380 100644 --- a/compiler/src/dotty/tools/dotc/core/StdNames.scala +++ b/compiler/src/dotty/tools/dotc/core/StdNames.scala @@ -197,6 +197,7 @@ object StdNames { final val Nothing: N = "Nothing" final val Null: N = "Null" final val Object: N = "Object" + final val Product: N = "Product" final val PartialFunction: N = "PartialFunction" final val PrefixType: N = "PrefixType" final val S: N = "S" diff --git a/compiler/src/dotty/tools/dotc/core/unpickleScala2/Scala2Unpickler.scala b/compiler/src/dotty/tools/dotc/core/unpickleScala2/Scala2Unpickler.scala index 7f1e79aa3b8a..6cd15e737264 100644 --- a/compiler/src/dotty/tools/dotc/core/unpickleScala2/Scala2Unpickler.scala +++ b/compiler/src/dotty/tools/dotc/core/unpickleScala2/Scala2Unpickler.scala @@ -467,6 +467,14 @@ class Scala2Unpickler(bytes: Array[Byte], classRoot: ClassDenotation, moduleClas def completeRoot(denot: ClassDenotation, completer: LazyType): Symbol = { denot.setFlag(flags) denot.resetFlag(Touched) // allow one more completion + + // Temporary measure, as long as we do not read these classes from Tasty. + // Scala-2 classes don't have NoInits set even if they are pure. We override this + // for Product and Serializable so that case classes can be pure. A full solution + // requires that we read all Scala code from Tasty. + if (owner == defn.ScalaPackageClass && ((name eq tpnme.Serializable) || (name eq tpnme.Product))) + denot.setFlag(NoInits) + denot.info = completer denot.symbol } From 8e29d38e1509b038632bbb6656aee7d359dc3ae4 Mon Sep 17 00:00:00 2001 From: Guillaume Martres Date: Thu, 28 Mar 2019 15:27:38 +0100 Subject: [PATCH 3/7] Fix classpath issues with the dotty-sbt-bridge build When compiling and running the tests for the bridge, we should have the bootstrapped library on the classpath since this is what people will actually use in the end. This is tricky to do since the bridge itself cannot be compiled with anything bootstrapped on the classpath since the bootstrapped projects need the bridge to be compiled ! We sidestep this issue by using two separate projects for the bridge and its tests. --- build.sbt | 1 + project/Build.scala | 34 +++++++++++++++++++++++++++------- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/build.sbt b/build.sbt index 2afa25b44bbc..342a9d834f4c 100644 --- a/build.sbt +++ b/build.sbt @@ -8,6 +8,7 @@ val `dotty-compiler-bootstrapped` = Build.`dotty-compiler-bootstrapped` val `dotty-library` = Build.`dotty-library` val `dotty-library-bootstrapped` = Build.`dotty-library-bootstrapped` val `dotty-sbt-bridge` = Build.`dotty-sbt-bridge` +val `dotty-sbt-bridge-tests` = Build.`dotty-sbt-bridge-tests` val `dotty-language-server` = Build.`dotty-language-server` val `dotty-bench` = Build.`dotty-bench` val `dotty-bench-bootstrapped` = Build.`dotty-bench-bootstrapped` diff --git a/project/Build.scala b/project/Build.scala index e7eeff2babc8..7d04202c1375 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -680,19 +680,39 @@ object Build { case Bootstrapped => `dotty-library-bootstrapped` } - lazy val `dotty-sbt-bridge` = project.in(file("sbt-bridge")). - dependsOn(dottyCompiler(NonBootstrapped) % Provided). + lazy val `dotty-sbt-bridge` = project.in(file("sbt-bridge/src")). + // We cannot depend on any bootstrapped project to compile the bridge, since the + // bridge is needed to compile these projects. dependsOn(dottyDoc(NonBootstrapped) % Provided). settings(commonJavaSettings). settings( description := "sbt compiler bridge for Dotty", - libraryDependencies ++= Seq( - Dependencies.`compiler-interface` % Provided, - (Dependencies.`zinc-api-info` % Test).withDottyCompat(scalaVersion.value) - ), + + sources in Test := Seq(), + scalaSource in Compile := baseDirectory.value, + javaSource in Compile := baseDirectory.value, + + // Referring to the other project using a string avoids an infinite loop + // when sbt reads the settings. + test in Test := (test in (LocalProject("dotty-sbt-bridge-tests"), Test)).value, + + libraryDependencies += Dependencies.`compiler-interface` % Provided + ) + + // We use a separate project for the bridge tests since they can only be run + // with the bootstrapped library on the classpath. + lazy val `dotty-sbt-bridge-tests` = project.in(file("sbt-bridge/test")). + dependsOn(dottyCompiler(Bootstrapped) % Test). + settings(commonBootstrappedSettings). + settings( + sources in Compile := Seq(), + scalaSource in Test := baseDirectory.value, + javaSource in Test := baseDirectory.value, fork in Test := true, - parallelExecution in Test := false + parallelExecution in Test := false, + + libraryDependencies += (Dependencies.`zinc-api-info` % Test).withDottyCompat(scalaVersion.value) ) lazy val `dotty-language-server` = project.in(file("language-server")). From a2be3dded427f8a0cf25e9a1fbaa183e6f6424bf Mon Sep 17 00:00:00 2001 From: Guillaume Martres Date: Wed, 27 Mar 2019 20:45:59 +0100 Subject: [PATCH 4/7] Get .tasty files from the correct jar 1. If we don't give a parent to the URLClassLoader constructor, it will use the system ClassLoader by default, which includes the jars in the JVM classpath. 2. `getResourceAsStream` will first look for the resource in a parent ClassLoader. 3. The reference compiler is in the JVM classpath. So if the reference compiler is Dotty-compiled, we might pick up .tasty files from it instead of the current compiler. Using `null` as the parent ClassLoader avoids this problem. --- .../src/dotty/tools/dotc/core/classfile/ClassfileParser.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala b/compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala index 790b6cc403e8..ec640e792548 100644 --- a/compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala +++ b/compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala @@ -804,7 +804,7 @@ class ClassfileParser( ctx.error("Could not load TASTY from .tasty for virtual file " + classfile) Array.empty case Some(jar: ZipArchive) => // We are in a jar - val cl = new URLClassLoader(Array(jar.jpath.toUri.toURL)) + val cl = new URLClassLoader(Array(jar.jpath.toUri.toURL), /*parent =*/ null) val path = classfile.path.stripSuffix(".class") + ".tasty" val stream = cl.getResourceAsStream(path) if (stream != null) { From 15d166437aa9106fa4daef905dfbae4f343cb0b1 Mon Sep 17 00:00:00 2001 From: Guillaume Martres Date: Thu, 28 Mar 2019 16:52:53 +0100 Subject: [PATCH 5/7] sjsSandbox: simplify and fix build We don't need to do anything special to compile code with dotty anymore, but we do need to have the bootstrapped library on the classpath. --- project/Build.scala | 100 +------------------------------------------- 1 file changed, 2 insertions(+), 98 deletions(-) diff --git a/project/Build.scala b/project/Build.scala index 7d04202c1375..429bab53705d 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -786,6 +786,7 @@ object Build { */ lazy val sjsSandbox = project.in(file("sandbox/scalajs")). enablePlugins(ScalaJSPlugin). + dependsOn(dottyLibrary(Bootstrapped)). settings(commonBootstrappedSettings). settings( /* Remove the Scala.js compiler plugin for scalac, and enable the @@ -808,8 +809,7 @@ object Build { scalaJSLinkerConfig ~= { _.withCheckIR(true).withParallel(false) } - ). - settings(compileWithDottySettings) + ) lazy val `dotty-bench` = project.in(file("bench")).asDottyBench(NonBootstrapped) lazy val `dotty-bench-bootstrapped` = project.in(file("bench")).asDottyBench(Bootstrapped) @@ -1056,102 +1056,6 @@ object Build { ) ) - // Compile with dotty - lazy val compileWithDottySettings = { - inConfig(Compile)(inTask(compile)(Defaults.runnerTask) ++ Seq( - // Compile with dotty - fork in compile := true, - - compile := { - val inputs = (compileInputs in compile).value - val inputOptions = inputs.options() - import inputOptions._ - - val s = streams.value - val logger = s.log - val cacheDir = s.cacheDirectory - - // Discover classpaths - - def cpToString(cp: Seq[File]) = - cp.map(_.getAbsolutePath).mkString(File.pathSeparator) - - val compilerCp = Attributed.data((fullClasspath in (`dotty-compiler`, Compile)).value) - val cpStr = cpToString(classpath ++ compilerCp) - - // List all my dependencies (recompile if any of these changes) - - val allMyDependencies = classpath filterNot (_ == classesDirectory) flatMap { cpFile => - if (cpFile.isDirectory) (cpFile ** "*.class").get - else Seq(cpFile) - } - - // Compile - - val run = (runner in compile).value - val cachedCompile = FileFunction.cached(cacheDir / "compile", - FilesInfo.lastModified, FilesInfo.exists) { dependencies => - - logger.info( - "Compiling %d Scala sources to %s..." format ( - sources.size, classesDirectory)) - - if (classesDirectory.exists) - IO.delete(classesDirectory) - IO.createDirectory(classesDirectory) - - val sourcesArgs = sources.map(_.getAbsolutePath()).toList - - /* run.run() below in doCompile() will emit a call to its - * logger.info("Running dotty.tools.dotc.Main [...]") - * which we do not want to see. We use this patched logger to - * filter out that particular message. - */ - val patchedLogger = new Logger { - def log(level: Level.Value, message: => String) = { - val msg = message - if (level != Level.Info || - !msg.startsWith("Running dotty.tools.dotc.Main")) - logger.log(level, msg) - } - def success(message: => String) = logger.success(message) - def trace(t: => Throwable) = logger.trace(t) - } - - def doCompile(sourcesArgs: List[String]): Unit = { - run.run("dotty.tools.dotc.Main", compilerCp, - "-classpath" :: cpStr :: - "-d" :: classesDirectory.getAbsolutePath :: - scalacOptions ++: - sourcesArgs, - patchedLogger) - } - - // Work around the Windows limitation on command line length. - val isWindows = - System.getProperty("os.name").toLowerCase().indexOf("win") >= 0 - if ((fork in compile).value && isWindows && - (sourcesArgs.map(_.length).sum > 1536)) { - IO.withTemporaryFile("sourcesargs", ".txt") { sourceListFile => - IO.writeLines(sourceListFile, sourcesArgs) - doCompile(List("@"+sourceListFile.getAbsolutePath)) - } - } else { - doCompile(sourcesArgs) - } - - // Output is all files in classesDirectory - (classesDirectory ** AllPassFilter).get.toSet - } - - cachedCompile((sources ++ allMyDependencies).toSet) - - // We do not have dependency analysis when compiling externally - sbt.internal.inc.Analysis.Empty - } - )) - } - lazy val commonDistSettings = Seq( packMain := Map(), publishArtifact := false, From c33d3cf41aa6fbc5e648f681970ba8a627dd3fe7 Mon Sep 17 00:00:00 2001 From: Guillaume Martres Date: Fri, 29 Mar 2019 15:09:08 +0100 Subject: [PATCH 6/7] Make InterfaceEntryPointTest a bootstrapped-only test This test runs the compiler using -usejavacp which doesn't really work when a non-bootstrapped Dotty is on the classpath. --- compiler/test/dotty/tools/dotc/InterfaceEntryPointTest.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compiler/test/dotty/tools/dotc/InterfaceEntryPointTest.scala b/compiler/test/dotty/tools/dotc/InterfaceEntryPointTest.scala index 1967a832fcbe..fd998e9d7c7f 100644 --- a/compiler/test/dotty/tools/dotc/InterfaceEntryPointTest.scala +++ b/compiler/test/dotty/tools/dotc/InterfaceEntryPointTest.scala @@ -3,6 +3,7 @@ package tools.dotc import org.junit.Test import org.junit.Assert._ +import org.junit.experimental.categories.Category import interfaces._ import scala.collection.mutable.ListBuffer import java.nio.file._ @@ -18,6 +19,7 @@ import java.nio.file._ * * @see [[OtherEntryPointsTest]] */ +@Category(Array(classOf[BootstrappedOnlyTests])) class InterfaceEntryPointTest { @Test def runCompilerFromInterface = { val sources = From dc0c818d990313a62452697a8e4daf41b02d906f Mon Sep 17 00:00:00 2001 From: Guillaume Martres Date: Fri, 29 Mar 2019 17:14:33 +0100 Subject: [PATCH 7/7] Plugin tests should be bootstrapped-only They require having the current compiler on the classpath, and the non-bootstrapped compiler cannot be loaded if it was compiled with an old Dotty. --- .../BootstrappedOnlyCompilationTests.scala | 25 +++++++++++++++++++ .../dotty/tools/dotc/CompilationTests.scala | 23 ----------------- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/compiler/test/dotty/tools/dotc/BootstrappedOnlyCompilationTests.scala b/compiler/test/dotty/tools/dotc/BootstrappedOnlyCompilationTests.scala index 6a11c0dc548e..4669335268a0 100644 --- a/compiler/test/dotty/tools/dotc/BootstrappedOnlyCompilationTests.scala +++ b/compiler/test/dotty/tools/dotc/BootstrappedOnlyCompilationTests.scala @@ -10,6 +10,8 @@ import org.junit.experimental.categories.Category import scala.concurrent.duration._ import vulpix._ +import java.nio.file._ + @Category(Array(classOf[BootstrappedOnlyTests])) class BootstrappedOnlyCompilationTests extends ParallelTesting { import ParallelTesting._ @@ -109,4 +111,27 @@ class BootstrappedOnlyCompilationTests extends ParallelTesting { compileDir("compiler/src/dotty/tools/dotc/core/tasty", picklingWithCompilerOptions) + compileDir("compiler/src/dotty/tools/dotc/core/unpickleScala2", picklingWithCompilerOptions) }.limitThreads(4).checkCompile() + + @Test def testPlugins: Unit = { + val pluginFile = "plugin.properties" + + // 1. hack with absolute path for -Xplugin + // 2. copy `pluginFile` to destination + def compileFilesInDir(dir: String): CompilationTest = { + val outDir = defaultOutputDir + "testPlugins/" + val sourceDir = new java.io.File(dir) + + val dirs = sourceDir.listFiles.toList.filter(_.isDirectory) + val targets = dirs.map { dir => + val compileDir = createOutputDirsForDir(dir, sourceDir, outDir) + Files.copy(dir.toPath.resolve(pluginFile), compileDir.toPath.resolve(pluginFile), StandardCopyOption.REPLACE_EXISTING) + val flags = TestFlags(withCompilerClasspath, noCheckOptions).and("-Xplugin:" + compileDir.getAbsolutePath) + SeparateCompilationSource("testPlugins", dir, flags, compileDir) + } + + new CompilationTest(targets) + } + + compileFilesInDir("tests/plugins/neg").checkExpectedErrors() + } } diff --git a/compiler/test/dotty/tools/dotc/CompilationTests.scala b/compiler/test/dotty/tools/dotc/CompilationTests.scala index ece4f316ce65..772ab1b3b544 100644 --- a/compiler/test/dotty/tools/dotc/CompilationTests.scala +++ b/compiler/test/dotty/tools/dotc/CompilationTests.scala @@ -272,29 +272,6 @@ class CompilationTests extends ParallelTesting { tests.foreach(_.delete()) } - - @Test def testPlugins: Unit = { - val pluginFile = "plugin.properties" - - // 1. hack with absolute path for -Xplugin - // 2. copy `pluginFile` to destination - def compileFilesInDir(dir: String): CompilationTest = { - val outDir = defaultOutputDir + "testPlugins/" - val sourceDir = new java.io.File(dir) - - val dirs = sourceDir.listFiles.toList.filter(_.isDirectory) - val targets = dirs.map { dir => - val compileDir = createOutputDirsForDir(dir, sourceDir, outDir) - Files.copy(dir.toPath.resolve(pluginFile), compileDir.toPath.resolve(pluginFile), StandardCopyOption.REPLACE_EXISTING) - val flags = TestFlags(withCompilerClasspath, noCheckOptions).and("-Xplugin:" + compileDir.getAbsolutePath) - SeparateCompilationSource("testPlugins", dir, flags, compileDir) - } - - new CompilationTest(targets) - } - - compileFilesInDir("tests/plugins/neg").checkExpectedErrors() - } } object CompilationTests {