diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 71d3b984783e..66d0e1c73bef 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -97,7 +97,7 @@ jobs: - name: Test run: | - ./project/scripts/sbt ";scala3-bootstrapped/compile ;scala3-bootstrapped/test;sjsSandbox/run;sjsSandbox/test;sjsJUnitTests/test;sjsCompilerTests/test ;sbt-dotty/scripted scala2-compat/* ;configureIDE" + ./project/scripts/sbt ";scala3-bootstrapped/compile ;scala3-bootstrapped/test;sjsSandbox/run;sjsSandbox/test;sjsJUnitTests/test;sjsCompilerTests/test ;sbt-dotty/scripted scala2-compat/* ;configureIDE ;stdlib-bootstrapped/test:run ;stdlib-bootstrapped-tasty-tests/test" ./project/scripts/bootstrapCmdTests ## Only run bootstrapped tests for Windows since that's a superset of the diff --git a/build.sbt b/build.sbt index e4cd3adb6286..b2f7208dc478 100644 --- a/build.sbt +++ b/build.sbt @@ -15,6 +15,8 @@ val `scala3-tasty-inspector` = Build.`scala3-tasty-inspector` val `scala3-language-server` = Build.`scala3-language-server` val `scala3-bench` = Build.`scala3-bench` val `scala3-bench-bootstrapped` = Build.`scala3-bench-bootstrapped` +val `stdlib-bootstrapped` = Build.`stdlib-bootstrapped` +val `stdlib-bootstrapped-tasty-tests` = Build.`stdlib-bootstrapped-tasty-tests` val `tasty-core` = Build.`tasty-core` val `tasty-core-bootstrapped` = Build.`tasty-core-bootstrapped` val `tasty-core-scala2` = Build.`tasty-core-scala2` diff --git a/compiler/src/dotty/tools/backend/sjs/JSDefinitions.scala b/compiler/src/dotty/tools/backend/sjs/JSDefinitions.scala index b4b2d038bc0e..4025446c8cd8 100644 --- a/compiler/src/dotty/tools/backend/sjs/JSDefinitions.scala +++ b/compiler/src/dotty/tools/backend/sjs/JSDefinitions.scala @@ -255,7 +255,7 @@ final class JSDefinitions()(using Context) { def isJSThisFunctionClass(cls: Symbol): Boolean = isScalaJSVarArityClass(cls, "ThisFunction") - /** Definitions related to the treatment of JUnit boostrappers. */ + /** Definitions related to the treatment of JUnit bootstrappers. */ object junit { @threadUnsafe lazy val TestAnnotType: TypeRef = requiredClassRef("org.junit.Test") def TestAnnotClass(using Context): ClassSymbol = TestAnnotType.symbol.asClass diff --git a/compiler/src/dotty/tools/dotc/core/Symbols.scala b/compiler/src/dotty/tools/dotc/core/Symbols.scala index 7ba320159531..0f6edc1f5f57 100644 --- a/compiler/src/dotty/tools/dotc/core/Symbols.scala +++ b/compiler/src/dotty/tools/dotc/core/Symbols.scala @@ -470,8 +470,8 @@ object Symbols { flags: FlagSet = sym.flags, info: Type = sym.info, privateWithin: Symbol = sym.privateWithin, - coord: Coord = NoCoord, // Can be `= owner.coord` once we boostrap - associatedFile: AbstractFile = null // Can be `= owner.associatedFile` once we boostrap + coord: Coord = NoCoord, // Can be `= owner.coord` once we bootstrap + associatedFile: AbstractFile = null // Can be `= owner.associatedFile` once we bootstrap ): Symbol = { val coord1 = if (coord == NoCoord) owner.coord else coord val associatedFile1 = if (associatedFile == null) owner.associatedFile else associatedFile diff --git a/compiler/src/dotty/tools/dotc/transform/sjs/JUnitBootstrappers.scala b/compiler/src/dotty/tools/dotc/transform/sjs/JUnitBootstrappers.scala index b1caa01ca5c7..a2964d802198 100644 --- a/compiler/src/dotty/tools/dotc/transform/sjs/JUnitBootstrappers.scala +++ b/compiler/src/dotty/tools/dotc/transform/sjs/JUnitBootstrappers.scala @@ -98,7 +98,7 @@ import dotty.tools.backend.sjs.JSDefinitions.jsdefn * framework with official asynchronous support instead. * * Because `Booststrapper` is annotated with `@EnableReflectiveInstantiation`, - * the run-time implementation of JUnit for Scala.js can load the boostrapper + * the run-time implementation of JUnit for Scala.js can load the bootstrapper * module using `scala.scalajs.reflect.Reflect`, and then use the methods of * Bootstrapper, which are implemented in the bootstrapper object, to perform * test discovery and invocation. @@ -145,7 +145,7 @@ class JUnitBootstrappers extends MiniPhase { private def genBootstrapper(testClass: ClassSymbol)(using Context): TypeDef = { val junitdefn = jsdefn.junit - /* The name of the boostrapper module. It is derived from the test class name by + /* The name of the bootstrapper module. It is derived from the test class name by * appending a specific suffix string mandated "by spec". It will indeed also be * computed as such at run-time by the Scala.js JUnit Runtime support. Therefore, * it must *not* be a dotc semantic name. diff --git a/project/Build.scala b/project/Build.scala index b8816306b0a8..8c3092876a0c 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -247,7 +247,7 @@ object Build { excludeFromIDE := true ) - // Settings used when compiling dotty (both non-boostrapped and bootstrapped) + // Settings used when compiling dotty (both non-bootstrapped and bootstrapped) lazy val commonDottySettings = commonSettings ++ Seq( // Manually set the standard library to use autoScalaLibrary := false @@ -805,6 +805,75 @@ object Build { javaOptions := (javaOptions in `scala3-compiler-bootstrapped`).value ) + /** Scala library compiled by dotty using the latest published sources of the library */ + lazy val `stdlib-bootstrapped` = project.in(file("stdlib-bootstrapped")). + withCommonSettings(Bootstrapped). + dependsOn(dottyCompiler(Bootstrapped) % "provided; compile->runtime; test->test"). + dependsOn(`scala3-tasty-inspector` % "test->test"). + settings(commonBootstrappedSettings). + settings( + moduleName := "scala-library", + javaOptions := (javaOptions in `scala3-compiler-bootstrapped`).value, + scalacOptions -= "-Xfatal-warnings", + ivyConfigurations += SourceDeps.hide, + transitiveClassifiers := Seq("sources"), + libraryDependencies += + ("org.scala-lang" % "scala-library" % stdlibVersion(Bootstrapped) % "sourcedeps"), + sourceGenerators in Compile += Def.task { + val s = streams.value + val cacheDir = s.cacheDirectory + val trgDir = (sourceManaged in Compile).value / "scala-library-src" + + val report = updateClassifiers.value + val scalaLibrarySourcesJar = report.select( + configuration = configurationFilter("sourcedeps"), + module = (_: ModuleID).name == "scala-library", + artifact = artifactFilter(`type` = "src")).headOption.getOrElse { + sys.error(s"Could not fetch scala-library sources") + } + + FileFunction.cached(cacheDir / s"fetchScalaLibrarySrc", + FilesInfo.lastModified, FilesInfo.exists) { dependencies => + s.log.info(s"Unpacking scala-library sources to $trgDir...") + if (trgDir.exists) + IO.delete(trgDir) + IO.createDirectory(trgDir) + IO.unzip(scalaLibrarySourcesJar, trgDir) + + ((trgDir ** "*.scala") +++ (trgDir ** "*.java")).get.toSet + } (Set(scalaLibrarySourcesJar)).toSeq + }.taskValue, + sources in Compile ~= (_.filterNot(file => + // sources from https://github.com/scala/scala/tree/2.13.x/src/library-aux + file.getPath.endsWith("scala-library-src/scala/Any.scala") || + file.getPath.endsWith("scala-library-src/scala/AnyVal.scala") || + file.getPath.endsWith("scala-library-src/scala/AnyRef.scala") || + file.getPath.endsWith("scala-library-src/scala/Nothing.scala") || + file.getPath.endsWith("scala-library-src/scala/Null.scala") || + file.getPath.endsWith("scala-library-src/scala/Singleton.scala"))), + managedClasspath in Test ~= { + _.filterNot(file => file.data.getName == s"scala-library-${stdlibVersion(Bootstrapped)}.jar") + }, + ) + + /** Test the tasty generated by `stdlib-bootstrapped` + * + * The tests are run with the bootstrapped compiler and the tasty inpector on the classpath. + * The classpath has the default `scala-library` and not `stdlib-bootstrapped`. + * + * The jar of `stdlib-bootstrapped` is provided for to the tests. + * - inspector: test that we can load the contents of the jar using the tasty inspector + * - from-tasty: test that we can recompile the contents of the jar using `dotc -from-tasty` + */ + lazy val `stdlib-bootstrapped-tasty-tests` = project.in(file("stdlib-bootstrapped-tasty-tests")). + withCommonSettings(Bootstrapped). + dependsOn(`scala3-tasty-inspector` % "test->test"). + settings(commonBootstrappedSettings). + settings( + javaOptions := (javaOptions in `scala3-compiler-bootstrapped`).value, + javaOptions += "-Ddotty.scala.library=" + packageBin.in(`stdlib-bootstrapped`, Compile).value.getAbsolutePath + ) + lazy val `scala3-sbt-bridge` = project.in(file("sbt-bridge/src")). // We cannot depend on any bootstrapped project to compile the bridge, since the // bridge is needed to compile these projects. diff --git a/stdlib-bootstrapped-tasty-tests/test/BootstrappedStdLibTASYyTest.scala b/stdlib-bootstrapped-tasty-tests/test/BootstrappedStdLibTASYyTest.scala new file mode 100644 index 000000000000..93b6e97f41bb --- /dev/null +++ b/stdlib-bootstrapped-tasty-tests/test/BootstrappedStdLibTASYyTest.scala @@ -0,0 +1,209 @@ +package dotty.tools.dotc + +import org.junit.Test +import org.junit.Ignore +import org.junit.Assert._ + +import dotty.tools.io._ +import dotty.tools.dotc.util.ClasspathFromClassloader + +import scala.quoted._ + +import java.io.File.pathSeparator + +class BootstrappedStdLibTASYyTest: + + import BootstrappedStdLibTASYyTest._ + + /** Test that we can load trees from TASTy */ + @Test def testTastyInspector: Unit = + loadWithTastyInspector(loadBlacklisted) + + /** Test that we can load and compile trees from TASTy */ + @Test def testFromTasty: Unit = + compileFromTasty(loadBlacklisted.union(compileBlacklisted)) + + @Ignore + @Test def testWhiteListFromTasty: Unit = + val whitelist = Set( + "scala.collection.mutable.StringBuilder" + ) + compileFromTasty(x => !whitelist(x)) + + @Test def blacklistNoDuplicates = + def testDup(name: String, list: List[String], set: Set[String]) = + assert(list.size == set.size, + list.diff(set.toSeq).mkString(s"`$name` has duplicate entries:\n ", "\n ", "\n\n")) + testDup("loadBlacklist", loadBlacklist, loadBlacklisted) + testDup("compileBlacklist", compileBlacklist, compileBlacklisted) + + @Test def blacklistsNoIntersection = + val intersection = loadBlacklisted & compileBlacklisted + assert(intersection.isEmpty, + intersection.mkString( + "`compileBlacklist` contains names that are already in `loadBlacklist`: \n ", "\n ", "\n\n")) + + @Test def blacklistsOnlyContainsClassesThatExist = + val scalaLibJarTastyClassNamesSet = scalaLibJarTastyClassNames.toSet + val intersection = loadBlacklisted & compileBlacklisted + assert(loadBlacklisted.diff(scalaLibJarTastyClassNamesSet).isEmpty, + loadBlacklisted.diff(scalaLibJarTastyClassNamesSet).mkString( + "`loadBlacklisted` contains names that are not in `scalaLibJarTastyClassNames`: \n ", "\n ", "\n\n")) + assert(compileBlacklisted.diff(scalaLibJarTastyClassNamesSet).isEmpty, + compileBlacklisted.diff(scalaLibJarTastyClassNamesSet).mkString( + "`loadBlacklisted` contains names that are not in `scalaLibJarTastyClassNames`: \n ", "\n ", "\n\n")) + + @Ignore + @Test def testLoadBacklistIsMinimal = + var shouldBeWhitelisted = List.empty[String] + val size = loadBlacklisted.size + for (notBlacklisted, i) <- loadBlacklist.zipWithIndex do + val blacklist = loadBlacklisted - notBlacklisted + println(s"Trying withouth $notBlacklisted in the blacklist (${i+1}/$size)") + try { + loadWithTastyInspector(blacklist) + shouldBeWhitelisted = notBlacklisted :: shouldBeWhitelisted + } + catch { + case ex: Throwable => // ok + } + assert(shouldBeWhitelisted.isEmpty, + shouldBeWhitelisted.mkString("Some classes do not need to be blacklisted in `loadBlacklisted`\n ", "\n ", "\n\n")) + + @Ignore + @Test def testCompileBlacklistIsMinimal = + var shouldBeWhitelisted = List.empty[String] + val size = compileBlacklisted.size + val blacklist0 = loadBlacklisted.union(compileBlacklisted) + for (notBlacklisted, i) <- compileBlacklist.zipWithIndex do + val blacklist = blacklist0 - notBlacklisted + println(s"Trying withouth $notBlacklisted in the blacklist (${i+1}/$size)") + try { + compileFromTasty(blacklist) + shouldBeWhitelisted = notBlacklisted :: shouldBeWhitelisted + } + catch { + case ex: Throwable => // ok + } + assert(shouldBeWhitelisted.isEmpty, + shouldBeWhitelisted.mkString("Some classes do not need to be blacklisted in `compileBlacklisted`\n ", "\n ", "\n\n")) + +end BootstrappedStdLibTASYyTest + +object BootstrappedStdLibTASYyTest: + + val scalaLibJarPath = System.getProperty("dotty.scala.library") + + val scalaLibJarTastyClassNames = { + val scalaLibJar = Jar(new File(java.nio.file.Paths.get(scalaLibJarPath))) + scalaLibJar.toList.map(_.toString).filter(_.endsWith(".tasty")) + .map(_.stripSuffix(".tasty").replace("/", ".")) + .sorted + } + + def loadWithTastyInspector(blacklisted: String => Boolean): Unit = + val inspector = new scala.tasty.inspector.TastyInspector { + def processCompilationUnit(using QuoteContext)(root: qctx.reflect.Tree): Unit = + root.showExtractors // Check that we can traverse the full tree + () + } + val classNames = scalaLibJarTastyClassNames.filterNot(blacklisted) + val hasErrors = inspector.inspect(scalaLibJarPath, classNames) + assert(!hasErrors, "Errors reported while loading from TASTy") + + def compileFromTasty(blacklisted: String => Boolean): Unit = { + val driver = new dotty.tools.dotc.Driver + val currentClasspath = ClasspathFromClassloader(getClass.getClassLoader) + val classNames = scalaLibJarTastyClassNames.filterNot(blacklisted) + val args = Array( + "-classpath", s"$scalaLibJarPath$pathSeparator$currentClasspath", + "-from-tasty", + "-nowarn" + ) ++ classNames + val reporter = driver.process(args) + assert(reporter.errorCount == 0, "Errors while re-compiling") + } + + /** List of classes that cannot be loaded from TASTy */ + def loadBlacklist = List[String]( + // No issues :) + ) + + /** List of classes that cannot be recompilied from TASTy */ + def compileBlacklist = List[String]( + // See #10048 + // failed: java.lang.AssertionError: assertion failed: class Boolean + // at dotty.DottyPredef$.assertFail(DottyPredef.scala:17) + // at dotty.tools.backend.jvm.BCodeHelpers$BCInnerClassGen.assertClassNotArrayNotPrimitive(BCodeHelpers.scala:247) + // at dotty.tools.backend.jvm.BCodeHelpers$BCInnerClassGen.getClassBTypeAndRegisterInnerClass(BCodeHelpers.scala:265) + // at dotty.tools.backend.jvm.BCodeHelpers$BCInnerClassGen.getClassBTypeAndRegisterInnerClass$(BCodeHelpers.scala:210) + // at dotty.tools.backend.jvm.BCodeSkelBuilder$PlainSkelBuilder.getClassBTypeAndRegisterInnerClass(BCodeSkelBuilder.scala:62) + // at dotty.tools.backend.jvm.BCodeHelpers$BCInnerClassGen.internalName(BCodeHelpers.scala:237) + "scala.Array", + "scala.Boolean", + "scala.Byte", + "scala.Char", + "scala.Double", + "scala.Float", + "scala.Int", + "scala.Long", + "scala.Short", + "scala.Unit", + + // See #9994 + // -- Error: + // | def addOne(kv: (K, V)) = { + // | ^ + // |error overriding method addOne in trait Growable of type (elem: (K, V)): (TrieMap.this : scala.collection.concurrent.TrieMap[K, V]); + // | method addOne of type (kv: (K, V)): (TrieMap.this : scala.collection.concurrent.TrieMap[K, V]) has incompatible type + // -- Error: + // | def subtractOne(k: K) = { + // | ^ + // |error overriding method subtractOne in trait Shrinkable of type (elem: K): (TrieMap.this : scala.collection.concurrent.TrieMap[K, V]); + // | method subtractOne of type (k: K): (TrieMap.this : scala.collection.concurrent.TrieMap[K, V]) has incompatible type + "scala.collection.concurrent.TrieMap", + "scala.collection.immutable.HashMapBuilder", + "scala.collection.immutable.HashSetBuilder", + "scala.collection.immutable.LazyList", + "scala.collection.immutable.ListMapBuilder", + "scala.collection.immutable.MapBuilderImpl", + "scala.collection.immutable.SetBuilderImpl", + "scala.collection.immutable.TreeSeqMap", + "scala.collection.immutable.VectorBuilder", + "scala.collection.immutable.VectorMapBuilder", + "scala.collection.mutable.AnyRefMap", + "scala.collection.mutable.ArrayBuilder", + "scala.collection.mutable.CollisionProofHashMap", + "scala.collection.mutable.LongMap", + "scala.collection.mutable.SortedMap", + "scala.collection.mutable.StringBuilder", + "scala.jdk.AnyAccumulator", + "scala.jdk.DoubleAccumulator", + "scala.jdk.IntAccumulator", + "scala.jdk.LongAccumulator", + + // See #9994 + // -- Error: + // | override def filterInPlace(p: A => Boolean): this.type = { + // | ^ + // |error overriding method filterInPlace in trait SetOps of type (p: A => Boolean): (HashSet.this : scala.collection.mutable.HashSet[A]); + // | method filterInPlace of type (p: A => Boolean): (HashSet.this : scala.collection.mutable.HashSet[A]) has incompatible type + "scala.collection.mutable.HashSet", + + // See #9994 + // -- Error: + // | def force: this.type = { + // | ^ + // |error overriding method force in class Stream of type => (Cons.this : scala.collection.immutable.Stream.Cons[A]); + // | method force of type => (Cons.this : scala.collection.immutable.Stream.Cons[A]) has incompatible type + "scala.collection.immutable.Stream", + + ) + + /** Set of classes that cannot be loaded from TASTy */ + def loadBlacklisted = loadBlacklist.toSet + + /** Set of classes that cannot be recompilied from TASTy */ + def compileBlacklisted = compileBlacklist.toSet + +end BootstrappedStdLibTASYyTest diff --git a/stdlib-bootstrapped/test/Main.scala b/stdlib-bootstrapped/test/Main.scala new file mode 100644 index 000000000000..1dad89eceffc --- /dev/null +++ b/stdlib-bootstrapped/test/Main.scala @@ -0,0 +1,12 @@ +package hello + +enum Color: + case Red, Green, Blue + +object HelloWorld: + def main(args: Array[String]): Unit = { + println("hello dotty.superbootstrapped!") + println(Color.Red) + println(Color.Green) + println(Color.Blue) + } diff --git a/tasty-inspector/src/scala/tasty/inspector/TastyInspector.scala b/tasty-inspector/src/scala/tasty/inspector/TastyInspector.scala index 1c57582c47a9..9d336f4fbff5 100644 --- a/tasty-inspector/src/scala/tasty/inspector/TastyInspector.scala +++ b/tasty-inspector/src/scala/tasty/inspector/TastyInspector.scala @@ -24,8 +24,9 @@ trait TastyInspector: * * @param classpath Classpath where the classes are located * @param classes classes to be inspected + * @return if an error was reported */ - def inspect(classpath: String, classes: List[String]): Unit = + def inspect(classpath: String, classes: List[String]): Boolean = if (classes.isEmpty) throw new IllegalArgumentException("Parameter classes should no be empty") @@ -64,7 +65,9 @@ trait TastyInspector: val currentClasspath = ClasspathFromClassloader(getClass.getClassLoader) val args = "-from-tasty" :: "-Yretain-trees" :: "-classpath" :: s"$classpath$pathSeparator$currentClasspath" :: classes - (new InspectorDriver).process(args.toArray) + val reporter = (new InspectorDriver).process(args.toArray) + reporter.hasErrors + end inspect