Skip to content

Commit 3647e95

Browse files
nicolasstuckinicolasstucki
committed
Compile Scala library with Dotty and test its TASTy
1 parent bdf634c commit 3647e95

File tree

6 files changed

+294
-3
lines changed

6 files changed

+294
-3
lines changed

‎.github/workflows/ci.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ jobs:
9797

9898
- name: Test
9999
run: |
100-
./project/scripts/sbt ";scala3-bootstrapped/compile ;scala3-bootstrapped/test;sjsSandbox/run;sjsSandbox/test;sjsJUnitTests/test;sjsCompilerTests/test ;sbt-dotty/scripted scala2-compat/* ;configureIDE"
100+
./project/scripts/sbt ";scala3-bootstrapped/compile ;scala3-bootstrapped/test;sjsSandbox/run;sjsSandbox/test;sjsJUnitTests/test;sjsCompilerTests/test ;sbt-dotty/scripted scala2-compat/* ;configureIDE ;scala3-scala2-library/test:run ;scala3-scala2-library-tasty-tests/test"
101101
./project/scripts/bootstrapCmdTests
102102
103103
## Only run bootstrapped tests for Windows since that's a superset of the

‎build.sbt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ val `scala3-library-bootstrapped` = Build.`scala3-library-bootstrapped`
1010
val `scala3-library-bootstrappedJS` = Build.`scala3-library-bootstrappedJS`
1111
val `scala3-sbt-bridge` = Build.`scala3-sbt-bridge`
1212
val `scala3-sbt-bridge-tests` = Build.`scala3-sbt-bridge-tests`
13+
val `scala3-scala2-library` = Build.`scala3-scala2-library`
14+
val `scala3-scala2-library-tasty-tests` = Build.`scala3-scala2-library-tasty-tests`
1315
val `scala3-staging` = Build.`scala3-staging`
1416
val `scala3-tasty-inspector` = Build.`scala3-tasty-inspector`
1517
val `scala3-language-server` = Build.`scala3-language-server`

‎project/Build.scala

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -805,6 +805,74 @@ object Build {
805805
javaOptions := (javaOptions in `scala3-compiler-bootstrapped`).value
806806
)
807807

808+
/** Scala library compiled by dotty using the latest published sources of the library */
809+
lazy val `scala3-scala2-library` = project.in(file("scala3-scala2-library")).
810+
withCommonSettings(Bootstrapped).
811+
dependsOn(dottyCompiler(Bootstrapped) % "provided; compile->runtime; test->test").
812+
dependsOn(`scala3-tasty-inspector` % "test->test").
813+
settings(commonBootstrappedSettings).
814+
settings(
815+
javaOptions := (javaOptions in `scala3-compiler-bootstrapped`).value,
816+
scalacOptions -= "-Xfatal-warnings",
817+
ivyConfigurations += SourceDeps.hide,
818+
transitiveClassifiers := Seq("sources"),
819+
libraryDependencies +=
820+
("org.scala-lang" % "scala-library" % stdlibVersion(Bootstrapped) % "sourcedeps"),
821+
sourceGenerators in Compile += Def.task {
822+
val s = streams.value
823+
val cacheDir = s.cacheDirectory
824+
val trgDir = (sourceManaged in Compile).value / "scala-library-src"
825+
826+
val report = updateClassifiers.value
827+
val scalaLibrarySourcesJar = report.select(
828+
configuration = configurationFilter("sourcedeps"),
829+
module = (_: ModuleID).name == "scala-library",
830+
artifact = artifactFilter(`type` = "src")).headOption.getOrElse {
831+
sys.error(s"Could not fetch scala-library sources")
832+
}
833+
834+
FileFunction.cached(cacheDir / s"fetchScalaLibrarySrc",
835+
FilesInfo.lastModified, FilesInfo.exists) { dependencies =>
836+
s.log.info(s"Unpacking scala-library sources to $trgDir...")
837+
if (trgDir.exists)
838+
IO.delete(trgDir)
839+
IO.createDirectory(trgDir)
840+
IO.unzip(scalaLibrarySourcesJar, trgDir)
841+
842+
((trgDir ** "*.scala") +++ (trgDir ** "*.java")).get.toSet
843+
} (Set(scalaLibrarySourcesJar)).toSeq
844+
}.taskValue,
845+
sources in Compile ~= (_.filterNot(file =>
846+
// sources from https://github.com/scala/scala/tree/2.13.x/src/library-aux
847+
file.getPath.endsWith("scala-library-src/scala/Any.scala") ||
848+
file.getPath.endsWith("scala-library-src/scala/AnyVal.scala") ||
849+
file.getPath.endsWith("scala-library-src/scala/AnyRef.scala") ||
850+
file.getPath.endsWith("scala-library-src/scala/Nothing.scala") ||
851+
file.getPath.endsWith("scala-library-src/scala/Null.scala") ||
852+
file.getPath.endsWith("scala-library-src/scala/Singleton.scala"))),
853+
managedClasspath in Test ~= {
854+
_.filterNot(file => file.data.getName == s"scala-library-${stdlibVersion(Bootstrapped)}.jar")
855+
},
856+
)
857+
858+
/** Test the tasty generated by `scala3-scala2-library`
859+
*
860+
* The tests are run with the bootstrapped compiler and the tasty inpector on the classpath.
861+
* The classpath has the default `scala-library` and not `scala3-scala2-library`.
862+
*
863+
* The jar of `scala3-scala2-library` is provided for to the tests.
864+
* - inspector: test that we can load the contents of the jar using the tasty inspector
865+
* - from-tasty: test that we can recompile the contents of the jar using `dotc -from-tasty`
866+
*/
867+
lazy val `scala3-scala2-library-tasty-tests` = project.in(file("scala3-scala2-library-tasty-tests")).
868+
withCommonSettings(Bootstrapped).
869+
dependsOn(`scala3-tasty-inspector` % "test->test").
870+
settings(commonBootstrappedSettings).
871+
settings(
872+
javaOptions := (javaOptions in `scala3-compiler-bootstrapped`).value,
873+
javaOptions += "-Ddotty.scala.library=" + packageBin.in(`scala3-scala2-library`, Compile).value.getAbsolutePath
874+
)
875+
808876
lazy val `scala3-sbt-bridge` = project.in(file("sbt-bridge/src")).
809877
// We cannot depend on any bootstrapped project to compile the bridge, since the
810878
// bridge is needed to compile these projects.

‎scala3-scala2-library-tasty-tests/test/Scala3Scala2LibraryTASYyTest.scala

Lines changed: 206 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,206 @@
1+
package dotty.tools.dotc
2+
3+
import org.junit.Test
4+
import org.junit.Ignore
5+
import org.junit.Assert._
6+
7+
import dotty.tools.io._
8+
import dotty.tools.dotc.util.ClasspathFromClassloader
9+
10+
import scala.quoted._
11+
12+
import java.io.File.pathSeparator
13+
14+
class Scala3Scala2LibraryTASYyTest:
15+
16+
import Scala3Scala2LibraryTASYyTest._
17+
18+
/** Test that we can load trees from TASTy */
19+
@Test def testTastyInspector: Unit =
20+
loadWithTastyInspector(loadBlacklisted)
21+
22+
/** Test that we can load and compile trees from TASTy */
23+
@Test def testFromTasty: Unit =
24+
compileFromTasty(loadBlacklisted.union(compileBlacklisted))
25+
26+
@Ignore
27+
@Test def testWhiteListFromTasty: Unit =
28+
val whitelist = Set(
29+
"scala.collection.mutable.StringBuilder"
30+
)
31+
compileFromTasty(x => !whitelist(x))
32+
33+
@Test def blacklistNoDuplicates =
34+
def testDup(name: String, list: List[String], set: Set[String]) =
35+
assert(list.size == set.size,
36+
list.diff(set.toSeq).mkString(s"`$name` has duplicate entries:\n ", "\n ", "\n\n"))
37+
testDup("loadBlacklist", loadBlacklist, loadBlacklisted)
38+
testDup("compileBlacklist", compileBlacklist, compileBlacklisted)
39+
40+
@Test def blacklistsNoIntersection =
41+
val intersection = loadBlacklisted & compileBlacklisted
42+
assert(intersection.isEmpty,
43+
intersection.mkString(
44+
"`compileBlacklist` contains names that are already in `compileBlacklist`: \n ", "\n ", "\n\n"))
45+
46+
@Test def blacklistsOnlyContainsClassesThatExist =
47+
val scalaLibJarTastyClassNamesSet = scalaLibJarTastyClassNames.toSet
48+
val intersection = loadBlacklisted & compileBlacklisted
49+
assert(loadBlacklisted.diff(scalaLibJarTastyClassNamesSet).isEmpty,
50+
loadBlacklisted.diff(scalaLibJarTastyClassNamesSet).mkString(
51+
"`loadBlacklisted` contains names that are not in `scalaLibJarTastyClassNames`: \n ", "\n ", "\n\n"))
52+
assert(compileBlacklisted.diff(scalaLibJarTastyClassNamesSet).isEmpty,
53+
compileBlacklisted.diff(scalaLibJarTastyClassNamesSet).mkString(
54+
"`loadBlacklisted` contains names that are not in `scalaLibJarTastyClassNames`: \n ", "\n ", "\n\n"))
55+
56+
@Ignore
57+
@Test def testLoadBacklistIsMinimal =
58+
var shouldBeWhitelisted = List.empty[String]
59+
val size = loadBlacklisted.size
60+
for (notBlacklisted, i) <- loadBlacklist.zipWithIndex do
61+
val blacklist = loadBlacklisted - notBlacklisted
62+
println(s"Trying withouth $notBlacklisted in the blacklist (${i+1}/$size)")
63+
try {
64+
loadWithTastyInspector(blacklist)
65+
shouldBeWhitelisted = notBlacklisted :: shouldBeWhitelisted
66+
}
67+
catch {
68+
case ex: Throwable => // ok
69+
}
70+
assert(shouldBeWhitelisted.isEmpty,
71+
shouldBeWhitelisted.mkString("Some classes do not need to be blacklisted in `loadBlacklisted`\n ", "\n ", "\n\n"))
72+
73+
@Ignore
74+
@Test def testCompileBlacklistIsMinimal =
75+
var shouldBeWhitelisted = List.empty[String]
76+
val size = compileBlacklisted.size
77+
val blacklist0 = loadBlacklisted.union(compileBlacklisted)
78+
for (notBlacklisted, i) <- compileBlacklist.zipWithIndex do
79+
val blacklist = blacklist0 - notBlacklisted
80+
println(s"Trying withouth $notBlacklisted in the blacklist (${i+1}/$size)")
81+
try {
82+
compileFromTasty(blacklist)
83+
shouldBeWhitelisted = notBlacklisted :: shouldBeWhitelisted
84+
}
85+
catch {
86+
case ex: Throwable => // ok
87+
}
88+
assert(shouldBeWhitelisted.isEmpty,
89+
shouldBeWhitelisted.mkString("Some classes do not need to be blacklisted in `compileBlacklisted`\n ", "\n ", "\n\n"))
90+
91+
end Scala3Scala2LibraryTASYyTest
92+
93+
object Scala3Scala2LibraryTASYyTest:
94+
95+
val scalaLibJarPath = System.getProperty("dotty.scala.library")
96+
97+
val scalaLibJarTastyClassNames = {
98+
val scalaLibJar = Jar(new File(java.nio.file.Paths.get(scalaLibJarPath)))
99+
scalaLibJar.toList.map(_.toString).filter(_.endsWith(".tasty"))
100+
.map(_.stripSuffix(".tasty").replace("/", "."))
101+
.sorted
102+
}
103+
104+
def loadWithTastyInspector(blacklisted: String => Boolean): Unit =
105+
val inspector = new scala.tasty.inspector.TastyInspector {
106+
def processCompilationUnit(using QuoteContext)(root: qctx.reflect.Tree): Unit =
107+
root.showExtractors // Check that we can traverse the full tree
108+
()
109+
}
110+
val classNames = scalaLibJarTastyClassNames.filterNot(blacklisted)
111+
val hasErrors = inspector.inspect(scalaLibJarPath, classNames)
112+
assert(!hasErrors, "Errors reported while loading from TASTy")
113+
114+
def compileFromTasty(blacklisted: String => Boolean): Unit = {
115+
val driver = new dotty.tools.dotc.Driver
116+
val currentClasspath = ClasspathFromClassloader(getClass.getClassLoader)
117+
val classNames = scalaLibJarTastyClassNames.filterNot(blacklisted)
118+
val args = Array(
119+
"-classpath", s"$scalaLibJarPath$pathSeparator$currentClasspath",
120+
"-from-tasty",
121+
"-nowarn"
122+
) ++ classNames
123+
val reporter = driver.process(args)
124+
assert(reporter.errorCount == 0, "Errors while re-compiling")
125+
}
126+
127+
/** List of classes that cannot be loaded from TASTy */
128+
def loadBlacklist = List[String](
129+
// No issues :)
130+
)
131+
132+
/** List of classes that cannot be recompilied from TASTy */
133+
def compileBlacklist = List[String](
134+
// OK?
135+
// failed: java.lang.AssertionError: assertion failed: class Boolean
136+
// at dotty.DottyPredef$.assertFail(DottyPredef.scala:17)
137+
// at dotty.tools.backend.jvm.BCodeHelpers$BCInnerClassGen.assertClassNotArrayNotPrimitive(BCodeHelpers.scala:247)
138+
// at dotty.tools.backend.jvm.BCodeHelpers$BCInnerClassGen.getClassBTypeAndRegisterInnerClass(BCodeHelpers.scala:265)
139+
// at dotty.tools.backend.jvm.BCodeHelpers$BCInnerClassGen.getClassBTypeAndRegisterInnerClass$(BCodeHelpers.scala:210)
140+
// at dotty.tools.backend.jvm.BCodeSkelBuilder$PlainSkelBuilder.getClassBTypeAndRegisterInnerClass(BCodeSkelBuilder.scala:62)
141+
// at dotty.tools.backend.jvm.BCodeHelpers$BCInnerClassGen.internalName(BCodeHelpers.scala:237)
142+
"scala.Array",
143+
"scala.Boolean",
144+
"scala.Byte",
145+
"scala.Char",
146+
"scala.Double",
147+
"scala.Float",
148+
"scala.Int",
149+
"scala.Long",
150+
"scala.Short",
151+
"scala.Unit",
152+
153+
// -- Error:
154+
// | def addOne(kv: (K, V)) = {
155+
// | ^
156+
// |error overriding method addOne in trait Growable of type (elem: (K, V)): (TrieMap.this : scala.collection.concurrent.TrieMap[K, V]);
157+
// | method addOne of type (kv: (K, V)): (TrieMap.this : scala.collection.concurrent.TrieMap[K, V]) has incompatible type
158+
// -- Error:
159+
// | def subtractOne(k: K) = {
160+
// | ^
161+
// |error overriding method subtractOne in trait Shrinkable of type (elem: K): (TrieMap.this : scala.collection.concurrent.TrieMap[K, V]);
162+
// | method subtractOne of type (k: K): (TrieMap.this : scala.collection.concurrent.TrieMap[K, V]) has incompatible type
163+
"scala.collection.concurrent.TrieMap",
164+
"scala.collection.immutable.HashMapBuilder",
165+
"scala.collection.immutable.HashSetBuilder",
166+
"scala.collection.immutable.LazyList",
167+
"scala.collection.immutable.ListMapBuilder",
168+
"scala.collection.immutable.MapBuilderImpl",
169+
"scala.collection.immutable.SetBuilderImpl",
170+
"scala.collection.immutable.TreeSeqMap",
171+
"scala.collection.immutable.VectorBuilder",
172+
"scala.collection.immutable.VectorMapBuilder",
173+
"scala.collection.mutable.AnyRefMap",
174+
"scala.collection.mutable.ArrayBuilder",
175+
"scala.collection.mutable.CollisionProofHashMap",
176+
"scala.collection.mutable.LongMap",
177+
"scala.collection.mutable.SortedMap",
178+
"scala.collection.mutable.StringBuilder",
179+
"scala.jdk.AnyAccumulator",
180+
"scala.jdk.DoubleAccumulator",
181+
"scala.jdk.IntAccumulator",
182+
"scala.jdk.LongAccumulator",
183+
184+
// -- Error:
185+
// | override def filterInPlace(p: A => Boolean): this.type = {
186+
// | ^
187+
// |error overriding method filterInPlace in trait SetOps of type (p: A => Boolean): (HashSet.this : scala.collection.mutable.HashSet[A]);
188+
// | method filterInPlace of type (p: A => Boolean): (HashSet.this : scala.collection.mutable.HashSet[A]) has incompatible type
189+
"scala.collection.mutable.HashSet",
190+
191+
// -- Error:
192+
// | def force: this.type = {
193+
// | ^
194+
// |error overriding method force in class Stream of type => (Cons.this : scala.collection.immutable.Stream.Cons[A]);
195+
// | method force of type => (Cons.this : scala.collection.immutable.Stream.Cons[A]) has incompatible type
196+
"scala.collection.immutable.Stream",
197+
198+
)
199+
200+
/** Set of classes that cannot be loaded from TASTy */
201+
def loadBlacklisted = loadBlacklist.toSet
202+
203+
/** Set of classes that cannot be recompilied from TASTy */
204+
def compileBlacklisted = compileBlacklist.toSet
205+
206+
end Scala3Scala2LibraryTASYyTest

‎scala3-scala2-library/test/Main.scala

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
package hello
2+
3+
enum Color:
4+
case Red, Green, Blue
5+
6+
object HelloWorld:
7+
def main(args: Array[String]): Unit = {
8+
println("hello dotty.superbootstrapped!")
9+
println(Color.Red)
10+
println(Color.Green)
11+
println(Color.Blue)
12+
}

‎tasty-inspector/src/scala/tasty/inspector/TastyInspector.scala

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,9 @@ trait TastyInspector:
2424
*
2525
* @param classpath Classpath where the classes are located
2626
* @param classes classes to be inspected
27+
* @return if an error was reported
2728
*/
28-
def inspect(classpath: String, classes: List[String]): Unit =
29+
def inspect(classpath: String, classes: List[String]): Boolean =
2930
if (classes.isEmpty)
3031
throw new IllegalArgumentException("Parameter classes should no be empty")
3132

@@ -64,7 +65,9 @@ trait TastyInspector:
6465

6566
val currentClasspath = ClasspathFromClassloader(getClass.getClassLoader)
6667
val args = "-from-tasty" :: "-Yretain-trees" :: "-classpath" :: s"$classpath$pathSeparator$currentClasspath" :: classes
67-
(new InspectorDriver).process(args.toArray)
68+
val reporter = (new InspectorDriver).process(args.toArray)
69+
reporter.hasErrors
70+
6871
end inspect
6972

7073

0 commit comments

Comments
 (0)