Skip to content

Commit 59e7a4f

Browse files
nicolasstuckinicolasstucki
committed
Compile Scala library with Dotty and test its TASTy
1 parent 4b8a1de commit 59e7a4f

File tree

6 files changed

+381
-3
lines changed

6 files changed

+381
-3
lines changed

‎.github/workflows/ci.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ jobs:
9393

9494
- name: Test
9595
run: |
96-
./project/scripts/sbt ";dotty-bootstrapped/compile ;dotty-bootstrapped/test;sjsSandbox/run;sjsSandbox/test;sjsJUnitTests/test;sjsCompilerTests/test ;configureIDE"
96+
./project/scripts/sbt ";dotty-bootstrapped/compile ;dotty-bootstrapped/test;sjsSandbox/run;sjsSandbox/test;sjsJUnitTests/test;sjsCompilerTests/test; dotty-scala-library-sandbox/run; dotty-scala-library-from-tasty-tests/test ;configureIDE"
9797
./project/scripts/bootstrapCmdTests
9898
9999
community_build:

‎build.sbt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@ val `dotty-library-bootstrapped` = Build.`dotty-library-bootstrapped`
1010
val `dotty-library-bootstrappedJS` = Build.`dotty-library-bootstrappedJS`
1111
val `dotty-sbt-bridge` = Build.`dotty-sbt-bridge`
1212
val `dotty-sbt-bridge-tests` = Build.`dotty-sbt-bridge-tests`
13+
val `dotty-scala-library` = Build.`dotty-scala-library`
14+
val `dotty-scala-library-from-tasty-tests` = Build.`dotty-scala-library-from-tasty-tests`
15+
val `dotty-scala-library-sandbox` = Build.`dotty-scala-library-sandbox`
1316
val `dotty-staging` = Build.`dotty-staging`
1417
val `dotty-tasty-inspector` = Build.`dotty-tasty-inspector`
1518
val `dotty-language-server` = Build.`dotty-language-server`

‎dotty-scala-library-from-tasty-tests/test/LibraryTest.scala

Lines changed: 292 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,292 @@
1+
package dotty.tools.dotc
2+
3+
import org.junit.Test
4+
import org.junit.Ignore
5+
import org.junit.Assert._
6+
7+
import dotty.tools.io._
8+
import dotty.tools.dotc.util.ClasspathFromClassloader
9+
10+
import scala.quoted._
11+
12+
import java.io.File.pathSeparator
13+
14+
class TastyInspectorTest:
15+
16+
import TastyInspectorTest._
17+
18+
/** Test that we can load trees from TASTy */
19+
@Test def testTastyInspector: Unit =
20+
loadWithTastyInspector(loadBlacklisted)
21+
22+
/** Test that we can load and compile trees from TASTy */
23+
@Test def testFromTasty: Unit =
24+
compileFromTasty(loadBlacklisted.union(compileBlacklisted))
25+
26+
@Test def blacklistNoDuplicates =
27+
def testDup(name: String, list: List[String], set: Set[String]) =
28+
assert(list.size == set.size,
29+
list.diff(set.toSeq).mkString(s"`$name` has duplicate entries:\n ", "\n ", "\n\n"))
30+
testDup("loadBlacklist", loadBlacklist, loadBlacklisted)
31+
testDup("compileBlacklist", compileBlacklist, compileBlacklisted)
32+
33+
@Test def blacklistsNoIntersection =
34+
val intersection = loadBlacklisted & compileBlacklisted
35+
assert(intersection.isEmpty,
36+
intersection.mkString(
37+
"`compileBlacklist` contains names that are already in `compileBlacklist`: \n ", "\n ", "\n\n"))
38+
39+
@Test def blacklistsOnlyContainsClassesThatExist =
40+
val scalaLibJarTastyClassNamesSet = scalaLibJarTastyClassNames.toSet
41+
val intersection = loadBlacklisted & compileBlacklisted
42+
assert(loadBlacklisted.diff(scalaLibJarTastyClassNamesSet).isEmpty,
43+
loadBlacklisted.diff(scalaLibJarTastyClassNamesSet).mkString(
44+
"`loadBlacklisted` contains names that are not in `scalaLibJarTastyClassNames`: \n ", "\n ", "\n\n"))
45+
assert(compileBlacklisted.diff(scalaLibJarTastyClassNamesSet).isEmpty,
46+
compileBlacklisted.diff(scalaLibJarTastyClassNamesSet).mkString(
47+
"`loadBlacklisted` contains names that are not in `scalaLibJarTastyClassNames`: \n ", "\n ", "\n\n"))
48+
49+
@Ignore
50+
@Test def testLoadBacklistIsMinimal =
51+
var shouldBeWhitelisted = List.empty[String]
52+
val size = loadBlacklisted.size
53+
for (notBlacklisted, i) <- loadBlacklist.zipWithIndex do
54+
val blacklist = loadBlacklisted - notBlacklisted
55+
println(s"Trying withouth $notBlacklisted in the blacklist (${i+1}/$size)")
56+
try {
57+
loadWithTastyInspector(blacklist)
58+
shouldBeWhitelisted = notBlacklisted :: shouldBeWhitelisted
59+
}
60+
catch {
61+
case ex: Throwable => // ok
62+
}
63+
assert(shouldBeWhitelisted.isEmpty,
64+
shouldBeWhitelisted.mkString("Some classes do not need to be blacklisted in `loadBlacklisted`\n ", "\n ", "\n\n"))
65+
66+
@Ignore
67+
@Test def testCompileBlacklistIsMinimal =
68+
var shouldBeWhitelisted = List.empty[String]
69+
val size = compileBlacklisted.size
70+
val blacklist0 = loadBlacklisted.union(compileBlacklisted)
71+
for (notBlacklisted, i) <- compileBlacklist.zipWithIndex do
72+
val blacklist = blacklist0 - notBlacklisted
73+
println(s"Trying withouth $notBlacklisted in the blacklist (${i+1}/$size)")
74+
try {
75+
compileFromTasty(blacklist)
76+
shouldBeWhitelisted = notBlacklisted :: shouldBeWhitelisted
77+
}
78+
catch {
79+
case ex: Throwable => // ok
80+
}
81+
assert(shouldBeWhitelisted.isEmpty,
82+
shouldBeWhitelisted.mkString("Some classes do not need to be blacklisted in `compileBlacklisted`\n ", "\n ", "\n\n"))
83+
84+
end TastyInspectorTest
85+
86+
object TastyInspectorTest:
87+
88+
val scalaLibJarPath = System.getProperty("dotty.scala.library")
89+
90+
val scalaLibJarTastyClassNames = {
91+
val scalaLibJar = Jar(new File(java.nio.file.Paths.get(scalaLibJarPath)))
92+
scalaLibJar.toList.map(_.toString).filter(_.endsWith(".tasty"))
93+
.map(_.stripSuffix(".tasty").replace("/", "."))
94+
.sorted
95+
}
96+
97+
def loadWithTastyInspector(blacklisted: String => Boolean): Unit =
98+
val inspector = new scala.tasty.inspector.TastyInspector {
99+
def processCompilationUnit(using QuoteContext)(root: qctx.tasty.Tree): Unit =
100+
root.showExtractors // Check that we can traverse the full tree
101+
()
102+
}
103+
val classNames = scalaLibJarTastyClassNames.filterNot(blacklisted)
104+
val hasErrors = inspector.inspect(scalaLibJarPath, classNames)
105+
assert(!hasErrors, "Errors reported while loading from TASTy")
106+
107+
def compileFromTasty(blacklisted: String => Boolean): Unit = {
108+
val driver = new dotty.tools.dotc.Driver
109+
val currentClasspath = ClasspathFromClassloader(getClass.getClassLoader)
110+
val classNames = scalaLibJarTastyClassNames.filterNot(blacklisted)
111+
val args = Array(
112+
"-classpath", s"$scalaLibJarPath$pathSeparator$currentClasspath",
113+
"-from-tasty",
114+
"-nowarn"
115+
) ++ classNames
116+
val reporter = driver.process(args)
117+
assert(reporter.errorCount == 0, "Errors while re-compiling")
118+
}
119+
120+
def loadBlacklist = List[String](
121+
// object s in class StringContext does not take parameters
122+
"scala.collection.ArrayOps",
123+
"scala.collection.BitSetOps",
124+
"scala.collection.generic.DefaultSerializationProxy",
125+
"scala.collection.immutable.LazyList",
126+
"scala.collection.immutable.List",
127+
"scala.collection.immutable.Node",
128+
"scala.collection.immutable.NumericRange",
129+
"scala.collection.immutable.StrictOptimizedSeqOps",
130+
"scala.collection.immutable.TreeSeqMap",
131+
"scala.collection.immutable.Vector",
132+
"scala.collection.immutable.Vector0",
133+
"scala.collection.immutable.VectorBuilder",
134+
"scala.collection.immutable.VectorSliceBuilder",
135+
"scala.collection.immutable.WrappedString",
136+
"scala.collection.LazyZip2",
137+
"scala.collection.LazyZip3",
138+
"scala.collection.LazyZip4",
139+
"scala.collection.MapOps",
140+
"scala.collection.mutable.ArrayBuffer",
141+
"scala.collection.mutable.ArrayBufferView",
142+
"scala.collection.mutable.ArrayDeque",
143+
"scala.collection.mutable.ArrayDequeOps",
144+
"scala.collection.mutable.CollisionProofHashMap",
145+
"scala.collection.mutable.HashMap",
146+
"scala.collection.mutable.HashSet",
147+
"scala.collection.mutable.ListBuffer",
148+
"scala.collection.mutable.UnrolledBuffer",
149+
"scala.collection.SeqView",
150+
"scala.collection.StringView",
151+
"scala.concurrent.duration.Duration",
152+
"scala.concurrent.Future",
153+
"scala.Enumeration",
154+
"scala.io.Source",
155+
"scala.jdk.Accumulator",
156+
"scala.jdk.DoubleAccumulator",
157+
"scala.jdk.IntAccumulator",
158+
"scala.jdk.javaapi.DurationConverters",
159+
"scala.jdk.LongAccumulator",
160+
"scala.Product",
161+
"scala.Product1",
162+
"scala.Product10",
163+
"scala.Product11",
164+
"scala.Product12",
165+
"scala.Product13",
166+
"scala.Product14",
167+
"scala.Product15",
168+
"scala.Product16",
169+
"scala.Product17",
170+
"scala.Product18",
171+
"scala.Product19",
172+
"scala.Product2",
173+
"scala.Product20",
174+
"scala.Product21",
175+
"scala.Product22",
176+
"scala.Product3",
177+
"scala.Product4",
178+
"scala.Product5",
179+
"scala.Product6",
180+
"scala.Product7",
181+
"scala.Product8",
182+
"scala.Product9",
183+
"scala.reflect.ClassTag",
184+
"scala.runtime.ArrayCharSequence",
185+
"scala.runtime.LazyBoolean",
186+
"scala.runtime.LazyByte",
187+
"scala.runtime.LazyChar",
188+
"scala.runtime.LazyDouble",
189+
"scala.runtime.LazyFloat",
190+
"scala.runtime.LazyInt",
191+
"scala.runtime.LazyLong",
192+
"scala.runtime.LazyRef",
193+
"scala.runtime.LazyShort",
194+
"scala.runtime.LazyUnit",
195+
"scala.runtime.Tuple2Zipped",
196+
"scala.runtime.Tuple3Zipped",
197+
"scala.StringContext",
198+
"scala.sys.process.ProcessImpl",
199+
200+
// undefined: _root_.scala.StringContext.apply(["Scala "," "," -- ","" : String]:String*).f # -1: TermRef(TypeRef(TermRef(ThisType(TypeRef(NoPrefix,module class <root>)),module scala),class StringContext),f) at readTasty
201+
"scala.collection.immutable.Range",
202+
"scala.collection.Iterator",
203+
"scala.util.PropertiesTrait",
204+
)
205+
206+
def compileBlacklist = List[String](
207+
// java.lang.AssertionError: assertion failed: private value DefaultSentinel in object Map
208+
"scala.Array",
209+
"scala.Boolean",
210+
"scala.Byte",
211+
"scala.Char",
212+
"scala.collection.convert.JavaCollectionWrappers",
213+
"scala.collection.convert.StreamExtensions",
214+
"scala.collection.immutable.HashMap",
215+
"scala.collection.immutable.ListSet",
216+
"scala.collection.immutable.Map",
217+
"scala.collection.immutable.Set",
218+
"scala.collection.immutable.TreeMap",
219+
"scala.collection.immutable.VectorMap",
220+
"scala.collection.Map",
221+
"scala.collection.mutable.PriorityQueue",
222+
"scala.collection.SeqOps",
223+
"scala.collection.View",
224+
"scala.Double",
225+
"scala.Float",
226+
"scala.Int",
227+
"scala.jdk.AnyAccumulatorStepper",
228+
"scala.jdk.DoubleAccumulatorStepper",
229+
"scala.jdk.IntAccumulatorStepper",
230+
"scala.jdk.LongAccumulatorStepper",
231+
"scala.Long",
232+
"scala.math.Equiv",
233+
"scala.runtime.LambdaDeserialize",
234+
"scala.runtime.StructuralCallSite",
235+
"scala.Short",
236+
"scala.sys.process.ProcessBuilderImpl",
237+
"scala.sys.ShutdownHookThread",
238+
"scala.Unit",
239+
"scala.util.control.Breaks",
240+
241+
// class StringContext does not have a member method f
242+
"scala.collection.immutable.RedBlackTree",
243+
"scala.collection.mutable.HashTable",
244+
"scala.collection.mutable.RedBlackTree",
245+
246+
// -- Error:
247+
// | def addOne(kv: (K, V)) = {
248+
// | ^
249+
// |error overriding method addOne in trait Growable of type (elem: (K, V)): (TrieMap.this : scala.collection.concurrent.TrieMap[K, V]);
250+
// | method addOne of type (kv: (K, V)): (TrieMap.this : scala.collection.concurrent.TrieMap[K, V]) has incompatible type
251+
// -- Error:
252+
// | def subtractOne(k: K) = {
253+
// | ^
254+
// |error overriding method subtractOne in trait Shrinkable of type (elem: K): (TrieMap.this : scala.collection.concurrent.TrieMap[K, V]);
255+
// | method subtractOne of type (k: K): (TrieMap.this : scala.collection.concurrent.TrieMap[K, V]) has incompatible type
256+
"scala.collection.concurrent.TrieMap",
257+
"scala.collection.immutable.HashMapBuilder",
258+
"scala.collection.immutable.HashSetBuilder",
259+
"scala.collection.immutable.ListMapBuilder",
260+
"scala.collection.immutable.MapBuilderImpl",
261+
"scala.collection.immutable.SetBuilderImpl",
262+
"scala.collection.immutable.VectorMapBuilder",
263+
"scala.collection.mutable.AnyRefMap",
264+
"scala.collection.mutable.ArrayBuilder",
265+
"scala.collection.mutable.LongMap",
266+
"scala.collection.mutable.SortedMap",
267+
"scala.collection.mutable.StringBuilder",
268+
"scala.jdk.AnyAccumulator",
269+
270+
// -- Error:
271+
// | def force: this.type = {
272+
// | ^
273+
// |error overriding method force in class Stream of type => (Cons.this : scala.collection.immutable.Stream.Cons[A]);
274+
// | method force of type => (Cons.this : scala.collection.immutable.Stream.Cons[A]) has incompatible type
275+
"scala.collection.immutable.Stream",
276+
277+
// assertion failed: private method checkFallback in object PartialFunction
278+
"scala.PartialFunction",
279+
280+
// fails on CI only
281+
// at dotty.DottyPredef$.assertFail(DottyPredef.scala:17)
282+
// at dotty.tools.dotc.transform.ExpandPrivate.ensurePrivateAccessible(ExpandPrivate.scala:84)
283+
// at dotty.tools.dotc.transform.ExpandPrivate.transformSelect(ExpandPrivate.scala:97)
284+
// at dotty.tools.dotc.transform.ExpandPrivate.transformSelect(ExpandPrivate.scala:96)
285+
"scala.util.DynamicVariable",
286+
"scala.util.Using",
287+
)
288+
289+
def loadBlacklisted = loadBlacklist.toSet
290+
def compileBlacklisted = compileBlacklist.toSet
291+
292+
end TastyInspectorTest

‎project/Build.scala

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -817,6 +817,74 @@ object Build {
817817
javaOptions := (javaOptions in `dotty-compiler-bootstrapped`).value
818818
)
819819

820+
lazy val `dotty-scala-library-from-tasty-tests` = project.in(file("dotty-scala-library-from-tasty-tests")).
821+
withCommonSettings(Bootstrapped).
822+
dependsOn(`dotty-tasty-inspector` % "test->test").
823+
settings(commonBootstrappedSettings).
824+
settings(
825+
javaOptions := (javaOptions in `dotty-compiler-bootstrapped`).value,
826+
javaOptions += "-Ddotty.scala.library=" + packageBin.in(`dotty-scala-library`, Compile).value.getAbsolutePath
827+
)
828+
829+
830+
// Scala library compiled by dotty using the latest published sources of the library
831+
lazy val `dotty-scala-library` = project.in(file("dotty-scala-library")).
832+
withCommonSettings(Bootstrapped).
833+
dependsOn(dottyCompiler(Bootstrapped) % "provided; compile->runtime; test->test").
834+
dependsOn(`dotty-tasty-inspector` % "test->test").
835+
settings(commonBootstrappedSettings).
836+
settings(
837+
javaOptions := (javaOptions in `dotty-compiler-bootstrapped`).value,
838+
scalacOptions -= "-Xfatal-warnings",
839+
ivyConfigurations += SourceDeps.hide,
840+
transitiveClassifiers := Seq("sources"),
841+
libraryDependencies +=
842+
("org.scala-lang" % "scala-library" % stdlibVersion(Bootstrapped) % "sourcedeps"),
843+
sourceGenerators in Compile += Def.task {
844+
val s = streams.value
845+
val cacheDir = s.cacheDirectory
846+
val trgDir = (sourceManaged in Compile).value / "scala-library-src"
847+
848+
val report = updateClassifiers.value
849+
val scalaLibrarySourcesJar = report.select(
850+
configuration = configurationFilter("sourcedeps"),
851+
module = (_: ModuleID).name == "scala-library",
852+
artifact = artifactFilter(`type` = "src")).headOption.getOrElse {
853+
sys.error(s"Could not fetch scala-library sources")
854+
}
855+
856+
FileFunction.cached(cacheDir / s"fetchScalaLibrarySrc",
857+
FilesInfo.lastModified, FilesInfo.exists) { dependencies =>
858+
s.log.info(s"Unpacking scala-library sources to $trgDir...")
859+
if (trgDir.exists)
860+
IO.delete(trgDir)
861+
IO.createDirectory(trgDir)
862+
IO.unzip(scalaLibrarySourcesJar, trgDir)
863+
864+
((trgDir ** "*.scala") +++ (trgDir ** "*.java")).get.toSet
865+
} (Set(scalaLibrarySourcesJar)).toSeq
866+
}.taskValue,
867+
sources in Compile ~= (_.filterNot(file =>
868+
// sources from https://github.com/scala/scala/tree/2.13.x/src/library-aux
869+
file.getPath.endsWith("scala/Any.scala") ||
870+
file.getPath.endsWith("scala/AnyVal.scala") ||
871+
file.getPath.endsWith("scala/AnyRef.scala") ||
872+
file.getPath.endsWith("scala/Nothing.scala") ||
873+
file.getPath.endsWith("scala/Null.scala") ||
874+
file.getPath.endsWith("scala/Singleton.scala")))
875+
)
876+
877+
// Tests that we can compile and run an app using the scala-library compiled by dotty
878+
lazy val `dotty-scala-library-sandbox` = project.in(file("sandbox/dotty-scala-library")).
879+
withCommonSettings(Bootstrapped).
880+
dependsOn(`dotty-scala-library`).
881+
settings(
882+
managedClasspath in Compile ~= {
883+
_.filterNot(file => file.data.getName == "scala-library-2.13.3.jar")
884+
}
885+
)
886+
887+
820888
lazy val `dotty-sbt-bridge` = project.in(file("sbt-bridge/src")).
821889
// We cannot depend on any bootstrapped project to compile the bridge, since the
822890
// bridge is needed to compile these projects.

‎sandbox/dotty-scala-library/src/hello.scala

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
package hello
2+
3+
enum Color:
4+
case Red, Green, Blue
5+
6+
object HelloWorld:
7+
def main(args: Array[String]): Unit = {
8+
println("hello dotty.superbootstrapped!")
9+
println(Color.Red)
10+
println(Color.Green)
11+
println(Color.Blue)
12+
}

0 commit comments

Comments
 (0)