Skip to content

Commit 8e2dd34

Browse files
OlivierBlanvillainOlivierBlanvillain
committed
Fix #5809: Run the community-build on every PR
This commit is meant to replace the infrastructure previously setup in https://github.com/lampepfl/dotty-community-build. A new sbt project, community-build, is added to dotty's Build.scala and contains all the integration tests for the community build. In case of failure the tests will output instructions on how to reproduce the error without JUnit. Hopefully these instructions are sufficient to have everyone fix regressions as they are introduced, or as a very last resort, update the code for the community build. This should be the main improvement over the previous approach, where the community build maintenance was centralized (Allan did all the work) and asynchronous (it's now broken for more than a month...). Another improvement is that a published local version of the sbt plugin is used when running the integration test, so breakage to sbt-dotty should be cough earlier than before.
1 parent f9e58c9 commit 8e2dd34

File tree

27 files changed

+260
-11
lines changed

27 files changed

+260
-11
lines changed

‎.drone.yml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,13 @@ pipeline:
3737
- ./project/scripts/sbt ";dotty-bootstrapped/compile ;dotty-bootstrapped/test ;sjsSandbox/run"
3838
- ./project/scripts/bootstrapCmdTests
3939

40+
test_community_build:
41+
group: test
42+
image: lampepfl/dotty:2018-12-12
43+
commands:
44+
- cp -R . /tmp/3/ && cd /tmp/3/
45+
- ./project/scripts/sbt community-build/test
46+
4047
test_sbt:
4148
group: test
4249
image: lampepfl/dotty:2018-12-12

‎.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,3 +79,5 @@ bench/compile.txt
7979

8080
# The vscode app for testing
8181
vscode-dotty/.vscode-test
82+
dotty-bootstrapped.version
83+
sbt-dotty.sbt

‎.gitmodules

Lines changed: 42 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,45 @@
66
path = scala2-library
77
url = https://github.com/lampepfl/scala.git
88
branch = dotty-library2.12
9-
[submodule "collection-strawman"]
10-
path = collection-strawman
11-
url = https://github.com/dotty-staging/collection-strawman.git
12-
branch = dotty
9+
[submodule "community-build/community-projects/algebra"]
10+
path = community-build/community-projects/algebra
11+
url = https://github.com/dotty-staging/algebra
12+
[submodule "community-build/community-projects/betterfiles"]
13+
path = community-build/community-projects/betterfiles
14+
url = https://github.com/dotty-staging/better-files
15+
[submodule "community-build/community-projects/scalacheck"]
16+
path = community-build/community-projects/scalacheck
17+
url = https://github.com/dotty-staging/scalacheck
18+
[submodule "community-build/community-projects/ScalaPB"]
19+
path = community-build/community-projects/ScalaPB
20+
url = https://github.com/dotty-staging/ScalaPB
21+
[submodule "community-build/community-projects/scalatest"]
22+
path = community-build/community-projects/scalatest
23+
url = https://github.com/dotty-staging/scalatest
24+
[submodule "community-build/community-projects/scopt"]
25+
path = community-build/community-projects/scopt
26+
url = https://github.com/dotty-staging/scopt
27+
[submodule "community-build/community-projects/squants"]
28+
path = community-build/community-projects/squants
29+
url = https://github.com/dotty-staging/squants
30+
[submodule "community-build/community-projects/collection-strawman"]
31+
path = community-build/community-projects/collection-strawman/
32+
url = https://github.com/dotty-staging/collection-strawman
33+
[submodule "community-build/community-projects/scalap"]
34+
path = community-build/community-projects/scalap
35+
url = https://github.com/dotty-staging/scala
36+
[submodule "community-build/community-projects/minitest"]
37+
path = community-build/community-projects/minitest
38+
url = https://github.com/dotty-staging/minitest
39+
[submodule "community-build/community-projects/fastparse"]
40+
path = community-build/community-projects/fastparse
41+
url = https://github.com/dotty-staging/fastparse
42+
[submodule "community-build/community-projects/stdLib213"]
43+
path = community-build/community-projects/stdLib213
44+
url = https://github.com/dotty-staging/scala
45+
[submodule "community-build/community-projects/pdbp"]
46+
path = community-build/community-projects/pdbp
47+
url = https://github.com/dotty-staging/pdbp
48+
[submodule "community-build/community-projects/sourcecode"]
49+
path = community-build/community-projects/sourcecode
50+
url = https://github.com/dotty-staging/sourcecode

‎.vscode-template/settings.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
"**/*.class": true,
1010
"**/*.hasTasty": true,
1111
"**/target/": true,
12+
"community-build/community-projects": true,
1213
"scala2-library/{doc,docs,lib,META-INF,scripts,spec,test,tools}/": true, // only allow scala-backend/src
1314
"scala2-library/src/[abcefimprs]*": true, // only allow scala-backend/src/library
1415
"scala-backend/{doc,docs,lib,META-INF,scripts,spec,test,tools}/": true, // only allow scala-backend/src

‎bench/scripts/collection-strawman-cold.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
#!/usr/bin/env bash
2-
find collection-strawman/collections/src/main/ -type f \( -name "*.scala" -or -name "*.java" \) -exec echo "dotty-bench-bootstrapped/jmh:run 0 1 10" {} + | sbt
2+
find community-build/community-projects/collection-strawman/collections/src/main/ -type f \( -name "*.scala" -or -name "*.java" \) -exec echo "dotty-bench-bootstrapped/jmh:run 0 1 10" {} + | sbt

‎bench/scripts/collection-strawman.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
#!/usr/bin/env bash
2-
find collection-strawman/collections/src/main/ -type f \( -name "*.scala" -or -name "*.java" \) -exec echo "dotty-bench-bootstrapped/jmh:run" {} + | sbt
2+
find community-build/community-projects/collection-strawman/collections/src/main/ -type f \( -name "*.scala" -or -name "*.java" \) -exec echo "dotty-bench-bootstrapped/jmh:run" {} + | sbt

‎build.sbt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ val `scala-reflect` = Build.`scala-reflect`
1919
val scalap = Build.scalap
2020
val dist = Build.dist
2121
val `dist-bootstrapped` = Build.`dist-bootstrapped`
22+
val `community-build` = Build.`community-build`
2223

2324
val sjsSandbox = Build.sjsSandbox
2425

‎collection-strawman

Lines changed: 0 additions & 1 deletion
This file was deleted.

‎community-build/community-projects/ScalaPB

Submodule ScalaPB added at 748609b

‎community-build/community-projects/algebra

Submodule algebra added at a964062

‎community-build/community-projects/betterfiles

Submodule betterfiles added at 81ed503

‎community-build/community-projects/collection-strawman

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Subproject commit 93f43aa5eaafb54fc272a0561d4523e50a2b95ca

‎community-build/community-projects/fastparse

Submodule fastparse added at ed1a044

‎community-build/community-projects/minitest

Submodule minitest added at 305cc8c

‎community-build/community-projects/pdbp

Submodule pdbp added at 604c06c

‎community-build/community-projects/scalacheck

Submodule scalacheck added at 83ce0bb

‎community-build/community-projects/scalap

Submodule scalap added at 625ae7d

‎community-build/community-projects/scalatest

Submodule scalatest added at 4af2ea2

‎community-build/community-projects/scopt

Submodule scopt added at d5aad6a

‎community-build/community-projects/sourcecode

Submodule sourcecode added at 888c747

‎community-build/community-projects/squants

Submodule squants added at 625b92d

‎community-build/community-projects/stdLib213

Submodule stdLib213 added at 655bca8

‎community-build/src/test/scala/dotty/communitybuild/CommunityBuildTest.scala

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
package dotty.communitybuild
2+
3+
import java.nio.file._
4+
import java.io.{PrintWriter, File}
5+
import java.nio.charset.StandardCharsets.UTF_8
6+
import org.junit.{Ignore, Test}
7+
import org.junit.Assert.{assertEquals, fail}
8+
9+
class CommunityBuildTest {
10+
lazy val communitybuildDir: Path = Paths.get(sys.props("user.dir") + "/community-build/")
11+
12+
lazy val compilerVersion: String = {
13+
val file = communitybuildDir.resolve("dotty-bootstrapped.version")
14+
new String(Files.readAllBytes(file), UTF_8)
15+
}
16+
17+
/** Build the given project with the published local compiler and sbt plugin.
18+
*
19+
* This test reads the compiler version from community-build/dotty-bootstrapped.version
20+
* and expects community-build/sbt-dotty.sbt to set the compiler plugin.
21+
*
22+
* @param project The project name, should be a git submodule in community-build/
23+
* @param command The sbt command used to build the project
24+
*/
25+
def test(project: String, command: String): Unit = {
26+
def log(msg: String) = println(Console.GREEN + msg + Console.RESET)
27+
28+
log(s"Building $project with dotty-bootstrapped $compilerVersion...")
29+
30+
val projectDir = communitybuildDir.resolve("community-projects").resolve(project)
31+
32+
if (!Files.exists(projectDir.resolve(".git"))) {
33+
fail(s"""
34+
|
35+
|Missing community-build/community-projects/$project submodule. You can initialize this module using
36+
|
37+
| git submodule update --init $projectDir
38+
|
39+
|""".stripMargin)
40+
}
41+
42+
/** Executes shell command, returns false in case of error. */
43+
def exec(binary: String, arguments: String*): Boolean = {
44+
val command = binary +: arguments
45+
log(command.mkString(" "))
46+
val builder = new ProcessBuilder(command: _*).directory(projectDir.toFile).inheritIO()
47+
val process = builder.start()
48+
val exitCode = process.waitFor()
49+
exitCode == 0
50+
}
51+
52+
// Workarounds for https://github.com/sbt/sbt/issues/4395
53+
new File(sys.props("user.home") + "/.sbt/1.0/plugins").mkdirs()
54+
val pluginFilePath = communitybuildDir.resolve("sbt-dotty.sbt").toAbsolutePath().toString()
55+
56+
// Run the sbt command with the compiler version and sbt plugin set in the build
57+
if (!exec("sbt", s"--addPluginSbtFile=$pluginFilePath", s"++$compilerVersion!", command)) {
58+
fail(s"""
59+
|
60+
|sbt exited with an error code. To reproduce without JUnit, use:
61+
|
62+
| sbt prepare-for-comunity-build
63+
| cd community-build/community-projects/$project
64+
| sbt --addPluginSbtFile=$pluginFilePath "++$compilerVersion! $command"
65+
|
66+
|For a faster feedback loop, one can try to extract a direct call to dotc from sbt:
67+
|
68+
| cd community-build/community-projects/$project
69+
| sbt --addPluginSbtFile=$pluginFilePath "++$compilerVersion! export $command:compileIncremental"
70+
|
71+
|""".stripMargin)
72+
}
73+
74+
}
75+
76+
@Test def algebra = test(
77+
project = "algebra",
78+
command = "coreJVM/compile"
79+
)
80+
81+
@Test def betterfiles = test(
82+
project = "betterfiles",
83+
command = "dottyCompile"
84+
)
85+
86+
@Test def scalacheck = test(
87+
project = "scalacheck",
88+
command = "jvm/test:compile"
89+
)
90+
91+
@Test def ScalaPB = test(
92+
project = "ScalaPB",
93+
command = "dottyCompile"
94+
)
95+
96+
@Test def scalatest = test(
97+
project = "scalatest",
98+
command = "scalatest/compile"
99+
)
100+
101+
@Test def scopt = test(
102+
project = "scopt",
103+
command = "scoptJVM/compile"
104+
)
105+
106+
@Test def squants = test(
107+
project = "squants",
108+
command = "squantsJVM/compile"
109+
)
110+
111+
@Test def collectionstrawman = test(
112+
project = "collection-strawman",
113+
command = "dottyCompile"
114+
)
115+
116+
@Test def scalap = test(
117+
project = "scalap",
118+
command = "scalap/compile"
119+
)
120+
121+
@Test def minitest = test(
122+
project = "minitest",
123+
command = "dottyCompile"
124+
)
125+
126+
@Test def fastparse = test(
127+
project = "fastparse",
128+
command = "fastparseJVM/compile"
129+
)
130+
131+
// TODO @smarter?
132+
// @Test def stdLib213 = test(
133+
// project = "stdLib213",
134+
// command = "library/compile"
135+
// )
136+
137+
// TODO @oderky? It got broken by #5458
138+
// @Test def pdbp = test(
139+
// project = "pdbp",
140+
// command = "compile"
141+
// )
142+
143+
// TODO: revert to sourcecodeJVM/test
144+
@Test def sourcecode = test(
145+
project = "sourcecode",
146+
command = "sourcecodeJVM/compile"
147+
)
148+
}

‎community-build/src/test/scala/dotty/communitybuild/readme.md

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# Dotty Community Build
2+
3+
This project contains tests to build and test a corpus of open sources Scala projects against the latest version of Dotty.
4+
5+
To run the community build on a local machine, first fetch all the git submodules with `git submodule update --init` and run `sbt community-build/test` from the root of the dotty repo.
6+
7+
## Adding your project
8+
9+
To add your project to the community build you can follow these steps:
10+
11+
1. Get your project to compile with Dotty. Instructions can be found on the [dotty-example-project](https://github.com/lampepfl/dotty-example-project).
12+
See the submodules in [community-projects](https://github.com/lampepfl/dotty/tree/master/community-build/community-projects/) for examples of projects that compile with Dotty.
13+
14+
2. Open a PR against this repo that:
15+
- Adds your project as a new git submodule
16+
- Adds a test in [CommunityBuildTest.scala](https://github.com/lampepfl/dotty/blob/master/src/test/scala/dotty/community-build/src/test/scala/dotty/communitybuild/CommunityBuildTest.scala)

‎compiler/test/dotty/tools/dotc/CompilationTests.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ class CompilationTests extends ParallelTesting {
6363
),
6464
scala2Mode
6565
) +
66-
compileDir("collection-strawman/collections/src/main", defaultOptions.and("-Yno-imports")) +
66+
compileDir("community-build/community-projects/collection-strawman/collections/src/main", defaultOptions.and("-Yno-imports")) +
6767
compileFilesInDir("tests/pos-special/spec-t5545", defaultOptions) +
6868
compileFilesInDir("tests/pos-special/strawman-collections", defaultOptions) +
6969
compileFilesInDir("tests/pos-special/isInstanceOf", allowDeepSubtypes.and("-Xfatal-warnings")) +

‎compiler/test/dotty/tools/dotc/IdempotencyTests.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ class IdempotencyTests extends ParallelTesting {
3232

3333
def sourcesFrom(dir: Path) = CompilationTests.sources(Files.walk(dir))
3434

35-
val strawmanSources = sourcesFrom(Paths.get("collection-strawman/collections/src/main"))
35+
val strawmanSources = sourcesFrom(Paths.get("community-build/community-projects/collection-strawman/collections/src/main"))
3636
val strawmanSourcesSorted = strawmanSources.sorted
3737
val strawmanSourcesRevSorted = strawmanSourcesSorted.reverse
3838

‎project/Build.scala

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,6 @@ object Build {
100100
lazy val dotr =
101101
inputKey[Unit]("run compiled binary using the correct classpath, or the user supplied classpath")
102102

103-
104103
// Compiles the documentation and static site
105104
lazy val genDocs = taskKey[Unit]("run dottydoc to generate static documentation site")
106105

@@ -162,7 +161,7 @@ object Build {
162161
}
163162

164163
// Make sure all submodules are properly cloned
165-
val submodules = List("scala-backend", "scala2-library", "collection-strawman")
164+
val submodules = List("scala-backend", "scala2-library")
166165
if (!submodules.forall(exists)) {
167166
sLog.value.log(Level.Error,
168167
s"""Missing some of the submodules
@@ -1017,6 +1016,30 @@ object Build {
10171016
}.dependsOn(compile in Compile).evaluated
10181017
)
10191018

1019+
lazy val `prepare-for-comunity-build` = taskKey[Unit]("Publish local the compiler and the sbt plugin. Also store the versions of the published local artefacts in two files, community-build/{dotty-bootstrapped.version,sbt-dotty.sbt}.")
1020+
1021+
lazy val `community-build` = project.in(file("community-build")).
1022+
settings(
1023+
scalaVersion := scalacVersion,
1024+
`prepare-for-comunity-build` := {
1025+
(publishLocal in `dotty-sbt-bridge`).value
1026+
(publishLocal in `dotty-interfaces`).value
1027+
(publishLocal in `dotty-compiler-bootstrapped`).value
1028+
(publishLocal in `dotty-library-bootstrapped`).value
1029+
(publishLocal in `scala-library`).value
1030+
(publishLocal in `scala-reflect`).value
1031+
(publishLocal in `dotty-doc-bootstrapped`).value
1032+
(publishLocal in `dotty-bootstrapped`).value
1033+
(publishLocal in `sbt-dotty`).value
1034+
val pluginText = s"""addSbtPlugin("ch.epfl.lamp" % "sbt-dotty" % "$sbtDottyVersion")"""
1035+
IO.write(baseDirectory.value / "sbt-dotty.sbt", pluginText)
1036+
IO.write(baseDirectory.value / "dotty-bootstrapped.version", dottyVersion)
1037+
},
1038+
(Test / testOnly) := ((Test / testOnly) dependsOn `prepare-for-comunity-build`).evaluated,
1039+
(Test / test ) := ((Test / test ) dependsOn `prepare-for-comunity-build`).value,
1040+
libraryDependencies += "com.novocode" % "junit-interface" % "0.11" % Test,
1041+
testOptions in Test += Tests.Argument(TestFrameworks.JUnit, "-a", "-v")
1042+
)
10201043

10211044
lazy val publishSettings = Seq(
10221045
publishMavenStyle := true,

0 commit comments

Comments
 (0)