Skip to content

Commit f72e41b

Browse files
OlivierBlanvillainOlivierBlanvillain
committed
Fix scala#5809: Run the community-build on every PR
This commit is meant to replace the infrastructure previously setup in https://github.com/lampepfl/dotty-community-build. A new sbt project, community-build, is added to dotty's Build.scala and contains all the integration tests for the community build. In case of failure the tests will output instructions on how to reproduce the error without JUnit. Hopefully these instructions are sufficient to have everyone fix regressions as they are introduced, or as a very last resort, update the code for the community build. This should be the main improvement over the previous approach, where the community build maintenance was centralized (Allan did all the work) and asynchronous (it's now broken for more than a month...). Another improvement is that a published local version of the sbt plugin is used when running the integration test, so breakage to sbt-dotty should be caught earlier than before.
1 parent 5bec023 commit f72e41b

File tree

11 files changed

+271
-2
lines changed

11 files changed

+271
-2
lines changed

‎.drone.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ pipeline:
4141
group: test
4242
image: lampepfl/dotty:2018-12-12
4343
commands:
44-
- cp -R . /tmp/4/ && cd /tmp/4/
44+
- cp -R . /tmp/3/ && cd /tmp/3/
4545
- ./project/scripts/sbt sbt-dotty/scripted
4646
when:
4747
# sbt scripted tests are slow and only run on nightly or deployment

‎.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,3 +79,5 @@ bench/compile.txt
7979

8080
# The vscode app for testing
8181
vscode-dotty/.vscode-test
82+
dotty-bootstrapped.version
83+
sbt-dotty.sbt

‎.gitmodules

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,42 @@
66
path = scala2-library
77
url = https://github.com/lampepfl/scala.git
88
branch = dotty-library2.12
9+
[submodule "community-build/community-projects/algebra"]
10+
path = community-build/community-projects/algebra
11+
url = https://github.com/dotty-staging/algebra
12+
[submodule "community-build/community-projects/betterfiles"]
13+
path = community-build/community-projects/betterfiles
14+
url = https://github.com/dotty-staging/better-files
15+
[submodule "community-build/community-projects/scalacheck"]
16+
path = community-build/community-projects/scalacheck
17+
url = https://github.com/dotty-staging/scalacheck
18+
[submodule "community-build/community-projects/ScalaPB"]
19+
path = community-build/community-projects/ScalaPB
20+
url = https://github.com/dotty-staging/ScalaPB
21+
[submodule "community-build/community-projects/scalatest"]
22+
path = community-build/community-projects/scalatest
23+
url = https://github.com/dotty-staging/scalatest
24+
[submodule "community-build/community-projects/scopt"]
25+
path = community-build/community-projects/scopt
26+
url = https://github.com/dotty-staging/scopt
27+
[submodule "community-build/community-projects/squants"]
28+
path = community-build/community-projects/squants
29+
url = https://github.com/dotty-staging/squants
30+
[submodule "community-build/community-projects/scalap"]
31+
path = community-build/community-projects/scalap
32+
url = https://github.com/dotty-staging/scala
33+
[submodule "community-build/community-projects/minitest"]
34+
path = community-build/community-projects/minitest
35+
url = https://github.com/dotty-staging/minitest
36+
[submodule "community-build/community-projects/fastparse"]
37+
path = community-build/community-projects/fastparse
38+
url = https://github.com/dotty-staging/fastparse
39+
[submodule "community-build/community-projects/stdLib213"]
40+
path = community-build/community-projects/stdLib213
41+
url = https://github.com/dotty-staging/scala
42+
[submodule "community-build/community-projects/pdbp"]
43+
path = community-build/community-projects/pdbp
44+
url = https://github.com/dotty-staging/pdbp
45+
[submodule "community-build/community-projects/sourcecode"]
46+
path = community-build/community-projects/sourcecode
47+
url = https://github.com/dotty-staging/sourcecode

‎.travis.yml

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
sudo: false
2+
language: scala
3+
jdk: oraclejdk8
4+
jobs:
5+
include:
6+
- script: sbt "community-build/testOnly dotty.communitybuild.CommunityBuildTest1"
7+
- script: sbt "community-build/testOnly dotty.communitybuild.CommunityBuildTest2"
8+
- script: sbt "community-build/testOnly dotty.communitybuild.CommunityBuildTest3"
9+
- script: sbt "community-build/testOnly dotty.communitybuild.CommunityBuildTest4"
10+
- script: sbt "community-build/testOnly dotty.communitybuild.CommunityBuildTest5"
11+
12+
cache:
13+
directories:
14+
- $HOME/.ivy2/cache
15+
- $HOME/.sbt
16+
17+
before_cache:
18+
- rm -fv $HOME/.ivy2/.sbt.ivy.lock
19+
- find $HOME/.ivy2/cache -name "ivydata-*.properties" -print -delete
20+
- find $HOME/.sbt -name "*.lock" -print -delete

‎.vscode-template/settings.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
"**/*.class": true,
1010
"**/*.hasTasty": true,
1111
"**/target/": true,
12+
"community-build/community-projects": true,
1213
"scala2-library/{doc,docs,lib,META-INF,scripts,spec,test,tools}/": true, // only allow scala-backend/src
1314
"scala2-library/src/[abcefimprs]*": true, // only allow scala-backend/src/library
1415
"scala-backend/{doc,docs,lib,META-INF,scripts,spec,test,tools}/": true, // only allow scala-backend/src

‎build.sbt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ val `scala-reflect` = Build.`scala-reflect`
1919
val scalap = Build.scalap
2020
val dist = Build.dist
2121
val `dist-bootstrapped` = Build.`dist-bootstrapped`
22+
val `community-build` = Build.`community-build`
2223

2324
val sjsSandbox = Build.sjsSandbox
2425

‎community-build/community-projects/squants

‎community-build/project/build.properties

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
sbt.version=1.2.7

‎community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
package dotty.communitybuild
2+
3+
import java.nio.file._
4+
import java.io.{PrintWriter, File}
5+
import java.nio.charset.StandardCharsets.UTF_8
6+
import org.junit.{Ignore, Test}
7+
import org.junit.Assert.{assertEquals, fail}
8+
9+
abstract class BuildTest {
10+
lazy val communitybuildDir: Path = Paths.get(sys.props("user.dir") + "/community-build/")
11+
12+
lazy val compilerVersion: String = {
13+
val file = communitybuildDir.resolve("dotty-bootstrapped.version")
14+
new String(Files.readAllBytes(file), UTF_8)
15+
}
16+
17+
/** Build the given project with the published local compiler and sbt plugin.
18+
*
19+
* This test reads the compiler version from community-build/dotty-bootstrapped.version
20+
* and expects community-build/sbt-dotty.sbt to set the compiler plugin.
21+
*
22+
* @param project The project name, should be a git submodule in community-build/
23+
* @param command The sbt command used to build the project
24+
*/
25+
def test(project: String, command: String): Unit = {
26+
def log(msg: String) = println(Console.GREEN + msg + Console.RESET)
27+
28+
log(s"Building $project with dotty-bootstrapped $compilerVersion...")
29+
30+
val projectDir = communitybuildDir.resolve("community-projects").resolve(project)
31+
32+
if (!Files.exists(projectDir.resolve(".git"))) {
33+
fail(s"""
34+
|
35+
|Missing community-build/community-projects/$project submodule. You can initialize this module using
36+
|
37+
| git submodule update --init $projectDir
38+
|
39+
|""".stripMargin)
40+
}
41+
42+
/** Executes shell command, returns false in case of error. */
43+
def exec(binary: String, arguments: String*): Int = {
44+
val command = binary +: arguments
45+
log(command.mkString(" "))
46+
val builder = new ProcessBuilder(command: _*).directory(projectDir.toFile).inheritIO()
47+
val process = builder.start()
48+
val exitCode = process.waitFor()
49+
exitCode
50+
}
51+
52+
// Workarounds for https://github.com/sbt/sbt/issues/4395
53+
new File(sys.props("user.home") + "/.sbt/1.0/plugins").mkdirs()
54+
val pluginFilePath = communitybuildDir.resolve("sbt-dotty.sbt").toAbsolutePath().toString()
55+
56+
// Run the sbt command with the compiler version and sbt plugin set in the build
57+
val arguments = Seq(
58+
"-sbt-version", "1.2.7",
59+
s"--addPluginSbtFile=$pluginFilePath",
60+
"clean",
61+
s"++$compilerVersion!",
62+
command
63+
)
64+
65+
val exitCode = exec("sbt", arguments: _*)
66+
67+
if (exitCode != 0) {
68+
fail(s"""
69+
|
70+
|sbt exited with an error code. To reproduce without JUnit, use:
71+
|
72+
| sbt community-build/prepareCommunityBuild
73+
| cd community-build/community-projects/$project
74+
| sbt ${arguments.mkString(" ")}
75+
|
76+
|For a faster feedback loop, one can try to extract a direct call to dotc from sbt:
77+
|
78+
| cd community-build/community-projects/$project
79+
| sbt ${arguments.init.mkString(" ")} export ${arguments.last.takeWhile(_ != ':')}:compileIncremental
80+
|
81+
|""".stripMargin)
82+
}
83+
}
84+
}
85+
86+
// The tests are divided into 5 classes, run in parrallele by Travis CI
87+
// (.travis.yml). When adding a new projects to the community build please
88+
// select the fastest CommunityBuildTestN class according to the running
89+
// times from https://travis-ci.org/lampepfl/dotty.
90+
91+
class CommunityBuildTest1 extends BuildTest {
92+
@Test def algebra = test(
93+
project = "algebra",
94+
command = "coreJVM/compile"
95+
)
96+
97+
@Test def scalacheck = test(
98+
project = "scalacheck",
99+
command = "jvm/test:compile"
100+
)
101+
}
102+
103+
class CommunityBuildTest2 extends BuildTest {
104+
@Test def scalatest = test(
105+
project = "scalatest",
106+
command = "scalatest/compile"
107+
)
108+
109+
@Test def scopt = test(
110+
project = "scopt",
111+
command = "scoptJVM/compile"
112+
)
113+
114+
@Test def scalap = test(
115+
project = "scalap",
116+
command = "scalap/compile"
117+
)
118+
}
119+
120+
class CommunityBuildTest3 extends BuildTest {
121+
@Test def squants = test(
122+
project = "squants",
123+
command = "squantsJVM/compile"
124+
)
125+
126+
@Test def betterfiles = test(
127+
project = "betterfiles",
128+
command = "dottyCompile"
129+
)
130+
131+
@Test def ScalaPB = test(
132+
project = "ScalaPB",
133+
command = "dottyCompile"
134+
)
135+
}
136+
137+
class CommunityBuildTest4 extends BuildTest {
138+
@Test def minitest = test(
139+
project = "minitest",
140+
command = "dottyCompile"
141+
)
142+
143+
@Test def fastparse = test(
144+
project = "fastparse",
145+
command = "fastparseJVM/compile"
146+
)
147+
148+
// TODO: revert to sourcecodeJVM/test
149+
@Test def sourcecode = test(
150+
project = "sourcecode",
151+
command = "sourcecodeJVM/compile"
152+
)
153+
}
154+
155+
class CommunityBuildTest5 extends BuildTest {
156+
// TODO @smarter?
157+
// @Test def stdLib213 = test(
158+
// project = "stdLib213",
159+
// command = "library/compile"
160+
// )
161+
162+
// TODO @oderky? It got broken by #5458
163+
// @Test def pdbp = test(
164+
// project = "pdbp",
165+
// command = "compile"
166+
// )
167+
}

‎community-build/test/scala/dotty/communitybuild/readme.md

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# Dotty Community Build
2+
3+
This project contains tests to build and test a corpus of open sources Scala projects against the latest version of Dotty.
4+
5+
To run the community build on a local machine, first fetch all the git submodules with `git submodule update --init` and run `sbt community-build/test` from the root of the dotty repo.
6+
7+
## Adding your project
8+
9+
To add your project to the community build you can follow these steps:
10+
11+
1. Get your project to compile with Dotty. Instructions can be found on the [dotty-example-project](https://github.com/lampepfl/dotty-example-project).
12+
See the submodules in [community-projects](https://github.com/lampepfl/dotty/tree/master/community-build/community-projects/) for examples of projects that compile with Dotty.
13+
14+
2. Open a PR against this repo that:
15+
- Adds your project as a new git submodule
16+
- Adds a test in [CommunityBuildTest.scala](https://github.com/lampepfl/dotty/blob/master/src/test/scala/dotty/community-build/src/test/scala/dotty/communitybuild/CommunityBuildTest.scala)

‎project/Build.scala

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1013,6 +1013,28 @@ object Build {
10131013
}.dependsOn(compile in Compile).evaluated
10141014
)
10151015

1016+
val prepareCommunityBuild = taskKey[Unit]("Publish local the compiler and the sbt plugin. Also store the versions of the published local artefacts in two files, community-build/{dotty-bootstrapped.version,sbt-dotty.sbt}.")
1017+
1018+
lazy val `community-build` = project.in(file("community-build")).
1019+
settings(commonNonBootstrappedSettings).
1020+
settings(
1021+
prepareCommunityBuild := {
1022+
(publishLocal in `dotty-sbt-bridge`).value
1023+
(publishLocal in `dotty-interfaces`).value
1024+
(publishLocal in `scala-library`).value
1025+
(publishLocal in `scala-reflect`).value
1026+
(publishLocal in `dotty-library-bootstrapped`).value
1027+
(publishLocal in `dotty-doc-bootstrapped`).value
1028+
(publishLocal in `dotty-compiler-bootstrapped`).value
1029+
(publishLocal in `sbt-dotty`).value
1030+
(publishLocal in `dotty-bootstrapped`).value
1031+
val pluginText = s"""addSbtPlugin("ch.epfl.lamp" % "sbt-dotty" % "$sbtDottyVersion")"""
1032+
IO.write(baseDirectory.value / "sbt-dotty.sbt", pluginText)
1033+
IO.write(baseDirectory.value / "dotty-bootstrapped.version", dottyVersion)
1034+
},
1035+
(Test / testOnly) := ((Test / testOnly) dependsOn prepareCommunityBuild).evaluated,
1036+
(Test / test ) := ((Test / test ) dependsOn prepareCommunityBuild).value
1037+
)
10161038

10171039
lazy val publishSettings = Seq(
10181040
publishMavenStyle := true,

0 commit comments

Comments
 (0)