Skip to content

Commit 5967bff

Browse files
OlivierBlanvillainOlivierBlanvillain
committed
Fix #5809: Run the community-build on every PR
This commit is meant to replace the infrastructure previously setup in https://github.com/lampepfl/dotty-community-build. A new sbt project, community-build, is added to dotty's Build.scala and contains all the integration tests for the community build. In case of failure the tests will output instructions on how to reproduce the error without JUnit. Hopefully these instructions are sufficient to have everyone fix regressions as they are introduced, or as a very last resort, update the code for the community build. This should be the main improvement over the previous approach, where the community build maintenance was centralized (Allan did all the work) and asynchronous (it's now broken for more than a month...). Another improvement is that a published local version of the sbt plugin is used when running the integration test, so breakage to sbt-dotty should be caught earlier than before.
1 parent 5bec023 commit 5967bff

File tree

10 files changed

+247
-8
lines changed

10 files changed

+247
-8
lines changed

‎.drone.yml

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,23 +23,31 @@ pipeline:
2323
# We run tests in parallel. Tests run in a copy of the working directory to avoid conflict
2424
test:
2525
group: test
26-
image: lampepfl/dotty:2018-12-12
26+
image: lampepfl/dotty:2019-02-06
2727
commands:
2828
- cp -R . /tmp/1/ && cd /tmp/1/
2929
- ./project/scripts/sbt ";compile ;test"
3030
- ./project/scripts/cmdTests
3131

3232
test_bootstrapped:
3333
group: test
34-
image: lampepfl/dotty:2018-12-12
34+
image: lampepfl/dotty:2019-02-06
3535
commands:
3636
- cp -R . /tmp/2/ && cd /tmp/2/
3737
- ./project/scripts/sbt ";dotty-bootstrapped/compile ;dotty-bootstrapped/test ;sjsSandbox/run"
3838
- ./project/scripts/bootstrapCmdTests
3939

40+
community_build:
41+
group: test
42+
image: lampepfl/dotty:2019-02-06
43+
commands:
44+
- cp -R . /tmp/3/ && cd /tmp/3/
45+
- export PATH=/tmp/4/project/scripts:$PATH
46+
- sbt community-build/test
47+
4048
test_sbt:
4149
group: test
42-
image: lampepfl/dotty:2018-12-12
50+
image: lampepfl/dotty:2019-02-06
4351
commands:
4452
- cp -R . /tmp/4/ && cd /tmp/4/
4553
- ./project/scripts/sbt sbt-dotty/scripted
@@ -49,7 +57,7 @@ pipeline:
4957

5058
# DOCUMENTATION:
5159
documentation:
52-
image: lampepfl/dotty:2018-12-12
60+
image: lampepfl/dotty:2019-02-06
5361
commands:
5462
- ./project/scripts/genDocs
5563
secrets: [ bot_token ]
@@ -61,7 +69,7 @@ pipeline:
6169
# PUBLISHING:
6270
# Publishing expect NIGHTLYBUILD or RELEASEBUILD to be set. See dottyVersion in Build.scala
6371
publish_nightly:
64-
image: lampepfl/dotty:2018-12-12
72+
image: lampepfl/dotty:2019-02-06
6573
environment:
6674
- NIGHTLYBUILD=yes
6775
commands:
@@ -72,7 +80,7 @@ pipeline:
7280
environment: nightly
7381

7482
publish_release:
75-
image: lampepfl/dotty:2018-12-12
83+
image: lampepfl/dotty:2019-02-06
7684
environment:
7785
- RELEASEBUILD=yes
7886
commands:
@@ -96,7 +104,7 @@ pipeline:
96104
event: tag
97105

98106
publish_sbt_release:
99-
image: lampepfl/dotty:2018-12-12
107+
image: lampepfl/dotty:2019-02-06
100108
environment:
101109
- RELEASEBUILD=yes
102110
commands:

‎.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,3 +79,5 @@ bench/compile.txt
7979

8080
# The vscode app for testing
8181
vscode-dotty/.vscode-test
82+
dotty-bootstrapped.version
83+
sbt-dotty.sbt

‎.gitmodules

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,42 @@
66
path = scala2-library
77
url = https://github.com/lampepfl/scala.git
88
branch = dotty-library2.12
9+
[submodule "community-build/community-projects/algebra"]
10+
path = community-build/community-projects/algebra
11+
url = https://github.com/dotty-staging/algebra
12+
[submodule "community-build/community-projects/betterfiles"]
13+
path = community-build/community-projects/betterfiles
14+
url = https://github.com/dotty-staging/better-files
15+
[submodule "community-build/community-projects/scalacheck"]
16+
path = community-build/community-projects/scalacheck
17+
url = https://github.com/dotty-staging/scalacheck
18+
[submodule "community-build/community-projects/ScalaPB"]
19+
path = community-build/community-projects/ScalaPB
20+
url = https://github.com/dotty-staging/ScalaPB
21+
[submodule "community-build/community-projects/scalatest"]
22+
path = community-build/community-projects/scalatest
23+
url = https://github.com/dotty-staging/scalatest
24+
[submodule "community-build/community-projects/scopt"]
25+
path = community-build/community-projects/scopt
26+
url = https://github.com/dotty-staging/scopt
27+
[submodule "community-build/community-projects/squants"]
28+
path = community-build/community-projects/squants
29+
url = https://github.com/dotty-staging/squants
30+
[submodule "community-build/community-projects/scalap"]
31+
path = community-build/community-projects/scalap
32+
url = https://github.com/dotty-staging/scala
33+
[submodule "community-build/community-projects/minitest"]
34+
path = community-build/community-projects/minitest
35+
url = https://github.com/dotty-staging/minitest
36+
[submodule "community-build/community-projects/fastparse"]
37+
path = community-build/community-projects/fastparse
38+
url = https://github.com/dotty-staging/fastparse
39+
[submodule "community-build/community-projects/stdLib213"]
40+
path = community-build/community-projects/stdLib213
41+
url = https://github.com/dotty-staging/scala
42+
[submodule "community-build/community-projects/pdbp"]
43+
path = community-build/community-projects/pdbp
44+
url = https://github.com/dotty-staging/pdbp
45+
[submodule "community-build/community-projects/sourcecode"]
46+
path = community-build/community-projects/sourcecode
47+
url = https://github.com/dotty-staging/sourcecode

‎.vscode-template/settings.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
"**/*.class": true,
1010
"**/*.hasTasty": true,
1111
"**/target/": true,
12+
"community-build/community-projects": true,
1213
"scala2-library/{doc,docs,lib,META-INF,scripts,spec,test,tools}/": true, // only allow scala-backend/src
1314
"scala2-library/src/[abcefimprs]*": true, // only allow scala-backend/src/library
1415
"scala-backend/{doc,docs,lib,META-INF,scripts,spec,test,tools}/": true, // only allow scala-backend/src

‎build.sbt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ val `scala-reflect` = Build.`scala-reflect`
1919
val scalap = Build.scalap
2020
val dist = Build.dist
2121
val `dist-bootstrapped` = Build.`dist-bootstrapped`
22+
val `community-build` = Build.`community-build`
2223

2324
val sjsSandbox = Build.sjsSandbox
2425

‎community-build/community-projects/squants

‎community-build/project/build.properties

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
sbt.version=1.2.7

‎community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
package dotty.communitybuild
2+
3+
import java.nio.file._
4+
import java.io.{PrintWriter, File}
5+
import java.nio.charset.StandardCharsets.UTF_8
6+
import org.junit.{Ignore, Test}
7+
import org.junit.Assert.{assertEquals, fail}
8+
9+
class CommunityBuildTest {
10+
lazy val communitybuildDir: Path = Paths.get(sys.props("user.dir") + "/community-build/")
11+
12+
lazy val compilerVersion: String = {
13+
val file = communitybuildDir.resolve("dotty-bootstrapped.version")
14+
new String(Files.readAllBytes(file), UTF_8)
15+
}
16+
17+
/** Build the given project with the published local compiler and sbt plugin.
18+
*
19+
* This test reads the compiler version from community-build/dotty-bootstrapped.version
20+
* and expects community-build/sbt-dotty.sbt to set the compiler plugin.
21+
*
22+
* @param project The project name, should be a git submodule in community-build/
23+
* @param command The sbt command used to build the project
24+
*/
25+
def test(project: String, command: String): Unit = {
26+
def log(msg: String) = println(Console.GREEN + msg + Console.RESET)
27+
28+
log(s"Building $project with dotty-bootstrapped $compilerVersion...")
29+
30+
val projectDir = communitybuildDir.resolve("community-projects").resolve(project)
31+
32+
if (!Files.exists(projectDir.resolve(".git"))) {
33+
fail(s"""
34+
|
35+
|Missing $project submodule. You can initialize this module using
36+
|
37+
| git submodule update --init community-build/community-projects/$project
38+
|
39+
|""".stripMargin)
40+
}
41+
42+
/** Executes shell command, returns false in case of error. */
43+
def exec(binary: String, arguments: String*): Int = {
44+
val command = binary +: arguments
45+
log(command.mkString(" "))
46+
val builder = new ProcessBuilder(command: _*).directory(projectDir.toFile).inheritIO()
47+
val process = builder.start()
48+
val exitCode = process.waitFor()
49+
exitCode
50+
}
51+
52+
// Workaround for https://github.com/sbt/sbt/issues/4395
53+
new File(sys.props("user.home") + "/.sbt/1.0/plugins").mkdirs()
54+
val pluginFilePath = communitybuildDir.resolve("sbt-dotty.sbt").toAbsolutePath().toString()
55+
56+
// Run the sbt command with the compiler version and sbt plugin set in the build
57+
val arguments = Seq(
58+
"-sbt-version", "1.2.7",
59+
s"--addPluginSbtFile=$pluginFilePath",
60+
s";clean ;set updateOptions in Global ~= (_.withLatestSnapshots(false)) ;++$compilerVersion! $command"
61+
)
62+
63+
val exitCode = exec("sbt", arguments: _*)
64+
65+
if (exitCode != 0) {
66+
fail(s"""
67+
|
68+
|sbt exited with an error code. To reproduce without JUnit, use:
69+
|
70+
| sbt community-build/prepareCommunityBuild
71+
| cd community-build/community-projects/$project
72+
| sbt ${arguments.init.mkString(" ")} "${arguments.last}"
73+
|
74+
|For a faster feedback loop, one can try to extract a direct call to dotc from
75+
|usign the sbt export command. For instance, for scalacheck, use
76+
| sbt export jvm/test:compileIncremental
77+
|
78+
|""".stripMargin)
79+
}
80+
}
81+
82+
@Test def algebra = test(
83+
project = "algebra",
84+
command = "coreJVM/compile"
85+
)
86+
87+
@Test def scalacheck = test(
88+
project = "scalacheck",
89+
command = "jvm/test:compile"
90+
)
91+
92+
@Test def scalatest = test(
93+
project = "scalatest",
94+
command = "scalatest/compile"
95+
)
96+
97+
@Test def scopt = test(
98+
project = "scopt",
99+
command = "scoptJVM/compile"
100+
)
101+
102+
@Test def scalap = test(
103+
project = "scalap",
104+
command = "scalap/compile"
105+
)
106+
107+
@Test def squants = test(
108+
project = "squants",
109+
command = "squantsJVM/compile"
110+
)
111+
112+
@Test def betterfiles = test(
113+
project = "betterfiles",
114+
command = "dottyCompile"
115+
)
116+
117+
@Test def ScalaPB = test(
118+
project = "ScalaPB",
119+
command = "dottyCompile"
120+
)
121+
122+
@Test def minitest = test(
123+
project = "minitest",
124+
command = "dottyCompile"
125+
)
126+
127+
@Test def fastparse = test(
128+
project = "fastparse",
129+
command = "fastparseJVM/compile"
130+
)
131+
132+
// TODO: revert to sourcecodeJVM/test
133+
@Test def sourcecode = test(
134+
project = "sourcecode",
135+
command = "sourcecodeJVM/compile"
136+
)
137+
138+
// TODO @smarter?
139+
// @Test def stdLib213 = test(
140+
// project = "stdLib213",
141+
// command = "library/compile"
142+
// )
143+
144+
// TODO @oderky? It got broken by #5458
145+
// @Test def pdbp = test(
146+
// project = "pdbp",
147+
// command = "compile"
148+
// )
149+
}

‎community-build/test/scala/dotty/communitybuild/readme.md

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# Dotty Community Build
2+
3+
This project contains tests to build and test a corpus of open sources Scala projects against the latest version of Dotty.
4+
5+
To run the community build on a local machine, first fetch all the git submodules with `git submodule update --init` and run `sbt community-build/test` from the root of the dotty repo.
6+
7+
## Adding your project
8+
9+
To add your project to the community build you can follow these steps:
10+
11+
1. Get your project to compile with Dotty. Instructions can be found on the [dotty-example-project](https://github.com/lampepfl/dotty-example-project).
12+
See the submodules in [community-projects](https://github.com/lampepfl/dotty/tree/master/community-build/community-projects/) for examples of projects that compile with Dotty.
13+
14+
2. Open a PR against this repo that:
15+
- Adds your project as a new git submodule
16+
- Adds a test in [CommunityBuildTest.scala](https://github.com/lampepfl/dotty/blob/master/src/test/scala/dotty/community-build/src/test/scala/dotty/communitybuild/CommunityBuildTest.scala)

‎project/Build.scala

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1013,6 +1013,28 @@ object Build {
10131013
}.dependsOn(compile in Compile).evaluated
10141014
)
10151015

1016+
val prepareCommunityBuild = taskKey[Unit]("Publish local the compiler and the sbt plugin. Also store the versions of the published local artefacts in two files, community-build/{dotty-bootstrapped.version,sbt-dotty.sbt}.")
1017+
1018+
lazy val `community-build` = project.in(file("community-build")).
1019+
settings(commonNonBootstrappedSettings).
1020+
settings(
1021+
prepareCommunityBuild := {
1022+
(publishLocal in `dotty-sbt-bridge`).value
1023+
(publishLocal in `dotty-interfaces`).value
1024+
(publishLocal in `scala-library`).value
1025+
(publishLocal in `scala-reflect`).value
1026+
(publishLocal in `dotty-library-bootstrapped`).value
1027+
(publishLocal in `dotty-doc-bootstrapped`).value
1028+
(publishLocal in `dotty-compiler-bootstrapped`).value
1029+
(publishLocal in `sbt-dotty`).value
1030+
(publishLocal in `dotty-bootstrapped`).value
1031+
val pluginText = s"""addSbtPlugin("ch.epfl.lamp" % "sbt-dotty" % "$sbtDottyVersion")"""
1032+
IO.write(baseDirectory.value / "sbt-dotty.sbt", pluginText)
1033+
IO.write(baseDirectory.value / "dotty-bootstrapped.version", dottyVersion)
1034+
},
1035+
(Test / testOnly) := ((Test / testOnly) dependsOn prepareCommunityBuild).evaluated,
1036+
(Test / test ) := ((Test / test ) dependsOn prepareCommunityBuild).value
1037+
)
10161038

10171039
lazy val publishSettings = Seq(
10181040
publishMavenStyle := true,

0 commit comments

Comments
 (0)