Merge pull request #5834 from dotty-staging/community-build

Fix #5809: Run the community-build on every PR

Author: OlivierBlanvillain

.drone.yml

@@ -23,23 +23,31 @@ pipeline:
   # We run tests in parallel. Tests run in a copy of the working directory to avoid conflict
   test:
     group: test
-    image: lampepfl/dotty:2018-12-12
+    image: lampepfl/dotty:2019-02-06
     commands:
       - cp -R . /tmp/1/ && cd /tmp/1/
       - ./project/scripts/sbt ";compile ;test"
       - ./project/scripts/cmdTests
 
   test_bootstrapped:
     group: test
-    image: lampepfl/dotty:2018-12-12
+    image: lampepfl/dotty:2019-02-06
     commands:
       - cp -R . /tmp/2/ && cd /tmp/2/
       - ./project/scripts/sbt ";dotty-bootstrapped/compile ;dotty-bootstrapped/test ;sjsSandbox/run"
       - ./project/scripts/bootstrapCmdTests
 
+  community_build:
+    group: test
+    image: lampepfl/dotty:2019-02-06
+    commands:
+      - cp -R . /tmp/3/ && cd /tmp/3/
+      - export PATH=/tmp/4/project/scripts:$PATH
+      - sbt community-build/test
+
   test_sbt:
     group: test
-    image: lampepfl/dotty:2018-12-12
+    image: lampepfl/dotty:2019-02-06
     commands:
       - cp -R . /tmp/4/ && cd /tmp/4/
       - ./project/scripts/sbt sbt-dotty/scripted
@@ -49,7 +57,7 @@ pipeline:
 
   # DOCUMENTATION:
   documentation:
-    image: lampepfl/dotty:2018-12-12
+    image: lampepfl/dotty:2019-02-06
     commands:
       - ./project/scripts/genDocs
     secrets: [ bot_token ]
@@ -61,7 +69,7 @@ pipeline:
   # PUBLISHING:
   # Publishing expect NIGHTLYBUILD or RELEASEBUILD to be set. See dottyVersion in Build.scala
   publish_nightly:
-    image: lampepfl/dotty:2018-12-12
+    image: lampepfl/dotty:2019-02-06
     environment:
       - NIGHTLYBUILD=yes
     commands:
@@ -72,7 +80,7 @@ pipeline:
       environment: nightly
 
   publish_release:
-    image: lampepfl/dotty:2018-12-12
+    image: lampepfl/dotty:2019-02-06
     environment:
       - RELEASEBUILD=yes
     commands:
@@ -96,7 +104,7 @@ pipeline:
       event: tag
 
   publish_sbt_release:
-    image: lampepfl/dotty:2018-12-12
+    image: lampepfl/dotty:2019-02-06
     environment:
       - RELEASEBUILD=yes
     commands:

.gitignore

@@ -79,3 +79,5 @@ bench/compile.txt
 
 # The vscode app for testing
 vscode-dotty/.vscode-test
+dotty-bootstrapped.version
+sbt-dotty.sbt

.gitmodules

@@ -6,7 +6,42 @@
 	path = scala2-library
 	url = https://github.com/lampepfl/scala.git
 	branch = dotty-library2.12
-[submodule "collection-strawman"]
-	path = collection-strawman
-	url = https://github.com/dotty-staging/collection-strawman.git
-	branch = dotty
+[submodule "community-build/community-projects/algebra"]
+	path = community-build/community-projects/algebra
+	url = https://github.com/dotty-staging/algebra
+[submodule "community-build/community-projects/betterfiles"]
+	path = community-build/community-projects/betterfiles
+	url = https://github.com/dotty-staging/better-files
+[submodule "community-build/community-projects/scalacheck"]
+	path = community-build/community-projects/scalacheck
+	url = https://github.com/dotty-staging/scalacheck
+[submodule "community-build/community-projects/ScalaPB"]
+	path = community-build/community-projects/ScalaPB
+	url = https://github.com/dotty-staging/ScalaPB
+[submodule "community-build/community-projects/scalatest"]
+	path = community-build/community-projects/scalatest
+	url = https://github.com/dotty-staging/scalatest
+[submodule "community-build/community-projects/scopt"]
+	path = community-build/community-projects/scopt
+	url = https://github.com/dotty-staging/scopt
+[submodule "community-build/community-projects/squants"]
+	path = community-build/community-projects/squants
+	url = https://github.com/dotty-staging/squants
+[submodule "community-build/community-projects/scalap"]
+	path = community-build/community-projects/scalap
+	url = https://github.com/dotty-staging/scala
+[submodule "community-build/community-projects/minitest"]
+	path = community-build/community-projects/minitest
+	url = https://github.com/dotty-staging/minitest
+[submodule "community-build/community-projects/fastparse"]
+	path = community-build/community-projects/fastparse
+	url = https://github.com/dotty-staging/fastparse
+[submodule "community-build/community-projects/stdLib213"]
+	path = community-build/community-projects/stdLib213
+	url = https://github.com/dotty-staging/scala
+[submodule "community-build/community-projects/pdbp"]
+	path = community-build/community-projects/pdbp
+	url = https://github.com/dotty-staging/pdbp
+[submodule "community-build/community-projects/sourcecode"]
+	path = community-build/community-projects/sourcecode
+	url = https://github.com/dotty-staging/sourcecode

.vscode-template/settings.json

@@ -9,6 +9,7 @@
     "**/*.class": true,
     "**/*.hasTasty": true,
     "**/target/": true,
+    "community-build/community-projects": true,
     "scala2-library/{doc,docs,lib,META-INF,scripts,spec,test,tools}/": true, // only allow scala-backend/src
     "scala2-library/src/[abcefimprs]*": true,  // only allow scala-backend/src/library
     "scala-backend/{doc,docs,lib,META-INF,scripts,spec,test,tools}/": true, // only allow scala-backend/src

bench/scripts/collection-strawman-cold.sh

@@ -19,6 +19,7 @@ val `scala-reflect` = Build.`scala-reflect`
 val scalap = Build.scalap
 val dist = Build.dist
 val `dist-bootstrapped` = Build.`dist-bootstrapped`
+val `community-build` = Build.`community-build`
 
 val sjsSandbox = Build.sjsSandbox
 

bench/scripts/collection-strawman.sh

@@ -0,0 +1 @@
+0.13.0-bin-SNAPSHOT

build.sbt

@@ -0,0 +1 @@
+sbt.version=1.2.7

collection-strawman

@@ -0,0 +1 @@
+addSbtPlugin("ch.epfl.lamp" % "sbt-dotty" % "0.2.7-SNAPSHOT")

community-build/community-projects/ScalaPB

@@ -0,0 +1,149 @@
+package dotty.communitybuild
+
+import java.nio.file._
+import java.io.{PrintWriter, File}
+import java.nio.charset.StandardCharsets.UTF_8
+import org.junit.{Ignore, Test}
+import org.junit.Assert.{assertEquals, fail}
+
+class CommunityBuildTest {
+  lazy val communitybuildDir: Path = Paths.get(sys.props("user.dir") + "/community-build/")
+
+  lazy val compilerVersion: String = {
+    val file = communitybuildDir.resolve("dotty-bootstrapped.version")
+    new String(Files.readAllBytes(file), UTF_8)
+  }
+
+  /** Build the given project with the published local compiler and sbt plugin.
+   *
+   *  This test reads the compiler version from community-build/dotty-bootstrapped.version
+   *  and expects community-build/sbt-dotty.sbt to set the compiler plugin.
+   *
+   *  @param project  The project name, should be a git submodule in community-build/
+   *  @param command  The sbt command used to build the project
+   */
+  def test(project: String, command: String): Unit = {
+    def log(msg: String) = println(Console.GREEN + msg + Console.RESET)
+
+    log(s"Building $project with dotty-bootstrapped $compilerVersion...")
+
+    val projectDir = communitybuildDir.resolve("community-projects").resolve(project)
+
+    if (!Files.exists(projectDir.resolve(".git"))) {
+      fail(s"""
+        |
+        |Missing $project submodule. You can initialize this module using
+        |
+        |    git submodule update --init community-build/community-projects/$project
+        |
+        |""".stripMargin)
+    }
+
+    /** Executes shell command, returns false in case of error. */
+    def exec(binary: String, arguments: String*): Int = {
+      val command = binary +: arguments
+      log(command.mkString(" "))
+      val builder = new ProcessBuilder(command: _*).directory(projectDir.toFile).inheritIO()
+      val process = builder.start()
+      val exitCode = process.waitFor()
+      exitCode
+    }
+
+    // Workaround for https://github.com/sbt/sbt/issues/4395
+    new File(sys.props("user.home") + "/.sbt/1.0/plugins").mkdirs()
+    val pluginFilePath = communitybuildDir.resolve("sbt-dotty.sbt").toAbsolutePath().toString()
+
+    // Run the sbt command with the compiler version and sbt plugin set in the build
+    val arguments = Seq(
+      "-sbt-version", "1.2.7",
+      s"--addPluginSbtFile=$pluginFilePath",
+      s";clean ;set updateOptions in Global ~= (_.withLatestSnapshots(false)) ;++$compilerVersion! $command"
+    )
+
+    val exitCode = exec("sbt", arguments: _*)
+
+    if (exitCode != 0) {
+      fail(s"""
+          |
+          |sbt exited with an error code. To reproduce without JUnit, use:
+          |
+          |    sbt community-build/prepareCommunityBuild
+          |    cd community-build/community-projects/$project
+          |    sbt ${arguments.init.mkString(" ")} "${arguments.last}"
+          |
+          |For a faster feedback loop, one can try to extract a direct call to dotc from
+          |usign the sbt export command. For instance, for scalacheck, use
+          |    sbt export jvm/test:compileIncremental
+          |
+          |""".stripMargin)
+    }
+  }
+
+  @Test def algebra = test(
+    project = "algebra",
+    command = "coreJVM/compile"
+  )
+
+  @Test def scalacheck = test(
+    project = "scalacheck",
+    command = "jvm/test:compile"
+  )
+
+  @Test def scalatest = test(
+    project = "scalatest",
+    command = "scalatest/compile"
+  )
+
+  @Test def scopt = test(
+    project = "scopt",
+    command = "scoptJVM/compile"
+  )
+
+  @Test def scalap = test(
+    project = "scalap",
+    command = "scalap/compile"
+  )
+
+  @Test def squants = test(
+    project = "squants",
+    command = "squantsJVM/compile"
+  )
+
+  @Test def betterfiles = test(
+    project = "betterfiles",
+    command = "dottyCompile"
+  )
+
+  @Test def ScalaPB = test(
+    project = "ScalaPB",
+    command = "dottyCompile"
+  )
+
+  @Test def minitest = test(
+    project = "minitest",
+    command = "dottyCompile"
+  )
+
+  @Test def fastparse = test(
+    project = "fastparse",
+    command = "fastparseJVM/compile"
+  )
+
+  // TODO: revert to sourcecodeJVM/test
+  @Test def sourcecode = test(
+    project = "sourcecode",
+    command = "sourcecodeJVM/compile"
+  )
+
+  // TODO @smarter?
+  // @Test def stdLib213 = test(
+  //   project = "stdLib213",
+  //   command = "library/compile"
+  // )
+
+  // TODO @oderky? It got broken by #5458
+  // @Test def pdbp = test(
+  //   project = "pdbp",
+  //   command = "compile"
+  // )
+}

community-build/community-projects/algebra

@@ -0,0 +1,16 @@
+# Dotty Community Build
+
+This project contains tests to build and test a corpus of open sources Scala projects against the latest version of Dotty.
+
+To run the community build on a local machine, first fetch all the git submodules with `git submodule update --init` and run `sbt community-build/test` from the root of the dotty repo.
+
+## Adding your project
+
+To add your project to the community build you can follow these steps:
+
+1. Get your project to compile with Dotty. Instructions can be found on the [dotty-example-project](https://github.com/lampepfl/dotty-example-project).
+   See the submodules in [community-projects](https://github.com/lampepfl/dotty/tree/master/community-build/community-projects/) for examples of projects that compile with Dotty.
+
+2. Open a PR against this repo that:
+     - Adds your project as a new git submodule
+     - Adds a test in [CommunityBuildTest.scala](https://github.com/lampepfl/dotty/blob/master/src/test/scala/dotty/community-build/src/test/scala/dotty/communitybuild/CommunityBuildTest.scala)

community-build/community-projects/betterfiles

@@ -63,7 +63,6 @@ class CompilationTests extends ParallelTesting {
       ),
       scala2Mode
     ) +
-    compileDir("collection-strawman/collections/src/main", defaultOptions.and("-Yno-imports")) +
     compileFilesInDir("tests/pos-special/spec-t5545", defaultOptions) +
     compileFilesInDir("tests/pos-special/strawman-collections", defaultOptions) +
     compileFilesInDir("tests/pos-special/isInstanceOf", allowDeepSubtypes.and("-Xfatal-warnings")) +

community-build/community-projects/fastparse

@@ -32,9 +32,6 @@ class IdempotencyTests extends ParallelTesting {
 
     def sourcesFrom(dir: Path) = CompilationTests.sources(Files.walk(dir))
 
-    val strawmanSources = sourcesFrom(Paths.get("collection-strawman/collections/src/main"))
-    val strawmanSourcesSorted = strawmanSources.sorted
-    val strawmanSourcesRevSorted = strawmanSourcesSorted.reverse
 
     val posIdempotency = {
       compileFilesInDir("tests/pos", opt)(TestGroup("idempotency/posIdempotency1")) +
@@ -51,13 +48,6 @@ class IdempotencyTests extends ParallelTesting {
       }).reduce(_ + _)
     }
 
-    val strawmanIdempotency = {
-      compileList("strawman0", strawmanSources, opt) +
-      compileList("strawman1", strawmanSources, opt) +
-      compileList("strawman2", strawmanSourcesSorted, opt) +
-      compileList("strawman3", strawmanSourcesRevSorted, opt)
-    }
-
     def check(name: String) = {
       val files = List(s"tests/idempotency/$name.scala", "tests/idempotency/IdempotencyCheck.scala")
       compileList(name, files, defaultOptions)(TestGroup("idempotency/check"))
@@ -70,8 +60,6 @@ class IdempotencyTests extends ParallelTesting {
     }
 
     val allTests = {
-      // Disabled until strawman is fixed
-      // strawmanIdempotency +
       orderIdempotency +
       posIdempotency
     }