Add decompiler tests

Author: nicolasstucki

compiler/src/dotty/tools/dotc/decompiler/DecompilationPrinter.scala

@@ -1,9 +1,12 @@
 package dotty.tools.dotc
 package decompiler
 
+import java.io.{OutputStream, PrintStream}
+
 import dotty.tools.dotc.core.Contexts._
 import dotty.tools.dotc.core.Phases.Phase
 import dotty.tools.dotc.core.tasty.TastyPrinter
+import dotty.tools.io.{File, Path}
 
 /** Phase that prints the trees in all loaded compilation units.
  *
@@ -14,23 +17,39 @@ class DecompilationPrinter extends Phase {
   override def phaseName: String = "decompilationPrinter"
 
   override def run(implicit ctx: Context): Unit = {
-    val unit = ctx.compilationUnit
+    val outputDir = ctx.settings.outputDir.value
+    if (outputDir == ".") printToOutput(System.out)
+    else {
+      var os: OutputStream = null
+      var ps: PrintStream = null
+      try {
+        os = File(outputDir + ".decompiled").outputStream()
+        ps = new PrintStream(os)
+        printToOutput(ps)
+      } finally {
+        if (os ne null) os.close()
+        if (ps ne null) ps.close()
+      }
+    }
+  }
 
+  private def printToOutput(out: PrintStream)(implicit ctx: Context): Unit = {
+    val unit = ctx.compilationUnit
     val pageWidth = ctx.settings.pageWidth.value
 
     val doubleLine = "=" * pageWidth
     val line = "-" * pageWidth
 
-    println(doubleLine)
-    println(unit.source)
-    println(line)
+    out.println(doubleLine)
+    out.println(unit.source)
+    out.println(line)
 
-    println(unit.tpdTree.show)
-    println(line)
+    out.println(unit.tpdTree.show)
+    out.println(line)
 
     if (ctx.settings.printTasty.value) {
       new TastyPrinter(unit.pickled.head._2).printContents()
-      println(line)
+      out.println(line)
     }
   }
 }

compiler/test/dotty/tools/dotc/FromTastyTests.scala

@@ -22,7 +22,7 @@ class FromTastyTests extends ParallelTesting {
 
   @Test def posTestFromTasty: Unit = {
     implicit val testGroup: TestGroup = TestGroup("posTestFromTasty")
-    val (step1, step2) = compileTastyInDir("../tests/pos", defaultOptions,
+    val (step1, step2, step3) = compileTastyInDir("../tests/pos", defaultOptions,
       blacklist = Set(
         "Meter.scala",
         "NoCyclicReference.scala",
@@ -56,12 +56,13 @@ class FromTastyTests extends ParallelTesting {
     )
     step1.checkCompile() // Compile all files to generate the class files with tasty
     step2.checkCompile() // Compile from tasty
-    (step1 + step2).delete()
+    step3.checkCompile() // Decompile from tasty
+    (step1 + step2 + step3).delete()
   }
 
   @Test def runTestFromTasty: Unit = {
     implicit val testGroup: TestGroup = TestGroup("runTestFromTasty")
-    val (step1, step2) = compileTastyInDir("../tests/run", defaultOptions,
+    val (step1, step2, step3) = compileTastyInDir("../tests/run", defaultOptions,
        blacklist = Set(
          "Course-2002-13.scala",
          "NestedClasses.scala",
@@ -93,7 +94,8 @@ class FromTastyTests extends ParallelTesting {
     )
     step1.checkCompile() // Compile all files to generate the class files with tasty
     step2.checkRuns() // Compile from tasty and run the result
-    (step1 + step2).delete()
+    step3.checkCompile() // Decompile from tasty
+    (step1 + step2 + step3).delete()
   }
 
   private implicit class tastyCompilationTuples(tup: (CompilationTest, CompilationTest)) {

compiler/test/dotty/tools/vulpix/ParallelTesting.scala

@@ -22,6 +22,7 @@ import dotc.reporting.diagnostic.MessageContainer
 import dotc.interfaces.Diagnostic.ERROR
 import dotc.util.DiffUtil
 import dotc.{ Driver, Compiler }
+import dotc.decompiler
 
 /** A parallel testing suite whose goal is to integrate nicely with JUnit
  *
@@ -47,7 +48,7 @@ trait ParallelTesting extends RunnerOrchestration { self =>
   /** A test source whose files or directory of files is to be compiled
    *  in a specific way defined by the `Test`
    */
-  private sealed trait TestSource { self =>
+  protected sealed trait TestSource { self =>
     def name: String
     def outDir: JFile
     def flags: TestFlags
@@ -133,7 +134,8 @@ trait ParallelTesting extends RunnerOrchestration { self =>
     files: Array[JFile],
     flags: TestFlags,
     outDir: JFile,
-    fromTasty: Boolean = false
+    fromTasty: Boolean = false,
+    decompilation: Boolean = false
   ) extends TestSource {
     def sourceFiles: Array[JFile] = files.filter(isSourceFile)
 
@@ -215,7 +217,7 @@ trait ParallelTesting extends RunnerOrchestration { self =>
     private val filteredSources =
       if (!testFilter.isDefined) testSources
       else testSources.filter {
-        case JointCompilationSource(_, files, _, _, _) =>
+        case JointCompilationSource(_, files, _, _, _, _) =>
           files.exists(file => file.getAbsolutePath.contains(testFilter.get))
         case SeparateCompilationSource(_, dir, _, _) =>
           dir.getAbsolutePath.contains(testFilter.get)
@@ -422,6 +424,32 @@ trait ParallelTesting extends RunnerOrchestration { self =>
       reporter
     }
 
+    protected def decompile(flags0: TestFlags, suppressErrors: Boolean, targetDir: JFile): TestReporter = {
+      val decompilationOutput = new JFile(targetDir.getPath)
+      decompilationOutput.mkdir()
+      val flags = flags0 and ("-d", decompilationOutput.getAbsolutePath) and "-decompile"
+
+      def hasTastyFileToClassName(f: JFile): String =
+        targetDir.toPath.relativize(f.toPath).toString.dropRight(".hasTasty".length).replace('/', '.')
+      val classes = flattenFiles(targetDir).filter(isHasTastyFile).map(hasTastyFileToClassName)
+
+      val reporter =
+        TestReporter.reporter(realStdout, logLevel =
+          if (suppressErrors || suppressAllOutput) ERROR + 1 else ERROR)
+
+      val driver = decompiler.Main
+
+      // Compile with a try to catch any StackTrace generated by the compiler:
+      try {
+        driver.process(flags.all ++ classes, reporter = reporter)
+      }
+      catch {
+        case NonFatal(ex) => reporter.logStackTrace(ex)
+      }
+
+      reporter
+    }
+
     private[ParallelTesting] def executeTestSuite(): this.type = {
       assert(_testSourcesCompleted == 0, "not allowed to re-use a `CompileRun`")
 
@@ -474,9 +502,52 @@ trait ParallelTesting extends RunnerOrchestration { self =>
     protected def encapsulatedCompilation(testSource: TestSource) = new LoggedRunnable {
       def checkTestSource(): Unit = tryCompile(testSource) {
         testSource match {
-          case testSource @ JointCompilationSource(_, files, flags, outDir, fromTasty) =>
+          case testSource @ JointCompilationSource(name, files, flags, outDir, fromTasty, decompilation) =>
             val reporter =
-              if (fromTasty) compileFromTasty(flags, false, outDir)
+              if (decompilation) {
+                val rep = decompile(flags, false, outDir)
+
+                val checkFileOpt = files.flatMap { file =>
+                  if (file.isDirectory) Nil
+                  else {
+                    val fname = file.getAbsolutePath.reverse.dropWhile(_ != '.').reverse + "decompiled"
+                    val checkFile = new JFile(fname)
+                    if (checkFile.exists) List(checkFile)
+                    else Nil
+                  }
+                }.headOption
+                checkFileOpt match {
+                  case Some(checkFile) =>
+                    val stripTrailingWhitespaces = "(.*\\S|)\\s+".r
+                    val output = Source.fromFile(outDir + ".decompiled").getLines().map {line =>
+                      stripTrailingWhitespaces.unapplySeq(line).map(_.head).getOrElse(line)
+                    }.mkString("\n")
+                      .replaceFirst("@scala\\.annotation\\.internal\\.SourceFile\\([^\\)]+\\)( |\\n   )", "") // FIXME: should not be printed in the decompiler
+
+                    checkDiff(output, checkFile, testSource, 0) match {
+                      case Some(diff) =>
+                        println("Expected:")
+                        println(checkFile)
+                        println("Actual output;")
+                        println(output)
+                        println("Diff;")
+                        echo(diff)
+                        addFailureInstruction(diff)
+
+                        // Print build instructions to file and summary:
+                        val buildInstr = testSource.buildInstructions(0, rep.warningCount)
+                        addFailureInstruction(buildInstr)
+
+                        // Fail target:
+                        failTestSource(testSource)
+                      case None =>
+                    }
+                  case _ =>
+                }
+
+                rep
+              }
+              else if (fromTasty) compileFromTasty(flags, false, outDir)
               else compile(testSource.sourceFiles, flags, false, outDir)
             registerCompletion(reporter.errorCount)
 
@@ -526,39 +597,20 @@ trait ParallelTesting extends RunnerOrchestration { self =>
       if (Properties.testsNoRun) addNoRunWarning()
       else runMain(testSource.runClassPath) match {
         case Success(_) if !checkFile.isDefined || !checkFile.get.exists => // success!
-        case Success(output) => {
-          val outputLines = output.lines.toArray :+ DiffUtil.EOF
-          val checkLines: Array[String] = Source.fromFile(checkFile.get).getLines().toArray :+ DiffUtil.EOF
-          val sourceTitle = testSource.title
-
-          def linesMatch =
-            outputLines
-            .zip(checkLines)
-            .forall { case (x, y) => x == y }
-
-          if (outputLines.length != checkLines.length || !linesMatch) {
-            // Print diff to files and summary:
-            val expectedSize = DiffUtil.EOF.length max checkLines.map(_.length).max
-            val diff = outputLines.padTo(checkLines.length, "").zip(checkLines.padTo(outputLines.length, "")).map { case (act, exp) =>
-              DiffUtil.mkColoredLineDiff(exp, act, expectedSize)
-            }.mkString("\n")
-
-            val msg =
-              s"""|Output from '$sourceTitle' did not match check file.
-                  |Diff (expected on the left, actual right):
-                  |""".stripMargin + diff + "\n"
-            echo(msg)
-            addFailureInstruction(msg)
-
-            // Print build instructions to file and summary:
-            val buildInstr = testSource.buildInstructions(0, warnings)
-            addFailureInstruction(buildInstr)
-
-            // Fail target:
-            failTestSource(testSource)
+        case Success(output) =>
+          checkDiff(output, checkFile.get, testSource, warnings) match {
+            case Some(msg) =>
+              echo(msg)
+              addFailureInstruction(msg)
+
+              // Print build instructions to file and summary:
+              val buildInstr = testSource.buildInstructions(0, warnings)
+              addFailureInstruction(buildInstr)
+
+              // Fail target:
+              failTestSource(testSource)
+            case None =>
           }
-        }
-
         case Failure(output) =>
           echo(s"Test '${testSource.title}' failed with output:")
           echo(output)
@@ -573,7 +625,7 @@ trait ParallelTesting extends RunnerOrchestration { self =>
     protected def encapsulatedCompilation(testSource: TestSource) = new LoggedRunnable {
       def checkTestSource(): Unit = tryCompile(testSource) {
         val (compilerCrashed, errorCount, warningCount, verifier: Function0[Unit]) = testSource match {
-          case testSource @ JointCompilationSource(_, files, flags, outDir, fromTasty) =>
+          case testSource @ JointCompilationSource(_, files, flags, outDir, fromTasty, decompilation) =>
             val checkFile = files.flatMap { file =>
               if (file.isDirectory) Nil
               else {
@@ -682,7 +734,7 @@ trait ParallelTesting extends RunnerOrchestration { self =>
         }
 
         val (compilerCrashed, expectedErrors, actualErrors, hasMissingAnnotations, errorMap) = testSource match {
-          case testSource @ JointCompilationSource(_, files, flags, outDir, fromTasty) =>
+          case testSource @ JointCompilationSource(_, files, flags, outDir, fromTasty, decompilation) =>
             val sourceFiles = testSource.sourceFiles
             val (errorMap, expectedErrors) = getErrorMapAndExpectedCount(sourceFiles)
             val reporter = compile(sourceFiles, flags, true, outDir)
@@ -736,6 +788,31 @@ trait ParallelTesting extends RunnerOrchestration { self =>
     }
   }
 
+  private def checkDiff(output: String, checkFile: JFile, testSource: TestSource, warnings: Int): Option[String] = {
+    val outputLines = output.lines.toArray :+ DiffUtil.EOF
+    val checkLines: Array[String] = Source.fromFile(checkFile).getLines().toArray :+ DiffUtil.EOF
+    val sourceTitle = testSource.title
+
+    def linesMatch =
+      outputLines
+        .zip(checkLines)
+        .forall { case (x, y) => x == y }
+
+    if (outputLines.length != checkLines.length || !linesMatch) {
+      // Print diff to files and summary:
+      val expectedSize = DiffUtil.EOF.length max checkLines.map(_.length).max
+      val diff = outputLines.padTo(checkLines.length, "").zip(checkLines.padTo(outputLines.length, "")).map { case (act, exp) =>
+        DiffUtil.mkColoredLineDiff(exp, act, expectedSize)
+      }.mkString("\n")
+
+      val msg =
+        s"""|Output from '$sourceTitle' did not match check file.
+            |Diff (expected on the left, actual right):
+            |""".stripMargin + diff + "\n"
+      Some(msg)
+    } else None
+  }
+
   /** The `CompilationTest` is the main interface to `ParallelTesting`, it
    *  can be instantiated via one of the following methods:
    *
@@ -967,7 +1044,7 @@ trait ParallelTesting extends RunnerOrchestration { self =>
      */
     def copyToTarget(): CompilationTest = new CompilationTest (
       targets.map {
-        case target @ JointCompilationSource(_, files, _, outDir, _) =>
+        case target @ JointCompilationSource(_, files, _, outDir, _, _) =>
           target.copy(files = files.map(copyToDir(outDir,_)))
         case target @ SeparateCompilationSource(_, dir, _, outDir) =>
           target.copy(dir = copyToDir(outDir, dir))
@@ -1083,34 +1160,6 @@ trait ParallelTesting extends RunnerOrchestration { self =>
     new CompilationTest(target)
   }
 
-  /** Compiles a single file from the string path `f` using the supplied flags
-   *
-   *  Tests in the first part of the tuple must be executed before the second.
-   *  Both testsRequires explicit delete().
-   */
-  def compileTasty(f: String, flags: TestFlags)(implicit testGroup: TestGroup): (CompilationTest, CompilationTest) = {
-    val sourceFile = new JFile(f)
-    val parent = sourceFile.getParentFile
-    val outDir =
-      defaultOutputDir + testGroup + "/" +
-        sourceFile.getName.substring(0, sourceFile.getName.lastIndexOf('.')) + "/"
-
-    require(
-      sourceFile.exists && !sourceFile.isDirectory &&
-        (parent ne null) && parent.exists && parent.isDirectory,
-      s"Source file: $f, didn't exist"
-    )
-    val tastySource = createOutputDirsForFile(sourceFile, parent, outDir)
-    val target = JointCompilationSource(
-      testGroup.name,
-      Array(sourceFile),
-      flags.withClasspath(tastySource.getPath) and "-from-tasty",
-      tastySource,
-      fromTasty = true
-    )
-    (compileFile(f, flags).keepOutput, new CompilationTest(target).keepOutput)
-  }
-
   /** Compiles a directory `f` using the supplied `flags`. This method does
    *  deep compilation, that is - it compiles all files and subdirectories
    *  contained within the directory `f`.
@@ -1216,7 +1265,7 @@ trait ParallelTesting extends RunnerOrchestration { self =>
    *  Both testsRequires explicit delete().
    */
   def compileTastyInDir(f: String, flags0: TestFlags, blacklist: Set[String] = Set.empty)(
-      implicit testGroup: TestGroup): (CompilationTest, CompilationTest) = {
+      implicit testGroup: TestGroup): (CompilationTest, CompilationTest, CompilationTest) = {
     val outDir = defaultOutputDir + testGroup + "/"
     val flags = flags0 and "-Yretain-trees"
     val sourceDir = new JFile(f)
@@ -1231,9 +1280,22 @@ trait ParallelTesting extends RunnerOrchestration { self =>
       }
     // TODO add SeparateCompilationSource from tasty?
 
+    val targets2 =
+      files
+        .filter(f => dotty.tools.io.File(f.toPath).changeExtension("decompiled").exists)
+        .map { f =>
+        val classpath = createOutputDirsForFile(f, sourceDir, outDir)
+        JointCompilationSource(testGroup.name, Array(f), flags.withClasspath(classpath.getPath), classpath, decompilation = true)
+      }
+
     // Create a CompilationTest and let the user decide whether to execute a pos or a neg test
     val generateClassFiles = compileFilesInDir(f, flags0, blacklist)
-    (generateClassFiles.keepOutput, new CompilationTest(targets).keepOutput)
+
+    (
+      generateClassFiles.keepOutput,
+      new CompilationTest(targets).keepOutput,
+      new CompilationTest(targets2).keepOutput
+    )
   }