Merge pull request scala#5246 from jodersky/javadoc

SI-4826 Retain javadoc comments in scaladoc [ci: last-only]

Author: adriaanm

src/compiler/scala/tools/nsc/Global.scala

@@ -411,7 +411,7 @@ class Global(var currentSettings: Settings, var reporter: Reporter)
     override val initial = true
   }
 
-  import syntaxAnalyzer.{ UnitScanner, UnitParser }
+  import syntaxAnalyzer.{ UnitScanner, UnitParser, JavaUnitParser }
 
   // !!! I think we're overdue for all these phase objects being lazy vals.
   // There's no way for a Global subclass to provide a custom typer
@@ -1042,6 +1042,8 @@ class Global(var currentSettings: Settings, var reporter: Reporter)
   def newUnitParser(code: String, filename: String = "<console>"): UnitParser =
     newUnitParser(newCompilationUnit(code, filename))
 
+  def newJavaUnitParser(unit: CompilationUnit): JavaUnitParser = new JavaUnitParser(unit)
+
   /** A Run is a single execution of the compiler on a set of units.
    */
   class Run extends RunContextApi with RunReporting with RunParsing {

src/compiler/scala/tools/nsc/ast/parser/SyntaxAnalyzer.scala

@@ -82,7 +82,7 @@ abstract class SyntaxAnalyzer extends SubComponent with Parsers with MarkupParse
   }
 
   private def initialUnitBody(unit: CompilationUnit): Tree = {
-    if (unit.isJava) new JavaUnitParser(unit).parse()
+    if (unit.isJava) newJavaUnitParser(unit).parse()
     else if (currentRun.parsing.incompleteHandled) newUnitParser(unit).parse()
     else newUnitParser(unit).smartParse()
   }

src/compiler/scala/tools/nsc/javac/JavaParsers.scala

@@ -111,7 +111,7 @@ trait JavaParsers extends ast.parser.ParsersCommon with JavaScanners {
     def arrayOf(tpt: Tree) =
       AppliedTypeTree(scalaDot(tpnme.Array), List(tpt))
 
-    def blankExpr = Ident(nme.WILDCARD)
+    def blankExpr = EmptyTree
 
     def makePackaging(pkg: RefTree, stats: List[Tree]): PackageDef =
       atPos(pkg.pos) {  PackageDef(pkg, stats) }
@@ -135,6 +135,11 @@ trait JavaParsers extends ast.parser.ParsersCommon with JavaScanners {
       DefDef(Modifiers(Flags.JAVA), nme.CONSTRUCTOR, List(), List(vparams), TypeTree(), blankExpr)
     }
 
+    /** A hook for joining the comment associated with a definition.
+      * Overridden by scaladoc.
+      */
+    def joinComment(trees: => List[Tree]): List[Tree] = trees
+
     // ------------- general parsing ---------------------------
 
     /** skip parent or brace enclosed sequence of things */
@@ -581,7 +586,7 @@ trait JavaParsers extends ast.parser.ParsersCommon with JavaScanners {
       case CLASS | ENUM | INTERFACE | AT =>
         typeDecl(if (definesInterface(parentToken)) mods | Flags.STATIC else mods)
       case _ =>
-        termDecl(mods, parentToken)
+        joinComment(termDecl(mods, parentToken))
     }
 
     def makeCompanionObject(cdef: ClassDef, statics: List[Tree]): Tree =
@@ -833,10 +838,10 @@ trait JavaParsers extends ast.parser.ParsersCommon with JavaScanners {
     }
 
     def typeDecl(mods: Modifiers): List[Tree] = in.token match {
-      case ENUM      => enumDecl(mods)
-      case INTERFACE => interfaceDecl(mods)
+      case ENUM      => joinComment(enumDecl(mods))
+      case INTERFACE => joinComment(interfaceDecl(mods))
       case AT        => annotationDecl(mods)
-      case CLASS     => classDecl(mods)
+      case CLASS     => joinComment(classDecl(mods))
       case _         => in.nextToken(); syntaxError("illegal start of type declaration", skipIt = true); List(errorTypeTree)
     }
 

src/compiler/scala/tools/nsc/javac/JavaScanners.scala

@@ -577,21 +577,29 @@ trait JavaScanners extends ast.parser.ScannersCommon {
       }
     }
 
-    protected def skipComment(): Boolean = {
-      @tailrec def skipLineComment(): Unit = in.ch match {
-        case CR | LF | SU =>
-        case _            => in.next; skipLineComment()
-      }
-      @tailrec def skipJavaComment(): Unit = in.ch match {
-        case SU  => incompleteInputError("unclosed comment")
-        case '*' => in.next; if (in.ch == '/') in.next else skipJavaComment()
-        case _   => in.next; skipJavaComment()
-      }
-      in.ch match {
-        case '/' => in.next ; skipLineComment() ; true
-        case '*' => in.next ; skipJavaComment() ; true
-        case _   => false
-      }
+    protected def putCommentChar(): Unit = in.next()
+
+    protected def skipBlockComment(isDoc: Boolean): Unit = in.ch match {
+      case SU  => incompleteInputError("unclosed comment")
+      case '*' => putCommentChar() ; if (in.ch == '/') putCommentChar() else skipBlockComment(isDoc)
+      case _   => putCommentChar() ; skipBlockComment(isDoc)
+    }
+
+    protected def skipLineComment(): Unit = in.ch match {
+      case CR | LF | SU =>
+      case _            => putCommentChar() ; skipLineComment()
+    }
+
+    protected def skipComment(): Boolean = in.ch match {
+      case '/' => putCommentChar() ; skipLineComment() ; true
+      case '*' =>
+        putCommentChar()
+        in.ch match {
+          case '*' => skipBlockComment(isDoc = true)
+          case _ => skipBlockComment(isDoc = false)
+        }
+        true
+      case _   => false
     }
 
 // Identifiers ---------------------------------------------------------------

src/compiler/scala/tools/nsc/typechecker/Typers.scala

@@ -2247,9 +2247,10 @@ trait Typers extends Adaptations with Tags with TypersTracking with PatternTyper
           transformedOrTyped(ddef.rhs, EXPRmode, tpt1.tpe)
         }
 
-      if (meth.isClassConstructor && !isPastTyper && !meth.owner.isSubClass(AnyValClass)) {
-        // At this point in AnyVal there is no supercall, which will blow up
-        // in computeParamAliases; there's nothing to be computed for Anyval anyway.
+      if (meth.isClassConstructor && !isPastTyper && !meth.owner.isSubClass(AnyValClass) && !meth.isJava) {
+        // There are no supercalls for AnyVal or constructors from Java sources, which
+        // would blow up in computeParamAliases; there's nothing to be computed for them
+        // anyway.
         if (meth.isPrimaryConstructor)
           computeParamAliases(meth.owner, vparamss1, rhs1)
         else

src/scaladoc/scala/tools/nsc/doc/ScaladocAnalyzer.scala

@@ -101,52 +101,6 @@ trait ScaladocAnalyzer extends Analyzer {
 abstract class ScaladocSyntaxAnalyzer[G <: Global](val global: G) extends SyntaxAnalyzer {
   import global._
 
-  class ScaladocJavaUnitParser(unit: CompilationUnit) extends {
-    override val in = new ScaladocJavaUnitScanner(unit)
-  } with JavaUnitParser(unit) { }
-
-  class ScaladocJavaUnitScanner(unit: CompilationUnit) extends JavaUnitScanner(unit) {
-    /** buffer for the documentation comment
-     */
-    var docBuffer: StringBuilder = null
-
-    /** add the given character to the documentation buffer
-     */
-    protected def putDocChar(c: Char) {
-      if (docBuffer ne null) docBuffer.append(c)
-    }
-
-    override protected def skipComment(): Boolean = {
-      if (in.ch == '/') {
-        do {
-          in.next
-        } while ((in.ch != CR) && (in.ch != LF) && (in.ch != SU))
-        true
-      } else if (in.ch == '*') {
-        docBuffer = null
-        in.next
-        val scaladoc = ("/**", "*/")
-        if (in.ch == '*')
-          docBuffer = new StringBuilder(scaladoc._1)
-        do {
-          do {
-            if (in.ch != '*' && in.ch != SU) {
-              in.next; putDocChar(in.ch)
-            }
-          } while (in.ch != '*' && in.ch != SU)
-          while (in.ch == '*') {
-            in.next; putDocChar(in.ch)
-          }
-        } while (in.ch != '/' && in.ch != SU)
-        if (in.ch == '/') in.next
-        else incompleteInputError("unclosed comment")
-        true
-      } else {
-        false
-      }
-    }
-  }
-
   class ScaladocUnitScanner(unit0: CompilationUnit, patches0: List[BracePatch]) extends UnitScanner(unit0, patches0) {
 
     private var docBuffer: StringBuilder = null        // buffer for comments (non-null while scanning)
@@ -259,4 +213,77 @@ abstract class ScaladocSyntaxAnalyzer[G <: Global](val global: G) extends Syntax
       else trees
     }
   }
+
+  class ScaladocJavaUnitScanner(unit: CompilationUnit) extends JavaUnitScanner(unit) {
+
+    private val docBuffer: StringBuilder = new StringBuilder
+    private var inDocComment = false
+    private var docStart: Int = 0
+    private var lastDoc: DocComment = null
+
+    // get last doc comment
+    def flushDoc(): DocComment = try lastDoc finally lastDoc = null
+
+    override protected def putCommentChar(): Unit = {
+      if (inDocComment) docBuffer append in.ch
+      in.next
+    }
+
+    override protected def skipBlockComment(isDoc: Boolean): Unit = {
+      // condition is true when comment is entered the first time,
+      // i.e. immediately after "/*" and when current character is "*"
+      if (!inDocComment && isDoc) {
+        docBuffer append "/*"
+        docStart = currentPos.start
+        inDocComment = true
+      }
+      super.skipBlockComment(isDoc)
+    }
+
+    override protected def skipComment(): Boolean = {
+      val skipped = super.skipComment()
+      if (skipped && inDocComment) {
+        val raw = docBuffer.toString
+        val position = Position.range(unit.source, docStart, docStart, in.cpos)
+        lastDoc = DocComment(raw, position)
+        signalParsedDocComment(raw, position)
+        docBuffer.setLength(0) // clear buffer
+        inDocComment = false
+        true
+      } else {
+        skipped
+      }
+    }
+
+  }
+
+  class ScaladocJavaUnitParser(unit: CompilationUnit) extends {
+    override val in = new ScaladocJavaUnitScanner(unit)
+  } with JavaUnitParser(unit) {
+
+    override def joinComment(trees: => List[Tree]): List[Tree] = {
+      val doc = in.flushDoc()
+
+      if ((doc ne null) && doc.raw.length > 0) {
+        log(s"joinComment(doc=$doc)")
+        val joined = trees map { t =>
+          DocDef(doc, t) setPos {
+            if (t.pos.isDefined) {
+              val pos = doc.pos.withEnd(t.pos.end)
+              pos.makeTransparent
+            } else {
+              t.pos
+            }
+          }
+        }
+        joined.find(_.pos.isOpaqueRange) foreach { main =>
+          val mains = List(main)
+          joined foreach { t => if (t ne main) ensureNonOverlapping(t, mains) }
+        }
+        joined
+      } else {
+        trees
+      }
+    }
+  }
 }

src/scaladoc/scala/tools/nsc/doc/ScaladocGlobal.scala

@@ -13,6 +13,7 @@ trait ScaladocGlobalTrait extends Global {
 
   override val useOffsetPositions = false
   override def newUnitParser(unit: CompilationUnit) = new syntaxAnalyzer.ScaladocUnitParser(unit, Nil)
+  override def newJavaUnitParser(unit: CompilationUnit) = new syntaxAnalyzer.ScaladocJavaUnitParser(unit)
 
   override lazy val syntaxAnalyzer = new ScaladocSyntaxAnalyzer[outer.type](outer) {
     val runsAfter = List[String]()
@@ -40,6 +41,8 @@ class ScaladocGlobal(settings: doc.Settings, reporter: Reporter) extends Global(
     phasesSet += analyzer.typerFactory
   }
   override def forScaladoc = true
+  override def createJavadoc = true
+
   override lazy val analyzer = new {
     val global: ScaladocGlobal.this.type = ScaladocGlobal.this
   } with ScaladocAnalyzer

src/scaladoc/scala/tools/partest/ScaladocModelTest.scala

@@ -81,7 +81,7 @@ abstract class ScaladocModelTest extends DirectTest {
   private[this] var settings: doc.Settings = null
 
   // create a new scaladoc compiler
-  private[this] def newDocFactory: DocFactory = {
+  def newDocFactory: DocFactory = {
     settings = new doc.Settings(_ => ())
     settings.scaladocQuietRun = true // yaay, no more "model contains X documentable templates"!
     val args = extraSettings + " " + scaladocSettings

test/files/run/t5699.check

@@ -1,10 +1,10 @@
 [[syntax trees at end of                    parser]] // annodef.java
 package <empty> {
   object MyAnnotation extends  {
-    def <init>() = _
+    def <init>()
   };
   class MyAnnotation extends scala.annotation.Annotation with _root_.java.lang.annotation.Annotation with scala.annotation.ClassfileAnnotation {
-    def <init>() = _;
+    def <init>();
     def value(): String
   }
 }

test/scaladoc/resources/SI-4826.java

@@ -0,0 +1,20 @@
+package test.scaladoc;
+
+/**
+ * Testing java comments. The presence of a :marker:
+ * tag is verified by tests.
+ */
+public class JavaComments {
+
+    /**
+     * Compute the answer to the ultimate question of life, the
+     * universe, and everything. :marker:
+     * @param factor scaling factor to the answer
+     * @return the answer to everything (42) scaled by factor
+     */
+    public int answer(int factor) {
+	return 42 * factor;
+    }
+
+}
+

test/scaladoc/run/SI-4826.check

@@ -0,0 +1 @@
+Done.

test/scaladoc/run/SI-4826.scala

@@ -0,0 +1,30 @@
+import scala.tools.nsc.doc.Universe
+import scala.tools.nsc.doc.model._
+import scala.tools.partest.ScaladocModelTest
+
+object Test extends ScaladocModelTest {
+
+  override def resourceFile = "SI-4826.java"
+
+  // overridden to pass explicit files to newDocFactory.makeUniverse (rather than code strings)
+  // since the .java file extension is required
+  override def model: Option[Universe] = {
+    val path = resourcePath + "/" + resourceFile
+    newDocFactory.makeUniverse(Left(List(path)))
+  }
+
+  // no need for special settings
+  def scaladocSettings = ""
+
+  def testModel(rootPackage: Package) = {
+    import access._
+    val Tag = ":marker:"
+
+    val base = rootPackage._package("test")._package("scaladoc")
+    val clazz = base._class("JavaComments")
+    val method = clazz._method("answer")
+
+    assert(extractCommentText(clazz.comment.get).contains(Tag))
+    assert(extractCommentText(method.comment.get).contains(Tag))
+  }
+}