diff --git a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala index 0ccc1743ced7..0633edc1389d 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala @@ -439,6 +439,21 @@ object Parsers { finally staged = saved } + private var strictIndent = false + private def withStrictIndent[T](body: => T): T = { + val saved = strictIndent + strictIndent = true + try body + finally strictIndent = saved + } + + private def withoutStrictIndent[T](body: => T): T = { + val saved = strictIndent + strictIndent = false + try body + finally strictIndent = saved + } + /* ---------- TREE CONSTRUCTION ------------------------------------------- */ /** Convert tree to formal parameter list @@ -1281,16 +1296,21 @@ object Parsers { in.sourcePos()) patch(source, Span(in.offset), " ") - def possibleTemplateStart(isNew: Boolean = false): Unit = + def possibleTemplateStart[A](isNew: Boolean = false)(rest: => A): A = in.observeColonEOL() - if in.token == COLONEOL || in.token == WITHEOL then - if in.lookahead.isIdent(nme.end) then in.token = NEWLINE + val indented = + if in.token == COLONEOL || in.token == WITHEOL then + if in.lookahead.isIdent(nme.end) then in.token = NEWLINE + else + in.nextToken() + if in.token != INDENT && in.token != LBRACE then + syntaxErrorOrIncomplete(i"indented definitions expected, ${in}") + true else - in.nextToken() - if in.token != INDENT && in.token != LBRACE then - syntaxErrorOrIncomplete(i"indented definitions expected, ${in}") - else - newLineOptWhenFollowedBy(LBRACE) + newLineOptWhenFollowedBy(LBRACE) + in.next.token == LBRACE + if indented then withStrictIndent(rest) + else rest def checkEndMarker[T <: Tree](stats: ListBuffer[T]): Unit = @@ -2309,19 +2329,22 @@ object Parsers { def newExpr(): Tree = val start = in.skipToken() def reposition(t: Tree) = t.withSpan(Span(start, in.lastOffset)) - possibleTemplateStart() - val parents = - if in.isNestedStart then Nil - else constrApps(commaOK = false) - colonAtEOLOpt() - possibleTemplateStart(isNew = true) - parents match { - case parent :: Nil if !in.isNestedStart => - reposition(if (parent.isType) ensureApplied(wrapNew(parent)) else parent) - case _ => - New(reposition(templateBodyOpt(emptyConstructor, parents, Nil))) + possibleTemplateStart() { + val parents = + if in.isNestedStart then Nil + else constrApps(commaOK = false) + colonAtEOLOpt() + possibleTemplateStart(isNew = true) { + parents match { + case parent :: Nil if !in.isNestedStart => + reposition(if (parent.isType) ensureApplied(wrapNew(parent)) else parent) + case _ => + New(reposition(templateBodyOpt(emptyConstructor, parents, Nil))) + } + } } + /** ExprsInParens ::= ExprInParens {`,' ExprInParens} */ def exprsInParensOpt(): List[Tree] = @@ -3660,12 +3683,13 @@ object Parsers { tokenSeparated(COMMA, () => convertToTypeId(qualId())) } else Nil - possibleTemplateStart() - if (isEnum) { - val (self, stats) = withinEnum(templateBody()) - Template(constr, parents, derived, self, stats) + possibleTemplateStart() { + if (isEnum) { + val (self, stats) = withinEnum(templateBody()) + Template(constr, parents, derived, self, stats) + } + else templateBodyOpt(constr, parents, derived) } - else templateBodyOpt(constr, parents, derived) } /** TemplateOpt = [Template] @@ -3675,12 +3699,13 @@ object Parsers { if in.token == EXTENDS || isIdent(nme.derives) then template(constr) else - possibleTemplateStart() - if in.isNestedStart then - template(constr) - else - checkNextNotIndented() - Template(constr, Nil, Nil, EmptyValDef, Nil) + possibleTemplateStart() { + if in.isNestedStart then + template(constr) + else + checkNextNotIndented() + Template(constr, Nil, Nil, EmptyValDef, Nil) + } /** TemplateBody ::= [nl] `{' TemplateStatSeq `}' * EnumBody ::= [nl] ‘{’ [SelfType] EnumStat {semi EnumStat} ‘}’ @@ -3705,10 +3730,11 @@ object Parsers { /** with Template, with EOL interpreted */ def withTemplate(constr: DefDef, parents: List[Tree]): Template = if in.token != WITHEOL then accept(WITH) - possibleTemplateStart() // consumes a WITHEOL token - val (self, stats) = templateBody() - Template(constr, parents, Nil, self, stats) - .withSpan(Span(constr.span.orElse(parents.head.span).start, in.lastOffset)) + possibleTemplateStart() { // consumes a WITHEOL token + val (self, stats) = templateBody() + Template(constr, parents, Nil, self, stats) + .withSpan(Span(constr.span.orElse(parents.head.span).start, in.lastOffset)) + } /* -------- STATSEQS ------------------------------------------- */ @@ -3778,6 +3804,7 @@ object Parsers { def templateStatSeq(): (ValDef, List[Tree]) = checkNoEscapingPlaceholders { var self: ValDef = EmptyValDef val stats = new ListBuffer[Tree] + var firstStatIndent: Option[IndentWidth] = None if (isExprIntro && !isDefIntro(modifierTokens)) { val first = expr1() if (in.token == ARROW) { @@ -3793,6 +3820,9 @@ object Parsers { in.nextToken() } else { + val indent = in.indentWidth(in.offset) + if (firstStatIndent.isEmpty) firstStatIndent = Some(indent) + else () stats += first acceptStatSepUnlessAtEnd(stats) } @@ -3800,6 +3830,12 @@ object Parsers { var exitOnError = false while (!isStatSeqEnd && !exitOnError) { setLastStatOffset() + val indent = in.indentWidth(in.offset) + if (firstStatIndent.isEmpty) firstStatIndent = Some(indent) + else if (strictIndent && firstStatIndent != Some(indent)) + syntaxErrorOrIncomplete(s"unexpected indent: found $indent expected ${firstStatIndent.get}") + else () + if (in.token == IMPORT) stats ++= importClause(IMPORT, mkImport()) else if (in.token == EXPORT) @@ -3919,18 +3955,19 @@ object Parsers { else val pkg = qualId() var continue = false - possibleTemplateStart() - if in.token == EOF then - ts += makePackaging(start, pkg, List()) - else if in.isNestedStart then - ts += inDefScopeBraces(makePackaging(start, pkg, topStatSeq()), rewriteWithColon = true) - continue = true - else - acceptStatSep() - ts += makePackaging(start, pkg, topstats()) - if continue then - acceptStatSepUnlessAtEnd(ts) - ts ++= topStatSeq() + possibleTemplateStart() { + if in.token == EOF then + ts += makePackaging(start, pkg, List()) + else if in.isNestedStart then + ts += inDefScopeBraces(makePackaging(start, pkg, topStatSeq()), rewriteWithColon = true) + continue = true + else + acceptStatSep() + ts += makePackaging(start, pkg, topstats()) + if continue then + acceptStatSepUnlessAtEnd(ts) + ts ++= topStatSeq() + } } else ts ++= topStatSeq(outermost = true) diff --git a/compiler/test/dotty/tools/DottyTest.scala b/compiler/test/dotty/tools/DottyTest.scala index b3b5c2f2fca6..aa4b0a934db1 100644 --- a/compiler/test/dotty/tools/DottyTest.scala +++ b/compiler/test/dotty/tools/DottyTest.scala @@ -13,7 +13,8 @@ import dotc.printing.Texts._ import dotc.reporting.ConsoleReporter import dotc.core.Decorators._ import dotc.ast.tpd -import dotc.Compiler +import dotc.{CompilationUnit,Compiler} +import dotc.util.SourceFile import dotc.core.Phases.Phase @@ -38,6 +39,12 @@ trait DottyTest extends ContextEscapeDetection { override def clearCtx() = { ctx = null } + def resetCtx(sourceFile: SourceFile) = { + clearCtx() + val c = initialCtx + c.setCompilationUnit(CompilationUnit(sourceFile, mustExist = false)) + ctx = c + } protected def initializeCtx(fc: FreshContext): Unit = { fc.setSetting(fc.settings.encoding, "UTF8") diff --git a/compiler/test/dotty/tools/dotc/parsing/DottyScannerTest.scala b/compiler/test/dotty/tools/dotc/parsing/DottyScannerTest.scala new file mode 100644 index 000000000000..56282bb2b505 --- /dev/null +++ b/compiler/test/dotty/tools/dotc/parsing/DottyScannerTest.scala @@ -0,0 +1,45 @@ +package dotty.tools +package dotc +package parsing + +import dotty.tools.io._ +import scala.io.Codec +import util._ +import Tokens._, Scanners._ +import org.junit.Test + +class DottyScannerTest extends ScannerTest { + + val blackList = List( + "/scaladoc/scala/tools/nsc/doc/html/page/Index.scala", + "/scaladoc/scala/tools/nsc/doc/html/page/Template.scala" + ) + + def scanDir(path: String): Unit = scanDir(Directory(path)) + + def scanDir(dir: Directory): Unit = { + if (blackList exists (dir.jpath.toString endsWith _)) + println(s"blacklisted package: ${dir.toAbsolute.jpath}") + else + for (f <- dir.files) + if (f.name.endsWith(".scala")) + if (blackList exists (f.jpath.toString endsWith _)) + println(s"blacklisted file: ${f.toAbsolute.jpath}") + else + scan(new PlainFile(f)) + for (d <- dir.dirs) + scanDir(d.path) + } + + @Test + def scanList() = { + println(System.getProperty("user.dir")) + scan("compiler/src/dotty/tools/dotc/core/Symbols.scala") + scan("compiler/src/dotty/tools/dotc/core/Symbols.scala") + } + + @Test + def scanDotty() = { + scanDir("compiler/src") + } +} \ No newline at end of file diff --git a/compiler/test/dotty/tools/dotc/parsing/IndentTest.scala b/compiler/test/dotty/tools/dotc/parsing/IndentTest.scala new file mode 100644 index 000000000000..28075733df4c --- /dev/null +++ b/compiler/test/dotty/tools/dotc/parsing/IndentTest.scala @@ -0,0 +1,61 @@ +package dotty.tools +package dotc +package parsing + +import org.junit.Test + +class IndentTest extends ParserTest: + + @Test + def parseBraces: Unit = + val code = s""" + |class A { + | val x = 1 + | val y = 2 + |}""".stripMargin + assert(parseTextEither(code).isRight) + + @Test + def parseIndents: Unit = + val code = s""" + |class A: + | val x = 1 + | val y = 2 + |""".stripMargin + assert(parseTextEither(code).isRight) + + @Test + def innerClassIndents: Unit = + val code = s""" + |class A: + | class B: + | val x = 1 + |""".stripMargin + assert(parseTextEither(code).isRight) + + @Test + def extendsClassIndents: Unit = + val code = s""" + |class A extends B: + | override def hasUnreportedErrors: Boolean = + | infos.exists(_.isInstanceOf[Error]) + |""".stripMargin + assert(parseTextEither(code).isRight) + + @Test + def superfluousIndents: Unit = + val code = s""" + |class A: + | val x = 1 + | val y = 2 + |""".stripMargin + assert(parseTextEither(code).isLeft) + + @Test + def superfluousIndents2: Unit = + val code = s""" + |class Test: + | test("hello") + | assert(1 == 1) + |""".stripMargin + assert(parseTextEither(code).isLeft) diff --git a/compiler/test/dotty/tools/dotc/parsing/IndentWidthTest.scala b/compiler/test/dotty/tools/dotc/parsing/IndentWidthTest.scala new file mode 100644 index 000000000000..ad0efe652e6b --- /dev/null +++ b/compiler/test/dotty/tools/dotc/parsing/IndentWidthTest.scala @@ -0,0 +1,18 @@ +package dotty.tools +package dotc +package parsing + +import org.junit.Test + +class IndentWidthTest extends ScannerTest: + + @Test + def innerObjectIndents: Unit = + val code = s""" + |object A: + | object B + | end B + | + | object C + |""".stripMargin + assert(scanTextEither(code).isRight) diff --git a/compiler/test/dotty/tools/dotc/parsing/ParserTest.scala b/compiler/test/dotty/tools/dotc/parsing/ParserTest.scala index b247ba1a525d..37b6a9894417 100644 --- a/compiler/test/dotty/tools/dotc/parsing/ParserTest.scala +++ b/compiler/test/dotty/tools/dotc/parsing/ParserTest.scala @@ -8,10 +8,12 @@ import core._ import scala.io.Codec import Tokens._, Parsers._ import ast.untpd._ +import reporting.Diagnostic import org.junit.Test import scala.collection.mutable.ListBuffer class ParserTest extends DottyTest { + import ParserTest._ def parse(name: String): Tree = parse(new PlainFile(File(name))) @@ -23,15 +25,28 @@ class ParserTest extends DottyTest { parsedTrees.clear() } - def parse(file: PlainFile): Tree = parseSource(new SourceFile(file, Codec.UTF8)) + def reset(source: SourceFile) = { + parsed = 0 + parsedTrees.clear() + resetCtx(source) + } + + def parse(file: PlainFile): Tree = parseSourceEither(new SourceFile(file, Codec.UTF8)).toTry.get - private def parseSource(source: SourceFile): Tree = { + private def parseSourceEither(source: SourceFile): Either[ParserError, Tree] = { //println("***** parsing " + source.file) + reset(source) val parser = new Parser(source) val tree = parser.parse() - parsed += 1 - parsedTrees += tree - tree + if (getCtx.reporter.hasErrors) { + val result = Left(ParserError(getCtx.reporter.allErrors)) + result + } + else { + parsed += 1 + parsedTrees += tree + Right(tree) + } } def parseDir(path: String): Unit = parseDir(Directory(path)) @@ -43,5 +58,12 @@ class ParserTest extends DottyTest { parseDir(d.path) } - def parseText(code: String): Tree = parseSource(SourceFile.virtual("", code)) + def parseText(code: String): Tree = parseTextEither(code).toTry.get + + def parseTextEither(code: String): Either[ParserError, Tree] = + parseSourceEither(SourceFile.virtual("", code)) +} + +object ParserTest { + case class ParserError(errors: List[Diagnostic.Error]) extends RuntimeException } diff --git a/compiler/test/dotty/tools/dotc/parsing/ScannerTest.scala b/compiler/test/dotty/tools/dotc/parsing/ScannerTest.scala index 9d3edb73ca78..80bc6a627bca 100644 --- a/compiler/test/dotty/tools/dotc/parsing/ScannerTest.scala +++ b/compiler/test/dotty/tools/dotc/parsing/ScannerTest.scala @@ -6,55 +6,32 @@ import dotty.tools.io._ import scala.io.Codec import util._ import Tokens._, Scanners._ -import org.junit.Test +import reporting.Diagnostic -class ScannerTest extends DottyTest { - - val blackList = List( - "/scaladoc/scala/tools/nsc/doc/html/page/Index.scala", - "/scaladoc/scala/tools/nsc/doc/html/page/Template.scala" - ) +trait ScannerTest extends DottyTest: + import ParserTest._ def scan(name: String): Unit = scan(new PlainFile(File(name))) - def scan(file: PlainFile): Unit = { + def scan(file: PlainFile): Unit = scanSourceEither(new SourceFile(file, Codec.UTF8)).toTry.get + + def reset(source: SourceFile) = resetCtx(source) + + private def scanSourceEither(source: SourceFile): Either[ParserError, Unit] = + reset(source) //println("***** scanning " + file) - val source = new SourceFile(file, Codec.UTF8) val scanner = new Scanner(source) var i = 0 - while (scanner.token != EOF) { + while scanner.token != EOF do // print("[" + scanner.token.show +"]") scanner.nextToken() // i += 1 // if (i % 10 == 0) println() - } - } - - def scanDir(path: String): Unit = scanDir(Directory(path)) - - def scanDir(dir: Directory): Unit = { - if (blackList exists (dir.jpath.toString endsWith _)) - println(s"blacklisted package: ${dir.toAbsolute.jpath}") - else - for (f <- dir.files) - if (f.name.endsWith(".scala")) - if (blackList exists (f.jpath.toString endsWith _)) - println(s"blacklisted file: ${f.toAbsolute.jpath}") - else - scan(new PlainFile(f)) - for (d <- dir.dirs) - scanDir(d.path) - } - - @Test - def scanList() = { - println(System.getProperty("user.dir")) - scan("compiler/src/dotty/tools/dotc/core/Symbols.scala") - scan("compiler/src/dotty/tools/dotc/core/Symbols.scala") - } - - @Test - def scanDotty() = { - scanDir("src") - } -} + + if getCtx.reporter.hasErrors || getCtx.reporter.hasWarnings then + val result = Left(ParserError(getCtx.reporter.allErrors)) + result + else Right(()) + + def scanTextEither(code: String): Either[ParserError, Unit] = scanSourceEither(SourceFile.virtual("", code)) + def scanText(code: String): Unit = scanTextEither(code).toTry.get