From a575d185e27f4ded23a09dbd1961ebb45fc1a5b3 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Sat, 1 May 2021 11:49:51 +0200 Subject: [PATCH 1/8] Drop lastOffset --- .../dotty/tools/dotc/parsing/Parsers.scala | 25 +++---------------- .../tasty/TastyHeaderUnpicklerTest.scala | 4 +-- 2 files changed, 5 insertions(+), 24 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala index 27ac53f5af34..07c8f5b10b2e 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala @@ -233,15 +233,6 @@ object Parsers { /* ------------- ERROR HANDLING ------------------------------------------- */ - /** The offset of the last time when a statement on a new line was definitely - * encountered in the current scope or an outer scope. - */ - private var lastStatOffset = -1 - - def setLastStatOffset(): Unit = - if (mustStartStat && in.isAfterLineEnd) - lastStatOffset = in.offset - /** Is offset1 less or equally indented than offset2? * This is the case if the characters between the preceding end-of-line and offset1 * are a prefix of the characters between the preceding end-of-line and offset2. @@ -533,11 +524,8 @@ object Parsers { if (in.rewriteToIndent) bracesToIndented(body, rewriteWithColon) else inBraces(body) - def inDefScopeBraces[T](body: => T, rewriteWithColon: Boolean = false): T = { - val saved = lastStatOffset - try inBracesOrIndented(body, rewriteWithColon) - finally lastStatOffset = saved - } + def inDefScopeBraces[T](body: => T, rewriteWithColon: Boolean = false): T = + inBracesOrIndented(body, rewriteWithColon) /** part { `separator` part } */ @@ -1538,10 +1526,7 @@ object Parsers { else t /** The block in a quote or splice */ - def stagedBlock() = - val saved = lastStatOffset - try inBraces(block(simplify = true)) - finally lastStatOffset = saved + def stagedBlock() = inBraces(block(simplify = true)) /** SimpleEpxr ::= spliceId | ‘$’ ‘{’ Block ‘}’) * SimpleType ::= spliceId | ‘$’ ‘{’ Block ‘}’) @@ -3642,7 +3627,6 @@ object Parsers { val meths = new ListBuffer[DefDef] val exitOnError = false while !isStatSeqEnd && !exitOnError do - setLastStatOffset() meths += extMethod(numLeadParams) acceptStatSepUnlessAtEnd(meths) if meths.isEmpty then syntaxError("`def` expected") @@ -3781,7 +3765,6 @@ object Parsers { def topStatSeq(outermost: Boolean = false): List[Tree] = { val stats = new ListBuffer[Tree] while (!isStatSeqEnd) { - setLastStatOffset() if (in.token == PACKAGE) { val start = in.skipToken() if (in.token == OBJECT) { @@ -3843,7 +3826,6 @@ object Parsers { } var exitOnError = false while (!isStatSeqEnd && !exitOnError) { - setLastStatOffset() if (in.token == IMPORT) stats ++= importClause(IMPORT, mkImport()) else if (in.token == EXPORT) @@ -3923,7 +3905,6 @@ object Parsers { val stats = new ListBuffer[Tree] var exitOnError = false while (!isStatSeqEnd && in.token != CASE && !exitOnError) { - setLastStatOffset() if (in.token == IMPORT) stats ++= importClause(IMPORT, mkImport()) else if (isExprIntro) diff --git a/tasty/test/dotty/tools/tasty/TastyHeaderUnpicklerTest.scala b/tasty/test/dotty/tools/tasty/TastyHeaderUnpicklerTest.scala index b9f2aff3f564..9f54c4b3061b 100644 --- a/tasty/test/dotty/tools/tasty/TastyHeaderUnpicklerTest.scala +++ b/tasty/test/dotty/tools/tasty/TastyHeaderUnpicklerTest.scala @@ -57,8 +57,8 @@ object TastyHeaderUnpicklerTest { buf.writeNat(exp) buf.writeNat(compilerBytes.length) buf.writeBytes(compilerBytes, compilerBytes.length) - buf.writeUncompressedLong(237478l) - buf.writeUncompressedLong(324789l) + buf.writeUncompressedLong(237478L) + buf.writeUncompressedLong(324789L) buf } From 542258b2b0d14da448f2f94ee2ee46990d5645e4 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Sat, 1 May 2021 12:44:11 +0200 Subject: [PATCH 2/8] Treat end markers as separate tokens Detect end markers in Scanner --- .../dotty/tools/dotc/parsing/Parsers.scala | 28 +++++------------- .../dotty/tools/dotc/parsing/Scanners.scala | 29 +++++++++++++++---- .../src/dotty/tools/dotc/parsing/Tokens.scala | 5 ++-- tests/neg/i12150.check | 6 ++++ tests/neg/i12150.scala | 3 ++ 5 files changed, 44 insertions(+), 27 deletions(-) create mode 100644 tests/neg/i12150.check create mode 100644 tests/neg/i12150.scala diff --git a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala index 07c8f5b10b2e..f2f323699b6d 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala @@ -167,6 +167,7 @@ object Parsers { class Parser(source: SourceFile)(using Context) extends ParserCommon(source) { val in: Scanner = new Scanner(source) + // in.debugTokenStream = true // uncomment to see the token stream of the standard scanner, but not syntax highlighting /** This is the general parse entry point. * Overridden by ScriptParser @@ -1242,7 +1243,7 @@ object Parsers { def possibleTemplateStart(isNew: Boolean = false): Unit = in.observeColonEOL() if in.token == COLONEOL then - if in.lookahead.isIdent(nme.end) then in.token = NEWLINE + if in.lookahead.token == END then in.token = NEWLINE else in.nextToken() if in.token != INDENT && in.token != LBRACE then @@ -1272,25 +1273,12 @@ object Parsers { case _: (ForYield | ForDo) => in.token == FOR case _ => false - if isIdent(nme.end) then - val start = in.offset - val isEndMarker = - val endLine = source.offsetToLine(start) - val lookahead = in.LookaheadScanner() - lookahead.nextToken() - source.offsetToLine(lookahead.offset) == endLine - && endMarkerTokens.contains(in.token) - && { - lookahead.nextToken() - lookahead.token == EOF - || source.offsetToLine(lookahead.offset) > endLine - } - if isEndMarker then - in.nextToken() - if stats.isEmpty || !matches(stats.last) then - syntaxError("misaligned end marker", Span(start, in.lastCharOffset)) - in.token = IDENTIFIER // Leaving it as the original token can confuse newline insertion - in.nextToken() + if in.token == END then + val start = in.skipToken() + if stats.isEmpty || !matches(stats.last) then + syntaxError("misaligned end marker", Span(start, in.lastCharOffset)) + in.token = IDENTIFIER // Leaving it as the original token can confuse newline insertion + in.nextToken() end checkEndMarker /* ------------- TYPES ------------------------------------------------------ */ diff --git a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala index d794c72ddc92..06e522a2009b 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala @@ -135,14 +135,16 @@ object Scanners { */ protected def putChar(c: Char): Unit = litBuf.append(c) - /** Clear buffer and set name and token */ - def finishNamed(idtoken: Token = IDENTIFIER, target: TokenData = this): Unit = { + /** Clear buffer and set name and token + * If `target` is different from `this`, don't treat identifiers as end tokens + */ + def finishNamed(idtoken: Token = IDENTIFIER, target: TokenData = this): Unit = target.name = termName(litBuf.chars, 0, litBuf.length) litBuf.clear() target.token = idtoken - if (idtoken == IDENTIFIER) - target.token = toToken(target.name) - } + if idtoken == IDENTIFIER then + val converted = toToken(target.name) + if converted != END || (target eq this) then target.token = converted /** The token for given `name`. Either IDENTIFIER or a keyword. */ def toToken(name: SimpleName): Token @@ -656,6 +658,8 @@ object Scanners { () /* skip the trailing comma */ else reset() + case END => + if !isEndMarker then token = IDENTIFIER case COLON => if fewerBracesEnabled then observeColonEOL() case RBRACE | RPAREN | RBRACKET => @@ -666,6 +670,21 @@ object Scanners { } } + protected def isEndMarker: Boolean = + if isAfterLineEnd then + val endLine = source.offsetToLine(offset) + val lookahead = new LookaheadScanner(): + override def isEndMarker = false + lookahead.nextToken() + if endMarkerTokens.contains(lookahead.token) + && source.offsetToLine(lookahead.offset) == endLine + then + lookahead.nextToken() + if lookahead.token == EOF + || source.offsetToLine(lookahead.offset) > endLine + then return true + false + /** Is there a blank line between the current token and the last one? * A blank line consists only of characters <= ' '. * @pre afterLineEnd(). diff --git a/compiler/src/dotty/tools/dotc/parsing/Tokens.scala b/compiler/src/dotty/tools/dotc/parsing/Tokens.scala index 27f533408deb..3d08463e3d3a 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Tokens.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Tokens.scala @@ -183,6 +183,7 @@ object Tokens extends TokensCommon { final val GIVEN = 63; enter(GIVEN, "given") final val EXPORT = 64; enter(EXPORT, "export") final val MACRO = 65; enter(MACRO, "macro") // TODO: remove + final val END = 66; enter(END, "end") /** special symbols */ final val NEWLINE = 78; enter(NEWLINE, "end of statement", "new line") @@ -207,7 +208,7 @@ object Tokens extends TokensCommon { /** XML mode */ final val XMLSTART = 99; enter(XMLSTART, "$XMLSTART$<") // TODO: deprecate - final val alphaKeywords: TokenSet = tokenRange(IF, MACRO) + final val alphaKeywords: TokenSet = tokenRange(IF, END) final val symbolicKeywords: TokenSet = tokenRange(USCORE, CTXARROW) final val keywords: TokenSet = alphaKeywords | symbolicKeywords @@ -256,7 +257,7 @@ object Tokens extends TokensCommon { final val canStartStatTokens2: TokenSet = canStartExprTokens2 | mustStartStatTokens | BitSet( AT, CASE) final val canStartStatTokens3: TokenSet = canStartExprTokens3 | mustStartStatTokens | BitSet( - AT, CASE) + AT, CASE, END) final val canEndStatTokens: TokenSet = atomicExprTokens | BitSet(TYPE, GIVEN, RPAREN, RBRACE, RBRACKET, OUTDENT) diff --git a/tests/neg/i12150.check b/tests/neg/i12150.check new file mode 100644 index 000000000000..884498ae2d1e --- /dev/null +++ b/tests/neg/i12150.check @@ -0,0 +1,6 @@ +-- [E018] Syntax Error: tests/neg/i12150.scala:1:13 -------------------------------------------------------------------- +1 |def f: Unit = // error + | ^ + | expression expected but end found + +longer explanation available when compiling with `-explain` diff --git a/tests/neg/i12150.scala b/tests/neg/i12150.scala new file mode 100644 index 000000000000..06fb5714f27d --- /dev/null +++ b/tests/neg/i12150.scala @@ -0,0 +1,3 @@ +def f: Unit = // error + +end f From 1100d43860cade19dc71b5c431e80546a142fada Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Sat, 1 May 2021 17:13:09 +0200 Subject: [PATCH 3/8] Refactor statement loops --- .../dotty/tools/dotc/parsing/Parsers.scala | 48 ++++++++++++++----- .../dotty/tools/dotc/reporting/messages.scala | 2 +- tests/neg-macros/i7603.scala | 2 +- tests/neg/indent.scala | 2 +- tests/neg/multiLineOps.scala | 2 +- 5 files changed, 41 insertions(+), 15 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala index f2f323699b6d..8006e266c463 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala @@ -254,6 +254,7 @@ object Parsers { || skipStopTokens.contains(in.token) && (in.currentRegion eq lastRegion) while !atStop do in.nextToken() + lastErrorOffset = in.offset def warning(msg: Message, sourcePos: SourcePosition): Unit = report.warning(msg, sourcePos) @@ -273,11 +274,9 @@ object Parsers { */ def syntaxErrorOrIncomplete(msg: Message, offset: Int = in.offset): Unit = if (in.token == EOF) incompleteInputError(msg) - else { + else syntaxError(msg, offset) skip() - lastErrorOffset = in.offset - } /** Consume one token of the specified type, or * signal an error if it is not there. @@ -312,6 +311,35 @@ object Parsers { def acceptStatSep(): Unit = if in.isNewLine then in.nextToken() else accept(SEMI) + def exitStats[T <: Tree](stats: ListBuffer[T], altEnd: Token = EOF, noPrevStat: Boolean): Boolean = + def recur(sepSeen: Boolean, endSeen: Boolean): Boolean = + if isStatSep then + in.nextToken() + recur(true, endSeen) + else if in.token == END then + if endSeen then syntaxError("duplicate end marker") + checkEndMarker(stats) + recur(sepSeen, true) + else if isStatSeqEnd || in.token == altEnd then + true + else if sepSeen || endSeen then + false + else + val found = in.token + syntaxError( + if noPrevStat then IllegalStartOfStatement(isModifier) + else i"end of statement expected but ${showToken(found)} found") + if mustStartStatTokens.contains(found) then + true // it's a statement that might be legal in an outer context + else + in.nextToken() // needed to ensure progress; otherwise we might cycle forever + skip() + false + + in.observeOutdented() + recur(false, false) + end exitStats + def acceptStatSepUnlessAtEnd[T <: Tree](stats: ListBuffer[T], altEnd: Token = EOF): Unit = def skipEmptyStats(): Unit = while (in.token == SEMI || in.token == NEWLINE || in.token == NEWLINES) do in.nextToken() @@ -3891,8 +3919,8 @@ object Parsers { */ def blockStatSeq(): List[Tree] = checkNoEscapingPlaceholders { val stats = new ListBuffer[Tree] - var exitOnError = false - while (!isStatSeqEnd && in.token != CASE && !exitOnError) { + while + var empty = false if (in.token == IMPORT) stats ++= importClause(IMPORT, mkImport()) else if (isExprIntro) @@ -3903,12 +3931,10 @@ object Parsers { stats += extension() else if isDefIntro(localModifierTokens, excludedSoftModifiers = Set(nme.`opaque`)) then stats +++= localDef(in.offset) - else if (!isStatSep && (in.token != CASE)) { - exitOnError = mustStartStat - syntaxErrorOrIncomplete(IllegalStartOfStatement(isModifier)) - } - acceptStatSepUnlessAtEnd(stats, CASE) - } + else + empty = true + !exitStats(stats, CASE, empty) + do () stats.toList } diff --git a/compiler/src/dotty/tools/dotc/reporting/messages.scala b/compiler/src/dotty/tools/dotc/reporting/messages.scala index 77766ed2a274..ff6ccab0e763 100644 --- a/compiler/src/dotty/tools/dotc/reporting/messages.scala +++ b/compiler/src/dotty/tools/dotc/reporting/messages.scala @@ -1826,7 +1826,7 @@ import transform.SymUtils._ class IllegalStartOfStatement(isModifier: Boolean)(using Context) extends SyntaxMsg(IllegalStartOfStatementID) { def msg = { - val addendum = if (isModifier) ": no modifiers allowed here" else "" + val addendum = if (isModifier) ": this modifier is not allowed here" else "" "Illegal start of statement" + addendum } def explain = "A statement is either an import, a definition or an expression." diff --git a/tests/neg-macros/i7603.scala b/tests/neg-macros/i7603.scala index c781513fb1ba..49bf1fe62471 100644 --- a/tests/neg-macros/i7603.scala +++ b/tests/neg-macros/i7603.scala @@ -2,6 +2,6 @@ import scala.quoted.* class Foo { def f[T2: Type](e: Expr[T2])(using Quotes) = e match { case '{ $x: ${'[List[$t]]} } => // error - case '{ $x: ${y @ '[List[$t]]} } => // error // error + case '{ $x: ${y @ '[List[$t]]} } => // error } } diff --git a/tests/neg/indent.scala b/tests/neg/indent.scala index 66ad2c7ca957..e6d6550bee19 100644 --- a/tests/neg/indent.scala +++ b/tests/neg/indent.scala @@ -2,7 +2,7 @@ object Test { extension (x: Int) def gt(y: Int) = x > y val y3 = - if (1) max 10 gt 0 // error: end of statement expected but integer literal found // error // error // error + if (1) max 10 gt 0 // error: end of statement expected but integer literal found // error // error 1 else 2 diff --git a/tests/neg/multiLineOps.scala b/tests/neg/multiLineOps.scala index 78a6ba4c3910..8499cc9fe710 100644 --- a/tests/neg/multiLineOps.scala +++ b/tests/neg/multiLineOps.scala @@ -6,7 +6,7 @@ val b1 = { 22 * 22 // ok */*one more*/22 // error: end of statement expected // error: not found: * -} // error: ';' expected, but '}' found +} val b2: Boolean = { println(x) From 021470080d3bdb3c11055b128ea9691f2549da02 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Sat, 1 May 2021 17:25:31 +0200 Subject: [PATCH 4/8] Tweak error reporting for illegal start of statement --- .../src/dotty/tools/dotc/parsing/Parsers.scala | 9 +++++---- .../dotty/tools/dotc/reporting/messages.scala | 16 ++++++++++------ 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala index 8006e266c463..d4724bafda59 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala @@ -311,7 +311,7 @@ object Parsers { def acceptStatSep(): Unit = if in.isNewLine then in.nextToken() else accept(SEMI) - def exitStats[T <: Tree](stats: ListBuffer[T], altEnd: Token = EOF, noPrevStat: Boolean): Boolean = + def exitStats[T <: Tree](stats: ListBuffer[T], noPrevStat: Boolean, altEnd: Token = EOF, what: String = "statement"): Boolean = def recur(sepSeen: Boolean, endSeen: Boolean): Boolean = if isStatSep then in.nextToken() @@ -326,9 +326,10 @@ object Parsers { false else val found = in.token + val statFollows = mustStartStatTokens.contains(found) syntaxError( - if noPrevStat then IllegalStartOfStatement(isModifier) - else i"end of statement expected but ${showToken(found)} found") + if noPrevStat then IllegalStartOfStatement(what, isModifier, statFollows) + else i"end of $what expected but ${showToken(found)} found") if mustStartStatTokens.contains(found) then true // it's a statement that might be legal in an outer context else @@ -3933,7 +3934,7 @@ object Parsers { stats +++= localDef(in.offset) else empty = true - !exitStats(stats, CASE, empty) + !exitStats(stats, empty, CASE) do () stats.toList } diff --git a/compiler/src/dotty/tools/dotc/reporting/messages.scala b/compiler/src/dotty/tools/dotc/reporting/messages.scala index ff6ccab0e763..eddd73e43941 100644 --- a/compiler/src/dotty/tools/dotc/reporting/messages.scala +++ b/compiler/src/dotty/tools/dotc/reporting/messages.scala @@ -1824,12 +1824,16 @@ import transform.SymUtils._ def explain = "" } - class IllegalStartOfStatement(isModifier: Boolean)(using Context) extends SyntaxMsg(IllegalStartOfStatementID) { - def msg = { - val addendum = if (isModifier) ": this modifier is not allowed here" else "" - "Illegal start of statement" + addendum - } - def explain = "A statement is either an import, a definition or an expression." + class IllegalStartOfStatement(what: String, isModifier: Boolean, isStat: Boolean)(using Context) extends SyntaxMsg(IllegalStartOfStatementID) { + def msg = + if isStat then + "this kind of statement is not allowed here" + else + val addendum = if isModifier then ": this modifier is not allowed here" else "" + s"Illegal start of $what$addendum" + def explain = + i"""A statement is an import or export, a definition or an expression. + |Some statements are only allowed in certain contexts""" } class TraitIsExpected(symbol: Symbol)(using Context) extends SyntaxMsg(TraitIsExpectedID) { From 2e5df7921893010e23ee25a982aa79cc7d58dca3 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Sat, 1 May 2021 18:27:32 +0200 Subject: [PATCH 5/8] Refactor remaining statement and declaration loops --- .../dotty/tools/dotc/parsing/Parsers.scala | 94 ++++++++----------- .../tools/dotc/reporting/ErrorMessageID.scala | 4 +- .../dotty/tools/dotc/reporting/messages.scala | 12 --- tests/neg/i10546.scala | 2 +- tests/neg/i4373.scala | 4 +- tests/neg/i8731.scala | 6 +- tests/neg/match-infix.scala | 2 +- tests/neg/parser-stability-17.scala | 2 +- tests/neg/parser-stability-19.scala | 2 +- tests/neg/parser-stability-23.scala | 2 +- tests/neg/spaces-vs-tabs.check | 6 +- tests/neg/spaces-vs-tabs.scala | 2 +- tests/neg/t6476.scala | 2 +- tests/neg/validate-parsing-2.scala | 1 + tests/neg/validate-parsing.scala | 1 - tests/pos/zipper.scala | 29 ++++++ 16 files changed, 82 insertions(+), 89 deletions(-) create mode 100644 tests/neg/validate-parsing-2.scala create mode 100644 tests/pos/zipper.scala diff --git a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala index d4724bafda59..8ff4b3e897a4 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala @@ -311,7 +311,11 @@ object Parsers { def acceptStatSep(): Unit = if in.isNewLine then in.nextToken() else accept(SEMI) - def exitStats[T <: Tree](stats: ListBuffer[T], noPrevStat: Boolean, altEnd: Token = EOF, what: String = "statement"): Boolean = + /** Parse statement separators and end markers. Ensure that there is at least + * one statement separator unless the next token terminates a statement sequence. + * @return true if the statement sequence continues, false if it terminates. + */ + def statSepOrEnd[T <: Tree](stats: ListBuffer[T], noPrevStat: Boolean = false, what: String = "statement", altEnd: Token = EOF): Boolean = def recur(sepSeen: Boolean, endSeen: Boolean): Boolean = if isStatSep then in.nextToken() @@ -321,9 +325,9 @@ object Parsers { checkEndMarker(stats) recur(sepSeen, true) else if isStatSeqEnd || in.token == altEnd then - true - else if sepSeen || endSeen then false + else if sepSeen || endSeen then + true else val found = in.token val statFollows = mustStartStatTokens.contains(found) @@ -331,32 +335,15 @@ object Parsers { if noPrevStat then IllegalStartOfStatement(what, isModifier, statFollows) else i"end of $what expected but ${showToken(found)} found") if mustStartStatTokens.contains(found) then - true // it's a statement that might be legal in an outer context + false // it's a statement that might be legal in an outer context else in.nextToken() // needed to ensure progress; otherwise we might cycle forever skip() - false + true in.observeOutdented() recur(false, false) - end exitStats - - def acceptStatSepUnlessAtEnd[T <: Tree](stats: ListBuffer[T], altEnd: Token = EOF): Unit = - def skipEmptyStats(): Unit = - while (in.token == SEMI || in.token == NEWLINE || in.token == NEWLINES) do in.nextToken() - - in.observeOutdented() - in.token match - case SEMI | NEWLINE | NEWLINES => - skipEmptyStats() - checkEndMarker(stats) - skipEmptyStats() - case `altEnd` => - case _ => - if !isStatSeqEnd then - syntaxError(i"end of statement expected but ${showToken(in.token)} found") - in.nextToken() // needed to ensure progress; otherwise we might cycle forever - accept(SEMI) + end statSepOrEnd def rewriteNotice(version: String = "3.0", additionalOption: String = "") = { val optionStr = if (additionalOption.isEmpty) "" else " " ++ additionalOption @@ -3642,10 +3629,10 @@ object Parsers { */ def extMethods(numLeadParams: Int): List[DefDef] = checkNoEscapingPlaceholders { val meths = new ListBuffer[DefDef] - val exitOnError = false - while !isStatSeqEnd && !exitOnError do + while meths += extMethod(numLeadParams) - acceptStatSepUnlessAtEnd(meths) + statSepOrEnd(meths, what = "extension method") + do () if meths.isEmpty then syntaxError("`def` expected") meths.toList } @@ -3781,7 +3768,8 @@ object Parsers { */ def topStatSeq(outermost: Boolean = false): List[Tree] = { val stats = new ListBuffer[Tree] - while (!isStatSeqEnd) { + while + var empty = false if (in.token == PACKAGE) { val start = in.skipToken() if (in.token == OBJECT) { @@ -3798,13 +3786,10 @@ object Parsers { stats += extension() else if isDefIntro(modifierTokens) then stats +++= defOrDcl(in.offset, defAnnotsMods(modifierTokens)) - else if !isStatSep then - if (in.token == CASE) - syntaxErrorOrIncomplete(OnlyCaseClassOrCaseObjectAllowed()) - else - syntaxErrorOrIncomplete(ExpectedToplevelDef()) - acceptStatSepUnlessAtEnd(stats) - } + else + empty = true + statSepOrEnd(stats, empty, "toplevel definition") + do () stats.toList } @@ -3836,13 +3821,12 @@ object Parsers { in.token = SELFARROW // suppresses INDENT insertion after `=>` in.nextToken() } - else { + else stats += first - acceptStatSepUnlessAtEnd(stats) - } + statSepOrEnd(stats) } - var exitOnError = false - while (!isStatSeqEnd && !exitOnError) { + while + var empty = false if (in.token == IMPORT) stats ++= importClause(IMPORT, mkImport()) else if (in.token == EXPORT) @@ -3853,12 +3837,10 @@ object Parsers { stats +++= defOrDcl(in.offset, defAnnotsMods(modifierTokens)) else if (isExprIntro) stats += expr1() - else if (!isStatSep) { - exitOnError = mustStartStat - syntaxErrorOrIncomplete("illegal start of definition") - } - acceptStatSepUnlessAtEnd(stats) - } + else + empty = true + statSepOrEnd(stats, empty) + do () (self, if (stats.isEmpty) List(EmptyTree) else stats.toList) } @@ -3887,16 +3869,14 @@ object Parsers { if problem.isEmpty then tree :: Nil else { syntaxError(problem, tree.span); Nil } - while (!isStatSeqEnd) { - if (isDclIntro) + while + val dclFound = isDclIntro + if dclFound then stats ++= checkLegal(defOrDcl(in.offset, Modifiers())) - else if (!isStatSep) - syntaxErrorOrIncomplete( - "illegal start of declaration" + - (if (inFunReturnType) " (possible cause: missing `=` in front of current method body)" - else "")) - acceptStatSepUnlessAtEnd(stats) - } + var what = "declaration" + if inFunReturnType then what += " (possible cause: missing `=` in front of current method body)" + statSepOrEnd(stats, !dclFound, what) + do () stats.toList } @@ -3934,7 +3914,7 @@ object Parsers { stats +++= localDef(in.offset) else empty = true - !exitStats(stats, empty, CASE) + statSepOrEnd(stats, empty, altEnd = CASE) do () stats.toList } @@ -3952,7 +3932,7 @@ object Parsers { in.nextToken() ts += objectDef(start, Modifiers(Package)) if (in.token != EOF) { - acceptStatSepUnlessAtEnd(ts) + statSepOrEnd(ts, what = "toplevel definition") ts ++= topStatSeq() } } @@ -3969,7 +3949,7 @@ object Parsers { acceptStatSep() ts += makePackaging(start, pkg, topstats()) if continue then - acceptStatSepUnlessAtEnd(ts) + statSepOrEnd(ts, what = "toplevel definition") ts ++= topStatSeq() } else diff --git a/compiler/src/dotty/tools/dotc/reporting/ErrorMessageID.scala b/compiler/src/dotty/tools/dotc/reporting/ErrorMessageID.scala index 23e1450f9225..ccdce7cfb1a4 100644 --- a/compiler/src/dotty/tools/dotc/reporting/ErrorMessageID.scala +++ b/compiler/src/dotty/tools/dotc/reporting/ErrorMessageID.scala @@ -87,8 +87,8 @@ enum ErrorMessageID extends java.lang.Enum[ErrorMessageID] { ValueClassesMayNotWrapAnotherValueClassID, ValueClassParameterMayNotBeAVarID, ValueClassNeedsExactlyOneValParamID, - OnlyCaseClassOrCaseObjectAllowedID, - ExpectedTopLevelDefID, + UNUSED1, + UNUSED2, AnonymousFunctionMissingParamTypeID, SuperCallsNotAllowedInlineableID, NotAPathID, diff --git a/compiler/src/dotty/tools/dotc/reporting/messages.scala b/compiler/src/dotty/tools/dotc/reporting/messages.scala index eddd73e43941..979a5e16ed6c 100644 --- a/compiler/src/dotty/tools/dotc/reporting/messages.scala +++ b/compiler/src/dotty/tools/dotc/reporting/messages.scala @@ -1656,18 +1656,6 @@ import transform.SymUtils._ def explain = "" } - class OnlyCaseClassOrCaseObjectAllowed()(using Context) - extends SyntaxMsg(OnlyCaseClassOrCaseObjectAllowedID) { - def msg = em"""Only ${hl("case class")} or ${hl("case object")} allowed""" - def explain = "" - } - - class ExpectedToplevelDef()(using Context) - extends SyntaxMsg(ExpectedTopLevelDefID) { - def msg = "Expected a toplevel definition" - def explain = "" - } - class SuperCallsNotAllowedInlineable(symbol: Symbol)(using Context) extends SyntaxMsg(SuperCallsNotAllowedInlineableID) { def msg = em"Super call not allowed in inlineable $symbol" diff --git a/tests/neg/i10546.scala b/tests/neg/i10546.scala index 3a5f42ec1fdf..e40f7117a3d4 100644 --- a/tests/neg/i10546.scala +++ b/tests/neg/i10546.scala @@ -1,5 +1,5 @@ object test: def times(num : Int)(block : => Unit) : Unit = () - times(10): println("ah") // error: end of statement expected but '(' found // error + times(10): println("ah") // error: end of statement expected but '(' found def foo: Set(Int) = Set(1) \ No newline at end of file diff --git a/tests/neg/i4373.scala b/tests/neg/i4373.scala index 2a7d0586f9d6..458dfc09c150 100644 --- a/tests/neg/i4373.scala +++ b/tests/neg/i4373.scala @@ -15,8 +15,8 @@ class A4 extends _ with Base // error object Test { type T1 = _ // error - type T2 = _[Int] // error // error - type T3 = _ { type S } // error // error + type T2 = _[Int] // error + type T3 = _ { type S } // error type T4 = [X] =>> _ // error // Open questions: diff --git a/tests/neg/i8731.scala b/tests/neg/i8731.scala index 0d4886c553ec..7eddcd42c030 100644 --- a/tests/neg/i8731.scala +++ b/tests/neg/i8731.scala @@ -11,9 +11,9 @@ object test: end if else // error: illegal start of definition () - end if // error: misaligned end marker + end if class Test { val test = 3 - end Test // error: misaligned end marker - } // error: eof expected, but unindent found \ No newline at end of file + end Test + } \ No newline at end of file diff --git a/tests/neg/match-infix.scala b/tests/neg/match-infix.scala index 12b57f4dc418..267dfbb720b7 100644 --- a/tests/neg/match-infix.scala +++ b/tests/neg/match-infix.scala @@ -1,3 +1,3 @@ def f = 1 + 1 match { case 2 => 3 -} + 1 // error // error +} + 1 // error diff --git a/tests/neg/parser-stability-17.scala b/tests/neg/parser-stability-17.scala index db1f212fc4ab..ff603a677378 100644 --- a/tests/neg/parser-stability-17.scala +++ b/tests/neg/parser-stability-17.scala @@ -1,2 +1,2 @@ trait x0[] { x0: x0 => } // error // error - class x0[x1] extends x0[x0 x0] x2 x0 // error // error // error + class x0[x1] extends x0[x0 x0] x2 x0 // error // error diff --git a/tests/neg/parser-stability-19.scala b/tests/neg/parser-stability-19.scala index 099c3d962c22..c320c7e8df74 100644 --- a/tests/neg/parser-stability-19.scala +++ b/tests/neg/parser-stability-19.scala @@ -1,5 +1,5 @@ object x0 { case class x0[](): // error def x0( ) ] // error - def x0 ( x0:x0 ):x0.type = x1 x0 // error // error // error + def x0 ( x0:x0 ):x0.type = x1 x0 // error // error // error \ No newline at end of file diff --git a/tests/neg/parser-stability-23.scala b/tests/neg/parser-stability-23.scala index d63059288b63..a27d79d5cc3e 100644 --- a/tests/neg/parser-stability-23.scala +++ b/tests/neg/parser-stability-23.scala @@ -1,3 +1,3 @@ object i0 { - import Ordering.{ implicitly as } (true: Boolean) match { case _: i1 as true } // error // error // error + import Ordering.{ implicitly as } (true: Boolean) match { case _: i1 as true } // error // error } diff --git a/tests/neg/spaces-vs-tabs.check b/tests/neg/spaces-vs-tabs.check index 51c2689f57bc..883e62b6aa00 100644 --- a/tests/neg/spaces-vs-tabs.check +++ b/tests/neg/spaces-vs-tabs.check @@ -23,12 +23,8 @@ | Previous indent : 2 tabs | Latest indent : 1 space -- Error: tests/neg/spaces-vs-tabs.scala:14:2 -------------------------------------------------------------------------- -14 | else 2 // error // error +14 | else 2 // error | ^ | The start of this line does not match any of the previous indentation widths. | Indentation width of current line : 1 tab, 2 spaces | This falls between previous widths: 1 tab and 1 tab, 4 spaces --- [E040] Syntax Error: tests/neg/spaces-vs-tabs.scala:14:7 ------------------------------------------------------------ -14 | else 2 // error // error - | ^ - | ';' expected, but integer literal found diff --git a/tests/neg/spaces-vs-tabs.scala b/tests/neg/spaces-vs-tabs.scala index ff8d1a1c328e..4f48d784eb7d 100644 --- a/tests/neg/spaces-vs-tabs.scala +++ b/tests/neg/spaces-vs-tabs.scala @@ -11,5 +11,5 @@ object Test: if true then 1 - else 2 // error // error + else 2 // error diff --git a/tests/neg/t6476.scala b/tests/neg/t6476.scala index 9d1415f55dd3..bd7868abe3e5 100644 --- a/tests/neg/t6476.scala +++ b/tests/neg/t6476.scala @@ -6,4 +6,4 @@ class C { s"\ " s"\\" s"\" // error -} // error (should not be one) +} diff --git a/tests/neg/validate-parsing-2.scala b/tests/neg/validate-parsing-2.scala new file mode 100644 index 000000000000..7457c649bad0 --- /dev/null +++ b/tests/neg/validate-parsing-2.scala @@ -0,0 +1 @@ +case class ByName(x: => Int) // error diff --git a/tests/neg/validate-parsing.scala b/tests/neg/validate-parsing.scala index d0eee526ae90..2b416c6eab27 100644 --- a/tests/neg/validate-parsing.scala +++ b/tests/neg/validate-parsing.scala @@ -10,4 +10,3 @@ class C () { } class D override() // error: ';' expected but 'override' found. -case class ByName(x: => Int) // error: `val' parameters may not be call-by-name diff --git a/tests/pos/zipper.scala b/tests/pos/zipper.scala new file mode 100644 index 000000000000..6ccd5462a43c --- /dev/null +++ b/tests/pos/zipper.scala @@ -0,0 +1,29 @@ +enum Tree[+A]: + case Branch(left: Tree[A], right: Tree[A]) + case Leaf(value: A) + +enum Context[+A]: + case Empty + case InLeft(right: Tree[A], parent: Context[A]) + case InRight(left: Tree[A], parent: Context[A]) + +import Tree.*, Context.* + +class Zipper[+A](val focus: Tree[A], val context: Context[A]): + def unfocus: Tree[A] = context match + case Empty => focus + case _ => moveUp.unfocus + def moveUp: Zipper[A] = context match + case Empty => this + case InLeft(right, parent) => Zipper(Branch(focus, right), parent) + case InRight(left, parent) => Zipper(Branch(left, focus), parent) + def moveLeft: Zipper[A] = focus match + case Leaf(_) => this + case Branch(left, right) => Zipper(left, InLeft(right, context)) + def moveRight: Zipper[A] = focus match + case Leaf(_) => this + case Branch(left, right) => Zipper(right, InRight(left, context)) + def replaceFocus[B >: A](newFocus: Tree[B]): Zipper[B] = + Zipper(newFocus, context) + +extension[A](tree: Tree[A]) def focus: Zipper[A] = Zipper(tree, Empty) From b7bd6a507b406b8b4cce7129e2e66bf00a555773 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Sat, 1 May 2021 19:09:49 +0200 Subject: [PATCH 6/8] Improve doc comments and fix repltests --- compiler/src/dotty/tools/dotc/parsing/Parsers.scala | 8 +++++++- compiler/test-resources/repl/i6676 | 6 ------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala index 8ff4b3e897a4..b81173650a2e 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala @@ -312,7 +312,13 @@ object Parsers { if in.isNewLine then in.nextToken() else accept(SEMI) /** Parse statement separators and end markers. Ensure that there is at least - * one statement separator unless the next token terminates a statement sequence. + * one statement separator unless the next token terminates a statement´sequence. + * @param stats the statements parsed to far + * @param noPrevStat true if there was no immediately preceding statement parsed + * @param what a string indicating what kin of statement is parsed + * @param altEnd a token that is also considered as a terminator of the statement + * sequence (the default `EOF` already assumes to terminate a statement + * sequence). * @return true if the statement sequence continues, false if it terminates. */ def statSepOrEnd[T <: Tree](stats: ListBuffer[T], noPrevStat: Boolean = false, what: String = "statement", altEnd: Token = EOF): Boolean = diff --git a/compiler/test-resources/repl/i6676 b/compiler/test-resources/repl/i6676 index 519225183e43..8e7f2c3d22ef 100644 --- a/compiler/test-resources/repl/i6676 +++ b/compiler/test-resources/repl/i6676 @@ -2,9 +2,6 @@ scala> xml" 1 | xml" | ^ | unclosed string literal -1 | xml" - | ^ - | ';' expected, but eof found scala> xml"" 1 | xml"" | ^^^^^ @@ -20,6 +17,3 @@ scala> s" 1 | s" | ^ | unclosed string literal -1 | s" - | ^ - | ';' expected, but eof found From b4182b10fc697896166a626165a94ead8cbe347a Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Sun, 2 May 2021 09:54:19 +0200 Subject: [PATCH 7/8] Fixes for handling end marker - Only recognize it under indentSyntax - But include it as a possible statement start even under Scala 2. END might be detected before it can be converted back to IDENTIFIER. So it needs to be included in canStartStatement2, just like IDENTIFIER. --- compiler/src/dotty/tools/dotc/parsing/Scanners.scala | 2 +- compiler/src/dotty/tools/dotc/parsing/Tokens.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala index 06e522a2009b..10905f93c80a 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala @@ -671,7 +671,7 @@ object Scanners { } protected def isEndMarker: Boolean = - if isAfterLineEnd then + if indentSyntax && isAfterLineEnd then val endLine = source.offsetToLine(offset) val lookahead = new LookaheadScanner(): override def isEndMarker = false diff --git a/compiler/src/dotty/tools/dotc/parsing/Tokens.scala b/compiler/src/dotty/tools/dotc/parsing/Tokens.scala index 3d08463e3d3a..cba07a6e5a34 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Tokens.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Tokens.scala @@ -255,7 +255,7 @@ object Tokens extends TokensCommon { final val mustStartStatTokens: TokenSet = defIntroTokens | modifierTokens | BitSet(IMPORT, EXPORT, PACKAGE) final val canStartStatTokens2: TokenSet = canStartExprTokens2 | mustStartStatTokens | BitSet( - AT, CASE) + AT, CASE, END) // END is included since it might be tested before being converted back to IDENTIFIER final val canStartStatTokens3: TokenSet = canStartExprTokens3 | mustStartStatTokens | BitSet( AT, CASE, END) From 78d370afd6b7a24f385ba6d8bafef91c033aba52 Mon Sep 17 00:00:00 2001 From: odersky Date: Tue, 4 May 2021 21:33:58 +0200 Subject: [PATCH 8/8] Update compiler/src/dotty/tools/dotc/parsing/Parsers.scala Co-authored-by: Nicolas Stucki --- compiler/src/dotty/tools/dotc/parsing/Parsers.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala index b81173650a2e..d1e880480aed 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala @@ -315,7 +315,7 @@ object Parsers { * one statement separator unless the next token terminates a statement´sequence. * @param stats the statements parsed to far * @param noPrevStat true if there was no immediately preceding statement parsed - * @param what a string indicating what kin of statement is parsed + * @param what a string indicating what kind of statement is parsed * @param altEnd a token that is also considered as a terminator of the statement * sequence (the default `EOF` already assumes to terminate a statement * sequence).