Skip to content

Commit 957fbde

Browse files
committed
Improve handling of end markers
1. Simplify the logic 2. Make end marker errors more reliable by recognizing end markers in more places 3. Allow nested end marker scopes at same indentation width Example in pos/end-nested.scala ```scala def f[T](x: Option[T]) = x match case Some(y) => case None => end f ``` Here, we should not give a "misaligned endmarker" error, which means we have to skip the nested `match` scope.
1 parent 5e47774 commit 957fbde

File tree

5 files changed

+93
-77
lines changed

5 files changed

+93
-77
lines changed

compiler/src/dotty/tools/dotc/parsing/Parsers.scala

Lines changed: 16 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1278,15 +1278,8 @@ object Parsers {
12781278
if silentTemplateIndent && !isNew then in.observeIndented()
12791279
newLineOptWhenFollowedBy(LBRACE)
12801280

1281-
def indentRegion[T](tag: EndMarkerTag)(op: => T): T = {
1282-
val iw = in.currentRegion.indentWidth
1283-
val t = op
1284-
in.consumeEndMarker(tag, iw)
1285-
t
1286-
}
1287-
1288-
def indentRegion[T](pid: Tree)(op: => T): T = pid match {
1289-
case pid: RefTree => indentRegion(pid.name.toTermName)(op)
1281+
def endMarkerScope[T](pid: Tree)(op: => T): T = pid match {
1282+
case pid: RefTree => in.endMarkerScope(pid.name.toTermName)(op)
12901283
case _ => op
12911284
}
12921285

@@ -1789,9 +1782,9 @@ object Parsers {
17891782

17901783
def expr1(location: Location.Value = Location.ElseWhere): Tree = in.token match {
17911784
case IF =>
1792-
indentRegion(IF) { ifExpr(in.offset, If) }
1785+
in.endMarkerScope(IF) { ifExpr(in.offset, If) }
17931786
case WHILE =>
1794-
indentRegion(WHILE) {
1787+
in.endMarkerScope(WHILE) {
17951788
atSpan(in.skipToken()) {
17961789
val cond = condExpr(DO)
17971790
newLinesOpt()
@@ -1826,7 +1819,7 @@ object Parsers {
18261819
WhileDo(Block(body, cond), Literal(Constant(())))
18271820
}
18281821
case TRY =>
1829-
indentRegion(TRY) {
1822+
in.endMarkerScope(TRY) {
18301823
val tryOffset = in.offset
18311824
atSpan(in.skipToken()) {
18321825
val body = expr()
@@ -1961,7 +1954,7 @@ object Parsers {
19611954
/** `match' (`{' CaseClauses `}' | CaseClause)
19621955
*/
19631956
def matchExpr(t: Tree, start: Offset, mkMatch: (Tree, List[CaseDef]) => Match) =
1964-
indentRegion(MATCH) {
1957+
in.endMarkerScope(MATCH) {
19651958
atSpan(start, in.skipToken()) {
19661959
mkMatch(t, casesExpr(caseClause))
19671960
}
@@ -2157,7 +2150,7 @@ object Parsers {
21572150
* | ‘new’ TemplateBody
21582151
*/
21592152
def newExpr(): Tree =
2160-
indentRegion(NEW) {
2153+
in.endMarkerScope(NEW) {
21612154
val start = in.skipToken()
21622155
def reposition(t: Tree) = t.withSpan(Span(start, in.lastOffset))
21632156
possibleBracesStart()
@@ -2323,7 +2316,7 @@ object Parsers {
23232316
* {nl} [`yield'] Expr
23242317
* | `for' Enumerators (`do' Expr | `yield' Expr)
23252318
*/
2326-
def forExpr(): Tree = indentRegion(FOR) {
2319+
def forExpr(): Tree = in.endMarkerScope(FOR) {
23272320
atSpan(in.skipToken()) {
23282321
var wrappedEnums = true
23292322
val start = in.offset
@@ -3074,7 +3067,7 @@ object Parsers {
30743067
else emptyType
30753068
val rhs =
30763069
if (tpt.isEmpty || in.token == EQUALS)
3077-
indentRegion(first) {
3070+
endMarkerScope(first) {
30783071
accept(EQUALS)
30793072
if (in.token == USCORE && !tpt.isEmpty && mods.is(Mutable) &&
30803073
(lhs.toList forall (_.isInstanceOf[Ident])))
@@ -3166,7 +3159,7 @@ object Parsers {
31663159
if (in.isScala2CompatMode) newLineOptWhenFollowedBy(LBRACE)
31673160
val rhs =
31683161
if (in.token == EQUALS)
3169-
indentRegion(name) {
3162+
in.endMarkerScope(name) {
31703163
in.nextToken()
31713164
subExpr()
31723165
}
@@ -3298,7 +3291,7 @@ object Parsers {
32983291
}
32993292

33003293
def classDefRest(start: Offset, mods: Modifiers, name: TypeName): TypeDef =
3301-
indentRegion(name.toTermName) {
3294+
in.endMarkerScope(name.toTermName) {
33023295
val constr = classConstr(isCaseClass = mods.is(Case))
33033296
val templ = templateOpt(constr)
33043297
finalizeDef(TypeDef(name, templ), mods, start)
@@ -3322,7 +3315,7 @@ object Parsers {
33223315
*/
33233316
def objectDef(start: Offset, mods: Modifiers): ModuleDef = atSpan(start, nameStart) {
33243317
val name = ident()
3325-
indentRegion(name) {
3318+
in.endMarkerScope(name) {
33263319
val templ = templateOpt(emptyConstructor)
33273320
finalizeDef(ModuleDef(name, templ), mods, start)
33283321
}
@@ -3332,7 +3325,7 @@ object Parsers {
33323325
*/
33333326
def enumDef(start: Offset, mods: Modifiers): TypeDef = atSpan(start, nameStart) {
33343327
val modulName = ident()
3335-
indentRegion(modulName) {
3328+
in.endMarkerScope(modulName) {
33363329
val clsName = modulName.toTypeName
33373330
val constr = classConstr()
33383331
val templ = template(constr, isEnum = true)
@@ -3412,7 +3405,7 @@ object Parsers {
34123405
else
34133406
(EmptyTermName, isIdent(nme.extension))
34143407

3415-
val gdef = indentRegion(if name.isEmpty then GIVEN else name) {
3408+
val gdef = in.endMarkerScope(if name.isEmpty then GIVEN else name) {
34163409
if isExtension then
34173410
if (in.token == COLON) in.nextToken()
34183411
assert(ident() == nme.extension)
@@ -3597,7 +3590,7 @@ object Parsers {
35973590
*/
35983591
def packaging(start: Int): Tree = {
35993592
val pkg = qualId()
3600-
indentRegion(pkg) {
3593+
endMarkerScope(pkg) {
36013594
possibleTemplateStart()
36023595
val stats = inDefScopeBraces(topStatSeq())
36033596
makePackaging(start, pkg, stats)
@@ -3785,7 +3778,7 @@ object Parsers {
37853778
else
37863779
val pkg = qualId()
37873780
var continue = false
3788-
indentRegion(pkg) {
3781+
endMarkerScope(pkg) {
37893782
possibleTemplateStart()
37903783
if in.token == EOF then
37913784
ts += makePackaging(start, pkg, List())

compiler/src/dotty/tools/dotc/parsing/Scanners.scala

Lines changed: 36 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -226,8 +226,8 @@ object Scanners {
226226
/** The current region. This is initially an Indented region with indentation width. */
227227
var currentRegion: Region = Indented(IndentWidth.Zero, Set(), EMPTY, null)
228228

229-
/** The end marker that was skipped last */
230-
val endMarkers = new mutable.ListBuffer[EndMarker]
229+
/** The number of open end marker scopes */
230+
var openEndMarkers: List[(EndMarkerTag, IndentWidth)] = Nil
231231

232232
// Scala 2 compatibility
233233

@@ -324,48 +324,47 @@ object Scanners {
324324
this.token = token
325325
}
326326

327-
/** If this token and the next constitute an end marker, skip them and append a new EndMarker
328-
* value at the end of the endMarkers queue.
327+
/** What can be referred to in an end marker */
328+
type EndMarkerTag = TermName | Token
329+
330+
/** Establish a scope for a passible end-marker with given tag, parsed by `op` */
331+
def endMarkerScope[T](tag: EndMarkerTag)(op: => T): T =
332+
val saved = openEndMarkers
333+
openEndMarkers = (tag, currentRegion.indentWidth) :: openEndMarkers
334+
try op finally openEndMarkers = saved
335+
336+
/** If this token and the next constitute an end marker, skip them and check they
337+
* align with an opening construct with the same end marker tag.
329338
*/
330-
private def handleEndMarkers(width: IndentWidth): Unit =
331-
if (next.token == IDENTIFIER && next.name == nme.end && width == currentRegion.indentWidth) {
339+
protected def skipEndMarker(width: IndentWidth): Unit =
340+
if next.token == IDENTIFIER && next.name == nme.end then
332341
val lookahead = LookaheadScanner()
333-
lookahead.nextToken() // skip the `end`
342+
val start = lookahead.offset
343+
344+
def handle(tag: EndMarkerTag) =
345+
def checkAligned(): Unit = openEndMarkers match
346+
case (etag, ewidth) :: rest if width <= ewidth =>
347+
if width < ewidth || tag != etag then
348+
openEndMarkers = rest
349+
checkAligned()
350+
case _ =>
351+
lexical.println(i"misaligned end marker $tag, $width, $openEndMarkers")
352+
errorButContinue("misaligned end marker", start)
334353

335-
def handle(tag: EndMarkerTag) = {
336354
val skipTo = lookahead.charOffset
337355
lookahead.nextToken()
338-
if (lookahead.isAfterLineEnd || lookahead.token == EOF) {
339-
lexical.println(i"produce end marker $tag $width")
340-
endMarkers += EndMarker(tag, width, offset)
356+
if lookahead.isAfterLineEnd || lookahead.token == EOF then
357+
checkAligned()
341358
next.token = EMPTY
342-
while (charOffset < skipTo) nextChar()
343-
}
344-
}
359+
while charOffset < skipTo do nextChar()
360+
end handle
345361

346-
lookahead.token match {
362+
lookahead.nextToken() // skip the `end`
363+
lookahead.token match
347364
case IDENTIFIER | BACKQUOTED_IDENT => handle(lookahead.name)
348365
case IF | WHILE | FOR | MATCH | TRY | NEW | GIVEN => handle(lookahead.token)
349366
case _ =>
350-
}
351-
}
352-
353-
/** Consume and cancel the head of the end markers queue if it has the given `tag` and width.
354-
* Flag end markers with higher indent widths as errors.
355-
*/
356-
def consumeEndMarker(tag: EndMarkerTag, width: IndentWidth): Unit = {
357-
lexical.println(i"consume end marker $tag $width")
358-
if (endMarkers.nonEmpty) {
359-
val em = endMarkers.head
360-
if (width <= em.width) {
361-
if (em.tag != tag || em.width != width) {
362-
lexical.println(i"misaligned end marker ${em.tag}, ${em.width} at ${width}")
363-
errorButContinue("misaligned end marker", em.offset)
364-
}
365-
endMarkers.trimStart(1)
366-
}
367-
}
368-
}
367+
end skipEndMarker
369368

370369
/** A leading symbolic or backquoted identifier is treated as an infix operator if
371370
* - it does not follow a blank line, and
@@ -501,6 +500,7 @@ object Scanners {
501500
&& !isLeadingInfixOperator()
502501
then
503502
insert(if (pastBlankLine) NEWLINES else NEWLINE, lineOffset)
503+
skipEndMarker(nextWidth)
504504
else if indentIsSignificant then
505505
if nextWidth < lastWidth
506506
|| nextWidth == lastWidth && (indentPrefix == MATCH || indentPrefix == CATCH) && token != CASE then
@@ -511,7 +511,7 @@ object Scanners {
511511
case r: Indented =>
512512
currentRegion = r.enclosing
513513
insert(OUTDENT, offset)
514-
handleEndMarkers(nextWidth)
514+
skipEndMarker(nextWidth)
515515
case r: InBraces if !closingRegionTokens.contains(token) =>
516516
ctx.warning("Line is indented too far to the left, or a `}' is missing",
517517
source.atSpan(Span(offset)))
@@ -883,6 +883,7 @@ object Scanners {
883883

884884
class LookaheadScanner(indent: Boolean = false) extends Scanner(source, offset) {
885885
override val indentSyntax = indent
886+
override def skipEndMarker(width: IndentWidth) = ()
886887
override protected def printState() = {
887888
print("la:")
888889
super.printState()
@@ -1416,16 +1417,6 @@ object Scanners {
14161417
val Zero = Run(' ', 0)
14171418
}
14181419

1419-
/** What can be referred to in an end marker */
1420-
type EndMarkerTag = TermName | Token
1421-
1422-
/** A processed end marker
1423-
* @param tag The name or token referred to in the marker
1424-
* @param width The indentation width where the marker occurred
1425-
* @param offset The offset of the `end`
1426-
*/
1427-
case class EndMarker(tag: EndMarkerTag, width: IndentWidth, offset: Int)
1428-
14291420
// ------------- keyword configuration -----------------------------------
14301421

14311422
private val (lastKeywordStart, kwArray) = buildKeywordArray(keywords)

tests/neg/endmarkers.scala

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,14 @@ object Test
33
locally {
44
var x = 0
55
while x < 10 do x += 1
6-
end while // error: end of statement expected but while found // error: not found: end
7-
val f = 10 // error: ';' expected, but 'val' found
6+
end while
7+
val f = 10
88
while
99
x += 1
1010
x < 10
1111
do ()
12-
end while // error: misaligned end marker // error: not found : end
13-
} // error: ';' expected, but '}' found
12+
end while // error: misaligned end marker
13+
}
1414

1515
def f(x: Int): Int =
1616
val y =
@@ -50,24 +50,24 @@ class Test2
5050
object x
5151
new Test2 {
5252
override def foo = 2
53-
end new // error: end of statement expected but new found // error: not found: end
54-
} // error: ';' expected, but '}' found
53+
end new // error: misaligned end marker
54+
}
5555
def bar = 2
5656
end Test2 // error: misaligned end marker
5757
end Test2
5858

5959
class Test3
6060
self =>
6161
def foo = 1
62-
end Test3 // error: not found: end
62+
end Test3 // error: misaligned end marker
6363

6464
import collection.mutable.HashMap
6565

6666
class Coder(words: List[String])
6767

6868
class Foo
6969
println()
70-
end Foo // error: not found: end
70+
end Foo // error: misaligned end marker
7171

7272
(2 -> "ABC", new ArrowAssoc('3') -> "DEF")
7373

@@ -101,4 +101,4 @@ class Coder(words: List[String])
101101
.flatMap {
102102
case (digit, str) => str map (ltr => ltr -> digit)
103103
}
104-
end Coder // error: The start of this line does not match any of the previous indentation widths.
104+
end Coder // error: misaligned end marker

tests/neg/endmarkers1.scala

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
2+
def f7[T](x: Option[T]) = x match
3+
case Some(y) =>
4+
case None =>
5+
end if // error: misaligned end marker
6+
7+
object Test4 with
8+
def f[T](x: Option[T]) = x match
9+
case Some(y) =>
10+
case None =>
11+
end if // error: misaligned end marker
12+
end Test // error: misaligned end marker
13+
14+
def f[T](x: Option[T]) = x match
15+
case Some(y) =>
16+
case None => "hello"
17+
end f // error: misaligned end marker
18+

tests/pos/end-nested.scala

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
def f[T](x: Option[T]) = x match
2+
case Some(y) =>
3+
case None =>
4+
end f
5+
6+
object Test with
7+
try List(1, 2, 3) match
8+
case x :: xs => println(x)
9+
case Nil => println("Nil")
10+
catch
11+
case ex: java.io.IOException => println(ex)
12+
case ex: Throwable => throw ex
13+
end try
14+

0 commit comments

Comments
 (0)