From 3292ed820de6b30d618a95d88409ff6591ceeaa4 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Sun, 19 Apr 2020 10:58:14 +0200 Subject: [PATCH 1/7] Tweaks - Use Eql derivation in Namer - Use end markers in Nullables - Fix a link in overview doc --- compiler/src/dotty/tools/dotc/core/Names.scala | 4 +--- compiler/src/dotty/tools/dotc/typer/Nullables.scala | 8 +++++++- docs/docs/reference/overview.md | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/core/Names.scala b/compiler/src/dotty/tools/dotc/core/Names.scala index c94043da0719..06d318165d61 100644 --- a/compiler/src/dotty/tools/dotc/core/Names.scala +++ b/compiler/src/dotty/tools/dotc/core/Names.scala @@ -25,8 +25,6 @@ object Names { def toTermName: TermName } - implicit def eqName: Eql[Name, Name] = Eql.derived - /** A common superclass of Name and Symbol. After bootstrap, this should be * just the type alias Name | Symbol */ @@ -37,7 +35,7 @@ object Names { * in a name table. A derived term name adds a tag, and possibly a number * or a further simple name to some other name. */ - abstract class Name extends Designator with PreName { + abstract class Name extends Designator, PreName derives Eql { /** A type for names of the same kind as this name */ type ThisName <: Name diff --git a/compiler/src/dotty/tools/dotc/typer/Nullables.scala b/compiler/src/dotty/tools/dotc/typer/Nullables.scala index d4f062b88a47..ad64a8f41f89 100644 --- a/compiler/src/dotty/tools/dotc/typer/Nullables.scala +++ b/compiler/src/dotty/tools/dotc/typer/Nullables.scala @@ -189,7 +189,7 @@ object Nullables: val mutables = infos.foldLeft(Set[TermRef]())((ms, info) => ms.union(info.asserted.filter(_.symbol.is(Mutable)))) infos.extendWith(NotNullInfo(Set(), mutables)) - // end notNullInfoOps + end notNullInfoOps extension refOps on (ref: TermRef): @@ -244,6 +244,7 @@ object Nullables: refSym.is(Mutable) // if it is immutable, we don't need to check the rest conditions && refOwner.isTerm && recur(ctx.owner) + end refOps extension treeOps on (tree: Tree): @@ -334,6 +335,7 @@ object Nullables: traverseChildren(tree) tree.computeNullable() }.traverse(tree) + end treeOps extension assignOps on (tree: Assign): def computeAssignNullable()(using Context): tree.type = tree.lhs match @@ -351,6 +353,7 @@ object Nullables: tree.withNotNullInfo(NotNullInfo(Set(ref), Set())) else tree case _ => tree + end assignOps private val analyzedOps = Set(nme.EQ, nme.NE, nme.eq, nme.ne, nme.ZAND, nme.ZOR, nme.UNARY_!) @@ -444,13 +447,16 @@ object Nullables: * xs = xs.next */ def whileContext(whileSpan: Span)(using Context): Context = + def isRetracted(ref: TermRef): Boolean = val sym = ref.symbol sym.span.exists && assignmentSpans.getOrElse(sym.span.start, Nil).exists(whileSpan.contains(_)) && ctx.notNullInfos.impliesNotNull(ref) + val retractedVars = ctx.notNullInfos.flatMap(_.asserted.filter(isRetracted)).toSet ctx.addNotNullInfo(NotNullInfo(Set(), retractedVars)) + end whileContext /** Post process all arguments to by-name parameters by removing any not-null * info that was used when typing them. Concretely: diff --git a/docs/docs/reference/overview.md b/docs/docs/reference/overview.md index 5d63e07ff236..d333a71e8933 100644 --- a/docs/docs/reference/overview.md +++ b/docs/docs/reference/overview.md @@ -33,7 +33,7 @@ These constructs replace existing constructs with the aim of making the language - [Trait Parameters](other-new-features/trait-parameters.md) replace [early initializers](dropped-features/early-initializers.md) with a more generally useful construct. - [Given Instances](contextual/delegates.md) replace implicit objects and defs, focussing on intent over mechanism. - - [Given Clauses](contextual/given-clauses.md) replace implicit parameters, avoiding their ambiguities. + - [Using Clauses](contextual/using-clauses.md) replace implicit parameters, avoiding their ambiguities. - [Extension Methods](contextual/extension-methods.md) replace implicit classes with a clearer and simpler mechanism. - [Opaque Type Aliases](other-new-features/opaques.md) replace most uses of value classes while guaranteeing absence of boxing. From bdf18e9d23c20ddd8fa85dbb50cc190770c5eae7 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Sun, 19 Apr 2020 15:33:42 +0200 Subject: [PATCH 2/7] Fix #8731: Revise end markers scheme Handle end markers in parser instead of in Scanner, adding end markers to the context free syntax. --- compiler/src/dotty/tools/dotc/ast/Trees.scala | 2 + .../dotty/tools/dotc/parsing/Parsers.scala | 309 +++++++++--------- .../dotty/tools/dotc/parsing/Scanners.scala | 73 +---- .../src/dotty/tools/dotc/parsing/Tokens.scala | 5 +- .../dotc/printing/SyntaxHighlighting.scala | 4 +- .../src/dotty/tools/dotc/typer/FrontEnd.scala | 1 + docs/docs/internals/syntax.md | 12 +- tests/neg/endmarkers.scala | 2 +- tests/neg/endmarkers1.scala | 2 +- tests/neg/i8731.scala | 29 ++ tests/pos/endmarkers.scala | 12 + 11 files changed, 225 insertions(+), 226 deletions(-) create mode 100644 tests/neg/i8731.scala create mode 100644 tests/pos/endmarkers.scala diff --git a/compiler/src/dotty/tools/dotc/ast/Trees.scala b/compiler/src/dotty/tools/dotc/ast/Trees.scala index 24bfa98401a7..7cbdb441fdca 100644 --- a/compiler/src/dotty/tools/dotc/ast/Trees.scala +++ b/compiler/src/dotty/tools/dotc/ast/Trees.scala @@ -369,6 +369,8 @@ object Trees { comment.map(putAttachment(DocComment, _)) this } + + def name: Name } /** A ValDef or DefDef tree */ diff --git a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala index 9dbb2a7efbb9..68b58c653c82 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala @@ -160,17 +160,12 @@ object Parsers { def skipBraces(): Unit = { accept(if (in.token == INDENT) INDENT else LBRACE) var openBraces = 1 - val savedCheckEndMarker = in.checkEndMarker - try - in.checkEndMarker = false - while (in.token != EOF && openBraces > 0) - skipBracesHook() getOrElse { - if (in.token == LBRACE || in.token == INDENT) openBraces += 1 - else if (in.token == RBRACE || in.token == OUTDENT) openBraces -= 1 - in.nextToken() - } - finally - in.checkEndMarker = savedCheckEndMarker + while (in.token != EOF && openBraces > 0) + skipBracesHook() getOrElse { + if (in.token == LBRACE || in.token == INDENT) openBraces += 1 + else if (in.token == RBRACE || in.token == OUTDENT) openBraces -= 1 + in.nextToken() + } } } @@ -368,14 +363,15 @@ object Parsers { def acceptStatSep(): Unit = if in.isNewLine then in.nextToken() else accept(SEMI) - def acceptStatSepUnlessAtEnd(altEnd: Token = EOF): Unit = + def acceptStatSepUnlessAtEnd(stats: ListBuffer[Tree], altEnd: Token = EOF): Unit = in.observeOutdented() if (!isStatSeqEnd) in.token match { case EOF => case `altEnd` => - case NEWLINE | NEWLINES => in.nextToken() - case SEMI => in.nextToken() + case SEMI | NEWLINE | NEWLINES => + in.nextToken() + checkEndMarker(stats) case _ => syntaxError(i"end of statement expected but ${showToken(in.token)} found") in.nextToken() // needed to ensure progress; otherwise we might cycle forever @@ -1292,10 +1288,47 @@ object Parsers { else newLineOptWhenFollowedBy(LBRACE) - def endMarkerScope[T](pid: Tree)(op: => T): T = pid match { - case pid: RefTree => in.endMarkerScope(pid.name.toTermName)(op) - case _ => op - } + def checkEndMarker(stats: ListBuffer[Tree]): Unit = + + def matches(stat: Tree): Boolean = stat match + case stat: MemberDef if !stat.name.isEmpty => + if stat.name == nme.CONSTRUCTOR then in.token == THIS + else in.isIdent && in.name == stat.name.toTermName + case ModuleDef(name, Template(_, Nil, _, _)) => + in.token == IDENTIFIER && in.name == nme.extension + case PackageDef(pid: RefTree, _) => + in.isIdent && in.name == pid.name + case PatDef(_, IdPattern(id, _) :: Nil, _, _) => + in.isIdent && in.name == id.name + case stat: MemberDef if stat.mods.is(Given) => in.token == GIVEN + case _: PatDef => in.token == VAL + case _: If => in.token == IF + case _: WhileDo => in.token == WHILE + case _: ParsedTry => in.token == TRY + case _: Match => in.token == MATCH + case _: New => in.token == NEW + case _: (ForYield | ForDo) => in.token == FOR + case _ => false + + if isIdent(nme.end) then + val start = in.offset + val isEndMarker = + val endLine = source.offsetToLine(start) + val lookahead = in.LookaheadScanner() + lookahead.nextToken() + source.offsetToLine(lookahead.offset) == endLine + && endMarkerTokens.contains(in.token) + && { + lookahead.nextToken() + lookahead.token == EOF + || source.offsetToLine(lookahead.offset) > endLine + } + if isEndMarker then + in.nextToken() + if stats.isEmpty || !matches(stats.last) then + syntaxError("misaligned end marker", Span(start, in.lastCharOffset)) + in.nextToken() + end checkEndMarker /* ------------- TYPES ------------------------------------------------------ */ @@ -1878,15 +1911,13 @@ object Parsers { def expr1(location: Location.Value = Location.ElseWhere): Tree = in.token match case IF => - in.endMarkerScope(IF) { ifExpr(in.offset, If) } + ifExpr(in.offset, If) case WHILE => - in.endMarkerScope(WHILE) { - atSpan(in.skipToken()) { - val cond = condExpr(DO) - newLinesOpt() - val body = subExpr() - WhileDo(cond, body) - } + atSpan(in.skipToken()) { + val cond = condExpr(DO) + newLinesOpt() + val body = subExpr() + WhileDo(cond, body) } case DO => ctx.errorOrMigrationWarning( @@ -1913,40 +1944,38 @@ object Parsers { WhileDo(Block(body, cond), Literal(Constant(()))) } case TRY => - in.endMarkerScope(TRY) { - val tryOffset = in.offset - atSpan(in.skipToken()) { - val body = expr() - val (handler, handlerStart) = - if in.token == CATCH then - val span = in.offset - in.nextToken() - (if in.token == CASE then Match(EmptyTree, caseClause(exprOnly = true) :: Nil) - else subExpr(), - span) - else (EmptyTree, -1) - - handler match { - case Block(Nil, EmptyTree) => - assert(handlerStart != -1) - syntaxError( - EmptyCatchBlock(body), - Span(handlerStart, endOffset(handler)) - ) - case _ => - } - - val finalizer = - if (in.token == FINALLY) { in.nextToken(); subExpr() } - else { - if (handler.isEmpty) warning( - EmptyCatchAndFinallyBlock(body), - source.atSpan(Span(tryOffset, endOffset(body))) - ) - EmptyTree - } - ParsedTry(body, handler, finalizer) + val tryOffset = in.offset + atSpan(in.skipToken()) { + val body = expr() + val (handler, handlerStart) = + if in.token == CATCH then + val span = in.offset + in.nextToken() + (if in.token == CASE then Match(EmptyTree, caseClause(exprOnly = true) :: Nil) + else subExpr(), + span) + else (EmptyTree, -1) + + handler match { + case Block(Nil, EmptyTree) => + assert(handlerStart != -1) + syntaxError( + EmptyCatchBlock(body), + Span(handlerStart, endOffset(handler)) + ) + case _ => } + + val finalizer = + if (in.token == FINALLY) { in.nextToken(); subExpr() } + else { + if (handler.isEmpty) warning( + EmptyCatchAndFinallyBlock(body), + source.atSpan(Span(tryOffset, endOffset(body))) + ) + EmptyTree + } + ParsedTry(body, handler, finalizer) } case THROW => atSpan(in.skipToken()) { Throw(expr()) } @@ -2045,10 +2074,8 @@ object Parsers { /** MatchClause ::= `match' `{' CaseClauses `}' */ def matchClause(t: Tree): Match = - in.endMarkerScope(MATCH) { - atSpan(t.span.start, in.skipToken()) { - Match(t, inBracesOrIndented(caseClauses(() => caseClause()))) - } + atSpan(t.span.start, in.skipToken()) { + Match(t, inBracesOrIndented(caseClauses(() => caseClause()))) } /** `match' `{' TypeCaseClauses `}' @@ -2249,21 +2276,19 @@ object Parsers { * | ‘new’ TemplateBody */ def newExpr(): Tree = - in.endMarkerScope(NEW) { - val start = in.skipToken() - def reposition(t: Tree) = t.withSpan(Span(start, in.lastOffset)) - possibleTemplateStart() - val parents = - if in.isNestedStart then Nil - else constrApps(commaOK = false, templateCanFollow = true) - colonAtEOLOpt() - possibleTemplateStart(isNew = true) - parents match { - case parent :: Nil if !in.isNestedStart => - reposition(if (parent.isType) ensureApplied(wrapNew(parent)) else parent) - case _ => - New(reposition(templateBodyOpt(emptyConstructor, parents, Nil))) - } + val start = in.skipToken() + def reposition(t: Tree) = t.withSpan(Span(start, in.lastOffset)) + possibleTemplateStart() + val parents = + if in.isNestedStart then Nil + else constrApps(commaOK = false, templateCanFollow = true) + colonAtEOLOpt() + possibleTemplateStart(isNew = true) + parents match { + case parent :: Nil if !in.isNestedStart => + reposition(if (parent.isType) ensureApplied(wrapNew(parent)) else parent) + case _ => + New(reposition(templateBodyOpt(emptyConstructor, parents, Nil))) } /** ExprsInParens ::= ExprInParens {`,' ExprInParens} @@ -2415,7 +2440,7 @@ object Parsers { * {nl} [`yield'] Expr * | `for' Enumerators (`do' Expr | `yield' Expr) */ - def forExpr(): Tree = in.endMarkerScope(FOR) { + def forExpr(): Tree = atSpan(in.skipToken()) { var wrappedEnums = true val start = in.offset @@ -2492,7 +2517,6 @@ object Parsers { ForDo(enums, expr()) } } - } /** CaseClauses ::= CaseClause {CaseClause} * TypeCaseClauses ::= TypeCaseClause {TypeCaseClause} @@ -3175,15 +3199,13 @@ object Parsers { else emptyType val rhs = if tpt.isEmpty || in.token == EQUALS then - endMarkerScope(first) { - accept(EQUALS) - subExpr() match - case rhs0 @ Ident(name) if placeholderParams.nonEmpty && name == placeholderParams.head.name - && !tpt.isEmpty && mods.is(Mutable) && lhs.forall(_.isInstanceOf[Ident]) => - placeholderParams = placeholderParams.tail - atSpan(rhs0.span) { Ident(nme.WILDCARD) } - case rhs0 => rhs0 - } + accept(EQUALS) + subExpr() match + case rhs0 @ Ident(name) if placeholderParams.nonEmpty && name == placeholderParams.head.name + && !tpt.isEmpty && mods.is(Mutable) && lhs.forall(_.isInstanceOf[Ident]) => + placeholderParams = placeholderParams.tail + atSpan(rhs0.span) { Ident(nme.WILDCARD) } + case rhs0 => rhs0 else EmptyTree lhs match { case IdPattern(id, t) :: Nil if t.isEmpty => @@ -3286,10 +3308,8 @@ object Parsers { if (migrateTo3) newLineOptWhenFollowedBy(LBRACE) val rhs = if (in.token == EQUALS) - in.endMarkerScope(name) { - in.nextToken() - subExpr() - } + in.nextToken() + subExpr() else if (!tpt.isEmpty) EmptyTree else if (scala2ProcedureSyntax(": Unit")) { @@ -3418,11 +3438,9 @@ object Parsers { } def classDefRest(start: Offset, mods: Modifiers, name: TypeName): TypeDef = - in.endMarkerScope(name.toTermName) { - val constr = classConstr(isCaseClass = mods.is(Case)) - val templ = templateOpt(constr) - finalizeDef(TypeDef(name, templ), mods, start) - } + val constr = classConstr(isCaseClass = mods.is(Case)) + val templ = templateOpt(constr) + finalizeDef(TypeDef(name, templ), mods, start) /** ClassConstr ::= [ClsTypeParamClause] [ConstrMods] ClsParamClauses */ @@ -3442,10 +3460,8 @@ object Parsers { */ def objectDef(start: Offset, mods: Modifiers): ModuleDef = atSpan(start, nameStart) { val name = ident() - in.endMarkerScope(name) { - val templ = templateOpt(emptyConstructor) - finalizeDef(ModuleDef(name, templ), mods, start) - } + val templ = templateOpt(emptyConstructor) + finalizeDef(ModuleDef(name, templ), mods, start) } private def checkAccessOnly(mods: Modifiers, where: String): Modifiers = @@ -3459,12 +3475,10 @@ object Parsers { def enumDef(start: Offset, mods: Modifiers): TypeDef = atSpan(start, nameStart) { val mods1 = checkAccessOnly(mods, "definitions") val modulName = ident() - in.endMarkerScope(modulName) { - val clsName = modulName.toTypeName - val constr = classConstr() - val templ = template(constr, isEnum = true) - finalizeDef(TypeDef(clsName, templ), mods1, start) - } + val clsName = modulName.toTypeName + val constr = classConstr() + val templ = template(constr, isEnum = true) + finalizeDef(TypeDef(clsName, templ), mods1, start) } /** EnumCase = `case' (id ClassConstr [`extends' ConstrApps] | ids) @@ -3532,7 +3546,7 @@ object Parsers { val nameStart = in.offset val name = if isIdent && hasGivenSig then ident() else EmptyTermName - val gdef = in.endMarkerScope(if name.isEmpty then GIVEN else name) { + val gdef = val tparams = typeParamClauseOpt(ParamOwner.Def) newLineOpt() val vparamss = @@ -3565,7 +3579,7 @@ object Parsers { val templ = templateBodyOpt(makeConstructor(tparams1, vparamss1), parents, Nil) if tparams.isEmpty && vparamss.isEmpty then ModuleDef(name, templ) else TypeDef(name.toTypeName, templ) - } + end gdef finalizeDef(gdef, mods1, start) } @@ -3575,23 +3589,21 @@ object Parsers { in.nextToken() val nameOffset = in.offset val name = if isIdent && !isIdent(nme.on) then ident() else EmptyTermName - in.endMarkerScope(if name.isEmpty then nme.extension else name) { - val (tparams, vparamss, extensionFlag) = - if isIdent(nme.on) then - in.nextToken() - val tparams = typeParamClauseOpt(ParamOwner.Def) - val extParams = paramClause(0, prefix = true) - val givenParamss = paramClauses(givenOnly = true) - (tparams, extParams :: givenParamss, Extension) - else - (Nil, Nil, EmptyFlags) - possibleTemplateStart() - if !in.isNestedStart then syntaxError("Extension without extension methods") - val templ = templateBodyOpt(makeConstructor(tparams, vparamss), Nil, Nil) - templ.body.foreach(checkExtensionMethod(tparams, vparamss, _)) - val edef = atSpan(start, nameOffset, in.offset)(ModuleDef(name, templ)) - finalizeDef(edef, addFlag(mods, Given | extensionFlag), start) - } + val (tparams, vparamss, extensionFlag) = + if isIdent(nme.on) then + in.nextToken() + val tparams = typeParamClauseOpt(ParamOwner.Def) + val extParams = paramClause(0, prefix = true) + val givenParamss = paramClauses(givenOnly = true) + (tparams, extParams :: givenParamss, Extension) + else + (Nil, Nil, EmptyFlags) + possibleTemplateStart() + if !in.isNestedStart then syntaxError("Extension without extension methods") + val templ = templateBodyOpt(makeConstructor(tparams, vparamss), Nil, Nil) + templ.body.foreach(checkExtensionMethod(tparams, vparamss, _)) + val edef = atSpan(start, nameOffset, in.offset)(ModuleDef(name, templ)) + finalizeDef(edef, addFlag(mods, Given | extensionFlag), start) /* -------- TEMPLATES ------------------------------------------- */ @@ -3695,14 +3707,11 @@ object Parsers { /** Packaging ::= package QualId [nl] `{' TopStatSeq `}' */ - def packaging(start: Int): Tree = { + def packaging(start: Int): Tree = val pkg = qualId() - endMarkerScope(pkg) { - possibleTemplateStart() - val stats = inDefScopeBraces(topStatSeq()) - makePackaging(start, pkg, stats) - } - } + possibleTemplateStart() + val stats = inDefScopeBraces(topStatSeq()) + makePackaging(start, pkg, stats) /** TopStatSeq ::= TopStat {semi TopStat} * TopStat ::= Import @@ -3735,7 +3744,7 @@ object Parsers { syntaxErrorOrIncomplete(OnlyCaseClassOrCaseObjectAllowed()) else syntaxErrorOrIncomplete(ExpectedToplevelDef()) - acceptStatSepUnlessAtEnd() + acceptStatSepUnlessAtEnd(stats) } stats.toList } @@ -3769,7 +3778,7 @@ object Parsers { } else { stats += first - acceptStatSepUnlessAtEnd() + acceptStatSepUnlessAtEnd(stats) } } var exitOnError = false @@ -3787,7 +3796,7 @@ object Parsers { exitOnError = mustStartStat syntaxErrorOrIncomplete("illegal start of definition") } - acceptStatSepUnlessAtEnd() + acceptStatSepUnlessAtEnd(stats) } (self, if (stats.isEmpty) List(EmptyTree) else stats.toList) } @@ -3825,7 +3834,7 @@ object Parsers { "illegal start of declaration" + (if (inFunReturnType) " (possible cause: missing `=` in front of current method body)" else "")) - acceptStatSepUnlessAtEnd() + acceptStatSepUnlessAtEnd(stats) } stats.toList } @@ -3864,7 +3873,7 @@ object Parsers { exitOnError = mustStartStat syntaxErrorOrIncomplete(IllegalStartOfStatement(isModifier)) } - acceptStatSepUnlessAtEnd(CASE) + acceptStatSepUnlessAtEnd(stats, CASE) } stats.toList } @@ -3889,19 +3898,17 @@ object Parsers { else val pkg = qualId() var continue = false - endMarkerScope(pkg) { - possibleTemplateStart() - if in.token == EOF then - ts += makePackaging(start, pkg, List()) - else if in.isNestedStart then - ts += inDefScopeBraces(makePackaging(start, pkg, topStatSeq())) - continue = true - else - acceptStatSep() - ts += makePackaging(start, pkg, topstats()) - } + possibleTemplateStart() + if in.token == EOF then + ts += makePackaging(start, pkg, List()) + else if in.isNestedStart then + ts += inDefScopeBraces(makePackaging(start, pkg, topStatSeq())) + continue = true + else + acceptStatSep() + ts += makePackaging(start, pkg, topstats()) if continue then - acceptStatSepUnlessAtEnd() + acceptStatSepUnlessAtEnd(ts) ts ++= topStatSeq() } else diff --git a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala index 4182c8795b2a..80d341eb5cce 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala @@ -89,9 +89,6 @@ object Scanners { /** the base of a number */ var base: Int = 0 - /** Set to false to disable end marker alignment checks, used for outline parsing. */ - var checkEndMarker: Boolean = true - def copyFrom(td: TokenData): Unit = { this.token = td.token this.offset = td.offset @@ -181,6 +178,8 @@ object Scanners { /** A switch whether operators at the start of lines can be infix operators */ private[Scanners] var allowLeadingInfixOperators = true + var debugTokenStream = false + val rewrite = ctx.settings.rewrite.value.isDefined val oldSyntax = ctx.settings.oldSyntax.value val newSyntax = ctx.settings.newSyntax.value @@ -265,9 +264,6 @@ object Scanners { /** The current region. This is initially an Indented region with indentation width. */ var currentRegion: Region = Indented(IndentWidth.Zero, Set(), EMPTY, null) - /** The number of open end marker scopes */ - var openEndMarkers: List[(EndMarkerTag, IndentWidth)] = Nil - // Get next token ------------------------------------------------------------ /** Are we directly in a multiline string interpolation expression? @@ -344,11 +340,11 @@ object Scanners { if (isAfterLineEnd) handleNewLine(lastToken) postProcessToken() - //printState() + printState() } - protected def printState() = - print("[" + show + "]") + final def printState() = + if debugTokenStream then print("[" + show + "]") /** Insert `token` at assumed `offset` in front of current one. */ def insert(token: Token, offset: Int) = { @@ -358,51 +354,6 @@ object Scanners { this.token = token } - /** What can be referred to in an end marker */ - type EndMarkerTag = TermName | Token - - /** Establish a scope for a passible end-marker with given tag, parsed by `op` */ - def endMarkerScope[T](tag: EndMarkerTag)(op: => T): T = - val saved = openEndMarkers - openEndMarkers = (tag, currentRegion.indentWidth) :: openEndMarkers - try op - finally openEndMarkers = saved - - /** If this token and the next constitute an end marker, skip them and check they - * align with an opening construct with the same end marker tag, - * unless `checkEndMarker` is false. - */ - protected def skipEndMarker(width: IndentWidth): Unit = - if next.token == IDENTIFIER && next.name == nme.end then - val lookahead = LookaheadScanner() - val start = lookahead.offset - - def handle(tag: EndMarkerTag) = - def checkAligned(): Unit = openEndMarkers match - case (etag, ewidth) :: rest if width <= ewidth => - if width < ewidth || tag != etag then - openEndMarkers = rest - checkAligned() - case _ => - if checkEndMarker - lexical.println(i"misaligned end marker $tag, $width, $openEndMarkers") - errorButContinue("misaligned end marker", start) - - val skipTo = lookahead.charOffset - lookahead.nextToken() - if lookahead.isAfterLineEnd || lookahead.token == EOF then - checkAligned() - next.token = EMPTY - while charOffset < skipTo do nextChar() - end handle - - lookahead.nextToken() // skip the `end` - lookahead.token match - case IDENTIFIER | BACKQUOTED_IDENT => handle(lookahead.name) - case IF | WHILE | FOR | MATCH | TRY | NEW | GIVEN => handle(lookahead.token) - case _ => - end skipEndMarker - /** A leading symbolic or backquoted identifier is treated as an infix operator if * - it does not follow a blank line, and * - it is followed on the same line by at least one ' ' @@ -535,7 +486,6 @@ object Scanners { && !isLeadingInfixOperator() then insert(if (pastBlankLine) NEWLINES else NEWLINE, lineOffset) - skipEndMarker(nextWidth) else if indentIsSignificant then if nextWidth < lastWidth || nextWidth == lastWidth && (indentPrefix == MATCH || indentPrefix == CATCH) && token != CASE then @@ -546,7 +496,6 @@ object Scanners { case r: Indented => currentRegion = r.enclosing insert(OUTDENT, offset) - skipEndMarker(nextWidth) case r: InBraces if !closingRegionTokens.contains(token) => ctx.warning("Line is indented too far to the left, or a `}` is missing", source.atSpan(Span(offset))) @@ -934,13 +883,7 @@ object Scanners { // Lookahead --------------------------------------------------------------- - class LookaheadScanner() extends Scanner(source, offset) { - override def skipEndMarker(width: IndentWidth) = () - override protected def printState() = { - print("la:") - super.printState() - } - } + class LookaheadScanner() extends Scanner(source, offset) /** Skip matching pairs of `(...)` or `[...]` parentheses. * @pre The current token is `(` or `[` @@ -1343,8 +1286,8 @@ object Scanners { def show: String = token match { case IDENTIFIER | BACKQUOTED_IDENT => s"id($name)" case CHARLIT => s"char($strVal)" - case INTLIT => s"int($strVal, $base)" - case LONGLIT => s"long($strVal, $base)" + case INTLIT => s"int($strVal, base = $base)" + case LONGLIT => s"long($strVal, base = $base)" case FLOATLIT => s"float($strVal)" case DOUBLELIT => s"double($strVal)" case STRINGLIT => s"string($strVal)" diff --git a/compiler/src/dotty/tools/dotc/parsing/Tokens.scala b/compiler/src/dotty/tools/dotc/parsing/Tokens.scala index 389407497ea9..b7e5501f1289 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Tokens.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Tokens.scala @@ -256,8 +256,7 @@ object Tokens extends TokensCommon { final val canStartStatTokens3: TokenSet = canStartExprTokens3 | mustStartStatTokens | BitSet( AT, CASE) - final val canEndStatTokens: TokenSet = atomicExprTokens | BitSet( - TYPE, RPAREN, RBRACE, RBRACKET, OUTDENT) // TODO: remove GIVEN once old import syntax is dropped + final val canEndStatTokens: TokenSet = atomicExprTokens | BitSet(TYPE, RPAREN, RBRACE, RBRACKET, OUTDENT) /** Tokens that stop a lookahead scan search for a `<-`, `then`, or `do`. * Used for disambiguating between old and new syntax. @@ -282,5 +281,7 @@ object Tokens extends TokensCommon { final val scala3keywords = BitSet(ENUM, ERASED, GIVEN) + final val endMarkerTokens = identifierTokens | BitSet(IF, WHILE, FOR, MATCH, TRY, NEW, GIVEN, VAL, THIS) + final val softModifierNames = Set(nme.inline, nme.opaque, nme.open, nme.transparent) } diff --git a/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala b/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala index f795781b05d9..56c6f5716b6d 100644 --- a/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala +++ b/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala @@ -50,9 +50,7 @@ object SyntaxHighlighting { else highlightRange(span.start, span.end, color) - val scanner = new Scanner(source) { - override protected def printState() = () - } + val scanner = new Scanner(source) while (scanner.token != EOF) { val start = scanner.offset val token = scanner.token diff --git a/compiler/src/dotty/tools/dotc/typer/FrontEnd.scala b/compiler/src/dotty/tools/dotc/typer/FrontEnd.scala index bcff04f10249..99be4cc440f8 100644 --- a/compiler/src/dotty/tools/dotc/typer/FrontEnd.scala +++ b/compiler/src/dotty/tools/dotc/typer/FrontEnd.scala @@ -53,6 +53,7 @@ class FrontEnd extends Phase { if (unit.isJava) new JavaParser(unit.source).parse() else { val p = new Parser(unit.source) + // p.in.debugTokenStream = true val tree = p.parse() if (p.firstXmlPos.exists && !firstXmlPos.exists) firstXmlPos = p.firstXmlPos diff --git a/docs/docs/internals/syntax.md b/docs/docs/internals/syntax.md index 96d0506c06d6..4b3867148b62 100644 --- a/docs/docs/internals/syntax.md +++ b/docs/docs/internals/syntax.md @@ -104,9 +104,8 @@ yield ### Soft keywords ``` -as derives extension inline on opaque open transparent -using -* + - +as derives end extension inline on opaque open transparent using +* + - ``` ## Context-free Syntax @@ -241,6 +240,7 @@ BlockStat ::= Import | {Annotation [nl]} [‘implicit’ | ‘lazy’] Def | {Annotation [nl]} {LocalModifier} TmplDef | Expr1 + | EndMarker ForExpr ::= ‘for’ (‘(’ Enumerators ‘)’ | ‘{’ Enumerators ‘}’) ForYield(enums, expr) {nl} [‘yield’] Expr @@ -344,6 +344,10 @@ ImportSelectors ::= id [‘=>’ id | ‘=>’ ‘_’] [‘,’ ImportSelect WildCardSelector ::= ‘given’ (‘_' | InfixType) | ‘_' Export ::= ‘export’ [‘given’] ImportExpr {‘,’ ImportExpr} + +EndMarker ::= ‘end’ EndMarkerTag -- when followed by EOL +EndMarkerTag ::= id | ‘if’ | ‘while’ | ‘for’ | ‘match’ | ‘try’ + | ‘new’ | ‘this’ | ‘given’ | ‘extension’ | ‘val’ ``` ### Declarations and Definitions @@ -406,6 +410,7 @@ TemplateStat ::= Import | {Annotation [nl]} {Modifier} Def | {Annotation [nl]} {Modifier} Dcl | Expr1 + | EndMarker | SelfType ::= id [‘:’ InfixType] ‘=>’ ValDef(_, name, tpt, _) | ‘this’ ‘:’ InfixType ‘=>’ @@ -422,6 +427,7 @@ TopStat ::= Import | {Annotation [nl]} {Modifier} Def | Packaging | PackageObject + | EndMarker | Packaging ::= ‘package’ QualId [nl | colonEol] ‘{’ TopStatSeq ‘}’ Package(qid, stats) PackageObject ::= ‘package’ ‘object’ ObjectDef object with package in mods. diff --git a/tests/neg/endmarkers.scala b/tests/neg/endmarkers.scala index 71d080a0c379..5c7c1c686cd0 100644 --- a/tests/neg/endmarkers.scala +++ b/tests/neg/endmarkers.scala @@ -9,7 +9,7 @@ object Test: x += 1 x < 10 do () - end while // error: misaligned end marker + end while // warning: line indented too far to the left } def f(x: Int): Int = diff --git a/tests/neg/endmarkers1.scala b/tests/neg/endmarkers1.scala index 9de2f9746522..b2b746c57371 100644 --- a/tests/neg/endmarkers1.scala +++ b/tests/neg/endmarkers1.scala @@ -14,5 +14,5 @@ end Test // error: misaligned end marker def f[T](x: Option[T]) = x match case Some(y) => case None => "hello" - end f // error: misaligned end marker + end f // error: The start of this line does not match any of the previous indentation widths. diff --git a/tests/neg/i8731.scala b/tests/neg/i8731.scala new file mode 100644 index 000000000000..bda0a4e7325e --- /dev/null +++ b/tests/neg/i8731.scala @@ -0,0 +1,29 @@ +object test: + + 3 match + case 3 => ??? + end match + case _ => () // error: missing parameter type + end match + + if 3 == 3 + () + end if + else // error: illegal start of definition + () + end if + + class Test { + val test = 3 + end Test // error: misaligned end marker + } + + while + 3 == 3 + end while // error: `do` expected + do () + + for + a <- Seq() + end for // error: `yield` or `do` expected + do () \ No newline at end of file diff --git a/tests/pos/endmarkers.scala b/tests/pos/endmarkers.scala new file mode 100644 index 000000000000..0433f373fd1d --- /dev/null +++ b/tests/pos/endmarkers.scala @@ -0,0 +1,12 @@ +trait T: + object O: + def foo = + 1 + end foo + def bar = 2 + end bar + end O +end T + +object T +end T \ No newline at end of file From 25b11486f067ab5cd4677a2e7e7b5e276fec549f Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Sun, 19 Apr 2020 16:30:39 +0200 Subject: [PATCH 3/7] Fixes - recognize end markers after toplevel package objects - don't let newline insertion be confused by endmarker tokens --- .../dotty/tools/dotc/parsing/Parsers.scala | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala index 68b58c653c82..82dd46fb6a11 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala @@ -365,18 +365,16 @@ object Parsers { def acceptStatSepUnlessAtEnd(stats: ListBuffer[Tree], altEnd: Token = EOF): Unit = in.observeOutdented() - if (!isStatSeqEnd) - in.token match { - case EOF => - case `altEnd` => - case SEMI | NEWLINE | NEWLINES => - in.nextToken() - checkEndMarker(stats) - case _ => + in.token match + case SEMI | NEWLINE | NEWLINES => + in.nextToken() + checkEndMarker(stats) + case `altEnd` => + case _ => + if !isStatSeqEnd then syntaxError(i"end of statement expected but ${showToken(in.token)} found") in.nextToken() // needed to ensure progress; otherwise we might cycle forever accept(SEMI) - } def rewriteNotice(additionalOption: String = "") = { val optionStr = if (additionalOption.isEmpty) "" else " " ++ additionalOption @@ -1327,6 +1325,7 @@ object Parsers { in.nextToken() if stats.isEmpty || !matches(stats.last) then syntaxError("misaligned end marker", Span(start, in.lastCharOffset)) + in.token = IDENTIFIER in.nextToken() end checkEndMarker @@ -3891,7 +3890,7 @@ object Parsers { in.nextToken() ts += objectDef(start, Modifiers(Package)) if (in.token != EOF) { - acceptStatSep() + acceptStatSepUnlessAtEnd(ts) ts ++= topStatSeq() } } From 7f4c828d097beb7fd84a46151fc4aee14d243126 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Sun, 19 Apr 2020 16:33:15 +0200 Subject: [PATCH 4/7] Adapt doc page to new scheme --- .../other-new-features/indentation.md | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/docs/reference/other-new-features/indentation.md b/docs/docs/reference/other-new-features/indentation.md index ce41be1f63cc..a76523a06edd 100644 --- a/docs/docs/reference/other-new-features/indentation.md +++ b/docs/docs/reference/other-new-features/indentation.md @@ -198,15 +198,21 @@ def largeMethod(...) = ... // more code end largeMethod ``` -An `end` marker consists of the identifier `end` which follows an `` token, and is in turn followed on the same line by exactly one other token, which is either an identifier or one of the reserved words +An `end` marker consists of the identifier `end` and a follow-on specifier token that together constitute all the tokes of a line. Possible specifier tokens are +identifiers or one of the following keywords ```scala -if while for match try new given extension +if while for match try new this val given ``` -If `end` is followed by a reserved word, the compiler checks that the marker closes an indentation region belonging to a construct that starts with the reserved word. If it is followed by an identifier _id_, the compiler checks that the marker closes a definition -that defines _id_ or a package clause that refers to _id_. - -`end` itself is a soft keyword. It is only treated as an `end` marker if it -occurs at the start of a line and is followed by an identifier or one of the reserved words above. +End markers are allowed in statement sequences. The specifier token `s` of an end marker must correspond to the statement that precedes it. This means: + + - If the statement defines a member `x` then `s` must be the same identifier `x`. + - If the statement defines a constructor then `s` must be `this`. + - If the statement defines an anonymous given, then `s` must be `given`. + - If the statement defines an anonymous extension, then `s` must be `extension`. + - If the statement defines an anonymous class, then `s` must be `new`. + - If the statement is a `val` definition binding a pattern, then `s` must be `val`. + - If the statement is a package clause that refers to package `p`, then `s` must be the same identifier `p`. + - If the statement is an `if`, `while`, `for`, `try`, or `match` statement, then `s` must be that same token. It is recommended that `end` markers are used for code where the extent of an indentation region is not immediately apparent "at a glance". Typically this is the case if an indentation region spans 20 lines or more. From 8062c41882a5f6e53fdaa923598896bfa05cf68d Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Sun, 19 Apr 2020 17:09:38 +0200 Subject: [PATCH 5/7] Improve docs --- compiler/src/dotty/tools/dotc/parsing/Parsers.scala | 2 +- docs/docs/reference/other-new-features/indentation.md | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala index 82dd46fb6a11..751f814d42d4 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala @@ -1325,7 +1325,7 @@ object Parsers { in.nextToken() if stats.isEmpty || !matches(stats.last) then syntaxError("misaligned end marker", Span(start, in.lastCharOffset)) - in.token = IDENTIFIER + in.token = IDENTIFIER // Leaving it as the original token can confuse newline insertion in.nextToken() end checkEndMarker diff --git a/docs/docs/reference/other-new-features/indentation.md b/docs/docs/reference/other-new-features/indentation.md index a76523a06edd..264d7a1eda07 100644 --- a/docs/docs/reference/other-new-features/indentation.md +++ b/docs/docs/reference/other-new-features/indentation.md @@ -216,6 +216,17 @@ End markers are allowed in statement sequences. The specifier token `s` of an en It is recommended that `end` markers are used for code where the extent of an indentation region is not immediately apparent "at a glance". Typically this is the case if an indentation region spans 20 lines or more. +**Syntax** + +``` +EndMarker ::= ‘end’ EndMarkerTag -- when followed by EOL +EndMarkerTag ::= id | ‘if’ | ‘while’ | ‘for’ | ‘match’ | ‘try’ + | ‘new’ | ‘this’ | ‘given’ | ‘extension’ | ‘val’ +BlockStat ::= ... | EndMarker +TemplateStat ::= ... | EndMarker +TopStat ::= ... | EndMarker +``` + ### Example Here is a (somewhat meta-circular) example of code using indentation. It provides a concrete representation of indentation widths as defined above together with efficient operations for constructing and comparing indentation widths. From 97112c2f3fe6e6725b490b8935056dd21fa33d00 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Sun, 19 Apr 2020 17:59:10 +0200 Subject: [PATCH 6/7] More docs on endmarkers --- .../other-new-features/indentation.md | 63 ++++++++++++++++++- tests/pos/endmarkers.scala | 48 +++++++++++++- 2 files changed, 107 insertions(+), 4 deletions(-) diff --git a/docs/docs/reference/other-new-features/indentation.md b/docs/docs/reference/other-new-features/indentation.md index 264d7a1eda07..6d5c08f6c4ff 100644 --- a/docs/docs/reference/other-new-features/indentation.md +++ b/docs/docs/reference/other-new-features/indentation.md @@ -187,7 +187,7 @@ println(".") Indentation-based syntax has many advantages over other conventions. But one possible problem is that it makes it hard to discern when a large indentation region ends, since there is no specific token that delineates the end. Braces are not much better since a brace by itself also contains no information about what region is closed. -To solve this problem, Scala 3 offers an optional `end` marker. Example +To solve this problem, Scala 3 offers an optional `end` marker. Example: ```scala def largeMethod(...) = ... @@ -214,9 +214,66 @@ End markers are allowed in statement sequences. The specifier token `s` of an en - If the statement is a package clause that refers to package `p`, then `s` must be the same identifier `p`. - If the statement is an `if`, `while`, `for`, `try`, or `match` statement, then `s` must be that same token. -It is recommended that `end` markers are used for code where the extent of an indentation region is not immediately apparent "at a glance". Typically this is the case if an indentation region spans 20 lines or more. +For instance, the following end markers are all legal: + ```scala + package p1.p2: + + abstract class C(): + + def this(x: Int) = + this() + if x > 0 then + val a :: b = + x :: Nil + end val + var y = + x + end y + while y > 0 do + println(y) + y -= 1 + end while + try + x match + case 0 => println("0") + case _ => + end match + finally + println("done") + end try + end if + end this + + def f: String + end C + + object C: + given C = + new C: + def f = "!" + end f + end new + end given + end C + + extension on (x: C): + def ff: String = x.f ++ x.f + end extension + + end p2 +``` + +#### When to Use End Markers + +It is recommended that `end` markers are used for code where the extent of an indentation region is not immediately apparent "at a glance". People will have different preferences what this means, but one can nevertheless give some guidelines that stem from experience. An end marker makes sense if + + - the construct contains blank lines, or + - the construct is long, say 15-20 lines or more, + - the construct ends heavily indented, say 4 indentation levels or more. + +If none of these criteria apply, it's often better to not use an end marker since the code will be just as clear and more concise. If there are several ending regions that satisfy one of the criteria above, we usually need an end marker only for the outermost closed reason. So cascades of end markers as in the example above are usually better avoided. -**Syntax** +#### Syntax ``` EndMarker ::= ‘end’ EndMarkerTag -- when followed by EOL diff --git a/tests/pos/endmarkers.scala b/tests/pos/endmarkers.scala index 0433f373fd1d..f889e7d63f12 100644 --- a/tests/pos/endmarkers.scala +++ b/tests/pos/endmarkers.scala @@ -9,4 +9,50 @@ trait T: end T object T -end T \ No newline at end of file +end T + +package p1.p2: + + abstract class C(): + + def this(x: Int) = + this() + if x > 0 then + val a :: b = + x :: Nil + end val + var y = + x + end y + while y > 0 do + println(y) + y -= 1 + end while + try + x match + case 0 => println("0") + case _ => + end match + finally + println("done") + end try + end if + end this + + def f: String + end C + + object C: + given C = + new C: + def f = "!" + end f + end new + end given + end C + + extension on (x: C): + def ff: String = x.f + end extension + +end p2 \ No newline at end of file From d7900c580814963b12e223e22ef593da9a24df8a Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Mon, 20 Apr 2020 10:13:29 +0200 Subject: [PATCH 7/7] Fix indentation width in example from 4 to 2 --- .../docs/reference/other-new-features/indentation.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/docs/reference/other-new-features/indentation.md b/docs/docs/reference/other-new-features/indentation.md index 6d5c08f6c4ff..d3b5db2a38fc 100644 --- a/docs/docs/reference/other-new-features/indentation.md +++ b/docs/docs/reference/other-new-features/indentation.md @@ -190,12 +190,12 @@ Indentation-based syntax has many advantages over other conventions. But one pos To solve this problem, Scala 3 offers an optional `end` marker. Example: ```scala def largeMethod(...) = - ... - if ... then ... - else - ... // a large block - end if - ... // more code + ... + if ... then ... + else + ... // a large block + end if + ... // more code end largeMethod ``` An `end` marker consists of the identifier `end` and a follow-on specifier token that together constitute all the tokes of a line. Possible specifier tokens are