Skip to content

Commit c5a37f8

Browse files
committed
Initial implementation featuring two different highlighters
One was implemted by hand and the other by using dotty's parser. The one built by hand is shorter, and behaves correctly. The scanner one is unfortunately not ready for testing - there are too many things that are workarounds for it to be a good solution as of now The code added from Ammonite is licensed under MIT, not sure where to put the license - but will add it once I know.
1 parent b56f5c9 commit c5a37f8

22 files changed

+2523
-26
lines changed

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
*.class
33
*.log
44
*~
5+
*.swp
56

67
# sbt specific
78
dist/*

src/dotty/tools/dotc/parsing/Scanners.scala

Lines changed: 38 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,9 @@ object Scanners {
4545
/** the string value of a literal */
4646
var strVal: String = null
4747

48+
/** the started parsing of a literal */
49+
var startedLiteral: String = null
50+
4851
/** the base of a number */
4952
var base: Int = 0
5053

@@ -174,8 +177,13 @@ object Scanners {
174177

175178
}
176179

177-
class Scanner(source: SourceFile, override val startFrom: Offset = 0)(implicit ctx: Context) extends ScannerCommon(source)(ctx) {
178-
val keepComments = ctx.settings.YkeepComments.value
180+
class Scanner(
181+
source: SourceFile,
182+
override val startFrom: Offset = 0,
183+
preserveWhitespace: Boolean = false
184+
)(implicit ctx: Context) extends ScannerCommon(source)(ctx) {
185+
val keepComments = ctx.settings.YkeepComments.value
186+
val whitespace = new StringBuilder
179187

180188
/** All doc comments as encountered, each list contains doc comments from
181189
* the same block level. Starting with the deepest level and going upward
@@ -239,13 +247,13 @@ object Scanners {
239247

240248
/** Are we directly in a string interpolation expression?
241249
*/
242-
private def inStringInterpolation =
250+
def inStringInterpolation =
243251
sepRegions.nonEmpty && sepRegions.head == STRINGLIT
244252

245253
/** Are we directly in a multiline string interpolation expression?
246254
* @pre inStringInterpolation
247255
*/
248-
private def inMultiLineInterpolation =
256+
def inMultiLineInterpolation =
249257
inStringInterpolation && sepRegions.tail.nonEmpty && sepRegions.tail.head == STRINGPART
250258

251259
/** read next token and return last offset
@@ -316,7 +324,7 @@ object Scanners {
316324
token = if (pastBlankLine()) NEWLINES else NEWLINE
317325
}
318326

319-
postProcessToken()
327+
if (!preserveWhitespace) postProcessToken()
320328
// print("[" + this +"]")
321329
}
322330

@@ -375,9 +383,20 @@ object Scanners {
375383
offset = charOffset - 1
376384
(ch: @switch) match {
377385
case ' ' | '\t' | CR | LF | FF =>
378-
nextChar()
379-
fetchToken()
380-
case 'A' | 'B' | 'C' | 'D' | 'E' |
386+
if (preserveWhitespace) {
387+
while ((' ' :: '\t' :: CR :: LF :: FF :: Nil) contains ch) {
388+
whitespace += ch
389+
nextChar()
390+
}
391+
token = WHITESPACE
392+
strVal = whitespace.toString
393+
whitespace.clear()
394+
} else {
395+
nextChar()
396+
fetchToken()
397+
}
398+
case c @ (
399+
'A' | 'B' | 'C' | 'D' | 'E' |
381400
'F' | 'G' | 'H' | 'I' | 'J' |
382401
'K' | 'L' | 'M' | 'N' | 'O' |
383402
'P' | 'Q' | 'R' | 'S' | 'T' |
@@ -388,12 +407,14 @@ object Scanners {
388407
'k' | 'l' | 'm' | 'n' | 'o' |
389408
'p' | 'q' | 'r' | 's' | 't' |
390409
'u' | 'v' | 'w' | 'x' | 'y' |
391-
'z' =>
410+
'z') =>
392411
putChar(ch)
393412
nextChar()
394413
getIdentRest()
395-
if (ch == '"' && token == IDENTIFIER)
414+
if (ch == '"' && token == IDENTIFIER) {
396415
token = INTERPOLATIONID
416+
startedLiteral = "\""
417+
}
397418
case '<' => // is XMLSTART?
398419
def fetchLT() = {
399420
val last = if (charOffset >= 2) buf(charOffset - 2) else ' '
@@ -494,9 +515,11 @@ object Scanners {
494515
getLitChar()
495516
if (ch == '\'') {
496517
nextChar()
518+
startedLiteral = null
497519
token = CHARLIT
498520
setStrVal()
499521
} else {
522+
startedLiteral = "\'"
500523
error("unclosed character literal")
501524
}
502525
}
@@ -686,8 +709,12 @@ object Scanners {
686709
if (ch == '"') {
687710
setStrVal()
688711
nextChar()
712+
startedLiteral = null
689713
token = STRINGLIT
690-
} else error("unclosed string literal")
714+
} else {
715+
startedLiteral = "\""
716+
error("unclosed string literal")
717+
}
691718
}
692719

693720
private def getRawStringLit(): Unit = {

src/dotty/tools/dotc/parsing/Tokens.scala

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,7 @@ abstract class TokensCommon {
141141

142142
object Tokens extends TokensCommon {
143143
final val minToken = EMPTY
144-
final val maxToken = XMLSTART
144+
final val maxToken = COMMENT
145145

146146
final val INTERPOLATIONID = 10; enter(INTERPOLATIONID, "string interpolator")
147147
final val SYMBOLLIT = 11; enter(SYMBOLLIT, "symbol literal") // TODO: deprecate
@@ -188,6 +188,11 @@ object Tokens extends TokensCommon {
188188
/** XML mode */
189189
final val XMLSTART = 96; enter(XMLSTART, "$XMLSTART$<") // TODO: deprecate
190190

191+
/** Whitespace */
192+
final val WHITESPACE = 97; enter(WHITESPACE, "whitespace")
193+
final val COMMENT = 98; enter(COMMENT, "comment")
194+
195+
191196
final val alphaKeywords = tokenRange(IF, FORSOME)
192197
final val symbolicKeywords = tokenRange(USCORE, VIEWBOUND)
193198
final val symbolicTokens = tokenRange(COMMA, VIEWBOUND)
Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
package dotty.tools
2+
package dotc
3+
package repl
4+
5+
import core.Contexts._
6+
import ammonite.terminal._
7+
import LazyList._
8+
import Ansi.Color
9+
import filters._
10+
import BasicFilters._
11+
import GUILikeFilters._
12+
import util.SourceFile
13+
14+
class AmmoniteReader extends InteractiveReader {
15+
val interactive = true
16+
17+
val reader = new java.io.InputStreamReader(System.in)
18+
val writer = new java.io.OutputStreamWriter(System.out)
19+
val cutPasteFilter = ReadlineFilters.CutPasteFilter()
20+
var history = List.empty[String]
21+
val selectionFilter = GUILikeFilters.SelectionFilter(indent = 2)
22+
val multilineFilter: Filter = Filter("multilineFilter") {
23+
case TermState(lb ~: rest, b, c, _)
24+
if (lb == 10 || lb == 13) => // Enter
25+
26+
BasicFilters.injectNewLine(b, c, rest)
27+
}
28+
def readLine(prompt: String)(implicit ctx: Context): String = {
29+
val historyFilter = new HistoryFilter(
30+
() => history.toVector,
31+
Console.BLUE,
32+
AnsiNav.resetForegroundColor
33+
)
34+
35+
val allFilters = Filter.merge(
36+
UndoFilter(),
37+
historyFilter,
38+
selectionFilter,
39+
GUILikeFilters.altFilter,
40+
GUILikeFilters.fnFilter,
41+
ReadlineFilters.navFilter,
42+
//autocompleteFilter,
43+
cutPasteFilter,
44+
//multilineFilter,
45+
BasicFilters.all
46+
)
47+
48+
Terminal.readLine(
49+
Console.BLUE + prompt + Console.RESET,
50+
reader,
51+
writer,
52+
allFilters,
53+
displayTransform = (buffer, cursor) => {
54+
val ansiBuffer = Ansi.Str.parse(SyntaxHighlighting(buffer))
55+
//val ansiBuffer = Ansi.Str.parse(SyntaxHighlighting(new SourceFile("<console>", buffer)))
56+
val (newBuffer, cursorOffset) = SelectionFilter.mangleBuffer(
57+
selectionFilter, ansiBuffer, cursor, Ansi.Reversed.On
58+
)
59+
val newNewBuffer = HistoryFilter.mangleBuffer(
60+
historyFilter, newBuffer, cursor,
61+
Ansi.Color.Green
62+
)
63+
64+
(newNewBuffer, cursorOffset)
65+
}
66+
) match {
67+
case Some(s) => history = s :: history; s
68+
case None => ":q"
69+
}
70+
}
71+
}

src/dotty/tools/dotc/repl/CompilingInterpreter.scala

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -727,10 +727,10 @@ class CompilingInterpreter(out: PrintWriter, ictx: Context) extends Compiler wit
727727
return str
728728

729729
val trailer = "..."
730-
if (maxpr >= trailer.length+1)
731-
return str.substring(0, maxpr-3) + trailer
732-
733-
str.substring(0, maxpr)
730+
if (maxpr >= trailer.length-1)
731+
str.substring(0, maxpr-3) + trailer + "\n"
732+
else
733+
str.substring(0, maxpr-1)
734734
}
735735

736736
/** Clean up a string for output */

src/dotty/tools/dotc/repl/InteractiveReader.scala

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,11 @@ package dotty.tools
22
package dotc
33
package repl
44

5+
import dotc.core.Contexts.Context
6+
57
/** Reads lines from an input stream */
68
trait InteractiveReader {
7-
def readLine(prompt: String): String
9+
def readLine(prompt: String)(implicit ctx: Context): String
810
val interactive: Boolean
911
}
1012

@@ -16,11 +18,12 @@ object InteractiveReader {
1618
* SimpleReader. */
1719
def createDefault(): InteractiveReader = {
1820
try {
19-
new JLineReader()
20-
} catch {
21-
case e =>
22-
//out.println("jline is not available: " + e) //debug
23-
new SimpleReader()
21+
new AmmoniteReader()
22+
} catch { case e =>
23+
//out.println("jline is not available: " + e) //debug
24+
e.printStackTrace()
25+
println("Could not use ammonite, falling back to simple reader")
26+
new SimpleReader()
2427
}
2528
}
2629
}

src/dotty/tools/dotc/repl/JLineReader.scala

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package dotty.tools
22
package dotc
33
package repl
44

5+
import dotc.core.Contexts.Context
56
import jline.console.ConsoleReader
67

78
/** Adaptor for JLine
@@ -11,5 +12,5 @@ class JLineReader extends InteractiveReader {
1112

1213
val interactive = true
1314

14-
def readLine(prompt: String) = reader.readLine(prompt)
15+
def readLine(prompt: String)(implicit ctx: Context) = reader.readLine(prompt)
1516
}

src/dotty/tools/dotc/repl/SimpleReader.scala

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package dotc
33
package repl
44

55
import java.io.{BufferedReader, PrintWriter}
6+
import dotc.core.Contexts.Context
67

78

89
/** Reads using standard JDK API */
@@ -13,7 +14,7 @@ class SimpleReader(
1314
extends InteractiveReader {
1415
def this() = this(Console.in, new PrintWriter(Console.out), true)
1516

16-
def readLine(prompt: String) = {
17+
def readLine(prompt: String)(implicit ctx: Context) = {
1718
if (interactive) {
1819
out.print(prompt)
1920
out.flush()

0 commit comments

Comments
 (0)