From 60e5494fe2f3c79d743c5fbc987e7382f0830cad Mon Sep 17 00:00:00 2001 From: Jentsch Date: Mon, 29 Apr 2019 14:04:34 +0200 Subject: [PATCH 1/3] Accept underscore as numeric literal separator The separator character is underscore as in Java. They may be mixed arbitrarily in a numeric literal, but only in the interior: the literal, and each part of double, must begin and end with a digit. Add spec for underscore numeric separator. This patch is a modification of a commit from som-snytt https://github.com/scala/scala/pull/6989/commits/c3591880d809985fd3de96462bf2742823b66234 --- .../dotty/tools/dotc/parsing/Scanners.scala | 65 +++++++++++++------ tests/neg/t6124.check | 20 ++++++ tests/neg/t6124.scala | 16 +++++ tests/pos/t6124.scala | 11 ++++ 4 files changed, 93 insertions(+), 19 deletions(-) create mode 100644 tests/neg/t6124.check create mode 100644 tests/neg/t6124.scala create mode 100644 tests/pos/t6124.scala diff --git a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala index b4cf20536bb6..839266d40a27 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala @@ -67,11 +67,15 @@ object Scanners { /** Generate an error at the given offset */ def error(msg: String, off: Offset = offset): Unit = { - ctx.error(msg, source atSpan Span(off)) + errorButContinue(msg, off) token = ERROR errOffset = off } + def errorButContinue(msg: String, off: Offset = offset): Unit = { + ctx.error(msg, source atSpan Span(off)) + } + /** signal an error where the input ended in the middle of a token */ def incompleteInputError(msg: String): Unit = { ctx.incompleteInputError(msg, source atSpan Span(offset)) @@ -129,19 +133,22 @@ object Scanners { var i = 0 val len = strVal.length while (i < len) { - val d = digit2int(strVal charAt i, base) - if (d < 0) { - error("malformed integer number") - return 0 - } - if (value < 0 || - limit / (base / divider) < value || - limit - (d / divider) < value * (base / divider) && - !(negated && limit == value * base - 1 + d)) { - error("integer number too large") - return 0 + val c = strVal charAt i + if (! isNumberSeparator(c)) { + val d = digit2int(c, base) + if (d < 0) { + error(s"malformed integer number") + return 0 + } + if (value < 0 || + limit / (base / divider) < value || + limit - (d / divider) < value * (base / divider) && + !(negated && limit == value * base - 1 + d)) { + error("integer number too large") + return 0 + } + value = value * base + d } - value = value * base + d i += 1 } if (negated) -value else value @@ -153,10 +160,11 @@ object Scanners { /** Convert current strVal, base to double value */ def floatVal(negated: Boolean): Double = { + val text = removeNumberSeparators(strVal) val limit: Double = if (token == DOUBLELIT) Double.MaxValue else Float.MaxValue try { - val value: Double = java.lang.Double.valueOf(strVal).doubleValue() + val value: Double = java.lang.Double.valueOf(text).doubleValue() if (value > limit) error("floating point number too large") if (negated) -value else value @@ -169,6 +177,17 @@ object Scanners { def floatVal: Double = floatVal(false) + @inline def isNumberSeparator(c: Char): Boolean = c == '_' + + @inline def removeNumberSeparators(s: String): String = + if (s.indexOf('_') > 0) s.replaceAllLiterally("_", "") /*.replaceAll("'","")*/ else s + + // disallow trailing numeric separator char, but continue lexing + def checkNoTrailingSeparator(): Unit = { + if (isNumberSeparator(litBuf.last)) + errorButContinue("trailing separator is not allowed", offset + litBuf.length - 1) + } + } class Scanner(source: SourceFile, override val startFrom: Offset = 0)(implicit ctx: Context) extends ScannerCommon(source)(ctx) { @@ -911,27 +930,29 @@ object Scanners { */ protected def getFraction(): Unit = { token = DOUBLELIT - while ('0' <= ch && ch <= '9') { + while ('0' <= ch && ch <= '9' || isNumberSeparator(ch)) { putChar(ch) nextChar() } + checkNoTrailingSeparator() if (ch == 'e' || ch == 'E') { val lookahead = lookaheadReader() lookahead.nextChar() if (lookahead.ch == '+' || lookahead.ch == '-') { lookahead.nextChar() } - if ('0' <= lookahead.ch && lookahead.ch <= '9') { + if ('0' <= lookahead.ch && lookahead.ch <= '9' || isNumberSeparator(ch)) { putChar(ch) nextChar() if (ch == '+' || ch == '-') { putChar(ch) nextChar() } - while ('0' <= ch && ch <= '9') { + while ('0' <= ch && ch <= '9' || isNumberSeparator(ch)) { putChar(ch) nextChar() } + checkNoTrailingSeparator() } token = DOUBLELIT } @@ -954,15 +975,18 @@ object Scanners { /** Read a number into strVal and set base */ protected def getNumber(): Unit = { - while (digit2int(ch, base) >= 0) { + while (isNumberSeparator(ch) || digit2int(ch, base) >= 0) { putChar(ch) nextChar() } + checkNoTrailingSeparator() token = INTLIT if (base == 10 && ch == '.') { val lch = lookaheadChar() if ('0' <= lch && lch <= '9') { - putChar('.'); nextChar(); getFraction() + putChar('.') + nextChar() + getFraction() } } else (ch: @switch) match { case 'e' | 'E' | 'f' | 'F' | 'd' | 'D' => @@ -972,6 +996,9 @@ object Scanners { token = LONGLIT case _ => } + + checkNoTrailingSeparator() + setStrVal() } diff --git a/tests/neg/t6124.check b/tests/neg/t6124.check new file mode 100644 index 000000000000..4374aab84c34 --- /dev/null +++ b/tests/neg/t6124.check @@ -0,0 +1,20 @@ +<273..273> in t6124.scala +trailing separator is not allowed +<235..235> in t6124.scala +Invalid literal number +<240..240> in t6124.scala +trailing separator is not allowed +<213..213> in t6124.scala +trailing separator is not allowed +<187..187> in t6124.scala +trailing separator is not allowed +<160..160> in t6124.scala +trailing separator is not allowed +<125..125> in t6124.scala +Invalid literal number +<101..101> in t6124.scala +trailing separator is not allowed +<67..67> in t6124.scala +trailing separator is not allowed +<33..33> in t6124.scala +trailing separator is not allowed diff --git a/tests/neg/t6124.scala b/tests/neg/t6124.scala new file mode 100644 index 000000000000..dbde52f911a4 --- /dev/null +++ b/tests/neg/t6124.scala @@ -0,0 +1,16 @@ + +trait T { + def i: Int = 123_456_ // error + def j: Long = 123_456_L * 1000 // error + + def f = 3_14_E-2 // error + def e = 3_14E-_2 // error + def d = 3_14E-2_ // error + + def p = 3.1_4_ // error + def q = 3.1_4_d // error + def r = 3.1_4_dd // error // error + def s = 3_.1 // error + + def z = 0 +} \ No newline at end of file diff --git a/tests/pos/t6124.scala b/tests/pos/t6124.scala new file mode 100644 index 000000000000..f83d96330a0e --- /dev/null +++ b/tests/pos/t6124.scala @@ -0,0 +1,11 @@ + +trait T { + def i: Int = 1_024 + def j: Long = 1_024L * 1_024 + //def k = 1'024 + + def f = 3_14e-2f + def d = 3_14E-2_1 + + def z = 0 +} \ No newline at end of file From 12d5f3d73193c9930a584f44eef27482ee2bcf62 Mon Sep 17 00:00:00 2001 From: Jentsch Date: Thu, 2 May 2019 13:03:03 +0200 Subject: [PATCH 2/3] Add check that floating point numbers are not too small --- .../tools/dotc/parsing/JavaScanners.scala | 2 +- .../dotty/tools/dotc/parsing/Parsers.scala | 2 +- .../dotty/tools/dotc/parsing/Scanners.scala | 40 +++++++++++++++---- tests/neg/t6124.check | 2 + tests/neg/t6124.scala | 2 + 5 files changed, 39 insertions(+), 9 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/parsing/JavaScanners.scala b/compiler/src/dotty/tools/dotc/parsing/JavaScanners.scala index 0c415d813889..0452b9f6e672 100644 --- a/compiler/src/dotty/tools/dotc/parsing/JavaScanners.scala +++ b/compiler/src/dotty/tools/dotc/parsing/JavaScanners.scala @@ -521,7 +521,7 @@ object JavaScanners { case FLOATLIT => "float(" + floatVal + ")" case DOUBLELIT => - "double(" + floatVal + ")" + "double(" + doubleVal + ")" case STRINGLIT => "string(" + name + ")" case SEMI => diff --git a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala index aa63467ad1b0..59af74177316 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala @@ -726,7 +726,7 @@ object Parsers { case INTLIT => in.intVal(isNegated).toInt case LONGLIT => in.intVal(isNegated) case FLOATLIT => in.floatVal(isNegated).toFloat - case DOUBLELIT => in.floatVal(isNegated) + case DOUBLELIT => in.doubleVal(isNegated) case STRINGLIT | STRINGPART => in.strVal case TRUE => true case FALSE => false diff --git a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala index 839266d40a27..a26ede38c6b6 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala @@ -157,16 +157,42 @@ object Scanners { def intVal: Long = intVal(false) + private val zeroFloat = raw"[0.]+(?:[eE][+-]?[0-9]+)?[fFdD]?".r + /** Convert current strVal, base to double value */ - def floatVal(negated: Boolean): Double = { + def floatVal(negated: Boolean): Float = { + assert(token == FLOATLIT) + val text = removeNumberSeparators(strVal) + try { + val value: Float = java.lang.Float.valueOf(text).floatValue() + if (value > Float.MaxValue) + errorButContinue("floating point number too large") + + if (value == 0.0f && !zeroFloat.pattern.matcher(text).matches) + errorButContinue("floating point number too small") + if (negated) -value else value + } catch { + case _: NumberFormatException => + error("malformed floating point number") + 0.0f + } + } + + def floatVal: Float = floatVal(false) + + /** Convert current strVal, base to double value + */ + def doubleVal(negated: Boolean): Double = { + assert(token == DOUBLELIT) val text = removeNumberSeparators(strVal) - val limit: Double = - if (token == DOUBLELIT) Double.MaxValue else Float.MaxValue try { val value: Double = java.lang.Double.valueOf(text).doubleValue() - if (value > limit) - error("floating point number too large") + if (value > Double.MaxValue) + errorButContinue("double precision floating point number too large") + + if (value == 0.0d && !zeroFloat.pattern.matcher(text).matches) + errorButContinue("double precision floating point number too small") if (negated) -value else value } catch { case _: NumberFormatException => @@ -175,7 +201,7 @@ object Scanners { } } - def floatVal: Double = floatVal(false) + def doubleVal: Double = doubleVal(false) @inline def isNumberSeparator(c: Char): Boolean = c == '_' @@ -1034,7 +1060,7 @@ object Scanners { case INTLIT => s"int($intVal)" case LONGLIT => s"long($intVal)" case FLOATLIT => s"float($floatVal)" - case DOUBLELIT => s"double($floatVal)" + case DOUBLELIT => s"double($doubleVal)" case STRINGLIT => s"string($strVal)" case STRINGPART => s"stringpart($strVal)" case INTERPOLATIONID => s"interpolationid($name)" diff --git a/tests/neg/t6124.check b/tests/neg/t6124.check index 4374aab84c34..9150e6c311c3 100644 --- a/tests/neg/t6124.check +++ b/tests/neg/t6124.check @@ -1,3 +1,5 @@ +<304..304> in t6124.scala +double precision floating point number too small <273..273> in t6124.scala trailing separator is not allowed <235..235> in t6124.scala diff --git a/tests/neg/t6124.scala b/tests/neg/t6124.scala index dbde52f911a4..78c74d63642b 100644 --- a/tests/neg/t6124.scala +++ b/tests/neg/t6124.scala @@ -12,5 +12,7 @@ trait T { def r = 3.1_4_dd // error // error def s = 3_.1 // error + def tooSmall = 1.0E-325 // error + def z = 0 } \ No newline at end of file From 11d533a64da6c975c8b7113b8a7f0ff4b0667e6b Mon Sep 17 00:00:00 2001 From: Jentsch Date: Thu, 2 May 2019 13:37:24 +0200 Subject: [PATCH 3/3] Add java examples and fix uncovered underscore case in hex literals Removed redundant test cases https://docs.oracle.com/javase/8/docs/technotes/guides/language/underscores-literals.html --- .../dotty/tools/dotc/parsing/Scanners.scala | 2 + tests/neg/t6124.check | 38 ++++++++++++------- tests/neg/t6124.scala | 17 +++++++-- tests/pos/t6124.scala | 7 ++++ 4 files changed, 48 insertions(+), 16 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala index a26ede38c6b6..895bf25fb03d 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala @@ -508,6 +508,8 @@ object Scanners { if (ch == 'x' || ch == 'X') { nextChar() base = 16 + if (isNumberSeparator(ch)) + errorButContinue("leading separator is not allowed", offset + 2) } else { /** * What should leading 0 be in the future? It is potentially dangerous diff --git a/tests/neg/t6124.check b/tests/neg/t6124.check index 9150e6c311c3..bf7a0ba4e68a 100644 --- a/tests/neg/t6124.check +++ b/tests/neg/t6124.check @@ -1,22 +1,34 @@ -<304..304> in t6124.scala -double precision floating point number too small -<273..273> in t6124.scala +[490..493] in t6124.scala +Not found: _52 +[404..406..412] in t6124.scala +value _1415F is not a member of Int +<594..594> in t6124.scala trailing separator is not allowed -<235..235> in t6124.scala -Invalid literal number -<240..240> in t6124.scala +<567..567> in t6124.scala +leading separator is not allowed +<540..540> in t6124.scala trailing separator is not allowed -<213..213> in t6124.scala +<516..516> in t6124.scala trailing separator is not allowed -<187..187> in t6124.scala +<467..467> in t6124.scala trailing separator is not allowed -<160..160> in t6124.scala +<375..375> in t6124.scala +trailing separator is not allowed +<228..228> in t6124.scala +double precision floating point number too small +<197..197> in t6124.scala trailing separator is not allowed -<125..125> in t6124.scala +<159..159> in t6124.scala Invalid literal number -<101..101> in t6124.scala +<164..164> in t6124.scala trailing separator is not allowed -<67..67> in t6124.scala +<137..137> in t6124.scala trailing separator is not allowed -<33..33> in t6124.scala +<111..111> in t6124.scala +trailing separator is not allowed +<84..84> in t6124.scala +trailing separator is not allowed +<49..49> in t6124.scala +Invalid literal number +<25..25> in t6124.scala trailing separator is not allowed diff --git a/tests/neg/t6124.scala b/tests/neg/t6124.scala index 78c74d63642b..ee233bd2372e 100644 --- a/tests/neg/t6124.scala +++ b/tests/neg/t6124.scala @@ -1,8 +1,5 @@ trait T { - def i: Int = 123_456_ // error - def j: Long = 123_456_L * 1000 // error - def f = 3_14_E-2 // error def e = 3_14E-_2 // error def d = 3_14E-2_ // error @@ -14,5 +11,19 @@ trait T { def tooSmall = 1.0E-325 // error + // Examples from + // https://docs.oracle.com/javase/8/docs/technotes/guides/language/underscores-literals.html + + val pi1 = 3_.1415F // error + val pi2 = 3._1415F // error + val socialSecurityNumber1 + = 999_99_9999_L // error + val x1 = _52 // error + val x3 = 52_ // error + + val x5 = 0_x52 // error + val x6 = 0x_52 // error + val x8 = 0x52_ // error + def z = 0 } \ No newline at end of file diff --git a/tests/pos/t6124.scala b/tests/pos/t6124.scala index f83d96330a0e..7fdfc141dfc8 100644 --- a/tests/pos/t6124.scala +++ b/tests/pos/t6124.scala @@ -8,4 +8,11 @@ trait T { def d = 3_14E-2_1 def z = 0 + + + // Examples from + // https://docs.oracle.com/javase/8/docs/technotes/guides/language/underscores-literals.html + val x2 = 5_2; // OK (decimal literal) + val x4 = 5_______2; // OK (decimal literal) + val x7 = 0x5_2; // OK (hexadecimal literal) } \ No newline at end of file