Accept underscore as numeric literal separator

compiler/src/dotty/tools/dotc/parsing/JavaScanners.scala

@@ -521,7 +521,7 @@ object JavaScanners {
       case FLOATLIT =>
         "float(" + floatVal + ")"
       case DOUBLELIT =>
-        "double(" + floatVal + ")"
+        "double(" + doubleVal + ")"
       case STRINGLIT =>
         "string(" + name + ")"
       case SEMI =>

compiler/src/dotty/tools/dotc/parsing/Parsers.scala

@@ -726,7 +726,7 @@ object Parsers {
           case INTLIT                 => in.intVal(isNegated).toInt
           case LONGLIT                => in.intVal(isNegated)
           case FLOATLIT               => in.floatVal(isNegated).toFloat
-          case DOUBLELIT              => in.floatVal(isNegated)
+          case DOUBLELIT              => in.doubleVal(isNegated)
           case STRINGLIT | STRINGPART => in.strVal
           case TRUE                   => true
           case FALSE                  => false

compiler/src/dotty/tools/dotc/parsing/Scanners.scala

@@ -67,11 +67,15 @@ object Scanners {
 
     /** Generate an error at the given offset */
     def error(msg: String, off: Offset = offset): Unit = {
-      ctx.error(msg, source atSpan Span(off))
+      errorButContinue(msg, off)
       token = ERROR
       errOffset = off
     }
 
+    def errorButContinue(msg: String, off: Offset = offset): Unit = {
+      ctx.error(msg, source atSpan Span(off))
+    }
+
     /** signal an error where the input ended in the middle of a token */
     def incompleteInputError(msg: String): Unit = {
       ctx.incompleteInputError(msg, source atSpan Span(offset))
@@ -129,19 +133,22 @@ object Scanners {
         var i = 0
         val len = strVal.length
         while (i < len) {
-          val d = digit2int(strVal charAt i, base)
-          if (d < 0) {
-            error("malformed integer number")
-            return 0
-          }
-          if (value < 0 ||
-            limit / (base / divider) < value ||
-            limit - (d / divider) < value * (base / divider) &&
-              !(negated && limit == value * base - 1 + d)) {
-            error("integer number too large")
-            return 0
+          val c = strVal charAt i
+          if (! isNumberSeparator(c)) {
+            val d = digit2int(c, base)
+            if (d < 0) {
+              error(s"malformed integer number")
+              return 0
+            }
+            if (value < 0 ||
+              limit / (base / divider) < value ||
+              limit - (d / divider) < value * (base / divider) &&
+                !(negated && limit == value * base - 1 + d)) {
+              error("integer number too large")
+              return 0
+            }
+            value = value * base + d
           }
-          value = value * base + d
           i += 1
         }
         if (negated) -value else value
@@ -150,15 +157,42 @@ object Scanners {
 
     def intVal: Long = intVal(false)
 
+    private val zeroFloat = raw"[0.]+(?:[eE][+-]?[0-9]+)?[fFdD]?".r
+
+    /** Convert current strVal, base to double value
+      */
+    def floatVal(negated: Boolean): Float = {
+      assert(token == FLOATLIT)
+      val text = removeNumberSeparators(strVal)
+      try {
+        val value: Float = java.lang.Float.valueOf(text).floatValue()
+        if (value > Float.MaxValue)
+          errorButContinue("floating point number too large")
+
+        if (value == 0.0f && !zeroFloat.pattern.matcher(text).matches)
+          errorButContinue("floating point number too small")
+        if (negated) -value else value
+      } catch {
+        case _: NumberFormatException =>
+          error("malformed floating point number")
+          0.0f
+      }
+    }
+
+    def floatVal: Float = floatVal(false)
+
     /** Convert current strVal, base to double value
       */
-    def floatVal(negated: Boolean): Double = {
-      val limit: Double =
-        if (token == DOUBLELIT) Double.MaxValue else Float.MaxValue
+    def doubleVal(negated: Boolean): Double = {
+      assert(token == DOUBLELIT)
+      val text = removeNumberSeparators(strVal)
       try {
-        val value: Double = java.lang.Double.valueOf(strVal).doubleValue()
-        if (value > limit)
-          error("floating point number too large")
+        val value: Double = java.lang.Double.valueOf(text).doubleValue()
+        if (value > Double.MaxValue)
+          errorButContinue("double precision floating point number too large")
+
+        if (value == 0.0d && !zeroFloat.pattern.matcher(text).matches)
+          errorButContinue("double precision floating point number too small")
         if (negated) -value else value
       } catch {
         case _: NumberFormatException =>
@@ -167,7 +201,18 @@ object Scanners {
       }
     }
 
-    def floatVal: Double = floatVal(false)
+    def doubleVal: Double = doubleVal(false)
+
+    @inline def isNumberSeparator(c: Char): Boolean = c == '_'
+
+    @inline def removeNumberSeparators(s: String): String =
+      if (s.indexOf('_') > 0) s.replaceAllLiterally("_", "") /*.replaceAll("'","")*/ else s
+
+    // disallow trailing numeric separator char, but continue lexing
+    def checkNoTrailingSeparator(): Unit = {
+      if (isNumberSeparator(litBuf.last))
+        errorButContinue("trailing separator is not allowed", offset + litBuf.length - 1)
+    }
 
   }
 
@@ -463,6 +508,8 @@ object Scanners {
             if (ch == 'x' || ch == 'X') {
               nextChar()
               base = 16
+              if (isNumberSeparator(ch))
+                errorButContinue("leading separator is not allowed", offset + 2)
             } else {
               /**
                * What should leading 0 be in the future? It is potentially dangerous
@@ -911,27 +958,29 @@ object Scanners {
      */
     protected def getFraction(): Unit = {
       token = DOUBLELIT
-      while ('0' <= ch && ch <= '9') {
+      while ('0' <= ch && ch <= '9' || isNumberSeparator(ch)) {
         putChar(ch)
         nextChar()
       }
+      checkNoTrailingSeparator()
       if (ch == 'e' || ch == 'E') {
         val lookahead = lookaheadReader()
         lookahead.nextChar()
         if (lookahead.ch == '+' || lookahead.ch == '-') {
           lookahead.nextChar()
         }
-        if ('0' <= lookahead.ch && lookahead.ch <= '9') {
+        if ('0' <= lookahead.ch && lookahead.ch <= '9' || isNumberSeparator(ch)) {
           putChar(ch)
           nextChar()
           if (ch == '+' || ch == '-') {
             putChar(ch)
             nextChar()
           }
-          while ('0' <= ch && ch <= '9') {
+          while ('0' <= ch && ch <= '9' || isNumberSeparator(ch)) {
             putChar(ch)
             nextChar()
           }
+          checkNoTrailingSeparator()
         }
         token = DOUBLELIT
       }
@@ -954,15 +1003,18 @@ object Scanners {
     /** Read a number into strVal and set base
     */
     protected def getNumber(): Unit = {
-      while (digit2int(ch, base) >= 0) {
+      while (isNumberSeparator(ch) || digit2int(ch, base) >= 0) {
         putChar(ch)
         nextChar()
       }
+      checkNoTrailingSeparator()
       token = INTLIT
       if (base == 10 && ch == '.') {
         val lch = lookaheadChar()
         if ('0' <= lch && lch <= '9') {
-          putChar('.'); nextChar(); getFraction()
+          putChar('.')
+          nextChar()
+          getFraction()
         }
       } else (ch: @switch) match {
         case 'e' | 'E' | 'f' | 'F' | 'd' | 'D' =>
@@ -972,6 +1024,9 @@ object Scanners {
           token = LONGLIT
         case _ =>
       }
+
+      checkNoTrailingSeparator()
+
       setStrVal()
     }
 
@@ -1007,7 +1062,7 @@ object Scanners {
       case INTLIT => s"int($intVal)"
       case LONGLIT => s"long($intVal)"
       case FLOATLIT => s"float($floatVal)"
-      case DOUBLELIT => s"double($floatVal)"
+      case DOUBLELIT => s"double($doubleVal)"
       case STRINGLIT => s"string($strVal)"
       case STRINGPART => s"stringpart($strVal)"
       case INTERPOLATIONID => s"interpolationid($name)"

tests/neg/t6124.check

@@ -0,0 +1,34 @@
+[490..493] in t6124.scala
+Not found: _52
+[404..406..412] in t6124.scala
+value _1415F is not a member of Int
+<594..594> in t6124.scala
+trailing separator is not allowed
+<567..567> in t6124.scala
+leading separator is not allowed
+<540..540> in t6124.scala
+trailing separator is not allowed
+<516..516> in t6124.scala
+trailing separator is not allowed
+<467..467> in t6124.scala
+trailing separator is not allowed
+<375..375> in t6124.scala
+trailing separator is not allowed
+<228..228> in t6124.scala
+double precision floating point number too small
+<197..197> in t6124.scala
+trailing separator is not allowed
+<159..159> in t6124.scala
+Invalid literal number
+<164..164> in t6124.scala
+trailing separator is not allowed
+<137..137> in t6124.scala
+trailing separator is not allowed
+<111..111> in t6124.scala
+trailing separator is not allowed
+<84..84> in t6124.scala
+trailing separator is not allowed
+<49..49> in t6124.scala
+Invalid literal number
+<25..25> in t6124.scala
+trailing separator is not allowed

tests/neg/t6124.scala

@@ -0,0 +1,29 @@
+
+trait T {
+  def f = 3_14_E-2 // error
+  def e = 3_14E-_2 // error
+  def d = 3_14E-2_ // error
+
+  def p = 3.1_4_ // error
+  def q = 3.1_4_d // error
+  def r = 3.1_4_dd // error // error
+  def s = 3_.1 // error
+
+  def tooSmall = 1.0E-325 // error
+
+  // Examples from
+  // https://docs.oracle.com/javase/8/docs/technotes/guides/language/underscores-literals.html
+
+  val pi1 = 3_.1415F // error
+  val pi2 = 3._1415F // error
+  val socialSecurityNumber1
+    = 999_99_9999_L // error
+  val x1 = _52 // error
+  val x3 = 52_ // error
+
+  val x5 = 0_x52 // error
+  val x6 = 0x_52 // error
+  val x8 = 0x52_ // error
+
+  def z = 0
+}

tests/pos/t6124.scala

@@ -0,0 +1,18 @@
+
+trait T {
+  def i: Int = 1_024
+  def j: Long = 1_024L * 1_024
+  //def k = 1'024
+
+  def f = 3_14e-2f
+  def d = 3_14E-2_1
+
+  def z = 0
+
+
+  // Examples from
+  // https://docs.oracle.com/javase/8/docs/technotes/guides/language/underscores-literals.html
+  val x2 = 5_2;              // OK (decimal literal)
+  val x4 = 5_______2;        // OK (decimal literal)
+  val x7 = 0x5_2;            // OK (hexadecimal literal)
+}