Merge pull request #382 from felixge/bignumber

Changes parseLengthCodedNumber to parse big numbers

Author: dresende

Readme.md

@@ -145,6 +145,7 @@ When establishing a connection, you can set the following options:
 * `typeCast`: Determines if column values should be converted to native
    JavaScript types. (Default: `true`)
 * `queryFormat`: A custom query format function. See [Custom format](#custom-format).
+* `supportBigNumbers`: When dealing with big numbers in the database, you should enable this option.
 * `debug`: Prints protocol details to stdout. (Default: `false`)
 * `multipleStatements`: Allow multiple mysql statements per query. Be careful
   with this, it exposes you to SQL injection attacks. (Default: `false`)
@@ -372,6 +373,13 @@ connection.query('INSERT INTO posts SET ?', {title: 'test'}, function(err, resul
 });
 ```
 
+When dealing with big numbers (above JavaScript Number precision limit), you should
+consider enabling `supportBigNumbers` option to be able to read the insert id as a
+string, otherwise it will throw.
+
+This option is also required when fetching big numbers from the database, otherwise
+you will get values rounded to hundreds or thousands due to the precision limit.
+
 ## Executing queries in parallel
 
 The MySQL protocol is sequential, this means that you need multiple connections

lib/ConnectionConfig.js

@@ -8,18 +8,19 @@ function ConnectionConfig(options) {
     options = ConnectionConfig.parseUrl(options);
   }
 
-  this.host         = options.host || 'localhost';
-  this.port         = options.port || 3306;
-  this.socketPath   = options.socketPath;
-  this.user         = options.user || undefined;
-  this.password     = options.password || undefined;
-  this.database     = options.database;
-  this.insecureAuth = options.insecureAuth || false;
-  this.debug        = options.debug;
-  this.timezone     = options.timezone || 'local';
-  this.flags        = options.flags || '';
-  this.queryFormat  = options.queryFormat;
-  this.typeCast     = (options.typeCast === undefined)
+  this.host              = options.host || 'localhost';
+  this.port              = options.port || 3306;
+  this.socketPath        = options.socketPath;
+  this.user              = options.user || undefined;
+  this.password          = options.password || undefined;
+  this.database          = options.database;
+  this.insecureAuth      = options.insecureAuth || false;
+  this.supportBigNumbers = options.supportBigNumbers || false;
+  this.debug             = options.debug;
+  this.timezone          = options.timezone || 'local';
+  this.flags             = options.flags || '';
+  this.queryFormat       = options.queryFormat;
+  this.typeCast          = (options.typeCast === undefined)
     ? true
     : options.typeCast;
 

lib/protocol/Parser.js

@@ -1,11 +1,13 @@
 var IEEE_754_BINARY_64_PRECISION = Math.pow(2, 53);
 var MAX_PACKET_LENGTH            = Math.pow(2, 24) - 1;
 var PacketHeader                 = require('./PacketHeader');
+var BigNumber                    = require("bignumber.js");
 
 module.exports = Parser;
 function Parser(options) {
   options = options || {};
 
+  this._supportBigNumbers = options.config && options.config.supportBigNumbers;
   this._buffer            = new Buffer(0);
   this._longPacketBuffers = [];
   this._offset            = 0;
@@ -145,29 +147,47 @@ Parser.prototype.parseLengthCodedBuffer = function() {
 };
 
 Parser.prototype.parseLengthCodedNumber = function() {
-  var byte = this._buffer[this._offset++];
+  var bits = this._buffer[this._offset++];
 
-  if (byte <= 251) {
-    return (byte === 251)
+  if (bits <= 251) {
+    return (bits === 251)
       ? null
-      : byte;
+      : bits;
   }
 
   var length;
-  if (byte === 252) {
+  var bigNumber = false;
+  var value = 0;
+
+  if (bits === 252) {
     length = 2;
-  } else if (byte === 253) {
+  } else if (bits === 253) {
     length = 3;
-  } else if (byte === 254) {
+  } else if (bits === 254) {
     length = 8;
+
+    if (this._supportBigNumbers) {
+      if (this._buffer[this._offset + 6] > 31 || this._buffer[this._offset + 7]) {
+        value = new BigNumber(0);
+        bigNumber = true;
+      }
+    }
   } else {
-    throw new Error('parseLengthCodedNumber: Unexpected first byte: ' + byte);
+    throw new Error('parseLengthCodedNumber: Unexpected first byte: ' + bits);
   }
 
-  var value = 0;
   for (var bytesRead = 0; bytesRead < length; bytesRead++) {
-    var byte = this._buffer[this._offset++];
-    value += Math.pow(256, bytesRead) * byte;
+    bits = this._buffer[this._offset++];
+
+    if (bigNumber) {
+      value = value.plus((new BigNumber(256)).pow(bytesRead).times(bits));
+    } else {
+      value += Math.pow(256, bytesRead) * bits;
+    }
+  }
+
+  if (bigNumber) {
+    return value.toString();
   }
 
   if (value >= IEEE_754_BINARY_64_PRECISION) {
@@ -194,7 +214,7 @@ Parser.prototype.parseNullTerminatedBuffer = function() {
 
 Parser.prototype.parseNullTerminatedString = function() {
   var end      = this._nullByteOffset();
-  var value    = this._buffer.toString(this._encoding, this._offset, end)
+  var value    = this._buffer.toString(this._encoding, this._offset, end);
   this._offset = end + 1;
 
   return value;

lib/protocol/Protocol.js

@@ -16,7 +16,6 @@ function Protocol(options) {
   this.readable = true;
   this.writable = true;
 
-  this._parser                        = new Parser({onPacket: this._parsePacket.bind(this)});
   this._config                        = options.config || {};
   this._connection                    = options.connection;
   this._callback                      = null;
@@ -26,6 +25,11 @@ function Protocol(options) {
   this._destroyed                     = false;
   this._queue                         = [];
   this._handshakeInitializationPacket = null;
+
+  this._parser = new Parser({
+    onPacket : this._parsePacket.bind(this),
+    config   : this._config
+  });
 }
 
 Protocol.prototype.write = function(buffer) {

lib/protocol/packets/RowDataPacket.js

@@ -1,6 +1,7 @@
-var Types    = require('../constants/types');
-var Charsets = require('../constants/charsets');
-var Field    = require('./Field');
+var Types                        = require('../constants/types');
+var Charsets                     = require('../constants/charsets');
+var Field                        = require('./Field');
+var IEEE_754_BINARY_64_PRECISION = Math.pow(2, 53);
 
 module.exports = RowDataPacket;
 function RowDataPacket() {
@@ -9,7 +10,7 @@ function RowDataPacket() {
 RowDataPacket.prototype.parse = function(parser, fieldPackets, typeCast, nestTables, connection) {
   var self = this;
   var next = function () {
-    return self._typeCast(fieldPacket, parser, connection.config.timezone);
+    return self._typeCast(fieldPacket, parser, connection.config.timezone, connection.config.supportBigNumbers);
   };
 
   for (var i = 0; i < fieldPackets.length; i++) {
@@ -20,7 +21,7 @@ RowDataPacket.prototype.parse = function(parser, fieldPackets, typeCast, nestTab
       value = typeCast.apply(connection, [ new Field({ packet: fieldPacket, parser: parser }), next ]);
     } else {
       value = (typeCast)
-        ? this._typeCast(fieldPacket, parser, connection.config.timezone)
+        ? this._typeCast(fieldPacket, parser, connection.config.timezone, connection.config.supportBigNumbers)
         : ( (fieldPacket.charsetNr === Charsets.BINARY)
           ? parser.parseLengthCodedBuffer()
           : parser.parseLengthCodedString() );
@@ -37,7 +38,7 @@ RowDataPacket.prototype.parse = function(parser, fieldPackets, typeCast, nestTab
   }
 };
 
-RowDataPacket.prototype._typeCast = function(field, parser, timeZone) {
+RowDataPacket.prototype._typeCast = function(field, parser, timeZone, supportBigNumbers) {
   switch (field.type) {
     case Types.TIMESTAMP:
     case Types.DATE:
@@ -69,7 +70,9 @@ RowDataPacket.prototype._typeCast = function(field, parser, timeZone) {
       var numberString = parser.parseLengthCodedString();
       return (numberString === null || (field.zeroFill && numberString[0] == "0"))
         ? numberString
-        : Number(numberString);
+        : ((supportBigNumbers && Number(numberString) > IEEE_754_BINARY_64_PRECISION)
+          ? numberString
+          : Number(numberString));
     case Types.BIT:
       return parser.parseLengthCodedBuffer();
     case Types.STRING:

package.json

@@ -14,7 +14,8 @@
     "node": "*"
   },
   "dependencies": {
-    "require-all": "0.0.3"
+    "require-all": "0.0.3",
+    "bignumber.js": "1.0.1"
   },
   "devDependencies": {
     "utest": "0.0.6",