Fix receiving large text fields

fixes #922

Author: dougwilson

Changes.md

@@ -4,6 +4,10 @@ This file is a manually maintained list of changes for each release. Feel free
 to add your changes here when sending pull requests. Also send corrections if
 you spot any mistakes.
 
+## HEAD
+
+* Fix receiving large text fields #922
+
 ## v2.5.1 (2014-09-22)
 
 * Fix `pool.end` race conditions #915

lib/protocol/Parser.js

@@ -98,25 +98,36 @@ Parser.prototype.write = function(buffer) {
   }
 };
 
-Parser.prototype.append = function(newBuffer) {
-  // If resume() is called, we don't pass a buffer to write()
-  if (!newBuffer) {
+Parser.prototype.append = function append(chunk) {
+  if (!chunk || chunk.length === 0) {
     return;
   }
 
-  var oldBuffer = this._buffer;
-  var bytesRemaining = this._bytesRemaining();
-  var newLength = bytesRemaining + newBuffer.length;
+  var buffer      = chunk;
+  var sliceEnd    = this._buffer.length;
+  var sliceStart  = this._packetOffset === null
+    ? this._offset
+    : this._packetOffset;
+  var sliceLength = sliceEnd - sliceStart;
 
-  var combinedBuffer = (this._offset > newLength)
-    ? oldBuffer.slice(0, newLength)
-    : new Buffer(newLength);
+  if (sliceLength !== 0) {
+    // Create a new Buffer
+    buffer = new Buffer(sliceLength + chunk.length);
 
-  oldBuffer.copy(combinedBuffer, 0, this._offset);
-  newBuffer.copy(combinedBuffer, bytesRemaining);
+    // Copy data
+    this._buffer.copy(buffer, 0, sliceStart, sliceEnd);
+    chunk.copy(buffer, sliceLength);
+  }
 
-  this._buffer = combinedBuffer;
-  this._offset = 0;
+  // Adjust data-tracking pointers
+  this._buffer       = buffer;
+  this._offset       = this._offset - sliceStart;
+  this._packetEnd    = this._packetEnd !== null
+    ? this._packetEnd - sliceStart
+    : null;
+  this._packetOffset = this._packetOffset !== null
+    ? this._packetOffset - sliceStart
+    : null;
 };
 
 Parser.prototype.pause = function() {

test/integration/connection/test-send-and-receive-large-text.js

@@ -0,0 +1,85 @@
+var assert = require('assert');
+var common = require('../../common');
+var crypto = require('crypto');
+
+common.getTestConnection(function (err, connection) {
+  assert.ifError(err);
+
+  getMaxAllowedPacket(connection);
+});
+
+
+var oldMaxAllowedPacket;
+function getMaxAllowedPacket(connection) {
+  connection.query('SHOW VARIABLES WHERE Variable_name = ?', ['max_allowed_packet'], function (err, rows) {
+    assert.ifError(err);
+
+    oldMaxAllowedPacket = Number(rows[0].Value);
+
+    increaseMaxAllowedPacketIfNeeded(connection);
+  });
+}
+
+function increaseMaxAllowedPacketIfNeeded(connection) {
+  // Our test generates a SQL query a few bytes larger than 16 MB, but lets
+  // leave a little margin:
+  var minMaxAllowedPacket = 20 * 1024 * 1024;
+
+  var newMaxAllowedPacket = (oldMaxAllowedPacket < minMaxAllowedPacket)
+    ? minMaxAllowedPacket
+    : oldMaxAllowedPacket;
+
+  connection.query('SET GLOBAL max_allowed_packet = ?', [newMaxAllowedPacket], function (err, rows) {
+    assert.ifError(err);
+
+    // We need to re-connect for this change to take effect, bah
+    connection.end();
+    connection = common.createConnection();
+
+    // We need to wait for the re-connect to happen before starting the actual
+    // test. That's because our buffer to hex shim in 0.4.x takes ~12 sec on
+    // TravisCI, causing a MySQL connection timeout otherwise.
+    connection.connect(function (err) {
+      assert.ifError(err);
+
+      triggerLargeQueryAndResponsePackets(connection);
+    });
+  });
+}
+
+var length = (Math.pow(256, 3) / 2) + 10; // Half, because of hex encoding
+var random = crypto.pseudoRandomBytes || crypto.randomBytes; // Depends on node.js version
+var table  = 'large_text_test';
+
+function triggerLargeQueryAndResponsePackets(connection) {
+  random(length, function (err, buf) {
+    assert.ifError(err);
+    assert.equal(buf.length, length);
+
+    common.useTestDb(connection);
+
+    connection.query([
+      'CREATE TEMPORARY TABLE ?? (',
+      '`id` int(11) unsigned NOT NULL AUTO_INCREMENT,',
+      '`bt` longtext NOT NULL,',
+      'PRIMARY KEY (`id`)',
+      ') ENGINE=InnoDB DEFAULT CHARSET=utf8'
+    ].join('\n'), [table], assert.ifError);
+
+    var text = buf.toString('hex');
+
+    connection.query('INSERT INTO ?? SET ?', [table, {bt: text}], assert.ifError);
+
+    connection.query('SELECT `id`, `bt` FROM ??', [table], function (err, rows) {
+      assert.ifError(err);
+
+      connection.query('SET GLOBAL max_allowed_packet = ?', [oldMaxAllowedPacket], assert.ifError);
+      connection.end(function (err) {
+        assert.ifError(err);
+        assert.equal(rows.length, 1);
+        assert.equal(rows[0].bt.length, text.length);
+        assert.equal(rows[0].bt, text);
+      });
+    });
+  });
+}