Added support for custom tokenizer for diffWords

Author: jongaull-nimbly

src/diff/word.js

@@ -32,7 +32,8 @@ wordDiff.equals = function(left, right) {
   return left === right || (this.options.ignoreWhitespace && !reWhitespace.test(left) && !reWhitespace.test(right));
 };
 wordDiff.tokenize = function(value) {
-  let tokens = value.split(/(\s+|[()[\]{}'"]|\b)/);
+  const tokenizer = this.options.tokenizer || /(\s+|[()[\]{}'"]|\b)/; // Use the tokenizer regex in the options or use the default regex
+  const tokens = value.split(tokenizer); // Join the boundary splits that we do not consider to be boundaries. This is primarily the extended Latin character set.
 
   // Join the boundary splits that we do not consider to be boundaries. This is primarily the extended Latin character set.
   for (let i = 0; i < tokens.length - 1; i++) {

test/diff/word.js

@@ -171,6 +171,29 @@ describe('WordDiff', function() {
         done();
       });
     });
+
+    // With custom tokenizer
+    it('should utilize a custom tokenizer', function() {
+
+      const diff = diffWords('foo_bar', 'something_bar', {
+        tokenizer: /(\s+|[()[\]{}_'"]|\b)/
+      });
+
+      expect(diff).to.eql([{
+        count: 1,
+        added: undefined,
+        removed: true,
+        value: 'foo'
+      }, {
+        count: 1,
+        added: true,
+        removed: undefined,
+        value: 'something'
+      }, {
+        count: 2,
+        value: '_bar'
+      }]);
+    });
   });
 
   describe('#diffWordsWithSpace', function() {