Merge pull request #160 from Kakueeen/master

fix a bug of ChineseTokenizer

Author: alanw

src/contrib/analyzers/common/analysis/cn/ChineseFilter.cpp

@@ -38,7 +38,7 @@ bool ChineseFilter::incrementToken() {
                 if (text.length() > 1) {
                     return true;
                 }
-            } else if (UnicodeUtil::isOther(text[0])) {
+            } else if (UnicodeUtil::isOther(text[0]) || UnicodeUtil::isDigit(text[0])) {
                 // One Chinese character as one Chinese word.
                 // Chinese word extraction to be added later here.
                 return true;

src/contrib/analyzers/common/analysis/cn/ChineseTokenizer.cpp

@@ -65,6 +65,7 @@ bool ChineseTokenizer::incrementToken() {
 
     length = 0;
     start = offset;
+    bool last_is_en = false, last_is_num = false;
 
     while (true) {
         wchar_t c;
@@ -82,11 +83,30 @@ bool ChineseTokenizer::incrementToken() {
             c = ioBuffer[bufferIndex++];
         }
 
-        if (UnicodeUtil::isDigit(c) || UnicodeUtil::isLower(c) || UnicodeUtil::isUpper(c)) {
+        if (UnicodeUtil::isLower(c) || UnicodeUtil::isUpper(c)) {
+            if (last_is_num) {
+                --bufferIndex;
+                --offset;
+                return flush();
+            }
+
+            push(c);
+            if (length == MAX_WORD_LEN) {
+                return flush();
+            }
+            last_is_en = true;
+        } else if (UnicodeUtil::isDigit(c)) {
+            if (last_is_en) {
+                --bufferIndex;
+                --offset;
+                return flush();
+            }
+
             push(c);
             if (length == MAX_WORD_LEN) {
                 return flush();
             }
+            last_is_num = true;
         } else if (UnicodeUtil::isOther(c)) {
             if (length > 0) {
                 --bufferIndex;