fix: correct Kanji character detection

lib/core/regex.js

@@ -1,25 +1,17 @@
-const numeric = '[0-9]+'
-const alphanumeric = '[A-Z $%*+\\-./:]+'
-let kanji = '(?:[u3000-u303F]|[u3040-u309F]|[u30A0-u30FF]|' +
-  '[uFF00-uFFEF]|[u4E00-u9FAF]|[u2605-u2606]|[u2190-u2195]|u203B|' +
-  '[u2010u2015u2018u2019u2025u2026u201Cu201Du2225u2260]|' +
-  '[u0391-u0451]|[u00A7u00A8u00B1u00B4u00D7u00F7])+'
-kanji = kanji.replace(/u/g, '\\u')
-
-const byte = '(?:(?![A-Z0-9 $%*+\\-./:]|' + kanji + ')(?:.|[\r\n]))+'
-
-exports.KANJI = new RegExp(kanji, 'g')
-exports.BYTE_KANJI = new RegExp('[^A-Z0-9 $%*+\\-./:]+', 'g')
-exports.BYTE = new RegExp(byte, 'g')
-exports.NUMERIC = new RegExp(numeric, 'g')
-exports.ALPHANUMERIC = new RegExp(alphanumeric, 'g')
-
-const TEST_KANJI = new RegExp('^' + kanji + '$')
-const TEST_NUMERIC = new RegExp('^' + numeric + '$')
+const Utils = require('./utils')
+
+const TEST_NUMERIC = new RegExp('^[0-9]+$')
 const TEST_ALPHANUMERIC = new RegExp('^[A-Z0-9 $%*+\\-./:]+$')
 
 exports.testKanji = function testKanji (str) {
-  return TEST_KANJI.test(str)
+  if(!Utils.isKanjiModeEnabled()) {
+    return false
+  }
+  let flag = true
+  for(let char of str) {
+    flag = flag && Utils.toSJIS(char)
+  }
+  return flag
 }
 
 exports.testNumeric = function testNumeric (str) {
@@ -29,3 +21,32 @@ exports.testNumeric = function testNumeric (str) {
 exports.testAlphanumeric = function testAlphanumeric (str) {
   return TEST_ALPHANUMERIC.test(str)
 }
+
+/**
+ * @param {string} charCollection char collection of this mode
+ * @returns {function}
+ */
+function isModeFactory (charCollection) {
+  let map = new Map()
+  for (let i = 0; i < charCollection.length; ++i) {
+    map.set(charCollection.codePointAt(i), i)
+  }
+  /**
+   * Verify whether the character at a specified position in the string belongs to this set. 
+   * @param {string} str
+   * @param {number} pos
+   * @returns {boolean}
+   */
+  return function (str, pos) {
+    return map.has(str.codePointAt(pos))
+  }
+}
+
+exports.isNumeric = isModeFactory('0123456789')
+
+exports.isAlphanumeric = isModeFactory('0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ $%*+-./:')
+
+exports.isKanji = function (str, pos) {
+  let unicode = str.codePointAt(pos)
+  return Utils.isKanjiModeEnabled() && !!Utils.toSJIS(String.fromCodePoint(unicode))
+}

lib/core/segments.js

@@ -49,32 +49,36 @@ function getSegments (regex, mode, str) {
  * @return {Array}          Array of object with segments data
  */
 function getSegmentsFromString (dataStr) {
-  const numSegs = getSegments(Regex.NUMERIC, Mode.NUMERIC, dataStr)
-  const alphaNumSegs = getSegments(Regex.ALPHANUMERIC, Mode.ALPHANUMERIC, dataStr)
-  let byteSegs
-  let kanjiSegs
-
-  if (Utils.isKanjiModeEnabled()) {
-    byteSegs = getSegments(Regex.BYTE, Mode.BYTE, dataStr)
-    kanjiSegs = getSegments(Regex.KANJI, Mode.KANJI, dataStr)
-  } else {
-    byteSegs = getSegments(Regex.BYTE_KANJI, Mode.BYTE, dataStr)
-    kanjiSegs = []
+  const segs = []
+  let pos = 0
+  while (pos < dataStr.length) {
+    let mode
+    let utf16Len = dataStr.codePointAt(pos) > 0xffff ? 2 : 1
+    if (Regex.isNumeric(dataStr, pos)) {
+      mode = Mode.NUMERIC
+    } else if (Regex.isAlphanumeric(dataStr, pos)) {
+      mode = Mode.ALPHANUMERIC
+    } else if (Regex.isKanji(dataStr, pos)) {
+      mode = Mode.KANJI
+    } else {
+      mode = Mode.BYTE
+    }
+    if (segs.length === 0 || segs[segs.length - 1].mode !== mode) {
+      segs.push({
+        pos: pos,
+        mode: mode,
+        length: utf16Len,
+      })
+    } else {
+      segs[segs.length - 1].length += utf16Len
+    }
+    pos += utf16Len
   }
-
-  const segs = numSegs.concat(alphaNumSegs, byteSegs, kanjiSegs)
-
-  return segs
-    .sort(function (s1, s2) {
-      return s1.index - s2.index
-    })
-    .map(function (obj) {
-      return {
-        data: obj.data,
-        mode: obj.mode,
-        length: obj.length
-      }
-    })
+  return segs.map(o => {
+    o.data = dataStr.slice(o.pos, o.pos + o.length)
+    delete o.pos
+    return o
+  })
 }
 
 /**