diff --git a/package-lock.json b/package-lock.json index 5d414f216f..dee645e675 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9452,15 +9452,13 @@ }, "regenerate": { "version": "1.4.0", - "resolved": "https://registry.npmjs.org/regenerate/-/regenerate-1.4.0.tgz", - "integrity": "sha512-1G6jJVDWrt0rK99kBjvEtziZNCICAuvIPkSiUFIQxVP06RCVpq3dmDo2oi6ABpYaDYaTRr67BEhL8r1wgEZZKg==", - "dev": true + "resolved": "https://registry.npm.taobao.org/regenerate/download/regenerate-1.4.0.tgz", + "integrity": "sha1-SoVuxLVuQHfFV1icroXnpMiGmhE=" }, "regenerate-unicode-properties": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/regenerate-unicode-properties/-/regenerate-unicode-properties-8.1.0.tgz", - "integrity": "sha512-LGZzkgtLY79GeXLm8Dp0BVLdQlWICzBnJz/ipWUgo59qBaZ+BHtq51P2q1uVZlppMuUAT37SDk39qUbjTWB7bA==", - "dev": true, + "version": "8.2.0", + "resolved": "https://registry.npm.taobao.org/regenerate-unicode-properties/download/regenerate-unicode-properties-8.2.0.tgz", + "integrity": "sha1-5d5xEdZV57pgwFfb6f83yH5lzew=", "requires": { "regenerate": "^1.4.0" } @@ -9496,39 +9494,35 @@ "dev": true }, "regexpu-core": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/regexpu-core/-/regexpu-core-4.6.0.tgz", - "integrity": "sha512-YlVaefl8P5BnFYOITTNzDvan1ulLOiXJzCNZxduTIosN17b87h3bvG9yHMoHaRuo88H4mQ06Aodj5VtYGGGiTg==", - "dev": true, + "version": "4.7.0", + "resolved": "https://registry.npm.taobao.org/regexpu-core/download/regexpu-core-4.7.0.tgz", + "integrity": "sha1-/L9FjFBDGwu3tF1pZ7gZLZHz2Tg=", "requires": { "regenerate": "^1.4.0", - "regenerate-unicode-properties": "^8.1.0", - "regjsgen": "^0.5.0", - "regjsparser": "^0.6.0", + "regenerate-unicode-properties": "^8.2.0", + "regjsgen": "^0.5.1", + "regjsparser": "^0.6.4", "unicode-match-property-ecmascript": "^1.0.4", - "unicode-match-property-value-ecmascript": "^1.1.0" + "unicode-match-property-value-ecmascript": "^1.2.0" } }, "regjsgen": { "version": "0.5.1", - "resolved": "https://registry.npmjs.org/regjsgen/-/regjsgen-0.5.1.tgz", - "integrity": "sha512-5qxzGZjDs9w4tzT3TPhCJqWdCc3RLYwy9J2NB0nm5Lz+S273lvWcpjaTGHsT1dc6Hhfq41uSEOw8wBmxrKOuyg==", - "dev": true + "resolved": "https://registry.npm.taobao.org/regjsgen/download/regjsgen-0.5.1.tgz", + "integrity": "sha1-SPC/Gl6iBRlpKcDZeYtC0e2YRDw=" }, "regjsparser": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/regjsparser/-/regjsparser-0.6.0.tgz", - "integrity": "sha512-RQ7YyokLiQBomUJuUG8iGVvkgOLxwyZM8k6d3q5SAXpg4r5TZJZigKFvC6PpD+qQ98bCDC5YelPeA3EucDoNeQ==", - "dev": true, + "version": "0.6.4", + "resolved": "https://registry.npm.taobao.org/regjsparser/download/regjsparser-0.6.4.tgz", + "integrity": "sha1-p2n4aEMIQBpm6bUp0kNv9NBmYnI=", "requires": { "jsesc": "~0.5.0" }, "dependencies": { "jsesc": { "version": "0.5.0", - "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-0.5.0.tgz", - "integrity": "sha1-597mbjXW/Bb3EP6R1c9p9w8IkR0=", - "dev": true + "resolved": "https://registry.npm.taobao.org/jsesc/download/jsesc-0.5.0.tgz", + "integrity": "sha1-597mbjXW/Bb3EP6R1c9p9w8IkR0=" } } }, @@ -10915,31 +10909,27 @@ }, "unicode-canonical-property-names-ecmascript": { "version": "1.0.4", - "resolved": "https://registry.npmjs.org/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-1.0.4.tgz", - "integrity": "sha512-jDrNnXWHd4oHiTZnx/ZG7gtUTVp+gCcTTKr8L0HjlwphROEW3+Him+IpvC+xcJEFegapiMZyZe02CyuOnRmbnQ==", - "dev": true + "resolved": "https://registry.npm.taobao.org/unicode-canonical-property-names-ecmascript/download/unicode-canonical-property-names-ecmascript-1.0.4.tgz", + "integrity": "sha1-JhmADEyCWADv3YNDr33Zkzy+KBg=" }, "unicode-match-property-ecmascript": { "version": "1.0.4", - "resolved": "https://registry.npmjs.org/unicode-match-property-ecmascript/-/unicode-match-property-ecmascript-1.0.4.tgz", - "integrity": "sha512-L4Qoh15vTfntsn4P1zqnHulG0LdXgjSO035fEpdtp6YxXhMT51Q6vgM5lYdG/5X3MjS+k/Y9Xw4SFCY9IkR0rg==", - "dev": true, + "resolved": "https://registry.npm.taobao.org/unicode-match-property-ecmascript/download/unicode-match-property-ecmascript-1.0.4.tgz", + "integrity": "sha1-jtKjJWmWG86SJ9Cc0/+7j+1fAgw=", "requires": { "unicode-canonical-property-names-ecmascript": "^1.0.4", "unicode-property-aliases-ecmascript": "^1.0.4" } }, "unicode-match-property-value-ecmascript": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/unicode-match-property-value-ecmascript/-/unicode-match-property-value-ecmascript-1.1.0.tgz", - "integrity": "sha512-hDTHvaBk3RmFzvSl0UVrUmC3PuW9wKVnpoUDYH0JDkSIovzw+J5viQmeYHxVSBptubnr7PbH2e0fnpDRQnQl5g==", - "dev": true + "version": "1.2.0", + "resolved": "https://registry.npm.taobao.org/unicode-match-property-value-ecmascript/download/unicode-match-property-value-ecmascript-1.2.0.tgz", + "integrity": "sha1-DZH2AO7rMJaqlisdb8iIduZOpTE=" }, "unicode-property-aliases-ecmascript": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/unicode-property-aliases-ecmascript/-/unicode-property-aliases-ecmascript-1.0.5.tgz", - "integrity": "sha512-L5RAqCfXqAwR3RriF8pM0lU0w4Ryf/GgzONwi6KnL1taJQa7x1TCxdJnILX59WIGOwR57IVxn7Nej0fz1Ny6fw==", - "dev": true + "version": "1.1.0", + "resolved": "https://registry.npm.taobao.org/unicode-property-aliases-ecmascript/download/unicode-property-aliases-ecmascript-1.1.0.tgz?cache=0&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Funicode-property-aliases-ecmascript%2Fdownload%2Funicode-property-aliases-ecmascript-1.1.0.tgz", + "integrity": "sha1-3Vepn2IHvt/0Yoq++5TFDblByPQ=" }, "union-value": { "version": "1.0.1", diff --git a/package.json b/package.json index 760e5a64b0..17b9e24a56 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,8 @@ }, "dependencies": { "@babel/polyfill": "^7.7.0", - "lodash": "^4.17.15" + "lodash": "^4.17.15", + "regexpu-core": "^4.7.0" }, "devDependencies": { "@babel/cli": "^7.7.0", diff --git a/src/core/Tokenizer.js b/src/core/Tokenizer.js index ec6a4a3e8e..ef45117861 100644 --- a/src/core/Tokenizer.js +++ b/src/core/Tokenizer.js @@ -1,6 +1,7 @@ import isEmpty from 'lodash/isEmpty'; import escapeRegExp from 'lodash/escapeRegExp'; import tokenTypes from './tokenTypes'; +import rewritePattern from 'regexpu-core'; export default class Tokenizer { /** @@ -66,9 +67,13 @@ export default class Tokenizer { createWordRegex(specialChars = []) { return new RegExp( - `^([\\p{Alphabetic}\\p{Mark}\\p{Decimal_Number}\\p{Connector_Punctuation}\\p{Join_Control}${specialChars.join( - '' - )}]+)`, + rewritePattern( + `^([\\p{Alphabetic}\\p{Mark}\\p{Decimal_Number}\\p{Connector_Punctuation}\\p{Join_Control}${specialChars.join( + '' + )}]+)`, + 'u', + { unicodePropertyEscape: true } + ), 'u' ); }