another fix for tokenizeOnWords()

apemsel · Mar 8, 2016 · 88bec4e · 88bec4e
1 parent 24f053c
commit 88bec4e
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/TokenizedAttributedString.php b/src/TokenizedAttributedString.php
@@ -163,7 +163,7 @@ public static function tokenizeOnWhitespace($string) {
    * @return array array of two arrays, with tokens at index 0 and their offsets at index 1
    */
   public static function tokenizeOnWords($string) {
-    return self::tokenizeOnRegex($string, '/(\p{L}+)/u');
+    return self::tokenizeOnRegex($string, '/([\p{L}\p{S}\p{N}]+)/u');
   }
 
   /**