fix bug in tokenizeOnWords

apemsel · Mar 8, 2016 · 24f053c · 24f053c
1 parent a1bd8a1
commit 24f053c
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/TokenizedAttributedString.php b/src/TokenizedAttributedString.php
@@ -163,7 +163,7 @@ public static function tokenizeOnWhitespace($string) {
    * @return array array of two arrays, with tokens at index 0 and their offsets at index 1
    */
   public static function tokenizeOnWords($string) {
-    return self::tokenizeOnRegex($string, '/([\w]+)/u');
+    return self::tokenizeOnRegex($string, '/(\p{L}+)/u');
   }
 
   /**