diff --git a/doc/class-apemsel.AttributedString.TokenizedAttributedString.html b/doc/class-apemsel.AttributedString.TokenizedAttributedString.html index 8d24685..588a9cf 100644 --- a/doc/class-apemsel.AttributedString.TokenizedAttributedString.html +++ b/doc/class-apemsel.AttributedString.TokenizedAttributedString.html @@ -534,7 +534,7 @@

Returns

public static - string[] + array @@ -559,7 +559,7 @@

Parameters

Returns

- string[]
tokens + array
array of two arrays, with tokens at index 0 and their offsets at index 1
@@ -572,7 +572,7 @@

Returns

public static - string[] + array @@ -597,7 +597,7 @@

Parameters

Returns

- string[]
tokens + array
array of two arrays, with tokens at index 0 and their offsets at index 1
@@ -610,7 +610,7 @@

Returns

public static - string[] + array @@ -637,7 +637,7 @@

Parameters

Returns

- string[]
tokens + array
array of two arrays, with tokens at index 0 and their offsets at index 1
diff --git a/doc/source-class-apemsel.AttributedString.TokenizedAttributedString.html b/doc/source-class-apemsel.AttributedString.TokenizedAttributedString.html index 6217ee5..4a0c2b1 100644 --- a/doc/source-class-apemsel.AttributedString.TokenizedAttributedString.html +++ b/doc/source-class-apemsel.AttributedString.TokenizedAttributedString.html @@ -174,7 +174,7 @@

Classes

90 public function setTokenAttribute($i, $attribute, $state = true) { 91 $token = $this->tokens[$i]; 92 $offset = $this->tokenOffsets[$i]; - 93 $length = strlen($token); + 93 $length = mb_strlen($token, "utf-8"); 94 95 return $this->setLength($offset, $length, $attribute, $state); 96 } @@ -189,7 +189,7 @@

Classes

105 */ 106 public function setTokenRangeAttribute($from, $to, $attribute, $state = true) { 107 $fromOffset = $this->tokenOffsets[$from]; -108 $toOffset = $this->tokenOffsets[$to] + strlen($this->tokens[$to]); +108 $toOffset = $this->tokenOffsets[$to] + mb_strlen($this->tokens[$to], "utf-8"); 109 110 return $this->setRange($fromOffset, $toOffset, $attribute, $state); 111 } @@ -232,7 +232,7 @@

Classes

148 * Tokenize a string on whitespace 149 * 150 * @param string $string string to be tokenized -151 * @return string[] tokens +151 * @return array array of two arrays, with tokens at index 0 and their offsets at index 1 152 */ 153 public static function tokenizeOnWhitespace($string) { 154 // Matches pontential whitespace in front of the token and the token itself. @@ -244,7 +244,7 @@

Classes

160 * Tokenize a string on words 161 * 162 * @param string $string string to be tokenized -163 * @return string[] tokens +163 * @return array array of two arrays, with tokens at index 0 and their offsets at index 1 164 */ 165 public static function tokenizeOnWords($string) { 166 return self::tokenizeOnRegex($string, '/([\w]+)/u'); @@ -255,7 +255,7 @@

Classes

171 * 172 * @param string $string string to be tokenized 173 * @param string $pattern regex. The token must be captured in the first subgroup. -174 * @return string[] tokens +174 * @return array array of two arrays, with tokens at index 0 and their offsets at index 1 175 */ 176 public static function tokenizeOnRegex($string, $pattern) 177 {