From f5c94b4be62d2cc18dcedf0bcdd51dc24b50a16c Mon Sep 17 00:00:00 2001
From: Adrian Pemsel <apemsel@gmail.com>
Date: Thu, 3 Mar 2016 09:50:00 +0100
Subject: [PATCH] update documentation

---
 ...ributedString.MutableAttributedString.html |   8 +-
 ...butedString.TokenizedAttributedString.html | 126 +++++--
 ...sel.AttributedString.AttributedString.html |   2 +-
 ...ributedString.MutableAttributedString.html | 233 ++++++------
 ...butedString.TokenizedAttributedString.html | 349 ++++++++++--------
 5 files changed, 403 insertions(+), 315 deletions(-)

diff --git a/doc/class-apemsel.AttributedString.MutableAttributedString.html b/doc/class-apemsel.AttributedString.MutableAttributedString.html
index e26f6bb..55b33d7 100644
--- a/doc/class-apemsel.AttributedString.MutableAttributedString.html
+++ b/doc/class-apemsel.AttributedString.MutableAttributedString.html
@@ -124,7 +124,7 @@ <h1>Class MutableAttributedString</h1>
 
 				<b>Author:</b>
 				Adrian Pemsel <a href="&#x6d;&#x61;&#x69;&#x6c;&#116;&#111;&#58;&#97;&#112;&#x65;&#x6d;&#x73;&#x65;&#x6c;&#64;&#103;&#109;&#97;&#105;&#x6c;&#x2e;&#x63;&#x6f;&#x6d;">&#97;&#112;&#x65;&#x6d;&#x73;&#x65;&#x6c;&#64;&#103;&#109;&#97;&#105;&#x6c;&#x2e;&#x63;&#x6f;&#x6d;</a><br>
-			<b>Located at</b> <a href="source-class-apemsel.AttributedString.MutableAttributedString.html#5-117" title="Go to source code">MutableAttributedString.php</a>
+			<b>Located at</b> <a href="source-class-apemsel.AttributedString.MutableAttributedString.html#4-116" title="Go to source code">MutableAttributedString.php</a>
 		<br>
 	</div>
 
@@ -144,7 +144,7 @@ <h1>Class MutableAttributedString</h1>
 
 			<td class="name"><div>
 			<a class="anchor" href="#_insert">#</a>
-			<code><a href="source-class-apemsel.AttributedString.MutableAttributedString.html#14-45" title="Go to source code">insert</a>( <span>integer <var>$pos</var></span>, <span>string <var>$string</var></span> )</code>
+			<code><a href="source-class-apemsel.AttributedString.MutableAttributedString.html#13-44" title="Go to source code">insert</a>( <span>integer <var>$pos</var></span>, <span>string <var>$string</var></span> )</code>
 
 			<div class="description short">
 				<p>Insert string at given offset</p>
@@ -180,7 +180,7 @@ <h4>Parameters</h4>
 
 			<td class="name"><div>
 			<a class="anchor" href="#_delete">#</a>
-			<code><a href="source-class-apemsel.AttributedString.MutableAttributedString.html#47-70" title="Go to source code">delete</a>( <span>integer <var>$pos</var></span>, <span>integer <var>$length</var></span> )</code>
+			<code><a href="source-class-apemsel.AttributedString.MutableAttributedString.html#46-69" title="Go to source code">delete</a>( <span>integer <var>$pos</var></span>, <span>integer <var>$length</var></span> )</code>
 
 			<div class="description short">
 				<p>Delete substring of given offset and length</p>
@@ -216,7 +216,7 @@ <h4>Parameters</h4>
 
 			<td class="name"><div>
 			<a class="anchor" href="#_mb_substr_replace">#</a>
-			<code><a href="source-class-apemsel.AttributedString.MutableAttributedString.html#72-116" title="Go to source code">mb_substr_replace</a>( <span>string <var>$string</var></span>, <span>string <var>$replacement</var></span>, <span>integer <var>$start</var></span>, <span>integer <var>$length</var> = <span class="php-keyword1">NULL</span></span> )</code>
+			<code><a href="source-class-apemsel.AttributedString.MutableAttributedString.html#71-115" title="Go to source code">mb_substr_replace</a>( <span>string <var>$string</var></span>, <span>string <var>$replacement</var></span>, <span>integer <var>$start</var></span>, <span>integer <var>$length</var> = <span class="php-keyword1">NULL</span></span> )</code>
 
 			<div class="description short">
 				<p>Missing mb_substr_replace() implementation</p>
diff --git a/doc/class-apemsel.AttributedString.TokenizedAttributedString.html b/doc/class-apemsel.AttributedString.TokenizedAttributedString.html
index 768402d..8d24685 100644
--- a/doc/class-apemsel.AttributedString.TokenizedAttributedString.html
+++ b/doc/class-apemsel.AttributedString.TokenizedAttributedString.html
@@ -125,7 +125,7 @@ <h1>Class TokenizedAttributedString</h1>
 
 				<b>Author:</b>
 				Adrian Pemsel <a href="&#x6d;&#x61;&#x69;&#x6c;&#116;&#111;&#58;&#97;&#112;&#x65;&#x6d;&#x73;&#x65;&#x6c;&#64;&#103;&#109;&#97;&#105;&#x6c;&#x2e;&#x63;&#x6f;&#x6d;">&#97;&#112;&#x65;&#x6d;&#x73;&#x65;&#x6c;&#64;&#103;&#109;&#97;&#105;&#x6c;&#x2e;&#x63;&#x6f;&#x6d;</a><br>
-			<b>Located at</b> <a href="source-class-apemsel.AttributedString.TokenizedAttributedString.html#5-161" title="Go to source code">TokenizedAttributedString.php</a>
+			<b>Located at</b> <a href="source-class-apemsel.AttributedString.TokenizedAttributedString.html#4-188" title="Go to source code">TokenizedAttributedString.php</a>
 		<br>
 	</div>
 
@@ -145,7 +145,7 @@ <h1>Class TokenizedAttributedString</h1>
 
 			<td class="name"><div>
 			<a class="anchor" href="#___construct">#</a>
-			<code><a href="source-class-apemsel.AttributedString.TokenizedAttributedString.html#18-35" title="Go to source code">__construct</a>( <span>string|<code><a href="class-apemsel.AttributedString.AttributedString.html">apemsel\AttributedString\AttributedString</a></code> <var>$string</var></span>, <span>string <var>$tokenizer</var> = <span class="php-quote">&quot;whitespace&quot;</span></span> )</code>
+			<code><a href="source-class-apemsel.AttributedString.TokenizedAttributedString.html#17-34" title="Go to source code">__construct</a>( <span>string|<code><a href="class-apemsel.AttributedString.AttributedString.html">apemsel\AttributedString\AttributedString</a></code> <var>$string</var></span>, <span>string <var>$tokenizer</var> = <span class="php-quote">&quot;whitespace&quot;</span></span> )</code>
 
 			<div class="description short">
 				
@@ -183,7 +183,7 @@ <h4>Overrides</h4>
 
 			<td class="name"><div>
 			<a class="anchor" href="#_getTokens">#</a>
-			<code><a href="source-class-apemsel.AttributedString.TokenizedAttributedString.html#37-44" title="Go to source code">getTokens</a>( )</code>
+			<code><a href="source-class-apemsel.AttributedString.TokenizedAttributedString.html#36-43" title="Go to source code">getTokens</a>( )</code>
 
 			<div class="description short">
 				<p>Return all tokens</p>
@@ -209,14 +209,14 @@ <h4>Returns</h4>
 
 			<td class="attributes"><code>
 				 public 
-				in[]
+				int[]
 				
 				</code>
 			</td>
 
 			<td class="name"><div>
 			<a class="anchor" href="#_getTokenOffsets">#</a>
-			<code><a href="source-class-apemsel.AttributedString.TokenizedAttributedString.html#46-53" title="Go to source code">getTokenOffsets</a>( )</code>
+			<code><a href="source-class-apemsel.AttributedString.TokenizedAttributedString.html#45-52" title="Go to source code">getTokenOffsets</a>( )</code>
 
 			<div class="description short">
 				<p>Return all tokens' offsets</p>
@@ -229,7 +229,40 @@ <h4>Returns</h4>
 
 					<h4>Returns</h4>
 					<div class="list">
-						in[]<br>offsets
+						int[]<br>offsets
+					</div>
+
+
+
+
+			</div>
+			</div></td>
+		</tr>
+		<tr data-order="getTokenCount" id="_getTokenCount">
+
+			<td class="attributes"><code>
+				 public 
+				integer
+				
+				</code>
+			</td>
+
+			<td class="name"><div>
+			<a class="anchor" href="#_getTokenCount">#</a>
+			<code><a href="source-class-apemsel.AttributedString.TokenizedAttributedString.html#54-61" title="Go to source code">getTokenCount</a>( )</code>
+
+			<div class="description short">
+				<p>Return the number of tokens</p>
+			</div>
+
+			<div class="description detailed hidden">
+				<p>Return the number of tokens</p>
+
+
+
+					<h4>Returns</h4>
+					<div class="list">
+						integer<br>count
 					</div>
 
 
@@ -249,7 +282,7 @@ <h4>Returns</h4>
 
 			<td class="name"><div>
 			<a class="anchor" href="#_getToken">#</a>
-			<code><a href="source-class-apemsel.AttributedString.TokenizedAttributedString.html#55-63" title="Go to source code">getToken</a>( <span>integer <var>$i</var></span> )</code>
+			<code><a href="source-class-apemsel.AttributedString.TokenizedAttributedString.html#63-71" title="Go to source code">getToken</a>( <span>integer <var>$i</var></span> )</code>
 
 			<div class="description short">
 				<p>Get indicated token</p>
@@ -287,7 +320,7 @@ <h4>Returns</h4>
 
 			<td class="name"><div>
 			<a class="anchor" href="#_getTokenOffset">#</a>
-			<code><a href="source-class-apemsel.AttributedString.TokenizedAttributedString.html#65-73" title="Go to source code">getTokenOffset</a>( <span>integer <var>$i</var></span> )</code>
+			<code><a href="source-class-apemsel.AttributedString.TokenizedAttributedString.html#73-81" title="Go to source code">getTokenOffset</a>( <span>integer <var>$i</var></span> )</code>
 
 			<div class="description short">
 				<p>Get indicated token offset</p>
@@ -325,7 +358,7 @@ <h4>Returns</h4>
 
 			<td class="name"><div>
 			<a class="anchor" href="#_setTokenAttribute">#</a>
-			<code><a href="source-class-apemsel.AttributedString.TokenizedAttributedString.html#75-88" title="Go to source code">setTokenAttribute</a>( <span>integer <var>$i</var></span>, <span>string <var>$attribute</var></span>, <span>boolean <var>$state</var> = <span class="php-keyword1">true</span></span> )</code>
+			<code><a href="source-class-apemsel.AttributedString.TokenizedAttributedString.html#83-96" title="Go to source code">setTokenAttribute</a>( <span>integer <var>$i</var></span>, <span>string <var>$attribute</var></span>, <span>boolean <var>$state</var> = <span class="php-keyword1">true</span></span> )</code>
 
 			<div class="description short">
 				<p>Set a token to a given attribute and state</p>
@@ -363,7 +396,7 @@ <h4>Parameters</h4>
 
 			<td class="name"><div>
 			<a class="anchor" href="#_setTokenRangeAttribute">#</a>
-			<code><a href="source-class-apemsel.AttributedString.TokenizedAttributedString.html#90-103" title="Go to source code">setTokenRangeAttribute</a>( <span>integer <var>$from</var></span>, <span>integer <var>$to</var></span>, <span>string <var>$attribute</var></span>, <span>boolean <var>$state</var> = <span class="php-keyword1">true</span></span> )</code>
+			<code><a href="source-class-apemsel.AttributedString.TokenizedAttributedString.html#98-111" title="Go to source code">setTokenRangeAttribute</a>( <span>integer <var>$from</var></span>, <span>integer <var>$to</var></span>, <span>string <var>$attribute</var></span>, <span>boolean <var>$state</var> = <span class="php-keyword1">true</span></span> )</code>
 
 			<div class="description short">
 				<p>Set a range of tokens to a given attribute and state</p>
@@ -403,7 +436,7 @@ <h4>Parameters</h4>
 
 			<td class="name"><div>
 			<a class="anchor" href="#_setTokenDictionaryAttribute">#</a>
-			<code><a href="source-class-apemsel.AttributedString.TokenizedAttributedString.html#105-118" title="Go to source code">setTokenDictionaryAttribute</a>( <span>string[] <var>$dictionary</var></span>, <span>string <var>$attribute</var></span>, <span>boolean <var>$state</var> = <span class="php-keyword1">true</span></span> )</code>
+			<code><a href="source-class-apemsel.AttributedString.TokenizedAttributedString.html#113-126" title="Go to source code">setTokenDictionaryAttribute</a>( <span>string[] <var>$dictionary</var></span>, <span>string <var>$attribute</var></span>, <span>boolean <var>$state</var> = <span class="php-keyword1">true</span></span> )</code>
 
 			<div class="description short">
 				<p>Set all tokens matching given dictionary to attribute and state</p>
@@ -441,7 +474,7 @@ <h4>Parameters</h4>
 
 			<td class="name"><div>
 			<a class="anchor" href="#_attributesAtToken">#</a>
-			<code><a href="source-class-apemsel.AttributedString.TokenizedAttributedString.html#120-128" title="Go to source code">attributesAtToken</a>( <span>integer <var>$i</var></span> )</code>
+			<code><a href="source-class-apemsel.AttributedString.TokenizedAttributedString.html#128-136" title="Go to source code">attributesAtToken</a>( <span>integer <var>$i</var></span> )</code>
 
 			<div class="description short">
 				<p>Get all attribute of token at given index</p>
@@ -479,14 +512,14 @@ <h4>Returns</h4>
 
 			<td class="name"><div>
 			<a class="anchor" href="#_lowercaseTokens">#</a>
-			<code><a href="source-class-apemsel.AttributedString.TokenizedAttributedString.html#133-137" title="Go to source code">lowercaseTokens</a>( )</code>
+			<code><a href="source-class-apemsel.AttributedString.TokenizedAttributedString.html#138-145" title="Go to source code">lowercaseTokens</a>( )</code>
 
 			<div class="description short">
-				
+				<p>Convert all tokens to lower case</p>
 			</div>
 
 			<div class="description detailed hidden">
-				
+				<p>Convert all tokens to lower case</p>
 
 
 
@@ -500,25 +533,34 @@ <h4>Returns</h4>
 		<tr data-order="tokenizeOnWhitespace" id="_tokenizeOnWhitespace">
 
 			<td class="attributes"><code>
-				 protected static
-				
+				 public static
+				string[]
 				
 				</code>
 			</td>
 
 			<td class="name"><div>
 			<a class="anchor" href="#_tokenizeOnWhitespace">#</a>
-			<code><a href="source-class-apemsel.AttributedString.TokenizedAttributedString.html#139-143" title="Go to source code">tokenizeOnWhitespace</a>( <span> <var>$string</var></span> )</code>
+			<code><a href="source-class-apemsel.AttributedString.TokenizedAttributedString.html#147-157" title="Go to source code">tokenizeOnWhitespace</a>( <span>string <var>$string</var></span> )</code>
 
 			<div class="description short">
-				
+				<p>Tokenize a string on whitespace</p>
 			</div>
 
 			<div class="description detailed hidden">
-				
+				<p>Tokenize a string on whitespace</p>
 
 
+					<h4>Parameters</h4>
+					<div class="list"><dl>
+						<dt><var>$string</var></dt>
+						<dd>string to be tokenized</dd>
+					</dl></div>
 
+					<h4>Returns</h4>
+					<div class="list">
+						string[]<br>tokens
+					</div>
 
 
 
@@ -529,25 +571,34 @@ <h4>Returns</h4>
 		<tr data-order="tokenizeOnWords" id="_tokenizeOnWords">
 
 			<td class="attributes"><code>
-				 protected static
-				
+				 public static
+				string[]
 				
 				</code>
 			</td>
 
 			<td class="name"><div>
 			<a class="anchor" href="#_tokenizeOnWords">#</a>
-			<code><a href="source-class-apemsel.AttributedString.TokenizedAttributedString.html#145-147" title="Go to source code">tokenizeOnWords</a>( <span> <var>$string</var></span> )</code>
+			<code><a href="source-class-apemsel.AttributedString.TokenizedAttributedString.html#159-167" title="Go to source code">tokenizeOnWords</a>( <span>string <var>$string</var></span> )</code>
 
 			<div class="description short">
-				
+				<p>Tokenize a string on words</p>
 			</div>
 
 			<div class="description detailed hidden">
-				
+				<p>Tokenize a string on words</p>
 
 
+					<h4>Parameters</h4>
+					<div class="list"><dl>
+						<dt><var>$string</var></dt>
+						<dd>string to be tokenized</dd>
+					</dl></div>
 
+					<h4>Returns</h4>
+					<div class="list">
+						string[]<br>tokens
+					</div>
 
 
 
@@ -558,25 +609,36 @@ <h4>Returns</h4>
 		<tr data-order="tokenizeOnRegex" id="_tokenizeOnRegex">
 
 			<td class="attributes"><code>
-				 protected static
-				
+				 public static
+				string[]
 				
 				</code>
 			</td>
 
 			<td class="name"><div>
 			<a class="anchor" href="#_tokenizeOnRegex">#</a>
-			<code><a href="source-class-apemsel.AttributedString.TokenizedAttributedString.html#149-160" title="Go to source code">tokenizeOnRegex</a>( <span> <var>$string</var></span>, <span> <var>$pattern</var></span> )</code>
+			<code><a href="source-class-apemsel.AttributedString.TokenizedAttributedString.html#169-187" title="Go to source code">tokenizeOnRegex</a>( <span>string <var>$string</var></span>, <span>string <var>$pattern</var></span> )</code>
 
 			<div class="description short">
-				
+				<p>Tokenize a string with a given regex</p>
 			</div>
 
 			<div class="description detailed hidden">
-				
+				<p>Tokenize a string with a given regex</p>
 
 
+					<h4>Parameters</h4>
+					<div class="list"><dl>
+						<dt><var>$string</var></dt>
+						<dd>string to be tokenized</dd>
+						<dt><var>$pattern</var></dt>
+						<dd>regex. The token must be captured in the first subgroup.</dd>
+					</dl></div>
 
+					<h4>Returns</h4>
+					<div class="list">
+						string[]<br>tokens
+					</div>
 
 
 
@@ -631,7 +693,7 @@ <h4>Returns</h4>
 			</code></td>
 
 			<td class="name">
-					<a href="source-class-apemsel.AttributedString.TokenizedAttributedString.html#15" title="Go to source code"><var>$tokens</var></a>
+					<a href="source-class-apemsel.AttributedString.TokenizedAttributedString.html#14" title="Go to source code"><var>$tokens</var></a>
 
 				<div class="description short">
 					
@@ -656,7 +718,7 @@ <h4>Returns</h4>
 			</code></td>
 
 			<td class="name">
-					<a href="source-class-apemsel.AttributedString.TokenizedAttributedString.html#16" title="Go to source code"><var>$tokenOffsets</var></a>
+					<a href="source-class-apemsel.AttributedString.TokenizedAttributedString.html#15" title="Go to source code"><var>$tokenOffsets</var></a>
 
 				<div class="description short">
 					
diff --git a/doc/source-class-apemsel.AttributedString.AttributedString.html b/doc/source-class-apemsel.AttributedString.AttributedString.html
index 56220aa..94499ef 100644
--- a/doc/source-class-apemsel.AttributedString.AttributedString.html
+++ b/doc/source-class-apemsel.AttributedString.AttributedString.html
@@ -87,7 +87,7 @@ <h3>Classes</h3>
 </span><span id="3" class="l"><a href="#3">  3 </a>
 </span><span id="4" class="l"><a href="#4">  4 </a><span class="php-comment">/**
 </span></span><span id="5" class="l"><a href="#5">  5 </a><span class="php-comment"> * Basic class to work with attributed strings.
-</span></span><span id="6" class="l"><a href="#6">  6 </a><span class="php-comment"> * 
+</span></span><span id="6" class="l"><a href="#6">  6 </a><span class="php-comment"> *
 </span></span><span id="7" class="l"><a href="#7">  7 </a><span class="php-comment"> * Attributed strings are strings that can have multiple attributes per character of the string
 </span></span><span id="8" class="l"><a href="#8">  8 </a><span class="php-comment"> *
 </span></span><span id="9" class="l"><a href="#9">  9 </a><span class="php-comment"> * @author Adrian Pemsel &lt;apemsel@gmail.com&gt;
diff --git a/doc/source-class-apemsel.AttributedString.MutableAttributedString.html b/doc/source-class-apemsel.AttributedString.MutableAttributedString.html
index b977637..ca49c77 100644
--- a/doc/source-class-apemsel.AttributedString.MutableAttributedString.html
+++ b/doc/source-class-apemsel.AttributedString.MutableAttributedString.html
@@ -83,123 +83,122 @@ <h3>Classes</h3>
 <div id="right">
 	<div id="rightInner">
 <pre id="source"><code><span id="1" class="l"><a href="#1">  1 </a><span class="xlang">&lt;?php</span>
-</span><span id="2" class="l"><a href="#2">  2 </a>
-</span><span id="3" class="l"><a href="#3">  3 </a><span class="php-keyword1">namespace</span> apemsel\AttributedString;
-</span><span id="4" class="l"><a href="#4">  4 </a>
-</span><span id="5" class="l"><a href="#5">  5 </a><span class="php-comment">/**
-</span></span><span id="6" class="l"><a href="#6">  6 </a><span class="php-comment"> * Extends AttributedString to support a mutable (changeable) string.
-</span></span><span id="7" class="l"><a href="#7">  7 </a><span class="php-comment"> *
-</span></span><span id="8" class="l"><a href="#8">  8 </a><span class="php-comment"> * During insert and delete operations the attribute layers are updated as smart as possible.
-</span></span><span id="9" class="l"><a href="#9">  9 </a><span class="php-comment"> *
-</span></span><span id="10" class="l"><a href="#10"> 10 </a><span class="php-comment"> * @author Adrian Pemsel &lt;apemsel@gmail.com&gt;
-</span></span><span id="11" class="l"><a href="#11"> 11 </a><span class="php-comment"> */</span>
-</span><span id="12" class="l"><a href="#12"> 12 </a><span class="php-keyword1">class</span> MutableAttributedString <span class="php-keyword1">extends</span> AttributedString
-</span><span id="13" class="l"><a href="#13"> 13 </a>{
-</span><span id="14" class="l"><a href="#14"> 14 </a>  <span class="php-comment">/**
-</span></span><span id="15" class="l"><a href="#15"> 15 </a><span class="php-comment">   * Insert string at given offset
-</span></span><span id="16" class="l"><a href="#16"> 16 </a><span class="php-comment">   *
-</span></span><span id="17" class="l"><a href="#17"> 17 </a><span class="php-comment">   * @param int $pos offset
-</span></span><span id="18" class="l"><a href="#18"> 18 </a><span class="php-comment">   * @param string $string string to be inserted
-</span></span><span id="19" class="l"><a href="#19"> 19 </a><span class="php-comment">   */</span>
-</span><span id="20" class="l"><a href="#20"> 20 </a>  <span class="php-keyword1">public</span> <span class="php-keyword1">function</span> insert(<span class="php-var">$pos</span>, <span class="php-var">$string</span>) {
-</span><span id="21" class="l"><a href="#21"> 21 </a>    <span class="php-var">$length</span> = <span class="php-keyword2">mb_strlen</span>(<span class="php-var">$string</span>, <span class="php-quote">&quot;utf-8&quot;</span>);
-</span><span id="22" class="l"><a href="#22"> 22 </a>    
-</span><span id="23" class="l"><a href="#23"> 23 </a>    <span class="php-keyword1">if</span> (<span class="php-var">$pos</span> == <span class="php-var">$this</span>-&gt;length) { <span class="php-comment">// append instead</span>
-</span><span id="24" class="l"><a href="#24"> 24 </a>      <span class="php-var">$this</span>-&gt;string .= <span class="php-var">$string</span>;
-</span><span id="25" class="l"><a href="#25"> 25 </a>    } <span class="php-keyword1">else</span> { <span class="php-comment">// insert at $pos</span>
-</span><span id="26" class="l"><a href="#26"> 26 </a>      <span class="php-var">$this</span>-&gt;string = self::mb_substr_replace(<span class="php-var">$this</span>-&gt;string, <span class="php-var">$string</span>, <span class="php-var">$pos</span>, <span class="php-num">0</span>);
-</span><span id="27" class="l"><a href="#27"> 27 </a>    }
-</span><span id="28" class="l"><a href="#28"> 28 </a>    
-</span><span id="29" class="l"><a href="#29"> 29 </a>    <span class="php-var">$this</span>-&gt;length += <span class="php-var">$length</span>;
-</span><span id="30" class="l"><a href="#30"> 30 </a>    <span class="php-var">$this</span>-&gt;byte2Char = []; <span class="php-comment">// invalidate cache</span>
-</span><span id="31" class="l"><a href="#31"> 31 </a>    
-</span><span id="32" class="l"><a href="#32"> 32 </a>    <span class="php-keyword1">foreach</span> (<span class="php-var">$this</span>-&gt;attributes <span class="php-keyword1">as</span> <span class="php-var">$attribute</span> =&gt; &amp;<span class="php-var">$map</span>) {
-</span><span id="33" class="l"><a href="#33"> 33 </a>      <span class="php-comment">// Check state of surrounding map to determine state of inserted part</span>
-</span><span id="34" class="l"><a href="#34"> 34 </a>      <span class="php-var">$state</span> = <span class="php-keyword1">false</span>;
-</span><span id="35" class="l"><a href="#35"> 35 </a>      <span class="php-var">$maxPos</span> = <span class="php-keyword2">count</span>(<span class="php-var">$map</span>) - <span class="php-num">1</span>;
-</span><span id="36" class="l"><a href="#36"> 36 </a>      <span class="php-var">$leftState</span> = <span class="php-var">$map</span>[<span class="php-keyword2">min</span>(<span class="php-var">$maxPos</span>, <span class="php-var">$pos</span>)];
-</span><span id="37" class="l"><a href="#37"> 37 </a>      <span class="php-var">$rightState</span> = <span class="php-var">$map</span>[<span class="php-keyword2">min</span>(<span class="php-var">$maxPos</span>, <span class="php-var">$pos</span> + <span class="php-num">1</span>)];
-</span><span id="38" class="l"><a href="#38"> 38 </a>      
-</span><span id="39" class="l"><a href="#39"> 39 </a>      <span class="php-keyword1">if</span> (<span class="php-var">$leftState</span> == <span class="php-var">$rightState</span>) {
-</span><span id="40" class="l"><a href="#40"> 40 </a>        <span class="php-var">$state</span> = <span class="php-var">$leftState</span>;
-</span><span id="41" class="l"><a href="#41"> 41 </a>      }
-</span><span id="42" class="l"><a href="#42"> 42 </a>      
-</span><span id="43" class="l"><a href="#43"> 43 </a>      <span class="php-keyword2">array_splice</span>(<span class="php-var">$map</span>, <span class="php-var">$pos</span>, <span class="php-num">0</span>, <span class="php-keyword2">array_fill</span>(<span class="php-num">0</span>, <span class="php-var">$length</span>, <span class="php-var">$state</span>));
-</span><span id="44" class="l"><a href="#44"> 44 </a>    }
-</span><span id="45" class="l"><a href="#45"> 45 </a>  }
-</span><span id="46" class="l"><a href="#46"> 46 </a>  
-</span><span id="47" class="l"><a href="#47"> 47 </a>  <span class="php-comment">/**
-</span></span><span id="48" class="l"><a href="#48"> 48 </a><span class="php-comment">   * Delete substring of given offset and length
-</span></span><span id="49" class="l"><a href="#49"> 49 </a><span class="php-comment">   *
-</span></span><span id="50" class="l"><a href="#50"> 50 </a><span class="php-comment">   * @param int $pos offset
-</span></span><span id="51" class="l"><a href="#51"> 51 </a><span class="php-comment">   * @param int $length length
-</span></span><span id="52" class="l"><a href="#52"> 52 </a><span class="php-comment">   */</span>
-</span><span id="53" class="l"><a href="#53"> 53 </a>  <span class="php-keyword1">public</span> <span class="php-keyword1">function</span> <span class="php-keyword2">delete</span>(<span class="php-var">$pos</span>, <span class="php-var">$length</span>) {
-</span><span id="54" class="l"><a href="#54"> 54 </a>    <span class="php-var">$leftPart</span> = <span class="php-quote">&quot;&quot;</span>;
-</span><span id="55" class="l"><a href="#55"> 55 </a>    <span class="php-keyword1">if</span> (<span class="php-var">$pos</span> &gt; <span class="php-num">0</span>) {
-</span><span id="56" class="l"><a href="#56"> 56 </a>      <span class="php-var">$leftPart</span> = <span class="php-keyword2">mb_substr</span>(<span class="php-var">$this</span>-&gt;string, <span class="php-num">0</span>, <span class="php-var">$pos</span> - <span class="php-num">1</span>, <span class="php-quote">&quot;utf-8&quot;</span>);
-</span><span id="57" class="l"><a href="#57"> 57 </a>    }
-</span><span id="58" class="l"><a href="#58"> 58 </a>    
-</span><span id="59" class="l"><a href="#59"> 59 </a>    <span class="php-var">$rightPart</span> = <span class="php-quote">&quot;&quot;</span>;
-</span><span id="60" class="l"><a href="#60"> 60 </a>    <span class="php-keyword1">if</span> (<span class="php-var">$pos</span> + <span class="php-var">$length</span> &lt; <span class="php-var">$this</span>-&gt;length) {
-</span><span id="61" class="l"><a href="#61"> 61 </a>      <span class="php-var">$rightPart</span> = <span class="php-keyword2">mb_substr</span>(<span class="php-var">$this</span>-&gt;string, <span class="php-var">$pos</span> + <span class="php-var">$length</span>, <span class="php-keyword1">NULL</span>, <span class="php-quote">&quot;utf-8&quot;</span>);
-</span><span id="62" class="l"><a href="#62"> 62 </a>    }
-</span><span id="63" class="l"><a href="#63"> 63 </a>    
-</span><span id="64" class="l"><a href="#64"> 64 </a>    <span class="php-var">$this</span>-&gt;string = <span class="php-var">$leftPart</span>.<span class="php-var">$rightPart</span>;
-</span><span id="65" class="l"><a href="#65"> 65 </a>    <span class="php-var">$this</span>-&gt;length -= <span class="php-var">$length</span>;
-</span><span id="66" class="l"><a href="#66"> 66 </a>    
-</span><span id="67" class="l"><a href="#67"> 67 </a>    <span class="php-keyword1">foreach</span> (<span class="php-var">$this</span>-&gt;attributes <span class="php-keyword1">as</span> <span class="php-var">$attribute</span> =&gt; &amp;<span class="php-var">$map</span>) {
-</span><span id="68" class="l"><a href="#68"> 68 </a>      <span class="php-keyword2">array_splice</span>(<span class="php-var">$map</span>, <span class="php-var">$pos</span>, <span class="php-var">$length</span>);
-</span><span id="69" class="l"><a href="#69"> 69 </a>    }
-</span><span id="70" class="l"><a href="#70"> 70 </a>  }
-</span><span id="71" class="l"><a href="#71"> 71 </a>  
-</span><span id="72" class="l"><a href="#72"> 72 </a>  <span class="php-comment">/**
-</span></span><span id="73" class="l"><a href="#73"> 73 </a><span class="php-comment">   * Missing mb_substr_replace() implementation
-</span></span><span id="74" class="l"><a href="#74"> 74 </a><span class="php-comment">   *
-</span></span><span id="75" class="l"><a href="#75"> 75 </a><span class="php-comment">   * @see https://gist.github.com/stemar/8287074 Original source
-</span></span><span id="76" class="l"><a href="#76"> 76 </a><span class="php-comment">   * @param string $string string to work on
-</span></span><span id="77" class="l"><a href="#77"> 77 </a><span class="php-comment">   * @param string $replacement replacement string
-</span></span><span id="78" class="l"><a href="#78"> 78 </a><span class="php-comment">   * @param int $start offset
-</span></span><span id="79" class="l"><a href="#79"> 79 </a><span class="php-comment">   * @param int $length length
-</span></span><span id="80" class="l"><a href="#80"> 80 </a><span class="php-comment">   * @return string modified string
-</span></span><span id="81" class="l"><a href="#81"> 81 </a><span class="php-comment">   */</span>
-</span><span id="82" class="l"><a href="#82"> 82 </a>  <span class="php-keyword1">protected</span> <span class="php-keyword1">static</span> <span class="php-keyword1">function</span> mb_substr_replace(<span class="php-var">$string</span>, <span class="php-var">$replacement</span>, <span class="php-var">$start</span>, <span class="php-var">$length</span> = <span class="php-keyword1">NULL</span>) {
-</span><span id="83" class="l"><a href="#83"> 83 </a>    
-</span><span id="84" class="l"><a href="#84"> 84 </a>    <span class="php-keyword1">if</span> (<span class="php-keyword2">is_array</span>(<span class="php-var">$string</span>)) {
-</span><span id="85" class="l"><a href="#85"> 85 </a>      <span class="php-var">$num</span> = <span class="php-keyword2">count</span>(<span class="php-var">$string</span>);
-</span><span id="86" class="l"><a href="#86"> 86 </a>      <span class="php-comment">// $replacement</span>
-</span><span id="87" class="l"><a href="#87"> 87 </a>      <span class="php-var">$replacement</span> = <span class="php-keyword2">is_array</span>(<span class="php-var">$replacement</span>) ? <span class="php-keyword2">array_slice</span>(<span class="php-var">$replacement</span>, <span class="php-num">0</span>, <span class="php-var">$num</span>) : <span class="php-keyword2">array_pad</span>(<span class="php-keyword1">array</span>(<span class="php-var">$replacement</span>), <span class="php-var">$num</span>, <span class="php-var">$replacement</span>);
-</span><span id="88" class="l"><a href="#88"> 88 </a>      <span class="php-comment">// $start</span>
-</span><span id="89" class="l"><a href="#89"> 89 </a>      <span class="php-keyword1">if</span> (<span class="php-keyword2">is_array</span>(<span class="php-var">$start</span>)) {
-</span><span id="90" class="l"><a href="#90"> 90 </a>        <span class="php-var">$start</span> = <span class="php-keyword2">array_slice</span>(<span class="php-var">$start</span>, <span class="php-num">0</span>, <span class="php-var">$num</span>);
-</span><span id="91" class="l"><a href="#91"> 91 </a>        <span class="php-keyword1">foreach</span> (<span class="php-var">$start</span> <span class="php-keyword1">as</span> <span class="php-var">$key</span> =&gt; <span class="php-var">$value</span>) {
-</span><span id="92" class="l"><a href="#92"> 92 </a>          <span class="php-var">$start</span>[<span class="php-var">$key</span>] = <span class="php-keyword2">is_int</span>(<span class="php-var">$value</span>) ? <span class="php-var">$value</span> : <span class="php-num">0</span>;
-</span><span id="93" class="l"><a href="#93"> 93 </a>        }
-</span><span id="94" class="l"><a href="#94"> 94 </a>      } <span class="php-keyword1">else</span> {
-</span><span id="95" class="l"><a href="#95"> 95 </a>        <span class="php-var">$start</span> = <span class="php-keyword2">array_pad</span>(<span class="php-keyword1">array</span>(<span class="php-var">$start</span>), <span class="php-var">$num</span>, <span class="php-var">$start</span>);
-</span><span id="96" class="l"><a href="#96"> 96 </a>      }
-</span><span id="97" class="l"><a href="#97"> 97 </a>      <span class="php-comment">// $length</span>
-</span><span id="98" class="l"><a href="#98"> 98 </a>      <span class="php-keyword1">if</span> (!<span class="php-keyword1">isset</span>(<span class="php-var">$length</span>)) {
-</span><span id="99" class="l"><a href="#99"> 99 </a>        <span class="php-var">$length</span> = <span class="php-keyword2">array_fill</span>(<span class="php-num">0</span>, <span class="php-var">$num</span>, <span class="php-num">0</span>);
-</span><span id="100" class="l"><a href="#100">100 </a>      } <span class="php-keyword1">elseif</span> (<span class="php-keyword2">is_array</span>(<span class="php-var">$length</span>)) {
-</span><span id="101" class="l"><a href="#101">101 </a>        <span class="php-var">$length</span> = <span class="php-keyword2">array_slice</span>(<span class="php-var">$length</span>, <span class="php-num">0</span>, <span class="php-var">$num</span>);
-</span><span id="102" class="l"><a href="#102">102 </a>        <span class="php-keyword1">foreach</span> (<span class="php-var">$length</span> <span class="php-keyword1">as</span> <span class="php-var">$key</span> =&gt; <span class="php-var">$value</span>)
-</span><span id="103" class="l"><a href="#103">103 </a>          <span class="php-var">$length</span>[<span class="php-var">$key</span>] = <span class="php-keyword1">isset</span>(<span class="php-var">$value</span>) ? (<span class="php-keyword2">is_int</span>(<span class="php-var">$value</span>) ? <span class="php-var">$value</span> : <span class="php-var">$num</span>) : <span class="php-num">0</span>;
-</span><span id="104" class="l"><a href="#104">104 </a>      } <span class="php-keyword1">else</span> {
-</span><span id="105" class="l"><a href="#105">105 </a>        <span class="php-var">$length</span> = <span class="php-keyword2">array_pad</span>(<span class="php-keyword1">array</span>(<span class="php-var">$length</span>), <span class="php-var">$num</span>, <span class="php-var">$length</span>);
-</span><span id="106" class="l"><a href="#106">106 </a>      }
-</span><span id="107" class="l"><a href="#107">107 </a>      <span class="php-comment">// Recursive call</span>
-</span><span id="108" class="l"><a href="#108">108 </a>      <span class="php-keyword1">return</span> <span class="php-keyword2">array_map</span>(__FUNCTION__, <span class="php-var">$string</span>, <span class="php-var">$replacement</span>, <span class="php-var">$start</span>, <span class="php-var">$length</span>);
-</span><span id="109" class="l"><a href="#109">109 </a>    }
-</span><span id="110" class="l"><a href="#110">110 </a>    <span class="php-keyword2">preg_match_all</span>(<span class="php-quote">'/./us'</span>, (string)<span class="php-var">$string</span>, <span class="php-var">$smatches</span>);
-</span><span id="111" class="l"><a href="#111">111 </a>    <span class="php-keyword2">preg_match_all</span>(<span class="php-quote">'/./us'</span>, (string)<span class="php-var">$replacement</span>, <span class="php-var">$rmatches</span>);
-</span><span id="112" class="l"><a href="#112">112 </a>    <span class="php-keyword1">if</span> (<span class="php-var">$length</span> === <span class="php-keyword1">NULL</span>) <span class="php-var">$length</span> = <span class="php-keyword2">mb_strlen</span>(<span class="php-var">$string</span>, <span class="php-quote">&quot;utf-8&quot;</span>);
-</span><span id="113" class="l"><a href="#113">113 </a>    <span class="php-keyword2">array_splice</span>(<span class="php-var">$smatches</span>[<span class="php-num">0</span>], <span class="php-var">$start</span>, <span class="php-var">$length</span>, <span class="php-var">$rmatches</span>[<span class="php-num">0</span>]);
-</span><span id="114" class="l"><a href="#114">114 </a>    
-</span><span id="115" class="l"><a href="#115">115 </a>    <span class="php-keyword1">return</span> <span class="php-keyword2">join</span>(<span class="php-var">$smatches</span>[<span class="php-num">0</span>]);
-</span><span id="116" class="l"><a href="#116">116 </a>  }
-</span><span id="117" class="l"><a href="#117">117 </a>}
-</span><span id="118" class="l"><a href="#118">118 </a></span></code></pre>
+</span><span id="2" class="l"><a href="#2">  2 </a><span class="php-keyword1">namespace</span> apemsel\AttributedString;
+</span><span id="3" class="l"><a href="#3">  3 </a>
+</span><span id="4" class="l"><a href="#4">  4 </a><span class="php-comment">/**
+</span></span><span id="5" class="l"><a href="#5">  5 </a><span class="php-comment"> * Extends AttributedString to support a mutable (changeable) string.
+</span></span><span id="6" class="l"><a href="#6">  6 </a><span class="php-comment"> *
+</span></span><span id="7" class="l"><a href="#7">  7 </a><span class="php-comment"> * During insert and delete operations the attribute layers are updated as smart as possible.
+</span></span><span id="8" class="l"><a href="#8">  8 </a><span class="php-comment"> *
+</span></span><span id="9" class="l"><a href="#9">  9 </a><span class="php-comment"> * @author Adrian Pemsel &lt;apemsel@gmail.com&gt;
+</span></span><span id="10" class="l"><a href="#10"> 10 </a><span class="php-comment"> */</span>
+</span><span id="11" class="l"><a href="#11"> 11 </a><span class="php-keyword1">class</span> MutableAttributedString <span class="php-keyword1">extends</span> AttributedString
+</span><span id="12" class="l"><a href="#12"> 12 </a>{
+</span><span id="13" class="l"><a href="#13"> 13 </a>  <span class="php-comment">/**
+</span></span><span id="14" class="l"><a href="#14"> 14 </a><span class="php-comment">   * Insert string at given offset
+</span></span><span id="15" class="l"><a href="#15"> 15 </a><span class="php-comment">   *
+</span></span><span id="16" class="l"><a href="#16"> 16 </a><span class="php-comment">   * @param int $pos offset
+</span></span><span id="17" class="l"><a href="#17"> 17 </a><span class="php-comment">   * @param string $string string to be inserted
+</span></span><span id="18" class="l"><a href="#18"> 18 </a><span class="php-comment">   */</span>
+</span><span id="19" class="l"><a href="#19"> 19 </a>  <span class="php-keyword1">public</span> <span class="php-keyword1">function</span> insert(<span class="php-var">$pos</span>, <span class="php-var">$string</span>) {
+</span><span id="20" class="l"><a href="#20"> 20 </a>    <span class="php-var">$length</span> = <span class="php-keyword2">mb_strlen</span>(<span class="php-var">$string</span>, <span class="php-quote">&quot;utf-8&quot;</span>);
+</span><span id="21" class="l"><a href="#21"> 21 </a>    
+</span><span id="22" class="l"><a href="#22"> 22 </a>    <span class="php-keyword1">if</span> (<span class="php-var">$pos</span> == <span class="php-var">$this</span>-&gt;length) { <span class="php-comment">// append instead</span>
+</span><span id="23" class="l"><a href="#23"> 23 </a>      <span class="php-var">$this</span>-&gt;string .= <span class="php-var">$string</span>;
+</span><span id="24" class="l"><a href="#24"> 24 </a>    } <span class="php-keyword1">else</span> { <span class="php-comment">// insert at $pos</span>
+</span><span id="25" class="l"><a href="#25"> 25 </a>      <span class="php-var">$this</span>-&gt;string = self::mb_substr_replace(<span class="php-var">$this</span>-&gt;string, <span class="php-var">$string</span>, <span class="php-var">$pos</span>, <span class="php-num">0</span>);
+</span><span id="26" class="l"><a href="#26"> 26 </a>    }
+</span><span id="27" class="l"><a href="#27"> 27 </a>    
+</span><span id="28" class="l"><a href="#28"> 28 </a>    <span class="php-var">$this</span>-&gt;length += <span class="php-var">$length</span>;
+</span><span id="29" class="l"><a href="#29"> 29 </a>    <span class="php-var">$this</span>-&gt;byte2Char = []; <span class="php-comment">// invalidate cache</span>
+</span><span id="30" class="l"><a href="#30"> 30 </a>    
+</span><span id="31" class="l"><a href="#31"> 31 </a>    <span class="php-keyword1">foreach</span> (<span class="php-var">$this</span>-&gt;attributes <span class="php-keyword1">as</span> <span class="php-var">$attribute</span> =&gt; &amp;<span class="php-var">$map</span>) {
+</span><span id="32" class="l"><a href="#32"> 32 </a>      <span class="php-comment">// Check state of surrounding map to determine state of inserted part</span>
+</span><span id="33" class="l"><a href="#33"> 33 </a>      <span class="php-var">$state</span> = <span class="php-keyword1">false</span>;
+</span><span id="34" class="l"><a href="#34"> 34 </a>      <span class="php-var">$maxPos</span> = <span class="php-keyword2">count</span>(<span class="php-var">$map</span>) - <span class="php-num">1</span>;
+</span><span id="35" class="l"><a href="#35"> 35 </a>      <span class="php-var">$leftState</span> = <span class="php-var">$map</span>[<span class="php-keyword2">min</span>(<span class="php-var">$maxPos</span>, <span class="php-var">$pos</span>)];
+</span><span id="36" class="l"><a href="#36"> 36 </a>      <span class="php-var">$rightState</span> = <span class="php-var">$map</span>[<span class="php-keyword2">min</span>(<span class="php-var">$maxPos</span>, <span class="php-var">$pos</span> + <span class="php-num">1</span>)];
+</span><span id="37" class="l"><a href="#37"> 37 </a>      
+</span><span id="38" class="l"><a href="#38"> 38 </a>      <span class="php-keyword1">if</span> (<span class="php-var">$leftState</span> == <span class="php-var">$rightState</span>) {
+</span><span id="39" class="l"><a href="#39"> 39 </a>        <span class="php-var">$state</span> = <span class="php-var">$leftState</span>;
+</span><span id="40" class="l"><a href="#40"> 40 </a>      }
+</span><span id="41" class="l"><a href="#41"> 41 </a>      
+</span><span id="42" class="l"><a href="#42"> 42 </a>      <span class="php-keyword2">array_splice</span>(<span class="php-var">$map</span>, <span class="php-var">$pos</span>, <span class="php-num">0</span>, <span class="php-keyword2">array_fill</span>(<span class="php-num">0</span>, <span class="php-var">$length</span>, <span class="php-var">$state</span>));
+</span><span id="43" class="l"><a href="#43"> 43 </a>    }
+</span><span id="44" class="l"><a href="#44"> 44 </a>  }
+</span><span id="45" class="l"><a href="#45"> 45 </a>  
+</span><span id="46" class="l"><a href="#46"> 46 </a>  <span class="php-comment">/**
+</span></span><span id="47" class="l"><a href="#47"> 47 </a><span class="php-comment">   * Delete substring of given offset and length
+</span></span><span id="48" class="l"><a href="#48"> 48 </a><span class="php-comment">   *
+</span></span><span id="49" class="l"><a href="#49"> 49 </a><span class="php-comment">   * @param int $pos offset
+</span></span><span id="50" class="l"><a href="#50"> 50 </a><span class="php-comment">   * @param int $length length
+</span></span><span id="51" class="l"><a href="#51"> 51 </a><span class="php-comment">   */</span>
+</span><span id="52" class="l"><a href="#52"> 52 </a>  <span class="php-keyword1">public</span> <span class="php-keyword1">function</span> <span class="php-keyword2">delete</span>(<span class="php-var">$pos</span>, <span class="php-var">$length</span>) {
+</span><span id="53" class="l"><a href="#53"> 53 </a>    <span class="php-var">$leftPart</span> = <span class="php-quote">&quot;&quot;</span>;
+</span><span id="54" class="l"><a href="#54"> 54 </a>    <span class="php-keyword1">if</span> (<span class="php-var">$pos</span> &gt; <span class="php-num">0</span>) {
+</span><span id="55" class="l"><a href="#55"> 55 </a>      <span class="php-var">$leftPart</span> = <span class="php-keyword2">mb_substr</span>(<span class="php-var">$this</span>-&gt;string, <span class="php-num">0</span>, <span class="php-var">$pos</span> - <span class="php-num">1</span>, <span class="php-quote">&quot;utf-8&quot;</span>);
+</span><span id="56" class="l"><a href="#56"> 56 </a>    }
+</span><span id="57" class="l"><a href="#57"> 57 </a>    
+</span><span id="58" class="l"><a href="#58"> 58 </a>    <span class="php-var">$rightPart</span> = <span class="php-quote">&quot;&quot;</span>;
+</span><span id="59" class="l"><a href="#59"> 59 </a>    <span class="php-keyword1">if</span> (<span class="php-var">$pos</span> + <span class="php-var">$length</span> &lt; <span class="php-var">$this</span>-&gt;length) {
+</span><span id="60" class="l"><a href="#60"> 60 </a>      <span class="php-var">$rightPart</span> = <span class="php-keyword2">mb_substr</span>(<span class="php-var">$this</span>-&gt;string, <span class="php-var">$pos</span> + <span class="php-var">$length</span>, <span class="php-keyword1">NULL</span>, <span class="php-quote">&quot;utf-8&quot;</span>);
+</span><span id="61" class="l"><a href="#61"> 61 </a>    }
+</span><span id="62" class="l"><a href="#62"> 62 </a>    
+</span><span id="63" class="l"><a href="#63"> 63 </a>    <span class="php-var">$this</span>-&gt;string = <span class="php-var">$leftPart</span>.<span class="php-var">$rightPart</span>;
+</span><span id="64" class="l"><a href="#64"> 64 </a>    <span class="php-var">$this</span>-&gt;length -= <span class="php-var">$length</span>;
+</span><span id="65" class="l"><a href="#65"> 65 </a>    
+</span><span id="66" class="l"><a href="#66"> 66 </a>    <span class="php-keyword1">foreach</span> (<span class="php-var">$this</span>-&gt;attributes <span class="php-keyword1">as</span> <span class="php-var">$attribute</span> =&gt; &amp;<span class="php-var">$map</span>) {
+</span><span id="67" class="l"><a href="#67"> 67 </a>      <span class="php-keyword2">array_splice</span>(<span class="php-var">$map</span>, <span class="php-var">$pos</span>, <span class="php-var">$length</span>);
+</span><span id="68" class="l"><a href="#68"> 68 </a>    }
+</span><span id="69" class="l"><a href="#69"> 69 </a>  }
+</span><span id="70" class="l"><a href="#70"> 70 </a>  
+</span><span id="71" class="l"><a href="#71"> 71 </a>  <span class="php-comment">/**
+</span></span><span id="72" class="l"><a href="#72"> 72 </a><span class="php-comment">   * Missing mb_substr_replace() implementation
+</span></span><span id="73" class="l"><a href="#73"> 73 </a><span class="php-comment">   *
+</span></span><span id="74" class="l"><a href="#74"> 74 </a><span class="php-comment">   * @see https://gist.github.com/stemar/8287074 Original source
+</span></span><span id="75" class="l"><a href="#75"> 75 </a><span class="php-comment">   * @param string $string string to work on
+</span></span><span id="76" class="l"><a href="#76"> 76 </a><span class="php-comment">   * @param string $replacement replacement string
+</span></span><span id="77" class="l"><a href="#77"> 77 </a><span class="php-comment">   * @param int $start offset
+</span></span><span id="78" class="l"><a href="#78"> 78 </a><span class="php-comment">   * @param int $length length
+</span></span><span id="79" class="l"><a href="#79"> 79 </a><span class="php-comment">   * @return string modified string
+</span></span><span id="80" class="l"><a href="#80"> 80 </a><span class="php-comment">   */</span>
+</span><span id="81" class="l"><a href="#81"> 81 </a>  <span class="php-keyword1">protected</span> <span class="php-keyword1">static</span> <span class="php-keyword1">function</span> mb_substr_replace(<span class="php-var">$string</span>, <span class="php-var">$replacement</span>, <span class="php-var">$start</span>, <span class="php-var">$length</span> = <span class="php-keyword1">NULL</span>) {
+</span><span id="82" class="l"><a href="#82"> 82 </a>    
+</span><span id="83" class="l"><a href="#83"> 83 </a>    <span class="php-keyword1">if</span> (<span class="php-keyword2">is_array</span>(<span class="php-var">$string</span>)) {
+</span><span id="84" class="l"><a href="#84"> 84 </a>      <span class="php-var">$num</span> = <span class="php-keyword2">count</span>(<span class="php-var">$string</span>);
+</span><span id="85" class="l"><a href="#85"> 85 </a>      <span class="php-comment">// $replacement</span>
+</span><span id="86" class="l"><a href="#86"> 86 </a>      <span class="php-var">$replacement</span> = <span class="php-keyword2">is_array</span>(<span class="php-var">$replacement</span>) ? <span class="php-keyword2">array_slice</span>(<span class="php-var">$replacement</span>, <span class="php-num">0</span>, <span class="php-var">$num</span>) : <span class="php-keyword2">array_pad</span>(<span class="php-keyword1">array</span>(<span class="php-var">$replacement</span>), <span class="php-var">$num</span>, <span class="php-var">$replacement</span>);
+</span><span id="87" class="l"><a href="#87"> 87 </a>      <span class="php-comment">// $start</span>
+</span><span id="88" class="l"><a href="#88"> 88 </a>      <span class="php-keyword1">if</span> (<span class="php-keyword2">is_array</span>(<span class="php-var">$start</span>)) {
+</span><span id="89" class="l"><a href="#89"> 89 </a>        <span class="php-var">$start</span> = <span class="php-keyword2">array_slice</span>(<span class="php-var">$start</span>, <span class="php-num">0</span>, <span class="php-var">$num</span>);
+</span><span id="90" class="l"><a href="#90"> 90 </a>        <span class="php-keyword1">foreach</span> (<span class="php-var">$start</span> <span class="php-keyword1">as</span> <span class="php-var">$key</span> =&gt; <span class="php-var">$value</span>) {
+</span><span id="91" class="l"><a href="#91"> 91 </a>          <span class="php-var">$start</span>[<span class="php-var">$key</span>] = <span class="php-keyword2">is_int</span>(<span class="php-var">$value</span>) ? <span class="php-var">$value</span> : <span class="php-num">0</span>;
+</span><span id="92" class="l"><a href="#92"> 92 </a>        }
+</span><span id="93" class="l"><a href="#93"> 93 </a>      } <span class="php-keyword1">else</span> {
+</span><span id="94" class="l"><a href="#94"> 94 </a>        <span class="php-var">$start</span> = <span class="php-keyword2">array_pad</span>(<span class="php-keyword1">array</span>(<span class="php-var">$start</span>), <span class="php-var">$num</span>, <span class="php-var">$start</span>);
+</span><span id="95" class="l"><a href="#95"> 95 </a>      }
+</span><span id="96" class="l"><a href="#96"> 96 </a>      <span class="php-comment">// $length</span>
+</span><span id="97" class="l"><a href="#97"> 97 </a>      <span class="php-keyword1">if</span> (!<span class="php-keyword1">isset</span>(<span class="php-var">$length</span>)) {
+</span><span id="98" class="l"><a href="#98"> 98 </a>        <span class="php-var">$length</span> = <span class="php-keyword2">array_fill</span>(<span class="php-num">0</span>, <span class="php-var">$num</span>, <span class="php-num">0</span>);
+</span><span id="99" class="l"><a href="#99"> 99 </a>      } <span class="php-keyword1">elseif</span> (<span class="php-keyword2">is_array</span>(<span class="php-var">$length</span>)) {
+</span><span id="100" class="l"><a href="#100">100 </a>        <span class="php-var">$length</span> = <span class="php-keyword2">array_slice</span>(<span class="php-var">$length</span>, <span class="php-num">0</span>, <span class="php-var">$num</span>);
+</span><span id="101" class="l"><a href="#101">101 </a>        <span class="php-keyword1">foreach</span> (<span class="php-var">$length</span> <span class="php-keyword1">as</span> <span class="php-var">$key</span> =&gt; <span class="php-var">$value</span>)
+</span><span id="102" class="l"><a href="#102">102 </a>          <span class="php-var">$length</span>[<span class="php-var">$key</span>] = <span class="php-keyword1">isset</span>(<span class="php-var">$value</span>) ? (<span class="php-keyword2">is_int</span>(<span class="php-var">$value</span>) ? <span class="php-var">$value</span> : <span class="php-var">$num</span>) : <span class="php-num">0</span>;
+</span><span id="103" class="l"><a href="#103">103 </a>      } <span class="php-keyword1">else</span> {
+</span><span id="104" class="l"><a href="#104">104 </a>        <span class="php-var">$length</span> = <span class="php-keyword2">array_pad</span>(<span class="php-keyword1">array</span>(<span class="php-var">$length</span>), <span class="php-var">$num</span>, <span class="php-var">$length</span>);
+</span><span id="105" class="l"><a href="#105">105 </a>      }
+</span><span id="106" class="l"><a href="#106">106 </a>      <span class="php-comment">// Recursive call</span>
+</span><span id="107" class="l"><a href="#107">107 </a>      <span class="php-keyword1">return</span> <span class="php-keyword2">array_map</span>(__FUNCTION__, <span class="php-var">$string</span>, <span class="php-var">$replacement</span>, <span class="php-var">$start</span>, <span class="php-var">$length</span>);
+</span><span id="108" class="l"><a href="#108">108 </a>    }
+</span><span id="109" class="l"><a href="#109">109 </a>    <span class="php-keyword2">preg_match_all</span>(<span class="php-quote">'/./us'</span>, (string)<span class="php-var">$string</span>, <span class="php-var">$smatches</span>);
+</span><span id="110" class="l"><a href="#110">110 </a>    <span class="php-keyword2">preg_match_all</span>(<span class="php-quote">'/./us'</span>, (string)<span class="php-var">$replacement</span>, <span class="php-var">$rmatches</span>);
+</span><span id="111" class="l"><a href="#111">111 </a>    <span class="php-keyword1">if</span> (<span class="php-var">$length</span> === <span class="php-keyword1">NULL</span>) <span class="php-var">$length</span> = <span class="php-keyword2">mb_strlen</span>(<span class="php-var">$string</span>, <span class="php-quote">&quot;utf-8&quot;</span>);
+</span><span id="112" class="l"><a href="#112">112 </a>    <span class="php-keyword2">array_splice</span>(<span class="php-var">$smatches</span>[<span class="php-num">0</span>], <span class="php-var">$start</span>, <span class="php-var">$length</span>, <span class="php-var">$rmatches</span>[<span class="php-num">0</span>]);
+</span><span id="113" class="l"><a href="#113">113 </a>    
+</span><span id="114" class="l"><a href="#114">114 </a>    <span class="php-keyword1">return</span> <span class="php-keyword2">join</span>(<span class="php-var">$smatches</span>[<span class="php-num">0</span>]);
+</span><span id="115" class="l"><a href="#115">115 </a>  }
+</span><span id="116" class="l"><a href="#116">116 </a>}
+</span><span id="117" class="l"><a href="#117">117 </a></span></code></pre>
 	</div>
 
 	<div id="footer">
diff --git a/doc/source-class-apemsel.AttributedString.TokenizedAttributedString.html b/doc/source-class-apemsel.AttributedString.TokenizedAttributedString.html
index 42e4b3d..6217ee5 100644
--- a/doc/source-class-apemsel.AttributedString.TokenizedAttributedString.html
+++ b/doc/source-class-apemsel.AttributedString.TokenizedAttributedString.html
@@ -83,167 +83,194 @@ <h3>Classes</h3>
 <div id="right">
 	<div id="rightInner">
 <pre id="source"><code><span id="1" class="l"><a href="#1">  1 </a><span class="xlang">&lt;?php</span>
-</span><span id="2" class="l"><a href="#2">  2 </a>
-</span><span id="3" class="l"><a href="#3">  3 </a><span class="php-keyword1">namespace</span> apemsel\AttributedString;
-</span><span id="4" class="l"><a href="#4">  4 </a>
-</span><span id="5" class="l"><a href="#5">  5 </a><span class="php-comment">/**
-</span></span><span id="6" class="l"><a href="#6">  6 </a><span class="php-comment"> * Extends AttributedString to support a tokenized string.
-</span></span><span id="7" class="l"><a href="#7">  7 </a><span class="php-comment"> *
-</span></span><span id="8" class="l"><a href="#8">  8 </a><span class="php-comment"> * You can mix working with tokens with working on string ranges using the AttributedString methods.
-</span></span><span id="9" class="l"><a href="#9">  9 </a><span class="php-comment"> * The original string is preserved.
-</span></span><span id="10" class="l"><a href="#10"> 10 </a><span class="php-comment"> *
-</span></span><span id="11" class="l"><a href="#11"> 11 </a><span class="php-comment"> * @author Adrian Pemsel &lt;apemsel@gmail.com&gt;
-</span></span><span id="12" class="l"><a href="#12"> 12 </a><span class="php-comment"> */</span>
-</span><span id="13" class="l"><a href="#13"> 13 </a><span class="php-keyword1">class</span> TokenizedAttributedString <span class="php-keyword1">extends</span> AttributedString
-</span><span id="14" class="l"><a href="#14"> 14 </a>{
-</span><span id="15" class="l"><a href="#15"> 15 </a>  <span class="php-keyword1">protected</span> <span class="php-var">$tokens</span>;
-</span><span id="16" class="l"><a href="#16"> 16 </a>  <span class="php-keyword1">protected</span> <span class="php-var">$tokenOffsets</span>;
-</span><span id="17" class="l"><a href="#17"> 17 </a>  
-</span><span id="18" class="l"><a href="#18"> 18 </a>  <span class="php-comment">/**
-</span></span><span id="19" class="l"><a href="#19"> 19 </a><span class="php-comment">   * @param string|AttributedString $string String to work on
-</span></span><span id="20" class="l"><a href="#20"> 20 </a><span class="php-comment">   * @param string $tokenizer Tokenizer to use, either &quot;whitespace&quot;, &quot;word&quot; or a custom regex
-</span></span><span id="21" class="l"><a href="#21"> 21 </a><span class="php-comment">   */</span>
-</span><span id="22" class="l"><a href="#22"> 22 </a>  <span class="php-keyword1">public</span> <span class="php-keyword1">function</span> __construct(<span class="php-var">$string</span>, <span class="php-var">$tokenizer</span> = <span class="php-quote">&quot;whitespace&quot;</span>) {
-</span><span id="23" class="l"><a href="#23"> 23 </a>    <span class="php-var">$tokenizerFunction</span> = <span class="php-quote">&quot;tokenizeOn&quot;</span>.<span class="php-keyword2">ucfirst</span>(<span class="php-var">$tokenizer</span>);
-</span><span id="24" class="l"><a href="#24"> 24 </a>
-</span><span id="25" class="l"><a href="#25"> 25 </a>    <span class="php-keyword1">if</span> (<span class="php-var">$tokenizer</span>[<span class="php-num">0</span>] == <span class="php-quote">&quot;/&quot;</span>) {
-</span><span id="26" class="l"><a href="#26"> 26 </a>      <span class="php-keyword1">list</span>(<span class="php-var">$this</span>-&gt;tokens, <span class="php-var">$this</span>-&gt;tokenOffsets) = self::tokenizeOnRegex(<span class="php-var">$string</span>, <span class="php-var">$tokenizer</span>);
-</span><span id="27" class="l"><a href="#27"> 27 </a>    } <span class="php-keyword1">else</span> {
-</span><span id="28" class="l"><a href="#28"> 28 </a>      <span class="php-keyword1">if</span> (!<span class="php-keyword2">method_exists</span>(<span class="php-quote">&quot;apemsel\AttributedString\TokenizedAttributedString&quot;</span>, <span class="php-var">$tokenizerFunction</span>)) {
-</span><span id="29" class="l"><a href="#29"> 29 </a>        <span class="php-keyword1">throw</span> <span class="php-keyword1">new</span> \InvalidArgumentException(<span class="php-quote">&quot;Unknown tokenizer </span><span class="php-var">$tokenizer</span><span class="php-quote">&quot;</span>);
-</span><span id="30" class="l"><a href="#30"> 30 </a>      }
-</span><span id="31" class="l"><a href="#31"> 31 </a>      <span class="php-keyword1">list</span>(<span class="php-var">$this</span>-&gt;tokens, <span class="php-var">$this</span>-&gt;tokenOffsets) = self::<span class="php-var">$tokenizerFunction</span>(<span class="php-var">$string</span>);
-</span><span id="32" class="l"><a href="#32"> 32 </a>    }
-</span><span id="33" class="l"><a href="#33"> 33 </a>    
-</span><span id="34" class="l"><a href="#34"> 34 </a>    parent::__construct(<span class="php-var">$string</span>);
-</span><span id="35" class="l"><a href="#35"> 35 </a>  }
-</span><span id="36" class="l"><a href="#36"> 36 </a>  
-</span><span id="37" class="l"><a href="#37"> 37 </a>  <span class="php-comment">/**
-</span></span><span id="38" class="l"><a href="#38"> 38 </a><span class="php-comment">   * Return all tokens
-</span></span><span id="39" class="l"><a href="#39"> 39 </a><span class="php-comment">   *
-</span></span><span id="40" class="l"><a href="#40"> 40 </a><span class="php-comment">   * @return string[] tokens
-</span></span><span id="41" class="l"><a href="#41"> 41 </a><span class="php-comment">   */</span>
-</span><span id="42" class="l"><a href="#42"> 42 </a>  <span class="php-keyword1">public</span> <span class="php-keyword1">function</span> getTokens() {
-</span><span id="43" class="l"><a href="#43"> 43 </a>    <span class="php-keyword1">return</span> <span class="php-var">$this</span>-&gt;tokens;
-</span><span id="44" class="l"><a href="#44"> 44 </a>  }
-</span><span id="45" class="l"><a href="#45"> 45 </a>  
-</span><span id="46" class="l"><a href="#46"> 46 </a>  <span class="php-comment">/**
-</span></span><span id="47" class="l"><a href="#47"> 47 </a><span class="php-comment">   * Return all tokens' offsets
-</span></span><span id="48" class="l"><a href="#48"> 48 </a><span class="php-comment">   *
-</span></span><span id="49" class="l"><a href="#49"> 49 </a><span class="php-comment">   * @return in[] offsets
-</span></span><span id="50" class="l"><a href="#50"> 50 </a><span class="php-comment">   */</span>
-</span><span id="51" class="l"><a href="#51"> 51 </a>  <span class="php-keyword1">public</span> <span class="php-keyword1">function</span> getTokenOffsets() {
-</span><span id="52" class="l"><a href="#52"> 52 </a>    <span class="php-keyword1">return</span> <span class="php-var">$this</span>-&gt;tokenOffsets;
-</span><span id="53" class="l"><a href="#53"> 53 </a>  }
-</span><span id="54" class="l"><a href="#54"> 54 </a>
-</span><span id="55" class="l"><a href="#55"> 55 </a>  <span class="php-comment">/**
-</span></span><span id="56" class="l"><a href="#56"> 56 </a><span class="php-comment">   * Get indicated token
-</span></span><span id="57" class="l"><a href="#57"> 57 </a><span class="php-comment">   *
-</span></span><span id="58" class="l"><a href="#58"> 58 </a><span class="php-comment">   * @param int $i token index
-</span></span><span id="59" class="l"><a href="#59"> 59 </a><span class="php-comment">   * @return string token
-</span></span><span id="60" class="l"><a href="#60"> 60 </a><span class="php-comment">   */</span>
-</span><span id="61" class="l"><a href="#61"> 61 </a>  <span class="php-keyword1">public</span> <span class="php-keyword1">function</span> getToken(<span class="php-var">$i</span>) {
-</span><span id="62" class="l"><a href="#62"> 62 </a>    <span class="php-keyword1">return</span> <span class="php-var">$this</span>-&gt;tokens[<span class="php-var">$i</span>];
-</span><span id="63" class="l"><a href="#63"> 63 </a>  }
-</span><span id="64" class="l"><a href="#64"> 64 </a>  
-</span><span id="65" class="l"><a href="#65"> 65 </a>  <span class="php-comment">/**
-</span></span><span id="66" class="l"><a href="#66"> 66 </a><span class="php-comment">   * Get indicated token offset
-</span></span><span id="67" class="l"><a href="#67"> 67 </a><span class="php-comment">   *
-</span></span><span id="68" class="l"><a href="#68"> 68 </a><span class="php-comment">   * @param int $i token index
-</span></span><span id="69" class="l"><a href="#69"> 69 </a><span class="php-comment">   * @return int offset
-</span></span><span id="70" class="l"><a href="#70"> 70 </a><span class="php-comment">   */</span>
-</span><span id="71" class="l"><a href="#71"> 71 </a>  <span class="php-keyword1">public</span> <span class="php-keyword1">function</span> getTokenOffset(<span class="php-var">$i</span>) {
-</span><span id="72" class="l"><a href="#72"> 72 </a>    <span class="php-keyword1">return</span> <span class="php-var">$this</span>-&gt;tokenOffsets[<span class="php-var">$i</span>];
-</span><span id="73" class="l"><a href="#73"> 73 </a>  }
-</span><span id="74" class="l"><a href="#74"> 74 </a>  
-</span><span id="75" class="l"><a href="#75"> 75 </a>  <span class="php-comment">/**
-</span></span><span id="76" class="l"><a href="#76"> 76 </a><span class="php-comment">   * Set a token to a given attribute and state
-</span></span><span id="77" class="l"><a href="#77"> 77 </a><span class="php-comment">   *
-</span></span><span id="78" class="l"><a href="#78"> 78 </a><span class="php-comment">   * @param int $i token index
-</span></span><span id="79" class="l"><a href="#79"> 79 </a><span class="php-comment">   * @param string $attribute attribute name
-</span></span><span id="80" class="l"><a href="#80"> 80 </a><span class="php-comment">   * @param bool $state attribute state
-</span></span><span id="81" class="l"><a href="#81"> 81 </a><span class="php-comment">   */</span>
-</span><span id="82" class="l"><a href="#82"> 82 </a>  <span class="php-keyword1">public</span> <span class="php-keyword1">function</span> setTokenAttribute(<span class="php-var">$i</span>, <span class="php-var">$attribute</span>, <span class="php-var">$state</span> = <span class="php-keyword1">true</span>) {
-</span><span id="83" class="l"><a href="#83"> 83 </a>    <span class="php-var">$token</span> = <span class="php-var">$this</span>-&gt;tokens[<span class="php-var">$i</span>];
-</span><span id="84" class="l"><a href="#84"> 84 </a>    <span class="php-var">$offset</span> = <span class="php-var">$this</span>-&gt;tokenOffsets[<span class="php-var">$i</span>];
-</span><span id="85" class="l"><a href="#85"> 85 </a>    <span class="php-var">$length</span> = <span class="php-keyword2">strlen</span>(<span class="php-var">$token</span>);
-</span><span id="86" class="l"><a href="#86"> 86 </a>    
-</span><span id="87" class="l"><a href="#87"> 87 </a>    <span class="php-keyword1">return</span> <span class="php-var">$this</span>-&gt;setLength(<span class="php-var">$offset</span>, <span class="php-var">$length</span>, <span class="php-var">$attribute</span>, <span class="php-var">$state</span>);
-</span><span id="88" class="l"><a href="#88"> 88 </a>  }
-</span><span id="89" class="l"><a href="#89"> 89 </a>  
-</span><span id="90" class="l"><a href="#90"> 90 </a>  <span class="php-comment">/**
-</span></span><span id="91" class="l"><a href="#91"> 91 </a><span class="php-comment">   * Set a range of tokens to a given attribute and state
-</span></span><span id="92" class="l"><a href="#92"> 92 </a><span class="php-comment">   *
-</span></span><span id="93" class="l"><a href="#93"> 93 </a><span class="php-comment">   * @param int $from token start index
-</span></span><span id="94" class="l"><a href="#94"> 94 </a><span class="php-comment">   * @param int $to token end index
-</span></span><span id="95" class="l"><a href="#95"> 95 </a><span class="php-comment">   * @param string $attribute attribute name
-</span></span><span id="96" class="l"><a href="#96"> 96 </a><span class="php-comment">   * @param bool $state attribute state
-</span></span><span id="97" class="l"><a href="#97"> 97 </a><span class="php-comment">   */</span>
-</span><span id="98" class="l"><a href="#98"> 98 </a>  <span class="php-keyword1">public</span> <span class="php-keyword1">function</span> setTokenRangeAttribute(<span class="php-var">$from</span>, <span class="php-var">$to</span>, <span class="php-var">$attribute</span>, <span class="php-var">$state</span> = <span class="php-keyword1">true</span>) {
-</span><span id="99" class="l"><a href="#99"> 99 </a>    <span class="php-var">$fromOffset</span> = <span class="php-var">$this</span>-&gt;tokenOffsets[<span class="php-var">$from</span>];
-</span><span id="100" class="l"><a href="#100">100 </a>    <span class="php-var">$toOffset</span> = <span class="php-var">$this</span>-&gt;tokenOffsets[<span class="php-var">$to</span>] + <span class="php-keyword2">strlen</span>(<span class="php-var">$this</span>-&gt;tokens[<span class="php-var">$to</span>]);
-</span><span id="101" class="l"><a href="#101">101 </a>    
-</span><span id="102" class="l"><a href="#102">102 </a>    <span class="php-keyword1">return</span> <span class="php-var">$this</span>-&gt;setRange(<span class="php-var">$fromOffset</span>, <span class="php-var">$toOffset</span>, <span class="php-var">$attribute</span>, <span class="php-var">$state</span>);
-</span><span id="103" class="l"><a href="#103">103 </a>  }
-</span><span id="104" class="l"><a href="#104">104 </a>  
-</span><span id="105" class="l"><a href="#105">105 </a>  <span class="php-comment">/**
-</span></span><span id="106" class="l"><a href="#106">106 </a><span class="php-comment">   * Set all tokens matching given dictionary to attribute and state
-</span></span><span id="107" class="l"><a href="#107">107 </a><span class="php-comment">   *
-</span></span><span id="108" class="l"><a href="#108">108 </a><span class="php-comment">   * @param string[] $dictionary dictionary
-</span></span><span id="109" class="l"><a href="#109">109 </a><span class="php-comment">   * @param string $attribute attribute name
-</span></span><span id="110" class="l"><a href="#110">110 </a><span class="php-comment">   * @param bool $state attribute state
-</span></span><span id="111" class="l"><a href="#111">111 </a><span class="php-comment">   */</span>
-</span><span id="112" class="l"><a href="#112">112 </a>  <span class="php-keyword1">public</span> <span class="php-keyword1">function</span> setTokenDictionaryAttribute(<span class="php-var">$dictionary</span>, <span class="php-var">$attribute</span>, <span class="php-var">$state</span> = <span class="php-keyword1">true</span>) {
-</span><span id="113" class="l"><a href="#113">113 </a>    <span class="php-keyword1">foreach</span>(<span class="php-var">$this</span>-&gt;tokens <span class="php-keyword1">as</span> <span class="php-var">$i</span> =&gt; <span class="php-var">$token</span>) {
-</span><span id="114" class="l"><a href="#114">114 </a>      <span class="php-keyword1">if</span> (<span class="php-keyword2">in_array</span>(<span class="php-var">$token</span>, <span class="php-var">$dictionary</span>)) {
-</span><span id="115" class="l"><a href="#115">115 </a>        <span class="php-var">$this</span>-&gt;setTokenAttribute(<span class="php-var">$i</span>, <span class="php-var">$attribute</span>, <span class="php-var">$state</span>);
-</span><span id="116" class="l"><a href="#116">116 </a>      }
-</span><span id="117" class="l"><a href="#117">117 </a>    }
-</span><span id="118" class="l"><a href="#118">118 </a>  }
-</span><span id="119" class="l"><a href="#119">119 </a>  
-</span><span id="120" class="l"><a href="#120">120 </a>  <span class="php-comment">/**
-</span></span><span id="121" class="l"><a href="#121">121 </a><span class="php-comment">   * Get all attribute of token at given index
-</span></span><span id="122" class="l"><a href="#122">122 </a><span class="php-comment">   *
-</span></span><span id="123" class="l"><a href="#123">123 </a><span class="php-comment">   * @param int token index
-</span></span><span id="124" class="l"><a href="#124">124 </a><span class="php-comment">   * @return string[] attributes
-</span></span><span id="125" class="l"><a href="#125">125 </a><span class="php-comment">   */</span>
-</span><span id="126" class="l"><a href="#126">126 </a>  <span class="php-keyword1">public</span> <span class="php-keyword1">function</span> attributesAtToken(<span class="php-var">$i</span>) {
-</span><span id="127" class="l"><a href="#127">127 </a>    <span class="php-keyword1">return</span> <span class="php-var">$this</span>-&gt;attributesAt(<span class="php-var">$this</span>-&gt;tokenOffsets[<span class="php-var">$i</span>]);
-</span><span id="128" class="l"><a href="#128">128 </a>  }
-</span><span id="129" class="l"><a href="#129">129 </a>  
-</span><span id="130" class="l"><a href="#130">130 </a>  <span class="php-comment">/*
-</span></span><span id="131" class="l"><a href="#131">131 </a><span class="php-comment">   * Convert all tokens to lower case
-</span></span><span id="132" class="l"><a href="#132">132 </a><span class="php-comment">   */</span>
-</span><span id="133" class="l"><a href="#133">133 </a>  <span class="php-keyword1">public</span> <span class="php-keyword1">function</span> lowercaseTokens() {
-</span><span id="134" class="l"><a href="#134">134 </a>    <span class="php-var">$this</span>-&gt;tokens = <span class="php-keyword2">array_map</span>(<span class="php-keyword1">function</span>(<span class="php-var">$token</span>) {
-</span><span id="135" class="l"><a href="#135">135 </a>      <span class="php-keyword1">return</span> <span class="php-keyword2">mb_strtolower</span>(<span class="php-var">$token</span>, <span class="php-quote">&quot;utf-8&quot;</span>);
-</span><span id="136" class="l"><a href="#136">136 </a>    }, <span class="php-var">$this</span>-&gt;tokens);
-</span><span id="137" class="l"><a href="#137">137 </a>  }
-</span><span id="138" class="l"><a href="#138">138 </a>  
-</span><span id="139" class="l"><a href="#139">139 </a>  <span class="php-keyword1">protected</span> <span class="php-keyword1">static</span> <span class="php-keyword1">function</span> tokenizeOnWhitespace(<span class="php-var">$string</span>) {
-</span><span id="140" class="l"><a href="#140">140 </a>    <span class="php-comment">// Matches pontential whitespace in front of the token and the token itself.</span>
-</span><span id="141" class="l"><a href="#141">141 </a>    <span class="php-comment">// Matching the whitespace could be omitted, but that results in slower execution ;-)</span>
-</span><span id="142" class="l"><a href="#142">142 </a>    <span class="php-keyword1">return</span> self::tokenizeOnRegex(<span class="php-var">$string</span>, <span class="php-quote">'/[\s\n\r]*([^\s\n\r]+)/u'</span>);
-</span><span id="143" class="l"><a href="#143">143 </a>  }
-</span><span id="144" class="l"><a href="#144">144 </a>  
-</span><span id="145" class="l"><a href="#145">145 </a>  <span class="php-keyword1">protected</span> <span class="php-keyword1">static</span> <span class="php-keyword1">function</span> tokenizeOnWords(<span class="php-var">$string</span>) {
-</span><span id="146" class="l"><a href="#146">146 </a>    <span class="php-keyword1">return</span> self::tokenizeOnRegex(<span class="php-var">$string</span>, <span class="php-quote">'/([\w]+)/u'</span>);
-</span><span id="147" class="l"><a href="#147">147 </a>  }
-</span><span id="148" class="l"><a href="#148">148 </a>  
-</span><span id="149" class="l"><a href="#149">149 </a>  <span class="php-keyword1">protected</span> <span class="php-keyword1">static</span> <span class="php-keyword1">function</span> tokenizeOnRegex(<span class="php-var">$string</span>, <span class="php-var">$pattern</span>)
-</span><span id="150" class="l"><a href="#150">150 </a>  {
-</span><span id="151" class="l"><a href="#151">151 </a>    <span class="php-comment">// Fastest way to get both tokens and their offsets, but not easy to understand.</span>
-</span><span id="152" class="l"><a href="#152">152 </a>    <span class="php-keyword2">preg_match_all</span>(<span class="php-var">$pattern</span>, <span class="php-var">$string</span>, <span class="php-var">$matches</span>, PREG_OFFSET_CAPTURE);
-</span><span id="153" class="l"><a href="#153">153 </a>
-</span><span id="154" class="l"><a href="#154">154 </a>    <span class="php-comment">// $matches[1] contains an array of all matched subexpressions (= tokens)</span>
-</span><span id="155" class="l"><a href="#155">155 </a>    <span class="php-comment">// with their offset in column 1 and the matched token in column 0</span>
-</span><span id="156" class="l"><a href="#156">156 </a>    <span class="php-var">$tokens</span> = array_column(<span class="php-var">$matches</span>[<span class="php-num">1</span>], <span class="php-num">0</span>);
-</span><span id="157" class="l"><a href="#157">157 </a>    <span class="php-var">$tokenOffsets</span> = array_column(<span class="php-var">$matches</span>[<span class="php-num">1</span>], <span class="php-num">1</span>);
-</span><span id="158" class="l"><a href="#158">158 </a>    
-</span><span id="159" class="l"><a href="#159">159 </a>    <span class="php-keyword1">return</span> [<span class="php-var">$tokens</span>, <span class="php-var">$tokenOffsets</span>];
-</span><span id="160" class="l"><a href="#160">160 </a>  }
-</span><span id="161" class="l"><a href="#161">161 </a>}
-</span><span id="162" class="l"><a href="#162">162 </a></span></code></pre>
+</span><span id="2" class="l"><a href="#2">  2 </a><span class="php-keyword1">namespace</span> apemsel\AttributedString;
+</span><span id="3" class="l"><a href="#3">  3 </a>
+</span><span id="4" class="l"><a href="#4">  4 </a><span class="php-comment">/**
+</span></span><span id="5" class="l"><a href="#5">  5 </a><span class="php-comment"> * Extends AttributedString to support a tokenized string.
+</span></span><span id="6" class="l"><a href="#6">  6 </a><span class="php-comment"> *
+</span></span><span id="7" class="l"><a href="#7">  7 </a><span class="php-comment"> * You can mix working with tokens with working on string ranges using the AttributedString methods.
+</span></span><span id="8" class="l"><a href="#8">  8 </a><span class="php-comment"> * The original string is preserved.
+</span></span><span id="9" class="l"><a href="#9">  9 </a><span class="php-comment"> *
+</span></span><span id="10" class="l"><a href="#10"> 10 </a><span class="php-comment"> * @author Adrian Pemsel &lt;apemsel@gmail.com&gt;
+</span></span><span id="11" class="l"><a href="#11"> 11 </a><span class="php-comment"> */</span>
+</span><span id="12" class="l"><a href="#12"> 12 </a><span class="php-keyword1">class</span> TokenizedAttributedString <span class="php-keyword1">extends</span> AttributedString
+</span><span id="13" class="l"><a href="#13"> 13 </a>{
+</span><span id="14" class="l"><a href="#14"> 14 </a>  <span class="php-keyword1">protected</span> <span class="php-var">$tokens</span>;
+</span><span id="15" class="l"><a href="#15"> 15 </a>  <span class="php-keyword1">protected</span> <span class="php-var">$tokenOffsets</span>;
+</span><span id="16" class="l"><a href="#16"> 16 </a>  
+</span><span id="17" class="l"><a href="#17"> 17 </a>  <span class="php-comment">/**
+</span></span><span id="18" class="l"><a href="#18"> 18 </a><span class="php-comment">   * @param string|AttributedString $string String to work on
+</span></span><span id="19" class="l"><a href="#19"> 19 </a><span class="php-comment">   * @param string $tokenizer Tokenizer to use, either &quot;whitespace&quot;, &quot;word&quot; or a custom regex
+</span></span><span id="20" class="l"><a href="#20"> 20 </a><span class="php-comment">   */</span>
+</span><span id="21" class="l"><a href="#21"> 21 </a>  <span class="php-keyword1">public</span> <span class="php-keyword1">function</span> __construct(<span class="php-var">$string</span>, <span class="php-var">$tokenizer</span> = <span class="php-quote">&quot;whitespace&quot;</span>) {
+</span><span id="22" class="l"><a href="#22"> 22 </a>    <span class="php-var">$tokenizerFunction</span> = <span class="php-quote">&quot;tokenizeOn&quot;</span>.<span class="php-keyword2">ucfirst</span>(<span class="php-var">$tokenizer</span>);
+</span><span id="23" class="l"><a href="#23"> 23 </a>
+</span><span id="24" class="l"><a href="#24"> 24 </a>    <span class="php-keyword1">if</span> (<span class="php-var">$tokenizer</span>[<span class="php-num">0</span>] == <span class="php-quote">&quot;/&quot;</span>) {
+</span><span id="25" class="l"><a href="#25"> 25 </a>      <span class="php-keyword1">list</span>(<span class="php-var">$this</span>-&gt;tokens, <span class="php-var">$this</span>-&gt;tokenOffsets) = self::tokenizeOnRegex(<span class="php-var">$string</span>, <span class="php-var">$tokenizer</span>);
+</span><span id="26" class="l"><a href="#26"> 26 </a>    } <span class="php-keyword1">else</span> {
+</span><span id="27" class="l"><a href="#27"> 27 </a>      <span class="php-keyword1">if</span> (!<span class="php-keyword2">method_exists</span>(<span class="php-quote">&quot;apemsel\AttributedString\TokenizedAttributedString&quot;</span>, <span class="php-var">$tokenizerFunction</span>)) {
+</span><span id="28" class="l"><a href="#28"> 28 </a>        <span class="php-keyword1">throw</span> <span class="php-keyword1">new</span> \InvalidArgumentException(<span class="php-quote">&quot;Unknown tokenizer </span><span class="php-var">$tokenizer</span><span class="php-quote">&quot;</span>);
+</span><span id="29" class="l"><a href="#29"> 29 </a>      }
+</span><span id="30" class="l"><a href="#30"> 30 </a>      <span class="php-keyword1">list</span>(<span class="php-var">$this</span>-&gt;tokens, <span class="php-var">$this</span>-&gt;tokenOffsets) = self::<span class="php-var">$tokenizerFunction</span>(<span class="php-var">$string</span>);
+</span><span id="31" class="l"><a href="#31"> 31 </a>    }
+</span><span id="32" class="l"><a href="#32"> 32 </a>    
+</span><span id="33" class="l"><a href="#33"> 33 </a>    parent::__construct(<span class="php-var">$string</span>);
+</span><span id="34" class="l"><a href="#34"> 34 </a>  }
+</span><span id="35" class="l"><a href="#35"> 35 </a>  
+</span><span id="36" class="l"><a href="#36"> 36 </a>  <span class="php-comment">/**
+</span></span><span id="37" class="l"><a href="#37"> 37 </a><span class="php-comment">   * Return all tokens
+</span></span><span id="38" class="l"><a href="#38"> 38 </a><span class="php-comment">   *
+</span></span><span id="39" class="l"><a href="#39"> 39 </a><span class="php-comment">   * @return string[] tokens
+</span></span><span id="40" class="l"><a href="#40"> 40 </a><span class="php-comment">   */</span>
+</span><span id="41" class="l"><a href="#41"> 41 </a>  <span class="php-keyword1">public</span> <span class="php-keyword1">function</span> getTokens() {
+</span><span id="42" class="l"><a href="#42"> 42 </a>    <span class="php-keyword1">return</span> <span class="php-var">$this</span>-&gt;tokens;
+</span><span id="43" class="l"><a href="#43"> 43 </a>  }
+</span><span id="44" class="l"><a href="#44"> 44 </a>  
+</span><span id="45" class="l"><a href="#45"> 45 </a>  <span class="php-comment">/**
+</span></span><span id="46" class="l"><a href="#46"> 46 </a><span class="php-comment">   * Return all tokens' offsets
+</span></span><span id="47" class="l"><a href="#47"> 47 </a><span class="php-comment">   *
+</span></span><span id="48" class="l"><a href="#48"> 48 </a><span class="php-comment">   * @return int[] offsets
+</span></span><span id="49" class="l"><a href="#49"> 49 </a><span class="php-comment">   */</span>
+</span><span id="50" class="l"><a href="#50"> 50 </a>  <span class="php-keyword1">public</span> <span class="php-keyword1">function</span> getTokenOffsets() {
+</span><span id="51" class="l"><a href="#51"> 51 </a>    <span class="php-keyword1">return</span> <span class="php-var">$this</span>-&gt;tokenOffsets;
+</span><span id="52" class="l"><a href="#52"> 52 </a>  }
+</span><span id="53" class="l"><a href="#53"> 53 </a>  
+</span><span id="54" class="l"><a href="#54"> 54 </a>  <span class="php-comment">/**
+</span></span><span id="55" class="l"><a href="#55"> 55 </a><span class="php-comment">   * Return the number of tokens
+</span></span><span id="56" class="l"><a href="#56"> 56 </a><span class="php-comment">   *
+</span></span><span id="57" class="l"><a href="#57"> 57 </a><span class="php-comment">   * @return int count
+</span></span><span id="58" class="l"><a href="#58"> 58 </a><span class="php-comment">   */</span>
+</span><span id="59" class="l"><a href="#59"> 59 </a>  <span class="php-keyword1">public</span> <span class="php-keyword1">function</span> getTokenCount() {
+</span><span id="60" class="l"><a href="#60"> 60 </a>    <span class="php-keyword1">return</span> <span class="php-keyword2">count</span>(<span class="php-var">$this</span>-&gt;tokens);
+</span><span id="61" class="l"><a href="#61"> 61 </a>  }
+</span><span id="62" class="l"><a href="#62"> 62 </a>
+</span><span id="63" class="l"><a href="#63"> 63 </a>  <span class="php-comment">/**
+</span></span><span id="64" class="l"><a href="#64"> 64 </a><span class="php-comment">   * Get indicated token
+</span></span><span id="65" class="l"><a href="#65"> 65 </a><span class="php-comment">   *
+</span></span><span id="66" class="l"><a href="#66"> 66 </a><span class="php-comment">   * @param int $i token index
+</span></span><span id="67" class="l"><a href="#67"> 67 </a><span class="php-comment">   * @return string token
+</span></span><span id="68" class="l"><a href="#68"> 68 </a><span class="php-comment">   */</span>
+</span><span id="69" class="l"><a href="#69"> 69 </a>  <span class="php-keyword1">public</span> <span class="php-keyword1">function</span> getToken(<span class="php-var">$i</span>) {
+</span><span id="70" class="l"><a href="#70"> 70 </a>    <span class="php-keyword1">return</span> <span class="php-var">$this</span>-&gt;tokens[<span class="php-var">$i</span>];
+</span><span id="71" class="l"><a href="#71"> 71 </a>  }
+</span><span id="72" class="l"><a href="#72"> 72 </a>  
+</span><span id="73" class="l"><a href="#73"> 73 </a>  <span class="php-comment">/**
+</span></span><span id="74" class="l"><a href="#74"> 74 </a><span class="php-comment">   * Get indicated token offset
+</span></span><span id="75" class="l"><a href="#75"> 75 </a><span class="php-comment">   *
+</span></span><span id="76" class="l"><a href="#76"> 76 </a><span class="php-comment">   * @param int $i token index
+</span></span><span id="77" class="l"><a href="#77"> 77 </a><span class="php-comment">   * @return int offset
+</span></span><span id="78" class="l"><a href="#78"> 78 </a><span class="php-comment">   */</span>
+</span><span id="79" class="l"><a href="#79"> 79 </a>  <span class="php-keyword1">public</span> <span class="php-keyword1">function</span> getTokenOffset(<span class="php-var">$i</span>) {
+</span><span id="80" class="l"><a href="#80"> 80 </a>    <span class="php-keyword1">return</span> <span class="php-var">$this</span>-&gt;tokenOffsets[<span class="php-var">$i</span>];
+</span><span id="81" class="l"><a href="#81"> 81 </a>  }
+</span><span id="82" class="l"><a href="#82"> 82 </a>  
+</span><span id="83" class="l"><a href="#83"> 83 </a>  <span class="php-comment">/**
+</span></span><span id="84" class="l"><a href="#84"> 84 </a><span class="php-comment">   * Set a token to a given attribute and state
+</span></span><span id="85" class="l"><a href="#85"> 85 </a><span class="php-comment">   *
+</span></span><span id="86" class="l"><a href="#86"> 86 </a><span class="php-comment">   * @param int $i token index
+</span></span><span id="87" class="l"><a href="#87"> 87 </a><span class="php-comment">   * @param string $attribute attribute name
+</span></span><span id="88" class="l"><a href="#88"> 88 </a><span class="php-comment">   * @param bool $state attribute state
+</span></span><span id="89" class="l"><a href="#89"> 89 </a><span class="php-comment">   */</span>
+</span><span id="90" class="l"><a href="#90"> 90 </a>  <span class="php-keyword1">public</span> <span class="php-keyword1">function</span> setTokenAttribute(<span class="php-var">$i</span>, <span class="php-var">$attribute</span>, <span class="php-var">$state</span> = <span class="php-keyword1">true</span>) {
+</span><span id="91" class="l"><a href="#91"> 91 </a>    <span class="php-var">$token</span> = <span class="php-var">$this</span>-&gt;tokens[<span class="php-var">$i</span>];
+</span><span id="92" class="l"><a href="#92"> 92 </a>    <span class="php-var">$offset</span> = <span class="php-var">$this</span>-&gt;tokenOffsets[<span class="php-var">$i</span>];
+</span><span id="93" class="l"><a href="#93"> 93 </a>    <span class="php-var">$length</span> = <span class="php-keyword2">strlen</span>(<span class="php-var">$token</span>);
+</span><span id="94" class="l"><a href="#94"> 94 </a>    
+</span><span id="95" class="l"><a href="#95"> 95 </a>    <span class="php-keyword1">return</span> <span class="php-var">$this</span>-&gt;setLength(<span class="php-var">$offset</span>, <span class="php-var">$length</span>, <span class="php-var">$attribute</span>, <span class="php-var">$state</span>);
+</span><span id="96" class="l"><a href="#96"> 96 </a>  }
+</span><span id="97" class="l"><a href="#97"> 97 </a>  
+</span><span id="98" class="l"><a href="#98"> 98 </a>  <span class="php-comment">/**
+</span></span><span id="99" class="l"><a href="#99"> 99 </a><span class="php-comment">   * Set a range of tokens to a given attribute and state
+</span></span><span id="100" class="l"><a href="#100">100 </a><span class="php-comment">   *
+</span></span><span id="101" class="l"><a href="#101">101 </a><span class="php-comment">   * @param int $from token start index
+</span></span><span id="102" class="l"><a href="#102">102 </a><span class="php-comment">   * @param int $to token end index
+</span></span><span id="103" class="l"><a href="#103">103 </a><span class="php-comment">   * @param string $attribute attribute name
+</span></span><span id="104" class="l"><a href="#104">104 </a><span class="php-comment">   * @param bool $state attribute state
+</span></span><span id="105" class="l"><a href="#105">105 </a><span class="php-comment">   */</span>
+</span><span id="106" class="l"><a href="#106">106 </a>  <span class="php-keyword1">public</span> <span class="php-keyword1">function</span> setTokenRangeAttribute(<span class="php-var">$from</span>, <span class="php-var">$to</span>, <span class="php-var">$attribute</span>, <span class="php-var">$state</span> = <span class="php-keyword1">true</span>) {
+</span><span id="107" class="l"><a href="#107">107 </a>    <span class="php-var">$fromOffset</span> = <span class="php-var">$this</span>-&gt;tokenOffsets[<span class="php-var">$from</span>];
+</span><span id="108" class="l"><a href="#108">108 </a>    <span class="php-var">$toOffset</span> = <span class="php-var">$this</span>-&gt;tokenOffsets[<span class="php-var">$to</span>] + <span class="php-keyword2">strlen</span>(<span class="php-var">$this</span>-&gt;tokens[<span class="php-var">$to</span>]);
+</span><span id="109" class="l"><a href="#109">109 </a>    
+</span><span id="110" class="l"><a href="#110">110 </a>    <span class="php-keyword1">return</span> <span class="php-var">$this</span>-&gt;setRange(<span class="php-var">$fromOffset</span>, <span class="php-var">$toOffset</span>, <span class="php-var">$attribute</span>, <span class="php-var">$state</span>);
+</span><span id="111" class="l"><a href="#111">111 </a>  }
+</span><span id="112" class="l"><a href="#112">112 </a>  
+</span><span id="113" class="l"><a href="#113">113 </a>  <span class="php-comment">/**
+</span></span><span id="114" class="l"><a href="#114">114 </a><span class="php-comment">   * Set all tokens matching given dictionary to attribute and state
+</span></span><span id="115" class="l"><a href="#115">115 </a><span class="php-comment">   *
+</span></span><span id="116" class="l"><a href="#116">116 </a><span class="php-comment">   * @param string[] $dictionary dictionary
+</span></span><span id="117" class="l"><a href="#117">117 </a><span class="php-comment">   * @param string $attribute attribute name
+</span></span><span id="118" class="l"><a href="#118">118 </a><span class="php-comment">   * @param bool $state attribute state
+</span></span><span id="119" class="l"><a href="#119">119 </a><span class="php-comment">   */</span>
+</span><span id="120" class="l"><a href="#120">120 </a>  <span class="php-keyword1">public</span> <span class="php-keyword1">function</span> setTokenDictionaryAttribute(<span class="php-var">$dictionary</span>, <span class="php-var">$attribute</span>, <span class="php-var">$state</span> = <span class="php-keyword1">true</span>) {
+</span><span id="121" class="l"><a href="#121">121 </a>    <span class="php-keyword1">foreach</span>(<span class="php-var">$this</span>-&gt;tokens <span class="php-keyword1">as</span> <span class="php-var">$i</span> =&gt; <span class="php-var">$token</span>) {
+</span><span id="122" class="l"><a href="#122">122 </a>      <span class="php-keyword1">if</span> (<span class="php-keyword2">in_array</span>(<span class="php-var">$token</span>, <span class="php-var">$dictionary</span>)) {
+</span><span id="123" class="l"><a href="#123">123 </a>        <span class="php-var">$this</span>-&gt;setTokenAttribute(<span class="php-var">$i</span>, <span class="php-var">$attribute</span>, <span class="php-var">$state</span>);
+</span><span id="124" class="l"><a href="#124">124 </a>      }
+</span><span id="125" class="l"><a href="#125">125 </a>    }
+</span><span id="126" class="l"><a href="#126">126 </a>  }
+</span><span id="127" class="l"><a href="#127">127 </a>  
+</span><span id="128" class="l"><a href="#128">128 </a>  <span class="php-comment">/**
+</span></span><span id="129" class="l"><a href="#129">129 </a><span class="php-comment">   * Get all attribute of token at given index
+</span></span><span id="130" class="l"><a href="#130">130 </a><span class="php-comment">   *
+</span></span><span id="131" class="l"><a href="#131">131 </a><span class="php-comment">   * @param int token index
+</span></span><span id="132" class="l"><a href="#132">132 </a><span class="php-comment">   * @return string[] attributes
+</span></span><span id="133" class="l"><a href="#133">133 </a><span class="php-comment">   */</span>
+</span><span id="134" class="l"><a href="#134">134 </a>  <span class="php-keyword1">public</span> <span class="php-keyword1">function</span> attributesAtToken(<span class="php-var">$i</span>) {
+</span><span id="135" class="l"><a href="#135">135 </a>    <span class="php-keyword1">return</span> <span class="php-var">$this</span>-&gt;attributesAt(<span class="php-var">$this</span>-&gt;tokenOffsets[<span class="php-var">$i</span>]);
+</span><span id="136" class="l"><a href="#136">136 </a>  }
+</span><span id="137" class="l"><a href="#137">137 </a>  
+</span><span id="138" class="l"><a href="#138">138 </a>  <span class="php-comment">/**
+</span></span><span id="139" class="l"><a href="#139">139 </a><span class="php-comment">   * Convert all tokens to lower case
+</span></span><span id="140" class="l"><a href="#140">140 </a><span class="php-comment">   */</span>
+</span><span id="141" class="l"><a href="#141">141 </a>  <span class="php-keyword1">public</span> <span class="php-keyword1">function</span> lowercaseTokens() {
+</span><span id="142" class="l"><a href="#142">142 </a>    <span class="php-var">$this</span>-&gt;tokens = <span class="php-keyword2">array_map</span>(<span class="php-keyword1">function</span>(<span class="php-var">$token</span>) {
+</span><span id="143" class="l"><a href="#143">143 </a>      <span class="php-keyword1">return</span> <span class="php-keyword2">mb_strtolower</span>(<span class="php-var">$token</span>, <span class="php-quote">&quot;utf-8&quot;</span>);
+</span><span id="144" class="l"><a href="#144">144 </a>    }, <span class="php-var">$this</span>-&gt;tokens);
+</span><span id="145" class="l"><a href="#145">145 </a>  }
+</span><span id="146" class="l"><a href="#146">146 </a>  
+</span><span id="147" class="l"><a href="#147">147 </a>  <span class="php-comment">/**
+</span></span><span id="148" class="l"><a href="#148">148 </a><span class="php-comment">   * Tokenize a string on whitespace
+</span></span><span id="149" class="l"><a href="#149">149 </a><span class="php-comment">   *
+</span></span><span id="150" class="l"><a href="#150">150 </a><span class="php-comment">   * @param string $string string to be tokenized
+</span></span><span id="151" class="l"><a href="#151">151 </a><span class="php-comment">   * @return string[] tokens
+</span></span><span id="152" class="l"><a href="#152">152 </a><span class="php-comment">   */</span>
+</span><span id="153" class="l"><a href="#153">153 </a>  <span class="php-keyword1">public</span> <span class="php-keyword1">static</span> <span class="php-keyword1">function</span> tokenizeOnWhitespace(<span class="php-var">$string</span>) {
+</span><span id="154" class="l"><a href="#154">154 </a>    <span class="php-comment">// Matches pontential whitespace in front of the token and the token itself.</span>
+</span><span id="155" class="l"><a href="#155">155 </a>    <span class="php-comment">// Matching the whitespace could be omitted, but that results in slower execution ;-)</span>
+</span><span id="156" class="l"><a href="#156">156 </a>    <span class="php-keyword1">return</span> self::tokenizeOnRegex(<span class="php-var">$string</span>, <span class="php-quote">'/[\s\n\r]*([^\s\n\r]+)/u'</span>);
+</span><span id="157" class="l"><a href="#157">157 </a>  }
+</span><span id="158" class="l"><a href="#158">158 </a>  
+</span><span id="159" class="l"><a href="#159">159 </a>  <span class="php-comment">/**
+</span></span><span id="160" class="l"><a href="#160">160 </a><span class="php-comment">   * Tokenize a string on words
+</span></span><span id="161" class="l"><a href="#161">161 </a><span class="php-comment">   *
+</span></span><span id="162" class="l"><a href="#162">162 </a><span class="php-comment">   * @param string $string string to be tokenized
+</span></span><span id="163" class="l"><a href="#163">163 </a><span class="php-comment">   * @return string[] tokens
+</span></span><span id="164" class="l"><a href="#164">164 </a><span class="php-comment">   */</span>
+</span><span id="165" class="l"><a href="#165">165 </a>  <span class="php-keyword1">public</span> <span class="php-keyword1">static</span> <span class="php-keyword1">function</span> tokenizeOnWords(<span class="php-var">$string</span>) {
+</span><span id="166" class="l"><a href="#166">166 </a>    <span class="php-keyword1">return</span> self::tokenizeOnRegex(<span class="php-var">$string</span>, <span class="php-quote">'/([\w]+)/u'</span>);
+</span><span id="167" class="l"><a href="#167">167 </a>  }
+</span><span id="168" class="l"><a href="#168">168 </a>  
+</span><span id="169" class="l"><a href="#169">169 </a>  <span class="php-comment">/**
+</span></span><span id="170" class="l"><a href="#170">170 </a><span class="php-comment">   * Tokenize a string with a given regex
+</span></span><span id="171" class="l"><a href="#171">171 </a><span class="php-comment">   *
+</span></span><span id="172" class="l"><a href="#172">172 </a><span class="php-comment">   * @param string $string string to be tokenized
+</span></span><span id="173" class="l"><a href="#173">173 </a><span class="php-comment">   * @param string $pattern regex. The token must be captured in the first subgroup.
+</span></span><span id="174" class="l"><a href="#174">174 </a><span class="php-comment">   * @return string[] tokens
+</span></span><span id="175" class="l"><a href="#175">175 </a><span class="php-comment">   */</span>
+</span><span id="176" class="l"><a href="#176">176 </a>  <span class="php-keyword1">public</span> <span class="php-keyword1">static</span> <span class="php-keyword1">function</span> tokenizeOnRegex(<span class="php-var">$string</span>, <span class="php-var">$pattern</span>)
+</span><span id="177" class="l"><a href="#177">177 </a>  {
+</span><span id="178" class="l"><a href="#178">178 </a>    <span class="php-comment">// Fastest way to get both tokens and their offsets, but not easy to understand.</span>
+</span><span id="179" class="l"><a href="#179">179 </a>    <span class="php-keyword2">preg_match_all</span>(<span class="php-var">$pattern</span>, <span class="php-var">$string</span>, <span class="php-var">$matches</span>, PREG_OFFSET_CAPTURE);
+</span><span id="180" class="l"><a href="#180">180 </a>
+</span><span id="181" class="l"><a href="#181">181 </a>    <span class="php-comment">// $matches[1] contains an array of all matched subexpressions (= tokens)</span>
+</span><span id="182" class="l"><a href="#182">182 </a>    <span class="php-comment">// with their offset in column 1 and the matched token in column 0</span>
+</span><span id="183" class="l"><a href="#183">183 </a>    <span class="php-var">$tokens</span> = array_column(<span class="php-var">$matches</span>[<span class="php-num">1</span>], <span class="php-num">0</span>);
+</span><span id="184" class="l"><a href="#184">184 </a>    <span class="php-var">$tokenOffsets</span> = array_column(<span class="php-var">$matches</span>[<span class="php-num">1</span>], <span class="php-num">1</span>);
+</span><span id="185" class="l"><a href="#185">185 </a>    
+</span><span id="186" class="l"><a href="#186">186 </a>    <span class="php-keyword1">return</span> [<span class="php-var">$tokens</span>, <span class="php-var">$tokenOffsets</span>];
+</span><span id="187" class="l"><a href="#187">187 </a>  }
+</span><span id="188" class="l"><a href="#188">188 </a>}
+</span><span id="189" class="l"><a href="#189">189 </a></span></code></pre>
 	</div>
 
 	<div id="footer">