implementation for TokenizedAttributedString->setTokenDictionaryAttri…

…bute() and ->lowercaseTokens()

src/TokenizedAttributedString.php

@@ -58,10 +58,24 @@ public function setTokenRangeAttribute($from, $to, $attribute, $state = true) {
     return $this->setRange($fromOffset, $toOffset, $attribute, $state);
   }
 
+  public function setTokenDictionaryAttribute($dictionary, $attribute) {
+    foreach($this->tokens as $i => $token) {
+      if (in_array($token, $dictionary)) {
+        $this->setTokenAttribute($i, $attribute);
+      }
+    }
+  }
+
   public function attributesAtToken($i) {
     return $this->attributesAt($this->tokenOffsets[$i]);
   }
 
+  public function lowercaseTokens() {
+    $this->tokens = array_map(function($token) {
+      return mb_strtolower($token, "utf-8");
+    }, $this->tokens);
+  }
+
   protected static function tokenizeOnWhitespace($string) {
     // Matches pontential whitespace in front of the token and the token itself.
     // Matching the whitespace could be omitted, but that results in slower execution ;-)