fix issues #8

weixsong · Mar 7, 2016 · 4c286bf · 4c286bf
1 parent e492d9b
commit 4c286bf
Show file tree

Hide file tree

Showing 2 changed files with 33 additions and 1 deletion.
diff --git a/lib/tokenizer.js b/lib/tokenizer.js
@@ -28,9 +28,17 @@ elasticlunr.tokenizer = function (str) {
       return true;
     });
 
-    return arr.map(function (t) {
+    arr = arr.map(function (t) {
       return elasticlunr.utils.toString(t).toLowerCase();
     });
+
+    var out = [];
+    arr.forEach(function(item) {
+      var tokens = item.split(elasticlunr.tokenizer.seperator);
+      out = out.concat(tokens);
+    }, this);
+
+    return out;
   }
 
   return str.toString().trim().toLowerCase().split(elasticlunr.tokenizer.seperator);

diff --git a/test/tokenizer_test.js b/test/tokenizer_test.js
@@ -117,3 +117,27 @@ test("test get seperator function", function () {
   elasticlunr.tokenizer.setSeperator(sep2);
   deepEqual(elasticlunr.tokenizer.getSeperator(), sep2);
 });
+
+test("tokenize array", function () {
+  var str = ['hello world', 'glad to see you'];
+  var tokens = elasticlunr.tokenizer(str);
+  deepEqual(tokens, ['hello', 'world', 'glad', 'to', 'see', 'you']);
+});
+
+test("tokenize array 2", function () {
+  var str = ['helloworld', 'glad to see you'];
+  var tokens = elasticlunr.tokenizer(str);
+  deepEqual(tokens, ['helloworld', 'glad', 'to', 'see', 'you']);
+});
+
+test("tokenize array", function () {
+  var str = ['helloworld', null, undefined, 'glad to see you'];
+  var tokens = elasticlunr.tokenizer(str);
+  deepEqual(tokens, ['helloworld', 'glad', 'to', 'see', 'you']);
+});
+
+test("tokenize array", function () {
+  var str = ['helloworld', 'glad to see you', 'hyper-parameters'];
+  var tokens = elasticlunr.tokenizer(str);
+  deepEqual(tokens, ['helloworld', 'glad', 'to', 'see', 'you', 'hyper', 'parameters']);
+});