Skip to content

Commit

Permalink
Build version 0.8.9
Browse files Browse the repository at this point in the history
  • Loading branch information
@weixsong
weixsong committed Apr 17, 2016
1 parent b20a10a commit b12fe19
Show file tree
Hide file tree
Showing 13 changed files with 387 additions and 174 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.mdown
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@

* fix issue [#8](https://github.com/weixsong/elasticlunr.js/issues/8#issuecomment-193266856), boolean "AND" error.
* need to add test cases for issue fix #8
* Optimized AND boolean search, merged pull request [#14](https://github.com/weixsong/elasticlunr.js/pull/14), thanks for [mhalle](https://github.com/mhalle), this is really brilliant idea.
* Fixed documents update issue, merged pull request [#12](https://github.com/weixsong/elasticlunr.js/pull/12), thanks [deerawan](https://github.com/deerawan).

## 0.8.8

Expand Down
2 changes: 1 addition & 1 deletion VERSION
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0.8.8
0.8.9
2 changes: 1 addition & 1 deletion bower.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "elasticlunr.js",
"version": "0.8.8",
"version": "0.8.9",
"main": "elasticlunr.js",
"ignore": [
"tests/",
Expand Down
2 changes: 1 addition & 1 deletion component.json
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "elasticlunr",
"repo": "weixsong/elasticlunr.js",
"version": "0.8.8",
"version": "0.8.9",
"description": "Lightweight full-text search engine in Javascript for browser search and offline search.",
"license": "MIT",
"main": "elasticlunr.js",
Expand Down
174 changes: 119 additions & 55 deletions docs/index.js.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,9 +115,9 @@ <h1>elasticlunr</h1>
</li>
<li><a href="#fieldSearch"><i class="alert alert-info"></i><span>fieldSearch</span></a>
</li>
<li><a href="#fieldSearchStats"><i class="alert alert-info"></i><span>fieldSearchStats</span></a>
<li><a href="#mergeScores"><i class="alert alert-info"></i><span>mergeScores</span></a>
</li>
<li><a href="#intersect"><i class="alert alert-info"></i><span>intersect</span></a>
<li><a href="#fieldSearchStats"><i class="alert alert-info"></i><span>fieldSearchStats</span></a>
</li>
<li><a href="#coordNorm"><i class="alert alert-info"></i><span>coordNorm</span></a>
</li>
Expand Down Expand Up @@ -455,7 +455,6 @@ <h5 class="subheader"></h5>
<pre><code class="language-javascript">elasticlunr.Index.prototype.removeDocByRef = function (docRef, emitEvent) {
if (!docRef) return;
if (this.documentStore.isDocStored() === false) {
elasticlunr.utils.warn('remove doc by ref is not allowed, because currectly not storing documents in DocumentStore');
return;
}

Expand Down Expand Up @@ -632,6 +631,38 @@ <h5 class="subheader"></h5>
results.sort(function (a, b) { return b.score - a.score; });
return results;
};</code></pre>
<table class="table table-bordered table-striped">
<thead>
<tr>
<th style="width:20%">Option name</th>
<th style="width:20%">Type</th>
<th>Description</th>
</tr>
</thead>
<tbody>
<tr>
<td>queryTokens</td>
<td>Array</td>
<td><p>The query tokens to query in this field.</p></td>
</tr>
<tr>
<td>field</td>
<td>String</td>
<td><p>Field to query in.</p></td>
</tr>
<tr>
<td>config</td>
<td>elasticlunr.Configuration</td>
<td><p>The user query config, JSON format.</p></td>
</tr>
<tr>
<td>return</td>
<td>Object</td>
<td></td>
</tr>
</tbody>
</table>
<div class="description"><p>search queryTokens in specified field.</p> </div>
<section id="fieldSearch">
<h1>fieldSearch</h1>
<h5 class="subheader"></h5>
Expand Down Expand Up @@ -674,19 +705,49 @@ <h5 class="subheader"></h5>
<pre><code class="language-javascript">elasticlunr.Index.prototype.fieldSearch = function (queryTokens, fieldName, config) {
var booleanType = config[fieldName].bool;
var expand = config[fieldName].expand;
var scores = {};
var scores = null;
var docTokens = {};

queryTokens.forEach(function (token) {
var tokens = [token];
if (expand == true) {
tokens = this.index[fieldName].expandToken(token);
}

// Consider every query token in turn. If expanded, each query token
// corresponds to a set of tokens, which is all tokens in the
// index matching the pattern queryToken* .
// For the set of tokens corresponding to a query token, find and score
// all matching documents. Store those scores in queryTokenScores,
// keyed by docRef.
// Then, depending on the value of booleanType, combine the scores
// for this query token with previous scores. If booleanType is OR,
// then merge the scores by summing into the accumulated total, adding
// new document scores are required (effectively a union operator).
// If booleanType is AND, accumulate scores only if the document
// has previously been scored by another query token (an intersection
// operation0.
// Furthermore, since when booleanType is AND, additional
// query tokens can't add new documents to the result set, use the
// current document set to limit the processing of each new query
// token for efficiency (i.e., incremental intersection).

var queryTokenScores = {};
tokens.forEach(function (key) {
var docs = this.index[fieldName].getDocs(key);
var idf = this.idf(key, fieldName);


if (scores &amp;&amp; booleanType == 'AND') {
// special case, we can rule out documents that have been
// already been filtered out because they weren't scored
// by previous query token passes.
var filteredDocs = {};
for (var docRef in scores) {
if (docRef in docs) {
filteredDocs[docRef] = docs[docRef];
}
}
docs = filteredDocs;
}
// only record appeared token for retrieved documents for the
// original token, not for expaned token.
// beause for doing coordNorm for a retrieved document, coordNorm only care how many
Expand Down Expand Up @@ -714,28 +775,25 @@ <h5 class="subheader"></h5>

var score = tf * idf * fieldLengthNorm * penality;

if (docRef in scores) {
scores[docRef] += score;
if (docRef in queryTokenScores) {
queryTokenScores[docRef] += score;
} else {
scores[docRef] = score;
queryTokenScores[docRef] = score;
}
}
}, this);

scores = this.mergeScores(scores, queryTokenScores, booleanType);
}, this);

if (booleanType == 'AND') {
scores = this.intersect(scores, docTokens, queryTokens.length);
}

scores = this.coordNorm(scores, docTokens, queryTokens.length);

return scores;
};</code></pre>
<section id="fieldSearchStats">
<h1>fieldSearchStats</h1>
<section id="mergeScores">
<h1>mergeScores</h1>
<h5 class="subheader"></h5>
<p>
<div class="label label-info radius ctx-type">method</div><span>elasticlunr.Index.prototype.fieldSearchStats()</span>
<div class="label label-info radius ctx-type">method</div><span>elasticlunr.Index.prototype.mergeScores()</span>
</p>
</section>
<table class="table table-bordered table-striped">
Expand All @@ -748,37 +806,51 @@ <h5 class="subheader"></h5>
</thead>
<tbody>
<tr>
<td>docTokens</td>
<td>bool</td>
<td>Object</td>
<td><p>a data structure stores which token appears in the retrieved doc.</p></td>
<td><p>accumulated scores. Should be null on first call.</p></td>
</tr>
<tr>
<td>token</td>
<td>scores</td>
<td>String</td>
<td><p>query token</p></td>
<td><p>new scores to merge into accumScores.</p></td>
</tr>
<tr>
<td>docs</td>
<td>op</td>
<td>Object</td>
<td><p>the retrieved documents of the query token</p></td>
<td><p>merge operation (should be &#39;AND&#39; or &#39;OR&#39;).</p></td>
</tr>
</tbody>
</table>
<div class="description"><p>Record the occuring query token of retrieved doc specified by doc field.<br />Only for inner user.</p> </div>
<pre><code class="language-javascript">elasticlunr.Index.prototype.fieldSearchStats = function (docTokens, token, docs) {
for (var doc in docs) {
if (doc in docTokens) {
docTokens[doc].push(token);
<div class="description"><p>Merge the scores from one set of tokens into an accumulated score table.<br />Exact operation depends on the op parameter. If op is &#39;AND&#39;, then only the<br />intersection of the two score lists is retained. Otherwise, the union of<br />the two score lists is returned. For internal use only.</p> </div>
<pre><code class="language-javascript">elasticlunr.Index.prototype.mergeScores = function (accumScores, scores, op) {
if (!accumScores) {
return scores;
}
if (op == 'AND') {
var intersection = {};
for (var docRef in scores) {
if (docRef in accumScores) {
intersection[docRef] = accumScores[docRef] + scores[docRef];
}
}
return intersection;
} else {
docTokens[doc] = [token];
for (var docRef in scores) {
if (docRef in accumScores) {
accumScores[docRef] += scores[docRef];
} else {
accumScores[docRef] = scores[docRef];
}
}
return accumScores;
}
}
};</code></pre>
<section id="intersect">
<h1>intersect</h1>
<section id="fieldSearchStats">
<h1>fieldSearchStats</h1>
<h5 class="subheader"></h5>
<p>
<div class="label label-info radius ctx-type">method</div><span>elasticlunr.Index.prototype.intersect()</span>
<div class="label label-info radius ctx-type">method</div><span>elasticlunr.Index.prototype.fieldSearchStats()</span>
</p>
</section>
<table class="table table-bordered table-striped">
Expand All @@ -791,39 +863,31 @@ <h5 class="subheader"></h5>
</thead>
<tbody>
<tr>
<td>results</td>
<td>Object</td>
<td><p>first results</p></td>
</tr>
<tr>
<td>docs</td>
<td>docTokens</td>
<td>Object</td>
<td><p>field search results of a token</p></td>
<td><p>a data structure stores which token appears in the retrieved doc.</p></td>
</tr>
<tr>
<td>n</td>
<td>Integer</td>
<td><p>query token number</p></td>
<td>token</td>
<td>String</td>
<td><p>query token</p></td>
</tr>
<tr>
<td>return</td>
<td>docs</td>
<td>Object</td>
<td></td>
<td><p>the retrieved documents of the query token</p></td>
</tr>
</tbody>
</table>
<div class="description"><p>find documents contain all the query tokens.<br />only for inner use.</p> </div>
<pre><code class="language-javascript">elasticlunr.Index.prototype.intersect = function (scores, docTokens, n) {
var res = {};

for (var doc in scores) {
if (!(doc in docTokens)) continue;
if (docTokens[doc].length == n) {
res[doc] = scores[doc];
<div class="description"><p>Record the occuring query token of retrieved doc specified by doc field.<br />Only for inner user.</p> </div>
<pre><code class="language-javascript">elasticlunr.Index.prototype.fieldSearchStats = function (docTokens, token, docs) {
for (var doc in docs) {
if (doc in docTokens) {
docTokens[doc].push(token);
} else {
docTokens[doc] = [token];
}
}

return res;
};</code></pre>
<section id="coordNorm">
<h1>coordNorm</h1>
Expand Down
Loading

0 comments on commit b12fe19

Please sign in to comment.