Merge branch 'feature/bulk-indexing'

README.md

@@ -85,6 +85,8 @@ Finally, add `importance` as a custom ranking attribute in the ranking tab under
        "index": "[YOUR_ALGOLIA_INDEX]"
      }
    ```
+   **Important note on bulk indexing**: setting `active` to `true` triggers both real-time indexing (when an CUD action is carried out on a post) and bulk indexing, which consists in indexing all published posts in one go. After installing this module, Ghost will automatically check your Algolia index during its next restart. If the index is empty, it will start sending fragments of all published posts (including the first few default posts coming with a fresh Ghost install). You might want to remove or unpublish those posts to save on operations.
+
 5. Apply the `ghost_algolia_register_events.patch` patch found in the app download by running the following command from the ghost root:
 
    ```shell
@@ -96,18 +98,29 @@ Finally, add `importance` as a custom ranking attribute in the ranking tab under
 
 # Usage
 
+## Real-time indexing
+
 Triggering indexing is transparent once the app is installed and happens on the following ghost panel operations:
 
 - publishing a new post (add a new record)
 - updating a published post (update an existing record)
 - unpublishing a post (remove a record)
 - deleting a post (remove a record)
 
+**Cost**: as many operations as fragments in the current post
+
+## Bulk indexing
+
+Bulk indexing happens automatically when Ghost is started, provided your Algolia index is empty and `active` is `true`  (See Installation, #4 for more information).
+
+**Cost**: 1 Algolia operation per restart + as many operations as fragments in all published posts
+
 # Compatibility
+
 Tested against Ghost 1.x.x releases.
 
 # Roadmap
 
 - ~~Switching to [fragment indexing](https://github.com/mlbrgl/kirby-algolia#principle).~~
-- Bulk indexing existing articles.
+- ~~Bulk indexing existing articles.~~
 - Upgrade to App API when available, to remove core hacking and simplify the installation process.

ghost_algolia_register_events.patch

@@ -1,16 +1,16 @@
-diff --git a/current/core/server/models/post.orig.js b/current/core/server/models/post.js
-index 4576534..69bc016 100644
---- a/current/core/server/models/post.orig.js
+diff --git a/current/core/server/models/post-orig.js b/current/core/server/models/post.js
+index f4c58c5..13e401f 100644
+--- a/current/core/server/models/post-orig.js
 +++ b/current/core/server/models/post.js
-@@ -15,6 +15,13 @@ var _               = require('lodash'),
+@@ -16,6 +16,13 @@ var _               = require('lodash'),
      Post,
      Posts;
 
 +// -- BEGIN ghost-algolia --
-+// Temporary hack added by ghost-algolia 
++// Temporary hack added by ghost-algolia
 +// (https://github.com/mlbrgl/ghost-algolia)
 +var ghostAlgolia    = require(config.getContentPath('apps') + 'ghost-algolia');
-+ghostAlgolia.registerEvents(events);
++ghostAlgolia.init(events, config, utils);
 +// -- END ghost-algolia --
 +
  Post = ghostBookshelf.Model.extend({

index.js

@@ -18,35 +18,75 @@
 // module.exports = GhostAlgolia;
 
 const converter = require('../../../current/core/server/utils/markdown-converter'),
+      client = require('../../../current/core/server/models').Client,
       indexFactory = require('./lib/indexFactory'),
       parserFactory = require('./lib/parserFactory');
 
-const GhostAlgolia = {};
+const GhostAlgolia = {
+  init: (events, config, utils) => {
+    bulkIndex(events, config, utils);
+    registerEvents(events, config);
+  }
+};
+
+/*
+ * Index all published posts at server start if Algolia indexing activated in config
+ */
+function bulkIndex(events, config, utils){
+
+  // Emitted in ghost-server.js
+  events.on('server:start', function(){
+    client.findOne({slug: 'ghost-frontend'}, {context: {internal: true}})
+    .then((client) => getContent(utils.url.urlFor('api', true) + 'posts/?formats=mobiledoc&client_id=ghost-frontend&client_secret=' + client.attributes.secret))
+    .then((data) => {
+      let posts = JSON.parse(data).posts;
+      if(posts.length > 0) {
+        let index = indexFactory(config);
+        if(index.connect()) {
+          index.countRecords().then((nbRecords) => {
+            if(nbRecords === 0) {
+              let parser = parserFactory(),
+                  nbFragments = 0;
+
+              posts.forEach((post) => {
+                post.attributes = post; // compatibility with posts returned internally in events (below)
+                nbFragments += parser.parse(post, index);
+              });
+              if(nbFragments) { index.save(); }
+            }
+          })
+          .catch((err) => console.error(err));
+        }
+      }
+    })
+    .catch((err) => console.error(err));
+  });
+}
 
 
 /*
  * Register (post) events to react to admin panel actions
  */
-GhostAlgolia.registerEvents = function registerEvents(events) {
+function registerEvents(events, config){
 
   // React to post being published (from unpublished)
   events.on('post.published', function(post) {
-    let index = indexFactory();
+    let index = indexFactory(config);
     if(index.connect() && parserFactory().parse(post, index)) {
-      index.add(post)
+      index.save()
       .then(() => { console.log('GhostAlgolia: post "' + post.attributes.title + '" has been added to the index.'); })
       .catch((err) => console.log(err));
     };
   });
 
   // React to post being edited in a published state
   events.on('post.published.edited', function(post) {
-    let index = indexFactory();
+    let index = indexFactory(config);
     if(index.connect()) {
       let promisePublishedEdited;
       if(parserFactory().parse(post, index)) {
         promisePublishedEdited = index.delete(post)
-                                 .then(() => { index.add(post) });
+        .then(() => index.save());
       } else {
         promisePublishedEdited = index.delete(post);
       }
@@ -61,14 +101,36 @@ GhostAlgolia.registerEvents = function registerEvents(events) {
   // before the deleted event which becomes redundant. Deletion of unpublished posts
   // is of no concern as they never made it to the index.
   events.on('post.unpublished', function(post) {
-    let index = indexFactory();
+    let index = indexFactory(config);
     if(index.connect()) {
       index.delete(post)
       .then(() => { console.log('GhostAlgolia: post "' + post.attributes.title + '" has been removed from the index.'); })
       .catch((err) => console.log(err));
     };
   });
-
 }
 
+// https://www.tomas-dvorak.cz/posts/nodejs-request-without-dependencies/
+function getContent (url) {
+  // return new pending promise
+  return new Promise((resolve, reject) => {
+    // select http or https module, depending on reqested url
+    const lib = url.startsWith('https') ? require('https') : require('http');
+    const request = lib.get(url, (response) => {
+      // handle http errors
+      if (response.statusCode < 200 || response.statusCode > 299) {
+         reject(new Error('Failed to load page, status code: ' + response.statusCode));
+       }
+      // temporary data holder
+      const body = [];
+      // on every content chunk, push it to the data array
+      response.on('data', (chunk) => body.push(chunk));
+      // we are done, resolve promise with those joined chunks
+      response.on('end', () => resolve(body.join('')));
+    });
+    // handle connection errors of the request
+    request.on('error', (err) => reject(err))
+    })
+};
+
 module.exports = GhostAlgolia;

lib/indexFactory.js

@@ -1,9 +1,7 @@
-const algoliaSearch = require('algoliasearch'),
-      config = require('../../../../current/core/server/config'),
-      algoliaSettings = config.get('algolia');
+const algoliaSearch = require('algoliasearch');
 
-
-const indexFactory = () => {
+const indexFactory = (config) => {
+  const algoliaSettings = config.get('algolia');
   let _fragments = [];
   let index;
 
@@ -28,15 +26,21 @@ const indexFactory = () => {
         _fragments.push(fragment);
       }
     },
-    hasFragments: () => {
-      return _fragments.length > 0;
+    fragmentsCount: () => {
+      return _fragments.length;
     },
-    add: (post) => {
+    save: () => {
       return index.addObjects(_fragments);
     },
     delete: (post) => {
       return index.deleteByQuery(post.attributes.uuid, {restrictSearchableAttributes: 'post_uuid'});
     },
+    getFragments: () => {
+      return _fragments;
+    },
+    countRecords: () => {
+      return index.search({query: '', hitsPerPage: 0}).then((queryResult) => queryResult.nbHits);
+    }
   }
 }
 

lib/parserFactory.js

@@ -48,7 +48,7 @@ const parserFactory = () => {
         index.addFragment(fragment);
       }
 
-      return index.hasFragments();
+      return index.fragmentsCount();
     }
   }
 }