diff --git a/README.md b/README.md index e7ae046..456e8bc 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,8 @@ Finally, add `importance` as a custom ranking attribute in the ranking tab under "index": "[YOUR_ALGOLIA_INDEX]" } ``` +   **Important note on bulk indexing**: setting `active` to `true` triggers both real-time indexing (when an CUD action is carried out on a post) and bulk indexing, which consists in indexing all published posts in one go. After installing this module, Ghost will automatically check your Algolia index during its next restart. If the index is empty, it will start sending fragments of all published posts (including the first few default posts coming with a fresh Ghost install). You might want to remove or unpublish those posts to save on operations. + 5. Apply the `ghost_algolia_register_events.patch` patch found in the app download by running the following command from the ghost root: ```shell @@ -96,6 +98,8 @@ Finally, add `importance` as a custom ranking attribute in the ranking tab under # Usage +## Real-time indexing + Triggering indexing is transparent once the app is installed and happens on the following ghost panel operations: - publishing a new post (add a new record) @@ -103,11 +107,20 @@ Triggering indexing is transparent once the app is installed and happens on the - unpublishing a post (remove a record) - deleting a post (remove a record) +**Cost**: as many operations as fragments in the current post + +## Bulk indexing + +Bulk indexing happens automatically when Ghost is started, provided your Algolia index is empty and `active` is `true` (See Installation, #4 for more information). + +**Cost**: 1 Algolia operation per restart + as many operations as fragments in all published posts + # Compatibility + Tested against Ghost 1.x.x releases. # Roadmap - ~~Switching to [fragment indexing](https://github.com/mlbrgl/kirby-algolia#principle).~~ -- Bulk indexing existing articles. +- ~~Bulk indexing existing articles.~~ - Upgrade to App API when available, to remove core hacking and simplify the installation process. diff --git a/ghost_algolia_register_events.patch b/ghost_algolia_register_events.patch index d438935..c1dc661 100644 --- a/ghost_algolia_register_events.patch +++ b/ghost_algolia_register_events.patch @@ -1,16 +1,16 @@ -diff --git a/current/core/server/models/post.orig.js b/current/core/server/models/post.js -index 4576534..69bc016 100644 ---- a/current/core/server/models/post.orig.js +diff --git a/current/core/server/models/post-orig.js b/current/core/server/models/post.js +index f4c58c5..13e401f 100644 +--- a/current/core/server/models/post-orig.js +++ b/current/core/server/models/post.js -@@ -15,6 +15,13 @@ var _ = require('lodash'), +@@ -16,6 +16,13 @@ var _ = require('lodash'), Post, Posts; +// -- BEGIN ghost-algolia -- -+// Temporary hack added by ghost-algolia ++// Temporary hack added by ghost-algolia +// (https://github.com/mlbrgl/ghost-algolia) +var ghostAlgolia = require(config.getContentPath('apps') + 'ghost-algolia'); -+ghostAlgolia.registerEvents(events); ++ghostAlgolia.init(events, config, utils); +// -- END ghost-algolia -- + Post = ghostBookshelf.Model.extend({ diff --git a/index.js b/index.js index f4e8173..5d755c4 100644 --- a/index.js +++ b/index.js @@ -18,22 +18,62 @@ // module.exports = GhostAlgolia; const converter = require('../../../current/core/server/utils/markdown-converter'), + client = require('../../../current/core/server/models').Client, indexFactory = require('./lib/indexFactory'), parserFactory = require('./lib/parserFactory'); -const GhostAlgolia = {}; +const GhostAlgolia = { + init: (events, config, utils) => { + bulkIndex(events, config, utils); + registerEvents(events, config); + } +}; + +/* + * Index all published posts at server start if Algolia indexing activated in config + */ +function bulkIndex(events, config, utils){ + + // Emitted in ghost-server.js + events.on('server:start', function(){ + client.findOne({slug: 'ghost-frontend'}, {context: {internal: true}}) + .then((client) => getContent(utils.url.urlFor('api', true) + 'posts/?formats=mobiledoc&client_id=ghost-frontend&client_secret=' + client.attributes.secret)) + .then((data) => { + let posts = JSON.parse(data).posts; + if(posts.length > 0) { + let index = indexFactory(config); + if(index.connect()) { + index.countRecords().then((nbRecords) => { + if(nbRecords === 0) { + let parser = parserFactory(), + nbFragments = 0; + + posts.forEach((post) => { + post.attributes = post; // compatibility with posts returned internally in events (below) + nbFragments += parser.parse(post, index); + }); + if(nbFragments) { index.save(); } + } + }) + .catch((err) => console.error(err)); + } + } + }) + .catch((err) => console.error(err)); + }); +} /* * Register (post) events to react to admin panel actions */ -GhostAlgolia.registerEvents = function registerEvents(events) { +function registerEvents(events, config){ // React to post being published (from unpublished) events.on('post.published', function(post) { - let index = indexFactory(); + let index = indexFactory(config); if(index.connect() && parserFactory().parse(post, index)) { - index.add(post) + index.save() .then(() => { console.log('GhostAlgolia: post "' + post.attributes.title + '" has been added to the index.'); }) .catch((err) => console.log(err)); }; @@ -41,12 +81,12 @@ GhostAlgolia.registerEvents = function registerEvents(events) { // React to post being edited in a published state events.on('post.published.edited', function(post) { - let index = indexFactory(); + let index = indexFactory(config); if(index.connect()) { let promisePublishedEdited; if(parserFactory().parse(post, index)) { promisePublishedEdited = index.delete(post) - .then(() => { index.add(post) }); + .then(() => index.save()); } else { promisePublishedEdited = index.delete(post); } @@ -61,14 +101,36 @@ GhostAlgolia.registerEvents = function registerEvents(events) { // before the deleted event which becomes redundant. Deletion of unpublished posts // is of no concern as they never made it to the index. events.on('post.unpublished', function(post) { - let index = indexFactory(); + let index = indexFactory(config); if(index.connect()) { index.delete(post) .then(() => { console.log('GhostAlgolia: post "' + post.attributes.title + '" has been removed from the index.'); }) .catch((err) => console.log(err)); }; }); - } +// https://www.tomas-dvorak.cz/posts/nodejs-request-without-dependencies/ +function getContent (url) { + // return new pending promise + return new Promise((resolve, reject) => { + // select http or https module, depending on reqested url + const lib = url.startsWith('https') ? require('https') : require('http'); + const request = lib.get(url, (response) => { + // handle http errors + if (response.statusCode < 200 || response.statusCode > 299) { + reject(new Error('Failed to load page, status code: ' + response.statusCode)); + } + // temporary data holder + const body = []; + // on every content chunk, push it to the data array + response.on('data', (chunk) => body.push(chunk)); + // we are done, resolve promise with those joined chunks + response.on('end', () => resolve(body.join(''))); + }); + // handle connection errors of the request + request.on('error', (err) => reject(err)) + }) +}; + module.exports = GhostAlgolia; diff --git a/lib/indexFactory.js b/lib/indexFactory.js index 866eab2..f9fde78 100644 --- a/lib/indexFactory.js +++ b/lib/indexFactory.js @@ -1,9 +1,7 @@ -const algoliaSearch = require('algoliasearch'), - config = require('../../../../current/core/server/config'), - algoliaSettings = config.get('algolia'); +const algoliaSearch = require('algoliasearch'); - -const indexFactory = () => { +const indexFactory = (config) => { + const algoliaSettings = config.get('algolia'); let _fragments = []; let index; @@ -28,15 +26,21 @@ const indexFactory = () => { _fragments.push(fragment); } }, - hasFragments: () => { - return _fragments.length > 0; + fragmentsCount: () => { + return _fragments.length; }, - add: (post) => { + save: () => { return index.addObjects(_fragments); }, delete: (post) => { return index.deleteByQuery(post.attributes.uuid, {restrictSearchableAttributes: 'post_uuid'}); }, + getFragments: () => { + return _fragments; + }, + countRecords: () => { + return index.search({query: '', hitsPerPage: 0}).then((queryResult) => queryResult.nbHits); + } } } diff --git a/lib/parserFactory.js b/lib/parserFactory.js index 26afbfb..7a891a3 100644 --- a/lib/parserFactory.js +++ b/lib/parserFactory.js @@ -48,7 +48,7 @@ const parserFactory = () => { index.addFragment(fragment); } - return index.hasFragments(); + return index.fragmentsCount(); } } }