From 389d934c6df83275a176330f7d5bdd05ec136853 Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Mon, 12 Oct 2015 23:25:35 +0700 Subject: [PATCH] async indexing / deletion done --- Gemfile | 1 + Gemfile.lock | 58 ++++++++++++++++++++++++++++++++++ app/jobs/indexer_job.rb | 20 ++++++++++++ app/models/movie.rb | 16 ++++++++-- config/application.rb | 1 + config/initializers/sidekiq.rb | 5 +++ test/jobs/indexer_job_test.rb | 7 ++++ 7 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 app/jobs/indexer_job.rb create mode 100644 config/initializers/sidekiq.rb create mode 100644 test/jobs/indexer_job_test.rb diff --git a/Gemfile b/Gemfile index 0bfe199..1810efb 100644 --- a/Gemfile +++ b/Gemfile @@ -27,6 +27,7 @@ gem 'sdoc', '~> 0.4.0', group: :doc gem 'elasticsearch-rails' gem 'elasticsearch-model' gem 'bootstrap-sass' +gem 'sidekiq' gem 'sequel' diff --git a/Gemfile.lock b/Gemfile.lock index e8e5e19..4d4e38f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -49,6 +49,47 @@ GEM builder (3.2.2) byebug (4.0.5) columnize (= 0.9.0) + celluloid (0.17.1.2) + bundler + celluloid-essentials + celluloid-extras + celluloid-fsm + celluloid-pool + celluloid-supervision + dotenv + nenv + rspec-logsplit (>= 0.1.2) + timers (>= 4.1.1) + celluloid-essentials (0.20.2.1) + bundler + dotenv + nenv + rspec-logsplit (>= 0.1.2) + timers (>= 4.1.1) + celluloid-extras (0.20.1) + bundler + dotenv + nenv + rspec-logsplit (>= 0.1.2) + timers (>= 4.1.1) + celluloid-fsm (0.20.1) + bundler + dotenv + nenv + rspec-logsplit (>= 0.1.2) + timers (>= 4.1.1) + celluloid-pool (0.20.1) + bundler + dotenv + nenv + rspec-logsplit (>= 0.1.2) + timers (>= 4.1.1) + celluloid-supervision (0.20.1.1) + bundler + dotenv + nenv + rspec-logsplit (>= 0.1.2) + timers (>= 4.1.1) coderay (1.1.0) coffee-rails (4.1.0) coffee-script (>= 2.2.0) @@ -58,7 +99,9 @@ GEM execjs coffee-script-source (1.9.1.1) columnize (0.9.0) + connection_pool (2.2.0) debug_inspector (0.0.2) + dotenv (2.0.2) elasticsearch (1.0.8) elasticsearch-api (= 1.0.7) elasticsearch-transport (= 1.0.7) @@ -79,6 +122,7 @@ GEM globalid (0.3.6) activesupport (>= 4.1.0) hashie (3.4.1) + hitimes (1.2.3) i18n (0.7.0) jbuilder (2.2.13) activesupport (>= 3.0.0, < 5) @@ -103,6 +147,7 @@ GEM ruby-progressbar multi_json (1.11.0) multipart-post (2.0.0) + nenv (0.2.0) nokogiri (1.6.6.2) mini_portile (~> 0.6.0) pg (0.18.1) @@ -141,6 +186,10 @@ GEM thor (>= 0.18.1, < 2.0) rake (10.4.2) rdoc (4.2.0) + redis (3.2.1) + redis-namespace (1.5.2) + redis (~> 3.0, >= 3.0.4) + rspec-logsplit (0.1.3) ruby-progressbar (1.7.5) sass (3.4.13) sass-rails (5.0.3) @@ -153,6 +202,12 @@ GEM json (~> 1.7, >= 1.7.7) rdoc (~> 4.0) sequel (4.21.0) + sidekiq (3.5.0) + celluloid (~> 0.17.0) + connection_pool (~> 2.2, >= 2.2.0) + json (~> 1.0) + redis (~> 3.2, >= 3.2.1) + redis-namespace (~> 1.5, >= 1.5.2) slop (3.6.0) spring (1.3.6) sprockets (3.3.3) @@ -164,6 +219,8 @@ GEM thor (0.19.1) thread_safe (0.3.5) tilt (1.4.1) + timers (4.1.1) + hitimes turbolinks (2.5.3) coffee-rails tzinfo (1.2.2) @@ -196,6 +253,7 @@ DEPENDENCIES sass-rails (~> 5.0) sdoc (~> 0.4.0) sequel + sidekiq spring turbolinks uglifier (>= 1.3.0) diff --git a/app/jobs/indexer_job.rb b/app/jobs/indexer_job.rb new file mode 100644 index 0000000..6862fff --- /dev/null +++ b/app/jobs/indexer_job.rb @@ -0,0 +1,20 @@ +class IndexerJob < ActiveJob::Base + queue_as :elasticsearch + + def perform(operation, record_id) + logger.debug [operation, "Movie: #{record_id}"] + self.send(operation, record_id) + end + +private + + def index(record_id) + record = Movie.find(record_id) + record.__elasticsearch__.index_document + end + + def delete(record_id) + record = Movie.find(record_id) + record.__elasticsearch__.delete_document + end +end diff --git a/app/models/movie.rb b/app/models/movie.rb index 30be020..b5a8e9b 100644 --- a/app/models/movie.rb +++ b/app/models/movie.rb @@ -1,6 +1,19 @@ class Movie < ActiveRecord::Base include Elasticsearch::Model - include Elasticsearch::Model::Callbacks + # we don't need the line below because we're + # implementing our own + # include Elasticsearch::Model::Callbacks + + after_commit :index_document, on: [:create, :update] + after_commit :delete_document, on: :destroy + + def index_document + IndexerJob.perform_later('index', self.id) + end + + def delete_document + IndexerJob.perform_later('delete', self.id) + end validates :name, uniqueness: true @@ -46,7 +59,6 @@ def multi_match_query(query) end end - class RelationError < StandardError def initialize(msg = "That Relationship Type doesn't exist") super(msg) diff --git a/config/application.rb b/config/application.rb index e7dae9a..bdc8d67 100644 --- a/config/application.rb +++ b/config/application.rb @@ -20,6 +20,7 @@ class Application < Rails::Application # The default locale is :en and all translations from config/locales/*.rb,yml are auto loaded. # config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s] # config.i18n.default_locale = :de + config.active_job.queue_adapter = :sidekiq # Do not swallow errors in after_commit/after_rollback callbacks. config.active_record.raise_in_transactional_callbacks = true diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb new file mode 100644 index 0000000..34bbc3c --- /dev/null +++ b/config/initializers/sidekiq.rb @@ -0,0 +1,5 @@ +Sidekiq.configure_server do |config| + config.server_middleware do |chain| + chain.remove Sidekiq::Middleware::Server::RetryJobs + end +end \ No newline at end of file diff --git a/test/jobs/indexer_job_test.rb b/test/jobs/indexer_job_test.rb new file mode 100644 index 0000000..42cacea --- /dev/null +++ b/test/jobs/indexer_job_test.rb @@ -0,0 +1,7 @@ +require 'test_helper' + +class IndexerJobTest < ActiveJob::TestCase + # test "the truth" do + # assert true + # end +end