Created
October 10, 2013 00:45
-
-
Save simonmorley/6911186 to your computer and use it in GitHub Desktop.
A bulk indexer for tire and elasticsearch. Based on Karmi's version with a couple of additions.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
task :reindex => :environment do | |
require 'ansi/progressbar' | |
params = {} | |
@date = Time.now.strftime '%Y%m%d%H%M%S' | |
def delete_index(index) | |
puts "[IMPORT] Deleting index '#{index.name}'" | |
index.delete | |
end | |
def create_index(index, klass) | |
unless index.exists? | |
mapping = MultiJson.encode(klass.tire.mapping_to_hash, :pretty => Tire::Configuration.pretty) | |
puts "[IMPORT] Creating index '#{index.name}' with mapping:", mapping | |
unless index.create(:mappings => klass.tire.mapping_to_hash, :settings => klass.tire.settings) | |
puts "[ERROR] There has been an error when creating the index -- Elasticsearch returned:", | |
index.response | |
exit(1) | |
end | |
end | |
end | |
def import_model(index, klass, params) | |
puts params | |
unless progress_bar(klass) | |
puts "[IMPORT] Importing '#{klass.to_s}'" | |
end | |
klass.tire.import(params) do |documents| | |
progress_bar(klass).inc documents.size if progress_bar(klass) | |
documents | |
end | |
progress_bar(klass).finish if progress_bar(klass) | |
end | |
def progress_bar(klass, total=nil) | |
@progress_bars ||= {} | |
if total | |
@progress_bars[klass.to_s] ||= ANSI::Progressbar.new(klass.to_s, total) | |
else | |
@progress_bars[klass.to_s] | |
end | |
end | |
[Announcement, Email, Location, Box].each do |klass| | |
puts "[IMPORT] #{klass.name} now being imported" | |
total = klass.all.size rescue nil | |
new_index = "#{klass.tire.index.name}_#{@date}" | |
index = Tire::Index.new(new_index) | |
params[:index] = index.name | |
params[:batch_size] = 1000 | |
delete_index(index) | |
create_index(index, klass) | |
import_model(index, klass, params) | |
if a = Tire::Alias.find(klass.tire.index.name) | |
puts "[IMPORT] aliases found: #{Tire::Alias.find(klass.tire.index.name).indices.to_ary.join(',')}. deleting." | |
old_indices = Tire::Alias.find(klass.tire.index.name).indices | |
old_indices.each do |index| | |
a.indices.delete index | |
end | |
a.indices.delete_if do |i| | |
Time.parse( i.gsub(/klass.name_/, '') ) < 4.weeks.ago rescue false | |
end | |
a.indices.add new_index | |
a.save | |
else | |
puts "[IMPORT] no aliases found. deleting index. creating new one and setting up alias." | |
klass.tire.index.delete | |
a = Tire::Alias.new | |
a.name(klass.tire.index.name) | |
a.index(new_index) | |
a.save | |
puts "Saved alias #{klass.tire.index.name} pointing to #{new_index}" | |
end | |
end | |
end |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment