Skip to content

Instantly share code, notes, and snippets.

@simonmorley
Created October 10, 2013 00:45
Show Gist options
  • Save simonmorley/6911186 to your computer and use it in GitHub Desktop.
Save simonmorley/6911186 to your computer and use it in GitHub Desktop.
A bulk indexer for tire and elasticsearch. Based on Karmi's version with a couple of additions.
task :reindex => :environment do
require 'ansi/progressbar'
params = {}
@date = Time.now.strftime '%Y%m%d%H%M%S'
def delete_index(index)
puts "[IMPORT] Deleting index '#{index.name}'"
index.delete
end
def create_index(index, klass)
unless index.exists?
mapping = MultiJson.encode(klass.tire.mapping_to_hash, :pretty => Tire::Configuration.pretty)
puts "[IMPORT] Creating index '#{index.name}' with mapping:", mapping
unless index.create(:mappings => klass.tire.mapping_to_hash, :settings => klass.tire.settings)
puts "[ERROR] There has been an error when creating the index -- Elasticsearch returned:",
index.response
exit(1)
end
end
end
def import_model(index, klass, params)
puts params
unless progress_bar(klass)
puts "[IMPORT] Importing '#{klass.to_s}'"
end
klass.tire.import(params) do |documents|
progress_bar(klass).inc documents.size if progress_bar(klass)
documents
end
progress_bar(klass).finish if progress_bar(klass)
end
def progress_bar(klass, total=nil)
@progress_bars ||= {}
if total
@progress_bars[klass.to_s] ||= ANSI::Progressbar.new(klass.to_s, total)
else
@progress_bars[klass.to_s]
end
end
[Announcement, Email, Location, Box].each do |klass|
puts "[IMPORT] #{klass.name} now being imported"
total = klass.all.size rescue nil
new_index = "#{klass.tire.index.name}_#{@date}"
index = Tire::Index.new(new_index)
params[:index] = index.name
params[:batch_size] = 1000
delete_index(index)
create_index(index, klass)
import_model(index, klass, params)
if a = Tire::Alias.find(klass.tire.index.name)
puts "[IMPORT] aliases found: #{Tire::Alias.find(klass.tire.index.name).indices.to_ary.join(',')}. deleting."
old_indices = Tire::Alias.find(klass.tire.index.name).indices
old_indices.each do |index|
a.indices.delete index
end
a.indices.delete_if do |i|
Time.parse( i.gsub(/klass.name_/, '') ) < 4.weeks.ago rescue false
end
a.indices.add new_index
a.save
else
puts "[IMPORT] no aliases found. deleting index. creating new one and setting up alias."
klass.tire.index.delete
a = Tire::Alias.new
a.name(klass.tire.index.name)
a.index(new_index)
a.save
puts "Saved alias #{klass.tire.index.name} pointing to #{new_index}"
end
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment