threads.rb

# source: Jan Aerts http://saaientist.blogspot.com/2009/01/threads-in-ruby-probably-not-how-to-use.html

require 'rubygems'
require 'progressbar'

MAX_NR_OF_THREADS = 5
nr_of_lines = `wc -l input_file.tsv`.split(/ /)[0].to_i
pbar = ProgressBar.new('processing', nr_of_lines.to_f/MAX_NR_OF_THREADS)
File.open(input_file.tsv).each_slice(MAX_NR_OF_THREADS) do |slice|
  pbar.inc
  threads = Hash.new
  slice.each do |line|
    threads[line] = Thread.new do
      # do the actual line parsing, DB lookup and DB updates
    end
  end
  threads.values.each do |thread|
    thread.join
  end
end
pbar.finish