-
-
Save asccigcc/c4dda5290be88b4cb8397006cdf1abcd to your computer and use it in GitHub Desktop.
Simple way to do multi-threaded chunking in Ruby : http://proccli.com/super-simple-thread-pooling-ruby
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
require 'thread' | |
# Stupid simple "multi-threading" - it doesn't use mutex or queues but | |
# it does have access to local variables, which is convenient. This will | |
# break a data set into equal slices and process them, but it is not | |
# perfect in that it will not start the next set until the first is | |
# completely processed -- so, if you have 1 slow item it loses benefit | |
# NOTE: this is not thread-safe! | |
class ThreadPool | |
def self.process!(data, size = 2, &block) | |
Array(data).each_slice(size) do |slice| | |
slice.map do |item| | |
Thread.new{ block.call(item) } | |
end.map{ |t| t.join } | |
end | |
end | |
def initialize(size) | |
@size = size | |
end | |
def process!(data, &block) | |
self.class.process!(data, @size, &block) | |
end | |
end | |
# Playing around with it on the alphabet | |
# adjust the +size+ to adjust how many threads are | |
# being used at once | |
if $0 == __FILE__ | |
require 'pp' | |
require 'benchmark' | |
size = 10 | |
words = ('a'..'z').to_a | |
list = [] | |
pool = ThreadPool.new(size) | |
puts "Starting (P: %d W: %d)" % [size, words.size] | |
b = Benchmark.realtime{ | |
pool.process!(words) do |word| | |
b = Benchmark.realtime{ | |
list << word | |
sleep rand*2 | |
} | |
puts "\t\tFinished: #{word} -- %0.2f seconds" % b | |
end | |
} | |
# Output some times | |
puts "Finished all: %0.2f seconds" % b | |
puts "\nList: %s\n\n" % list.join(', ') | |
end |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment