Limit open threads in ruby

download_pages.rb

# Pretty simple solution to harness the capability of multithreading without running into problems 
# caused by doing too many things at once.

# In this example, I was trying to download hundreds of web pages, but opening all those connections 
# simultaneously caused a variety of errors

# Contains download_link(link,save_dir,save_name) to download and save webpages locally (irrelevant)
require 'download_page'

# keep the threads in here
threads = []

(1967..2010).each do |year| # (irrelevant to gist)

  # excluded irrelevant variable definitions :  link,save_dir,save_name ...

  # Only open 10 threads at a time
  if(Thread.list.count % 10 != 0) 
    download_thread = Thread.new do
      download_link(link,save_dir,save_name)
    end
    threads << download_thread
  else
    # Wait for open threads to finish executing before starting new one
    threads.each do |thread|
      thread.join
    end
    # Start downloading again
    download_thread = Thread.new do
      download_link(link,save_dir,save_name)
    end
    threads << download_thread
  end
end

# Wait for threads to finish executing before exiting the program
threads.each do |thread|
  thread.join
end