RailsCastsMultiStream

RailsCastsMultiStream.rb

# Copyright © 2013 Ishkov Dmitry <[email protected]>
# This work is free. You can redistribute it and/or modify it under the
# terms of the Do What The Fuck You Want To Public License, Version 2,
# as published by Sam Hocevar. See http://www.wtfpl.net/ for more details.

require 'open-uri'
require 'nokogiri'
require 'typhoeus'

def make_name link
  if link[-1]=='/'
		link=link[0..-2]
	end
	link[link.rindex(/\//)+1..-1]   
end

feed_burner_link = 'http://feeds.feedburner.com/railscasts'
doc = Nokogiri::HTML open(feed_burner_link)
links = doc.css('enclosure').map { |e| e['url'] }.reverse
while links.length>0 do
	hydra = Typhoeus::Hydra.new(max_concurrency: 20)
	memory_usage = 0
	for link in links do 
		request = Typhoeus::Request.new(link)
		request.on_complete do |response|
			links.delete response.request.url
			file_name=make_name response.request.url
			file = File.new (file_name), 'wb' 
			puts "Starting #{file_name}"
			memory_usage+=response.response_body.length
			file << response.response_body
			file.close
			puts "Memory usage #{memory_usage/1000000} Mb"
			puts "Links left: #{links.length}"
			puts
			if memory_usage/1000000 > 300 # Mb
				hydra.abort
				memory_usage=0
				GC.start
			end
		end 
		hydra.queue request
	end
	hydra.run   
end