thread-xp.rb

an experiment in a Celluloid-based mass downloader

require 'celluloid'
require 'zip/zipfilesystem'
require 'uri'
require 'net/http'

class ZipManifest
  include Celluloid

  def initialize(filename)
    @urls = {}
    @filename = filename
  end

  def add(url)
    @urls[url] = false
  end

  def package_files
    @packager = ZipPackager.new_link(Celluloid.current_actor, @filename)
    start_workers
    wait :all_packaged
    puts "ALL PACKAGED"
  ensure
    @packager.finish if @packager
  end

  def packaged(url)
    puts "DONE PACKAGING #{url}"
    @urls[url] = true
    signal :all_packaged if all_packaged?
  end

  def all_packaged?
    @urls.values.all?
  end

  def start_workers
    workers = 50.times.map do |i|
      FileDownloader.new_link(@packager)
    end
    @urls.each do |url,stat|
      wk = workers.shift
      wk.download! url
      workers.push wk
    end
  end

end

class FileDownloader
  include Celluloid

  def initialize(packager)
    @packager = packager
  end

  def download(url)
    uri = URI.parse(url)
    puts "FETCHING #{url}"
    Net::HTTP.start(uri.host) do |http|
      http.request_get(uri.path) do |response|
        @packager.package!(url, response.body)
      end
    end
  end

end

class ZipPackager
  include Celluloid

  def initialize(manifest, filename)
    @manifest = manifest
    @zipfile = Zip::ZipFile.open(filename, Zip::ZipFile::CREATE)
  end

  def finish

    @zipfile.close
  end

  def package(url, file)
    file_path = url.sub(/^.+\/events/, 'events')
    file_dir = File.dirname(file_path)
    puts "PACKAGING #{url} to #{file_path} (#{file.length} bytes)"
    unless @zipfile.file.directory?(file_dir)
      @zipfile.dir.mkdir(file_dir)
    end
    @zipfile.file.open(file_path, 'w') {|f|
      f.write file
    }
    @manifest.packaged!(url)
  end
end

manifest = ZipManifest.new "zip-out.zip"

File.readlines("somefiles.txt").each do |url|
  manifest.add url.chomp
end

manifest.package_files