worker_get_url.rb

Gemfile

source 'https://rubygems.org'
gem 'open_uri_redirections'

get_url.sh

#!/bin/bash -xe
curl $1 |
    egrep -o 'href="(.*?)"' |
    tr '=' ' ' |
    awk '{print $2}' |
    tr -d '"' |
    grep -E '^https?://'

worker_get_url.rb

require 'thread'
require 'open-uri'
require 'open_uri_redirections'

class Workers
  def initialize(count)
    @q = Queue.new
    @count = count
    @jobs = (0...@count).map.with_index {|id|
      Thread.new do
        while (job = @q.pop)
          total = job.(id, total)
        end
      end
    }
  end
  def push(&proc)
    @q.push(proc)
  end
  def wait
    @count.times { @q.push(nil) }
    @jobs.each {|job| job.join.value}
  ensure
    @q.close
  end
end

def run(uri)
  try = 0
  begin
    puts "open: #{uri}"
    res = open(uri, :allow_redirections => :safe)
    puts "success: #{uri} #{res.read.size} #{res.status}"
  rescue OpenURI::HTTPRedirect => redirect
    uri = redirect.uri
    p uri
    retry
  rescue => e
    try += 1
    warn "warn: #{try} #{e.message}\n"
    retry if try <= 4
  end
end

w = Workers.new(60)
$stdin.read.split("\n").each do |uri|
  w.push do |id|
    run(uri)
  end
end
w.wait