Split a url file into some files. Same hosts are written in the same file.

split_url.rb

#!/usr/bin/env ruby
# -*- coding: utf-8 -*-

require 'URI'

file = ARGV.shift
num_split = ARGV.shift.to_i
order = Math::log10(num_split).truncate + 1

url_hash = {}

open(file).each_line do |line|
  line ? line.chomp! : next

	uri = begin
		URI.parse(line)
	rescue
		next
	end

	#p uri.host.intern.object_id
	key = uri.host.intern.hash % num_split
	#p key
	(url_hash[key] ||= []) << line
end

url_hash.each_with_index do |(k,v),i|
	filename = "%s_%0#{order}d" % [file, i]
	#p filename
	open(filename, "w"){|f|
		v.each{|url| f.puts url}
	}
end