Sample downloader tool in Ruby. it reads HTTP or FTP sites from /local/misc/misc/sites and stores results into /local/misc/mirrors.

downloader.rb

#!/usr/bin/ruby
require 'net/ftp'
require 'net/http'
require 'fileutils'

class String
    def to_l(srcl,dstl)
    val = gsub(/(["'\&\(\)])/) { "\\" + $1 }
    `echo #{val} |iconv -f #{srcl} -t #{dstl}`.sub(/\n/,"")
    end
end
class Net::FTP
    def mirror(folder)
	printf("Curdir %s\n",  FileUtils.pwd) if $verbose > 1
	printf("Remdir %s\n",  pwd) if $verbose > 1
	name = folder.to_l("cp1251","utf-8")
	unless $simulate > 0 then
	    begin
		    FileUtils.cd(name)
	    rescue Errno::ENOENT
		    FileUtils.makedirs(name)
		    retry
	    end
	end
	printf("Folder %s\n", name) if $verbose > 0
	chdir(folder) 
	list.each {|line|
	    line =~ /^([d-])[rwx-]{1,9} +\d+ \w+ +\w+ +(\d+) (\w{3} +\d{1,2} +[\d:]+) (.*)$/
	    next if $4 == "." or $4 == ".."
	    if $1 == "d" then
		    printf("Down to folder %s\n", $4) if $verbose > 1
		    mirror($4) 
	    else
		    file = $4.to_l("cp1251","utf-8")
		    if File.size?(file) == $2.to_i then
		        printf("File %s is exist and the same as remote: not downloading\n", file) if $verbose > 0
		        next;
		    end
		    printf("Downloading file %s\n", file) if $verbose > 0
		    begin
		        getbinaryfile($4, $4.to_l("cp1251","utf-8"), 1024) unless $simulate > 0
#		    rescue Errno::EEXIST
		    end
	    end
	}
	FileUtils.cd("..")
	chdir("..")
    end
end

def log(msg)
    puts msg
end

def error(msg)
    log msg
    exit
end

#puts "Initialization"
p = `ps aux`#|grep downloader.rb`
c = 0
val = $0.gsub(/.*\//,"")
while p =~ /ruby.*#{val}/o
    c = c + 1 
    p = p.sub(/#{val}/o, "")
end 
exit if c > 1

#init vars
$verbose = 1
$simulate = 0

root_folder = "/local/misc/mirrors"

sitef = IO.read('/local/misc/misc/sites');

#change to specified path
begin
    FileUtils.cd(root_folder)
rescue Errno::ENOENT
    FileUtils.mkdir(root_folder)
    retry
end

sitef.each { |line|
    FileUtils.cd(root_folder)
    next if line == "\n"
    puts "Reading the new line" if $verbose > 0
    if line =~ /^(.*) +(.*?)$/ then
	    site = $1
	    folder_path = $2
    else
	    site = line
    end
    if (site.gsub(/%(..)/) {|s| $1.hex.chr } !~ /^ *(ftp|http):\/\/(?:(.*?)\/)?(?:(.*)\/)*(?:(.*)\/?)?$/) then
	    puts "Wrong site name, please check"
	    exit 1
    end
    ftp_addr = $2
    folder_path = $2 + '/' + $3.to_s.to_l("cp1251","utf-8") if !folder_path
    remote_dir = $3
    folder = $4
    puts ftp_addr, folder_path, remote_dir, folder if $verbose > 2
    puts "Connection to site #{ftp_addr}" if $verbose > 0
    printf("Site = %s, folder = %s, folder = %s\n", ftp_addr.to_l("cp1251","utf-8"), folder_path, folder.to_l("cp1251","utf-8")) if $verbose > 1
    begin
	    ftp = Net::FTP.new(ftp_addr.gsub(/\//,""))
    rescue Errno::ECONNREFUSED
	    printf("Connection to server %s refused\n", ftp_addr.gsub(/\//,""))
	    exit 1
    end
    begin
	    ftp.login
    rescue Net::FTPTempError
	next
    end
    begin
	    FileUtils.cd(folder_path)
    rescue Errno::ENOENT
        FileUtils.makedirs(folder_path)
	retry
    end
    ftp.chdir(remote_dir) if remote_dir
    ftp.mirror(folder)
    ftp.close
}