Skip to content

Instantly share code, notes, and snippets.

@mps
Created March 7, 2011 17:00
Show Gist options
  • Save mps/858787 to your computer and use it in GitHub Desktop.
Save mps/858787 to your computer and use it in GitHub Desktop.
Refactored using FasterCSV
#!/usr/bin/env ruby
require 'optparse'
require 'rubygems'
#require 'excelsior'
require 'mongo'
require 'fastercsv'
load 'progressbar.rb'
options = { :server => "localhost", :port => "27017", :db => "AccuAuto" }
optparse = OptionParser.new do |opts|
opts.banner = "Usage: import-vins.rb [options] file-to-import.csv"
opts.on("-s", "--server SERVER_URL", "The URL of the server to connect to. The default is 'localhost'.") do |server|
options[:server] = server
end
opts.on("-p", "--port SERVER_PORT", "The port number to connect on. The default is '27017'.") do |port|
options[:port] = port
end
opts.on("-d", "--db DATABASE_NAME", "The database to import to. The default is 'AccuAuto'.") do |db|
options[:db] = db
end
opts.on("-h", "--help', 'Display this screen") do
puts opts
exit
end
end
optparse.parse!
if (ARGV.count == 0)
puts optparse
exit
end
csv_file = ARGV[0]
puts "Parsing #{csv_file}"
rows = FasterCSV.read(csv_file)
total = rows.count - 1;
puts "Found #{total} rows."
puts "Connecting to mongodb://#{options[:server]}:#{options[:port]}/#{options[:db]}"
conn = Mongo::ReplSetConnection.new([options[:server], options[:port]], :read_secondary => true)
db = conn.db(options[:db])
vins = db["VinInfo"]
puts "Removing #{vins.count} existing VINs."
vins.drop unless vins.count == 0
bar = ProgressBar.new("Adding VINs", rows.count)
(1..total).each do |i|
r = rows[i]
r.each {|f| f.strip! unless f.nil?}
vin = {
"vin" => r[7],
"yr" => r[0].to_i,
"mka" => r[1],
"mk" => r[2],
"md" => r[3],
"sm" => r[4],
"mn" => r[5].to_i,
"t" => r[6],
"bipd" => r[8],
"pipmp" => r[9],
"cmp" => r[10],
"col" => r[11],
"pns" => r[12],
"bs" => r[13],
"ss" => r[14],
"cyl" => r[15],
"es" => r[16],
"eh" => r[17],
"wb" => r[19],
"ht" => r[20],
"cw" => r[21],
"gw" => r[22],
"cls" => r[23],
"n" => r[24]
}
svchars = r[18]
vchars = []
idx = 0
until idx >= svchars.length
vc = svchars[idx,4]
if !vc.nil?
vc.strip!
vchars << vc
end
idx += 4
end
vin["vchars"] = vchars
if r[25] =~ %r{(\d+)/(\d+)/(\d+)}
vin["cd"] = Time.utc($3.to_i, $1.to_i, $2.to_i)
end
vins.insert(vin)
bar.inc
end
bar.finish
puts "Inserted #{vins.count} VINs into the collection."
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment