Created
December 13, 2014 05:48
-
-
Save lucascaton/7a3f1bbc963472a1a970 to your computer and use it in GitHub Desktop.
Car Sales checker (Ruby script)
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
source 'https://rubygems.org' | |
gem 'mechanize' | |
gem 'pry' |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env ruby | |
require 'mechanize' | |
puts 'Starting...' | |
base_url = 'http://www.carsales.com.au' | |
list_url = "#{base_url}/all-cars/results.aspx?WT.z_srchsrcx=makemodel&q=(((((((((SiloType%3d%5bBrand+new+cars+in+stock%5d%7cSiloType%3d%5bBrand+new+cars+available%5d)%7cSiloType%3d%5bDealer+used+cars%5d)%7cSiloType%3d%5bDemo+and+near+new+cars%5d)%7cSiloType%3d%5bPrivate+seller+cars%5d)%26(Make%7b%3d%7d%5bHyundai%5d%7b%26%7dModel%7b%3d%7d%5bix35%5d))%26GenericGearType%7b%3d%7d%5bAutomatic%5d)%26Service%3d%5bCarsales%5d)%26Location%3dlocation%5b-27.45076%2c153.0341x250000%5d))%26(Odometer%3drange%5b..150000%5d)&vertical=car&postcode=4006&distance=250&silo=stock&sortby=~Price" | |
agent = Mechanize.new | |
list_page = agent.get(list_url) | |
cars = list_page.search('.result-item a:contains("View")') | |
results = [] | |
cars.each do |car| | |
print "Fetching car #{cars.index(car) + 1} from #{cars.count}\r" | |
car_url = "#{base_url}#{car[:href]}" | |
car_page = agent.get(car_url) | |
result = {} | |
%i[badge year color fuel price].each do |attr| | |
result[attr] = car_page.search("meta[name='WT.z_#{attr}']").first.attributes['content'].value | |
end | |
%i[title description image].each do |attr| | |
result[attr] = car_page.search("meta[name='og:#{attr}']").first.attributes['content'].value rescue nil | |
end | |
result[:kilometres] = car_page.search("td.term:contains('Kilometres')").first.parent.children. | |
search('td.definition').children.first.content.gsub(/[\.,]/, '') | |
result[:link] = car_url | |
results << result | |
end | |
file_name = "#{Time.now.to_s.match(/(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2})/).captures.join('_')}.csv" | |
puts "\nGenerating CSV #{file_name}..." | |
File.open(file_name, 'w') do |csv| | |
csv.write "#{[results.first.keys.map(&:to_s)].join(',')}\n" | |
results.each { |result| csv.write("#{result.values.join(',')}\n") } | |
end | |
puts 'Done!' |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Result => https://gist.github.com/lucascaton/d7fa30406d6316ad5baa