yellow page email scrp

gistfile1.rb

class YellowPagesCrawler < Pioneer::Base

  attr_reader :locations
  def initialize(opts = {})
    @locations_business = opts.fetch(:locations, {})
    @locations = @locations_business.keys
    super
  end

  def processing(req)
    htmldoc = Nokogiri::HTML(req.response_body)

    htmldoc.css('.busCardLeftLinks a').each do |a|
      if a.content.strip == "Email"
        business_details = @locations_business[req.url]
        email = a[:title].strip

        puts "Found email for: #{business_details} (#{email})"

        contact = Contact.new
        contact.province = business_details[0]
        contact.city = business_details[1]
        contact.name = business_details[2]
        contact.country = "Canada"
        contact.email = email
        contact.source = "YP"

        contact.save
      end
    end
  end
end