rets api scrapper

require 'nokogiri'
require 'zlib'
require 'open-uri'
require 'net/http'
require 'net/https'

=begin
To get up and running on local:
git clone [email protected]:mphteam/list-hub.git
change directories to list-hub (cd list-hub)
bundle install (This will pull down and install required Ruby Gems)
Database configuration: 'rake db:migrate'
Run Test suite: 'rake test' (WIP)
View available rake tasks: 'rake -T | grep bdx_list_feed'
Run a task: rake <namespace><task> EX: 'rake bdx_list_feed:pull_data'

=end

namespace :bdx_list_feed do

  desc "Retrieve listhub xml data from api"
    task pull_data: :environment do

      CONN = ActiveRecord::Base.connection

      puts 'starting automated rake task...'

      puts "\n#{Time.now}"
=begin
      # make our connection and authenticate into the listhub api:
      easy = Curl::Easy.new("https://feeds.listhub.com/pickup/marketplace/marketplace.xml.gz")
      easy.follow_location = true
      easy.http_auth_types = :basic
      easy.username = 'some_user_name'
      easy.password = 'some_password'

      # snatch and open gzipped data
      File.open("marketplace.xml.gz", 'wb') do|f|
        easy.on_progress {|dl_total, dl_now, ul_total, ul_now| print "="; true }
        easy.on_body {|data| f << data; data.size }
        easy.perform
      end

      File.open('marketplace.xml', 'w') do |file|
        file.write(Zlib::GzipReader.new(File.open("marketplace.xml.gz")).read)
      end
=end
    # xml_file = File.open("marketplace.xml")
      xml_file = File.open("sample_xml.xml")

      doc = Nokogiri::XML(xml_file)

      doc.remove_namespaces!

      # write data.
      doc.css("Listing").each do |listing|

        street = listing.at("Address FullStreetAddress") ? listing.at("Address FullStreetAddress").text : 'street'
        city = listing.at("Address City") ? listing.at("Address City").text : 'city'
        state = listing.at("Address StateOrProvince") ? listing.at("Address StateOrProvince").text : 'state'
        zip = listing.at("Address PostalCode") ? listing.at("Address PostalCode").text : 0
        info = listing.at("ListingDescription") ? listing.at("ListingDescription").text : ""

        price = listing.at("ListPrice") ? listing.at("ListPrice").text : 0
        bedrooms = listing.at("Bedrooms") ? listing.at("Bedrooms").text : 0
        bathrooms = listing.at("Bathrooms") ? listing.at("Bathrooms").text : 0

        lat = listing.at("Location Latitude") ? listing.at("Location Latitude").text : 0
        long = listing.at("Location Longitude") ? listing.at("Location Longitude").text : 0

        name = listing.at("Location Community Subdivision") ? listing.at("Location Community Subdivision").text : 'Unknown'

        builder_name = listing.at("Builder Name") ? listing.at("Builder Name").text : "Builder"
        
        builder_address = listing.at("Builder Address")
        
        builder_address.children.each do |b|
          if b.name == "commons:StateOrProvince"
            state = b.text
            puts state
          end
        end

        # STATES / BUILDERS
        MPH_Homes = { 
                      "GA" =>         [ "DR Horton", "Acadia", "Piedmont Residential", "Brock Built", "Ryland",
                                            "Knight Homes", "Kerely Family Homes", "Crown Communities", "Paran Homes",
                                            "Blue Ribbon Homes", "Rockland", "Edward Andrews", "Traton Homes", "Ashton Woods",
                                            "O'Dwyer Homes", "Legendary/Meritage", "KM Homes", "Stonecrest", "Jim Chapman (Active Adult)", "Sharp Residential" ], 

                      "MI" =>       [ "Timothy Patrick", "Lombardo", "Hunter Pasteur", "Pulte", "Pinnacle", "Palazzolo Brothers", "Soave Homes",
                                            "Infinity Homes", "Newmark", "Allen Edwin", "Evergreen Homes", "Guenther Homes", "Windmill", "Echelon", "Norfolk" ],

                      "SC" => [ "DR Horton", "Crown Communities", "Blue Ribbon Homes" ],

                      "FL" =>        [ "DR Horton", "Dream Finders Homes", "Drees Homes" ],

                      "OH" =>           [ "Brookestone Builders", "Maronda Homes", "Drees Homes", "Cristo Homes", "Potterhill Homes", "New Generation Homes",
                                            "Schumacher Homes", "Ball Homes", "Maplecraft Custom Homes", "TK Constructors", "Wayne Homes", "Rockford Homes", "Schottenstein Homes",
                                            "P & D Custom Homes", "New Generation Homes", "Ashford Homes", "Maronda Homes", "Westport Homes" ],

                      "MO" =>       [ "McBride and Sons" ],

                      "IN" =>        [ "Ryan Homes", "Ryland Homes", "Olthof Homes", "Drees Homes", "West Pointe Homes" ],

                      "NV" =>         [ "DR Horton" ],

                      "TN" =>      [ "DR Horton", "Drees Homes" ]
                    }

        MPH_Homes.each do |key, value|
          puts key
          value.each do |k,v|
            puts k
            puts v
          end
        end

        if MPH_Homes["OH"].include? "Ball Homes"
          puts "\n\n\n\n\n\nYES\n\n\n\n\n\n"
        else
          puts "\n\n\n\n\n\nNO\n\n\n\n\n\n"
        end

        community = Community.where('name = ? AND zip = ?', name, zip).first

        if community
          community_id = community.id
        else
          new_community = Community.create(
            :street_address => street,
            :city => city,
            :state => state,
            :zip => zip,
            :max_price => 0,
            :max_square_ft => 0,
            :min_price => 0,
            :min_square_ft => 0,
            :name => name,
            :info => info,
            :lat => lat,
            :lng => long)
          community_id = new_community.id
        end

        sql  = "INSERT INTO home (`bathroom_count`, `bedroom_count`, `community_id`, `garage_count`, `model_name`, `price`, `quick_move_in`, `square_ft`, `street_address`, `city`, `state`, `zip`) VALUES (#{bathrooms}, #{bedrooms}, #{community_id}, 0, '', #{price}, 0, 0, #{ActiveRecord::Base.connection.quote(street)}, '#{city}', '#{state}', #{zip})"
        CONN.execute sql
      end

      xml_file.close
      puts 'task completed'
      puts "\n#{Time.now}"
  end
end