Download stock price data from yahoo and insert them into database

stock_daily_price.rb

require 'rubygems'
require 'active_record'
require 'yaml'

dbconfig = YAML::load(File.new(File.join(File.dirname(__FILE__), "../../config/database.yml")))
ActiveRecord::Base.establish_connection(dbconfig["development"])

class StockDailyPrice < ActiveRecord::Base
  def initialize(stock_values, stock_no = "")
    super()
    if stock_values.nil? || stock_values[2].to_s == 'N/A' || stock_values[4].to_s == 'N/A'
      raise ArgumentError
    else
      case stock_values.size
      when 7
        self.stock_no = stock_no
        self.trade_date = stock_values[0]
        self.open = stock_values[1]
        self.close = stock_values[2]
        self.high = stock_values[3]
        self.low = stock_values[4]
        self.volume = stock_values[5]
        self.adjusted = true
      when 9
        self.stock_no = stock_values[0]  
        self.close = stock_values[1]
        self.trade_date = Date.strptime(stock_values[2], "%m/%d/%Y")
        self.low = stock_values[5]
        self.high = stock_values[6]      
        self.open = stock_values[7]
        self.volume = stock_values[8]      
        self.adjusted = false
      else
      end
    end
  end 
end

yahoo_loader_helper.rb

require 'open-uri'
require 'date'
require 'logger'
require 'csv'
require 'yahoo_data_parser'

class YahooLoaderHelper
  def initialize(options)
    @stocks = Array.new
    all_stock_numbers(options).each { |stock_no| 
      @stocks << {
        :stock_no => stock_no,
        :history => "http://ichart.yahoo.com/table.csv?s=#{stock_no}&d=0&e=1&f=2050&g=d&a=0&b=1&c=1980&ignore=.csv",
        :last_day => "http://hk.finance.yahoo.com/d/quotes.csv?s=#{stock_no}&f=sl1d1t1c1ohgv&e=.csv"
      }
    }
  end

  def retrieve_all
    stock_prices = nil
    @stocks.each { |stock|
      stock_no = stock[:stock_no]

      last_day_stock_price = YahooParser.download_last_day_snapshot(stock)
      if last_day_stock_price && last_day_stock_price.valid?
        logger.debug "downloading #{stock_no}"
        
        stock_prices = YahooParser.download_history_stock(last_day_stock_price, stock)
      end
    }
    stock_prices
  end

  def retrieve_incremental
    @stocks.each { |stock|
      stock_no = stock[:stock_no]

      if false
        download_missing_stock_prices
      end
    }
  end

  protected

  def all_stock_numbers(options)
    stock_nos = (options[:from]..options[:to]).to_a.map {|num| "#{('%04d' % num).to_s}.HK"}
    stock_nos << "%5EHSI"
  end

  def logger
    @logger ||= Logger.new(File.join(File.dirname(__FILE__), "../../log/data_loader.log"),3,5*1024*1024)
  end
end

yahoo_parser.rb

require 'csv'
require 'stock_daily_price'

class YahooParser
  def self.download_last_day_snapshot(stock)
    last_day_value = nil
    open_with_error_handled(stock[:last_day])  { |f|
      last_day_value = StockDailyPrice.new(CSV.parse_line(f.readline))
    }
    last_day_value
  end

  def self.download_history_stock(last_day_stock_price, stock)
    stock_prices = Array.new
    open_with_error_handled(stock[:history]) { |f|
      skip_header(f)
      
      stock_no = last_day_stock_price.stock_no
      first_history_stock_price = StockDailyPrice.new(CSV.parse_line(f.readline), stock_no)
      if !history_include?(first_history_stock_price, last_day_stock_price)
        stock_prices << last_day_stock_price
      end
      stock_prices << first_history_stock_price
      f.readlines.each { |line|
        stock_prices << StockDailyPrice.new(CSV.parse_line(line), stock_no)
      }
      # TODO convert to use ar-extension
      #require 'ar-extensions'
      #class User
      #end
      #Then to use it, it really couldn't be more straightforward. Use the following in your controller or rake task to import your data:

      #fields = [:first_name, :last_name, :email]
      #data = [["glenn", "gillen", "[email protected]"],
      #        ["john", "jones", "[email protected]"],
      #        ["steve", "smith", "[email protected]"]]

      #User.import fields, data
    }
    stock_prices
  end
  
  protected
  
  def self.history_include?(history_stock_price, last_day_stock_price)
    true
  end

  def self.open_with_error_handled(url)
    begin
      open(url) { |f|
        yield f
      }
    rescue OpenURI::HTTPError
      error_logger.error "http connection error: #{url}"          
    end
  end
  
  def self.skip_header(f)
    f.readline
  end 
  
  def self.error_logger
    @logger ||= Logger.new(File.join(File.dirname(__FILE__), "../../log/data_loader_error.log"),3,5*1024*1024)
  end  
end