kivagrabber.rb

#!/usr/bin/env ruby
require 'rubygems'
require 'httparty'
require 'json'
require 'fileutils'
require 'pp'

module SaveUrl
  UA = 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; ja-JP-mac; rv:1.9.2.8) Gecko/20100722 Firefox/3.6.8'
  DIR = './download'

  def save(url, dir=DIR)
    FileUtils.mkdir dir unless File.exist? dir
    filename = url.split(/.*\/(.*?\..*?)$/)[-1]
    p filename
    `curl -A '#{UA}' -L -o '#{dir}/#{filename}' #{url}`
  end
end

class KivaGrabber
  class Parser < HTTParty::Parser
    include SaveUrl
    def parse
      JSON.parse(body)['progressData'].each {|k,v| v['image_id']}.map do |k,v|
        url = "http://www.kiva.org/img/w800/#{k}.jpg"
        save url
        url
      end
    end
  end

  include HTTParty
  parser Parser

  def initialize()
    @default_params = {
      'pageID' => 1,
      'perPage' => 20,
      'borrower_type' => 'All',
      'countries[]' => 'All',
      'gender' => 'All',
      'partner_id' => '',
      'queryString' => '',
      'regions[]' => 'All',
      'sectors[]' => 'All',
      'sortBy' => 'Popularity',
      'status' => 'fundRaising',
    }
  end

  def grab(params)
    params = @default_params.merge(params)
    res = self.class.post('http://www.kiva.org/ajax/queryBusinessesListing', :query => params)
  end
end

kg = KivaGrabber.new
(1..10).map do |i|
  puts "page #{i} ----"
  pp kg.grab('pageID' => i)
  sleep 1
end