pager.rb

require "rubygems"
require "nokogiri"
require "httpclient"
require "uri"
require "json"

class Pager
  @@siteinfo_url = "http://wedata.net/databases/AutoPagerize/items.json"

  attr_accessor :doc

  def initialize(url)
    @url = url
    @client = HTTPClient.new
    @doc = Nokogiri::HTML(@client.get_content(@url))

    @siteinfo = JSON.parse(@client.get_content(@@siteinfo_url))

    @next_link_xpath
    @siteinfo.each do |item|
      regexp = Regexp.new(item["data"]["url"])
      if regexp =~ @url
        puts item["name"]
        @next_link_xpath = item["data"]["nextLink"]
        return @next_link_xpath
      end
    end

    return nil
  end

  def next
    next_link = @doc.xpath(@next_link_xpath).first["href"]
    next_link = URI.split(@url)[0] + "://" + URI.split(@url)[2] + next_link unless /^http/ =~ next_link
    puts next_link
    @doc = Nokogiri::HTML(@client.get_content(next_link))
  end
end