README

Async URLFetch on appengine-jruby

appengine_apis_ext.rb

module AppEngine
  module Labs
    module TaskQueue
      class Task
        def tx_add(txn, queue=nil)
          queue = Queue.new(queue) unless queue.kind_of? Queue
          @handle = queue.java_queue.add(txn, _task)
          self
        end
      end
    end
  end
  module URLFetch
    def fetch_with_async(url, options={})
      if options && options[:headers] && options[:headers]["X-URLFetch-Async"]
        request = build_urlfetch_request(url, options)
        begin
          future = urlfetch_service.fetch_async(request)
          raise AsyncFuture.new(future)
          ## return convert_urlfetch_body(java_response)
        rescue java.lang.IllegalArgumentException => ex
          raise ArgumentError, ex.message
        rescue java.net.MalformedURLException => ex
          raise InvalidURLError, ex.message
        rescue java.io.IOException => ex
          raise DownloadError, ex.message
        rescue ResponseTooLargeException => ex
          raise ResponseTooLargeError, ex.message
        end
      else
        fetch_without_async(url, options)
      end
    end

    alias :fetch_without_async :fetch
    alias :fetch               :fetch_with_async
    module_function :fetch_without_async, :fetch_with_async, :fetch

    class AsyncFuture < Exception
      attr_reader :future
      def initialize(future)
        @future = future
        super("This is async URLFetch future.")
      end
    end
  end
end

test.rb

require "open-uri"
require 'benchmark'

keywords = %w(
java.util.concurrent.Future
com.google.appengine.api.urlfetch.URLFetchService
rubyopenuri
)

puts Benchmark.measure{
  keywords.each do |kw|
    open("http://www.google.co.jp/search?q=#{kw}"){|f|
      puts "#{f.read.size} kw=#{kw}"
    }
  end
}

def async_http_get(url)
  begin
    open(url, "X-URLFetch-Async"=>"1") 
  rescue AppEngine::URLFetch::AsyncFuture => f
    return f.future
  end
end

puts Benchmark.measure{
  futures = []
  keywords.each do |kw|
    futures << async_http_get("http://www.google.co.jp/search?q=#{kw}")
  end
  futures.zip(keywords){|f,kw|
    puts "#{f.get.getContent.size} kw=#{kw}"
  }
}

=begin
36526 kw=java.util.concurrent.Future
37357 kw=com.google.appengine.api.urlfetch.URLFetchService
25072 kw=rubyopenuri
  0.641000   0.000000   0.641000 (  0.641000)
36634 kw=java.util.concurrent.Future
37357 kw=com.google.appengine.api.urlfetch.URLFetchService
25030 kw=rubyopenuri
  0.241000   0.000000   0.241000 (  0.241000)
=end