-
-
Save ramieblatt/361658 to your computer and use it in GitHub Desktop.
Fixed a few issues I came across and added a Memcached (from the gem) wrapper class for the AbstractStore
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# = Icebox : Caching for HTTParty | |
# | |
# Cache responses in HTTParty models [http://github.com/jnunemaker/httparty] | |
# | |
# === Usage | |
# | |
# class Foo | |
# include HTTParty | |
# include HTTParty::Icebox | |
# cache :store => 'file', :timeout => 600, :location => MY_APP_ROOT.join('tmp', 'cache') | |
# end | |
# | |
# Modeled after Martyn Loughran's APICache [http://github.com/newbamboo/api_cache] | |
# and Ruby On Rails's caching [http://api.rubyonrails.org/classes/ActiveSupport/Cache.html] | |
# | |
# Author: Karel Minarik [www.karmi.cz] | |
# | |
# === Notes | |
# | |
# Thanks to Amit Chakradeo for pointing out response objects have to be stored marhalled on FS | |
# Thanks to Marlin Forbes for pointing out the query parameters have to be included in the cache key | |
# | |
# | |
require 'logger' | |
require 'ftools' | |
require 'tmpdir' | |
require 'pathname' | |
require 'digest/md5' | |
module HTTParty #:nodoc: | |
# == Caching for HTTParty | |
# See documentation in HTTParty::Icebox::ClassMethods.cache | |
# | |
module Icebox | |
module ClassMethods | |
# Enable caching and set cache options | |
# Returns memoized cache object | |
# | |
# Following options are available, default values are in []: | |
# | |
# +store+:: Storage mechanism for cached data (memory, filesystem, your own) [memory] | |
# +timeout+:: Cache expiration in seconds [60] | |
# +logger+:: Path to logfile or logger instance [nil, silent] | |
# | |
# Any additional options are passed to the Cache constructor | |
# | |
# Usage: | |
# | |
# # Enable caching in HTTParty, in memory, for 1 minute | |
# cache # Use default values | |
# | |
# # Enable caching in HTTParty, on filesystem (/tmp), for 10 minutes | |
# cache :store => 'file', :timeout => 600, :location => '/tmp/' | |
# | |
# # Use your own cache store (see +AbstractStore+ class below) | |
# cache :store => 'memcached', :timeout => 600, :server => '192.168.1.1:1001' | |
# | |
def cache(options={}) | |
options[:store] ||= 'memory' | |
options[:timeout] ||= 60 | |
logger = options[:logger] | |
@cache ||= Cache.new( options.delete(:store), options ) | |
end | |
end | |
# When included, extend class with +cache+ method | |
# and redefine +get+ method to use cache | |
# | |
def self.included(receiver) #:nodoc: | |
receiver.extend ClassMethods | |
receiver.class_eval do | |
# Get reponse from network | |
# | |
# TODO: Why alias :new :old is not working here? Returns NoMethodError | |
# | |
def self.get_without_caching(path, options={}) | |
perform_request Net::HTTP::Get, path, options | |
end | |
# Get response from cache, if available | |
# | |
def self.get_with_caching(path, options={}) | |
key = path.downcase # this makes a copy of path | |
key << options[:query].to_s if defined? options[:query] | |
if cache.exists?(key) and not cache.stale?(key) | |
Cache.logger.debug "CACHE -- GET #{path}#{options[:query]}" | |
return cache.get(key) | |
else | |
Cache.logger.debug "/!\\ NETWORK -- GET #{path}#{options[:query]}" | |
response = get_without_caching(path, options) | |
cache.set(key, response) if response.code.to_s == "200" # this works for string and integer response codes | |
return response | |
end | |
end | |
# Redefine original HTTParty +get+ method to use cache | |
# | |
def self.get(path, options={}) | |
self.get_with_caching(path, options) | |
end | |
end | |
end | |
# === Cache container | |
# | |
# Pass a store name ('memory', etc) to new | |
# | |
class Cache | |
attr_accessor :store | |
def initialize(store, options={}) | |
self.class.logger = options[:logger] | |
@store = self.class.lookup_store(store).new(options) | |
end | |
def get(key); @store.get encode(key) unless stale?(key); end | |
def set(key, value);puts "Cache.set, key: #{key}, value: #{value}"; @store.set encode(key), value; end | |
def exists?(key); @store.exists? encode(key); end | |
def stale?(key); @store.stale? encode(key); end | |
def self.logger; @logger || default_logger; end | |
def self.default_logger; logger = ::Logger.new(STDERR); end | |
# Pass a filename (String), IO object, Logger instance or +nil+ to silence the logger | |
def self.logger=(device); @logger = device.kind_of?(::Logger) ? device : ::Logger.new(device); end | |
private | |
# Return store class based on passed name | |
def self.lookup_store(name) | |
store_name = "#{name.capitalize}Store" | |
return Store::const_get(store_name) | |
rescue NameError => e | |
raise Store::StoreNotFound, "The cache store '#{store_name}' was not found. Did you load any such class?" | |
end | |
def encode(key); Digest::MD5.hexdigest(key); end | |
end | |
# === Cache stores | |
# | |
module Store | |
class StoreNotFound < StandardError; end #:nodoc: | |
# ==== Abstract Store | |
# Inherit your store from this class | |
# *IMPORTANT*: Do not forget to call +super+ in your +initialize+ method! | |
# | |
class AbstractStore | |
def initialize(options={}) | |
raise ArgumentError, "You need to set the :timeout parameter" unless options[:timeout] | |
@timeout = options[:timeout] | |
message = "Cache: Using #{self.class.to_s.split('::').last}" | |
message << " in location: #{options[:location]}" if options[:location] | |
message << " with timeout #{options[:timeout]} sec" | |
Cache.logger.info message unless options[:logger].nil? | |
return self | |
end | |
%w{set get exists? stale?}.each do |method_name| | |
define_method(method_name) { raise NoMethodError, "Please implement method #{method_name} in your store class" } | |
end | |
end | |
# ==== Store objects in memory | |
# See HTTParty::Icebox::ClassMethods.cache | |
# | |
class MemoryStore < AbstractStore | |
def initialize(options={}) | |
super; @store = {}; self | |
end | |
def set(key, value) | |
Cache.logger.info("Cache: set (#{key})") | |
@store[key] = [Time.now, value]; true | |
end | |
def get(key) | |
data = @store[key][1] | |
Cache.logger.info("Cache: #{data.nil? ? "miss" : "hit"} (#{key})") | |
data | |
end | |
def exists?(key) | |
!@store[key].nil? | |
end | |
def stale?(key) | |
return true unless exists?(key) | |
Time.now - created(key) > @timeout | |
end | |
private | |
def created(key) | |
@store[key][0] | |
end | |
end | |
# ==== Store objects on the filesystem | |
# See HTTParty::Icebox::ClassMethods.cache | |
# | |
class FileStore < AbstractStore | |
def initialize(options={}) | |
super | |
options[:location] ||= Dir::tmpdir | |
@path = Pathname.new( options[:location] ) | |
FileUtils.mkdir_p( @path ) | |
self | |
end | |
def set(key, value) | |
Cache.logger.info("Cache: set (#{key})") | |
File.open( @path.join(key), 'w' ) { |file| file << Marshal.dump(value) } | |
true | |
end | |
def get(key) | |
data = Marshal.load(File.read( @path.join(key))) | |
Cache.logger.info("Cache: #{data.nil? ? "miss" : "hit"} (#{key})") | |
data | |
end | |
def exists?(key) | |
File.exists?( @path.join(key) ) | |
end | |
def stale?(key) | |
return true unless exists?(key) | |
Time.now - created(key) > @timeout | |
end | |
private | |
def created(key) | |
File.mtime( @path.join(key) ) | |
end | |
end | |
end | |
end | |
end | |
# Major parts of this code are based on architecture of ApiCache. | |
# Copyright (c) 2008 Martyn Loughran | |
# | |
# Other parts are inspired by the ActiveSupport::Cache in Ruby On Rails. | |
# Copyright (c) 2005-2009 David Heinemeier Hansson | |
# | |
# Permission is hereby granted, free of charge, to any person obtaining | |
# a copy of this software and associated documentation files (the | |
# "Software"), to deal in the Software without restriction, including | |
# without limitation the rights to use, copy, modify, merge, publish, | |
# distribute, sublicense, and/or sell copies of the Software, and to | |
# permit persons to whom the Software is furnished to do so, subject to | |
# the following conditions: | |
# | |
# The above copyright notice and this permission notice shall be | |
# included in all copies or substantial portions of the Software. | |
# | |
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE | |
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION | |
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION | |
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# | |
# Assumes that MEMCACHE is instantiated, using the gem, use the following lines in environment.rb: | |
# | |
# require 'memcached' | |
# MEMCACHE = Memcached.new("localhost:11211") | |
# | |
# Usage: | |
# | |
# require 'httparty' | |
# require 'httparty_icebox' | |
# require 'httparty_icebox_memcached' | |
# | |
# class Foo | |
# include HTTParty | |
# include HTTParty::Icebox | |
# cache :store => 'memcached', :timeout => 3600*1 # cached 1 hour | |
# end | |
# | |
# | |
require 'httparty_icebox' | |
include HTTParty::Icebox::Store | |
class MemcachedStore < AbstractStore | |
include HTTParty::Icebox | |
def initialize(options={}) | |
super;self | |
end | |
def set(key, value) | |
res = MEMCACHE.set(key, value, @timeout) | |
#puts "MemcachedStore.set, key: #{key}, result: #{res}" | |
Cache.logger.info("Cache: set (#{key})") | |
true | |
end | |
def get(key) | |
data = MEMCACHE.get(key) rescue nil | |
Cache.logger.info("Cache: #{data.nil? ? "miss" : "hit"} (#{key})") | |
data | |
end | |
def exists?(key) | |
data = MEMCACHE.get(key) rescue nil | |
!data.nil? | |
end | |
def stale?(key) | |
return true unless exists?(key) | |
end | |
end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
require 'rubygems' | |
require 'fakeweb' | |
require 'ftools' | |
require 'test/unit' | |
require 'shoulda' | |
require 'httparty' | |
require 'httparty_icebox' | |
FakeWeb.register_uri :get, "http://example.com/", | |
[ {:body => "Hello, World!"}, {:body => "Goodbye, World!"} ] | |
FakeWeb.register_uri :get, "http://example.com/?name=Joshua", | |
[ {:body => "Hello, Joshua!"}, {:body => "Goodbye, Joshua!"} ] | |
FakeWeb.register_uri :get, "http://example.com/teapot", | |
{ :body => "I'm a teapot", :status => 200, :'x-powered-by'=>"Teapot 1.0" } | |
FakeWeb.register_uri :get, "http://example.com/bad", {:body => "Not Found", :status => 404} | |
FakeWeb.register_uri :post, "http://example.com/form", {:body => "Processed", :status => 200} | |
class HTTPartyIceboxTest < Test::Unit::TestCase | |
context "When Icebox is included in a class, it" do | |
setup do | |
MyResource.class_eval do | |
@cache = nil | |
cache :store => 'memory', :timeout => 0.1#, :logger => Logger.new(STDOUT) | |
# cache :store => 'file', :timeout => 0.1, :location => File.dirname(__FILE__), :logger => Logger.new(STDOUT) | |
end | |
end | |
should "allow setting cache" do | |
assert_respond_to MyResource, :cache | |
end | |
should "set the cache" do | |
MyResource.cache :store => 'memory', :timeout => 5, :logger => nil | |
assert_not_nil MyResource.cache | |
assert_not_nil MyResource.cache.store | |
assert_instance_of HTTParty::Icebox::Store::MemoryStore, MyResource.cache.store | |
end | |
should "get the response from network and cache it" do | |
MyResource.get('http://example.com') | |
assert_not_nil MyResource.cache.get('http://example.com') | |
end | |
should "get the cached response when the cache still fresh" do | |
MyResource.get('http://example.com') | |
assert_equal 'Hello, World!', MyResource.get('http://example.com').body | |
end | |
should "get the fresh response when the cache is stale" do | |
MyResource.get('http://example.com') | |
sleep 0.3 | |
assert_equal 'Goodbye, World!', MyResource.get('http://example.com').body | |
end | |
should "include the query params in key" do | |
MyResource.get('http://example.com/?name=Joshua') | |
assert_equal 'Hello, Joshua!', MyResource.get('http://example.com/?name=Joshua').body | |
end | |
should "not cache the response when receiving error from network" do | |
MyResource.get('http://example.com/bad') | |
assert_nil MyResource.cache.get('http://example.com/bad') | |
end | |
should "not cache the response when not a GET request" do | |
MyResource.post('http://example.com/form') | |
assert_nil MyResource.cache.get('http://example.com/form') | |
end | |
should "store reponse with code, body and headers" do | |
MyResource.get('http://example.com/teapot') | |
cached = MyResource.get('http://example.com/teapot') | |
assert_equal 200, cached.code | |
assert_equal "I'm a teapot", cached.body | |
assert_not_nil cached.headers['x-powered-by'] | |
assert_not_nil cached.headers['x-powered-by'].first | |
assert_equal 'Teapot 1.0', cached.headers['x-powered-by'].first | |
end | |
end | |
# --------------------------------------------------------------------------- | |
context "A logger for Icebox" do | |
setup do | |
HTTParty::Icebox::Cache.class_eval { @logger = nil } | |
@logpath = Pathname(__FILE__).dirname.expand_path.join('test.log').to_s | |
end | |
should "return default logger" do | |
assert_instance_of Logger, HTTParty::Icebox::Cache.default_logger | |
end | |
should "return default logger when none was set" do | |
assert_instance_of Logger, HTTParty::Icebox::Cache.logger | |
end | |
should "set be set to nil when given nil" do | |
HTTParty::Icebox::Cache.logger=(nil) | |
assert_instance_of Logger, HTTParty::Icebox::Cache.logger | |
HTTParty::Icebox::Cache.logger.info "Hi" # Should not see this | |
assert_equal nil, # UH :/ | |
HTTParty::Icebox::Cache.logger.instance_variable_get(:@logdev).instance_eval{defined?(@filename)} | |
end | |
should "should set logger to a Logger instance" do | |
HTTParty::Icebox::Cache.logger = ::Logger.new(STDOUT) | |
assert_instance_of Logger, HTTParty::Icebox::Cache.logger | |
end | |
should "create a logger to log to file" do | |
HTTParty::Icebox::Cache.logger = @logpath | |
assert_equal @logpath, # UH :/ | |
HTTParty::Icebox::Cache.logger.instance_variable_get(:@logdev).instance_variable_get(:@filename) | |
FileUtils.rm_rf(@logpath) | |
end | |
end | |
# --------------------------------------------------------------------------- | |
context "When looking up store, it" do | |
should "use default store" do | |
# 'memory' is default | |
assert_equal HTTParty::Icebox::Store::MemoryStore, HTTParty::Icebox::Cache.lookup_store('memory') | |
end | |
should "find existing store" do | |
assert_equal HTTParty::Icebox::Store::FileStore, HTTParty::Icebox::Cache.lookup_store('file') | |
end | |
should "raise when passed non-existing store" do | |
assert_raise(HTTParty::Icebox::Store::StoreNotFound) { HTTParty::Icebox::Cache.lookup_store('ether') } | |
end | |
end | |
# --------------------------------------------------------------------------- | |
context "AbstractStore class" do | |
should "raise unless passed :timeout option" do | |
assert_raise(ArgumentError) { HTTParty::Icebox::Store::AbstractStore.new( :timeout => nil ) } | |
end | |
should "raise when abstract methods are called" do | |
s = HTTParty::Icebox::Store::AbstractStore.new( :timeout => 1, :logger => nil ) | |
assert_raise(NoMethodError) { s.set } | |
end | |
end | |
# --------------------------------------------------------------------------- | |
context "Memory cache" do | |
setup do | |
@cache = HTTParty::Icebox::Cache.new('memory', :timeout => 0.1, :logger => nil) | |
@key = 'abc' | |
@value = { :one => [1, 2, 3], :two => 'Hello', :three => 1...5 } | |
@cache.set @key, @value | |
end | |
should "store and retrieve the value" do | |
assert_equal @value, @cache.get('abc') | |
end | |
should "miss when retrieving the value after timeout" do | |
sleep 0.1 | |
assert_nil @cache.get('abc') | |
end | |
end | |
# --------------------------------------------------------------------------- | |
context "File store cache" do | |
setup do | |
@cache = HTTParty::Icebox::Cache.new('file', :timeout => 1) | |
@key = 'abc' | |
@value = { :one => [1, 2, 3], :two => 'Hello', :three => 1...5 } | |
@cache.set @key, @value | |
end | |
should "store and retrieve the value" do | |
assert_equal @value, @cache.get('abc') | |
end | |
should "miss when retrieving the value after timeout" do | |
sleep 1.1 | |
assert_nil @cache.get('abc') | |
end | |
end | |
# --------------------------------------------------------------------------- | |
end | |
# Fake Resource to mixin the functionality into | |
class MyResource | |
include HTTParty | |
include HTTParty::Icebox | |
cache :store => 'memory', :timeout => 0.2 | |
end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
require 'rubygems' | |
require 'httparty' | |
require 'httparty_icebox' | |
class CachedRepresentative | |
include HTTParty | |
include HTTParty::Icebox | |
cache :store => 'memory', :timeout => 0.5, :logger => Logger.new(STDOUT) | |
end | |
# ----- | |
response1 = CachedRepresentative.get('http://whoismyrepresentative.com/whoismyrep.php?zip=46544') | |
response2 = CachedRepresentative.get('http://whoismyrepresentative.com/whoismyrep.php?zip=46544') | |
puts response1.headers.inspect | |
puts "\nFresh: " << response1.inspect << " [Code #{response1.code}]" | |
puts "\nCached: " << response2.inspect << " [Code #{response2.code}]" | |
puts "\nHeaders work too! => #{response2.headers.inspect}" | |
# ----- | |
puts "\n\nzzzz...\n\n" | |
sleep 1.5 | |
# ----- | |
puts CachedRepresentative.get('http://whoismyrepresentative.com/whoismyrep.php?zip=46544').inspect | |
puts CachedRepresentative.get('http://whoismyrepresentative.com/whoismyrep.php?zip=46544').inspect | |
puts CachedRepresentative.get('http://whoismyrepresentative.com/whoismyrep.php?zip=46544').inspect |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
I fixed a few issues I found and added a Memcached implementation with some comments to guide usage. Performance seems great with memcached gem. This is live on some sites I develop for.
Thanks to karmi for the great code.