Created
August 16, 2013 10:28
-
-
Save fred/6248817 to your computer and use it in GitHub Desktop.
ruby script to clone/copy Iron IO cache (using celluloid pool)
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
require 'celluloid' | |
class IronQueueWorker | |
include Celluloid | |
def initialize | |
@token = "your-token" | |
@old_project_id = "source-project-id" | |
@new_project_id = "source-project-id" | |
@old_iron_cache = IronCache::Client.new(token: @token, project_id: @old_project_id) | |
@new_iron_cache = IronCache::Client.new(token: @token, project_id: @new_project_id) | |
@old_cache = @old_iron_cache.cache("links_prakard") | |
@new_cache = @new_iron_cache.cache("links_prakard") | |
end | |
def get(url) | |
@old_cache.get CGI::escape(url) | |
end | |
def put(url, data) | |
@new_cache.put(CGI::escape(url), data.to_json) | |
end | |
def copy(url) | |
item = @old_cache.get CGI::escape(url) | |
if item && item.value | |
@new_cache.put(CGI::escape(url), item.value.to_json) | |
end | |
end | |
end | |
# Run 100 parallel threads to clone the cache. | |
# if server is slow, run 20-40 | |
pool = IronQueueWorker.pool(size: 100) | |
# I use URLs as the cache key for the Web crawler | |
Site.each do |site| | |
if url = site.url | |
pool.copy(url) | |
end | |
end;nil | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment