bench.rb

# Trying to optimize the gemcutter indexing process.
# Since we're dumping out the index ~150 times daily, this is going to help the server load/cpu out immensely.

require 'config/environment'
require 'benchmark'

redis = Redis.new

def pack(value)
  final = StringIO.new
  gzip = Zlib::GzipWriter.new(final)
  gzip.write(Marshal.dump(value))
  gzip.close
  final.string
end

Benchmark.bmbm do |bm|
  bm.report('with_indexed') do
    # this is how the index works now
    value = Version.with_indexed(true).map(&:to_index)
    pack(value)
  end
  bm.report('all') do
    # trying to optimize the above query
    versions = Version.all(:select => "number, platform, position, rubygem_id, indexed, rubygems.name",
                           :joins => :rubygem,
                           :order => "rubygems.name asc, position desc",
                           :conditions => {:indexed => true});
    indexed = versions.map { |v| [ v['name'], v.to_gem_version, v.platform] }
    pack(indexed)
  end
  bm.report('redis with marshalled entries') do
    # index:all is a set of all Versions's full names... so something like: [rails-2.3.5, rack-1.0.0...]
    members = redis.smembers('index:all').map { |member| "marshal-#{member}" }

    # at each key is a Marshalled version of the index entry ( [rubygem name, Gem::Version, platform] )
    marshals = redis.mget(members).map { |m| Marshal.load(m) }
    pack(marshals.join)
  end
  bm.report('redis without marshalled entries') do
    members = redis.smembers('index:all')
    index = members.map do |member|
      list = redis.lrange "list:#{member}", 0, -1
      list[1] = Gem::Version.new(list[1])
      list
    end
    pack(index)
  end
end

development.log

## from Version.with_indexed

  Version Load Including Associations (2695.5ms)   SELECT "versions"."id" AS t0_r0, "versions"."authors" AS t0_r1, "versions"."description" AS t0_r2, "versions"."number" AS t0_r3, "versions"."rubygem_id" AS t0_r4, "versions"."built_at" AS t0_r5, "versions"."updated_at" AS t0_r6, "versions"."rubyforge_project" AS t0_r7, "versions"."summary" AS t0_r8, "versions"."platform" AS t0_r9, "versions"."created_at" AS t0_r10, "versions"."indexed" AS t0_r11, "versions"."prerelease" AS t0_r12, "versions"."position" AS t0_r13, "versions"."downloads_count" AS t0_r14, "versions"."latest" AS t0_r15, "versions"."full_name" AS t0_r16, "rubygems"."id" AS t1_r0, "rubygems"."name" AS t1_r1, "rubygems"."created_at" AS t1_r2, "rubygems"."updated_at" AS t1_r3, "rubygems"."downloads" AS t1_r4, "rubygems"."slug" AS t1_r5 FROM "versions" LEFT OUTER JOIN "rubygems" ON "rubygems".id = "versions".rubygem_id WHERE ("versions"."indexed" = 't') ORDER BY position

## from trying to optimize it

  Version Load (696.6ms)   SELECT number, platform, position, rubygem_id, indexed, rubygems.name FROM "versions" INNER JOIN "rubygems" ON "rubygems".id = "versions".rubygem_id WHERE ("versions"."indexed" = 't') ORDER BY rubygems.name asc, position desc, position

output

gemcutter master % ruby bench.rb
** [NewRelic] Cannot find newrelic.yml file at /Users/qrush/Dev/rails/gemcutter/config/newrelic.yml.
Rehearsal --------------------------------------------------------------------
with_indexed                      21.130000   0.400000  21.530000 ( 24.162104)
all                                8.740000   0.060000   8.800000 (  9.483408)
redis with marshalled entries      5.090000   0.100000   5.190000 (  6.683802)
redis without marshalled entries  11.160000   2.020000  13.180000 ( 17.084345)
---------------------------------------------------------- total: 48.700000sec

                                       user     system      total        real
with_indexed                      20.930000   0.150000  21.080000 ( 23.606389)
all                                8.240000   0.050000   8.290000 (  9.007170)
redis with marshalled entries      4.770000   0.080000   4.850000 (  6.328967)
redis without marshalled entries  10.930000   2.020000  12.950000 ( 16.818897)

populate_redis.rb

r = Redis.new
v = Version.with_indexed(true); nil
v.each { |onev| r["marshal-#{onev.full_name}"] = Marshal.dump(onev.to_index) }; nil
v.each { |onev| r.sadd 'index:all', onev.full_name }; nil
v.each { |onev| key = "list:#{onev.full_name}"; r.rpush key, onev.rubygem.name; r.rpush key, onev.number; r.rpush key, onev.platform }; nil