01_redis_set_results.md

Comparison of setting a key to Redis

note: Redis is running locally with a maximum of 10K connections

 set LZ4 + MessagePack:    401.7 i/s
           set Marshal:    232.3 i/s - 1.73x  slower
     set LZ4 + Marshal:    176.4 i/s - 2.28x  slower
              set to_s:    141.5 i/s - 2.84x  slower
  set Snappy + Marshal:     92.6 i/s - 4.34x  slower
set zlib + MessagePack:     51.9 i/s - 7.73x  slower
    set zlib + Marshal:     41.4 i/s - 9.69x  slower

Compressed sizes of the object

note: everything is set to their defaults

Hash.to_s size 680000
  Marshal size 740007
     zlib size 380620
   Snappy size 627216
      LZ4 size 634926

02_bench.rb

require 'redis'
require 'zlib'
require 'lz4-ruby'
require 'msgpack'
require 'snappy'
require 'benchmark/ips'
require 'json'
require 'securerandom'
require 'active_support'

redis_pool = ConnectionPool.new(size: 1000, timeout: 5) do
  Redis.new(host: 'localhost', port: 6379, db: 15)
end


# Generate a large Hash with high entropy
large_hash = {}

10000.times do
  key = SecureRandom.hex(10)
  value = SecureRandom.hex(20)
  large_hash[key] = value
end

serialized_value        = Marshal.dump(large_hash)
msgpack_value           = MessagePack.pack(large_hash)
compressed_value_zlib   = Zlib::Deflate.deflate(serialized_value)
compressed_value_lz4    = LZ4.compress(serialized_value)
compressed_value_snappy = Snappy.compress(serialized_value)

puts "------------------------------------------------------------------------------------"

Benchmark.ips do |x|
  x.report('.to_s') do
    large_hash.to_s
  end

  x.report('MessagePack') do
    MessagePack.pack(large_hash)
  end

  x.report('Marshal') do
    Marshal.dump(large_hash)
  end

  x.report('zlib') do
    Zlib::Deflate.deflate(serialized_value)
  end

  x.report('Snappy') do
    Snappy.deflate(serialized_value)
  end

  x.report('LZ4') do
    LZ4.compress(serialized_value)
  end

  x.compare!
end

puts "------------------------------------------------------------------------------------"

puts "Hash.to_s size #{large_hash.to_s.size}"
puts "  Marshal size #{serialized_value.size}"
puts "     zlib size #{compressed_value_zlib.size}"
puts "   Snappy size #{compressed_value_snappy.size}"
puts "      LZ4 size #{compressed_value_lz4.size}"

Benchmark.ips do |x|
  x.report('set.to_s') do
    redis_pool.with do |conn|
      conn.set('sample_key', large_hash.to_s)
    end
  end

  x.report('set.marshal') do
    redis_pool.with do |conn|
      conn.set('sample_key', Marshal.dump(large_hash))
    end
  end

  x.report('set.zlib') do
    redis_pool.with do |conn|
      conn.set('sample_key', Zlib::Deflate.deflate(Marshal.dump(large_hash)))
    end
  end

  x.report('set.zlib + MessagePack') do
    redis_pool.with do |conn|
      conn.set('sample_key', Zlib::Deflate.deflate(MessagePack.pack(large_hash)))
    end
  end

  x.report('set.Snappy') do
    redis_pool.with do |conn|
      conn.set('sample_key', Snappy.deflate(Marshal.dump(large_hash)))
    end
  end

  x.report('set.LZ4') do
    redis_pool.with do |conn|
      conn.set('sample_key', LZ4.compress(Marshal.dump(large_hash)))
    end
  end

  x.report('set.LZ4 + MessagePack') do
    redis_pool.with do |conn|
      conn.set('sample_key', LZ4.compress(MessagePack.pack(large_hash)))
    end
  end

  x.compare!
end

# Clean up
redis_pool.with do |conn|
  conn.del('sample_key')
end

How to get the maximum number of connections from your local Redis instance:

> redis-cli
127.0.0.1:6379> CONFIG GET maxclients
1) "maxclients"
2) "10000"