alekseyl

alekseyl

alekseyl

class CommentsController < ApplicationController
  # I assumed the usual naming for models and foreign columns 
  def users_comments
    # I don't like where( post_id: Post.all ) but if table comments 
    # contains deleted posts comments or even NULLs than we need to keep it this way 
    @user_comments = Comment.where( post_id: Post.all )
                            .where(author_id: Author.where( username: params[:username]) )

  end
end

alekseyl

Redis.include( MeRedis )
# zip key crumbs user, card, card_preview, zips integer crumbs to base62, 
# for keys starting with card_preview prefix compress values with Zlib 
# for keys starting with user or card compress values with ActiveRecordJSONCompressor
Redis.configure( 
  hash_max_ziplist_entries: 256,
  zip_crumbs: %i[user card card_preview], # -> { user: :u, card: :c, card_preview: :c0 }
  integers_to_base62: true,
  compress_namespaces: {
      :card_preview => MeRedis::ZipValues::ZlibCompressor,

alekseyl

module RSmazCompressor
  def self.compress(value); RSmaz.compress(value) end
  def self.decompress(value); RSmaz.decompress(value) end
end

module ZLibJSONCompressor
  def self.compress(value); Zlib.deflate(value.to_json) end
  def self.decompress(value); JSON.load( Zlib.inflate( value ) )end
end

alekseyl

 def calc_me_redis_perfomance_hash_zipping( zip_max = 128, sample_size = 10000 )
    Redis.include(MeRedis)
    redis = Redis.new

    Redis.configure do |c|
      c.zip_crumbs = :user
      c.integers_to_base62 = true
      c.hash_max_ziplist_entries = zip_max
    end
    

alekseyl

# Columns: 
# redis - clear memory usage of 100K values in Mb, 
# gz - gz compressed value, smaz = smaz compressed value, 
# key zip = zipping key crumbs to cingle char, 
# me_ - memory_efficient_ hash optimization

#integers, html pieces, plain english text pieces 
+----------+--------+--------+--------+---------+--------+----------+------------+
| type     | redis  | gz     | smaz   | key zip | me_*   | kz+me+gz | kz+me+smaz |
+----------+--------+--------+--------+---------+--------+----------+------------+

alekseyl

def redis_memory_test
    results = {}
    
    redis = Redis.new
    redis.flushdb
    results[:clear] = redis.info(:memory)['used_memory'].to_i
    
    (2..128).each do |i|
      redis.pipelined do
        10000.times { redis.incr( SecureRandom.urlsafe_base64( i  ) ) }

alekseyl

#  def to_base62
#    Base62.encode(to_s)
#  end

2.4.1 :001 > redis = Redis.new
 => #<Redis client v4.0.1 for redis://127.0.0.1:6379/0> 
2.4.1 :002 > redis.flushdb
2.4.1 :003 > redis.info(:memory)['used_memory_human']
 => "670.55K" 
2.4.1 :004 > redis.pipelined { 100000.times{|i| redis.incr( "user:#{i}" ) }; }; :done

alekseyl

+----------------+-----------------+-----------------+-----------------+---------------+---------------+-----------------+----------------+----------------+
| N/cards/M      | order by random | random() < 0.xx | generate_series | M partials    | ruby way      | Unfold          | ORM random(rw) | ORM random(mp) |
| 10 / 10 / 5    | 0.32ms / 4.27   | 0.26ms / 3.55   | 0.25ms / 3.44   | 0.31ms / 4.13 | 0.28ms / 3.82 | 0.07ms / 1.0    | 0.17ms / 2.3   | 0.19ms / 2.55  |
| 100 / 10 / 5   | 0.66ms / 9.69   | 0.47ms / 6.9    | 0.26ms / 3.85   | 0.29ms / 4.22 | 0.31ms / 4.59 | 0.07ms / 1.0    | 0.2ms / 3.01   | 0.19ms / 2.74  |
| 1000 / 10 / 5  | 4.39ms / 10.09  | 2.98ms / 6.86   | 0.51ms / 1.16   | 0.57ms / 1.32 | 0.66ms / 1.51 | 2.89ms / 6.63   | 0.55ms / 1.26  | 0.44ms / 1.0   |
| 10000 / 10 / 5 | 43.06ms / 22.11 | 26.48ms / 13.6  | 1.79ms / 0.92   | 2.17ms / 1.11 | 3.18ms / 1.63 | 22.89ms / 11.75 | 3.01ms / 1.55  | 1.95ms / 1.0   |
+----------------+-----------------+-----------------+-----------------+----------