dynamodb_scan.rb

TABLE_NAME = 'devices'.freeze
SCAN_WORKERS = 1 # 1 ~ 32


Aws.config.update({
  region: 'region',
  credentials: Aws::Credentials.new('...', '...')
})

client = Aws::DynamoDB::Client.new

# parallel scan
Benchmark.bm 10 do |r|
  r.report 'multi process' do
    Parallel.each((0...SCAN_WORKERS).to_a, in_processes: SCAN_WORKERS) do |var|
      exclusive_start_key = nil

      loop do
        res = client.scan(
          table_name: TABLE_NAME,
          total_segments: SCAN_WORKERS,
          segment: var,
          exclusive_start_key: exclusive_start_key
        )

        exclusive_start_key = res.last_evaluated_key
        item = res.items.first
        p "var:#{var} count:#{res.count} user_id:#{item['user_id']} exclusive_start_key:#{exclusive_start_key}"
        sleep 1
        break if res.last_evaluated_key.blank?
      end
    end
  end

  sleep 2

  r.report 'multi thread' do
    Parallel.each((0...SCAN_WORKERS).to_a, in_threads: SCAN_WORKERS) do |var|
      exclusive_start_key = nil

      loop do
        res = client.scan(
          table_name: TABLE_NAME,
          total_segments: SCAN_WORKERS,
          segment: var,
          exclusive_start_key: exclusive_start_key
        )

        exclusive_start_key = res.last_evaluated_key
        item = res.items.first
        p "var:#{var} count:#{res.count} user_id:#{item['user_id']} exclusive_start_key:#{exclusive_start_key}"
        sleep 1
        break if res.last_evaluated_key.blank?
      end
    end
  end
end


# series scan
Benchmark.bm 10 do |r|
  r.report 'series_scan' do
    exclusive_start_key = nil

    loop do
      options = options.merge(exclusive_start_key: exclusive_start_key)
      res = client.scan(options)

      res = cli.scan(table_name: 'devices_development', exclusive_start_key: exclusive_start_key)
      puts "res.count:#{res.count} exclusive_start_key:#{exclusive_start_key}"
      sleep 1
      break if res.last_evaluated_key.blank?
      exclusive_start_key = res.last_evaluated_key
    end
  end
end


# 結果
#
# ワーカー数 4              user     system      total        real
# multi process          0.000000   0.010000   1.960000 (  2.222439)
# multi thread           0.790000   0.120000   0.910000 (  8.817508)
# 
# ワーカー数 8              user     system      total        real
# multi process          0.010000   0.010000   4.700000 (  5.428759)
# multi thread           0.880000   0.130000   1.010000 (  2.382630)
# 
# ワーカー数 16              user     system      total        real
# multi process          0.010000   0.020000   5.770000 ( 16.346010)
# multi thread           1.000000   0.130000   1.130000 (  2.196733)
# 
# ワーカー数 32              user     system      total        real
# multi process          0.010000   0.050000  13.950000 ( 29.365831)
# multi thread           0.950000   0.180000   1.130000 (  4.588469)
# 
# 参考) 直列スキャン
#                            user     system      total        real
# series_scan            1.010000   0.090000   1.100000 (  6.624715)