Calculate kafka topics sizes

kafka_topics.sh

#!/usr/bin/ruby

require 'net/ssh'
require 'pp'

def fetch(host)
  data = []
  Net::SSH.start(host, 'virool') do |ssh|
    index_folders = ssh.exec!("ls -d /var/lib/kafka/*").split("\n")
    data = index_folders.flat_map do |ifolder|
       topics = ssh.exec!("ls -d #{ifolder}/*").split("\n")
       calculate_size(topics, ssh)
    end
  end
end

def calculate_size(topics, ssh)
  topics.map do |topic|
    size_in_bytes, _ = ssh.exec!("du -cs #{topic}/*.log | tail -1").split("\t")
    t = File.basename(topic).gsub(/-\d/, '').gsub(/\d/, '')
    p "#{topic} with #{size_in_bytes.to_i/1024} MB"
    [t, size_in_bytes.to_i/1024]
  end
end

def calculate_topics_size(nodes)
  Hash.new(0).tap do |result|
    nodes.each do |host|
      fetch(host).each do |k,v|
        result[k] += v
      end
    end
  end
end

def humanize(data)
  data.each do |k,v|
    data[k] = "#{v/1024} GB"
  end
end

pp humanize calculate_topics_size(['kafka01'])