Hadoop Streaming Map/Reduce with Ruby

data.txt

Now is definitely the time

mapper.rb

#!/usr/bin/ruby

while line = gets
  words = line.split /\s/
  words.each do |word|
    puts "#{word[0,1]}\t#{word.size}"
  end
end

reducer.rb

#!/usr/bin/ruby

curr_key = nil
curr_total = 0
curr_key_count = 0

while line = gets
  character, length = line.split /\t/
  length = length.to_f

  if curr_key == character
    curr_total += length
    curr_key_count += 1
  else
    unless curr_key.nil?
      avg = curr_total / curr_key_count
      puts "#{curr_key}\t#{avg}"
    end
    curr_key = character
    curr_total = length
    curr_key_count = 1
  end

end

unless curr_key.nil?
  avg = curr_total / curr_key_count
  puts "#{curr_key}\t#{avg}"
end

test.sh

$ cat data.txt | ./mapper.rb | ./reducer.rb

N       3.0
i       2.0
d       10.0
t       3.5