concat_csv.rb

#!/usr/bin/env ruby
require 'kconv'
if ARGV.size < 2
  puts <<-HELP
HELP:
  ruby concat_csv.rb [number of header rows] [list of files with identical headers]
    e.g.
    ruby concat_csv.rb 2 census2005_miyagi_age/*C*.txt
HELP
  exit
end

header_size = (ARGV.first =~ /\A\d+\z/) ? ARGV.shift.to_i : 1

output_handle = "concat.csv"
inc = 0
output_handle = "concat#{".#{inc += 1}"}.csv" while File.exists?(output_handle)

puts "writing to #{output_handle}"

File.open(output_handle, "w") do |output|
  first_header = nil
  ARGV.each do |path|
    File.open(path) do |handle|
      header = [handle.first(header_size)].flatten # flattening, as Enumerable.first returns either a string or array :|
      first_header ||= header

      (raise RuntimeError, "ABORTING: #{handle.path} does not have the same header as the previous files") unless first_header == header

      first_header.each{ |row| output << Kconv.toutf8(row) }
      output << Kconv.toutf8(handle.read)
    end
  end
end

to_utf8.kconv.rb

require 'Kconv'
if ARGV.size == 2 
  File.open(ARGV.last,'w') do |f| 
    puts "writing to #{ARGV.last}"
    f << Kconv.toutf8(File.open(ARGV.first).read)
  end
else 
  puts("1st arg: input file, 2nd arg: output name")
end