separate a large Rails log file into multiple files by date

readme.md

You can use this script to separate a very large Rails log file into serveral files by date very quickly.

The Rails log file may have special characters that mawk awk may not work well with. It is highly recommended to use gawk instead.

# this would print a shell script
gawk -f log.awk production.log

# run directly
gawk -f log.awk production.log | bash

The Rails log file may look like this:

I, [2014-09-10T00:00:52.413000 #5702]  INFO -- : Completed 200 OK in 2ms ...

log.awk

function p () {
  if (last != "") {
    month = substr(last, 1, 7)
    day = substr(last, 9, 2)
    print "mkdir -p", month"/"day
    file = month"/"day"/production.log"
    print "file="file
    print "test -f $file && file=${file}.2"
    printf "%s production.log", "tail -n+"NRs[last]
    if (date != "") {
      printf " | head -n%d", NRs[date]-NRs[last]
    }
    print " > $file"
  }
}
{
  date = substr($0, 5, 10)
  if (match(date, /^20[0-1][0-9]-[0-1][0-9]-[0-3][0-9]$/)) {
    if (!(date in NRs)) {
      NRs[date] = NR
      dates[dateslen++] = date
    }
  }
}
END {
  for (i = 0; i < dateslen; i++) {
    date = dates[i]
    p()
    last = date
  }
  date = ""
  p()
}