Extract only the latest log lines from apache/nginx logs

extract-latest-log-date.awk

#!/usr/bin/awk -f

# Usage:
# zcat -f access.log.* | ./extract-latest-log-date.awk > latest-log.txt

BEGIN {
  # pre-compute months field lookup
  m=split("Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec",d,"|")
  for(o=1;o<=m;o++){
    months[d[o]]=sprintf("%02d",o)
  }
  # date field separators
  FS="[]]|[[]"	
}
# for each line
{
  # break the date field into components
  split($2, dt, "[: /]")
  # build a new comparable date sequence
  datestamp = (dt[3] " " months[dt[2]] " " dt[1] " " dt[4] " " dt[5] " " dt[6])
  # find if this line has a more recent date than the latest known one
  if (datestamp > latest) {
    latest = datestamp;
    latest_stamp = $2;
  }
}
# once we're done, output the result
END {
  print latest_stamp
}

log-lines-after-date.awk

#!/usr/bin/awk -f

# Usage:
# (once you've got a date from parsing some previous logs above)
# zcat -f access.log.* | ./log-lines-after-date.awk -vcheck="`cat latest-log.txt`"

BEGIN {
  if (check == "" ) {
    print "Usage: log-lines-after-date.awk -vcheck='DATE'"
    print "Where DATE is e.g. 04/Jun/2017:07:44:10"
    exit 1
  }
  # pre-compute months field lookup
  m=split("Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec",d,"|")
  for(o=1;o<=m;o++){
    months[d[o]]=sprintf("%02d",o)
  }
  # pre-compute our check date in a comparable format
  split(check, dti, "[: /]")
  dcheck = (dti[3] " " months[dti[2]] " " dti[1] " " dti[4] " " dti[5] " " dti[6])
  # date field separators
  FS="[]]|[[]"
}
# for each line
{
  # break the date field into components
  split($2, dt, "[: /]")
  # build a new comparable date sequence
  datestamp = (dt[3] " " months[dt[2]] " " dt[1] " " dt[4] " " dt[5] " " dt[6])
  # find if this line has a more recent date than the check date
  if (datestamp > dcheck) {
    print $0
  }
}

If you have a bunch of log files (access.log.*) and you want to only process newly added/changed lines when they are updated you can use these scripts to do so.