A simple awk script for http request duration statistic

http_log_statistic.sh

# tornado http request duration histogram
awk '$1 == "[I" && $5 == 200 {sum += 1; duration = int(substr($10, 0, length($10) - 2)); if(duration < 100) r100 +=1; else if(duration < 200) r200 += 1; else if(duration < 500) r500 += 1; else r1000 += 1;} END {print "总请求量", "100ms以内", "200ms以内", "500ms以内", "500ms以外"; print sum, r100, r200, r500, r1000, (r100 + r200) / sum}' /path/app.log

# tornado http request top ten qps time
awk '$1 == "[I" && $5 == 200 {split($3, time, ","); second = $2" "time[1]; agg[second] += 1} END {for(second in agg) print second, agg[second]}' /path/app.log | sort -nr -k 3 | head

# nginx top ten qps time
awk '{second = substr($4, 2, length($4) - 1); agg[second] += 1} END {for(second in agg) print second, agg[second]}' logs/access.log | sort -nr -k 2 | head

# tornado http request duration greater than 100ms
awk '$1 == "[INFO]" && $9 == 200 {sum += 1; duration = int(substr($NF, 0, length($NF) - 2)); if(duration > 100) print $0}' /path/app.log

# calculate the %90 and %99 duration
awk '/2016-11-03/ && /logics.cache/ {split($NF, a, "ms"); b[NR] = a[1]} END {n = asort(b); p90 = int(n * 0.9) + 1; p99 = int(n * 0.99) + 1; print b[1], b[n]; print p90, b[p90]; print p99, b[p99]}' ~/backend/logs/apps/david-calendar/slowlog