Count lines per second in access.log

get-rps.md

get-rps

Counts "Lines per second" in access.log file.

Usage

whoami@hostname ~ $ get-rps --help

 Usage: get-rps [OPTIONS] LOG_FILE
   OPTIONS are:
    --help, -h    - Print information about basic usage.
    --debug, -d   - Debug mode
    --json, -j    - Output as JSON
    --logfile, -l - Log file
    --avg   - Outputs only average RPS

Examples

basic

whoami@hostname ~ $ get-rps
"timestamp":	"19/Jun/2019:17:48"
"rps_max":	"32"
"rps_min":	"6"
"rps_avg":	16

output as JSON

whoami@hostname ~ $ get-rps --json | jq
{
  "timestamp": "19/Jun/2019:17:48",
  "rps_max": 32,
  "rps_min": 6,
  "rps_avg": 16
}

output only average (int)

whoami@hostname ~ $ get-rps --logfile /var/log/nginx/access.log --avg
15

debug output with time

whoami@hostname ~ $ time get-rps --debug
  LOG_PATH:	/var/log/nginx/access.log
	DEBUG:		true
	JSON:		false
	LOG_NAME:	access.log
	TS:		19/Jun/2019:17:50
	TOTAL_LINES:	597059
	FIRST_LINE:	596033
	LAST_LINE:	596972
	LINES_TO_TAIL:	1026
	LINES_TO_HEAD:	939
1      19/Jun/2019:17:50:00 4          4
2      19/Jun/2019:17:50:01 24         24
3      19/Jun/2019:17:50:02 21         21
4      19/Jun/2019:17:50:03 16         16
5      19/Jun/2019:17:50:04 11         11
6      19/Jun/2019:17:50:05 15         15
7      19/Jun/2019:17:50:06 25         25
8      19/Jun/2019:17:50:07 22         22
9      19/Jun/2019:17:50:08 12         12
10     19/Jun/2019:17:50:09 18         18
11     19/Jun/2019:17:50:10 18         18
12     19/Jun/2019:17:50:11 16         16
13     19/Jun/2019:17:50:12 17         17
14     19/Jun/2019:17:50:13 21         21
15     19/Jun/2019:17:50:14 5          5
16     19/Jun/2019:17:50:15 11         11
17     19/Jun/2019:17:50:16 16         16
18     19/Jun/2019:17:50:17 16         16
19     19/Jun/2019:17:50:18 9          9
20     19/Jun/2019:17:50:19 15         15
21     19/Jun/2019:17:50:20 21         21
22     19/Jun/2019:17:50:21 16         16
23     19/Jun/2019:17:50:22 16         16
24     19/Jun/2019:17:50:23 26         26
25     19/Jun/2019:17:50:24 11         11
26     19/Jun/2019:17:50:25 13         13
27     19/Jun/2019:17:50:26 18         18
28     19/Jun/2019:17:50:27 17         17
29     19/Jun/2019:17:50:28 5          5
30     19/Jun/2019:17:50:29 13         13
31     19/Jun/2019:17:50:30 8          8
32     19/Jun/2019:17:50:31 14         14
33     19/Jun/2019:17:50:32 25         25
34     19/Jun/2019:17:50:33 19         19
35     19/Jun/2019:17:50:34 21         21
36     19/Jun/2019:17:50:35 14         14
37     19/Jun/2019:17:50:36 12         12
38     19/Jun/2019:17:50:37 18         18
39     19/Jun/2019:17:50:38 21         21
40     19/Jun/2019:17:50:39 8          8
41     19/Jun/2019:17:50:40 19         19
42     19/Jun/2019:17:50:41 22         22
43     19/Jun/2019:17:50:42 11         11
44     19/Jun/2019:17:50:43 8          8
45     19/Jun/2019:17:50:44 13         13
46     19/Jun/2019:17:50:45 8          8
47     19/Jun/2019:17:50:46 14         14
48     19/Jun/2019:17:50:47 15         15
49     19/Jun/2019:17:50:48 16         16
50     19/Jun/2019:17:50:49 15         15
51     19/Jun/2019:17:50:50 11         11
52     19/Jun/2019:17:50:51 6          6
53     19/Jun/2019:17:50:52 18         18
54     19/Jun/2019:17:50:53 13         13
55     19/Jun/2019:17:50:54 23         23
56     19/Jun/2019:17:50:55 23         23
57     19/Jun/2019:17:50:56 28         28
58     19/Jun/2019:17:50:57 17         17
59     19/Jun/2019:17:50:58 12         12
60     19/Jun/2019:17:50:59 13         13
          TIMESTAMP:    19/Jun/2019:17:50
        MAXIMUM RPS:    28
        MINIMUM RPS:    4
        AVERAGE RPS:    15

real	0m0.487s
user	0m0.164s
sys	0m0.120s

get-rps.sh

#!/bin/bash

NAME=$(basename $0)
DEBUG=${DEBUG:-false}
JSON=${JSON:-false}
LOG_PROVIDED=false
AVG_ONLY=false
TS="$(date +'%d/%b/%Y:%H:%M' --date='1 minute ago')"


function main(){
  LOG_PATH="${LOG_PATH:-/var/log/nginx/access.log}"
  LOG_NAME="$(basename ${LOG_PATH})"

  ${DEBUG} && __debug_inputs

  TOTAL_LINES=$(wc -l ${LOG_PATH} | awk '{print $1}')
  FIRST_LINE=$(_line_numbers_by_ts ${TS} | head -n 1)
  LAST_LINE=$(_line_numbers_by_ts ${TS} | tail -n 1)
  LINES_TO_TAIL=$((${TOTAL_LINES}-${FIRST_LINE}))
  LINES_TO_HEAD=$((${LAST_LINE}-${FIRST_LINE}))
  ${DEBUG} && __debug_lines

  TMPFILE=$(/bin/mktemp /tmp/${NAME}.tmp.XXXXXXXXX)
  trap "{ /bin/rm -f $TMPFILE ; exit 255; }" EXIT
  tail -n ${LINES_TO_TAIL} ${LOG_PATH} | head -n ${LINES_TO_HEAD} > ${TMPFILE}


  SECONDS_ARRAY=$(echo {00..59})
  REQS=()
  CNT=0

  for SEC in ${SECONDS_ARRAY}; do
    ((++CNT))
    LPS=$(_lines_per_second ${SEC} ${TMPFILE} | wc -l)
    REQS[$CNT]="${LPS}"
    ${DEBUG} && printf '%-5s  %s:%s %-10s %-15s\n' "${CNT}" "${TS}" "${SEC}" "${LPS}" "${REQS[$CNT]}"
  done

  ${AVG_ONLY} || MAX_RPS=$(echo ${REQS[@]} | jq -s max)
  ${AVG_ONLY} || MIN_RPS=$(echo ${REQS[@]} | jq -s min)
  AVG_RPS=$(echo ${REQS[@]} | jq -s add/length | cut -d '.' -f 1)

  __print_results

}

#### MISC FUNCTIONS AND UTILS

## get line numbers for timestamp (previous minute)
function _line_numbers_by_ts(){
  grep -h -n  "${TS}" ${LOG_PATH} | cut -d ':' -f 1
}

## get lines count per second
function _lines_per_second(){
  SECOND=${1:-\d\d}
  FILE=${2:-${LOG_PATH}}
  egrep "${TS}:${SECOND}" ${FILE}
}

## print inputs
function __debug_inputs(){
  cat <<EOF
	LOG_PATH:	${LOG_PATH}
	DEBUG:		${DEBUG}
	JSON:		${JSON}
	LOG_NAME:	${LOG_NAME}
	TS:		${TS}
EOF
}

## print variables with lines calc
function __debug_lines(){
  cat <<EOF
	TOTAL_LINES:	${TOTAL_LINES}
	FIRST_LINE:	${FIRST_LINE}
	LAST_LINE:	${LAST_LINE}
	LINES_TO_TAIL:	${LINES_TO_TAIL}
	LINES_TO_HEAD:	${LINES_TO_HEAD}
EOF
}

## print result
function __print_results(){
  if ${DEBUG}; then
    cat <<EOF
          TIMESTAMP:    ${TS}
        MAXIMUM RPS:    ${MAX_RPS}
        MINIMUM RPS:    ${MIN_RPS}
        AVERAGE RPS:    ${AVG_RPS}
EOF
  else
    if ${AVG_ONLY}; then
      printf '%s\n' "${AVG_RPS}"
    elif ${JSON}; then
      printf '{"timestamp":"%s","rps_max":%s,"rps_min":%s,"rps_avg":%s}\n' "${TS}" "${MAX_RPS}" "${MIN_RPS}" "${AVG_RPS}"
    else
      printf '"timestamp":\t"%s"\n"rps_max":\t"%s"\n"rps_min":\t"%s"\n"rps_avg":\t%s\n' "${TS}" "${MAX_RPS}" "${MIN_RPS}" "${AVG_RPS}"
    fi
  fi
}

## print usage info
function __print_help(){
  cat <<EOF

  Usage: ${NAME} [OPTIONS] LOG_FILE

   OPTIONS are:
    --help, -h		- Print information about basic usage.
    --debug, -d		- Debug mode
    --json, -j		- Output as JSON
    --logfile, -l	- Log file
    --avg		- Outputs only average RPS
EOF
}



##################################


for OPT in "$@"; do
  ${DEBUG} && echo "OPT: ${OPT}"
  case $OPT in
    --help|-h|-\?|help)		__print_help ; exit 1 ;;
    --debug|-d)			DEBUG=true ;;
    --json|-j)			JSON=true  ;;
    --logfile|-l)		LOG_PROVIDED=true ;;
    --avg)			AVG_ONLY=true ;;
    *)				VALUE=${OPT} ;;
  esac
done

#${DEBUG} && echo "[--logfile] LOG_PROVIDED: ${LOG_PROVIDED}"

if ${LOG_PROVIDED}; then
  if [[ -f ${VALUE} ]]; then
    LOG_PATH=${VALUE}
  else
    echo "FAIL: seems ${VALUE} is not a file"
    exit 1
  fi
fi

#__debug_inputs

##############################
main