Loop though all rotated access logs and get the top 20 list of User AgentsThis can be used to identify bots

top-20-user-agents-and-ips-from-directadmin-logs.sh

# Loop though all rotated Directadmin access logs and get the top 20 list of User Agents
# This can be used to identify bots
zcat /home/USER/domains/DOMAIN/logs/LOGNAME.tar.gz* | awk -F\" '{print $6}' | sort | uniq -c | sort -nr | head -20

# Get top 20 IP addresses
zcat /home/USER/domains/DOMAIN/logs/Feb-2019.tar.gz | awk -F\" '{print $1}' | sort | uniq -c | sort -nr | head -20

# Get top 20 most visited URLs
zcat /home/USER/domains/DOMAIN/logs/Feb-2019.tar.gz | awk -F\" '{print $4}' | sort | uniq -c | sort -nr | head -20

# Get top 20 most visited URLs
zcat /home/USER/domains/DOMAIN/logs/Feb-2019.tar.gz | awk -F\" '{print $3}' | sort | uniq -c | sort -nr | head -20

# Get the top 20 404 responses from all rotated Directadmin access logs
zgrep -a ' 404 ' /home/USER/domains/DOMAIN/logs/Feb-2019.tar.gz* | awk -F\" '{print $2}' | sort | uniq -c | sort -nr | head -20