A simple script that scours logs (incrementally) looking for keywords and emails you if it finds at least 1 of your keywords. Perfect to be scheduled with CRON since it uses logtail to scan only parts of logs that it has not scanned before!

logalert.sh

#!/bin/bash

####
# Scours log files for keywords, then and emails
# if it finds them.
#
# This script also has keywords to exclude. This is, within the matches
# if the exclude keywords appear, then those lines are excluded from
# the output report. This allows you to "match" on a long keyword (
# e.g. "/some/directory/with/tools") yet exclude a specific match (
# e.g. "specific-tool.sh").
#
# Note that you can use the command `logger` with your admin shell
# scripts in order to output values into syslog.
# See https://www.commandlinux.com/man-page/man1/logger.1.html)
#
# When defining log files, the user that will execute the
# script must have access otherwise this script won't be able to inspect.
# Output is both emailed and saved to the "reports" subdirectory (under
# the directory where the script resides!)
#
# Dependencies: logtail, egrep, sendmail
# Note: sendmail must be configured to properly send outbound email!
#
# Eric A. Soto, 2017-10-28
#
# Tested on Ubuntu 16.04
#
####

# ###########
# The following section has variables which should not need to be changed!
# Skip to constants for values you do want to change.
#
at_least_one=0 # used to determine if we have at least 1 hit
# path where this script resides
currPath="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# reports path (will be a subdirectory)
report_sub_directory="/reports"
# offsets path (will be a subdirectory where logtail offset files are kept for each log inspected)
offset_sub_directory="/offsets"
# Log changes file (temporary), Report path
logtail_new_file=${currPath}/logtail.new
# log out temp files (used to store items before the excluded keywords)
logalert_out_temp_file_0=${currPath}${report_sub_directory}/logalert-temp-0.tmp
logalert_out_temp_file_1=${currPath}${report_sub_directory}/logalert-temp-1.tmp
# computer the name for our report file
logalert_out_file=${currPath}${report_sub_directory}/logalert-`date +%Y-%m-%d-%H-%M-%S`.txt

# ###########
# Set our constants (must change these to suit!)
# 
email_from="WP02 <[email protected]>"
email_to="[email protected]"
email_subject="WP02 Log Alert"
# Define the logs to scour. Must be in the format "name:/full/path/to/log"
logs=("syslog:/var/log/syslog")
# Define the keywords to scour for - bash array & quoted please!
keywords=("keyword to seek" "another keyword")
# Define the keywords to exclude, if found in lines that match the above keywords - bash array & quoted please!
keywords_excluded=("logalert.sh" "exclude me also")
# Set debug_flag="1" to produce debug output and not send any email
# Can also be done by calling the script with the --debug command line argument
debug_flag="0"

## Do Work!

# Check to see if we were passed "--debug_flag" as command line argument
if [[ "$1" == "--debug"  ]]; then
  debug_flag="1"
fi

# If in debug_flag, we setup a few things differently
if [[ "$debug_flag" -eq "1" ]]; then
  echo "debug mode active..."
  logtail_options="-t"
fi

# Ensure we have the directories we need
if [ "$report_sub_directory" != "" ] && [ ! -d "${currPath}${report_sub_directory}" ]; then
  mkdir -p ${currPath}${report_sub_directory}
fi
if [ "$offset_sub_directory" != "" ] && [ ! -d "${currPath}${offset_sub_directory}" ]; then
  mkdir -p ${currPath}${offset_sub_directory}
fi

# cleanup from a prior run
rm $logalert_out_file > /dev/null 2>&1
rm $logtail_new_file > /dev/null 2>&1

# Initialize the email block of the output
echo "To: ${email_to}" >> $logalert_out_file
echo "Subject: ${email_subject}" >> $logalert_out_file
echo "From: ${email_from}" >> $logalert_out_file
echo "" >> $logalert_out_file

# Initialize the output file
echo ">>>>> logalert output for `date '+%Y-%m-%d %H:%M:%S'`" >> $logalert_out_file
echo ">>>>> logfile: ${logalert_out_file}" >> $logalert_out_file
echo "" >> $logalert_out_file

echo "***** Preparing log files: " >> $logalert_out_file

# Loop through the log files and add all of them to the inspection file
for i in "${logs[@]}"
do
  name=`echo "$i" | cut -d':' -f1`
  path=`echo "$i" | cut -d':' -f2`
  # Log
  echo "  Logfile: ${path}" >> $logalert_out_file
    # begin by looking at the logtail of the syslog
    /usr/sbin/logtail ${logtail_options} -f $path -o ${currPath}${offset_sub_directory}/${name} >> $logtail_new_file
done
echo "" >> $logalert_out_file

# Loop through the keywords, looking for each in the inspection file
for keyword in "${keywords[@]}"
do
    # Escape any slashes in the keyword (from "/" to "\/")
    escaped_slash_keyword=${keyword//\//\\\/}
    echo "***** keyword: ${keyword} | with escaped slashes: ${escaped_slash_keyword} ********************" >> $logalert_out_file
    echo "" >> $logalert_out_file
    egrep -i --regexp="$escaped_slash_keyword" $logtail_new_file >> $logalert_out_temp_file_0
    grep_result=$?
    if [[ $grep_result -eq 1 ]]; then
      echo "  KEYWORD NOT FOUND" >> $logalert_out_file
    else
      # Loop through the exclude keywords, eliminating lines that match
      for keyword_excluded in "${keywords_excluded[@]}"
      do
          # Escape any slashes in the keyword (from "/" to "\/")
          escaped_slash_keyword=${keyword_excluded//\//\\\/}
          echo "***** exclude keyword: ${keyword_excluded} | with escaped slashes: ${escaped_slash_keyword} ********************" >> $logalert_out_file
          echo "" >> $logalert_out_file
          egrep -i -v --regexp="$escaped_slash_keyword" $logalert_out_temp_file_0 > $logalert_out_temp_file_1
          grep_result=$?

          # shuffle the temp files
          rm $logalert_out_temp_file_0 > /dev/null 2>&1
          mv $logalert_out_temp_file_1 $logalert_out_temp_file_0 > /dev/null 2>&1

          # Check to see how things went
          if [[ $grep_result -eq 1 ]]; then
            # Some exclusions filtered out
            echo "  EXCLUSION FOUND" >> $logalert_out_file
            # Do we have anything left in the file?
            lines_left=`wc -l $logalert_out_temp_file_0 | cut -d' ' -f1`
            # check to see
            if [[ "$lines_left" -gt "0" ]]; then
              echo "  OTHER ITEMS STILL MATCHED" >> $logalert_out_file
              # Some lines were left, so we have at least one match
              at_least_one=1
              # Output the temp file that has [keyword matches minus exclusions]
              cat $logalert_out_temp_file_0 >> $logalert_out_file
            else 
              echo "  NO OTHER ITEMS MATCHED" >> $logalert_out_file
            fi
          else
            # No exclusions performed
            echo "  EXCLUSION NOT FOUND" >> $logalert_out_file
            # We found at least 1 match that was not excluded, so we can set our flag since we're keeping matches!
            at_least_one=1
            # Output the temp file that has [keyword matches with no exclusions]
            cat $logalert_out_temp_file_0 >> $logalert_out_file
          fi
          echo "" >> $logalert_out_file
      done
    fi
    # Clear out the temp-working file so it's not there for the next match
    rm $logalert_out_temp_file_0 > /dev/null 2>&1
    rm $logalert_out_temp_file_1 > /dev/null 2>&1
    echo "" >> $logalert_out_file
done

# If in debug_flag, we output the report we just created but we don't email!
# If not in debug_flag, we delete the inspection file and we email the report.
if [[ "$debug_flag" -eq "1" ]]; then
  cat ${logalert_out_file}
else 
  # delete the logtail file since we're done
  rm $logtail_new_file > /dev/null 2>&1 
  # delete reports older than 1 day
  /usr/bin/find "${currPath}${report_sub_directory}" -mtime +1 -type f -delete
  # send email IF we had at least 1 log hit. We don't send an email otherwise.
  if [[ "$at_least_one" -eq "1" ]]; then
    /usr/sbin/sendmail -vt < ${logalert_out_file} > /dev/null 2>&1
    sendmail_result=$?
    if [[ "$sendmail_result" -eq 0 ]]; then
      echo "Email sent successfully." >> $logalert_out_file
    else
      echo "Unable to send email. Sendmail exit=$sendmail_result" >> $logalert_out_file
    fi
  fi
fi