Magento 2 - Send nightly error email, grouped and sorted by count

daily_errors_report.sh

#!/bin/bash
# stop on errors
set -e
# turn on debugging if running into issues
#set -x

# This script sends a nightly email, showing errors, grouped and sorted by count. 
# Getting these errors emailed  can be especially useful for the first few weeks after launching a new Magento site.

# Assumptions: 
# - Capistrano is being used to deploy
# - This CRON has been configured:
## Send daily errors report
#55 23 * * * /home/www-prod/daily_errors_report.sh

DATE=$(TZ=Universal date +"%F")
[email protected]
[email protected]
EMAIL_SUBJECT='<SITE> daily errors report'

ERRORS=""

ERRORS="Date: $DATE
"

if [[ -r '/var/log/php-fpm/www-prod-error.log' ]]; then
    ERRORS+="/var/log/php-fpm/www-prod-error.log:
    "
    ERRORS+=$(cat /var/log/php-fpm/www-prod-error.log | grep "\[$(date +'%d-%b-%Y')" | awk '{gsub(/releases\/[0-9]*\//, "date/")}1' | awk '{gsub(/-'$(date +'%Y')'\s[0-9]*.*UTC/, "")}1' |  sort | uniq -c | sort -nr)
fi

ERRORS+="

exception.log:
"
ERRORS+=$(cat /var/www/prod/current/var/log/exception.log | grep "\[$(date +'%F')" | grep -v "was processed" | grep -v "Cannot gather stats" | grep -v "Unable to get content for" | sed -n 's/.*\(report.*\)/\1/p' | awk '{gsub(/releases\/[0-9]*\//, "date/")}1' | sort | uniq -c | sort -nr)

ERRORS+="

support_report.log:
"
ERRORS+=$(cat /var/www/prod/current/var/log/support_report.log | grep "\[$(date +'%F')" | grep -v "is successfully finished" | grep -v "was processed" | grep -v "Cannot gather stats" | grep -v "Unable to get content for" | sed -n 's/.*\(report.*\)/\1/p' | awk '{gsub(/releases\/[0-9]*\//, "date/")}1' | sort | uniq -c | sort -nr)

ERRORS+="

This email was sent by the ~/daily_errors_report.sh script on the [email protected] account
"

mail -s "$EMAIL_SUBJECT" $EMAIL <<< "$ERRORS"
mail -s "$EMAIL_SUBJECT" $EMAIL2 <<< "$ERRORS"

example_output.txt

Date: 
/var/log/php-fpm/www-prod-error.log:
    1 [15-Jan] PHP Fatal error:  Uncaught TypeError: Argument 1 passed to Magento\Catalog\Model\ProductRepository::getProductFromLocalCache() must be of the type string, null given, called in /var/www/prod/date/vendor/magento/module-catalog/Model/ProductRepository.php on line 240 and defined in /var/www/prod/date/vendor/magento/module-catalog/Model/ProductRepository.php:645

exception.log:
    210 report.CRITICAL: You may need more permissions to access this product. {"exception":"[object] (Magento\\Framework\\Exception\\LocalizedException(code: 0): You may need more permissions to access this product. at /var/www/prod/date/vendor/magento/module-catalog-permissions/Observer/ApplyProductPermissionObserver.php:94)"} []
    136 report.CRITICAL: You may need more permissions to access this category. {"exception":"[object] (Magento\\Framework\\Exception\\LocalizedException(code: 0): You may need more permissions to access this category. at /var/www/prod/date/vendor/magento/module-catalog-permissions/Observer/ApplyCategoryPermissionObserver.php:113)"} []
     12 report.CRITICAL: DOMDocument::loadHTML(): htmlParseEntityRef: expecting ';' in Entity, line: 1 {"exception":"[object] (Exception(code: 2): DOMDocument::loadHTML(): htmlParseEntityRef: expecting ';' in Entity, line: 1 at /var/www/prod/date/vendor/magento/framework/Escaper.php:77)"} []
     10 report.CRITICAL: Invalid account number {"exception":"[object] (Magento\\Framework\\Exception\\PaymentException(code: 0): Invalid account number at /var/www/prod/date/vendor/magento/module-paypal/Model/Payflowpro.php:662)"} []
     10 report.CRITICAL: Declined {"exception":"[object] (Magento\\Framework\\Exception\\PaymentException(code: 0): Declined at /var/www/prod/date/vendor/magento/module-paypal/Model/Payflowpro.php:662)"} []
      2 report.CRITICAL: Report ID: webapi-5c3e1d57f40d7; Message: Warning: session_start(): Failed to read session data: user (path: /var/www/prod/date/var/session/) in /var/www/prod/date/vendor/magento/framework/Session/SessionManager.php on line 194 {"exception":"[object] (Exception(code: 0): Report ID: webapi-5c3e1d57f40d7; Message: Warning: session_start(): Failed to read session data: user (path: /var/www/prod/date/var/session/) in /var/www/prod/date/vendor/magento/framework/Session/SessionManager.php on line 194 at /var/www/prod/date/vendor/magento/framework/Webapi/ErrorProcessor.php:205, Exception(code: 0): Warning: session_start(): Failed to read session data: user (path: /var/www/prod/date/var/session/) in /var/www/prod/date/vendor/magento/framework/Session/SessionManager.php on line 194 at /var/www/prod/date/vendor/magento/framework/App/ErrorHandler.php:61)"} []
      2 report.CRITICAL: Report ID: webapi-5c3e1188671fe; Message: Warning: session_start(): Failed to read session data: user (path: /var/www/prod/date/var/session/) in /var/www/prod/date/vendor/magento/framework/Session/SessionManager.php on line 194 {"exception":"[object] (Exception(code: 0): Report ID: webapi-5c3e1188671fe; Message: Warning: session_start(): Failed to read session data: user (path: /var/www/prod/date/var/session/) in /var/www/prod/date/vendor/magento/framework/Session/SessionManager.php on line 194 at /var/www/prod/date/vendor/magento/framework/Webapi/ErrorProcessor.php:205, Exception(code: 0): Warning: session_start(): Failed to read session data: user (path: /var/www/prod/date/var/session/) in /var/www/prod/date/vendor/magento/framework/Session/SessionManager.php on line 194 at /var/www/prod/date/vendor/magento/framework/App/ErrorHandler.php:61)"} []
      2 report.CRITICAL: Report ID: webapi-5c3d9317bed3b; Message: SQLSTATE[40001]: Serialization failure: 1213 Deadlock found when trying to get lock; try restarting transaction
      2 report.CRITICAL: Report ID: webapi-5c3d930eadcfd; Message: SQLSTATE[40001]: Serialization failure: 1213 Deadlock found when trying to get lock; try restarting transaction
      2 report.CRITICAL: Report ID: webapi-5c3d773dabe62; Message: SQLSTATE[40001]: Serialization failure: 1213 Deadlock found when trying to get lock; try restarting transaction
      2 report.CRITICAL: Report ID: webapi-5c3d68e7b9e9d; Message: SQLSTATE[40001]: Serialization failure: 1213 Deadlock found when trying to get lock; try restarting transaction
      2 report.CRITICAL: PayPal NVP CURL connection error #56: TCP connection reset by peer {"exception":"[object] (Exception(code: 0): PayPal NVP CURL connection error #56: TCP connection reset by peer at /var/www/prod/date/vendor/magento/module-paypal/Model/Api/Nvp.php:1214)"} []

support_report.log:
   2502 report.WARNING: Attempt to load value of nonexistent EAV attribute '259' 
   2498 report.WARNING: Attempt to load value of nonexistent EAV attribute '182' 
   2063 report.WARNING: Attempt to load value of nonexistent EAV attribute '318' 
   2063 report.WARNING: Attempt to load value of nonexistent EAV attribute '256' 
   1474 report.WARNING: Attempt to load value of nonexistent EAV attribute '333' 
   1472 report.WARNING: Attempt to load value of nonexistent EAV attribute '337' 
   1472 report.WARNING: Attempt to load value of nonexistent EAV attribute '335' 
   1391 report.WARNING: Attempt to load value of nonexistent EAV attribute '317' 
   1279 report.WARNING: Attempt to load value of nonexistent EAV attribute '271' 
   1274 report.WARNING: Attempt to load value of nonexistent EAV attribute '316' 
   1273 report.WARNING: Attempt to load value of nonexistent EAV attribute '356' 
   1273 report.WARNING: Attempt to load value of nonexistent EAV attribute '351' 
   1173 report.INFO: Cron Job staging_synchronize_entities_period is run [] []
   1173 report.INFO: Cron Job staging_remove_updates is run [] []
   1173 report.INFO: Cron Job staging_apply_version is run [] []
   1173 report.INFO: Cron Job consumers_runner is run [] []
   1173 report.INFO: Cron Job catalog_event_status_checker is run [] []
   1173 report.INFO: Consumer "async.operations.all" skipped as required connection "amqp" is not configured. Unknown connection name amqp [] []
   1162 report.INFO: Cron Job xtento_xtcore_register_last_cron_execution is run [] []
   1162 report.INFO: Cron Job sales_send_order_shipment_emails is run [] []