check_robots.sh

#!/bin/bash
_file=$(mktemp -t "check_robots_internal_XXXXXX")
find  /www/ -maxdepth 2 -name robots.txt | xargs  grep "^[^#]*Disallow: /\s*$" -L > $_file
 
# First, try to copy the default robots.txt to all the dirs missing it
find /www/ -mindepth 1 -maxdepth 1 -type d '!' -exec test -e "{}/robots.txt" ';' -print0 | xargs -0 -I target_folder cp /www/0default/robots.txt target_fo
# If it somehow failed for some of the folders, then report
find /www/ -mindepth 1 -maxdepth 1 -type d '!' -exec test -e "{}/robots.txt" ';' -print >> $_file
 
if [ -s "$_file" ]
then
  mails=`cat /www/sitechecker/mails4internal.txt`
   # Send it to email
    /bin/mail -s "There are incorrect robots.txt on the server!" "$mails" < $_file
fi
 
rm $_file

www_sitechecker_mails4internal.txt

[email protected] [email protected]