check_remote_robots.sh

#!/bin/bash

result_file=$(mktemp -t "check_robots_final_XXXXX")

echo "# Checking servers..."

# Need to remove the Windows file endings
sites_file=$(mktemp -t "sites_XXXXXX")
sed 's/^M$//' /www/sitechecker/sites.txt > $sites_file

while read line; do
  myfile=$(mktemp -t "check_robotsXXXXXXXXX")
  printf "\t$line"
  wget -O $myfile "$line/robots.txt" > /dev/null 2>&1
  if grep -q "^[^#]*Disallow: /\s*$" "$myfile"; then
    echo "$line/robots.txt" >> $result_file
    printf " <<<<<<< It has incorrect robots.txt!"
  fi
  printf "\n"
  rm $myfile
done < "$sites_file"

rm $sites_file

if [ -s "$result_file" ]
then
  echo "# Sending errors to emails"
  mails=`cat /www/sitechecker/mails4external.txt`
  /bin/mail -s "There are incorrect robots.txt on the WORKING servers!" "$mails" < $result_file
fi

rm $result_file

echo "# Done"

www_sitechecker_mails4external.txt

test.example.com
example.org
ya.ru
market.yandex.ru
google.com

www_sitechecker_sites.txt

[email protected] [email protected] [email protected]