Check CentOS Mirrors Validity via ISO Size and repo XML enumeration

check-centos-mirrors.sh

#!/bin/sh

## PoC: Validate CentOS mirrors using a set of release data
##      - ISO image sizes
##      - ISO image digests of bytes 32768-65535
##      - Torrent digests
##      - Enumerate repo XML files to verify SHA-256 hash
##
## @oogali
##

CURL=`which curl 2>/dev/null`
CURL_ARGS="-s --connect-timeout 3 --max-time 15"
if [ -z "${CURL}" ]; then
  echo "ERROR: curl is not installed."
  exit 1
fi

OPENSSL=`which openssl 2>/dev/null`
if [ -z "${OPENSSL}" ]; then
  echo "ERROR: openssl is not installed."
  exit 1
fi

MIRMON_STATE_URL=http://mirror-status.centos.org/mstat.txt

if [ $# -lt 2 ] || [ $# -gt 3 ]; then
  echo "$0 <CentOS Mirror Hostname> <CentOS Release Version> [Base URI]"
  exit 1
fi

mirror=${1}
version=${2}

absolute_path=`${CURL} ${CURL_ARGS} ${MIRMON_STATE_URL} | awk '{ print $1 }' | grep -i "/${mirror}/"`
if [ $? -ne 0 ]; then
  if [ $# -ne 3 ]; then
    echo "$0: ${mirror} is not a registered mirror, cannot perform any checks without a specified base URI"
    exit 1
  fi

  mirror="${mirror}${3}"
else
  mirror=${absolute_path}
fi

echo "Checking mirror ${mirror}..."
${CURL} ${CURL_ARGS} -o /dev/null ${mirror}
if [ $? -ne 0 ]; then
  echo "ERROR: could not connect to mirror"
  exit 1
fi

# XXX: this should be broken out into a separate file and fed in as a command line argument
checkdata=$(
cat <<EOF
size|/${version}/isos/i386/CentOS-${version}-i386-bin-DVD.iso|4705456128
size|/${version}/isos/i386/CentOS-${version}-i386-netinstall.iso|181403648
size|/${version}/isos/x86_64/CentOS-${version}-x86_64-bin-DVD1.iso|4238800896
size|/${version}/isos/x86_64/CentOS-${version}-x86_64-bin-DVD2.iso|1182699520
size|/${version}/isos/x86_64/CentOS-${version}-x86_64-netinstall.iso|221249536
isodigest|/${version}/isos/i386/CentOS-${version}-i386-bin-DVD1.iso|sha256:97e843ba12889ea7af83ceb0b4c142063d8cfee709155980cbb7b2da0149c862
isodigest|/${version}/isos/i386/CentOS-${version}-i386-netinstall.iso|sha256:f9152ddfcb5f0d1ebf7245918ea11d281b18288522e41b84e451f99eaecb31f3
isodigest|/${version}/isos/x86_64/CentOS-${version}-x86_64-bin-DVD1.iso|sha256:612adfaf522468c906eac6e69740a02c3a71e0c35f29c99a867f0ab026990707
isodigest|/${version}/isos/x86_64/CentOS-${version}-x86_64-bin-DVD2.iso|sha256:4abc7a193b65930c59af88352eab43bcfa1ca5b996f83d200f2a9c38f6c47da8
isodigest|/${version}/isos/x86_64/CentOS-${version}-x86_64-netinstall.iso|sha256:4dad13cd3c97115154c421298de4478617d91bf8799a47fd064c4dba2ecb83db
digest|${version}/isos/i386/CentOS-${version}-i386-bin-DVD.torrent|sha256:49439a88260f8f6d640e28e1e4923b51292994a3057b8e8c4b939b3c9b6876eb
digest|${version}/isos/x86_64/CentOS-${version}-x86_64-bin-DVD.torrent|sha256:a9ce0b150c57750b2ceb9080fb60b6c7b6765cc9b8d314b2722ab311f5304343
repoenum|/${version}/os/i386/repodata/repomd.xml
repoenum|/${version}/os/x86_64/repodata/repomd.xml
repoenum|/${version}/updates/i386/repodata/repomd.xml
repoenum|/${version}/updates/x86_64/repodata/repomd.xml
repoenum|/${version}/centosplus/i386/repodata/repomd.xml
repoenum|/${version}/centosplus/x86_64/repodata/repomd.xml
EOF
)

do_http_size_check() {
  if [ $# -ne 2 ]; then
    echo "do_http_size_check: need file and expected size"
    return 1
  fi

  uri=${1}
  expected_size=${2}

  hosted_size=`${CURL} ${CURL_ARGS} -I "${mirror}${uri}" | awk '/^Content-Length: / { print $2 }' | tr -d '\015'`
  if [ "${hosted_size}" != "${expected_size}" ]; then
    return 1
  fi
}

do_repo_enum_check() {
  if [ $# -ne 1 ]; then
    echo "do_repo_enum: need xml file"
    return
  fi

  uri=${1}
  base_uri=`echo ${uri} | sed 's/repomd.xml//'`

  for fn in `${CURL} ${CURL_ARGS} "${mirror}${uri}" | xpath '//location[@href]/@href' 2>/dev/null`; do
    xml=`echo ${fn} | cut -f2 -d '/' | sed 's/"//g'`
    expected_digest=`echo ${xml} | cut -f1 -d '-'`
    # XXX: currently assuming all digests are sha256, should parse checksum type out of xml
    do_digest_check "${base_uri}${xml}" "sha256" "${expected_digest}"
    if [ $? -eq 1 ]; then
      # XXX: error is printed by do_digest_check() see refactor note below
      return 1
    fi
  done
}

# XXX: should we refactor do_digest_check into digest_calc and do_digest_check?
#      - digest_calc would simply calculate digest given url and digest method
#      - do_digest_check would call digest_calc and display any necessary alerts
#      - do_repo_enum_check would call digest_calc and display any necessary alerts
do_digest_check() {
  if [ $# -lt 3 ]; then
    echo "do_digest_check: need file, digest method, and expected digest; optionally start and end range"
    return
  fi

  uri=${1}
  digest_method=${2}
  expected_digest=${3}

  # were we passed a begin/end range?
  if [ $# -eq 5 ]; then
    range="-r ${4}-${5}"
  else
    range=""
  fi

  our_digest=`${CURL} ${CURL_ARGS} ${range} "${mirror}${uri}" | ${OPENSSL} "${digest_method}" | awk '{ print $2 }' 2>/dev/null`
  if [ "${expected_digest}" != "${our_digest}" ]; then
    echo "file ${uri} failed digest check (expected ${expected_digest} != received ${our_digest})"
    return 1
  fi
}

for check in ${checkdata} ; do
  action=`echo ${check} | cut -f1 -d '|'`
  uri=`echo ${check} | cut -f2 -d '|'`
  expected=`echo ${check} | cut -f3 -d '|'`

  case "${action}" in
    size)
      do_http_size_check ${uri} ${expected}
      if [ $? -eq 1 ]; then
        echo "file ${uri} failed size check"
      fi
      ;;

    repoenum)
      do_repo_enum_check ${uri}
      if [ $? -eq 1 ]; then
        echo "${uri} failed repo enumeration check"
      fi
      ;;

    digest)
      method=`echo ${expected} | cut -f1 -d ':'`
      expected_digest=`echo ${expected} | cut -f2 -d ':'`
      do_digest_check ${uri} ${method} ${expected_digest}
      ;;

    isodigest)
      method=`echo ${expected} | cut -f1 -d ':'`
      expected_digest=`echo ${expected} | cut -f2 -d ':'`
      # XXX: allow offset and end to be configurable
      do_digest_check ${uri} ${method} ${expected_digest} 32768 65535
      ;;

    *)
      ;;
  esac
done