Fedalto · January 16, 2019 07:23 · Fedalto · May 27, 2013 · rajeshpal007 · Oct 24, 2015
diff --git a/LICENSE b/LICENSE
 The MIT License (MIT)

 Copyright (c) 2015 Leonardo Fedalto

 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
 in the Software without restriction, including without limitation the rights
 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 copies of the Software, and to permit persons to whom the Software is
 furnished to do so, subject to the following conditions:

 The above copyright notice and this permission notice shall be included in all
 copies or substantial portions of the Software.

 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
diff --git a/solr-monitor.sh b/solr-monitor.sh
 #!/bin/bash

 # This script will get the list af active cores and issues a ping request to them.
 # If a single request failes, the check will return ERROR
 # All requests must return OK

 # Local variables
 EXIT_CODE=0
 SOLR_HOST="localhost"
 SOLR_HOST_PORT="8983"
 # Last replication event must have been at least xxx sec ago.
 REPLICATION_TIME_TOLERANCE=900 # = 15 min
 OUT=""

 set -u

 parse_args() {
  # If no arguments are given
  if [ "$#" -eq 0 ]; then
    pod2usage "$0"
    exit 1
  fi

  # Parse arguments
  while [ $# -gt 0 ]
  do
    case "$1" in
      --help|-h|-\?)
        pod2usage -verbose 1 "$0"
        exit 1
      ;;

      --man)
        pod2usage -verbose 2 "$0"
        exit 1
      ;;

      --host|-H)
        SOLR_HOST="$2"
        shift 2
      ;;

      --port|-P)
        SOLR_HOST_PORT="$2"
        shift 2
      ;;

      --diff|-D)
        REPLICATION_TIME_TOLERANCE="$2"
        shift 2
      ;;

      *)
        # Non option argument
        break # Finish for loop
      ;;
    esac
  done

  # main code
  [ -n "$SOLR_HOST" ] || {
    echo "UNKNOWN: --host argument is not set"
    exit 2
  }
 }

 call_solr_api() {
  curl --retry 3 --max-time 15 --fail --silent "http://${SOLR_HOST}:${SOLR_HOST_PORT}/solr/${1}"

  if [ "$?" != "0" ]; then
    echo "CRITICAL: server \"$SOLR_HOST\" is not responding or returned incorrect data." >&2
    exit 2
  fi
 }

 get_cores_status() {
  call_solr_api "admin/cores"
 }

 get_replication_details() {
  call_solr_api "${1}/replication?command=details"
 }

 ping_solr_core() {
  call_solr_api "${1}/admin/ping"
 }

 get_solr_cores() {
  local response=$(<<< "$1" xmllint --format - | grep '<str name="name"'  | sed -e 's/[ ]*<str name="name">//g' -e 's/<\/str>//g')

  if [ -z "$response"  ]; then
    echo "CRITICAL: server \"$SOLR_HOST\" returned an empty list of cores."
    exit 2
  fi

  echo "$response"
 }

 is_master() {
  [[ $(xmlstarlet sel -t -v "/response/lst[@name='details']/str[@name='isMaster']" <<< "$1") == "true" ]]
 }

 check_init_failures() {
  local cores_status="$1"
  local cores_with_init_failures=$(xmlstarlet sel -t -m "/response/lst[@name='initFailures']/str" -m "@name" -v . -n <<< "${cores_status="$1"}" | grep -v '^$')
  for bad_core in $cores_with_init_failures; do
    local problem=$(xmlstarlet sel -t -v "/response/lst[@name='initFailures']/str[@name='$bad_core']" <<< "${cores_status="$1"}")
    OUT="${OUT}Core \"$bad_core\" could not initialize: ${problem}\n"
    EXIT_CODE=2
  done
 }

 check_master_core() {
  local core="$1"
  local result=$(ping_solr_core "$core")
  result=$(xmlstarlet sel -t -v "/response/str[@name='status']" <<< "$result")
  OUT="${OUT}Core \"${core}\" returned \"${result}\".\n"
  [ "$result" = "OK" ] || EXIT_CODE=2
 }


 check_slave_core() {
  local core="$1"
  local replication_details="$2"

  #Verifying replication health is harder, so there will be several checks.
  # 1) Check if master host is defined.
  # 2) Check indexReplicatedAtList and compare it with replicationFailedAtList. This will determine if replication is working.
  # 3) Check if indexReplicatedAtList is more then 2 hours behind now().  If it is then something might be wrong.
  # 4) Check if indexVersion on master and slave match.
  # Get the URL to the corresponding master core
  SOLR_SLAVE_MASTERURL=$(xmlstarlet sel -t -v "/response/lst[@name='details']/lst[@name='slave']/str[@name='masterUrl']" <<< "$replication_details")

  # Get the master index version
  SOLR_MASTER_INDEXVERSION=$(xmlstarlet sel -t -v "/response/lst[@name='details']/lst[@name='slave']/lst[@name='masterDetails']/long[@name='indexVersion']" <<< "$replication_details")

  # Get slave index version
  SOLR_SLAVE_INDEXVERSION=$(xmlstarlet sel -t -v "/response/lst[@name='details']/long[@name='indexVersion']" <<< "$replication_details")

  # Get the last time the core replicated correctly.
  SOLR_SLAVE_REPLICATEDAT=$(xmlstarlet sel -t -v "/response/lst[@name='details']/lst[@name='slave']/arr[@name='indexReplicatedAtList']/str[1]" <<< "$replication_details")

  # Get the last time the core failed to replicate.
  SOLR_SLAVE_FAILEDAT=$(xmlstarlet sel -t -v "/response/lst[@name='details']/lst[@name='slave']/arr[@name='replicationFailedAtList']/str[1]" <<< "$replication_details")

  # Is this core replicating (aka pulling index from master) right now?
  SOLR_SLAVE_REPLICATING=$(xmlstarlet sel -t -v "/response/lst[@name='details']/lst[@name='slave']/str[@name='isReplicating']" <<< "$replication_details")

  # masterUrl is not set, replication is broken.
  if [ -z "$SOLR_SLAVE_MASTERURL" ]; then
    OUT="$OUT\n"."Core \"${core}\" has no masterUrl set, solr.xml misconfiguration or orphaned core."
    EXIT_CODE=2
    continue
  fi

  # Slave could not get the master index version. Maybe the master is down or network issues.
  if [[ -z "$SOLR_MASTER_INDEXVERSION" ]]; then
    OUT="${OUT}Core \"$core\" could not get master index version.\n"
    EXIT_CODE=2
    continue
  fi

  # If SOLR_SLAVE_REPLICATEDAT is empty then the instance has not replicated once, this is an error.
  # Actually it's possible that this check will cause a false alert if the index is being replicated for the first time just now, so check for this also.
  if [ -z "$SOLR_SLAVE_FAILEDAT" -a -z "$SOLR_SLAVE_REPLICATEDAT" -a "$SOLR_SLAVE_REPLICATING" == "true" ]; then
    # Everything is ok, this is the first time this core has been triggered to replicate.
    OUT="${OUT}Core \"${core}\" is replicating for the first time.\n"
    continue
  fi

  if [ \( -n "$SOLR_SLAVE_FAILEDAT" -a -z "$SOLR_SLAVE_REPLICATEDAT" \) -o \( -z "$SOLR_SLAVE_FAILEDAT" -a -z "$SOLR_SLAVE_REPLICATEDAT" \) ]; then
    # This core has never replicated, this is an error.
    OUT="${OUT}Core \"${core}\" has problems replicating.\n"
    EXIT_CODE=2
    continue
  fi

  # We need to calculate if the last replication attempt was successfull.
  LAST_REPLICATION_SUCCESSFUL=`date -d "$SOLR_SLAVE_REPLICATEDAT" +%s`
  TOLERANCE=`date -d "now - $REPLICATION_TIME_TOLERANCE seconds" +%s`

  # Verify that either the timestamp is not older then REPLICATION_TIME_TOLERANCE seconds and if it is check that indexversions match.
  if [ $LAST_REPLICATION_SUCCESSFUL -gt $TOLERANCE -o $SOLR_MASTER_INDEXVERSION = $SOLR_SLAVE_INDEXVERSION ]; then
    # Everything is ok
    OUT="${OUT}Core \"${core}\" is up to date.\n"
  else
    # Slave if outdated.
    OUT="${OUT}Core \"${core}\" is behind master more then $REPLICATION_TIME_TOLERANCE seconds.\n"
    EXIT_CODE=2
  fi
 }

 main() {
  parse_args "$@"

  local cores_status=$(get_cores_status)

  local solr_cores=$(get_solr_cores "$cores_status")

  check_init_failures "$cores_status"

  for core in $solr_cores; do
    local replication_details=$(get_replication_details "$core")

    if is_master "$replication_details"; then
      check_master_core "$core"
    else
      check_slave_core "$core" "$replication_details"
    fi
  done

  echo -e -n "$OUT"
  exit $EXIT_CODE
 }

 main "$@"

 __END__

 =pod

 =head1 NAME

 solr-monitor - Simple script to check the status of all defined cores on a solr server

 =head1 SYNOPSIS

 solr-monitor [OPTIONS]

 =head1 OPTIONS

 =over 4

 =item B<--help> | B<-h>

 Print the brief help message and exit.

 =item B<--man>

 Print the manual page and exit.

 =item B<--host> | B<-H> HOST

 Check this host instead of localhost.

 =item B<--port> | B<-P> Port

 Use this port instead of the default(8983) to connect.

 =item B<--diff> | B<-D> Time difference between now and when solr last replicated

 Use this option to set the maximum difference in seconds between the time when the solr slave replicated and now.

 =back

 Use '--' to separate options and argument if it starts with '-'.

 =head1 DESCRIPTION

 Simple script to run that will query the solr server for a list of defined cores
 and then verify that all of them respond to ping requests

 =head1 EXAMPLES

  solr-monitor --host qa-c1-solrmst1

  solr-monitor -H qa-c1-solrmst1 -P 80 -D 900

 =head1 AUTHOR

 Alexander V. Chykysh <[email protected]>

 Leonardo Fedalto <[email protected]>

 =cut
	The MIT License (MIT)

	Copyright (c) 2015 Leonardo Fedalto

	Permission is hereby granted, free of charge, to any person obtaining a copy
	of this software and associated documentation files (the "Software"), to deal
	in the Software without restriction, including without limitation the rights
	to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
	copies of the Software, and to permit persons to whom the Software is
	furnished to do so, subject to the following conditions:

	The above copyright notice and this permission notice shall be included in all
	copies or substantial portions of the Software.

	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
	AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
	OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	SOFTWARE.
	#!/bin/bash

	# This script will get the list af active cores and issues a ping request to them.
	# If a single request failes, the check will return ERROR
	# All requests must return OK

	# Local variables
	EXIT_CODE=0
	SOLR_HOST="localhost"
	SOLR_HOST_PORT="8983"
	# Last replication event must have been at least xxx sec ago.
	REPLICATION_TIME_TOLERANCE=900 # = 15 min
	OUT=""

	set -u

	parse_args() {
	# If no arguments are given
	if [ "$#" -eq 0 ]; then
	pod2usage "$0"
	exit 1
	fi

	# Parse arguments
	while [ $# -gt 0 ]
	do
	case "$1" in
	--help\|-h\|-\?)
	pod2usage -verbose 1 "$0"
	exit 1
	;;

	--man)
	pod2usage -verbose 2 "$0"
	exit 1
	;;

	--host\|-H)
	SOLR_HOST="$2"
	shift 2
	;;

	--port\|-P)
	SOLR_HOST_PORT="$2"
	shift 2
	;;

	--diff\|-D)
	REPLICATION_TIME_TOLERANCE="$2"
	shift 2
	;;

	*)
	# Non option argument
	break # Finish for loop
	;;
	esac
	done

	# main code
	[ -n "$SOLR_HOST" ] \|\| {
	echo "UNKNOWN: --host argument is not set"
	exit 2
	}
	}

	call_solr_api() {
	curl --retry 3 --max-time 15 --fail --silent "http://${SOLR_HOST}:${SOLR_HOST_PORT}/solr/${1}"

	if [ "$?" != "0" ]; then
	echo "CRITICAL: server \"$SOLR_HOST\" is not responding or returned incorrect data." >&2
	exit 2
	fi
	}

	get_cores_status() {
	call_solr_api "admin/cores"
	}

	get_replication_details() {
	call_solr_api "${1}/replication?command=details"
	}

	ping_solr_core() {
	call_solr_api "${1}/admin/ping"
	}

	get_solr_cores() {
	local response=$(<<< "$1" xmllint --format - \| grep '<str name="name"' \| sed -e 's/[ ]*<str name="name">//g' -e 's/<\/str>//g')

	if [ -z "$response" ]; then
	echo "CRITICAL: server \"$SOLR_HOST\" returned an empty list of cores."
	exit 2
	fi

	echo "$response"
	}

	is_master() {
	[[ $(xmlstarlet sel -t -v "/response/lst[@name='details']/str[@name='isMaster']" <<< "$1") == "true" ]]
	}

	check_init_failures() {
	local cores_status="$1"
	local cores_with_init_failures=$(xmlstarlet sel -t -m "/response/lst[@name='initFailures']/str" -m "@name" -v . -n <<< "${cores_status="$1"}" \| grep -v '^$')
	for bad_core in $cores_with_init_failures; do
	local problem=$(xmlstarlet sel -t -v "/response/lst[@name='initFailures']/str[@name='$bad_core']" <<< "${cores_status="$1"}")
	OUT="${OUT}Core \"$bad_core\" could not initialize: ${problem}\n"
	EXIT_CODE=2
	done
	}

	check_master_core() {
	local core="$1"
	local result=$(ping_solr_core "$core")
	result=$(xmlstarlet sel -t -v "/response/str[@name='status']" <<< "$result")
	OUT="${OUT}Core \"${core}\" returned \"${result}\".\n"
	[ "$result" = "OK" ] \|\| EXIT_CODE=2
	}


	check_slave_core() {
	local core="$1"
	local replication_details="$2"

	#Verifying replication health is harder, so there will be several checks.
	# 1) Check if master host is defined.
	# 2) Check indexReplicatedAtList and compare it with replicationFailedAtList. This will determine if replication is working.
	# 3) Check if indexReplicatedAtList is more then 2 hours behind now(). If it is then something might be wrong.
	# 4) Check if indexVersion on master and slave match.
	# Get the URL to the corresponding master core
	SOLR_SLAVE_MASTERURL=$(xmlstarlet sel -t -v "/response/lst[@name='details']/lst[@name='slave']/str[@name='masterUrl']" <<< "$replication_details")

	# Get the master index version
	SOLR_MASTER_INDEXVERSION=$(xmlstarlet sel -t -v "/response/lst[@name='details']/lst[@name='slave']/lst[@name='masterDetails']/long[@name='indexVersion']" <<< "$replication_details")

	# Get slave index version
	SOLR_SLAVE_INDEXVERSION=$(xmlstarlet sel -t -v "/response/lst[@name='details']/long[@name='indexVersion']" <<< "$replication_details")

	# Get the last time the core replicated correctly.
	SOLR_SLAVE_REPLICATEDAT=$(xmlstarlet sel -t -v "/response/lst[@name='details']/lst[@name='slave']/arr[@name='indexReplicatedAtList']/str[1]" <<< "$replication_details")

	# Get the last time the core failed to replicate.
	SOLR_SLAVE_FAILEDAT=$(xmlstarlet sel -t -v "/response/lst[@name='details']/lst[@name='slave']/arr[@name='replicationFailedAtList']/str[1]" <<< "$replication_details")

	# Is this core replicating (aka pulling index from master) right now?
	SOLR_SLAVE_REPLICATING=$(xmlstarlet sel -t -v "/response/lst[@name='details']/lst[@name='slave']/str[@name='isReplicating']" <<< "$replication_details")

	# masterUrl is not set, replication is broken.
	if [ -z "$SOLR_SLAVE_MASTERURL" ]; then
	OUT="$OUT\n"."Core \"${core}\" has no masterUrl set, solr.xml misconfiguration or orphaned core."
	EXIT_CODE=2
	continue
	fi

	# Slave could not get the master index version. Maybe the master is down or network issues.
	if [[ -z "$SOLR_MASTER_INDEXVERSION" ]]; then
	OUT="${OUT}Core \"$core\" could not get master index version.\n"
	EXIT_CODE=2
	continue
	fi

	# If SOLR_SLAVE_REPLICATEDAT is empty then the instance has not replicated once, this is an error.
	# Actually it's possible that this check will cause a false alert if the index is being replicated for the first time just now, so check for this also.
	if [ -z "$SOLR_SLAVE_FAILEDAT" -a -z "$SOLR_SLAVE_REPLICATEDAT" -a "$SOLR_SLAVE_REPLICATING" == "true" ]; then
	# Everything is ok, this is the first time this core has been triggered to replicate.
	OUT="${OUT}Core \"${core}\" is replicating for the first time.\n"
	continue
	fi

	if [ \( -n "$SOLR_SLAVE_FAILEDAT" -a -z "$SOLR_SLAVE_REPLICATEDAT" \) -o \( -z "$SOLR_SLAVE_FAILEDAT" -a -z "$SOLR_SLAVE_REPLICATEDAT" \) ]; then
	# This core has never replicated, this is an error.
	OUT="${OUT}Core \"${core}\" has problems replicating.\n"
	EXIT_CODE=2
	continue
	fi

	# We need to calculate if the last replication attempt was successfull.
	LAST_REPLICATION_SUCCESSFUL=`date -d "$SOLR_SLAVE_REPLICATEDAT" +%s`
	TOLERANCE=`date -d "now - $REPLICATION_TIME_TOLERANCE seconds" +%s`

	# Verify that either the timestamp is not older then REPLICATION_TIME_TOLERANCE seconds and if it is check that indexversions match.
	if [ $LAST_REPLICATION_SUCCESSFUL -gt $TOLERANCE -o $SOLR_MASTER_INDEXVERSION = $SOLR_SLAVE_INDEXVERSION ]; then
	# Everything is ok
	OUT="${OUT}Core \"${core}\" is up to date.\n"
	else
	# Slave if outdated.
	OUT="${OUT}Core \"${core}\" is behind master more then $REPLICATION_TIME_TOLERANCE seconds.\n"
	EXIT_CODE=2
	fi
	}

	main() {
	parse_args "$@"

	local cores_status=$(get_cores_status)

	local solr_cores=$(get_solr_cores "$cores_status")

	check_init_failures "$cores_status"

	for core in $solr_cores; do
	local replication_details=$(get_replication_details "$core")

	if is_master "$replication_details"; then
	check_master_core "$core"
	else
	check_slave_core "$core" "$replication_details"
	fi
	done

	echo -e -n "$OUT"
	exit $EXIT_CODE
	}

	main "$@"

	__END__

	=pod

	=head1 NAME

	solr-monitor - Simple script to check the status of all defined cores on a solr server

	=head1 SYNOPSIS

	solr-monitor [OPTIONS]

	=head1 OPTIONS

	=over 4

	=item B<--help> \| B<-h>

	Print the brief help message and exit.

	=item B<--man>

	Print the manual page and exit.

	=item B<--host> \| B<-H> HOST

	Check this host instead of localhost.

	=item B<--port> \| B<-P> Port

	Use this port instead of the default(8983) to connect.

	=item B<--diff> \| B<-D> Time difference between now and when solr last replicated

	Use this option to set the maximum difference in seconds between the time when the solr slave replicated and now.

	=back

	Use '--' to separate options and argument if it starts with '-'.

	=head1 DESCRIPTION

	Simple script to run that will query the solr server for a list of defined cores
	and then verify that all of them respond to ping requests

	=head1 EXAMPLES

	solr-monitor --host qa-c1-solrmst1

	solr-monitor -H qa-c1-solrmst1 -P 80 -D 900

	=head1 AUTHOR

	Alexander V. Chykysh <[email protected]>

	Leonardo Fedalto <[email protected]>

	=cut