Last active
April 23, 2024 21:14
-
-
Save CesarCapillas/a796c0e7cba10ac02213c7f3485d6e90 to your computer and use it in GitHub Desktop.
SOLR bash recipes for creating, deleting or truncating collections, monitoring and searching.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| COLLECTION=${2:-zylk} | |
| SERVER=${3:-localhost} | |
| PORT=${4:-8983} | |
| if [ -z "$1" ]; then | |
| # Usage | |
| echo 'Usage: add-by-id.sh <id> [<collection> <solr-server=localhost> <port=8383>]' | |
| else | |
| curl -X POST "http://${SERVER}:${PORT}/solr/${COLLECTION}/update?commit=true" -H "Content-Type: text/xml" --data-binary "<add><doc><field name='id'>$1</field><field name='url'>$1</field></doc></add>" | |
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| SERVER=${1:-localhost} | |
| PORT=${2:-8983} | |
| NUMSERVERS=${3:-1} | |
| if [ "$PORT" = "443" ]; then | |
| PROTOCOL="https" | |
| else | |
| PROTOCOL="http" | |
| fi | |
| ENDPOINT="$PROTOCOL://${SERVER}:${PORT}/solr/admin/collections?action=clusterstatus&wt=json" | |
| if [[ "$1" == "" ]]; then | |
| echo "Usage:" | |
| echo " check_solr.sh <SOLRSERVER=localhost> [<PORT=8983> <NUMSERVERS=1>]" | |
| exit | |
| fi | |
| CURL=`curl --silent -X GET ${ENDPOINT}` | |
| CHCK=`echo $CURL | grep "live_nodes"` | |
| if [[ "$CHCK" == "" ]]; then | |
| CHECK="Failed" | |
| else | |
| CHECK="OK" | |
| #SOLR_RES=`echo $CURL | jq ".cluster.live_nodes" | tr -d '\r\n'` | |
| SOLR_RES=`echo $CURL | jshon -e cluster | jshon -e live_nodes | tr -d '\r\n'` | |
| # tricky | |
| #SOLR_RES=`echo $CURL | jq ".cluster.live_nodes" | wc -w` | |
| SOLR_NUM=`echo $CURL | jshon -e cluster | jshon -e live_nodes | wc -w` | |
| SOLR_VAR=`expr $SOLR_NUM - 2` | |
| fi | |
| if [[ "$CHECK" == "OK" ]]; then | |
| if (($SOLR_VAR < $NUMSERVERS));then | |
| echo "CRITICAL: SOLR ($SOLR_VAR live nodes) = $SOLR_RES (<$NUMSERVERS)" | |
| exit 2 | |
| fi | |
| echo "INFO: SOLR ($SOLR_VAR live nodes) = $SOLR_RES " | |
| exit 0 | |
| elif [[ "$CHECK" == "Failed" ]]; then | |
| echo "CRITICAL: ${SERVER}" | |
| exit 2 | |
| else | |
| echo "Check failed." | |
| exit 3 | |
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| ALIAS=${1:-zylk-alias} | |
| COLLECTION=${3:-zylk} | |
| SERVER=${3:-localhost} | |
| PORT=${4:-8983} | |
| if [ -z "$1" ]; then | |
| # Usage | |
| echo 'Usage: create-alias.sh <alias-name> <collection-name-list> [<solr-server=localhost> <port=8983>]' | |
| #curl -s "http://${SERVER}:${PORT}/solr/admin/collections?action=LISTALIASES" | xmllint --format - | |
| else | |
| echo "Creating SOLR alias collection $ALIAS for ${COLLECTION}" | |
| curl -s "http://${SERVER}:${PORT}/solr/admin/collections?action=CREATEALIAS&name=${ALIAS}&collections=${COLLECTION}" | tidy -q -xml -i - | |
| fi | |
| curl -s "http://${SERVER}:${PORT}/solr/admin/collections?action=LISTALIASES" | tidy -q -xml -i - |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| COLLECTION=${1:-zylk} | |
| MYDATE=`/bin/date +%Y%m%d` | |
| MYBACK="${COLLECTION}-${MYDATE}-backup" | |
| BACKUPNAME=${2:-${MYBACK}} | |
| BACKUPDIR=${3:-/opt/solr6/backups} | |
| SERVER=${4:-localhost} | |
| PORT=${5:-8983} | |
| if [ -z "$1" ]; then | |
| # Usage | |
| echo 'Usage: create-backup.sh <collection-name> <backupname> <local-path> [<solr-server=localhost> <port=8983>]' | |
| else | |
| echo "Creating SOLR backup collection for ${COLLECTION}" | |
| curl -s "http://${SERVER}:${PORT}/solr/admin/collections?action=BACKUP&name=${BACKUPNAME}&collection=${COLLECTION}&location=${BACKUPDIR}" | |
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| #./bin/solr create -c zylk -shards 1 -replicationFactor 2 -p 8983 -d server/solr/configsets/nutchconfig-base | |
| SOLRHOME=/opt/solr6/solr-6.6.0 | |
| CONFIGSET=server/solr/configsets/nutchconfig-base | |
| COLLECTION=${1:-zylk} | |
| PORT=${2:-8983} | |
| if [ -z "$1" ]; then | |
| # Usage | |
| echo 'Usage: create-collection.sh <collection-name> [ <port=8080> ]' | |
| else | |
| echo "Creating SOLR collection ${COLLECTION}" | |
| (cd ${SOLRHOME} && ./bin/solr create -c ${COLLECTION} -shards 1 -replicationFactor 2 -p ${PORT} -d ${CONFIGSET}) | |
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| COLLECTION=${2:-zylk} | |
| SERVER=${3:-localhost} | |
| PORT=${4:-8983} | |
| if [ -z "$1" ]; then | |
| # Usage | |
| echo 'Usage: delete-by-id.sh <id> [<collection> <solr-server=localhost> <port=8983>]' | |
| else | |
| curl -X POST "http://${SERVER}:${PORT}/solr/${COLLECTION}/update?commit=true" -H "Content-Type: text/xml" --data-binary "<delete><id>$1</id></delete>" | |
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| SOLR_DIR=/opt/solr6/solr-6.6.0/ | |
| COLLECTION=${2:-gettingstarted} | |
| PORT=${3:-8983} | |
| if [ -z "$1" ]; then | |
| # Usage | |
| echo 'Usage: delete-by-id2.sh <id> [<collection>]' | |
| else | |
| cd SOLR_DIR | |
| ./bin/post -p $PORT -c $COLLECTION -d "<delete><id>$1</id></delete>" | |
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| COLLECTION=${2:-gettingstarted} | |
| SERVER=${3:-localhost} | |
| PORT=${4:-8983} | |
| if [ -z "$1" ]; then | |
| # Usage | |
| echo 'Usage: delete-by-id3.sh <id> [<collection> <solr-server=localhost> <port=8383>]' | |
| else | |
| curl -X POST "http://${SERVER}:${PORT}/solr/${COLLECTION}/update?commit=true" -H "Content-Type: application/json" --data-binary "{\"delete\": {\"id\":\"$1\"}}" | |
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #! /bin/bash | |
| ### | |
| ### Script for do SOLR querys from command line via REST API | |
| ### | |
| urlencode() { | |
| # urlencode <string> | |
| old_lc_collate=$LC_COLLATE | |
| LC_COLLATE=C | |
| local length="${#1}" | |
| for (( i = 0; i < length; i++ )); do | |
| local c="${1:i:1}" | |
| case $c in | |
| [a-zA-Z0-9.~_-]) printf "$c" ;; | |
| *) printf '%%%02X' "'$c" ;; | |
| esac | |
| done | |
| LC_COLLATE=$old_lc_collate | |
| } | |
| urldecode() { | |
| # urldecode <string> | |
| local url_encoded="${1//+/ }" | |
| printf '%b' "${url_encoded//%/\\x}" | |
| } | |
| # Usage functions | |
| usage() { echo "Usage: $0 [-t <term>] [-c <collection-name>] [-l <field-list-fl>] [-q <field-query-fq>] [-r <query-type-qt>] [-h <solr-server>] [-p <solr-port>]" 1>&2; exit 1; } | |
| # Command line options | |
| while getopts "t:c:l:q:r:h:p" o; do | |
| case "${o}" in | |
| t) | |
| TERM=${OPTARG} | |
| ;; | |
| l) | |
| FL=${OPTARG} | |
| ;; | |
| q) | |
| FQ=${OPTARG} | |
| ;; | |
| r) | |
| QT=${OPTARG} | |
| ;; | |
| c) | |
| COLLECTION=${OPTARG} | |
| ;; | |
| h) | |
| SERVER=${OPTARG} | |
| ;; | |
| p) | |
| PORT=${OPTARG} | |
| ;; | |
| \?) | |
| echo "Invalid Option: -$OPTARG" 1>&2 | |
| exit 1 | |
| ;; | |
| *) | |
| usage | |
| ;; | |
| esac | |
| done | |
| shift $((OPTIND-1)) | |
| # Needs at least COLLECTION and TERM as parameters | |
| if [ -z "${COLLECTION}" ] || [ -z "${TERM}" ] ; then | |
| usage | |
| fi | |
| # Exports SERVER,PORT | |
| #source ./exportENVARS.sh | |
| # Default parameters | |
| # Encode parameters (as blank spaces) | |
| TERM=`urlencode "$TERM"` | |
| #COLLECTION=${COLLECTION:-zylk} | |
| FL=${FL:-"url,title,lang,score"} | |
| FL=`urlencode $FL` | |
| FQ=${FQ:-"*"} | |
| QT=${QT:-select} | |
| SERVER=${SERVER:-localhost} | |
| PORT=${PORT:-8983} | |
| # Main | |
| NUMROWS=`curl -s "http://${SERVER}:${PORT}/solr/${COLLECTION}/${QT}?fq=$FQ&indent=on&q=${TERM}&wt=json" | jq '.response.numFound'` | |
| #echo "Number of results for search: $1 --> $NUMROWS" | |
| ROWS=10000 | |
| COUNT=0 | |
| START=0 | |
| while [ $START -lt $NUMROWS ]; do | |
| # Only 10 results finally | |
| #curl -s "http://${SERVER}:${PORT}/solr/${COLLECTION}/${QT}?fl=$FL&indent=on&q=$TERM&rows=$ROWS&start=$START&wt=json" | jq '.response.docs[]' | |
| echo "curl -s \"http://${SERVER}:${PORT}/solr/${COLLECTION}/${QT}?fq=$FQ&fl=$FL&indent=on&q=$TERM&rows=10&start=$START&wt=json\"" | |
| curl -s "http://${SERVER}:${PORT}/solr/${COLLECTION}/${QT}?fq=$FQ&fl=$FL&indent=on&q=$TERM&rows=10&start=$START&wt=json" | jq '.response.docs[]' | |
| COUNT=$((COUNT+1)) | |
| START=$((ROWS*COUNT)) | |
| done | |
| echo "Number of results for search: $1 --> $NUMROWS" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #! /bin/bash | |
| ### | |
| ### Script for getting indexed urls from SOLR | |
| ### | |
| # Usage functions | |
| usage() { echo "Usage: $0 [-c <collection-name>] [-h <solr-host>] [-p <solr-port>]" 1>&2; exit 1; } | |
| # Command line options | |
| while getopts "c:h:p:" o; do | |
| case "${o}" in | |
| c) | |
| COLLECTION=${OPTARG} | |
| ;; | |
| h) | |
| SERVER=${OPTARG} | |
| ;; | |
| p) | |
| PORT=${OPTARG} | |
| ;; | |
| \?) | |
| echo "Invalid Option: -$OPTARG" 1>&2 | |
| exit 1 | |
| ;; | |
| *) | |
| usage | |
| ;; | |
| esac | |
| done | |
| shift $((OPTIND-1)) | |
| # Needs at least COLLECTION as parameter | |
| if [ -z "${COLLECTION}" ]; then | |
| usage | |
| fi | |
| # Exports SERVER, PORT ? | |
| #source ./exportENVARS.sh | |
| # Default parameters | |
| #COLLECTION=${COLLECTION:-garbiker} | |
| SERVER=${SERVER:-localhost} | |
| PORT=${PORT:-8983} | |
| # Main | |
| NUMROWS=`curl -s "http://${SERVER}:${PORT}/solr/${COLLECTION}/select?indent=on&q=*:*&wt=json" | jq '.response.numFound'` | |
| echo "Number of indexed urls in $1 : $NUMROWS" | |
| ROWS=10000 | |
| COUNT=0 | |
| START=0 | |
| while [ $START -lt $NUMROWS ]; do | |
| curl -s "http://${SERVER}:${PORT}/solr/${COLLECTION}/select?fl=url&indent=on&q=*:*&rows=$ROWS&start=$START&sort=url%20desc&wt=json" | jq '.response.docs[].url' | |
| COUNT=$((COUNT+1)) | |
| START=$((ROWS*COUNT)) | |
| done |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #! /bin/bash | |
| ### | |
| ### Script for reloading SOLR collection via REST API | |
| ### | |
| # Usage functions | |
| usage() { echo "Usage: $0 [-c <collection-name>] [-h <solr-host>] [-p <solr-port>]" 1>&2; exit 1; } | |
| # Command line options | |
| while getopts "c:h:p:" o; do | |
| case "${o}" in | |
| c) | |
| COLLECTION=${OPTARG} | |
| ;; | |
| h) | |
| SERVER=${OPTARG} | |
| ;; | |
| p) | |
| PORT=${OPTARG} | |
| ;; | |
| \?) | |
| echo "Invalid Option: -$OPTARG" 1>&2 | |
| exit 1 | |
| ;; | |
| *) | |
| usage | |
| ;; | |
| esac | |
| done | |
| shift $((OPTIND-1)) | |
| # Needs at least COLLECTION as parameter | |
| if [ -z "${COLLECTION}" ]; then | |
| usage | |
| fi | |
| # Exports SERVER, PORT ? | |
| #source ./exportENVARS.sh | |
| # Default parameters | |
| #COLLECTION=${COLLECTION:-turismoberria} | |
| SERVER=${SERVER:-localhost} | |
| PORT=${PORT:-8983} | |
| # Reload | |
| curl "http://${SERVER}:${PORT}/solr/admin/collections?action=RELOAD&name=${COLLECTION}" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| ALIAS=${1:-zylk-alias} | |
| SERVER=${2:-localhost} | |
| PORT=${3:-8983} | |
| if [ -z "$1" ]; then | |
| # Usage | |
| echo 'Usage: remove-alias.sh <alias-name> [<solr-server=localhost> <port=8983>]' | |
| #curl -s "http://${SERVER}:${PORT}/solr/admin/collections?action=LISTALIASES" | xmllint --format - | |
| else | |
| echo "Removing SOLR alias collection ${ALIAS}" | |
| curl -s "http://${SERVER}:${PORT}/solr/admin/collections?action=DELETEALIAS&name=${ALIAS}" | tidy -q -xml -i - | |
| fi | |
| curl -s "http://${SERVER}:${PORT}/solr/admin/collections?action=LISTALIASES" | tidy -q -xml -i - |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| #./bin/solr delete -c zylk | |
| SOLRHOME=/opt/solr6/solr-6.6.0 | |
| COLLECTION=${1:-zylk} | |
| if [ -z "$1" ]; then | |
| # Usage | |
| echo 'Usage: remove-collection.sh <collection-name>' | |
| else | |
| echo "Removing SOLR collection ${COLLECTION}" | |
| (cd ${SOLRHOME} && ./bin/solr delete -c ${COLLECTION}) | |
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| COLLECTION=${1:-zylk-backup} | |
| BACKUPNAME=${2:-zylk-backup-name} | |
| BACKUPDIR=${3:-/opt/solr6/backups} | |
| SERVER=${4:-localhost} | |
| PORT=${5:-8983} | |
| if [ -z "$2" ]; then | |
| # Usage | |
| echo 'Usage: restore-backup.sh <collection-name> <backupname> <local-path> [<solr-server=localhost> <port=8983>]' | |
| else | |
| echo "Restoring SOLR backup collection for ${COLLECTION}" | |
| curl -s "http://${SERVER}:${PORT}/solr/admin/collections?action=RESTORE&name=${BACKUPNAME}&collection=${COLLECTION}&location=${BACKUPDIR}" | |
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #! /bin/bash | |
| ### | |
| ### Script for truncating SOLR collection via REST API | |
| ### | |
| # Usage functions | |
| usage() { echo "Usage: $0 [-c <collection-name>] [-h <solr-host>] [-p <solr-port>]" 1>&2; exit 1; } | |
| # Command line options | |
| while getopts "c:h:p:" o; do | |
| case "${o}" in | |
| c) | |
| COLLECTION=${OPTARG} | |
| ;; | |
| h) | |
| SERVER=${OPTARG} | |
| ;; | |
| p) | |
| PORT=${OPTARG} | |
| ;; | |
| \?) | |
| echo "Invalid Option: -$OPTARG" 1>&2 | |
| exit 1 | |
| ;; | |
| *) | |
| usage | |
| ;; | |
| esac | |
| done | |
| shift $((OPTIND-1)) | |
| # Needs at least COLLECTION as parameter | |
| if [ -z "${COLLECTION}" ]; then | |
| usage | |
| fi | |
| # Exports SERVER, PORT ? | |
| #source ./exportENVARS.sh | |
| # Default parameters | |
| #COLLECTION=${COLLECTION:-turismoberria} | |
| SERVER=${SERVER:-localhost} | |
| PORT=${PORT:-8983} | |
| # Truncate | |
| curl -X POST http://${SERVER}:${PORT}/solr/${COLLECTION}/update?commit=true -H "Content-Type: text/xml" --data-binary '<delete><query>*:*</query></delete>' | |
| # Reload | |
| curl "http://${SERVER}:${PORT}/solr/admin/collections?action=RELOAD&name=${COLLECTION}" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| json=$(cat <<EOF | |
| [ | |
| { | |
| "id": "$1", | |
| "url": "$1" | |
| } | |
| ] | |
| EOF | |
| ) | |
| COLLECTION=${2:-zylk} | |
| SERVER=${3:-localhost} | |
| PORT=${4:-8983} | |
| if [ -z "$1" ]; then | |
| # Usage | |
| echo 'Usage: update-json.sh <id> [<collection> <solr-server=localhost> <port=8983>]' | |
| else | |
| curl -X POST "http://${SERVER}:${PORT}/solr/${COLLECTION}/update?boost=0&commit=true" -H "Content-Type: application/json" --data-binary "$json" | |
| fi |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment