Undertow scripts

solr.conf

solr.undertow: {
  httpClusterPort: 8983
  # Shutdown options
  shutdown: {
     httpPort: 9983
     httpHost: "0.0.0.0"
     # if password is not set, shutdown is not enabled.  Make this a secure password!
     password: "THESCRETPASSWORD"
     gracefulDelay: "30s"
  }
  solrHome: "../../solr"
  solrLogs: "../../logs"
  tempDir: "../solr-temp"
  solrVersion: "5.2.1"
  solrWarFile: ../solr-wars/solr-${solr.undertow.solrVersion}.war
}

startSolr.sh

#!/bin/bash

. ${HOME}/config/undertow.environment

PID=`ps aux | grep -v grep | grep org.bremeld.solr.undertow.UndertowPackage | awk '{print $2}'`

if [[ "" ==  "$PID" ]]
then
	echo "**** Staring Solr server"
	$APP_HOME/undertow/bin/solr-undertow $APP_HOME/undertow/conf/solr.conf > /dev/null 2>&1 &
	echo "**** Solr server started - check log files for startup"    
else
	echo "**** Solr Server is running with pid=$PID"
fi

stopSolr.sh

#!/bin/bash

PID=`ps aux | grep -v grep | grep org.bremeld.solr.undertow.UndertowPackage | awk '{print $2}'`

if [[ "" !=  "$PID" ]]
then
    echo "Stopping Solr Server with pid=$PID"
    
    SHUTDOWN_RESPONSE=$(curl --write-out %{http_code} --silent --output /dev/null 'http://localhost:9983?password=THESCRETPASSWORD')
    
    if [ $SHUTDOWN_RESPONSE -eq 200 ]
    then
        echo "Graceful Solr server shutdown successful"
        exit
    elif  [ $SHUTDOWN_RESPONSE -eq 401 ]
    then
        echo "Graceful Solr server shutdown unauthorised"
        exit
    else
        echo "Graceful Solr server shutdown failed with status=$SHUTDOWN_RESPONSE, attempting kill"
    fi

    # Number of seconds to wait before using "kill -9"
    WAIT_SECONDS=10

    # Counter to keep count of how many seconds have passed
    count=0

    while kill $PID > /dev/null
    do
        # Wait for one second
        sleep 1
        # Increment the second counter
        ((count++))

        # Has the process been killed? If so, exit the loop.
        if ! ps -p $PID > /dev/null ; then
            break
        fi

        # Have we exceeded $WAIT_SECONDS? If so, kill the process with "kill -9"
        # and exit the loop
        if [ $count -gt $WAIT_SECONDS ]; then
            kill -9 $PID
            break
        fi
    done
    echo "Process has been killed after $count seconds."    
else
   echo "Solr Server is not running"
fi

undertow.environment

# User specific aliases and functions
export APP_HOME=${HOME}
export SOLR_HOME=$APP_HOME/solr

# Make sure umask is sane
umask 022
ulimit -c unlimited

# Set the path
PATH=$PATH:$JAVA_HOME/bin:$APP_HOME/bin:$APP_HOME/scripts
export PATH
export LC_ALL=C

alias btail='tail -f $APP_HOME/logs/solr.log'

# ------ UNDERTOW - JVM TUNING / SETTINGS
SOLR_UNDERTOW_OPTS="-Dsolr.undertow.solrHome=${SOLR_HOME} -Dsolr.solr.home=${SOLR_HOME} -Dsolr.data.dir=${SOLR_HOME}/data"

#### JVM TUNING OPTIONS
## Server config - enable 64 bit environment
SOLR_UNDERTOW_OPTS="$SOLR_UNDERTOW_OPTS -server -d64 -Djava.awt.headless=true"

## Garbage collection logging
SOLR_UNDERTOW_OPTS="$SOLR_UNDERTOW_OPTS -verbose:gc -Xloggc:${APP_HOME}/logs/jvm-gc.log"
SOLR_UNDERTOW_OPTS="$SOLR_UNDERTOW_OPTS -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintAdaptiveSizePolicy -XX:+PrintReferenceGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime"

## Heap memory
# We set heap min/max to same size for consistent results
# Max Heap = anticipated index size in memory + delta for new gen
# Min Heap = Max Heap to limit Hotspot optimizations
#
# -Xms3G (2GB index + 1GB eden)
# -Xmx3G
# -XX:PermSize=64m
# -XX:MaxPermSize=64m
SOLR_UNDERTOW_OPTS="$SOLR_UNDERTOW_OPTS -Xms3G -Xmx3G"

SOLR_UNDERTOW_OPTS="$SOLR_UNDERTOW_OPTS -XX:PermSize=64m -XX:MaxPermSize=64m"


## Use G1GC - Garbage First Garbage Collector (only works properly in JDK 1.8)
## With G1 we do not use young generation size refining options (-Xmn, -XX:SurvivorRatio)
##
## TOOLS:
## --- Use jstatd on the server via: jstatd -J-Djava.security.policy=config/jstatd.all.policy -J-Djava.rmi.server.hostname=10.0.0.112 -J-Djava.net.preferIPv4Stack=true  
## --- The policy file should contain one line:
## --- grant codebase "file:${java.home}/../lib/tools.jar" { permission java.security.AllPermission;};

##
## Tuning guidelines
## Look for "to-space overflow" / "to-space exhausted" and "Full GC" in G1 logs - those need to be avoided at all cost

## --- "to-space overflow" / "to-space exhausted"
## --- There are G1 region evacuation failures and occurs where no G1 regions are available to evacuate live options
## --- The region being evacuated could be an eden-/survivor or an old region and is not limited to survivor space overflows
## --- like what can occur with other Hotspot GCs
## --- Both evacuation failures are expensive events ot deal with, as any object that has been evacuated, its object references
## --- must be updated, and the region must be tenured. Any object that has not been evacuated, its object references must be
## --- self-forwarded and the region tenured in place. Those evacuation failures are usually followed by a Full GC which collects
## --- all regions and is single threaded.
SOLR_UNDERTOW_OPTS="$SOLR_UNDERTOW_OPTS -XX:+UseG1GC"

## --- "concurrent humongous allocation"
## --- Object allocations greater than or equal to 50% of G1's region size are considered a humongous allocation (humongous
## --- allocation & collection are not optimized in G1). The G1 region size is determined automatically at JVM launch time based
## --- on Java heap (-Xms). It can be overriden with XX:G1HeapRegionSize where the size must be a power of 2 ranging from 1MB - 32MB
## ---
## ----- EXAMPLE: "allocation request: 6440816 bytes ..., source: concurrent humongous allocation"
## ----- We need a region size, so that 6.4MB is less than 50% of the region size and where the region size is a power of 2 between
## ----- 1MB and 32MB:
## ----- Set -XX:G1HeapRegionSize=16m (16 MB * 50% = 8MB as 8MB > 6.4MB)
## ----- Note: -Xms & -Xms must be aligned with the region size (-Xms3G = 3072m/16m=192)
SOLR_UNDERTOW_OPTS="$SOLR_UNDERTOW_OPTS -XX:G1HeapRegionSize=16m"

## --- "ParNew (promotion failed)" - CMS GC promotion failures are problematic

SOLR_UNDERTOW_OPTS="$SOLR_UNDERTOW_OPTS -XX:MaxGCPauseMillis=250 -XX:InitiatingHeapOccupancyPercent=75"

## --- If marking cycles are running too frequently, look at space reclaimed in the mixed GC cycles after marking cycles completes
## --- If you are reclaiming very little space per marking cycle & mixed GC cycle, increase -XX:InitiatingHeapOccupancyPercent.
## --- Note: The more frequent the marking cycles & mixed GC cycles, you may observe higher CPU utilisation
## --- (Default is 45%)
SOLR_UNDERTOW_OPTS="$SOLR_UNDERTOW_OPTS -XX:InitiatingHeapOccupancyPercent=75"

## --- "GC pause (young)" - look for high pause times (anything over 200ms is bad)
## --- "Update RS (ms)" - look for high RSet update time
## --- When RSet update times are high, tune -XX:G1RSetUpdatingPauseTimePercent (default is 10). A lower value pushes more work onto
## --- G1 Concurrent Refinement threads. A higher value pushes more to be done during GC (stop the world) pause
## --- Tune -XX:G1ConcRefinementThreads (defaults to the value of -XX:ParallelGCThreads)
## --- Watch for change in CPU usage when changing those parameters
## ----- Add these options to observe RSet update/scan information to identify if some of the updating/coarsening of RSets is being pushed
## ----- onto the mutator (application threads)
## ----- SOLR_UNDERTOW_OPTS="$SOLR_UNDERTOW_OPTS -XX:+G1SummarizeRSetStats -XX:G1SummarizeRSetStatsPeriod=1 -XX:+UnlockDiagnosticVMOptions"
## SOLR_UNDERTOW_OPTS="$SOLR_UNDERTOW_OPTS -XX:ParallelGCThreads=NUMBER-OF-CPUs -XX:G1RSetUpdatingPauseTimePercent=10"

## --- If time from marking cycle start until mixed GCs start take a long time, increase -XX:ConcGCThreads (default ParallelGCThreads / 4)
## --- Note: Observe change/increase in CPU utilization
## SOLR_UNDERTOW_OPTS="$SOLR_UNDERTOW_OPTS -XX:ConcGCThreads=NUMBER-OF-CPUs/4"

## --- Parallel Reference Processing - look for "[Ref Proc:" or "[GC ref-proc" times or high reference processing times in remark pauses. The GC-ref proc
## --- time should be a fraction of the overall GC pause time.
## --- Enable -XX:+ParallelRefProcEnabled to address this
SOLR_UNDERTOW_OPTS="$SOLR_UNDERTOW_OPTS -XX:+ParallelRefProcEnabled"

## --- Solr specific tuning
SOLR_UNDERTOW_OPTS="$SOLR_UNDERTOW_OPTS -XX:+PerfDisableSharedMem -XX:+UseLargePages -XX:+AggressiveOpts"

# Enable JMX monitoring & New Relic monitoring
export NEW_RELIC_HOST_DISPLAY_NAME="$(/bin/hostname)"
SOLR_UNDERTOW_OPTS="$SOLR_UNDERTOW_OPTS -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9901 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false"

export SOLR_UNDERTOW_OPTS

The above JVM tuning runs perfectly fine on JDK 1.8 64bit on CentOS7 with Solr 5.2.1. We are running two SOLR cores (2GB + 400MB) on the instance with 10GB RAM and 4 CPU cores.

Heap usage is steady without any GC pauses and performance using G1GC is better than when we used JDK1.7 with CMS.

Donations are always welcome

🍺 Please support me: If the above helped you in any way, then follow me on Twitter or send me some coins:

(BTC)    36nBgsAhBBzkTvJMut851XVj47bUrdsmQx
(ETH)    0xE572b3B1187a3Ab77D72f7d6AeCd18DF26306cfC
(BAT)    0x48c65D6f768D92d4a23E4e9d25329E7De67c14d9
(LTC)    M8TNsiQWe591HTkDtLubZeftbejfPMcoUy
(Ripple) rw2ciyaNshpHe7bCHo4bRWq6pqqynnWKQg (Tag: 2478959347)
(XLM)    GDQP2KPQGKIHYJGXNUIYOMHARUARCA7DJT5FO2FFOOKY3B2WSQHG4W37 (Memo ID: 909493707)

Sign up to Cointracking which uses APIs to connect to all exchanges and helps you with tax. Use Binance Exchange to trade #altcoins. Join TradingView to get trend-reports. Sign up with Coinbase and instantly get $10 in BTC. I also accept old-school PayPal.

If you have no crypto, follow me at least on Twitter.