Bash script to monitor Oracle CDC Lag Time on StreamSets Control Hub 3.x

get-oracle-cdc-metrics-sch3.sh

#!/usr/bin/env bash

# Control Hub 3.x URL
export SCH_URL=https://cloud.streamsets.com

# SDC URL - The SDC where the Job is running
export SDC_URL=https://<sdc-host>:<sdc-port>

# Control Hub User in the form <user>@<org>
export SCH_USER=

# Control Hub Password
export SCH_PASSWORD=

# Job ID for the  Job running the Oracle CDC pipeline
export JOB_ID=

# SLEEP_SECONDS (how long to sleep between calls to get the lag time)
export SLEEP_SECONDS=30

echo '----------'
echo 'Getting Control Hub Auth Token'

# Get a Control Hub Login Token
curl -X POST -d "{\"userName\": \"${SCH_USER}\", \"password\": \"${SCH_PASSWORD}\"}" ${SCH_URL}/security/public-rest/v1/authentication/login --header "Content-Type:application/json" --header "X-Requested-By:SDC" -c authToken.txt

# Extract the auth token  
authToken=$(cat authToken.txt | grep SSO | rev | grep -o '^\S*' | rev)

# Delete the authToken.txt file
rm authToken.txt

# get the SDC Pipeline ID for the given JOB ID:
export PIPELINE_ID=`curl -X GET "${SCH_URL}/jobrunner/rest/v1/job/${JOB_ID}" --header "Content-Type:application/json" --header "X-Requested-By:SDC" --header "X-SS-REST-CALL:true" --header "X-SS-User-Auth-Token:$authToken"   -s | jq '[.currentJobStatus][0] | .pipelineStatus[] | .name' | tr -d '"'`
echo ''
echo '----------'
echo 'Getting Oracle CDC metrics for:'
echo 'Control Hub Job ID: ' ${JOB_ID}
echo 'SDC Pipeline ID:' ${PIPELINE_ID}
echo 'SDC URL: ' ${SDC_URL}
echo '----------'

while [ true ]
do

    # Call SDC REST API to get the metrics
    METRICS=`curl -X GET "${SDC_URL}/rest/v1/pipeline/${PIPELINE_ID}/metrics?rev=0" --header "Content-Type:application/json" --header "X-Requested-By:SDC" --header "X-SS-REST-CALL:true" --header "X-SS-User-Auth-Token:$authToken" -s`

    # Uncomment this line to see the full response from the metrics request:
    # echo $METRICS
    
    ## GET READ LAG ###############################

    # Extract "Read lag (seconds)" metric
    READ_LAG=`jq '.gauges | ."custom.OracleCDCClient_1.Work State A: RedoLog Archives.0.gauge" | .value | ."Read lag (seconds)"' <<< "${METRICS}"`

    # Trim the quotes
    READ_LAG=`echo $READ_LAG | tr -d '"'`

    echo `date`  ": Oracle CDC Read lag (seconds): " ${READ_LAG}
    
    ## GET CURRENT WINDOW ###############################
    
    # Extract "RedoLog start time" metric
    CURRENT_WINDOW_START_TIME=`jq '.gauges | ."custom.OracleCDCClient_1.Position A: Current Window.0.gauge" | .value | ."RedoLog start time"' <<< "${METRICS}"`

    # Extract "RedoLog end time" metric
    CURRENT_WINDOW_END_TIME=`jq '.gauges | ."custom.OracleCDCClient_1.Position A: Current Window.0.gauge" | .value | ."RedoLog end time"' <<< "${METRICS}"`

    # Trim the quotes
    CURRENT_WINDOW_START_TIME=`echo $CURRENT_WINDOW_START_TIME | tr -d '"'`
    CURRENT_WINDOW_END_TIME=`echo $CURRENT_WINDOW_END_TIME | tr -d '"'`

    echo `date`  ": Current Window Start Time: " ${CURRENT_WINDOW_START_TIME}
    echo `date`  ": Current Window End Time: " ${CURRENT_WINDOW_END_TIME}
    echo ""
    
    sleep ${SLEEP_SECONDS}
  
done