Bash script to monitor Oracle CDC Lag Time on StreamSets DataOps Platform

get-oracle-cdc-metrics-dataops.sh

#!/usr/bin/env bash

# DataOps Platform URL
export SCH_URL=https://na01.hub.streamsets.com

# SDC URL - The SDC where the Job is running
export SDC_URL=http://<host>:<port>

# CRED_ID -- Your API Credential CRED_ID.  
export CRED_ID=<redacted>

# CRED_TOKEN -- Your API Credential CRED_TOKEN
export CRED_TOKEN=<redacted>

# Job ID for the  Job running the Oracle CDC pipeline
export JOB_ID=

# SLEEP_SECONDS (how long to sleep between calls to get the lag time)
export SLEEP_SECONDS=5

# get the SDC Pipeline ID for the given JOB ID:
export PIPELINE_ID=`curl -X GET "${SCH_URL}/jobrunner/rest/v1/job/${JOB_ID}" --header "Content-Type:application/json" -H "X-Requested-By:SDC" -H "X-SS-REST-CALL:true" -H "X-SS-App-Component-Id: ${CRED_ID}" -H "X-SS-App-Auth-Token: ${CRED_TOKEN}"   -s | jq '[.currentJobStatus][0] | .pipelineStatus[] | .name' | tr -d '"'`
echo ''
echo '----------'
echo 'Getting Oracle CDC metrics for:'
echo 'Control Hub Job ID: ' ${JOB_ID}
echo 'SDC Pipeline ID:' ${PIPELINE_ID}
echo 'SDC URL: ' ${SDC_URL}
echo '----------'

while [ true ]
do

    # Call SDC REST API to get the metrics
    METRICS=`curl -X GET "${SDC_URL}/rest/v1/pipeline/${PIPELINE_ID}/metrics?rev=0" --header "Content-Type:application/json" --header "X-Requested-By:SDC" --header "X-SS-REST-CALL:true" --header "X-SS-User-Auth-Token:$authToken" -s`

    # Uncomment this line to see the full response from the metrics request:
    # echo $METRICS
    
    ## GET READ LAG ###############################

    # Extract "Read lag (seconds)" metric
    READ_LAG=`jq '.gauges | ."custom.OracleCDCClient_1.Work State A: RedoLog Archives.0.gauge" | .value | ."Read lag (seconds)"' <<< "${METRICS}"`

    # Trim the quotes
    READ_LAG=`echo $READ_LAG | tr -d '"'`

    echo `date`  ": Oracle CDC Read lag (seconds): " ${READ_LAG}
    
    ## GET CURRENT WINDOW ###############################
    
    # Extract "RedoLog start time" metric
    CURRENT_WINDOW_START_TIME=`jq '.gauges | ."custom.OracleCDCClient_1.Position A: Current Window.0.gauge" | .value | ."RedoLog start time"' <<< "${METRICS}"`

    # Extract "RedoLog end time" metric
    CURRENT_WINDOW_END_TIME=`jq '.gauges | ."custom.OracleCDCClient_1.Position A: Current Window.0.gauge" | .value | ."RedoLog end time"' <<< "${METRICS}"`

    # Trim the quotes
    CURRENT_WINDOW_START_TIME=`echo $CURRENT_WINDOW_START_TIME | tr -d '"'`
    CURRENT_WINDOW_END_TIME=`echo $CURRENT_WINDOW_END_TIME | tr -d '"'`

    echo `date`  ": Current Window Start Time: " ${CURRENT_WINDOW_START_TIME}
    echo `date`  ": Current Window End Time: " ${CURRENT_WINDOW_END_TIME}
    echo ""
    
    sleep ${SLEEP_SECONDS}
  
done