Shell script to pull CMON metrics from Joyent Triton (https://www.joyent.com/triton)

cmon.sh

#!/bin/bash

# shellcheck disable=SC2154
if [[ -n "$TRACE" ]]; then
    export PS4='[\D{%FT%TZ}] ${BASH_SOURCE}:${LINENO}: ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'
    set -o xtrace
fi

export PATH=/opt/pkg/bin:/opt/local/bin:/opt/custom/bin:$PATH

call() {
    curl --connect-timeout 10 $insecure "${header:--i}" -s --compressed --cert "$cert" --key "$key" "https://${1}/${2}"
    exit $?
}

do_genconfig() {
    local datacenters
    local dc_names

    if [[ -n $gz ]]; then
        printf 'Prometheus does not support discovery of CNs.\n'
        # shellcheck disable=SC2016
        printf 'Use file_sd and `cmon -g targets` instead.\n'
        exit 1
    fi

    datacenters=$(get_datacenters)
    # shellcheck disable=SC2207
    dc_names=( $(get_dcnames) )

    cat << EOF
global:
  scrape_interval:     10s
  evaluation_interval: 8s
  # scrape_timeout is set to the global default 10s

scrape_configs:
- job_name: prometheus
  scrape_interval: 5s
  static_configs:
    - targets: ['localhost:9090']

EOF

    for dc in "${dc_names[@]}"; do
        dc_url="$(json -Ma -c "this.key==\"$dc\"" value <<< "$datacenters")"
        print_prom_job "$dc" "$dc_url"
    done
}

do_keys() {
    keydir=$(get_cmonkeydir)
    mkdir -p ~/.triton/cmon/"${keydir}"
    cd ~/.triton/cmon/"${keydir}" || return
    export SDC_ACCOUNT="$user"
    triton -p "${profile}" -a "$SDC_ACCOUNT" profile cmon-certgen -y
}

do_targets() {
    local dcname
    local kind='cns'
    local objs
    dcname="$(get_dcname)"

    if [[ -z $gz ]]; then
        printf 'Prometheus supports triton_sd for containers directly.\n'
        # shellcheck disable=2016
        printf 'Use `cmon genconfig` instead.\n'
        exit 1
    fi

    # shellcheck disable=2207
    objs=( $(call "${socket:?}" "v1/${gz}discover" | json -Ha "$kind" | json -a -o jsony-0) )
    # shellcheck disable=2207
    targets=(
        $(for o in "${objs[@]}"; do
            read -r -a u <<< "$(json -ga server_uuid server_hostname <<< "$o")"
            printf '{"targets":'
            printf '["%s.%s"],' "${u[0]}" "$socket"
            printf '"labels":'
            printf '{"server_uuid":"%s","hostname":"%s","datacenter":"%s"' "${u[0]}" "${u[1]}" "$dcname"
            printf '}}\n'
        done)
    )

    t=$(join , "${targets[@]}")

    printf '[%s]' "$t" | json
}


get_cmonkeydir () {
    local url
    profile_j=$(triton -p "$profile" profile get -j)
    url=$(json url <<< "$profile_j" | sed -E 's/^https?//g' | tr -d :/ | tr '.' _)
    echo "${user}@${url}"
}

get_datacenters() {
    triton -p "$profile" datacenters -j
}

get_dcname() {
    cut -d '.' -f 2 <<< "$socket"
}

get_dcnames() {
    local datacenters
    datacenters="$(get_datacenters)"
    json -Ma key <<< "$datacenters"
}

get_profile() {
    triton -p "$profile" profile get -j
}

function join {
    local IFS="$1"; shift; echo "$*";
}

parse_cmon_url () {
    # shellcheck disable=2207
    parts=( $(tr -d '/' <<< "$1" | tr ':' ' ') )
    echo "${parts[@]}"
}

print_prom_job () {
    local dc="$1"
    local url="$2"
    # shellcheck disable=2207
    local cmon=($(parse_cmon_url "$(triton -a "$user" -p "$profile" -U "$url" services -j | json cmon)"))
    local scheme="${cmon[0]}"
    local endpoint="${cmon[1]}"
    # Not used
    #local port="${cmon[2]}"

    cat << EOF
- job_name: ${user}_${dc}
  scheme: ${scheme}
  tls_config:
    cert_file: ${cert}
    key_file: ${key}
    ${config_insecure}
  triton_sd_configs:
    - account: ${user}
      dns_suffix: ${endpoint}
      endpoint: ${endpoint}
      version: 1
      tls_config:
        cert_file: ${cert}
        key_file: ${key}
        ${config_insecure}
  relabel_configs:
    - source_labels: [__meta_triton_machine_alias]
      target_label: alias
    - source_labels: [__meta_triton_machine_id]
      target_label: zonename
    - source_labels: [__param_datacenter_name]
      target_label: datacenter_name
      replacement: ${dc}
    - source_labels: [__param_account]
      target_label: account_name
      replacement: ${user}

EOF

}

strip_proto() {
    #sed 's#https://##' <<< "$1"
    echo "${1/*:\/\/}"
}

usage () {
    printf '%s [ -g ] [ -p profile ] [ -c certficate ] [ -k key ] targets\n' "${0##*/}"
    printf '%s [ -g ] [ -p profile ] [ -c certficate ] [ -k key ] [ -G groups ] discover\n' "${0##*/}"
    printf '%s [ -p profile ] [ -c certficate ] [ -k key ] metrics vm_uuid|server_uuid\n' "${0##*/}"
    printf '%s [ -p profile ] -u username keysetup\n' "${0##*/}"
    printf '%s [ -p profile ] -u username genconfig\n' "${0##*/}"
    printf '\n    -g\tDiscover Compute Nodes (gz metrics)\n'
    exit "$1"
}

while getopts "e:c:k:p:u:G:ghiH" options; do
    case $options in
        H) header=' ';;
        c) cert="${OPTARG}";;
        e) socket="${OPTARG}";;
        g) gz="gz/";;
        G) groups="?groups=${OPTARG}" ;;
        i)
            insecure="-k";
            config_insecure="insecure_skip_verify: true";
            ;;
        k) key="${OPTARG}";;
        p) profile="${OPTARG}";;
        u) user="${OPTARG}";;
        h) usage 0;;
        *) usage 1;;
    esac
done

shift $(( OPTIND - 1 ))

if [[ -z $user ]] || [[ -z $socket ]]; then
    # Default profile
    profile_json=$(get_profile)
fi

if [[ -n $profile ]]; then
    profile=$(json name <<< "$profile_json")
fi

if [[ -z $user ]]; then
    user=$(json account <<< "$profile_json")
fi

if [[ -z $socket ]]; then
    socket=$(triton -p "$profile" services -j | json cmon)
fi

socket=$(strip_proto "$socket")
action="$1"
inst="$2"

if [[ -z $cert ]] || [[ -z $key ]]; then
    keydir=$(get_cmonkeydir)
fi

# Use certs generated by triton profile cmon-certgen
if [[ -z $cert ]]; then
    cert=~/.triton/cmon/"${keydir}"/cmon-${user}-cert.pem
fi
if [[ -z $key ]]; then
    key=~/.triton/cmon/"${keydir}"/cmon-${user}-key.pem
fi

if [[ -z $socket ]]; then
    printf 'Profile "%s" does not list a cmon endpoint\n' "$profile"
    exit 1
fi

case $action in
    disco|discover)
        call "${socket:?}" "v1/${gz}discover${groups}"
        ;;
    metrics)
        call "${inst:?}.${socket}" metrics
        ;;
    certgen|keysetup)
        do_keys
        ;;
    targets)
        do_targets
        ;;
    genconfig)
        if [[ -z $user ]]; then
            printf 'user is required for config generation\n'
            usage 1
        fi
        do_genconfig;
        ;;
    *)
        printf 'Not implemented\n'
        ;;
esac

Discovery

$ cmon -p east1a discover | json -H 
{
  "containers": [
    {
      "server_uuid": "44454c4c-5300-1037-8044-b8c04f524432",
      "source": "Bootstrapper",
      "vm_alias": "e3e1aa6b",
      "vm_brand": "joyent",
      "vm_image_uuid": "1f32508c-e6e9-11e6-bc05-8fea9e979940",
      "vm_owner_uuid": "72804dc9-afef-4228-af55-2e973063865b",
      "vm_uuid": "e3e1aa6b-ad2c-cdc1-8b7d-df5c8bbdda70",
      "cached_date": 1508548301529
    }
  ]
}

Metrics. This works for either compute nodes or instances. It doesn't matter if the uuid passed is a server_uuid or vm_uuid.

$ cmon -p east1a metrics e3e1aa6b-ad2c-cdc1-8b7d-df5c8bbdda70 | head -15
HTTP/1.1 200 OK
Content-Type: text/plain
Content-Length: 4526
Date: Fri, 27 Oct 2017 17:54:58 GMT
Server: cmon
x-request-id: 76789960-9433-4703-a548-86abe3147538
x-response-time: 85
x-server-name: 75f44180-07b1-4992-aec3-ca9ec28d3eab
Connection: keep-alive

# HELP time_of_day System time in seconds since epoch
# TYPE time_of_day counter
time_of_day 1509126898642
# HELP net_agg_packets_in Aggregate inbound packets
# TYPE net_agg_packets_in counter

Pipe metrics to exposition for json format.

$ cmon -p east1a metrics e3e1aa6b-ad2c-cdc1-8b7d-df5c8bbdda70 | exposition | json | head -15
[
  {
    "name": "time_of_day",
    "metrics": [
      {
        "value": "1509127000839"
      }
    ],
    "help": "System time in seconds since epoch",
    "type": "COUNTER"
  },
  {
    "name": "net_agg_packets_in",
    "metrics": [
      {

Compute Node discovery for Global Zone Metrics

> cmon -i -g -c ./cmon-admin-cert.pem -k ./cmon-admin-key.pem -e https://cmon.vt-coal-1.ext.example.com:9163 discover | json
HTTP/1.1 200 OK
Content-Type: application/json
Content-Length: 523
Date: Thu, 07 May 2020 15:57:53 GMT
Server: cmon
x-request-id: 25f78b09-c3ce-4418-add0-1f8161e1e3c3
x-response-time: 83
x-server-name: f2cccad9-5684-4a29-9106-75f717c4dc1f
Connection: keep-alive

{
  "cns": [
    {
      "server_uuid": "564d2b64-8862-278f-6181-cea59c4ec30d",
      "server_hostname": "headnode"
    },
    {
      "server_uuid": "298697d2-edde-4a3e-a921-3758b569621a",
      "server_hostname": "VC298697D2"
    },
    {
      "server_uuid": "33a55038-8383-45d5-9c5a-3e44ca2bebb6",
      "server_hostname": "VC33A55038"
    },
    {
      "server_uuid": "7b2fffab-cc38-4bb0-8713-edb9e97b8aba",
      "server_hostname": "VC7B2FFFAB"
    },
    {
      "server_uuid": "8ee0a055-fb1c-429b-822a-72f8b011405f",
      "server_hostname": "VC8EE0A055"
    },
    {
      "server_uuid": "eb8adc80-4ef1-4e5f-84a0-eb1094a1746d",
      "server_hostname": "VCEB8ADC80"
    }
  ]
}

Generate a json file to be used with Prometheus' file_sd.

> cmon -i -g -c ./cmon-admin-cert.pem -k ./cmon-admin-key.pem -e https://cmon.vt-coal-1.ext.example.com:9163 targets
[
  {
    "targets": [
      "564d2b64-8862-278f-6181-cea59c4ec30d.cmon.vt-coal-1.ext.example.com:9163"
    ],
    "labels": {
      "hostname": "headnode",
      "datacenter": "vt-coal-1"
    }
  },
  {
    "targets": [
      "298697d2-edde-4a3e-a921-3758b569621a.cmon.vt-coal-1.ext.example.com:9163"
    ],
    "labels": {
      "hostname": "VC298697D2",
      "datacenter": "vt-coal-1"
    }
  },
  {
    "targets": [
      "33a55038-8383-45d5-9c5a-3e44ca2bebb6.cmon.vt-coal-1.ext.example.com:9163"
    ],
    "labels": {
      "hostname": "VC33A55038",
      "datacenter": "vt-coal-1"
    }
  },
  {
    "targets": [
      "7b2fffab-cc38-4bb0-8713-edb9e97b8aba.cmon.vt-coal-1.ext.example.com:9163"
    ],
    "labels": {
      "hostname": "VC7B2FFFAB",
      "datacenter": "vt-coal-1"
    }
  },
  {
    "targets": [
      "8ee0a055-fb1c-429b-822a-72f8b011405f.cmon.vt-coal-1.ext.example.com:9163"
    ],
    "labels": {
      "hostname": "VC8EE0A055",
      "datacenter": "vt-coal-1"
    }
  },
  {
    "targets": [
      "eb8adc80-4ef1-4e5f-84a0-eb1094a1746d.cmon.vt-coal-1.ext.example.com:9163"
    ],
    "labels": {
      "hostname": "VCEB8ADC80",
      "datacenter": "vt-coal-1"
    }
  }
]
> 

All options

% cmon -h
cmon [ -g ] [ -p profile ] [ -c certficate ] [ -k key ] targets
cmon [ -g ] [ -p profile ] [ -c certficate ] [ -k key ] [ -G groups ] discover
cmon [ -p profile ] [ -c certficate ] [ -k key ] metrics vm_uuid|server_uuid
cmon [ -p profile ] -u username keysetup
cmon [ -p profile ] -u username genconfig

    -g	Discover Compute Nodes (gz metrics)