Mnfst Puppet Helper Scripts

README.md

Scripts which use the Puppet dashboard data and update the mnfst-neo graph information.

Prereqisites are curl, awk, jq and csvtool to be available in the system.

Usage:

• Make sure to set the DASHBOARD_HOST and MNFST_HOST environment variables
• Trigger ./run.sh NO once every 10-30 minutes (depending on the assumed change rate)
• Trigger ./run.sh YES once or twice a day to import new server / file information for each system and remove relations which are not relevant anymore.

import_tsv.sh

#!/bin/bash -ex

export THEHOST=${1/data_/}
export MNFST_HOST=${MNFST_HOST:-127.0.0.1:3000}

curl -sS -X DELETE "http://${MNFST_HOST}/server/${THEHOST}/files"
curl -sS -X PUT --header "Content-Type: text/csv"  --data-binary "@./data_${THEHOST}.tsv" "http://${MNFST_HOST}/server/${THEHOST}"

obsoleteServers.sh

#!/bin/bash -ex

export MNFST_HOST=${MNFST_HOST:-127.0.0.1:3000}

awk 'BEGIN{FS=OFS="\t"} {print $1}' | sort -u > listActual
curl -sS http://${MNFST_HOST}/server | jq -r ".data[] | .[0].data.fqdn" | sort -u >  listFull

for fqdn in $(comm -13 listActual listFull); do
    curl -sS -X DELETE "http://${MNFST_HOST}/server/${fqdn}"
done

run.sh

#!/bin/bash

export FULL_IMPORT=${1:-NO}
export DASHBOARD_HOST=${DASHBOARD_HOST:-puppet-dashboard.local}

curl -sSL "http://${DASHBOARD_HOST}/nodes.csv" > nodes.csv
csvtool -t , -u TAB cat nodes.csv > nodes.tsv
list=$( cat nodes.csv | cut -f1 -d, | tail -n +2 | sort -u)

if [ "${FULL_IMPORT}" == "YES" ]; then
  cat nodes.tsv | ./obsoleteServers.sh
  cat nodes.tsv | ./transformDataFile.sh
  rm data_name.tsv
  for F in data_*; do ./import_tsv.sh "${F/.tsv/}"; done
  
  ./weightServers.sh
else 
  cat nodes.tsv | ./transformChangeFile.sh
  cat nodes.tsv | ./transformFailuresFile.sh
  
  for H in $list; do
  	./update_tsv.sh "${H/.tsv/}"
  done
fi

transformChangeFile.sh

#!/bin/bash

awk 'BEGIN{FS=OFS="\t"} $13=="1" {print $7 "[" $8 "]" >> "changes_" $1 ".tsv"}'

transformDataFile.sh

#!/bin/bash

awk 'BEGIN{FS=OFS="\t"} {gsub(/\/etc\/puppet\//,""); print $10 "\t" $7"["$8"]" > "data_" $1 ".tsv"} '

transformFailuresFile.sh

#!/bin/bash

awk 'BEGIN{FS=OFS="\t"} $16=="true" {print $7 "[" $8 "]" >> "failures_" $1 ".tsv"}'
jq

update_tsv.sh

#!/bin/bash -ex

export THEHOST=${1/changes_/}
export MNFST_HOST=${MNFST_HOST:-127.0.0.1:3000}

if [ -f "./changes_${THEHOST}.tsv" ]; then
 CHANGE_LIST=$(cat "./changes_${THEHOST}.tsv" | sed '$!s/$/","/' | tr -d '\n')
fi

if [ -f "./failures_${THEHOST}.tsv" ]; then
 FAILURE_LIST=$(cat "./failures_${THEHOST}.tsv" | sed '$!s/$/","/' | tr -d '\n')
fi

if [[ -z "${CHANGE_LIST}${FAILURE_LIST}"  ]]; then
  PAYLOAD='{"changes":[], "failures":[]}'
elif [[ -z "${CHANGE_LIST}" ]]; then
  PAYLOAD='{"changes":[], "failures":["'${FAILURE_LIST}'"]}'
elif [[ -z "${FAILURE_LIST}" ]]; then
  PAYLOAD='{"changes":["'${CHANGE_LIST}'"], "failures":[]}'
else
  PAYLOAD='{"changes":["'${CHANGE_LIST}'"], "failures":["'${FAILURE_LIST}'"]}'
fi

curl -sS -X POST --header "Content-Type: application/json"  -d "$PAYLOAD" "http://${MNFST_HOST}/server/${THEHOST}/rates"

weights.json

[
  {
    "pattern": ".system.production.local",
    "value": 100
  },
  {
    "pattern": ".system.staging.local",
    "value": 20
  },
  {
    "pattern": ".system.testing.local",
    "value": 5
  }
]

weightServers.sh

#!/bin/bash

export MNFST_HOST=${MNFST_HOST:-127.0.0.1:3000}

list=$(curl -sS http://${MNFST_HOST}/server | jq -r ".data[] | .[0].data.fqdn" | sort -u)

for p in $(cat weights.json| jq -c ".[]"); do
    pattern=$(echo "$p" | jq -r .pattern);

    updates=$(echo "$list" | grep $pattern)

    for s in $updates; do
        weight=$(echo "$p" | jq -c "{weight: .value}")
        echo "$s => $weight"
        curl -s -X POST -H 'Content-Type:application/json' \
            -d "$weight" "http://${MNFST_HOST}/server/${s}" >/dev/null
    done
done