Install consul agent on a VCSA 6.5 vm and setup checks for all services / volume utilization

vcsa-6.5-consul-agent-install.txt

cd ~/
wget https://releases.hashicorp.com/consul/1.0.2/consul_1.0.2_linux_amd64.zip
unzip consul_1.0.2_linux_amd64.zip
mv consul /usr/sbin/

# create script for checking services
sudo tee /root/check_service.sh << 'EOF'
#!/bin/bash
export VMWARE_PYTHON_PATH="/usr/lib/vmware/site-packages"
export VMWARE_LOG_DIR="/var/log"
export VMWARE_DATA_DIR="/storage"
export VMWARE_CFG_DIR="/etc/vmware"
/usr/bin/service-control --status $1
[[ $(/usr/bin/service-control --status $1) =~ "Running" ]] || exit 1
EOF
chmod +x /root/check_service.sh

# create script for checking volume groups utilization
sudo tee /root/check_volume_group.sh << 'EOF'
#!/bin/bash
df $1
_used=$(df $1 | tail -n 1 | awk '{ print $5 }' | sed 's/\%//')
[[ $_used -lt 80 ]] || exit 1
EOF
chmod +x /root/check_volume_group.sh

# The assumption is that the IP addres of this server vCSA vm is 192.168.100.100
# and the other consul servers are 192.168.100.51, 192.168.100.52 & 192.168.100.53
sudo tee /etc/consul.d/consul.json << 'EOF'
{
  "bind_addr": "192.168.100.100",
  "datacenter": "dc1",
  "data_dir": "/var/consul",
  "log_level": "INFO",
  "enable_syslog": true,
  "enable_debug": true,
  "enable_script_checks": true,
  "node_name": "vcsa",
  "server": false,
  "services": [
    { "id": "vcsa", "name": "vcsa", "tags": [ "primary" ], "address": "", "check": { "id": "vcsa-api", "name": "HTTP API on port 9443", "tcp": "localhost:9443", "interval": "10s", "timeout": "1s"} }
  ],
  "checks": [ 
    { "id": "/dev/sda1 usage under 80%", "args": ["/root/check_volume_group.sh", "/dev/sda1"], "interval": "60s", "timeout": "1s" }, 
    { "id": "/dev/sda3 usage under 80%", "args": ["/root/check_volume_group.sh", "/dev/sda3"], "interval": "60s", "timeout": "1s" }, 
    { "id": "/dev/mapper/core_vg-core usage under 80%", "args": ["/root/check_volume_group.sh", "/dev/mapper/core_vg-core"], "interval": "60s", "timeout": "1s" }, 
    { "id": "/dev/mapper/log_vg-log usage under 80%", "args": ["/root/check_volume_group.sh", "/dev/mapper/log_vg-log"], "interval": "60s", "timeout": "1s" }, 
    { "id": "/dev/mapper/db_vg-db  usage under 80%", "args": ["/root/check_volume_group.sh", "/dev/mapper/db_vg-db "], "interval": "60s", "timeout": "1s" }, 
    { "id": "/dev/mapper/dblog_vg-dblog usage under 80%", "args": ["/root/check_volume_group.sh", "/dev/mapper/dblog_vg-dblog"], "interval": "60s", "timeout": "1s" }, 
    { "id": "/dev/mapper/seat_vg-seat usage under 80%", "args": ["/root/check_volume_group.sh", "/dev/mapper/seat_vg-seat"], "interval": "60s", "timeout": "1s" },       
    { "id": "/dev/mapper/netdump_vg-netdump usage under 80%", "args": ["/root/check_volume_group.sh", "/dev/mapper/netdump_vg-netdump"], "interval": "60s", "timeout": "1s" },   
    { "id": "/dev/mapper/autodeploy_vg-autodeploy usage under 80%", "args": ["/root/check_volume_group.sh", "/dev/mapper/autodeploy_vg-autodeploy"], "interval": "60s", "timeout": "1s" },   
    { "id": "/dev/mapper/imagebuilder_vg-imagebuilder usage under 80%", "args": ["/root/check_volume_group.sh", "/dev/mapper/imagebuilder_vg-imagebuilder"], "interval": "60s", "timeout": "1s" },   
    { "id": "/dev/mapper/updatemgr_vg-updatemgr usage under 80%", "args": ["/root/check_volume_group.sh", "/dev/mapper/updatemgr_vg-updatemgr"], "interval": "60s", "timeout": "1s" },
    { "id": "applmgmt service status", "args": ["/root/check_service.sh", "applmgmt"], "interval": "10s", "timeout": "1s" },
    { "id": "lwsmd service status", "args": ["/root/check_service.sh", "lwsmd"], "interval": "10s", "timeout": "1s"},
    { "id": "pschealth service status", "args": ["/root/check_service.sh", "pschealth"], "interval": "10s","timeout": "1s" },
    { "id": "vmafdd service status", "args": ["/root/check_service.sh", "vmafdd"], "interval": "10s", "timeout": "1s" },
    { "id": "vmcad service status", "args": ["/root/check_service.sh", "vmcad"], "interval": "10s", "timeout": "1s"},
    { "id": "vmdird service status", "args": ["/root/check_service.sh", "vmdird"], "interval": "10s", "timeout": "1s" },
    { "id": "vmdnsd service status", "args": ["/root/check_service.sh", "vmdnsd"], "interval": "10s", "timeout": "1s" },
    { "id": "vmonapi service status", "args": ["/root/check_service.sh", "vmonapi"], "interval": "10s", "timeout": "1s" },
    { "id": "vmware-cis-license service status", "args": ["/root/check_service.sh", "vmware-cis-license"], "interval": "10s", "timeout": "1s" },
    { "id": "vmware-cm service status", "args": ["/root/check_service.sh", "vmware-cm"], "interval": "10s","timeout": "1s" },
    { "id": "vmware-content-library service status", "args": ["/root/check_service.sh", "vmware-content-library"], "interval": "10s", "timeout": "1s" },
    { "id": "vmware-eam service status", "args": ["/root/check_service.sh", "vmware-eam"], "interval": "10s", "timeout": "1s" },
    { "id": "vmware-perfcharts service status", "args": ["/root/check_service.sh", "vmware-perfcharts"], "interval": "10s", "timeout": "1s" },
    { "id": "vmware-psc-client service status", "args": ["/root/check_service.sh", "vmware-psc-client"], "interval": "10s", "timeout": "1s" },
    { "id": "vmware-rhttpproxy service status", "args": ["/root/check_service.sh", "vmware-rhttpproxy"], "interval": "10s", "timeout": "1s" },
    { "id": "vmware-sca service status", "args": ["/root/check_service.sh", "vmware-sca"], "interval": "10s", "timeout": "1s" },
    { "id": "vmware-sps service status", "args": ["/root/check_service.sh", "vmware-sps"], "interval": "10s", "timeout": "1s" },
    { "id": "vmware-statsmonitor service status", "args": ["/root/check_service.sh", "vmware-statsmonitor"], "interval": "10s", "timeout": "1s" },
    { "id": "vmware-sts-idmd service status", "args": ["/root/check_service.sh", "vmware-sts-idmd"], "interval": "10s", "timeout": "1s" },
    { "id": "vmware-stsd service status", "args": ["/root/check_service.sh", "vmware-stsd"], "interval": "10s", "timeout": "1s" },
    { "id": "vmware-updatemgr service status", "args": ["/root/check_service.sh", "vmware-updatemgr"], "interval": "10s", "timeout": "1s" },
    { "id": "vmware-vapi-endpoint service status", "args": ["/root/check_service.sh","vmware-vapi-endpoint"], "interval": "10s", "timeout": "1s" },
    { "id": "vmware-vmon service status", "args": ["/root/check_service.sh", "vmware-vmon"], "interval": "10s", "timeout": "1s" },
    { "id": "vmware-vpostgres service status", "args": ["/root/check_service.sh", "vmware-vpostgres"], "interval": "10s", "timeout": "1s" },
    { "id": "vmware-vpxd service status", "args": ["/root/check_service.sh", "vmware-vpxd"], "interval": "10s", "timeout": "1s" },
    { "id": "vmware-vpxd-svcs service status", "args": ["/root/check_service.sh", "vmware-vpxd-svcs"], "interval": "10s", "timeout": "1s" },
    { "id": "vmware-vsan-health service status", "args": ["/root/check_service.sh", "vmware-vsan-health"], "interval": "10s", "timeout": "1s" },
    { "id": "vmware-vsm service status", "args": ["/root/check_service.sh", "vmware-vsm"], "interval": "10s", "timeout": "1s" },
    { "id": "vsphere-client service status", "args": ["/root/check_service.sh", "vsphere-client"], "interval": "10s", "timeout": "1s" },
    { "id": "vsphere-ui service status", "args": ["/root/check_service.sh", "vsphere-ui"], "interval": "10s", "timeout": "1s" }
  ],
  "rejoin_after_leave": true,
  "encrypt": "[output of consul-a 'consul keygen' command]",
  "retry_join": ["192.168.100.51","192.168.100.52","192.168.100.53"]
}
EOF

sudo tee /etc/systemd/system/consul.service << 'EOF'
[Unit]
Description=Consul service discovery agent
Requires=network-online.target
After=network.target

[Service]
User=root
Group=root
PIDFile=/run/consul/consul.pid
Restart=on-failure
Environment=GOMAXPROCS=2
ExecStart=/usr/sbin/consul agent $OPTIONS -config-dir=/etc/consul.d
ExecReload=/bin/kill -s HUP $MAINPID
KillSignal=SIGINT
TimeoutStopSec=5

[Install]
WantedBy=multi-user.target
EOF

systemctl start consul.service
systemctl enable consul.service