Installing Prometheus on Centos 7 with consul agent, node_exporter & consul_exporter

prometheus-notes.txt

cd ~/
wget https://releases.hashicorp.com/consul/1.0.2/consul_1.0.2_linux_amd64.zip
unzip consul_1.0.2_linux_amd64.zip
mv consul /usr/local/bin/
mkdir /etc/consul.d/
tee /etc/consul.d/consul.json << 'EOF'
{
  "bind_addr": "192.168.100.57",
  "datacenter": "dc1",
  "data_dir": "/var/consul",
  "log_level": "INFO",
  "enable_syslog": true,
  "enable_debug": true,
  "enable_script_checks": true,
  "node_name": "prometheus",
  "server": false,
  "checks": [ 
    { "id": "/dev/mapper/centos-root usage under 80%", "args": ["/root/check_volume_group.sh", "/dev/mapper/centos-root"], "interval": "60s", "timeout": "1s" }
  ],
  "rejoin_after_leave": true,
  "encrypt": "[consul_encryption_key]",
  "retry_join": ["192.168.100.51","192.168.100.52","192.168.100.53"]
}
EOF

sudo tee /etc/systemd/system/consul.service << 'EOF'
[Unit]
Description=Consul service discovery agent
Requires=network-online.target
After=network.target
[Service]
User=root
Group=root
PIDFile=/run/consul/consul.pid
Restart=on-failure
Environment=GOMAXPROCS=2
ExecStart=/usr/sbin/consul agent $OPTIONS -config-dir=/etc/consul.d
ExecReload=/bin/kill -s HUP $MAINPID
KillSignal=SIGINT
TimeoutStopSec=5
[Install]
WantedBy=multi-user.target
EOF

systemctl status consul.service
systemctl start consul.service
systemctl status consul.service

rm -rf ~/consul_1.0.2_linux_amd64.zip

wget https://github.com/prometheus/prometheus/releases/download/v2.0.0/prometheus-2.0.0.linux-amd64.tar.gz
tar xvfz prometheus-*.tar.gz
cd prometheus-2.0.0.linux-amd64

useradd --no-create-home --shell /bin/false prometheus
useradd --no-create-home --shell /bin/false node_exporter
useradd --no-create-home --shell /bin/false prometheus
mkdir /etc/prometheus
mkdir /var/lib/prometheus
chown prometheus:prometheus /etc/prometheus
chown prometheus:prometheus /var/lib/prometheus
cp prometheus-2.0.0.linux-amd64/prometheus /usr/local/bin/
cp prometheus-2.0.0.linux-amd64/promtool /usr/local/bin/
chown prometheus:prometheus /usr/local/bin/prometheus
chown prometheus:prometheus /usr/local/bin/promtool
cp -r prometheus-2.0.0.linux-amd64/consoles /etc/prometheus
cp -r prometheus-2.0.0.linux-amd64/console_libraries /etc/prometheus
chown -R prometheus:prometheus /etc/prometheus/consoles
chown -R prometheus:prometheus /etc/prometheus/console_libraries
chown -R prometheus:prometheus /etc/prometheus/consoles
rm -rf prometheus-2.0.0.linux-amd64.tar.gz prometheus-2.0.0.linux-amd64

tee /etc/prometheus/prometheus.yml << 'EOF'
global:
  scrape_interval: 15s

scrape_configs:
  - job_name: 'prometheus'
    scrape_interval: 5s
    static_configs:
      - targets: ['localhost:9090']
  - job_name: 'node_exporter'
    scrape_interval: 5s
    static_configs:
      - targets: ['localhost:9100']
  - job_name: 'consul_exporter'
    scrape_interval: 5s
    static_configs:
      - targets: ['localhost:9107']
EOF

chown prometheus:prometheus /etc/prometheus/prometheus.yml

tee /etc/systemd/system/prometheus.service << 'EOF'
[Unit]
Description=Prometheus
Wants=network-online.target
After=network-online.target

[Service]
User=prometheus
Group=prometheus
Type=simple
ExecStart=/usr/local/bin/prometheus \
    --config.file /etc/prometheus/prometheus.yml \
    --storage.tsdb.path /var/lib/prometheus/ \
    --web.console.templates=/etc/prometheus/consoles \
    --web.console.libraries=/etc/prometheus/console_libraries

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl start prometheus
systemctl status prometheus
systemctl enable prometheus

cd ~/
wget https://github.com/prometheus/node_exporter/releases/download/v0.15.2/node_exporter-0.15.2.linux-amd64.tar.gz
tar xvf node_exporter-0.15.2.linux-amd64.tar.gz 
sudo cp node_exporter-0.15.2.linux-amd64/node_exporter /usr/local/bin
sudo chown node_exporter:node_exporter /usr/local/bin/node_exporter

wget https://github.com/prometheus/consul_exporter/releases/download/v0.3.0/consul_exporter-0.3.0.linux-amd64.tar.gz
tar xvf consul_exporter-0.3.0.linux-amd64.tar.gz 
sudo cp consul_exporter-0.3.0.linux-amd64/consul_exporter /usr/local/bin
sudo chown node_exporter:node_exporter /usr/local/bin/consul_exporter


tee /etc/systemd/system/node_exporter.service << 'EOF'
[Unit]
Description=Node Exporter
Wants=network-online.target
After=network-online.target

[Service]
User=node_exporter
Group=node_exporter
Type=simple
ExecStart=/usr/local/bin/node_exporter

[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl start node_exporter
systemctl status node_exporter
systemctl enable node_exporter

tee /etc/systemd/system/consul_exporter.service << 'EOF'
[Unit]
Description=Node Exporter
Wants=network-online.target
After=network-online.target

[Service]
User=node_exporter
Group=node_exporter
Type=simple
ExecStart=/usr/local/bin/consul_exporter -consul.server http://localhost:8500

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl start consul_exporter.service
systemctl status consul_exporter.service
systemctl enable consul_exporter.service

systemctl restart prometheus