Install Prometheus, Prometheus Pushgateway and Grafana and add them to systemd startup

README.md

Install and use metrics with Prometheus and Grafana

Install metrics agents

Install Prometheus, Prometheus Pushgateway and Grafana. Put them behind NGINX reverse proxy, get a TLS certificate, and run the service on startup with systemd.

Do a replace all of prometheus-pushgateway.example.com and grafana.example.com with your own domains.

chmod u+x ./install_metrics_agents.sh
sudo ./install_metrics_agents.sh

Configure Grafana dashboard

Go to https://grafana.example.com, log in with admin:admin and set your new secure password.

In Connections > Data sources, Select Prometheus. In the host field, enter http://localhost:9090, and save. It should say that the connection was succesful.

Push some metrics in Node.js

Install required dependencies

pnpm i prom-client

In the entrypoint of your Node.js project, simply import metrics.mjs. Pass the host where your Prometheus Pushgateway is deployed with PROMETHEUS_GATEWAY_ENDPOINT=https://prometheus-pushgateway.example.com.

To increment the counter metric named my_metric, simply call:

MetricsUtils.my_metric.inc()

Check the prom-client repository for further documentation.

install_metrics_agents.sh

set -x
set -e

sudo mkdir -p /opt/

######################
# Install Prometheus #
######################
cd /tmp
wget https://github.com/prometheus/prometheus/releases/download/v2.47.0/prometheus-2.47.0.linux-amd64.tar.gz
tar xvfz prometheus-*.tar.gz
rm prometheus-*.tar.gz
sudo mv prometheus-*.linux-amd64 /opt/prometheus
cd /opt/prometheus
# Add Pushgateway as a target
sudo sed -i 's/- targets: \[/- targets: \["localhost:9091",/g' prometheus.yml
ll

set +x

# Run Prometheus on startup
sudo bash -c 'cat << EOF > /etc/systemd/system/prometheus.service
[Unit]
Description=Prometheus Server
Documentation=https://prometheus.io/docs/introduction/overview/
After=network-online.target

[Service]
Type=simple
ExecStart=/opt/prometheus/prometheus \
  --config.file=/opt/prometheus/prometheus.yml \
  --storage.tsdb.path=/opt/prometheus/data \
  --web.listen-address="127.0.0.1:9090" \
  --storage.tsdb.retention.time=1y

[Install]
WantedBy=multi-user.target
EOF'

set -x

##################################
# Install Prometheus Pushgateway #
##################################
cd /tmp
wget https://github.com/prometheus/pushgateway/releases/download/v1.6.1/pushgateway-1.6.1.linux-amd64.tar.gz
tar xvfz pushgateway-*.tar.gz
rm pushgateway-*.tar.gz
sudo mv pushgateway-*.linux-amd64 /opt/prometheus-pushgateway
cd /opt/prometheus-pushgateway
ll

set +x

# Run Prometheus Pushgateway on startup
sudo bash -c 'cat << EOF > /etc/systemd/system/prometheus-pushgateway.service
[Unit]
Description=Prometheus Pushgateway
Documentation=https://github.com/prometheus/pushgateway
After=network-online.target

[Service]
Type=simple
ExecStart=/opt/prometheus-pushgateway/pushgateway \
  --web.listen-address="127.0.0.1:9091" \
  --web.external-url="http://prometheus-pushgateway.example.com"

[Install]
WantedBy=multi-user.target
EOF'

set -x

# Set up NGINX reverse proxy for the public Prometheus Pushgateway endpoint
sudo sed -i 's/# server_names_hash_bucket_size 64;/server_names_hash_bucket_size 128;/g' /etc/nginx/nginx.conf
sudo bash -c 'cat << EOF > /etc/nginx/sites-available/prometheus-pushgateway.example.com
upstream prometheus-pushgateway.example.com {
    server 127.0.0.1:9091;
    keepalive 64;
}

server {
    # nginx listens to this
    listen 80;
    # uncomment the line if you want nginx to listen on IPv6 address
    #listen [::]:80;

    # the virtual host name of this
    server_name prometheus-pushgateway.example.com;

    location / {
        proxy_set_header X-Forwarded-Host \$host;
        proxy_set_header X-Forwarded-Server \$host;
        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
        proxy_pass http://prometheus-pushgateway.example.com;
        proxy_http_version 1.1;
        proxy_pass_request_headers on;
        proxy_set_header Connection "keep-alive";
        proxy_store off;
    }
}
EOF'

set -x

sudo ln -s /etc/nginx/sites-available/prometheus-pushgateway.example.com /etc/nginx/sites-enabled/prometheus-pushgateway.example.com
sudo service nginx restart
sudo certbot -d prometheus-pushgateway.example.com
sudo service nginx restart

###################
# Install Grafana #
###################
cd /tmp
wget https://dl.grafana.com/oss/release/grafana-10.1.2.linux-amd64.tar.gz
# extract and change name to "grafana-install"
tar -zxvf grafana-*.tar.gz
rm grafana-*.tar.gz
sudo mv grafana-* /opt/grafana
cd /opt/grafana
ll
sudo sed -i 's/^http_addr =.*$/http_addr = 127.0.0.1/g' conf/defaults.ini
sudo sed -i 's/^http_port =.*$/http_port = 9099/g' conf/defaults.ini
sudo sed -i 's/^enforce_domain =.*$/enforce_domain = true/g' conf/defaults.ini
sudo sed -i 's/^domain =.*$/domain = grafana.example.com/g' conf/defaults.ini

set +x

# Run Grafana on startup
sudo bash -c 'cat << EOF > /etc/systemd/system/grafana.service
[Unit]
Description=Grafana
Documentation=https://grafana.com/docs/grafana/latest/
After=network-online.target

[Service]
Type=simple
ExecStart=/opt/grafana/bin/grafana-server --config=/opt/grafana/conf/defaults.ini --homepath=/opt/grafana

[Install]
WantedBy=multi-user.target
EOF'

set -x

# Set up NGINX reverse proxy for the public Grafana endpoint
sudo bash -c 'cat << EOF > /etc/nginx/sites-available/grafana.example.com
upstream grafana.example.com {
    server 127.0.0.1:9099;
    keepalive 64;
}

server {
    # nginx listens to this
    listen 80;
    # uncomment the line if you want nginx to listen on IPv6 address
    #listen [::]:80;

    # the virtual host name of this
    server_name grafana.example.com;

    location / {
        proxy_set_header X-Forwarded-Host \$host;
        proxy_set_header X-Forwarded-Server \$host;
        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
        proxy_pass http://grafana.example.com;
        proxy_http_version 1.1;
        proxy_pass_request_headers on;
        proxy_set_header Connection "keep-alive";
        proxy_store off;
    }
}
EOF'

set -x

sudo ln -s /etc/nginx/sites-available/grafana.example.com /etc/nginx/sites-enabled/grafana.example.com
sudo service nginx restart
sudo certbot -d grafana.example.com
sudo service nginx restart

##################
# Start services #
##################
sudo systemctl daemon-reload
sudo systemctl enable prometheus
sudo systemctl start prometheus
sudo systemctl enable prometheus-pushgateway
sudo systemctl start prometheus-pushgateway
sudo systemctl enable grafana
sudo systemctl start grafana

metrics.mjs

//
// Push metrics to Prometheus
//
// Set `PROMETHEUS_GATEWAY_ENDPOINT=http://127.0.0.1:9091` with the endpoint of the Prometheus Push Gateweay
//
// See https://github.com/prometheus/exporter-toolkit/blob/master/docs/web-configuration.md
//

// @ts-check
import * as Prometheus from 'prom-client'

const CLIENT_LABEL = 'codingame-bot'

const register = new Prometheus.Registry()

if (process.env.PROMETHEUS_GATEWAY_ENDPOINT) {
  const GATEWAY_ENDPOINT = process.env.PROMETHEUS_GATEWAY_ENDPOINT

  console.log(`\n\n[Metrics] Metrics are enabled, will push them to ${GATEWAY_ENDPOINT} in the background\n\n`)

  const gateway = new Prometheus.Pushgateway(GATEWAY_ENDPOINT, [], register)

  register.setDefaultLabels({ app: CLIENT_LABEL })
  Prometheus.collectDefaultMetrics({ register })

  async function pushMetrics() {
    const res = await gateway.push({ jobName: CLIENT_LABEL })
    const resp = /** @type {any} */ (res.resp)
    if (resp?.statusCode !== 200 && resp?.statusCode !== 204) {
      throw new Error(`[Metrics] Bad response status code from gateway ${resp.statusCode}`)
    }
    // console.log('[Metrics] Pushed metrics to gateway')
  }

  pushMetrics()
    .catch(err => {
      throw new Error(
        '[Metrics] Error on first push of metrics to gateway, exiting program ',
        // @ts-ignore
        { cause: err }
      )
    })
    .then(() => {
      // Start pushing metrics on a regular basis in the background
      setInterval(() => {
        pushMetrics().catch(err => console.error('[Metrics] Error pushing metrics to gateway', err))
      }, 5_000)
    })
} else {
  console.log(
    '\n\n[Metrics] Metrics are disabled, set `PROMETHEUS_GATEWAY_ENDPOINT` in your environment to enable them\n\n'
  )
}

export const MetricsUtils = {
  my_metric: new Prometheus.Counter({
    name: 'my_metric',
    help: 'Count of times we pushed a my_metric',
    registers: [register],
  }),
}