Install Monitoring Tools with Docker

docker-compose-agent.yaml

# Using this composer in the machines that a specific app runs which opens the port for prometheus
# Using promtail collect logs and publish to external loki
version: '3.8'

networks:
  monitoring:
    driver: bridge

services:
  cadvisor:
    image: gcr.io/cadvisor/cadvisor:v0.47.2
    container_name: cadvisor
    volumes:
      - /:/rootfs:ro
      - /var/run:/var/run:rw
      - /sys:/sys:ro
      - /var/lib/docker/:/var/lib/docker:ro
      - /etc/machine-id:/etc/machine-id:ro
      - /dev/kmsg:/dev/kmsg:ro
    ports:
      - 8080:8080
    restart: always
    privileged: true
    networks:
      - monitoring
    logging:
        driver: "json-file"
        options:
            max-file: 2
            max-size: 5m

  promtail:
    image: grafana/promtail:latest
    container_name: promtail
    command: -config.file=/mnt/config/promtail-config.yaml -config.expand-env=true
    volumes:
      - vpromtail:/mnt/config
      - /var/log:/var/log
      - /var/run:/var/run:ro
      - /var/lib/docker/containers:/var/lib/docker/containers:ro
    restart: always
    networks:
      - monitoring
    logging:
        driver: "json-file"
        options:
            max-file: 2
            max-size: 5m

  node_exporter:
    image: quay.io/prometheus/node-exporter:latest
    container_name: node_exporter
    command:
      - '--path.rootfs=/host'
    pid: host
    restart: always
    volumes:
      - '/:/host:ro,rslave'
    ports:
      - 9100:9100
    networks:
      - monitoring
    logging:
        driver: "json-file"
        options:
            max-file: 2
            max-size: 5m

volumes:
  vpromtail:

docker-compose.yaml

version: '3.8'

networks:
  monitoring:
    driver: bridge

services:
  cadvisor:
    image: gcr.io/cadvisor/cadvisor:v0.47.2
    container_name: cadvisor
    volumes:
      - /:/rootfs:ro
      - /run/user/1000:/var/run:rw
      - /sys:/sys:ro
      - /home/pi/.local/share/docker/:/var/lib/docker:ro
    ports:
      - 8080:8080
    restart: unless-stopped
    networks:
      - monitoring

  loki:
    image: grafana/loki:latest
    container_name: loki
    command: -config.file=/mnt/config/loki-config.yaml
    volumes:
      - vloki:/mnt/config
    ports:
      - 3100:3100
    restart: unless-stopped
    networks:
      - monitoring

  promtail:
    image: grafana/promtail:latest
    container_name: promtail
    command: -config.file=/mnt/config/promtail-config.yaml
    volumes:
      - vpromtail:/mnt/config
      - /var/log:/var/log
      - /var/run:/var/run:ro
    restart: unless-stopped
    networks:
      - monitoring

  grafana:
    image: grafana/grafana-oss
    container_name: grafana
    volumes:
      - vgrafana:/var/lib/grafana
    ports:
      - 3000:3000
    restart: unless-stopped
    networks:
      - monitoring

  node_exporter:
    image: quay.io/prometheus/node-exporter:latest
    container_name: node_exporter
    command:
      - '--path.rootfs=/host'
    pid: host
    restart: unless-stopped
    volumes:
      - '/:/host:ro,rslave'
    networks:
      - monitoring

  prometheus:
    image: prom/prometheus
    container_name: prometheus
    command: --config.file=/etc/prometheus/prometheus.yml
    volumes:
      - vprometheus:/etc/prometheus
    ports:
      - 9090:9090
    restart: unless-stopped
    networks:
      - monitoring

volumes:
  vloki:
  vpromtail:
  vgrafana:
  vprometheus:

promtail-config.yaml

server:
  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: /tmp/positions.yaml

clients:
  - url: http://loki:3100/loki/api/v1/push

scrape_configs:
- job_name: system
  static_configs:
  - targets:
      - localhost
    labels:
      job: varlogs
      __path__: /var/log/*log

- job_name: docker
  pipeline_stages:
    - docker: {}
  static_configs:
  - targets:
      - localhost
    labels:
      job: docker
      __path__: /var/lib/docker/containers/*/*-json.log

# if the above config for docker does not work well
- job_name: dlogs
  docker_sd_configs:
    - host: unix:///var/run/docker.sock
      refresh_interval: 5s
  pipeline_stages:
  - docker: "{{ hostname }}"
  relabel_configs:
    - source_labels: ['__meta_docker_container_name']
      regex: '/(.*)'
      target_label: 'container'

setup.sh

#!/bin/bash

# Get current dir name
script_path=$(realpath "$0")
script_dir=$(dirname "$script_path")
cbase=$(basename "$script_dir")

# Create vars
cbase_vloki="${cbase}_vloki"
cbase_vpromtail="${cbase}_vpromtail"
cbase_vprometheus="${cbase}_vprometheus"
cbase_grafana="${cbase}_grafana"

# Create volumes
docker volume create $cbase_vloki
docker volume create $cbase_vpromtail
docker volume create $cbase_vprometheus
docker volume create $cbase_grafana

echo "Volumes are created."

# Download Loki and Promtail configuration files
cd "$(docker volume inspect --format '{{ .Mountpoint }}' "$cbase_vloki")"
wget https://raw.githubusercontent.com/grafana/loki/v2.8.0/cmd/loki/loki-local-config.yaml -O loki-config.yaml
cd "$(docker volume inspect --format '{{ .Mountpoint }}' "$cbase_vpromtail")"
wget https://raw.githubusercontent.com/grafana/loki/v2.8.0/clients/cmd/promtail/promtail-docker-config.yaml -O promtail-config.yaml
echo "Downloaded loki & promtail configs."

# Create Prometheus configuration file
cd "$(docker volume inspect --format '{{ .Mountpoint }}' "$cbase_vprometheus")"
cat > prometheus.yml <<EOL
global:
  scrape_interval:     15s # By default, scrape targets every 15 seconds.
  external_labels:
    monitor: 'promon'
scrape_configs:
  - job_name: 'prometheus'
    scrape_interval: 5s
    static_configs:
      - targets: ['localhost:9090']
  - job_name: 'cadvisor'
    scrape_interval: 5s
    static_configs:
      - targets: ['cadvisor:8080']
  - job_name: 'grafana'
    scrape_interval: 5s
    static_configs:
      - targets: ['grafana:3000']
  - job_name: 'loki'
    scrape_interval: 5s
    static_configs:
      - targets: ['loki:3100']
  - job_name: 'node_exporter'
    scrape_interval: 5s
    static_configs:
      - targets: ['node_exporter:9100']
EOL
echo "Created prometheus.yml"