prometheus and grafana setup example

clear_prometheus_data.sh

#!/bin/bash
# example:
# clear data more than 15 days
# ./clear_prometheus_data.sh https://localhost job_example 1296000


TIME=`date '+%s'`
TIME=`expr $TIME - $3`

BASE_DOMAIN=$1
JOB_NAME=$2

curl --location -g --request POST "$BASE_DOMAIN/api/v1/admin/tsdb/delete_series?end=$TIME&match[]={job=~'$JOB_NAME'}"
curl --location --request POST "$BASE_DOMAIN/api/v1/admin/tsdb/clean_tombstones"

docker-compose.monitoring.yml

version: '3.7'

services:
  prometheus:
    image: prom/prometheus:v2.21.0
    restart: unless-stopped
    user: root
    ports:
      - 9000:9090
    volumes:
      - ./prometheus:/etc/prometheus
      - ./prometheus/prometheus-data:/prometheus
    command: --web.enable-admin-api --web.enable-lifecycle  --config.file=/etc/prometheus/prometheus.yml
    
  grafana:
    image: grafana/grafana:7.5.7
    user: root
    ports:
      - 3000:3000
    restart: unless-stopped
    volumes:
      - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources
      - ./grafana/grafana-data:/var/lib/grafana
      - ./grafana/dashboards:/var/lib/grafana/dashboards

node-example.yml

- targets: ['monitoring:9100']
  labels:
   alias: 'monitoring'
   instance: monitoring

prometheus-example.yml

global:
  scrape_interval: 30s
  scrape_timeout: 10s

scrape_configs:

  - job_name: node
    file_sd_configs:
      - files:
        - /etc/prometheus/targets.d/node.yml