Simple monitoring

README.MD

To avoid permission error of loki container:

chown -R 10001:10001 /var/lib/docker/volumes/monitoring_loki_data_rules/*
chown -R 10001:10001 /var/lib/docker/volumes/monitoring_loki_data_chunks/*

docker-compose.yml

version: '3.9'

services:

  prometheus:
    image: prom/prometheus:v2.47.2
    user: ":"
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--web.console.libraries=/etc/prometheus/console_libraries'
      - '--web.console.templates=/etc/prometheus/consoles'
      - '--storage.tsdb.retention.time=60d'
      - '--web.enable-lifecycle'
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - ./prometheus_data:/prometheus
    networks:
      - monitoring
    restart: unless-stopped
  loki:
    image: grafana/loki:2.9.2
    ports:
      - "3100:3100"
    command: -config.file=/etc/loki/local-config.yaml
    volumes:
      - loki_data_rules:/loki/rules
      - loki_data_chunks:/loki/chunks
      - type: bind
        source: ./local-config.yaml
        target: /etc/loki/local-config.yaml
    networks:
      - monitoring

  grafana:
    user: ":"
    image: grafana/grafana:10.2.0
    environment:
      GF_SERVER_ROOT_URL: 'http://{{HOST_IP}}'
    ports:
      - "80:3000"
    volumes:
      - ./grafana:/var/lib/grafana
    #      - ./grafana.ini:/etc/grafana/grafana.ini
    networks:
      - monitoring
    restart: unless-stopped

networks:
  monitoring:

volumes:
  loki_data_rules:
  loki_data_chunks:

local-config.yaml

auth_enabled: false

target: all
http_prefix: ""
ballast_bytes: 0
use_buffered_logger: true
use_sync_logger: true

common:
  path_prefix: /loki
  storage:
    filesystem:
      chunks_directory: /loki/chunks
      rules_directory: /loki/rules
  replication_factor: 1
  ring:
    kvstore:
      store: inmemory

server:
  http_listen_port: 3100
  grpc_listen_port: 9095

schema_config:
  configs:
    - from: "2020-10-24"
      store: boltdb-shipper
      object_store: filesystem
      schema: v11
      index:
        prefix: index_
        period: 24h

limits_config:
  ingestion_rate_strategy: global
  ingestion_rate_mb: 4
  ingestion_burst_size_mb: 6
  max_label_name_length: 1024
  max_label_value_length: 2048
  max_label_names_per_series: 30
  reject_old_samples: true
  reject_old_samples_max_age: 1w
  creation_grace_period: 10m
  enforce_metric_name: true
  max_line_size: 0
  max_line_size_truncate: false
  increment_duplicate_timestamp: false
  max_streams_per_user: 0
  max_global_streams_per_user: 5000
  unordered_writes: true
  per_stream_rate_limit: 3145728
  per_stream_rate_limit_burst: 15728640
  max_chunks_per_query: 2000000
  max_query_series: 500
  max_query_lookback: 0s
  max_query_length: 10d1h
  max_query_parallelism: 32
  cardinality_limit: 100000
  max_streams_matchers_per_query: 1000
  max_concurrent_tail_requests: 10
  max_entries_limit_per_query: 5000
  max_cache_freshness_per_query: 1m
  max_queriers_per_tenant: 0
  query_ready_index_num_days: 0
  query_timeout: 1m
  split_queries_by_interval: 30m
  min_sharding_lookback: 0s

  deletion_mode: filter-only
  retention_period: 1d
  per_tenant_override_config: ""
  per_tenant_override_period: 10s
  allow_deletes: true
  shard_streams:
    enabled: false
    logging_enabled: false
    desired_rate: 3145728
table_manager:
  throughput_updates_disabled: false
  retention_deletes_enabled: true
  retention_period: 0s
  poll_interval: 2m0s
  creation_grace_period: 10m0s

compactor:
  working_directory: /loki/compactor
  shared_store: filesystem
  shared_store_key_prefix: index/
  compaction_interval: 10m0s
  apply_retention_interval: 0s
  retention_enabled: true
  retention_delete_delay: 15m0s
  retention_delete_worker_count: 150
  retention_table_timeout: 0s
  delete_batch_size: 70
  delete_request_cancel_period: 15m0s
  delete_max_interval: 0s
  max_compaction_parallelism: 1
  upload_parallelism: 10

prometheus.yml

global:
  scrape_interval: 10s
scrape_configs:
  - job_name: "server-explorer"
    static_configs:
      - targets: ["ip:port"]