Docker Observability Stack (Loki + Alloy + Mimir + Grafana)

README.md

Docker Observability Stack (Loki + Alloy + Mimir + Grafana)

This repository contains a self-hosted observability stack built with Docker Compose, designed for production-ready single-node setups using Grafana Labs components only.

The stack provides:

• 📄 Docker container logs → Loki
• 📊 Host metrics → Node Exporter → Alloy → Mimir
• 📈 Visualization & querying → Grafana
• 🔒 Only Grafana is exposed to the host
• 🧠 No classic Prometheus instance (Mimir acts as the PromQL backend)

---

🧩 Components

Service	Purpose
Loki	Log storage and querying
Alloy	Unified agent (logs + metrics)
Mimir	Prometheus-compatible metrics backend
Node Exporter	Exposes host metrics
Grafana	Visualization UI

---

Architecture Overview

Docker containers logs
→ Alloy
→ Logs → Loki
→ Metrics → remote_write → Mimir → Grafana

Key design decisions:

• Logs are read directly from Docker json-file logs
• Metrics are scraped from node-exporter
• Alloy handles both logs and metrics
• Mimir replaces the need for a standalone Prometheus server
• Only Grafana is accessible from the host or public internet

---

Grafana Data Sources

Loki (Logs)

• Type: Loki
• URL: http://loki:3100

Mimir (Metrics)

• Type: Prometheus
• URL: http://mimir:9009/prometheus

---

Security Notes

• Only Grafana is exposed to the host
• Loki, Mimir, Alloy, and Node Exporter are internal to the Docker network
• no-new-privileges is enabled where possible
• Read-only mounts are used when applicable

---

Notes

• Healthchecks are intentionally disabled for Loki and Alloy because their images are distroless and do not include a shell or HTTP client.
• Alloy retries automatically, making explicit health gating unnecessary.

---

License

MIT

alloy-config.river

// -----------------------------------------------------------------------------
// Alloy – Docker logs (json-file) → Loki, with container_name as label
// Source of truth: discovery.docker (metadata) + build __path__
// -----------------------------------------------------------------------------

// 1) Discover containers via Docker API (docker.sock)
discovery.docker "containers" {
  host = "unix:///var/run/docker.sock"
}

// 2) Relabel: build the json-file log path and useful labels
discovery.relabel "docker_targets" {
  targets = discovery.docker.containers.targets

  // Build the __path__ to the json log of the container:
  // /var/lib/docker/containers/<id>/<id>-json.log
  rule {
    source_labels = ["__meta_docker_container_id"]
    target_label  = "__path__"
    replacement   = "/var/lib/docker/containers/$1/$1-json.log"
  }

  // fixed job
  rule {
    target_label = "job"
    replacement  = "docker"
  }

  // container_name without the initial slash
  rule {
    source_labels = ["__meta_docker_container_name"]
    regex         = "/(.*)"
    target_label  = "container_name"
    replacement   = "$1"
  }

  // (Optional) image as label
  // rule {
  //   source_labels = ["__meta_docker_container_image"]
  //   target_label  = "image"
  // }

  // Host
  rule {
    target_label = "host"
    replacement  = constants.hostname
  }
}

// 3) Tail files (targets already have container_name)
loki.source.file "docker_files" {
  targets    = discovery.relabel.docker_targets.output
  forward_to = [loki.process.docker_json.receiver]
}

// 4) Pipeline: parse the Docker JSON
loki.process "docker_json" {
  forward_to = [loki.write.to_loki.receiver]

  stage.json {
    expressions = {
      message = "log",
      stream  = "stream",
      ts      = "time",
    }
  }

  stage.timestamp {
    source = "ts"
    format = "RFC3339Nano"
  }

  stage.output {
    source = "message"
  }

  // Final labels (low cardinality)
  stage.labels {
    values = {
      stream         = "stream",
      container_name = "container_name",
      host           = "host",
      job            = "job",
    }
  }
}

// 5) Send to Loki
loki.write "to_loki" {
  endpoint {
    url = "http://loki:3100/loki/api/v1/push"
  }
}



// -----------------------------------------------------------------------------
// METRICS: node_exporter -> Mimir (Prometheus remote_write)
// -----------------------------------------------------------------------------

// Scrape node-exporter inside the obs network
prometheus.scrape "node_exporter" {
  targets = [
    {
      __address__ = "node-exporter:9100",
      job         = "node_exporter",
      instance    = constants.hostname,
    },
  ]

  scrape_interval = "15s"
  scrape_timeout  = "10s"

  forward_to = [prometheus.remote_write.mimir.receiver]
}

// Send metrics to Mimir
prometheus.remote_write "mimir" {
  endpoint {
    url = "http://mimir:9009/api/v1/push"
  }
}

docker-compose.yml

volumes:
  loki-data:
  grafana-data:
  mimir-data:

networks:
  obs:
    name: obs
    driver: bridge

services:
  loki:
    image: grafana/loki:3.6.3
    command: ["-config.file=/etc/loki/config.yml"]
    restart: unless-stopped
    init: true
    volumes:
      - ./loki-config.yml:/etc/loki/config.yml:ro
      - loki-data:/loki
    networks:
      - obs
    expose:
      - "3100"
    security_opt:
      - no-new-privileges:true

  node-exporter:
    image: prom/node-exporter:v1.10.2
    pid: host
    restart: unless-stopped
    init: true
    networks:
      - obs
    expose:
      - "9100"
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro
    command:
      - "--path.procfs=/host/proc"
      - "--path.rootfs=/rootfs"
      - "--path.sysfs=/host/sys"
      - "--collector.filesystem.mount-points-exclude=^/(dev|proc|sys|run|var/lib/docker/.+|var/lib/containerd/.+)($$|/)"
      - "--collector.filesystem.fs-types-exclude=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|pstore|rpc_pipefs|securityfs|squashfs|sysfs|tracefs)$"
    healthcheck:
      test: ["CMD", "wget", "-qO-", "http://localhost:9100/metrics"]
      interval: 15s
      timeout: 5s
      retries: 10
    security_opt:
      - no-new-privileges:true

  mimir:
    image: grafana/mimir:3.0.2
    command: ["-config.file=/etc/mimir.yml"]
    restart: unless-stopped
    init: true
    volumes:
      - ./mimir.yml:/etc/mimir.yml:ro
      - mimir-data:/data
    networks:
      - obs
    expose:
      - "9009"
    security_opt:
      - no-new-privileges:true

  alloy:
    image: grafana/alloy:v1.12.2
    command:
      - run
      - --server.http.listen-addr=0.0.0.0:12345
      - /etc/alloy/config.river
    restart: unless-stopped
    init: true
    user: "0:0"
    depends_on:
      - mimir
      - loki
      - node-exporter
    volumes:
      - ./alloy-config.river:/etc/alloy/config.river:ro
      - /var/lib/docker/containers:/var/lib/docker/containers:ro
      - /var/run/docker.sock:/var/run/docker.sock:ro
    networks:
      - obs
    expose:
      - "12345"

  grafana:
    image: grafana/grafana:12.3.0
    restart: unless-stopped
    init: true
    environment:
      - GF_SERVER_DOMAIN=grafana.diegocornejo.com
      - GF_SERVER_ROOT_URL=https://grafana.diegocornejo.com/
      - GF_SERVER_SERVE_FROM_SUB_PATH=false
    volumes:
      - grafana-data:/var/lib/grafana
    networks:
      - obs
    ports:
      - "127.0.0.1:20003:3000"
    healthcheck:
      test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/health"]
      interval: 15s
      timeout: 5s
      retries: 15

loki-config.yml

# loki-config.yml
# Loki single-binary + TSDB + filesystem

auth_enabled: false

server:
  http_listen_port: 3100
  log_level: info

common:
  path_prefix: /loki
  replication_factor: 1

  ring:
    kvstore:
      store: inmemory

# ------------------------------------------------------------------------------
# Storage schema: TSDB (required for structured metadata)
# ------------------------------------------------------------------------------
schema_config:
  configs:
    - from: 2026-01-01
      store: tsdb
      object_store: filesystem
      schema: v13
      index:
        prefix: index_
        period: 24h

# ------------------------------------------------------------------------------
# TSDB shipper (filesystem local)
# ------------------------------------------------------------------------------
storage_config:
  tsdb_shipper:
    active_index_directory: /loki/index
    cache_location: /loki/index_cache

  filesystem:
    directory: /loki/chunks

# ------------------------------------------------------------------------------
# Compactor (required with TSDB, even in single-node)
# ------------------------------------------------------------------------------
compactor:
  working_directory: /loki/compactor
  compaction_interval: 5m

# ------------------------------------------------------------------------------
# Limits
# ------------------------------------------------------------------------------
limits_config:
  # Required for TSDB + modern Loki
  allow_structured_metadata: true

  # Avoid rejections if any container has clock skew
  reject_old_samples: false

  # Reasonable limits for a single server
  ingestion_rate_mb: 10
  ingestion_burst_size_mb: 20

mimir.yml

multitenancy_enabled: false

server:
  http_listen_port: 9009

ingester:
  ring:
    replication_factor: 1