Install Grafana Alloy and Node Exporter on EC2 Linux AMI 2023 and send metrics to Remote Mimir

INSTALL.md

Install Grafana Alloy and Node Exporter on EC2 Linux AMI 2023 and send metrics to Remote Mimir

Prerequisites

Create prometheus user

sudo useradd -M -U alloy

Create a directory for Grafana Alloy

sudo mkdir -p /opt/alloy/{bin,config,data,logs}
sudo chown -R alloy:alloy /opt/alloy
sudo chmod -R 755 /opt/alloy

Install Node Exporter

Download Node Exporter

Check the architecture and download the appropriate version

uname -m
# This will output the architecture, for example: x86_64 or arm64

Download the appropriate version

Replace the architecture with the output from the previous command

wget https://github.com/prometheus/node_exporter/releases/download/v1.9.1/node_exporter-1.9.1.linux-amd64.tar.gz

Extract Node Exporter

tar -xvf node_exporter-1.9.1.linux-amd64.tar.gz

Move Node Exporter to /opt/node_exporter

sudo mv node_exporter-1.9.1.linux-amd64 /opt/node_exporter

Set ownership and permissions

sudo chown -R alloy:alloy /opt/node_exporter

Create systemd service

sudo nano /etc/systemd/system/node_exporter.service

Add the following content to the file

[Unit]
Description=Node Exporter
After=network-online.target

[Service]
User=alloy
Group=alloy
Restart=on-failure
ExecStart=/opt/node_exporter/node_exporter

[Install]
WantedBy=multi-user.target

Reload systemd

sudo systemctl daemon-reload

Enable Node Exporter to start on boot and start it

sudo systemctl enable node_exporter
sudo systemctl start node_exporter

Check Node Exporter status and ensure it is running

sudo systemctl status node_exporter
curl http://localhost:9100/metrics

Install Grafana Alloy

Download the appropriate version

wget https://github.com/grafana/alloy/releases/download/v1.10.2/alloy-linux-amd64.zip

Extract Grafana Alloy

unzip alloy-linux-amd64.zip

Move Grafana Alloy to /opt/alloy

sudo mv alloy-linux-amd64 /opt/alloy/bin/alloy

Set ownership and permissions

sudo chown -R alloy:alloy /opt/alloy
sudo chmod -R 755 /opt/alloy
sudo chmod 750 /opt/alloy/data

Configure Grafana Alloy

sudo nano /opt/alloy/config/config.river

Add the following content to the file

// =============================
// Grafana Alloy - config.river
// Local Node Exporter -> Remote Mimir (single-tenant, no auth)
// =============================

prometheus.scrape "node" {
  // List of targets (each target is a map with labels). __address__ is required.
  // You can set job and instance here.
  targets = [
    {
      "__address__" = "127.0.0.1:9100",
      "job"         = "node",
      "instance"    = "your-instance-name", // or sys.env("INSTANCE") if you want to use the instance name from the environment variable
    },
  ]

  forward_to	  = [prometheus.remote_write.to_mimir.receiver]
  scrape_interval = "15s"
  scrape_timeout  = "10s"
}

prometheus.remote_write "to_mimir" {
  endpoint {
    url = "http://your-mimir-host:9009/api/v1/push"

    // If you are using basic auth in the reverse proxy, enable this block
    // basic_auth {
    //   username = "prom_rw"
    //   password = env("MIMIR_RW_PASSWORD")
    // }

    // If you are using multi-tenant, enable this block
    // headers = {
    //   "X-Scope-OrgID" = "test"
    // }

    // Reasonable queue tuning
    queue_config {
      capacity              = 10000
      max_shards            = 4
      max_samples_per_send  = 5000
      batch_send_deadline   = "5s"
      min_backoff           = "100ms"
      max_backoff           = "5s"
      retry_on_http_429     = true
    }
  }
}

Create systemd service

sudo nano /etc/systemd/system/alloy.service

Add the following content to the file

[Unit]
Description=Grafana Alloy (metrics scraper & remote_write)
After=network-online.target
Wants=network-online.target

[Service]
User=alloy
Group=alloy
# Uncomment this if you want to use the instance name from the environment variable
# Environment="INSTANCE=your-instance-name" 
WorkingDirectory=/opt/alloy
ExecStart=/opt/alloy/bin/alloy run \
  --server.http.listen-addr=127.0.0.1:12345 \
  --storage.path=/opt/alloy/data \
  /opt/alloy/config/config.river
Restart=always
RestartSec=5
LimitNOFILE=65535

[Install]
WantedBy=multi-user.target

Reload systemd

sudo systemctl daemon-reload

Enable Grafana Alloy to start on boot and start it

sudo systemctl enable --now alloy

Check Grafana Alloy status and ensure it is running

sudo systemctl status alloy

Check Grafana Alloy logs

sudo journalctl -u alloy -f

Check Grafana Alloy metrics

curl -s http://127.0.0.1:12345/metrics | egrep 'scrape|remote_write|alloy'