ELK set-up for local Rails development

.env

ELASTIC_PASSWORD=<your-elastic-password>
KIBANA_PASSWORD=<your-kibana-password>

config slash docker-logstash.conf

input {
  udp {
    host => "0.0.0.0"
    port => 5228
    codec => json_lines
  }
}

filter {
    json {
    source => "message"
    }
    mutate {
      rename => { "host" => "container_host" }
    }
    mutate {
      remove_field => [ "[container_host][ip]" ]
    }
}

output {
  elasticsearch {
    hosts => ["elasticsearch:9200"]
    user => "elastic"
    password => "${ELASTIC_PASSWORD}"
    codec => json_lines
  }
  stdout {
    codec => rubydebug
  }
}

docker-compose.yml

services:
  setup:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.15.1
    environment:
      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
      - KIBANA_PASSWORD=${KIBANA_PASSWORD}
    container_name: setup
    command:
      - bash
      - -c
      - |
        echo "Waiting for Elasticsearch availability";
        until curl -s http://elasticsearch:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
        echo "Setting kibana_system password";
        until curl -s -X POST -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" http://elasticsearch:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;
        echo "All done!";

  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.15.1
    # give the container a name
    # this will also set the container's hostname as elasticsearch
    container_name: elasticsearch
    # this will store the data permanently outside the elastissearch container
#    volumes:
#      - ./esdata:/usr/share/elasticsearch/data
    # this will allow access to the content from outside the container
    ports:
      - 9200:9200
    environment:
      - discovery.type=single-node
      - cluster.name=elasticsearch
      - bootstrap.memory_lock=true
      # limits elasticsearch to 1 GB of RAM
      - ES_JAVA_OPTS=-Xms1g -Xmx1g
      # The password for the 'elastic' user
      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
      - xpack.security.http.ssl.enabled=false

  logstash:
    command: logstash -f /etc/logstash/conf.d/logstash.conf
    image: logstash:8.15.2
    ports:
      - '5001:5001'
    depends_on:
      - elasticsearch
    volumes:
      - ./config/docker-logstash.conf:/etc/logstash/conf.d/logstash.conf
    environment:
      - ELASTIC_HOSTS=http://elasticsearch:9200
      - ELASTIC_USER=elastic
      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}

  kibana:
    image: docker.elastic.co/kibana/kibana:8.15.1
    container_name: kibana
    ports:
      - 5601:5601
    environment:
      # remember the container_name for elasticsearch?
      # we use it here to access that container
      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
      - ELASTICSEARCH_USERNAME=kibana_system
      - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD}
      # Change this to true if you want to sent
      # telemetry data to kibana developers
      - TELEMETRY_ENABLED=false
  rails:
    build: .
    command: bash -c "rm -f tmp/pids/server.pid && ./bin/thrust ./bin/rails server"
    environment:
      - ELASTICSEARCH_HOST=elasticsearch
      - LOGSTASH_HOST=logstash
      - RAILS_ENV=development
    ports:
      - '80:80'
    volumes:
      - .:/rails  # Mounting app for any dynamic changes
      - ./log:/rails/log  # Mounting log directory
      - ./tmp:/rails/tmp  # Mounting tmp for PID files, cache, etc.
volumes:
  esdata:
  bundle:

Credit to https://community.hetzner.com/tutorials/deploy-elk-stack-with-docker and https://ericlondon.com/2017/01/26/integrate-rails-logs-with-elasticsearch-logstash-kibana-in-docker-compose.html