Elastic Search, Logstash and Kibana via docker-compose for parsing key=value style log files

docker-compose.yml

version: '2'

services:

  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.2.2
    volumes:
       - esdata:/usr/share/elasticsearch/data
    ports:
      - "9200:9200"
      - "9300:9300"
    environment:
      ES_JAVA_OPTS: "-Xmx256m -Xms256m"
    networks:
      - elk

  logstash:
    image: docker.elastic.co/logstash/logstash-oss:6.2.2
    volumes:
      # - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro
      - ./logstash/pipeline:/usr/share/logstash/pipeline:ro
      - /var/log/GDPR/myapplication:/var/log/GDPR/myapplication:ro
    ports:
      - "5000:5000"
    environment:
      LS_JAVA_OPTS: "-Xmx256m -Xms256m"
    networks:
      - elk
    depends_on:
      - elasticsearch
    links:
      - elasticsearch

  kibana:
    image: docker.elastic.co/kibana/kibana-oss:6.2.2
    volumes:
      - ./kibana/config/:/usr/share/kibana/config:ro
    ports:
      - "5601:5601"
    networks:
      - elk
    depends_on:
      - elasticsearch
    links:
      - elasticsearch

networks:
  elk:
    driver: bridge

volumes:
  esdata:
    driver: local

gdpr-pipeline.conf

## Save in: logstash/pipeline/gdpr-pipeline.conf

input {
  file {
    path => [ "/var/log/GDPR/myapplication/myapplication_gdpr.log" ]
  }
}

filter {
  # Parsing three types of GDPR log entries, one at a time
  
  # Pattern for GDPR User Activity log entries
  grok {
    match => { "message" => "%{TIMESTAMP_ISO8601:time} %{LOGLEVEL:loglevel} : action=%{WORD:action} command=%{QS:command} host=%{IPORHOST:host} dest=%{QS:dest} status=%{WORD:status}( result=%{QS:result})?( result_id=%{QS:result_id})? src=%{QS:src} act_id=%{NUMBER:act_id} src_domain=%{IPORHOST:src_domain} user=%{QS:user} object=%{QS:object} object_category=%{QS:object_category} object_id=%{QS:object_id} object_attrs=%{QS:object_attrs}" }
    add_tag => [ "User Activity" ]
    overwrite => [ "host" ]
  }
 
 # Pattern for GDPR Access log entries
  if "_grokparsefailure" in [tags] {
      grok {
        match => { "message" => "%{TIMESTAMP_ISO8601:time} %{LOGLEVEL:loglevel} : action=%{WORD:action} command=%{QS:command} host=%{IPORHOST:host} dest=%{QS:dest} status=%{WORD:status}( result=%{QS:result})?( result_id=%{QS:result_id})? src=%{QS:src} act_id=%{NUMBER:act_id} src_domain=%{IPORHOST:src_domain} user=%{QS:user} duration=%{INT:duration} response_time=%{INT:response_time}" }
        remove_tag => [ "_grokparsefailure" ]
        add_tag => [ "Access" ]
        overwrite => [ "host" ]
      }
  }

  # Pattern for GDPR User Management log entries
  if "_grokparsefailure" in [tags] {
      grok {
        match => { "message" => "%{TIMESTAMP_ISO8601:time} %{LOGLEVEL:loglevel} : action=%{WORD:action} command=%{QS:command} host=%{IPORHOST:host} dest=%{QS:dest} status=%{WORD:status}( result=%{QS:result})?( result_id=%{QS:result_id})? src=%{QS:src} act_id=%{NUMBER:act_id} src_domain=%{IPORHOST:src_domain} user=%{QS:user} src_user=%{QS:src_user} user_attrs=%{QS:user_attrs}" }
        remove_tag => [ "_grokparsefailure" ]
        add_tag => [ "User Management" ]
        overwrite => [ "host" ]
      }
  }

  date {
    match => [ "time", "ISO8601" ]
    timezone => [ "Europe/Copenhagen" ]
  }
  
}

output {
  stdout { codec => rubydebug }
  elasticsearch {
      hosts => [ "elasticsearch:9200" ]
      user => 'elastic'
      password => 'changeme'
      index => "gdpr_logs"
      id => "gdpr_pipeline_id"
  }
}

kibana.yml

## Save in: kibana/config/kibana.yml
# Default Kibana configuration from kibana-docker.

server.name: kibana
server.host: "0"
elasticsearch.url: http://elasticsearch:9200
elasticsearch.username: elastic
elasticsearch.password: changeme