Logstash configuration for reading s3 buckets logs

logstash.conf.erb

input {
  s3 {
    access_key_id => "<%= @access_key_id %>"
    secret_access_key => "<%= @secret_access_key %>"
    bucket => "airhelp-logs-s3-<%= node.chef_environment %>"
    region => "eu-west-1"
    type => "s3"
    interval => 20
    codec => "plain"
    sincedb_path => "<%= node['logstash-s3']['sincedb_path'] %>"
  }
}

filter {
  grok {
    match => [ "message", "%{S3_ACCESS_LOG}" ]
    remove_field => ["message", "path"]
  }
  date {
    locale => "en"
    match => ["timestamp", "d/MMM/YYYY:HH:mm:ss Z"]
  }
}

output {
  # for debug
  # stdout { codec => rubydebug }
  redis {
    host => "<%= node['kibana']['redis_hostname'] %>"
    port => 6379
    data_type => "list"
    key => "logstash"
  }
}