Logstash S3 ELB Access Logs Conf

s3_elb_logs.conf

input {
  s3 {
    type => "elb-access-log"
    bucket => "elb-absencetracker"
    region => "us-west-2"
    access_key_id => "access_key_id_goes_here"
    secret_access_key => "secret_access_key_goes_here"
    sincedb_path => "/tmp/.prod_s3_elb_logs_since.db"
  }
}

filter {
  if [type] == "elb-access-log" {
    grok {
      match => [ "message", "%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:elb} %{IP:clientip}:%{INT:clientport:int} (?:(%{IP:backendip}:?:%{INT:backendport:int})|-) %{NUMBER:request_processing_time:float} %{NUMBER:backend_processing_time:float} %{NUMBER:response_processing_time:float} (?:-|%{INT:elb_status_code:int}) (?:-|%{INT:backend_status_code:int}) %{INT:received_bytes:int} %{INT:sent_bytes:int} \"%{ELB_REQUEST_LINE}\" \"(?:-|%{DATA:user_agent})\" (?:-|%{NOTSPACE:ssl_cipher}) (?:-|%{NOTSPACE:ssl_protocol})" ]
    }
    date {
      match => [ "timestamp", "ISO8601" ]
    }

    fingerprint {
       source => ["message"]
       target => "[@metadata][fingerprint]"
       method => "MURMUR3"
    }


    mutate {
      add_field => { "indexname" => "elb-%{elb}" }
    }

    mutate {
      lowercase => [ "indexname" ]
    }
  }
}

output {
   elasticsearch {
      hosts => ["elasticsearch_hostname_or_ip_goes_here:9200"]
      document_id => "%{[@metadata][fingerprint]}"
   }
}