@ Logstash 1.4.2 snippet

Logstash 1.4.2 snippet

clean-syslog-parsing.rb

filter {

    # Extract priority/facility/severity from syslog info

    grok {
      match => { "message" => "<(?<priority>\d{2,3})>(?<message>.*)" }
      overwrite => [ "message", "priority" ]
      remove_tag => [ "_grokparsefailure" ]
      tag_on_failure => [ "_groksyslogfailure" ]
    }
    syslog_pri {
      syslog_pri_field_name => "priority"
    }
    mutate {
      rename => [ "syslog_facility", "facility_label", "syslog_facility_code", "facility", "syslog_severity", "severity_label",  "syslog_severity_code", "severity"]
    }

}

De-escape rsyslog tokens.rb

filter {
  if [type] == "ludia_syslog" {
    mutate {
      gsub => [ "message", "#012", "\
      " ]
    }
    mutate {
      gsub => [
        "message", "\\$", "",
        "message", "#011", "",
        "message", "#015", ""
      ]
      strip => [ "message" ]
    }


    # Extract Key Value pairs

    kv {
      prefix => "kv_"
      field_split => " &?"
      trim => "<>\[\]\{\}\(\),"
      trimkey => "<>\[\]\{\}\(\),"
    }
  }
}

Deep parse.rb

filter {

    # Try to parse log deeper

    grok {
      match => { "message" => "module=%{NOTSPACE:module} function=%{NOTSPACE:function} line=%{NOTSPACE:line} %{GREEDYDATA:message}" }
      overwrite => [ "message" ]
      add_field => {
        "source_type" => "python"
        "source" => "%{module}.%{function}:%{line}"
      }
      tag_on_failure => []
    }

    grok {
      match => { "message" => "\[thread=%{NOTSPACE:thread} class=%{NOTSPACE:class}\] %{GREEDYDATA:message}" }
      overwrite => [ "message" ]
      add_field => {
        "source_type" => "java"
        "source" => "%{class}"
      }
      tag_on_failure => []
    }

}

drop-annoying-logs.rb

filter {

    if ([message] =~ /com\.ludia\.bingo\.controller\.error::ErrorManagementController/) {
      drop{}
    }

}

elasticsearch-config-with-logstash.json

{
  "template":"logstash-*",
  "settings":{
  },
  "mappings":{
     "_default_":{
        "dynamic_templates":[
           {
              "string_fields":{
                 "mapping":{
                    "index":"analyzed",
                    "omit_norms":true,
                    "type":"string",
                    "fields":{
                       "raw":{
                          "index":"not_analyzed",
                          "ignore_above":256,
                          "type":"string"
                       }
                    }
                 },
                 "match_mapping_type":"string",
                 "match":"*"
              }
           }
        ],
        "properties":{
           "@version":{
              "index":"not_analyzed",
              "type":"string"
           }
        },
        "_all":{
           "enabled":true
        }
     }
  }
}

extract key-value.rb

filter {
    kv {
      prefix => "kv_"
      field_split => " &?"
      trim => "<>\[\]\{\}\(\),"
      trimkey => "<>\[\]\{\}\(\),"
    }
}

logstash-full-redis2elasticsearch.rb

input {
  redis {
    host => 'host.of.redis.exemple.com'
    data_type => 'list'
    key => 'logstash:redis'
    type => 'redis-input'
  }
}

filter {
}

output {
  elasticsearch_http {
    host => 'localhost or host.of.elasticsearch.exemple.com'
    template => '/opt/logstash/lib/logstash/outputs/elasticsearch/elasticsearch-template.json'
    template_overwrite => true
  }
}

outputs.rb

output {
  if [type] == "ludia_syslog" {

    # Don't push into ES directly. Buffer through Redis
    #elasticsearch_http {
    #  host => "127.0.0.1"
    #  flush_size => 1
    #  template => "/srv/logstash/lib/logstash/outputs/elasticsearch/elasticsearch-template.json"
    #}

    file {
      path => "/srv/logs/logs-%{+YYYY-MM-dd}.log"
    }
  }

  if [type] == "events" {
    file {
      path => "/srv/logs/events-%{+YYYY-MM-dd}.log"
    }
  }

  redis {
    host => 'localhost'
    data_type => 'list'
    key => 'logstash:redis'
  }

}