couchpotato

# parses strings such as: 07-31 09:56:08
DATESTAMP_CP [0-9]{2}-[0-9]{2} %{TIME}
# parses strings such as: '\u001b\[0m' or '^[[0m' or '\e[0m'
METACHAR_CP ((\\u001b|\^\[|\e)\[\d+m)?
# assigns regular expression that matches Java classes to a new variable name. 
FACILITY_CP %{JAVACLASS}

logstash.conf

input {
# stdin { type => "stdin-type" }
  file {
    type => "syslog"
    path => [ "/var/log/syslog" ]
  }
  file {
    type => "nginx_access"
    path => [ "/var/log/nginx/access.log" ]
  }
  file {
    type => "nginx_error"
    path => [ "/var/log/nginx/error.log" ]
  }
  file {
    type => "plexmediaserver"
    path => ["/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Logs/Plex Media Server.log"]
  }
  file {
    type => "plexmediascanner"
    path => ["/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Logs/Plex Media Scanner.log"]
  }
  file {
    type => "plexdlnaserver"
    path => ["/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Logs/Plex DLNA*.log"]
  }
  file {
    type => "couchpotato"
    path => [ "/home/halkeye/.couchpotato/logs/CouchPotato.log" ]
    codec => multiline {
      patterns_dir => "/opt/logstash/patterns/"
      pattern => "^%{DATESTAMP_CP}"
      negate => true
      what => previous
    }
  }
}
filter {
  if [type] == "nginx_access" {
    grok {
      match => [
        "message", "%{IPORHOST:client_ip} - (?:%{USERNAME:username}|-) \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:http_verb} %{NOTSPACE:http_request}(?: HTTP/%{NUMBER:http_version})?|%{DATA:raw_http_request})\" %{NUMBER:http_status_code} (?:%{NUMBER:bytes_read}|-) %{QS:referrer} %{QS:agent} %{QS:forwarded_for}",
        "message", "%{HOSTPORT:http_host} %{IPORHOST:client_ip} \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:http_verb} %{NOTSPACE:http_request}(?: HTTP/%{NUMBER:http_version})?|%{DATA:raw_http_request})\" %{NUMBER:http_status_code} (?:%{NUMBER:bytes_read}|-) %{QS:referrer} %{QS:agent} %{NUMBER:time_duration:float} %{NUMBER:time_backend_response:float}",
        "message", "%{HOSTPORT:http_host} %{IPORHOST:client_ip} \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:http_verb} %{NOTSPACE:http_request}(?: HTTP/%{NUMBER:http_version})?|%{DATA:raw_http_request})\" %{NUMBER:http_status_code} (?:%{NUMBER:bytes_read}|-) %{QS:referrer} %{QS:agent} %{NUMBER:time_duration:float}"
      ]
    }
    date {
      match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
    }
  }
  if [type] =~ "^plex.*" {
    grok {
      match => [
        "message", "(?<timestamp>%{MONTH} %{MONTHDAY}, %{YEAR} %{TIME}) \[%{DATA:pid}\] %{LOGLEVEL:logLevel} - %{GREEDYDATA:log}"
      ]
    }
    date {
      locale => "en"
      match => [ "timestamp", "MMM dd, yyyy HH:mm:ss"]
    }
    mutate {
      remove_field => [ "timestamp" ]
    }
  }
  if [type] == "couchpotato" {
    grok {
      patterns_dir => "/opt/logstash/patterns"
      match => [ "message",
            "(?m)%{DATESTAMP_CP:date}%{SPACE}%{LOGLEVEL:logLevel}%{SPACE}%{METACHAR_CP}\[%{FACILITY_CP:facility}\]%{GREEDYDATA:msg}"
      ]
    }
    if "_grokparsefailure" in [_tags] {
      mutate {
        remove_tag => [ "_grokparsefailure" ]
      }
      grok {
        patterns_dir => "/opt/logstash/patterns"
        match => [ "message",
              "(?m)%{DATESTAMP_CP:date}%{SPACE}%{LOGLEVEL:logLevel}%{SPACE}%{METACHAR_CP}%{GREEDYDATA:msg}%{METACHAR_CP}%"
        ]
      }
    }
    mutate {
      gsub => [ "msg", "\e\[\d+m", " " ]
      strip => [ "msg" ]
    }
    date {
      locale => "en"
      match => [ "date" , "yyyy-MM-dd HH:mm:ss", "MM-dd HH:mm:ss" ]
    }
    mutate {
      remove_field => [ "date" ]
    }
  }
}
output {
  if "_grokparsefailure" in [_tags] {
    stdout { codec => rubydebug }
  }
  elasticsearch { host => localhost }
}