logstash api importer

.gitignore

.DS_Store
data/

iis.config

input {
  file {
    path => "/Users/shaw/apilogs/data/API-*/*.log"
    tags => [ "api" ]
    start_position => "beginning"
  }
}

filter{
  if [message] =~ "^#" {
    drop {}
  }

  grok {
    match => ["path", "data/(?<log_host>[^/]+)/(?<log_file>.*).log"] 
  }

  grok {
    match => ["message", "%{TIMESTAMP_ISO8601:log_timestamp} %{IPORHOST:ip} %{WORD:method} %{URIPATH:uri} %{NOTSPACE:username} %{IPORHOST:cip} %{NOTSPACE:httpversion} %{NOTSPACE:useragent} %{NOTSPACE:cookie} %{NOTSPACE:referer} (%{URIHOST:chost}|%{NOTSPACE:chost}) %{INT:scstatus} %{INT:scsubstatus} %{INT:scbytes} %{INT:csbytes} %{INT:timetaken}"]
  }

  date {
    match => [ "log_timestamp", "YYYY-MM-dd HH:mm:ss" ]
    timezone => "Etc/UTC"
  }   

  useragent {
    source => "useragent"
    prefix => "browser"
  }
  
  mutate {
    remove_field => ["log_timestamp"]
    gsub => [ "log_host", "-", "" ]
  }

  if [uri] =~ /v(2.7|2.8|2.9|3.0|max)/ {
    ruby {
      code => "event['api_version'] = /v(2.7|2.8|2.9|3.0|max)/.match(event['uri'])[1]"
    } 
  }
}

output {
  elasticsearch {
    host => localhost
    protocol => http
  }
}

reset.sh

#!/bin/sh

curl -XDELETE 'http://127.0.0.1:9200/metrics/'
curl -XDELETE 'http://127.0.0.1:9200/logstash-*/'

start.sh

#!/bin/sh

sudo elasticsearch --config=/usr/local/opt/elasticsearch/config/elasticsearch.yml

stash.sh

#!/bin/sh

logstash agent -f iis.config --verbose