Skip to content

Instantly share code, notes, and snippets.

@wzyboy

wzyboy/filebeat.template.json

Last active December 11, 2017 08:31
Show Gist options
  • Save wzyboy/e1b034bfdb1add0c091a65235d9ea81a to your computer and use it in GitHub Desktop.
Save wzyboy/e1b034bfdb1add0c091a65235d9ea81a to your computer and use it in GitHub Desktop.
Download ZIP
Raw
filebeat.template.json
{
"mappings": {
"_default_": {
"_meta": {
"version": "5.6.2"
},
"date_detection": false,
"dynamic_templates": [
{
"strings_as_keyword": {
"mapping": {
"ignore_above": 1024,
"type": "keyword"
},
"match_mapping_type": "string"
}
}
],
"properties": {
"@timestamp": {
"type": "date"
},
"beat": {
"properties": {
"hostname": {
"ignore_above": 1024,
"type": "keyword"
},
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"error": {
"ignore_above": 1024,
"type": "keyword"
},
"fileset": {
"properties": {
"module": {
"ignore_above": 1024,
"type": "keyword"
},
"name": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"input_type": {
"ignore_above": 1024,
"type": "keyword"
},
"message": {
"norms": false,
"type": "text"
},
"meta": {
"properties": {
"cloud": {
"properties": {
"availability_zone": {
"ignore_above": 1024,
"type": "keyword"
},
"instance_id": {
"ignore_above": 1024,
"type": "keyword"
},
"machine_type": {
"ignore_above": 1024,
"type": "keyword"
},
"project_id": {
"ignore_above": 1024,
"type": "keyword"
},
"provider": {
"ignore_above": 1024,
"type": "keyword"
},
"region": {
"ignore_above": 1024,
"type": "keyword"
}
}
}
}
},
"nginx": {
"properties": {
"access": {
"properties": {
"remote_addr": {
"type": "ip"
},
"remote_user": {
"ignore_above": 1024,
"type": "keyword"
},
"time_local": {
"ignore_above": 1024,
"type": "keyword"
},
"request": {
"ignore_above": 1024,
"type": "keyword"
},
"status": {
"type": "long"
},
"bytes_sent": {
"type": "long"
},
"http_referrer": {
"ignore_above": 1024,
"type": "keyword"
},
"http_user_agent": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"error": {
"properties": {
"connection_id": {
"type": "long"
},
"level": {
"ignore_above": 1024,
"type": "keyword"
},
"message": {
"norms": false,
"type": "text"
},
"pid": {
"type": "long"
},
"tid": {
"type": "long"
}
}
}
}
},
"offset": {
"type": "long"
},
"read_timestamp": {
"ignore_above": 1024,
"type": "keyword"
},
"source": {
"ignore_above": 1024,
"type": "keyword"
},
"system": {
"properties": {
"auth": {
"properties": {
"groupadd": {
"properties": {
"gid": {
"type": "long"
},
"name": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"hostname": {
"ignore_above": 1024,
"type": "keyword"
},
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"pid": {
"type": "long"
},
"program": {
"ignore_above": 1024,
"type": "keyword"
},
"ssh": {
"properties": {
"dropped_ip": {
"type": "ip"
},
"event": {
"ignore_above": 1024,
"type": "keyword"
},
"ip": {
"type": "ip"
},
"method": {
"ignore_above": 1024,
"type": "keyword"
},
"port": {
"type": "long"
},
"signature": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"sudo": {
"properties": {
"command": {
"ignore_above": 1024,
"type": "keyword"
},
"error": {
"ignore_above": 1024,
"type": "keyword"
},
"pwd": {
"ignore_above": 1024,
"type": "keyword"
},
"tty": {
"ignore_above": 1024,
"type": "keyword"
},
"user": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"timestamp": {
"ignore_above": 1024,
"type": "keyword"
},
"user": {
"ignore_above": 1024,
"type": "keyword"
},
"useradd": {
"properties": {
"gid": {
"type": "long"
},
"home": {
"ignore_above": 1024,
"type": "keyword"
},
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"shell": {
"ignore_above": 1024,
"type": "keyword"
},
"uid": {
"type": "long"
}
}
}
}
},
"syslog": {
"properties": {
"hostname": {
"ignore_above": 1024,
"type": "keyword"
},
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"pid": {
"ignore_above": 1024,
"type": "keyword"
},
"program": {
"ignore_above": 1024,
"type": "keyword"
},
"timestamp": {
"ignore_above": 1024,
"type": "keyword"
}
}
}
}
},
"tags": {
"ignore_above": 1024,
"type": "keyword"
},
"type": {
"ignore_above": 1024,
"type": "keyword"
}
}
}
},
"order": 0,
"settings": {
"index.mapping.total_fields.limit": 10000,
"index.refresh_interval": "5s",
"number_of_shards": 1,
"number_of_replicas": 0
},
"template": "filebeat-*"
}
Raw
log_format.conf
# vim: ft=nginx:
# Default "combined"
#log_format combined '$remote_addr - $remote_user [$time_local] '
# '"$request" $status $body_bytes_sent '
# '"$http_referer" "$http_user_agent"';
# Extending "combined" to include more fields
log_format main
'$remote_addr - $remote_user [$time_local] '
'"$request" $status $body_bytes_sent '
'"$http_referer" "$http_user_agent" '
'"'
'host=$host|'
'request_time=$request_time|'
'http_x_forwarded_for=$http_x_forwarded_for|'
'http_cloudfront_viewer_country=$http_cloudfront_viewer_country|'
'upstream_addr=$upstream_addr|'
'upstream_status=$upstream_status|'
'upstream_response_time=$upstream_response_time'
'"'
;
Raw
nginx.access.pipeline.json
{
"description": "Pipeline for parsing Nginx access logs.",
"processors": [
{
"grok": {
"field": "message",
"patterns": [
"%{IP:nginx.access.remote_addr} - %{DATA:nginx.access.remote_user} \\[%{HTTPDATE:nginx.access.time_local}\\] \"%{DATA:nginx.access.request}\" %{NUMBER:nginx.access.status} %{NUMBER:nginx.access.bytes_sent} \"%{DATA:nginx.access.http_referrer}\" \"%{DATA:nginx.access.http_user_agent}\"( \"%{DATA:nginx.access._kvs}\")?"
],
"ignore_missing": true
}
},
{
"remove": {
"field": "message"
}
},
{
"rename": {
"field": "@timestamp",
"target_field": "read_timestamp"
}
},
{
"date": {
"field": "nginx.access.time_local",
"target_field": "@timestamp",
"formats": [
"dd/MMM/YYYY:H:m:s Z"
]
}
},
{
"kv": {
"field": "nginx.access._kvs",
"field_split": "\\|",
"value_split": "=",
"target_field": "nginx.access",
"ignore_missing": true,
"ignore_failure": true
}
},
{
"remove": {
"field": "nginx.access._kvs",
"ignore_failure": true
}
}
],
"on_failure": [
{
"set": {
"field": "error.message",
"value": "{{ _ingest.on_failure_message }}"
}
}
]
}
Raw
nginx.error.pipeline.json
{
"description": "Pipeline for parsing the Nginx error logs",
"processors": [
{
"grok": {
"field": "message",
"patterns": [
"%{DATA:nginx.error.time} \\[%{DATA:nginx.error.level}\\] %{NUMBER:nginx.error.pid}#%{NUMBER:nginx.error.tid}: (\\*%{NUMBER:nginx.error.connection_id} )?%{GREEDYDATA:nginx.error.message}"
],
"ignore_missing": true
}
},
{
"remove": {
"field": "message"
}
},
{
"rename": {
"field": "@timestamp",
"target_field": "read_timestamp"
}
},
{
"date": {
"field": "nginx.error.time",
"target_field": "@timestamp",
"formats": [
"YYYY/MM/dd H:m:s"
],
"timezone": "America/New_York"
}
},
{
"remove": {
"field": "nginx.error.time"
}
}
],
"on_failure": [
{
"set": {
"field": "error.message",
"value": "{{ _ingest.on_failure_message }}"
}
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment