Skip to content

Instantly share code, notes, and snippets.

@hanikesn

hanikesn/content-packs.yml

Last active February 10, 2020 09:40
Show Gist options
  • Save hanikesn/f45ecc5bb3bb7628e226db7df9be4bc3 to your computer and use it in GitHub Desktop.
Save hanikesn/f45ecc5bb3bb7628e226db7df9be4bc3 to your computer and use it in GitHub Desktop.
Download ZIP
Graylog Kubernetes Setup
Raw
content-packs.yml
apiVersion: v1
kind: ConfigMap
metadata:
name: graylog-content-packs
data:
grok-patterns.json: |2
{
"name": "Core Grok Patterns",
"description": "Core grok patterns",
"category": "Grok",
"grok_patterns": [
{
"name": "USERNAME",
"pattern": "[a-zA-Z0-9._-]+"
},
{
"name": "USER",
"pattern": "%{USERNAME}"
},
{
"name": "EMAILLOCALPART",
"pattern": "[a-zA-Z][a-zA-Z0-9_.+-=:]+"
},
{
"name": "EMAILADDRESS",
"pattern": "%{EMAILLOCALPART}@%{HOSTNAME}"
},
{
"name": "HTTPDUSER",
"pattern": "%{EMAILADDRESS}|%{USER}"
},
{
"name": "INT",
"pattern": "(?:[+-]?(?:[0-9]+))"
},
{
"name": "BASE10NUM",
"pattern": "(?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\\.[0-9]+)?)|(?:\\.[0-9]+)))"
},
{
"name": "NUMBER",
"pattern": "(?:%{BASE10NUM})"
},
{
"name": "BASE16NUM",
"pattern": "(?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))"
},
{
"name": "BASE16FLOAT",
"pattern": "\\b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\\.[0-9A-Fa-f]*)?)|(?:\\.[0-9A-Fa-f]+)))\\b"
},
{
"name": "POSINT",
"pattern": "\\b(?:[1-9][0-9]*)\\b"
},
{
"name": "NONNEGINT",
"pattern": "\\b(?:[0-9]+)\\b"
},
{
"name": "WORD",
"pattern": "\\b\\w+\\b"
},
{
"name": "NOTSPACE",
"pattern": "\\S+"
},
{
"name": "SPACE",
"pattern": "\\s*"
},
{
"name": "DATA",
"pattern": ".*?"
},
{
"name": "GREEDYDATA",
"pattern": ".*"
},
{
"name": "QUOTEDSTRING",
"pattern": "(?>(?<!\\\\)(?>\"(?>\\\\.|[^\\\\\"]+)+\"|\"\"|(?>'(?>\\\\.|[^\\\\']+)+')|''|(?>`(?>\\\\.|[^\\\\`]+)+`)|``))"
},
{
"name": "UUID",
"pattern": "[A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}"
},
{
"name": "MAC",
"pattern": "(?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})"
},
{
"name": "CISCOMAC",
"pattern": "(?:(?:[A-Fa-f0-9]{4}\\.){2}[A-Fa-f0-9]{4})"
},
{
"name": "WINDOWSMAC",
"pattern": "(?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})"
},
{
"name": "COMMONMAC",
"pattern": "(?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})"
},
{
"name": "IPV6",
"pattern": "((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:)))(%.+)?"
},
{
"name": "IPV4",
"pattern": "(?<![0-9])(?:(?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5]))(?![0-9])"
},
{
"name": "IP",
"pattern": "(?:%{IPV6}|%{IPV4})"
},
{
"name": "HOSTNAME",
"pattern": "\\b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\\b)"
},
{
"name": "IPORHOST",
"pattern": "(?:%{IP}|%{HOSTNAME})"
},
{
"name": "HOSTPORT",
"pattern": "%{IPORHOST}:%{POSINT}"
},
{
"name": "PATH",
"pattern": "(?:%{UNIXPATH}|%{WINPATH})"
},
{
"name": "UNIXPATH",
"pattern": "(/([\\w_%!$@:.,~-]+|\\\\.)*)+"
},
{
"name": "TTY",
"pattern": "(?:/dev/(pts|tty([pq])?)(\\w+)?/?(?:[0-9]+))"
},
{
"name": "WINPATH",
"pattern": "(?>[A-Za-z]+:|\\\\)(?:\\\\[^\\\\?*]*)+"
},
{
"name": "URIPROTO",
"pattern": "[A-Za-z]+(\\+[A-Za-z+]+)?"
},
{
"name": "URIHOST",
"pattern": "%{IPORHOST}(?::%{POSINT:port})?"
},
{
"name": "URIPATH",
"pattern": "(?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\\-]*)+"
},
{
"name": "URIPARAM",
"pattern": "\\?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\\-\\[\\]<>]*"
},
{
"name": "URIPATHPARAM",
"pattern": "%{URIPATH}(?:%{URIPARAM})?"
},
{
"name": "URI",
"pattern": "%{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?"
},
{
"name": "MONTH",
"pattern": "\\b(?:Jan(?:uary|uar)?|Feb(?:ruary|ruar)?|M(?:a|ä)?r(?:ch|z)?|Apr(?:il)?|Ma(?:y|i)?|Jun(?:e|i)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|O(?:c|k)?t(?:ober)?|Nov(?:ember)?|De(?:c|z)(?:ember)?)\\b"
},
{
"name": "MONTHNUM",
"pattern": "(?:0?[1-9]|1[0-2])"
},
{
"name": "MONTHNUM2",
"pattern": "(?:0[1-9]|1[0-2])"
},
{
"name": "MONTHDAY",
"pattern": "(?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])"
},
{
"name": "DAY",
"pattern": "(?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)"
},
{
"name": "YEAR",
"pattern": "(?>\\d\\d){1,2}"
},
{
"name": "HOUR",
"pattern": "(?:2[0123]|[01]?[0-9])"
},
{
"name": "MINUTE",
"pattern": "(?:[0-5][0-9])"
},
{
"name": "SECOND",
"pattern": "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)"
},
{
"name": "TIME",
"pattern": "(?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])"
},
{
"name": "DATE_US",
"pattern": "%{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}"
},
{
"name": "DATE_EU",
"pattern": "%{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}"
},
{
"name": "ISO8601_TIMEZONE",
"pattern": "(?:Z|[+-]%{HOUR}(?::?%{MINUTE}))"
},
{
"name": "ISO8601_SECOND",
"pattern": "(?:%{SECOND}|60)"
},
{
"name": "TIMESTAMP_ISO8601",
"pattern": "%{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?"
},
{
"name": "DATE",
"pattern": "%{DATE_US}|%{DATE_EU}"
},
{
"name": "DATESTAMP",
"pattern": "%{DATE}[- ]%{TIME}"
},
{
"name": "TZ",
"pattern": "(?:[PMCE][SD]T|UTC)"
},
{
"name": "DATESTAMP_RFC822",
"pattern": "%{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}"
},
{
"name": "DATESTAMP_RFC2822",
"pattern": "%{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE}"
},
{
"name": "DATESTAMP_OTHER",
"pattern": "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}"
},
{
"name": "DATESTAMP_EVENTLOG",
"pattern": "%{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND}"
},
{
"name": "HTTPDERROR_DATE",
"pattern": "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"
},
{
"name": "SYSLOGTIMESTAMP",
"pattern": "%{MONTH} +%{MONTHDAY} %{TIME}"
},
{
"name": "PROG",
"pattern": "[\\x21-\\x5a\\x5c\\x5e-\\x7e]+"
},
{
"name": "SYSLOGPROG",
"pattern": "%{PROG:program}(?:\\[%{POSINT:pid}\\])?"
},
{
"name": "SYSLOGHOST",
"pattern": "%{IPORHOST}"
},
{
"name": "SYSLOGFACILITY",
"pattern": "<%{NONNEGINT:facility}.%{NONNEGINT:priority}>"
},
{
"name": "HTTPDATE",
"pattern": "%{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}"
},
{
"name": "QS",
"pattern": "%{QUOTEDSTRING}"
},
{
"name": "SYSLOGBASE",
"pattern": "%{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:"
},
{
"name": "COMMONAPACHELOG",
"pattern": "%{IPORHOST:clientip} %{HTTPDUSER:ident} %{USER:auth} \\[%{HTTPDATE:timestamp}\\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:response} (?:%{NUMBER:bytes}|-)"
},
{
"name": "COMBINEDAPACHELOG",
"pattern": "%{COMMONAPACHELOG} %{QS:referrer} %{QS:agent}"
},
{
"name": "HTTPD20_ERRORLOG",
"pattern": "\\[%{HTTPDERROR_DATE:timestamp}\\] \\[%{LOGLEVEL:loglevel}\\] (?:\\[client %{IPORHOST:clientip}\\] ){0,1}%{GREEDYDATA:errormsg}"
},
{
"name": "HTTPD24_ERRORLOG",
"pattern": "\\[%{HTTPDERROR_DATE:timestamp}\\] \\[%{WORD:module}:%{LOGLEVEL:loglevel}\\] \\[pid %{POSINT:pid}:tid %{NUMBER:tid}\\]( \\(%{POSINT:proxy_errorcode}\\)%{DATA:proxy_errormessage}:)?( \\[client %{IPORHOST:client}:%{POSINT:clientport}\\])? %{DATA:errorcode}: %{GREEDYDATA:message}"
},
{
"name": "HTTPD_ERRORLOG",
"pattern": "%{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG}"
},
{
"name": "LOGLEVEL",
"pattern": "([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)"
}
]
}
udp-input-graylog.json: |-
{
"name":"UDP GELF input on 12201",
"description":"Adds a global UDP GELF input on port 12201",
"category":"Inputs",
"inputs":[
{
"title":"udp input",
"configuration":{
"override_source":null,
"recv_buffer_size":262144,
"bind_address":"0.0.0.0",
"port":12201,
"decompress_size_limit":8388608
},
"static_fields":{},
"type":"org.graylog2.inputs.gelf.udp.GELFUDPInput",
"global":true,
"extractors":[]
}
],
"streams":[],
"outputs":[],
"dashboards":[],
"grok_patterns":[]
}
Raw
graylog.yml
apiVersion: v1
kind: Service
metadata:
name: mongodb
spec:
selector:
app: mongodb
ports:
- port: 27017
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: mongodb
spec:
template:
metadata:
labels:
app: mongodb
spec:
containers:
- name: mongodb
image: mongo:3.2
ports:
- containerPort: 27017
---
apiVersion: v1
kind: Service
metadata:
name: elasticsearch
spec:
selector:
app: elasticsearch
ports:
- name: management
port: 9200
- name: zen
port: 9300
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: elasticsearch
labels:
app: elasticsearch
spec:
template:
metadata:
labels:
app: elasticsearch
spec:
containers:
- name: elasticsearch
image: elasticsearch:2.4.4
args:
- -Des.insecure.allow.root=true
- -Des.cluster.name=graylog
- -Des.discovery.zen.ping.unicast.hosts=graylog:9350
- -Des.discovery.zen.ping.multicast.enabled=false
ports:
- containerPort: 9200
- containerPort: 9300
---
apiVersion: v1
kind: Service
metadata:
name: graylog
spec:
clusterIP: 10.0.0.100
selector:
app: graylog
ports:
- name: web
port: 80
targetPort: 9000
- name: elasticsearch
port: 9350
- name: gelf-udp
protocol: UDP
port: 12201
- name: gelf-tcp
protocol: TCP
port: 12201
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: graylog
labels:
app: graylog
spec:
template:
metadata:
labels:
app: graylog
spec:
containers:
- name: graylog
image: graylog2/server:2.2.3-1
env:
- name: GRAYLOG_MONGODB_URI
value: mongodb://mongodb/graylog
- name : GRAYLOG_PASSWORD_SECRET
value: "somepasswordpepper"
- name: GRAYLOG_ROOT_PASSWORD_SHA2
value: "8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918"
- name: GRAYLOG_WEB_ENDPOINT_URI
value: "http://10.0.0.100/api"
- name: ELASTICSEARCH_DISCOVERY_ZEN_PING_MULTICAST_ENABLED
value: "false"
- name: ELASTICSEARCH_DISCOVERY_ZEN_PING_UNICAST_HOSTS
value: "elasticsearch:9300"
- name: GRAYLOG_CONTENT_PACKS_AUTO_LOAD
value: "grok-patterns.json,udp-input-graylog.json"
ports:
- containerPort: 9000
volumeMounts:
- name: content-packs
mountPath: /usr/share/graylog/data/contentpacks
volumes:
- name: content-packs
configMap:
name: graylog-content-packs
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment