fluent-bit

fluent-bit-containers.conf

[SERVICE]
    Flush         5
    Daemon        Off
    Log_Level     info
    Parsers_File  /etc/fluent-bit/parsers.conf

[INPUT]
    Name           Tail
    Path           /mnt/data/docker/containers/*/*.log
    Path_Key       file
    DB             /run/fluent-bit-containers.state
    Parser         docker
    Refresh_Interval 20s

[FILTER]
    Name           kubernetes
    Match          *
    Merge_JSON_Log On
    Dummy_Meta     On

[FILTER]
    Name           record_modifier
    Match          *
    Remove_key     kubernetes
    Record         host_name ${HOST_NAME}
    Record         host_ip ${HOST_IP}

[OUTPUT]
    Name           es
    Match          *
    Host           ${ELASTICSEARCH_HOST}
    Port           ${ELASTICSEARCH_PORT}
    Logstash_Prefix ${ELASTICSEARCH_INDEX}
    Type           ${ELASTICSEARCH_TYPE}
    HTTP_User      ${ELASTICSEARCH_USER}
    HTTP_Passwd    ${ELASTICSEARCH_PASSWORD}
    tls            On
    tls.verify     On
    Retry_Limit    False
    Logstash_Format On

fluent-bit-system.conf

[SERVICE]
    Flush        5
    Daemon       Off
    Log_Level    info
    Parsers_File /etc/fluent-bit/parsers.conf

[INPUT]
    Name         Tail
    Path         /var/log/messages
    Path_Key     log_file
    DB           /run/fluent-bit-messages.state
    Parser       syslog-rfc3164

[INPUT]
    Name         Tail
    Path         /var/log/kern.log
    Path_Key     log_file
    DB           /run/fluent-bit-kern.state
    Parser       syslog-rfc3164

[INPUT]
    Name         Tail
    Path         /var/log/auth.log
    Path_Key     log_file
    DB           /run/fluent-bit-auth.state
    Parser       syslog-rfc3164

[INPUT]
    Name         Tail
    Path         /var/log/authpriv.log
    Path_Key     log_file
    DB           /run/fluent-bit-authpriv.state
    Parser       syslog-rfc3164

[INPUT]
    Name         Tail
    Path         /var/log/docker.log
    Path_Key     log_file
    DB           /run/fluent-bit-docker.state
    Parser       docker-daemon

[INPUT]
    Name         Tail
    Path         /var/log/fluent-bit-*.log
    Path_Key     log_file
    DB           /run/fluent-bit-fluent-bit.state

[INPUT]
    Name         Tail
    Path         /var/log/prometheus-*.log
    Path_Key     log_file
    DB           /run/fluent-bit-prometheus.state

[FILTER]
    Name           kubernetes
    Match          *
    Merge_JSON_Log On
    Dummy_Meta     On

[FILTER]
    Name           record_modifier
    Match          *
    Remove_key     kubernetes
    Record         host_name ${HOST_NAME}
    Record         host_ip ${HOST_IP}

[OUTPUT]
    Name           es
    Match          *
    Host           ${ELASTICSEARCH_HOST}
    Port           ${ELASTICSEARCH_PORT}
    Logstash_Prefix ${ELASTICSEARCH_INDEX}
    Type           ${ELASTICSEARCH_TYPE}
    HTTP_User      ${ELASTICSEARCH_USER}
    HTTP_Passwd    ${ELASTICSEARCH_PASSWORD}
    tls            On
    tls.verify     On
    Retry_Limit    False
    Logstash_Format On

init.d-fluent-bit-containers

#!/sbin/openrc-run
# Copyright (c) 2017 Marc Bachmann <[email protected]>
# Released under the MIT license.

description="fluent-bit docker container log forwarder"
pidfile="/run/${RC_SVCNAME}.pid"
command="/usr/bin/env"
command_args="HOST_IP=${HOST_IP} HOST_NAME=${HOST_NAME} ELASTICSEARCH_HOST=${ELASTICSEARCH_HOST} ELASTICSEARCH_PORT=${ELASTICSEARCH_PORT} ELASTICSEARCH_INDEX=${ELASTICSEARCH_INDEX} ELASTICSEARCH_TYPE=${ELASTICSEARCH_TYPE} ELASTICSEARCH_USER=${ELASTICSEARCH_USER} ELASTICSEARCH_PASSWORD=${ELASTICSEARCH_PASSWORD} /usr/bin/fluent-bit -q -c /etc/fluent-bit/fluent-bit-containers.conf"
start_stop_daemon_args="--stdout /var/log/fluent-bit-containers.log --stderr /var/log/fluent-bit-containers.log"
command_background="true"

depend() {
  need net root
  before docker
}

init.d-fluent-bit-system

#!/sbin/openrc-run
# Copyright (c) 2017 Marc Bachmann <[email protected]>
# Released under the MIT license.

description="fluent-bit system log forwarder"
pidfile="/run/${RC_SVCNAME}.pid"
command="/usr/bin/env"
command_args="HOST_IP=${HOST_IP} HOST_NAME=${HOST_NAME} ELASTICSEARCH_HOST=${ELASTICSEARCH_HOST} ELASTICSEARCH_PORT=${ELASTICSEARCH_PORT} ELASTICSEARCH_INDEX=${ELASTICSEARCH_INDEX} ELASTICSEARCH_TYPE=${ELASTICSEARCH_TYPE} ELASTICSEARCH_USER=${ELASTICSEARCH_USER} ELASTICSEARCH_PASSWORD=${ELASTICSEARCH_PASSWORD} /usr/bin/fluent-bit -q -c /etc/fluent-bit/fluent-bit-system.conf"
start_stop_daemon_args="--stdout /var/log/fluent-bit-system.log --stderr /var/log/fluent-bit-system.log"
command_background="true"

depend() {
  need net root
  before docker
}

parsers.conf

[PARSER]
    Name   apache
    Format regex
    Regex  ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
    Time_Key time
    Time_Format %d/%b/%Y:%H:%M:%S %z

[PARSER]
    Name   apache2
    Format regex
    Regex  ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
    Time_Key time
    Time_Format %d/%b/%Y:%H:%M:%S %z

[PARSER]
    Name   apache_error
    Format regex
    Regex  ^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$

[PARSER]
    Name   nginx
    Format regex
    Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
    Time_Key time
    Time_Format %d/%b/%Y:%H:%M:%S %z

[PARSER]
    Name   json-test
    Format json
    Time_Key time
    Time_Format %d/%b/%Y:%H:%M:%S %z

[PARSER]
    Name        docker
    Format      json
    Time_Key    time
    Time_Format %Y-%m-%dT%H:%M:%S.%L

[PARSER]
    Name        docker-daemon
    Format      regex
    Regex       time="(?<time>[^ ]*)" level=(?<level>[^ ]*) msg="(?<message>[^ ].*)"
    Time_Key    time
    Time_Format %Y-%m-%dT%H:%M:%S.%L

[PARSER]
    Name        syslog-rfc5424
    Format      regex
    Regex       ^\<(?<pri>[0-9]{1,5})\>1 (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|-)) (?<message>.+)$
    Time_Key    time
    Time_Format %Y-%m-%dT%H:%M:%S.%L

[PARSER]
    Name        syslog-rfc3164-local
    Format      regex
    Regex       ^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$
    Time_Key    time
    Time_Format %b %d %H:%M:%S

[PARSER]
    Name        syslog-rfc3164
    Format      regex
    Regex       /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
    Time_Key    time
    Time_Format %b %d %H:%M:%S
    Time_Format %Y-%m-%dT%H:%M:%S.%L

[PARSER]
    Name    mongodb
    Format  regex
    Regex   ^(?<time>[^ ]*)\s+(?<severity>\w)\s+(?<context>[^ ]+)\s+\[(?<connection>[^\]]+)]\s+(?<message>.*)$
    Time_Format %Y-%m-%dT%H:%M:%S.%L
    Time_Keep   On
    Time_Key time