Skip to content

Instantly share code, notes, and snippets.

@tennix

tennix/fluent-bit-config.yaml

Created May 6, 2020 04:04
Show Gist options
  • Save tennix/3eaa229ca30601db313029ab507e56ad to your computer and use it in GitHub Desktop.
Save tennix/3eaa229ca30601db313029ab507e56ad to your computer and use it in GitHub Desktop.
Download ZIP
fluend/fleunt-bit log parser config for TiDB
Raw
fluent-bit-config.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: fluent-bit-config
labels:
app: fluent-bit
data:
fluent-bit-service.conf: |-
[SERVICE]
Flush 1
Daemon Off
Log_Level info
Parsers_File parsers.conf
Parsers_File parsers_custom.conf
fluent-bit-input.conf: |-
[INPUT]
Name tail
Path /var/log/containers/*_slowlog-*.log
Parser docker
Tag slowlog.*
Refresh_Interval 5
Mem_Buf_Limit 1MB
Docker_Mode On
Skip_Long_Lines On
[INPUT]
Name tail
Path /var/log/containers/*_auditlog-*.log
Parser docker
Tag auditlog.*
Refresh_Interval 5
Mem_Buf_Limit 1MB
Docker_Mode On
Skip_Long_Lines On
[INPUT]
Name tail
Path /var/log/containers/*.log
Exclude_Path /var/log/containers/*_slowlog*.log, /var/log/containers/*_auditlog*.log
Parser docker
Tag kube.*
Refresh_Interval 5
Mem_Buf_Limit 5MB
Docker_Mode On
Skip_Long_Lines On
[INPUT]
Name tail
Path /var/log/messages
Parser sysmsg
Tag system.*
Refresh_Interval 5
Mem_Buf_Limit 1MB
Skip_Long_Lines On
fluent-bit-filter.conf: |-
[FILTER]
Name kubernetes
Match kube.*
Kube_Tag_Prefix kube.var.log.containers.
Kube_URL https://kubernetes.default.svc:443
Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token
Merge_Log On
K8S-Logging.Parser On
K8S-Logging.Exclude On
[FILTER]
Name kubernetes
Match slowlog.*
Kube_Tag_Prefix slowlog.var.log.containers.
Kube_URL https://kubernetes.default.svc:443
Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token
Merge_Log On
K8S-Logging.Parser Off
K8S-Logging.Exclude Off
[FILTER]
Name kubernetes
Match auditlog.*
Kube_Tag_Prefix auditlog.var.log.containers.
Kube_URL https://kubernetes.default.svc:443
Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token
Merge_Log On
K8S-Logging.Parser Off
K8S-Logging.Exclude Off
fluent-bit-output.conf: |-
[OUTPUT]
Name es
Match kube.*
Host infra-es-elasticsearch-client
Port 9200
Logstash_Format On
Retry_Limit False
Type flb_type
Time_Key @timestamp
Include_Tag_Key On
Replace_Dots On
Logstash_Prefix kubernetes_cluster
Generate_ID On
[OUTPUT]
Name es
Match system.*
Host infra-es-elasticsearch-client
Port 9200
Logstash_Format On
Retry_Limit False
Type flb_type
Time_Key @timestamp
Include_Tag_Key On
Replace_Dots On
Logstash_Prefix syslog
Generate_ID On
#Trace_Output On
[OUTPUT]
Name forward
Match slowlog.*
Host infra-fluentd
Port 24224
Retry_Limit false
[OUTPUT]
Name forward
Match auditlog.*
Host infra-fluentd
Port 24224
Retry_Limit false
fluent-bit.conf: |-
@INCLUDE fluent-bit-service.conf
@INCLUDE fluent-bit-input.conf
@INCLUDE fluent-bit-filter.conf
@INCLUDE fluent-bit-output.conf
parsers.conf: |-
[PARSER]
Name pingcap_log
Format regex
Regex \[(?<time>\d{4}\/\d{2}\/\d{2}.\d{2}:\d{2}:\d{2}.\d{3}).\+\d{2}:\d{2}\].\[(?<level>\w*)\].\[(?<source>\w.*):(?<line>\d*)\].(?<message>.*$)
Time_Key time
Time_Format %Y/%m/%d %H:%M:%S.%L
[PARSER]
Name sysmsg
Format regex
Regex /^(?<time>[^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:[^\:]*\:)? *(?<message>.*)$/
Time_Format %Y-%m-%dT%T.%N%z
Time_Key time
Time_Keep On
Raw
fluent-bit-ds.yaml
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
name: fluent-bit
labels:
app: fluent-bit
spec:
updateStrategy:
type: RollingUpdate
template:
metadata:
labels:
app: fluent-bit
spec:
hostNetwork: false
dnsPolicy: ClusterFirst
serviceAccountName: fluent-bit
containers:
- name: fluent-bit
image: "fluent/fluent-bit:1.1.0"
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: varlog
mountPath: /var/log
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
- name: etcmachineid
mountPath: /etc/machine-id
readOnly: true
- name: config
mountPath: /fluent-bit/etc/fluent-bit.conf
subPath: fluent-bit.conf
- name: config
mountPath: /fluent-bit/etc/fluent-bit-service.conf
subPath: fluent-bit-service.conf
- name: config
mountPath: /fluent-bit/etc/fluent-bit-input.conf
subPath: fluent-bit-input.conf
- name: config
mountPath: /fluent-bit/etc/fluent-bit-filter.conf
subPath: fluent-bit-filter.conf
- name: config
mountPath: /fluent-bit/etc/fluent-bit-output.conf
subPath: fluent-bit-output.conf
- name: config
mountPath: /fluent-bit/etc/parsers_custom.conf
subPath: parsers.conf
- mountPath: /data1/var/lib/docker/containers
name: dockerroot
readOnly: true
volumes:
- name: varlog
hostPath:
path: /var/log
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: etcmachineid
hostPath:
path: /etc/machine-id
type: File
- name: config
configMap:
name: fluent-bit-config
- hostPath:
path: /data1/var/lib/docker/containers
name: dockerroot
Raw
fluentd-config.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: fluentd
labels:
app: fluentd
data:
fluent.conf: |-
<source>
@type forward
port 24224
bind 0.0.0.0
</source>
# Prevent fluentd from handling records containing its own logs. Otherwise
# it can lead to an infinite loop, when error in sending one message generates
# another message which also fails to be sent and so on.
<match fluentd.**>
@type null
</match>
# Used for health checking
<source>
@type http
port 9880
bind 0.0.0.0
</source>
# Emits internal metrics to every minute, and also exposes them on port
# 24220. Useful for determining if an output plugin is retryring/erroring,
# or determining the buffer queue length.
<source>
@type monitor_agent
bind 0.0.0.0
port 24220
tag fluentd.monitor.metrics
</source>
<filter slowlog.**>
@type concat
key log
separator ""
stream_identity_key docker_id
multiline_start_regexp /^# Time/
multiline_end_regexp /.*;\n/
</filter>
<match slowlog.**>
@id elasticsearch
@type elasticsearch
@log_level info
include_tag_key true
# Replace with the host/port to your Elasticsearch cluster.
host "#{ENV['OUTPUT_HOST']}"
port "#{ENV['OUTPUT_PORT']}"
scheme "#{ENV['OUTPUT_SCHEME']}"
ssl_version "#{ENV['OUTPUT_SSL_VERSION']}"
logstash_format true
logstash_prefix slowlog
<buffer>
@type file
path /var/log/fluentd-buffers/kubernetes.system.buffer
flush_mode interval
retry_type exponential_backoff
flush_thread_count 2
flush_interval 5s
retry_forever
retry_max_interval 30
chunk_limit_size "#{ENV['OUTPUT_BUFFER_CHUNK_LIMIT']}"
queue_limit_length "#{ENV['OUTPUT_BUFFER_QUEUE_LIMIT']}"
overflow_action block
</buffer>
</match>
<filter auditlog.**>
@type parser
key_name log
reserve_data true
<parse>
@type regexp
expression /^.*\[TIMESTAMP=(?<timestamp>.*)\] \[EVENT_CLASS=(?<event>.*)\] \[EVENT_SUBCLASS=.* \[COST_TIME=(?<cost_time>.*)\] \[HOST=(?<host>.*)\] \[CLIENT_IP=(?<client>.*)\] \[USER=(?<user>.*)] \[DATABASES="\[(?<db>.?*)]"] \[TABLES="\[(?<table>.?*)]"] \[SQL_TEXT="(?<sql>.?*)"] \[ROWS=(?<row>\d?*)\].*$/
time_key timestamp
time_type string
time_format %Y/%m/%d %H:%M:%S.%L
</parse>
</filter>
<match auditlog.**>
@id auditlog
@type elasticsearch
@log_level debug
include_tag_key true
# Replace with the host/port to your Elasticsearch cluster.
host "#{ENV['OUTPUT_HOST']}"
port "#{ENV['OUTPUT_PORT']}"
scheme "#{ENV['OUTPUT_SCHEME']}"
ssl_version "#{ENV['OUTPUT_SSL_VERSION']}"
logstash_format true
logstash_prefix auditlog
<buffer>
@type file
path /var/log/fluentd-buffers/auditlog
flush_mode interval
retry_type exponential_backoff
flush_thread_count 2
flush_interval 5s
retry_forever
retry_max_interval 30
chunk_limit_size "#{ENV['OUTPUT_BUFFER_CHUNK_LIMIT']}"
queue_limit_length "#{ENV['OUTPUT_BUFFER_QUEUE_LIMIT']}"
overflow_action block
</buffer>
</match>
#<match **>
# @type stdout
#</match>
<system>
root_dir /tmp/fluentd-buffers/
log_level debug
</system>
Raw
fluentd-deploy.yaml
apiVersion: apps/v1beta1
kind: Deployment
metadata:
name: fluentd
labels:
app: fluentd
spec:
replicas: 2
selector:
matchLabels:
app: fluentd
template:
metadata:
labels:
app: fluentd
spec:
containers:
- name: fluentd
image: "gcr.io/google-containers/fluentd-elasticsearch:v2.3.2"
imagePullPolicy: IfNotPresent
env:
- name: OUTPUT_HOST
value: "es-elasticsearch-client"
- name: OUTPUT_PORT
value: "9200"
- name: OUTPUT_SCHEME
value: "http"
- name: OUTPUT_SSL_VERSION
value: "TLSv1"
- name: OUTPUT_BUFFER_CHUNK_LIMIT
value: "2M"
- name: OUTPUT_BUFFER_QUEUE_LIMIT
value: "8"
ports:
- name: forwarder-tcp
containerPort: 24224
protocol: TCP
- name: forwarder-udp
containerPort: 24224
protocol: UDP
- name: monitor-agent
containerPort: 24220
protocol: TCP
- name: http-input
containerPort: 9880
protocol: TCP
livenessProbe:
httpGet:
# Use percent encoding for query param.
# The value is {"log": "health check"}.
# the endpoint itself results in a new fluentd
# tag 'fluentd.pod-healthcheck'
path: /fluentd.pod.healthcheck?json=%7B%22log%22%3A+%22health+check%22%7D
port: 9880
initialDelaySeconds: 5
timeoutSeconds: 1
volumeMounts:
- name: config-volume-fluentd
mountPath: /etc/fluent/config.d
- name: buffer
mountPath: "/var/log/fluentd-buffers"
volumes:
- name: config-volume-fluentd
configMap:
name: fluentd
- name: buffer
emptyDir: {}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment