-
-
Save busyboy77/fafd75136d2db5e94c6cc27e3759fff8 to your computer and use it in GitHub Desktop.
banzaicloud/logging-operator working ElasticSearch example with index lifecycle management
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
apiVersion: v1 | |
kind: Secret | |
metadata: | |
name: index-template | |
stringData: | |
template: '{"index_patterns":["logstash-*"],"template":{"aliases":{"logstash":{}},"mappings":{"dynamic":"true","dynamic_date_formats":["strict_date_optional_time","yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"],"dynamic_templates":[]},"settings":{"index":{"lifecycle":{"name":"logstash_policy","rollover_alias":"logstash"},"number_of_shards":"1","number_of_replicas":"0"}}}}' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# ClusterFlows to deploy | |
clusterFlows: | |
- name: all | |
spec: | |
filters: | |
- record_modifier: # if you e.g. have multiple clusters | |
records: | |
- cluster: "CLUSTER_NAME" | |
# replaces dots in labels and annotations with dashes to avoid mapping issues (app=foo (text) vs. app.kubernetes.io/name=foo (object)) | |
# fixes error: existing mapping for [kubernetes.labels.app] must be of type object but found [text] | |
- dedot: | |
de_dot_separator: "-" | |
de_dot_nested: true | |
globalOutputRefs: | |
- elasticsearch | |
# ClusterOutputs to deploy | |
clusterOutputs: | |
- name: elasticsearch | |
spec: | |
elasticsearch: | |
host: ES_HOST | |
port: EST_PORT | |
user: ES_USER | |
password: | |
valueFrom: | |
secretKeyRef: | |
name: es-password | |
key: password | |
scheme: https | |
ssl_verify: true | |
logstash_format: true | |
include_timestamp: true | |
reconnect_on_error: true | |
reload_on_failure: true | |
buffer: | |
flush_mode: interval | |
flush_interval: 60s | |
flush_at_shutdown: true | |
flush_thread_count: 4 | |
enable_ilm: true | |
ilm_policy_id: logstash_policy | |
ilm_policy: '{"policy":{"phases":{"hot":{"min_age":"0ms","actions":{"readonly":{},"rollover":{"max_size":"8gb","max_primary_shard_size":"4gb","max_age":"7d"}}},"cold":{"min_age":"7d","actions":{"freeze":{},"readonly":{},"searchable_snapshot":{"snapshot_repository":"found-snapshots","force_merge_index":true},"set_priority":{"priority":0}}}}}}' | |
# ilm_policy_overwrite: false # # when updating the existing policy, this needs to be set to true and then redeployed with false to avoid versioning of this policy every 60secs | |
template_name: "logstash_template" | |
template_file: | |
mountFrom: | |
secretKeyRef: | |
name: index-template | |
key: template | |
customize_template: '{"settings.index.number_of_replicas": "0"}' | |
# enables logging of bad request reasons within the fluentd log file (in the pod /fluentd/log/out) | |
log_es_400_reason: true |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment