fluent-bit-issues-7658

gistfile1.txt

   parsers.conf: |
     [PARSER]
        Name docker
        Format json
        Time_Keep Off
        Time_Key time
        Time_Format %Y-%m-%dT%H:%M:%S.%L
     
   -------------------
   fluent-bit.conf: |
     [SERVICE]
        Log_Level info
        Parsers_File parsers.conf
        HTTP_Server on
        HTTP_Listen 0.0.0.0
        HTTP_Port 2020
    [INPUT]
        Name tail
        Path /var/log/containers/*.log
        Exclude_Path /var/log/containers/fluent-bit*
        Parser docker
        Tag kube.*
        Buffer_Max_Size 50MB
        Mem_Buf_Limit 200MB
        Skip_Long_Lines On
        DB logging.db
        DB.locking true
    [FILTER]
        Name kubernetes
        Buffer_Size 0
        Match kube.*
        Keep_Log Off
        Merge_Log On
        Merge_Log_Key       log_processed
        K8S-Logging.Parser  On
        K8S-Logging.Exclude On
        Kube_URL            https://kubernetes.default.svc:443
        Kube_CA_File        /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        Kube_Token_File     /var/run/secrets/kubernetes.io/serviceaccount/token
        Kube_Tag_Prefix     kube.var.log.containers.
    [FILTER]
        Name          rewrite_tag
        Match         kube.*
        Rule          $kubernetes['namespace_name'] ^(test1-.*) apps_test1.out false
    [FILTER]
        Name          rewrite_tag
        Match         kube.*
        Rule          $kubernetes['namespace_name'] ^(test2-.*) apps_test2.out false
    [FILTER]
        Name          rewrite_tag
        Match         kube.*
        Rule          $kubernetes['namespace_name'] ^(test3-.*) apps_test3.out false
   
   [OUTPUT] # This server is always available, so never any issues to send logs here
        Name opensearch
        Match apps_test1.*
        Host            SERVER1       
        Port            9200
        HTTP_User       ${LOGGING_USER}
        HTTP_Passwd     ${LOGGING_PASS}
        Logstash_Format On
        Logstash_Prefix test1
        Replace_Dots On
        tls on
        tls.verify Off
        Retry_Limit False
        Buffer_Size False
        Generate_ID On
        Workers 3
        Trace_Error On
        Suppress_Type_Name On

    [OUTPUT] # This server is NOT available for the last few days, so FluentBit cannot reach it
        Name opensearch
        Match apps_test2.*
        Host            SERVER2
        Port            9200
        HTTP_User       ${LOGGING_USER}
        HTTP_Passwd     ${LOGGING_PASS}
        Logstash_Format On
        Logstash_Prefix test2
        Replace_Dots On
        tls on
        tls.verify Off
        Retry_Limit False
        Buffer_Size False
        Generate_ID On
        Workers 3
        Trace_Error On
        Suppress_Type_Name On

    [OUTPUT] # This server is available, so never any issues to send logs here. But
             # what really happens is that fluentbit only sends logs to the SERVER1 (correctly). 
             # SERVER2 is ignored (correctly), because it is down. 
             # And where the issue really lies is SERVER3 where I would expect that logs can be forwarded to
             # but it does not happen, unfortunately. 
        Name opensearch
        Match apps_test3.*
        Host            SERVER3
        Port            9200
        HTTP_User       ${LOGGING_USER}
        HTTP_Passwd     ${LOGGING_PASS}
        Logstash_Format On
        Logstash_Prefix test3
        Replace_Dots On
        tls on
        tls.verify Off
        Retry_Limit False
        Buffer_Size False
        Generate_ID On
        Workers 3
        Trace_Error On
        Suppress_Type_Name On