Overcast's logstash config for parsing log files created by our custom log4j2 config

logstash.cfg

input {
    file {
        path => "/minecraft/logs/*/server.log"
        type => "server"

        # stacktrace java as one message
        codec => multiline {
            pattern => "^([0-9\-: ]+ \[[A-Z]+\] )?(((\w+\.)+\w+Exception.*)|((\s|\t)+at .+)|(\s+... \d+ more)|(\s*Caused by:.+))"
            what => "previous"
            multiline_tag => "stacktrace"
        }
    }
}

# attempt to simplify host by removing any suffixes
filter {
    grok {
        match => [ "host", "(?<simplehost>(phx|ams)\d\d)\.?" ]
    }

    if "_grokparsefailure" in [tags] {
        mutate {
            remove_tag => [ "_grokparsefailure" ]
        }
    } else {
        mutate {
            replace => [ "host", "%{simplehost}" ]
            remove_field => "simplehost"
        }
    }
}

# extract server name from path
filter {
    grok {
        match => [ "path", "/logs/(?<server>[A-Za-z0-9-]+)/*" ]
    }

    if "_grokparsefailure" in [tags] {
        mutate {
            replace => [ "server", "unknown" ]
            remove_tag => [ "_grokparsefailure" ]
        }
    } else {
        if [host] =~ /^phx\d\d/ {
            mutate {
                replace => [ "server", "us-%{server}" ]
            }
        } else if [host] =~ /^ams\d\d/ {
            mutate {
                replace => [ "server", "eu-%{server}" ]
            }
        }

        mutate {
            remove_field => "path"
        }
    }
}

# get rid of color codes
filter {
    mutate {
        gsub => [ "message", "\e\[(0;)?([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]", "" ]
    }
}

filter {
    # match lines
    grok {
        match => [ "message", "(?m)%{DATESTAMP:minecraft_timestamp} \[%{WORD:level}\] %{GREEDYDATA:minecraft_message}" ]
        add_field => [ "received_at", "%{@timestamp}" ]
        named_captures_only => true
    }

    # read timestamp from lines
    date {
        match => [ "minecraft_timestamp", "YY-MM-dd HH:mm:ss" ]
    }

    # set message
    if "_grokparsefailure" not in [tags] {
        mutate {
            replace => [ "message", "%{minecraft_message}" ]
        }
    }

    # remove parsed fieds
    mutate {
        remove_field => [ "minecraft_message", "minecraft_timestamp" ]
    }
}

output {
    elasticsearch_http {
        host => "es"
        index => "logstash-%{+xxxx.ww}"
    }
}