Logstash filter to be used with the Ruby filter: https://www.elastic.co/guide/en/logstash/current/plugins-filters-ruby.html#plugins-filters-ruby-path

gzip-decode-filter.rb

# Copyright 2018 Ry Biesemeyer (@yaauie)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.]

def register(params)
  params = params.dup
  # charset: the charset of the compressed data (default: `UTF-8`)
  @charset = params.delete('charset') || 'UTF-8'
  # source: the field from which to extract data
  @source = params.delete('source') || fail(LogStash::ConfigurationError, 'missing `source`')

  # target: the field into which the result will go
  @target = params.delete('target') || fail(LogStash::ConfigurationError, 'missing `target`')

  # base64: whether to decode-64 before decompressing
  base64 = params.delete('base64') || 'true'
  @base64 = case base64
            when 'true', true then true
            when 'false', false then false
            else
              fail(LogStash::ConfigurationError, "invalid value for `base64`: `#{base64.inspect}`")
            end

  params.empty? || fail(LogStash::ConfigurationError, "unexpected params: #{params}")

  @converter = LogStash::Util::Charset.new(@charset)

  require 'stringio'
end

def filter(event)
  data = event.get(@source)

  if !data.nil?
    data = Base64.decode64(data) if @base64
    decoder = Zlib::GzipReader.new(StringIO.new(data))
    value = ""
    decoder.each_line do |line|
      value << @converter.convert(line)
    end
    event.set(@target, value)
  end
rescue Zlib::Error => e
  logger.trace("Zlib Error decoding an event", exception: e.message, backtrace: e.backtrace)
  event.tag('_gzipdecompressionerror')
rescue => e
  logger.warn("Unhandled Exception", exception: e.message, backtrace: e.backtrace)
ensure
  return [event]
end

test 'basic decode' do
  parameters do
    { "source" => "message",
      "target" => "decoded" }
  end

  in_event do
    {"message" => "H4sIAEgC0FoAA8vPUMhIzFQoyUgtSuUCAA8xJ50NAAAA"}
  end

  expect("decoded") do |events|
    events.first.get('decoded') == "oh hai there\n"
  end
end

test 'error decode' do
  parameters do
    { "source" => "message",
      "target" => "decoded" }
  end

  in_event do
    {"message" => "definitely not valid"}
  end

  expect("decoded") do |events|
    events.first.get('tags').include?('_gzipdecompressionerror')
  end
end