This is an example of using the new file based Ruby filter to create a filter that removes keys with nil values from an event.

alternate_compact_event.rb

# this is an alternate compact function implementation which also removes keys with empty string values
# writing the tests for this is left as an excercise to the reader :D

def compact(h)
  h.inject({}) do |result, (k, v)|
    if v.is_a?(Hash)
      result[k] = compact(v)
    elsif v.is_a?(String)
      result[k] = v unless v.empty?
    elsif !v.nil?
      result[k] = v
    end
    result
  end
end

alternate_compact_event2.rb

# this is anoter alternate compact function implementation which also removes empty values and nil values in arrays.

def compact(h)
  h.inject({}) do |result, (k, v)|
    case v
    when Hash
      c = compact(v)
      result[k] = c unless c.empty?
    when String
      result[k] = v unless v.empty?
    when Array
      c = v.delete_if{|e| e.nil? || (e.is_a?(String) && e.empty?)}
      result[k] = c unless c.empty?
    when NilClass
      # nothing
    else
      result[k] = v
    end
    result
  end
end

compact_event.rb

def compact(h)
  h.inject({}) do |result, (k, v)|
    unless v.nil?
      result[k] = v.is_a?(Hash) ? compact(v) : v
    end
    result
  end
end

def filter(event)
  return [LogStash::Event.new(compact(event.to_hash_with_metadata))]
end

test "remove keys with nil values" do

  in_event { { "foo" => 1, "bar" => nil, "nested" => { "baz" => nil, "biz" => "yo" }} }

  expect("return a single event") do |events|
    events.size == 1
  end

  expect("kept the foo key") do |events|
    events.first.get("[foo]") == 1
  end

  expect("kept the [nested][biz] key") do |events|
    events.first.get("[nested][biz]") == "yo"
  end

  expect("remove the bar key") do |events|
    !events.first.include?("[bar]")
  end

  expect("remove the baz key") do |events|
    !events.first.include?("[nested][baz]")
  end
end

logstash.conf

filter {
  ruby {
    path => "path/to/compact_event.rb"
  }
}

output.md

# run filter script tests

$ bin/logstash -t -e 'filter{ruby{path => "path/to/compact_event.rb"}}'

[2018-06-07T14:20:22,777][INFO ][logstash.filters.ruby.script] Test run complete {:script_path=>"path/to/compact_event.rb", :results=>{:passed=>5, :failed=>0, :errored=>0}}
Configuration OK
[2018-06-07T14:20:22,790][INFO ][logstash.runner          ] Using config.test_and_exit mode. Config Validation Result: OK. Exiting Logstash