gistfile1.txt

Reading config file {:file=>"logstash/agent.rb", :level=>:debug, :line=>"301"}
Compiled pipeline code:
@inputs = []
@filters = []
@outputs = []
@input_redis_1 = plugin("input", "redis", LogStash::Util.hash_merge_many({ "host" => ("server".force_encoding("UTF-8")) }, { "data_type" => ("list".force_encoding("UTF-8")) }, { "key" => ("cloudtrail".force_encoding("UTF-8")) }, { "type" => ("json".force_encoding("UTF-8")) }))

@inputs << @input_redis_1
@filter_json_2 = plugin("filter", "json", LogStash::Util.hash_merge_many({ ("source".force_encoding("UTF-8")) => ("message".force_encoding("UTF-8")) }))

@filters << @filter_json_2
@filter_split_3 = plugin("filter", "split", LogStash::Util.hash_merge_many({ ("add_tag".force_encoding("UTF-8")) => [("splitted".force_encoding("UTF-8"))] }, { ("add_tag".force_encoding("UTF-8")) => [("cloudtrail".force_encoding("UTF-8"))] }, { ("field".force_encoding("UTF-8")) => ("Records".force_encoding("UTF-8")) }))

@filters << @filter_split_3
@filter_date_4 = plugin("filter", "date", LogStash::Util.hash_merge_many({ ("match".force_encoding("UTF-8")) => [("eventTime".force_encoding("UTF-8")), ("ISO8601".force_encoding("UTF-8"))] }, { ("remove_tag".force_encoding("UTF-8")) => [("splitted".force_encoding("UTF-8"))] }, { ("add_tag".force_encoding("UTF-8")) => [("cloudtrail".force_encoding("UTF-8"))] }))

@filters << @filter_date_4
@filter_ruby_5 = plugin("filter", "ruby", LogStash::Util.hash_merge_many({ ("code".force_encoding("UTF-8")) => ("[\"responseElements\",\"requestParameters\"].each { |field| event[field] = event[field].inspect }  ".force_encoding("UTF-8")) }))

@filters << @filter_ruby_5
@output_stdout_6 = plugin("output", "stdout")

@outputs << @output_stdout_6
@output_elasticsearch_7 = plugin("output", "elasticsearch", LogStash::Util.hash_merge_many({ "cluster" => ("logstash".force_encoding("UTF-8")) }, { "host" => ("server".force_encoding("UTF-8")) }, { "embedded" => ("false".force_encoding("UTF-8")) }, { "protocol" => ("http".force_encoding("UTF-8")) }))

@outputs << @output_elasticsearch_7
  @filter_func = lambda do |event, &block|
    extra_events = []
    @logger.debug? && @logger.debug("filter received", :event => event.to_hash)
    if ((!((((x = event["[tags]"]; x.respond_to?(:include?) && x.include?(("splitted".force_encoding("UTF-8")))))))))
      newevents = []
      extra_events.each do |event|
        @filter_json_2.filter(event) do |newevent|
          newevents << newevent
        end
      end
      extra_events += newevents
      @filter_json_2.filter(event) do |newevent|
        extra_events << newevent
      end
      if event.cancelled?
        extra_events.each(&block)
        return
      end
        newevents = []
      extra_events.each do |event|
        @filter_split_3.filter(event) do |newevent|
          newevents << newevent
        end
      end
      extra_events += newevents
      @filter_split_3.filter(event) do |newevent|
        extra_events << newevent
      end
      if event.cancelled?
        extra_events.each(&block)
        return
      end

    end
    if (((((x = event["[tags]"]; x.respond_to?(:include?) && x.include?(("splitted".force_encoding("UTF-8"))))))))
      newevents = []
      extra_events.each do |event|
        @filter_date_4.filter(event) do |newevent|
          newevents << newevent
        end
      end
      extra_events += newevents
      @filter_date_4.filter(event) do |newevent|
        extra_events << newevent
      end
      if event.cancelled?
        extra_events.each(&block)
        return
      end
        newevents = []
      extra_events.each do |event|
        @filter_ruby_5.filter(event) do |newevent|
          newevents << newevent
        end
      end
      extra_events += newevents
      @filter_ruby_5.filter(event) do |newevent|
        extra_events << newevent
      end
      if event.cancelled?
        extra_events.each(&block)
        return
      end

    end

    extra_events.each(&block)
  end
  @output_func = lambda do |event, &block|
    @logger.debug? && @logger.debug("output received", :event => event.to_hash)
    @output_stdout_6.handle(event)
    @output_elasticsearch_7.handle(event)

  end {:level=>:debug, :file=>"logstash/pipeline.rb", :line=>"26"}
Using milestone 2 input plugin 'redis'. This plugin should be stable, but if you see strange behavior, please let us know! For more information on plugin milestones, see http://logstash.net/docs/1.4.1/plugin-milestones {:level=>:warn, :file=>"logstash/config/mixin.rb", :line=>"209"}
config LogStash::Codecs::JSON/@charset = "UTF-8" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Inputs::Redis/@host = "server" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Inputs::Redis/@data_type = "list" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Inputs::Redis/@key = "cloudtrail" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Inputs::Redis/@type = "json" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Inputs::Redis/@debug = false {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Inputs::Redis/@codec = <LogStash::Codecs::JSON charset=>"UTF-8"> {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Inputs::Redis/@add_field = {} {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Inputs::Redis/@threads = 1 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Inputs::Redis/@name = "default" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Inputs::Redis/@port = 6379 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Inputs::Redis/@db = 0 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Inputs::Redis/@timeout = 5 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Inputs::Redis/@batch_count = 1 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
Using milestone 2 filter plugin 'json'. This plugin should be stable, but if you see strange behavior, please let us know! For more information on plugin milestones, see http://logstash.net/docs/1.4.1/plugin-milestones {:level=>:warn, :file=>"logstash/config/mixin.rb", :line=>"209"}
config LogStash::Filters::Json/@source = "message" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Filters::Json/@type = "" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Filters::Json/@tags = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Filters::Json/@exclude_tags = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Filters::Json/@add_tag = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Filters::Json/@remove_tag = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Filters::Json/@add_field = {} {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Filters::Json/@remove_field = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
Using milestone 2 filter plugin 'split'. This plugin should be stable, but if you see strange behavior, please let us know! For more information on plugin milestones, see http://logstash.net/docs/1.4.1/plugin-milestones {:level=>:warn, :file=>"logstash/config/mixin.rb", :line=>"209"}
config LogStash::Filters::Split/@add_tag = ["splitted", "cloudtrail"] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Filters::Split/@field = "Records" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Filters::Split/@type = "" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Filters::Split/@tags = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Filters::Split/@exclude_tags = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Filters::Split/@remove_tag = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Filters::Split/@add_field = {} {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Filters::Split/@remove_field = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Filters::Split/@terminator = "\n" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Filters::Date/@match = ["eventTime", "ISO8601"] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Filters::Date/@remove_tag = ["splitted"] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Filters::Date/@add_tag = ["cloudtrail"] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Filters::Date/@type = "" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Filters::Date/@tags = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Filters::Date/@exclude_tags = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Filters::Date/@add_field = {} {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Filters::Date/@remove_field = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Filters::Date/@target = "@timestamp" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
Using milestone 1 filter plugin 'ruby'. This plugin should work, but would benefit from use by folks like you. Please let us know if you find bugs or have suggestions on how to improve this plugin.  For more information on plugin milestones, see http://logstash.net/docs/1.4.1/plugin-milestones {:level=>:warn, :file=>"logstash/config/mixin.rb", :line=>"209"}
config LogStash::Filters::Ruby/@code = "[\"responseElements\",\"requestParameters\"].each { |field| event[field] = event[field].inspect }  " {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Filters::Ruby/@type = "" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Filters::Ruby/@tags = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Filters::Ruby/@exclude_tags = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Filters::Ruby/@add_tag = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Filters::Ruby/@remove_tag = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Filters::Ruby/@add_field = {} {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Filters::Ruby/@remove_field = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Codecs::Line/@charset = "UTF-8" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Outputs::Stdout/@type = "" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Outputs::Stdout/@tags = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Outputs::Stdout/@exclude_tags = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Outputs::Stdout/@codec = <LogStash::Codecs::Line charset=>"UTF-8"> {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Outputs::Stdout/@workers = 1 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Codecs::Plain/@charset = "UTF-8" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Outputs::ElasticSearch/@cluster = "logstash" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Outputs::ElasticSearch/@host = "server" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Outputs::ElasticSearch/@embedded = false {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Outputs::ElasticSearch/@protocol = "http" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Outputs::ElasticSearch/@type = "" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Outputs::ElasticSearch/@tags = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Outputs::ElasticSearch/@exclude_tags = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Outputs::ElasticSearch/@codec = <LogStash::Codecs::Plain charset=>"UTF-8"> {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Outputs::ElasticSearch/@workers = 1 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Outputs::ElasticSearch/@index = "logstash-%{+YYYY.MM.dd}" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Outputs::ElasticSearch/@manage_template = true {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Outputs::ElasticSearch/@template_name = "logstash" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Outputs::ElasticSearch/@template_overwrite = false {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Outputs::ElasticSearch/@document_id = nil {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Outputs::ElasticSearch/@embedded_http_port = "9200-9300" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Outputs::ElasticSearch/@max_inflight_requests = 50 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Outputs::ElasticSearch/@flush_size = 5000 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Outputs::ElasticSearch/@idle_flush_time = 1 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
config LogStash::Outputs::ElasticSearch/@action = "index" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}
Registering Redis {:identity=>"default", :level=>:info, :file=>"logstash/inputs/redis.rb", :line=>"84"}
Adding type with date config {:type=>"", :field=>"eventTime", :format=>"ISO8601", :level=>:debug, :file=>"logstash/filters/date.rb", :line=>"165"}
Pipeline started {:level=>:info, :file=>"logstash/pipeline.rb", :line=>"78"}
filter received {:event=>{"eventVersion"=>"1.0", "eventName"=>"DescribeInstances", "eventTime"=>"2014-01-02T12:09:16Z", "requestParameters"=>{"instancesSet"=>{"items"=>[{"instanceId"=>"id"}]}, "filterSet"=>{}}, "responseElements"=>"<responseOmitted>", "awsRegion"=>"us-east-1", "environment"=>"prod", "userIdentity"=>{"userName"=>"name", "principalId"=>"id", "accessKeyId"=>"access_id", "type"=>"IAMUser", "arn"=>"arn", "accountId"=>"accountid"}, "eventSource"=>"ec2.amazonaws.com", "userAgent"=>"agent", "sourceIPAddress"=>"ip", "type"=>"cloudtrail", "@version"=>"1", "@timestamp"=>"2014-05-19T20:17:44.946Z"}, :level=>:debug, :file=>"(eval)", :line=>"27"}
Running json filter {:event=>#<LogStash::Event:0x711185e7 @accessors=#<LogStash::Util::Accessors:0x356e3aaf @store={"eventVersion"=>"1.0", "eventName"=>"DescribeInstances", "eventTime"=>"2014-01-02T12:09:16Z", "requestParameters"=>{"instancesSet"=>{"items"=>[{"instanceId"=>"id"}]}, "filterSet"=>{}}, "responseElements"=>"<responseOmitted>", "awsRegion"=>"us-east-1", "environment"=>"prod", "userIdentity"=>{"userName"=>"name", "principalId"=>"id", "accessKeyId"=>"accesid", "type"=>"IAMUser", "arn"=>"arn", "accountId"=>"accountid"}, "eventSource"=>"ec2.amazonaws.com", "userAgent"=>"agent", "sourceIPAddress"=>"IP", "type"=>"cloudtrail", "@version"=>"1", "@timestamp"=>"2014-05-19T20:17:44.946Z"}, @lut={"type"=>[{"eventVersion"=>"1.0", "eventName"=>"DescribeInstances", "eventTime"=>"2014-01-02T12:09:16Z", "requestParameters"=>{"instancesSet"=>{"items"=>[{"instanceId"=>"id"}]}, "filterSet"=>{}}, "responseElements"=>"<responseOmitted>", "awsRegion"=>"us-east-1", "environment"=>"prod", "userIdentity"=>{"userName"=>"name", "principalId"=>"id", "accessKeyId"=>"accesid", "type"=>"IAMUser", "arn"=>"arn", "accountId"=>"accountid"}, "eventSource"=>"ec2.amazonaws.com", "userAgent"=>"agent", "sourceIPAddress"=>"IP", "type"=>"cloudtrail", "@version"=>"1", "@timestamp"=>"2014-05-19T20:17:44.946Z"}, "type"], "[tags]"=>[{"eventVersion"=>"1.0", "eventName"=>"DescribeInstances", "eventTime"=>"2014-01-02T12:09:16Z", "requestParameters"=>{"instancesSet"=>{"items"=>[{"instanceId"=>"id"}]}, "filterSet"=>{}}, "responseElements"=>"<responseOmitted>", "awsRegion"=>"us-east-1", "environment"=>"prod", "userIdentity"=>{"userName"=>"name", "principalId"=>"id", "accessKeyId"=>"id", "type"=>"IAMUser", "arn"=>"arn", "accountId"=>"id"}, "eventSource"=>"ec2.amazonaws.com", "userAgent"=>"agent", "sourceIPAddress"=>"ip", "type"=>"cloudtrail", "@version"=>"1", "@timestamp"=>"2014-05-19T20:17:44.946Z"}, "tags"]}>, @data={"eventVersion"=>"1.0", "eventName"=>"DescribeInstances", "eventTime"=>"2014-01-02T12:09:16Z", "requestParameters"=>{"instancesSet"=>{"items"=>[{"instanceId"=>"id"}]}, "filterSet"=>{}}, "responseElements"=>"<responseOmitted>", "awsRegion"=>"us-east-1", "environment"=>"prod", "userIdentity"=>{"userName"=>"name", "principalId"=>"id", "accessKeyId"=>"id", "type"=>"IAMUser", "arn"=>"arn", "accountId"=>"id"}, "eventSource"=>"ec2.amazonaws.com", "userAgent"=>"agent", "sourceIPAddress"=>"IP", "type"=>"cloudtrail", "@version"=>"1", "@timestamp"=>"2014-05-19T20:17:44.946Z"}, @cancelled=false>, :level=>:debug, :file=>"logstash/filters/json.rb", :line=>"59"}
Exception in filterworker {"exception"=>#<NoMethodError: undefined method `split' for nil:NilClass>, "backtrace"=>["/dir/logstash-1.4.1/lib/logstash/filters/split.rb:43:in `filter'", "(eval):50:in `initialize'", "org/jruby/RubyProc.java:271:in `call'", "/dir/logstash-1.4.1/lib/logstash/pipeline.rb:262:in `filter'", "/dir/logstash-1.4.1/lib/logstash/pipeline.rb:203:in `filterworker'", "/dir/logstash-1.4.1/lib/logstash/pipeline.rb:143:in `start_filters'"], :level=>:error, :file=>"logstash/pipeline.rb", :line=>"212"}
Plugin is finished {:plugin=><LogStash::Filters::Json source=>"message">, :level=>:info, :file=>"logstash/plugin.rb", :line=>"59"}
Plugin is finished {:plugin=><LogStash::Filters::Split add_tag=>["splitted", "cloudtrail"], field=>"Records", terminator=>"\n">, :level=>:info, :file=>"logstash/plugin.rb", :line=>"59"}
Plugin is finished {:plugin=><LogStash::Filters::Date match=>["ISO8601"], remove_tag=>["splitted"], add_tag=>["cloudtrail"], target=>"@timestamp">, :level=>:info, :file=>"logstash/plugin.rb", :line=>"59"}
Plugin is finished {:plugin=><LogStash::Filters::Ruby code=>"[\"responseElements\",\"requestParameters\"].each { |field| event[field] = event[field].inspect }  ">, :level=>:info, :file=>"logstash/plugin.rb", :line=>"59"}
New Elasticsearch output {:cluster=>"logstash", :host=>"server", :port=>"9200", :embedded=>false, :protocol=>"http", :level=>:info, :file=>"logstash/outputs/elasticsearch.rb", :line=>"252"}
Automatic template management enabled {:manage_template=>"true", :level=>:info, :file=>"logstash/outputs/elasticsearch.rb", :line=>"258"}
Using mapping template {:template=>"{  \"template\" : \"logstash-*\",  \"settings\" : {    \"index.refresh_interval\" : \"5s\"  },  \"mappings\" : {    \"_default_\" : {       \"_all\" : {\"enabled\" : true},       \"dynamic_templates\" : [ {         \"string_fields\" : {           \"match\" : \"*\",           \"match_mapping_type\" : \"string\",           \"mapping\" : {             \"type\" : \"string\", \"index\" : \"analyzed\", \"omit_norms\" : true,               \"fields\" : {                 \"raw\" : {\"type\": \"string\", \"index\" : \"not_analyzed\", \"ignore_above\" : 256}               }           }         }       } ],       \"properties\" : {         \"@version\": { \"type\": \"string\", \"index\": \"not_analyzed\" },         \"geoip\"  : {           \"type\" : \"object\",             \"dynamic\": true,             \"path\": \"full\",             \"properties\" : {               \"location\" : { \"type\" : \"geo_point\" }             }         }       }    }  }}", :level=>:info, :file=>"logstash/outputs/elasticsearch.rb", :line=>"284"}