marianogg9

def ec2_node_exists?
    states = get_valid_states
    filtered_instances = ec2.servers.select { |s| states.include?(s.state) }
    instance_names = filtered_instances.collect { |s| s.name }
    instance_names.each do |name|
      return true if name == @event['client']['name']
    end
    false # no match found, node doesn't exist
  end

marianogg9

input {
      s3 {
        bucket => "bucket_name"
        credentials => ["access","secret"]
        delete => "false"
        interval => "60"
        #prefix => "logs/"
        #'region' => 'us-east-1'
        region_endpoint => "us-east-1"
        type => "cloudtrail"

marianogg9

Automatic template management enabled {:manage_template=>"true", :level=>:info, :file=>"logstash/outputs/elasticsearch.rb", :line=>"258"}
Using mapping template {:template=>"{
    \"template\": \"logstash-*\",
    \"settings\": {
        \"index.refresh_interval\": \"5s\"
    },
    \"mappings\": {
        \"_default_\": {
            \"_all\": {
                \"enabled\": true

marianogg9

Plugin is finished {:plugin=><LogStash::Inputs::S3 bucket=>"bucket_name", credentials=>["access_key", "secret_key"], region_endpoint=>"us-east-1", type=>"elb-access", sincedb_path=>"/dev/null">, :level=>:info, :file=>"logstash/plugin.rb", :line=>"59"}
A plugin had an unrecoverable error. Will restart this plugin.
  Plugin: <LogStash::Inputs::S3 bucket=>"bucket_name", credentials=>["access_name", "secret_key"], region_endpoint=>"us-east-1", type=>"elb-access", sincedb_path=>"/dev/null">
  Error: no time information in ""
  Exception: ArgumentError
  Stack: file:/path/logstash-1.4.1/vendor/jar/jruby-complete-1.7.11.jar!/META-INF/jruby.home/lib/ruby/1.9/time.rb:267:in `parse'
/path/logstash-1.4.1/lib/logstash/inputs/s3.rb:261:in `sincedb_read'
/path/logstash-1.4.1/lib/logstash/inputs/s3.rb:147:in `process_new'
/path/logstash-1.4.1/lib/logstash/inputs/s3.rb:137:in `run'
org/jruby/RubyKernel.java:1521:in `loop'

marianogg9

Reading config file {:file=>"logstash/agent.rb", :level=>:debug, :line=>"301"}
Compiled pipeline code:
@inputs = []
@filters = []
@outputs = []
@input_s3_1 = plugin("input", "s3", LogStash::Util.hash_merge_many({ "bucket" => ("name".force_encoding("UTF-8")) }, { "credentials" => [("accesskey".force_encoding("UTF-8")), ("secretkey".force_encoding("UTF-8"))] }, { "delete" => ("false".force_encoding("UTF-8")) }, { "interval" => 60 }, { "region_endpoint" => ("us-east-1".force_encoding("UTF-8")) }, { "type" => ("elb-access".force_encoding("UTF-8")) }))

@inputs << @input_s3_1
@filter_date_2 = plugin("filter", "date", LogStash::Util.hash_merge_many({ "match" => [("eventTime".force_encoding("UTF-8")), ("ISO8601".force_encoding("UTF-8"))] }, { ("remove_tag".force_encoding("UTF-8")) => [("splitted".force_encoding("UTF-8"))] }, { ("add_tag".force_encoding("UTF-8")) => [("cloudtrail".force_encoding("UTF-8"))] }))

marianogg9

{
         "eventVersion" => "1.01",
              "eventID" => "ID",
            "eventTime" => "2014-05-19T02:12:02Z",
    "requestParameters" => nil,
            "eventName" => "DescribeDBInstances",
     "responseElements" => nil,
            "awsRegion" => "us-east-1",
          "environment" => "prod",
         "userIdentity" => {

marianogg9

input {

     redis {
       host => "redis_server"
       data_type => "list"
       key => "cloudtrail"
       type => "json"
       }
}

marianogg9

input {
      s3 {
        bucket => "bucket_name"
        credentials => ["access_key","secret_key"]
        delete => "false"
        interval => 60
        prefix => "logs/"
        #'region' => 'us-east-1'
        region_endpoint => "us-east-1"
        type => "json"

marianogg9

Reading config file {:file=>"logstash/agent.rb", :level=>:debug, :line=>"301"}
Compiled pipeline code:
@inputs = []
@filters = []
@outputs = []
@input_redis_1 = plugin("input", "redis", LogStash::Util.hash_merge_many({ "host" => ("server".force_encoding("UTF-8")) }, { "data_type" => ("list".force_encoding("UTF-8")) }, { "key" => ("cloudtrail".force_encoding("UTF-8")) }, { "type" => ("json".force_encoding("UTF-8")) }))

@inputs << @input_redis_1
@filter_json_2 = plugin("filter", "json", LogStash::Util.hash_merge_many({ ("source".force_encoding("UTF-8")) => ("message".force_encoding("UTF-8")) }))

marianogg9

cloudtrail.conf:

input {
      s3 {
        'bucket' => 'bucket_name'
        'credentials' => ['access_key','secret_key']
        'delete' => false
        'interval' => '30'
        'prefix' => 'logs/'
        #'region' => 'us-east-1'