agent.conf

input {
  exec {
     type => "dstat"
     command => "dstat -cdngypms --nocolor 1 0"
     interval => 13
   }

   exec {
      type => "apache-benchmark"
      command => "ab  -t 5 -n 1 -c 1 -d -S 'https://spratshop-rails.spratshop.com/' | grep -E '(requests:|Time taken)'"
      interval => 17
    }

  file {
    type => "linux-syslog"
    path => [ 
      "/var/log/alternatives.log",
      "/var/log/auth.log",
      "/var/log/daemon.log",
      "/var/log/dpkg.log",
      "/var/log/faillog",
      "/var/log/fontconfig.log",
      "/var/log/kern.log",
      "/var/log/lastlog",
      "/var/log/lpr.log",
      "/var/log/mail.log",
      "/var/log/mysql.log",
      "/var/log/pycentral.log",
#      "/var/log/rabbitmq_logger.log",
      "/var/log/user.log",

       "/var/log/messages", 
       "/var/log/syslog" 
     ]
    
    
  }

  file {
    type => "apache-access"
    path => [
      "/var/log/apache2/access.log",
      "/var/log/apache2/eth0-0-NOS-1-001_spratshop_images-access.log",
      "/var/log/apache2/eth0-0-NOS-1-002_spratshop_rails-access.log",
      "/var/log/apache2/eth0-0-SSL-1-001_spratshop_images-access.log",
      "/var/log/apache2/eth0-0-SSL-1-002_spratshop_rails-access.log",
      "/var/log/apache2/other_vhosts_access.log"
    ]    
  }
  
  file {
    type => "apache-rewrite"
    path => [
      "/var/log/apache2/eth0-0-NOS-1-002_spratshop_rails-rewrite.log",
      "/var/log/apache2/eth0-0-SSL-1-002_spratshop_rails-rewrite.log",
      "/var/log/apache2/rewrite.log"
    ]    
  }

  file {
    type => "apache-error"
    path => [
      "/var/log/apache2/error.log",
      "/var/log/apache2/eth0-0-NOS-1-001_spratshop_images-error.log",
      "/var/log/apache2/eth0-0-NOS-1-002_spratshop_rails-error.log",
      "/var/log/apache2/eth0-0-SSL-1-001_spratshop_images-error.log",
      "/var/log/apache2/eth0-0-SSL-1-002_spratshop_rails-error.log"
    ]
  }
  
  file {
    type => "rabbitmq-post-bridge"
    path => [
    "/var/log/rabbitmq_post_bridge.log"
    ]
  }
  
  file {
    type => "spratshop-rails"
    path => [
    "/opt/spratshop/rails/shared/log/production.log",
    "/opt/spratshop/rails/shared/log/development.log"
    ]
  }
  
  file {
    type => "spratshop-file-sender"
    path => [
    "/var/log/spratshop_file_sender.log"
    ]
  }
  
  file {
    type => "spratshop-ftp-retry-cron"
    path => [
    "/var/log/spratshop_ftp_retry_cron.log"
    ]
  }
  
  file {
    type => "spratshop-status-checker"
    path => [
    "/var/log/spratshop_status_checker.log"
    ]
  }
  
  file {
    type => "spratshop-usage-cron"
    path => [
    "/var/log/spratshop_usage_cron.log"
    ]
  }
  
  file {
    type => "vsftpd"
    path => [
    "/var/log/vsftpd.log"
    ]
  }
  
  file {
    type => "vsftpd-to-rabbitmq"
    path => [
    "/var/log/vsftpd_to_rabbitmq.log"
    ]
  }
  
  file {
    type => "spratshop-image-processor"
    path => [
      "/var/log/spratshop_image_processor-1.log",
      "/var/log/spratshop_image_processor-2.log",
      "/var/log/spratshop_image_processor-3.log",
      "/var/log/spratshop_image_processor-4.log",
      "/var/log/spratshop_image_processor.log"
    ]
  }
  
 
  amqp {      
      host => ...
      user => ...
      password => ...
      
      name => "from-vsftpd"
      key => "spratshop"
      type => "from-vsftpd"
      # message_format => ""
      
      durable => true
      exchange_type => "direct"
      format => "json"
      
      queue_durable => false
      queue_name => "logstash_from-vsftpd_spratshop"
      
      # port => ... # number, default: 5672
      # ssl => ... # boolean
      # tags => ... # array
      # 
      # verify_ssl => ... # boolean
      # vhost => ... # string, default: "/"
    }
  amqp {      
      host => ...
      user => ...
      password => ...
      
      name => "incoming-images"
      key => "spratshop"
      type => "incoming-images"
      # message_format => ""
      
      durable => true
      exchange_type => "direct"
      format => "json"
      
      queue_durable => false
      queue_name => "logstash_incoming-images_spratshop"
      
      # port => ... # number, default: 5672
      # ssl => ... # boolean
      # tags => ... # array
      # 
      # verify_ssl => ... # boolean
      # vhost => ... # string, default: "/"
    }
  amqp {      
      host => ...
      user => ...
      password => ...
      
      name => "processed-images"
      key => "spratshop"
      type => "processed-images"
      # message_format => ""
      
      durable => true
      exchange_type => "direct"
      format => "json"
      
      queue_durable => false
      queue_name => "logstash_processed-images_spratshop"
      
      # port => ... # number, default: 5672
      # ssl => ... # boolean
      # tags => ... # array
      # 
      # verify_ssl => ... # boolean
      # vhost => ... # string, default: "/"
    }
  amqp {      
      host => ...
      user => ...
      password => ...
      
      name => "send-file-todo"
      key => "spratshop"
      type => "send-file-todo"
      # message_format => ""
      
      durable => true
      exchange_type => "direct"
      format => "json"
      
      queue_durable => false
      queue_name => "logstash_send-file-todo_spratshop"
      
      # port => ... # number, default: 5672
      # ssl => ... # boolean
      # tags => ... # array
      # 
      # verify_ssl => ... # boolean
      # vhost => ... # string, default: "/"
    }
  amqp {      
      host => ...
      user => ...
      password => ...
      
      name => "sent-files"
      key => "spratshop"
      type => "sent-files"
      # message_format => ""
      
      durable => true
      exchange_type => "direct"
      format => "json"
      
      queue_durable => false
      queue_name => "logstash_sent-files_spratshop"
      
      # port => ... # number, default: 5672
      # ssl => ... # boolean
      # tags => ... # array
      # 
      # verify_ssl => ... # boolean
      # vhost => ... # string, default: "/"
    }
  amqp {      
      host => ...
      user => ...
      password => ...
      
      name => "status-checker-to-system"
      key => "spratshop"
      type => "status-checker-to-system"
      # message_format => ""
      
      durable => true
      exchange_type => "direct"
      format => "json"
      
      queue_durable => false
      queue_name => "logstash_status-checker-to-system_spratshop"
      
      # port => ... # number, default: 5672
      # ssl => ... # boolean
      # tags => ... # array
      # 
      # verify_ssl => ... # boolean
      # vhost => ... # string, default: "/"
    }
  amqp {      
      host => ...
      user => ...
      password => ...
      
      name => "status-system-to-checker"
      key => "spratshop"
      type => "status-system-to-checker"
      # message_format => ""
      
      durable => true
      exchange_type => "direct"
      format => "json"
      
      queue_durable => false
      queue_name => "logstash_status-system-to-checker_spratshop"
      
      # port => ... # number, default: 5672
      # ssl => ... # boolean
      # tags => ... # array
      # 
      # verify_ssl => ... # boolean
      # vhost => ... # string, default: "/"
    }
  
}



filter {
  grok {
    type => "linux-syslog"
    pattern => ["%{SYSLOG_SUDO}", "%{SYSLOG_KERNEL}", "%{SYSLOGLINE}" ]
  }

  date {
    type => "linux-syslog"
    timestamp => "MMM dd HH:mm:ss"
    timestamp8601 => ISO8601
  }

  grok {
    type => "apache-access"
    pattern => "%{COMBINEDAPACHELOG}"
  }
  
#  grok {
#    type => "apache-error"
#    match {
#      "APACHE_LOG_LEVEL" => "(?:emerg|alert|crit|error|warn|notice|info|debug)"
#      "APACHE_ERROR_LOG" => "\[%{DATESTAMP_OTHER:timestamp}\] \[%{APACHE_LOG_LEVEL:level}\] %{GREEDYDATA:message}"
#    }
#    pattern => "%{APACHE_ERROR_LOG}"
#  }

  multiline {
    type => "spratshop-file-sender"
    pattern => "^\\s"
    what => "previous"
  }
  multiline {
    type => "spratshop-image-processor"
    pattern => "^\\s"
    what => "previous"
  }
  multiline {
    type => "spratshop-status-checker"
    pattern => "^\\s"
    what => "previous"
  }
  
  multiline { 
    type => "spratshop-rails" 
    pattern => "^$" 
    negate => true 
    what => next 
  } 

  grep {
    type => "spratshop-rails" 
    match => [
      "@message", "^$"
    ]
    negate => true
  }

  
  multiline { 
    type => "spratshop-ftp-retry-cron" 
    pattern => "^$" 
    negate => true 
    what => next 
  }
  
}


output {
  # Emit events to stdout for easy debugging of what is going through
  # logstash.
  # stdout { }

  # This will use elasticsearch to store your logs.
  # The 'embedded' option will cause logstash to run the elasticsearch
  # server in the same process, so you don't have to worry about
  # how to download, configure, or run elasticsearch!
  # elasticsearch {
  #  cluster => "default"
  #  host => "127.0.0.1"
  #  bind_host => "127.0.0.1"
  #}

  amqp {
      host => ...
      user => ...
      password => ...

    exchange_type => "fanout"
    
    name => "rawlogs"
    durable => true
  }

}