logstash_main.conf

logstash_main.conf

input {
  file {
    path => ["/var/log/network.log"]
    sincedb_path => "/var/log/logstash"
    start_position => "end"
    type => "syslog"
    tags => [ "netsyslog" ]
  }
}

filter {
  grep {
    drop => true
    match => { "message" => "ethtool" }
    match => { "message" => "ps -el" }
    negate => true
  }


  if [type] == "syslog" {
    grok {
      patterns_dir => "/opt/logstash/patterns"
      match => [ "message",
        "%{SYSLOGTIMESTAMP:@timestamp} %{IP:sig_ip} %{HOST:hostname} %{GREEDYDATA:raw_message}"
      ]
      remove_field => [ "host", "path" ]
    }

  # ### secure logs
    grep {
      drop => false
      match => { "raw_message" => "dropbear" }
      add_tag => [ "secure" ]
    }
    grep {
      drop => false
      match => { "raw_message" => "sshd" }
      add_tag => [ "secure" ]
    }
    grep {
      drop => false
      match => { "raw_message" => "arp-scan" }
      add_tag => [ "secure", "arp-scan" ]
    }

  # ### messages / dhcpd / dhclient
    grep {
      drop => false
      match => { "raw_message" => "dhclient" }
      add_tag => [ "dhcp" ]
    }

    grep {
      drop => false
      match => { "raw_message" => "dhcpd" }
      add_tag => [ "dhcp" ]
    }

    if "dhcp" in [tags] {
      grep {
        drop => true
        negate => true
        match => { "raw_message" => "For info, please visit" }
        match => { "raw_message" => "Copyright" }
        match => { "raw_message" => "All rights reserved" }
        match => { "raw_message" => "Internet Systems Consortium DHCP Server" }
      }
    }
    # ***** ***** ***** ***** *****

  # ### firewall
    if "secure" not in [tags] {
       grok {
        patterns_dir => "/opt/logstash/patterns"
        match => [ "raw_message",
          "kernel: FIREWALL %{WORD:fw_action} IN=%{WORD:fw_in} OUT=%{WORD:fw_out} SRC=%{IPV4:fw_src} DST=%{IPV4:fw_dst} %{GREEDYDATA:fw_etc}"
        ]
        add_tag => [ "firewall", "sig6", "%{fw_action}" ]
        tag_on_failure => [ ]
      }

      if "firewall" in [tags] {
        grok {
          patterns_dir => "/opt/logstash/patterns"
          match => [ "fw_etc",
            "LEN=%{INT:length} TOS=%{BASE16NUM} PREC=%{BASE16NUM} TTL=%{INT:ttl} ID=%{INT:id} %{WORD} PROTO=%{WORD:proto} %{GREEDYDATA:fw_etc2}"
          ]
          add_tag => [ "%{proto}" ]
          remove_field => [ "fw_etc" ]
        }
        if [proto] == "TCP" {
          grok {
            patterns_dir => "/opt/logstash/patterns"
            match => [ "fw_etc2",
              "SPT=%{INT:sport} DPT=%{INT:dport} WINDOW=%{INT:window} RES=%{BASE16NUM} %{WORD:flag} URGP=%{INT}"
            ]
            remove_field => [ "fw_etc2" ]
          }
        } else if [proto] == "UDP" {
          grok {
            patterns_dir => "/opt/logstash/patterns"
            match => [ "fw_etc2",
              "SPT=%{INT:sport} DPT=%{INT:dport} LEN=%{INT}"
            ]
            remove_field => [ "fw_etc2" ]
          }
        } else if [proto] == "ICMP" {
          grok {
            patterns_dir => "/opt/logstash/patterns"
            match => [ "fw_etc2",
              "TYPE=%{INT:icmp_type} CODE=%{INT:icmp_code} ID=%{INT} SEQ=%{INT:icmp_seq}"
            ]
            remove_field => [ "fw_etc2" ]
          }
        }
      }
    # ***** ***** ***** ***** *****

    # ### ipsec
      mutate { gsub => ["raw_message","\"","'"] }
      grok {
        match => [ "raw_message", "pluto\[%{INT}\]\: '%{WORD:ipsec_conn}' %{GREEDYDATA:ipsec_msg}" ]
        add_tag => [ "vpn", "ipsec" ]
        tag_on_failure => []
      }
    # ***** ***** ***** ***** *****
    } # end of not secure log

  } # end of if syslog
}

output {
  elasticsearch {
    protocol => "node"
    host => "cacti.plus-i.co.kr"
    cluster => "elasticsearch"
  }
}