logstash enrichment example

enrichment.conf

input {
  syslog  {}
}

filter {
	if [program] == '(squid-1)' {
		grok {
			match => { "message" => ".*?\s(?<bytes_out>\d+)\s(?<src_ip>\d+\.\d+\.\d+\.\d+)\s.*?\/(?<status_code>\d+)\s(?<bytes_in>\d+)\s(?<request_method>\w+)\s(?<request_url>.*?)\s-\s.*?\/(?<dst_ip>\d+\.\d+\.\d+\.\d+)\s(?<file_type>.*?)$" }
		}

		mutate {
			convert => { "bytes_in" => "integer" }
			convert => { "bytes_out" => "integer" }
			convert => { "status_code" => "integer" }
		}

		if [request_url] {
			ruby {
				init => " require 'uri' "
				code => "
					url = event['request_url'].gsub('[', '').gsub(']', '').gsub('_', '-')
					my_uri = URI(url)
					event['domain'] = my_uri.host
					event['request_url_length'] = url.length
					event['request_url_entropy'] = url.each_char.group_by(&:to_s).values.map{|x|x.length/url.length.to_f}.reduce(0){|e,x|e-x*Math.log2(x)}
				"
			}
		}

		if [dst_ip] =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/ {
			geoip {
				source => "dst_ip"
				fields => ["country_name", "location"]
			}
		}
   }


  if [program] == 'dnsmasq'{
    grok {
      match => { "message" => "\d+\s(?<src_ip>.*?)\/.*?\s(?<dns_action>.*?)\s(?<domain>.*?)\s\w+\s(?<dns_response>.*)" }
    }
  }

	if [dns_response] =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/ {
		geoip {
			source => "dns_response"
			fields => ["country_name", "location"]
		}
	}


  if [domain] {
    tld {
      source => "domain"
		remove_field => "[tld][subdomain]"
    }

    if [tld][sld] {
      ruby {
        code => "
          sld = event['[tld][sld]']
          event['[tld][sld_length]'] = sld.length
          event['[tld][sld_entropy]'] = sld.each_char.group_by(&:to_s).values.map{|x|x.length/sld.length.to_f}.reduce(0){|e,x|e-x*Math.log2(x)}
        "
      }
    }

    if [tld][trd] {
      ruby {
        code => "
          trd = event['[tld][trd]']
          event['[tld][trd_length]'] = trd.length
          event['[tld][trd_entropy]'] = trd.each_char.group_by(&:to_s).values.map{|x|x.length/trd.length.to_f}.reduce(0){|e,x|e-x*Math.log2(x)}
        "
      }
    }


  }

}

output {

	if [program] == 'dnsmasq' {
		elasticsearch {
			index => "logstash-dns-%{+YYYY.MM.dd}"
		}
	}

	if [program] == '(squid-1)' {
		elasticsearch {
			index => "logstash-squid-%{+YYYY.MM.dd}"
		}
	}

	# stdout { codec => rubydebug }
	# elasticsearch {}
}