logstash meetup.com stream

logstash-meetup.conf

# curl -s http://stream.meetup.com/2/rsvps | ./bin/logstash -f logstash-meetup.conf
input {
  stdin {
    codec => json_lines
  }
}

# the lonlat array is needed for the map widget in kibana to work
# so we have to work a bit around the supplied data
filter {
  if [venue][lat] and [venue][lon] {
    # copy field, then merge to create array for bettermap
    mutate {
      add_field => [ "[venue][lonlat]", "%{[venue][lon]}",
                     "tmplat", "%{[venue][lat]}" ]
    }
    mutate {
      merge => [ "[venue][lonlat]", "tmplat" ]
    }
    mutate {
      convert => [ "[venue][lonlat]", "float" ]
      remove_field => [ "tmplat" ]
    }
  }


  # metrics, baby!
  # lets see the distribution by country if possible
  metrics {
    meter => "meetup.country.%{[group][group_country]}"
    meter => "meetup.country.total"
    add_tag => "metric"
    flush_interval => 60
  }
}

output {
  if "metric" in [tags] {
    stdout {}
    elasticsearch {
      index => "metrics-meetup-%{+YYYY.MM.dd}"
    }
  }
  if "metric" not in [tags] {
    elasticsearch {
      # CHANGE THIS PATH!
      template => '/Users/alr/elastic/presentations/2018/paderborn-technology-meetup/6.3.0/logstash-6.3.0/logstash-meetup.json'
      template_name => 'meetup'
      template_overwrite => true
      index => "meetup-%{+YYYY.MM.dd}"
    }
  }
}

logstash-meetup.json

{
  "index_patterns" : "meetup*",
  "settings" : {
    "number_of_shards" : 1
  },
  "mappings" : {
    "doc" : {
      "properties" : {
        "venue" : {
          "properties" : {
            "lonlat" : { "type": "geo_point" },
            "lat" : { "type": "float" },
            "lon" : { "type": "float" }
          }
        }
      }
    }
  }
}