es_data_test.rb

require 'open-uri'
require 'json'
require 'net/http'

# EventsIndex.purge!
# EventsIndex.reset!

# Event.create!(
#   occured_at: Time.now - 1.day,
#   original_payload: {
#     "email": "[email protected]",
#     "event": "processed",
#     "smtp-id": "234",
#     "subject": "123"
#   }
# )
# Event.create!(
#   occured_at: Time.now,
#   original_payload: {
#     "email": "[email protected]",
#     "event": "delivered",
#     "smtp-id": "234",
#     "subject": "123"
#   }
# )

# EventsIndex.reset!

RESULTS_PER_REQUEST = 9000
INITIAL_REQUEST_URI = URI('http://localhost:9200/development_events/_search?scroll=1m')
SCROLL_REQUEST_URI = URI('http://localhost:9200/_search/scroll')
TEMPLATE = <<EOS
{
  "query": {
    "filtered": {
      "query": {
        "match_all": {
        }
      },
      "filter": {
        "and": [
          {
            "range": {
              "occured_at": {
                "lte": "2016-11-07"
              }
            }
          },
          {
            "match_all": {
            }
          }
        ]
      }
    }
  },
  "sort": [
    {
      "occured_at": "asc"
    }
  ],
  "_source": [
    "id"
  ],
  "size": %d,
  "from": 0
}
EOS
SCROLL_TEMPLATE = <<EOS
{
    "scroll" : "1m",
    "scroll_id" : "%s"
}
EOS

def records
  data = initial_request
  begin
    remaining = data["hits"]["total"]
    data["hits"]["hits"].each {|h|
      yield Event.find(Integer(h["_id"]))
    }
    data = scroll_request(data["_scroll_id"])
  end while data["hits"]["hits"].size > 0
end

def scroll_request(id)
  request(SCROLL_REQUEST_URI, sprintf(SCROLL_TEMPLATE, id))
end

def initial_request
  request(INITIAL_REQUEST_URI, sprintf(TEMPLATE, RESULTS_PER_REQUEST))
end

def request(uri, body)
  req = Net::HTTP::Post.new(uri)
  req.body = body
  req.content_type = 'application/json'
  res = Net::HTTP.start(uri.hostname, uri.port) do |http|
    http.request(req)
  end

  case res
  when Net::HTTPSuccess, Net::HTTPRedirection
    return JSON.parse(res.body)
  else
    raise res.value
  end
end

highest_occured_at_data = Time.now - 2.day
records do |e|
  if e.occured_at > highest_occured_at_data
    highest_occured_at_data = e.occured_at
  end
end

puts highest_occured_at_data