Twitter Client with control loop, If any too many requests error raised. It'll sleep for a certain time period and will wake up and continue from last batch. This is an addition to Twitter::Client 'gem twitter'

twitter_client_patch.rb

# Twitter Client with control loop, If any too many requests error raised. 
# It'll sleep for a certain time period and will wake up and continue from last batch.
# 
# This is an addition to Twitter::Client 'gem twitter'
module BeatDeckMachine::Scrapers::OAuth::TwitterClientPatch

  MAX_ATTEMPTS = 20

  extend ActiveSupport::Concern

  included do
    attr_accessor :cursor
  end

  def each_item(&block)
    while has_next?
      attempts = 0

      begin
        attempts += 1
        find_in_batch.each { |item| yield item }

      rescue Twitter::Error::TooManyRequests => error
        if attempts <= MAX_ATTEMPTS
          # Based on https://github.com/sferik/twitter#rate-limiting
          # NOTE: Your process could go to sleep for up to 15 minutes but if you
          # retry any sooner, it will almost certainly fail with the same exception.
          puts "Attempt #{attempts}: Sleeping for #{error.rate_limit.reset_in}"
          sleep error.rate_limit.reset_in
          retry
        else
          raise error
        end
      end
    end
  end

  def find_in_batch
    offset = cursor.nil? ? next_cursor : cursor.next_cursor

    self.cursor = @client.send(@method_name.to_sym, @method_options.merge(:cursor => offset))
    self.cursor.collection
  end

  def has_next?
    (cursor.nil? && next_cursor > 0) || (cursor && cursor.next_cursor > 0)
  end

  Twitter::Cursor.send :include, BeatDeckMachine::Scrapers::OAuth::TwitterClientPatch
end