DelayedJob DeadlockRetryer

readme.md

Deadlock Retryer

when you decorate a class with retry_deadlocks_from :such_a_locker, :so_many_locks, :lock_it_to_me the deadlocks will be retried using a small random delay. THIS MEANS THAT THEY CAN NO LONGER BE TREATED SYNCHRONOUSLY

deadlock_retryer.rb

# Deadlock Retryer
# ================
# 
# when you decorate a class with retry_deadlocks_from :such_a_locker, :so_many_locks, :lock_it_to_me
# the deadlocks will be retried using a small random delay.  THIS MEANS THAT THEY CAN NO LONGER BE TREATED SYNCHRONOUSLY

module DeadlockRetryer
  def retry_deadlocks_from(*method_names)
    method_names -= retry_deadlocks.to_a 
    retry_deadlocks.merge method_names

    method_names.each do |method_name|
      original_method_name = "original_#{method_name}"
      while methods.include?(original_method_name)
        original_method_name = "original_#{method_name}_#{rand}"
      end

      alias_method original_method_name, method_name
      define_method(method_name) do |*args, &block|
        begin
          send original_method_name, *args
        rescue ActiveRecord::StatementInvalid => e
          # I see two different exceptions in Airbrake.
          # I think the second message is from mysql 5.5 and we now use 5.6 (just a guess)
          if e.message =~ /Deadlock found when trying to get lock/ || e.message =~ /Lock wait timeout exceeded; try restarting transaction/
            #random to stop deadlocks from retrying at same time.
            self.delay(run_at: rand(5).seconds.from_now).send method_name, *args
          else
            raise e
          end
        end
      end
    end
  end

  def retry_deadlocks
    @methods_retry_deadlocks ||= Set.new
  end
end

deadlock_retyer_spec.rb

require 'spec_helper'

class DeadlockProneClass
  extend DeadlockRetryer

  attr_accessor :deadlock_message

  
  def sometimes_i_deadlock  
    raise_deadlock if @deadlocked
    :butts
  end
  retry_deadlocks_from :sometimes_i_deadlock

  def deadlock_message
    @deadlock_message || 'Deadlock found when trying to get lock'
  end

  def deadlock!
    @deadlocked = true
  end

  def dont_deadlock!
    @deadlocked = false
  end

  private

  def raise_deadlock
    raise ActiveRecord::StatementInvalid, deadlock_message if @deadlocked
  end
end

describe DeadlockRetryer do
  describe 'retry_deadlocks_from :method_name' do
    let(:instance) { DeadlockProneClass.new }
    context 'with no deadlock' do
      before { instance.dont_deadlock! }
      it 'calls the original function' do
        instance.sometimes_i_deadlock.should == :butts
      end
    end

    context 'with a deadlock' do
      before { instance.deadlock! }
      it 'delays the original function call' do
        mock_delay = double('mock_delay').as_null_object
        instance.stub(delay: mock_delay)
        mock_delay.should_receive(:sometimes_i_deadlock)
        instance.sometimes_i_deadlock
      end
    end
  end
end