CB2 failing test shows that CB2 stays in half-open state forever after opening once

acceptance_test.rb

require "spec_helper"
require "thread"

describe "CB2 acceptance test" do
  WORKERS = 1
  RESPONSE_TIME_SEC = 0.01

  def make_breaker(redis)
    CB2::Breaker.new(
      service: "test service",
      strategy: :rolling_window,
      duration: 3,
      threshold: 10, # differs from percentage; threshold is an absolute number of failures within window
      reenable_after: 2,
      redis: redis
    )
  end

  before do
    @requests_queue = Queue.new
    @responses_queue = Queue.new
  end

  Request = Struct.new(:action, :duration)

  it "does all the things" do
    debug "== normal mode of operation =="
    workers = run_workers

    n = 50
    send_n_requests(n, :succeed)
    assert_equal n, get_n_responses(n).count { |r| r == :succeeded }, "expected #{n} requests to succeed"

    debug "== failure mode of operation =="
    n = 10
    send_n_requests(n, :fail)
    assert_equal n, get_n_responses(n).count { |r| r == :failed }, "expected #{n} requests to fail"

    debug "== circuit is open =="
    n = 50
    send_n_requests(n, :succeed)
    assert_equal n, get_n_responses(n).count { |r| r == :skipped }, "expected #{n} requests to be skipped by circuit breaker"

    sleep 3 # wait till circuit surely half-closes

    debug "== resume normal operation =="
    send_n_requests(25, :succeed)
    send_n_requests(5, :fail)
    send_n_requests(70, :succeed)
    responses = get_n_responses(100)
    assert_equal 0, responses.count { |r| r == :skipped }, "expected no requests to be skipped"
    assert_equal 5, responses.count { |r| r == :failed }, "expected 5 requests to fail"
    assert_equal 95, responses.count { |r| r == :succeeded }, "expected 20 requests to succeed"

    workers.each(&:kill)
  end

  def send_n_requests(n, action)
    n.times { @requests_queue.push(Request.new(action, RESPONSE_TIME_SEC)) }
  end

  def get_n_responses(n)
    n.times.map { @responses_queue.pop }
  end

  def run_workers
    WORKERS.times.map do
      Thread.new(@requests_queue, @responses_queue) do |requests, responses|
        redis = Redis.new
        breaker = make_breaker(redis)
        loop do
          request = requests.pop
          begin
            breaker.run do
              raise "service unavailable" if request.action == :fail
              responses.push :succeeded
              debug "success"
            end
          rescue CB2::BreakerOpen
            responses.push :skipped
            debug "breaker open"
          rescue
            responses.push :failed
            debug "failure"
          end
          sleep request.duration
        end
      end
    end
  end

  def debug(msg)
    puts "[#{Thread.current.object_id}] #{msg}" if ENV["DEBUG"]
  end
end