Fetches proxies from multiple urls and test them using multiple threads. Results are saved across runs to avoid re-testing known proxies.

proxy_tester.rb

require 'net/http'
require 'uri'
require 'logger'
require 'parallel'

# Fetch proxies from multiple sources and test them against a specified URL, measuring response code and latency.
# Results are saved and reloaded on subsequent calls to avoid retesting recently tested proxies.
# Working proxies are saved in a separate text file with their response code and latency for easy reference.
class ProxyTester
  # known exceptions that can be raised when testing a http proxy
  HANDLED_EXCEPTIONS = [
    EOFError,
    Errno::ECONNREFUSED,
    Errno::ECONNRESET,
    Errno::EHOSTUNREACH,
    Net::HTTPBadResponse,
    Net::HTTPClientException,
    Net::HTTPFatalError,
    Net::HTTPRetriableError,
    Net::OpenTimeout,
    Net::ReadTimeout,
    OpenSSL::SSL::SSLError
  ].freeze

  class TrapSafeLogger
    def initialize(logger)
      @logger = logger
      @queue = Queue.new

      @thread = Thread.new do
        loop do
          level, message = @queue.pop
          @logger.send(level, message)
        end
      end
    end

    def stop
      sleep 0.1 unless @queue.empty? # Give the logger thread a moment to process remaining messages
      @thread.kill if @thread
    end

    %i[info error warn debug].each do |level|
      define_method(level) { |message| @queue << [level, message] }
    end
  end

  def initialize(lists:, url_to_test:, in_threads: 100, proxied_request_options: { open_timeout: 5, read_timeout: 5 }, logger: nil)
    @url_to_test = URI(url_to_test)
    @lists = lists.map { |list| URI(list) }
    @in_threads = in_threads

    # Logger by itself does not work inside `trap` context. See: https://github.com/resque/resque/issues/1493
    @logger = TrapSafeLogger.new(logger || Logger.new($stderr))

    @proxied_request_options = proxied_request_options.merge({ use_ssl: @url_to_test.scheme == "https" })
    @tested_proxies = 0
  end

  def call
    @results = load_previous_results || {}
    @proxies = fetch_proxies(except: @results.keys)

    trap("INT") { process_results; exit }

    Parallel.each(
      @proxies,
      in_threads: @in_threads,
      finish: ->(proxy, _i, (code, latency)) { @tested_proxies += 1; @results[proxy] = [code, latency, Time.now.to_i] }
    ) do |proxy|
      now = Time.now.to_f
      response = test_proxy(proxy)
      print "." if response
      [response&.code, Time.now.to_f - now]
    end

    process_results

    @logger.stop
  end

  private

  def load_previous_results
    return unless File.exist?(".#{__FILE__}.results")

    previous_result = eval(File.binread(".#{__FILE__}.results"))
    return unless previous_result.is_a?(Hash)

    now = Time.now.to_i
    previous_result.select! { |_, (_, _, timestamp)| now - timestamp < 24 * 3600 } # Only keep results from the last 24 hours
    @logger.info "Loaded #{previous_result.size} previous results"
    previous_result
  rescue Exception => e
    @logger.error "Failed to load previous results: #{e.message}"
  end

  def process_results
    File.binwrite(".#{__FILE__}.results", @results)

    @logger.info "Tested #{@tested_proxies} proxies."
    working_proxies = @results.select { |_, (code, _, _)| (200..299).include?(code.to_i) }
    file = File.open("working_proxies.txt", "w")
    @logger.info "#{working_proxies.size} working proxies found:"
    working_proxies.sort_by(&:last).each do |proxy, (code, latency, _)|
      line = "* #{proxy} - #{code} - #{latency.round(2)}s"
      file.puts line
    @logger.info line
    end
  ensure
    file.close if file
  end

  def fetch_proxies(except: [])
    proxies = @lists.flat_map do |uri|
      proxy_response = Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |http|
        http.request(Net::HTTP::Get.new(uri))
      end
      partial_proxy_list = proxy_response.body.split("\n").filter_map do |line|
        next $1 if line.match(/([\d\.]+:\d+)/)

        @logger.error "[#{uri}] Skipping invalid line: #{line}"
      end

      @logger.info "[#{uri}] Found #{partial_proxy_list.size} proxies."
      partial_proxy_list
    end

    unique_proxies = proxies.uniq
    @logger.info "Total proxies: #{proxies.size} (#{unique_proxies.size} unique)."

    proxies_to_test = unique_proxies - except
    @logger.info "Will test #{proxies_to_test.size} proxies (skipping proxies already tested)."

    proxies_to_test
  end

  def test_proxy(proxy)
    addr, port = proxy.split(":")
    Net::HTTP.start(@url_to_test.host, @url_to_test.port, addr, port, **@proxied_request_options) do |http|
      http.request(Net::HTTP::Get.new(@url_to_test.request_uri))
    end
  rescue *HANDLED_EXCEPTIONS
  end
end

ProxyTester.new(
  url_to_test: 'https://api.coinbase.com/api/v3/brokerage/time',
  lists: [
    # https://github.com/proxifly/free-proxy-list
    'https://cdn.jsdelivr.net/gh/proxifly/free-proxy-list@main/proxies/protocols/http/data.txt',

    # https://github.com/TheSpeedX/PROXY-List
    'https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/http.txt'
  ]
).call