replicate-gem-inabox.rb

#!/usr/bin/env ruby
# frozen_string_literal: true

require 'net/http'
require 'open-uri'
require 'tempfile'

require File.expand_path('ruby_bash', __dir__).to_s
options = {}

parser = OptionParser.new do |opts|
  opts.banner = 'Usage: replicate gem inabox'

  opts.on('-f', '--from [HOST]', String) do |host|
    options[:from_host] = host
  end

  opts.on('-t', '--to [HOST]', String) do |host|
    options[:to_host] = host
  end

  opts.on('-d', '--deep-scan') do
    options[:deep_scan] = true
  end

  opts.on_tail('-h', '--help', 'Prints this help') do
    puts opts
    exit
  end
  options
end
parser.parse!

class DownloadGemInabox
  TOTAL_THREADS = 10
  Gem = Struct.new(:name, :version, :server, :download_path) do
    def older_versions_path
      "/gems/#{name}"
    end
  end
  attr_reader :to, :from, :deep_scan, :total_replicated, :completed

  def initialize(to, from, deep_scan)
    @to = URI(to)
    @from = URI(from)
    @deep_scan = deep_scan
    @total_replicated = 0
    @completed = 0
    @tries = 0
  end

  def call
    if `gem list | grep geminabox`.empty?
      putsl 'Geminabox not installed. running gem install geminabox'
      `gem install geminabox`
    end

    to_paths = to_gems_urls.map(&:download_path)
    putsl "Collected Gem Versions for #{to}"

    from_gem_urls.each do |gem|
      @completed += 1
      if to_paths.include?(gem.download_path)
        printl "#{progress} Already Exists #{gem.name}(#{gem.version})"
        next
      end
      printl "#{progress} Downloading #{gem.name}(#{gem.version})"
      file = download(gem)
      printl "#{progress} Uploading #{gem.name}(#{gem.version})"
      system!("gem inabox #{file.path} -g #{to}")
      file.unlink
      printl "#{progress} Uploaded #{gem.name}(#{gem.version})"
      @total_replicated += 1
    end
    if total_replicated == 1
      putsl "Finished and replicated #{total_replicated} gem"
    elsif total_replicated > 1
      putsl "Finished and replicated #{total_replicated} gems"
    else
      putsl 'Everything is in sync nothing to replicate'
    end
  end

  private

  def progress
    "#{((completed / total_from_gems) * 100).round(1)}%"
  end

  def thread_flat_map(array)
    threads = []
    array.each_slice((array.size.to_f / TOTAL_THREADS).ceil) do |group|
      threads << Thread.new do
        group.flat_map do |item|
          yield(item)
        end
      end
    end
    threads.each(&:join).flat_map(&:value)
  end

  def total_from_gems
    @total_from_gems ||= from_gem_urls.size.to_f
  end

  def download(gem)
    Tempfile.new(gem.name).tap do |file|
      Net::HTTP.start(gem.server.host, gem.server.port, use_ssl: gem.server.is_a?(URI::HTTPS)) do |http|
        request = Net::HTTP::Get.new gem.download_path
        file.write(http.request(request).body)
      end
      file.close
    end
  end

  def to_gems_urls
    @to_gems_urls ||= gem_urls(to, deep_scan: true)
  end

  def from_gem_urls
    @from_gem_urls ||= gem_urls(from, deep_scan: deep_scan)
  end

  def gem_urls(uri, deep_scan:)
    page = get_index(uri)
    gem_names = page.scan(%r{<h2>(.*)</h2>}).flatten.map { |x| x.split(' ')[0] }
    thread_flat_map(gem_names) { |name| create_gems(page, name, uri, deep_scan) }
  end

  def create_gems(page, name, uri, deep_scan)
    page = get_older_versions(uri, name) if page.match("/gems/#{name}.*more_link") && deep_scan
    page.scan(/gem install #{name}  -v "(.*)"/).flatten.flat_map do |version|
      page.scan(%r{/gems/#{name}-#{version}.*.gem}).uniq.map { |x| Gem.new(name, version, uri, x) }
    end
  end

  def get_older_versions(uri, name)
    printl "Scanning all Versions #{uri} - #{name}"
    Net::HTTP.start(uri.host, uri.port, use_ssl: uri.is_a?(URI::HTTPS)) do |http|
      request = Net::HTTP::Get.new "/gems/#{name}"
      http.request(request).body
    end
  end

  def get_index(uri)
    Net::HTTP.start(uri.host, uri.port, use_ssl: uri.is_a?(URI::HTTPS)) do |http|
      request = Net::HTTP::Get.new uri
      http.request(request).body
    end
  end

  def printl(text)
    text = text.ljust(@last_line.length) if @last_line && @last_line.length > text.length
    @last_line = text
    print("#{text}\r")
  end

  def putsl(text)
    puts if @last_line
    @last_line = nil
    puts(text)
  end
end
begin
  Errors.add('From host is missing --from') if options[:from_host].to_s == ''
  Errors.add('To host is missing --to') if options[:to_host].to_s == ''
  abort_and_display_errors
  DownloadGemInabox.new(options[:to_host], options[:from_host], options[:deep_scan]).call
rescue Interrupt => e
  putsl 'Exiting...'
  exit 1
end