build-rubies

build-rubies.rb

#!/usr/bin/env ruby

#
# This is a proof-of-concept script to build every commit of ruby over the past
# year and store all of the resulting builds in git. The purpose of using git
# for storage is to take advantage of its de-duplication and compression
# features.
#
# Requirements:
# - git
# - local clone of https://github.com/ruby/ruby
# - ruby-build: https://github.com/sstephenson/ruby-build
# - rugged gem: https://github.com/libgit2/rugged
#
# This script will build each commit of Ruby, starting with the HEAD commit of
# your local clone and going back 1 year.
#
# Built Rubies will be stored in a git repository, where each commit maps to one
# source commit. Built artifacts will be stored in a directory called 'work'
# under the CWD. work/ruby-builds will contain the git repository with the
# builds.
#
# To get a reliable estimate of the space taken up by all of the builds, be sure
# to run 'git gc' in the work/ruby-builds directory before measuring disk usage,
# since this will give git a chance to do de-duping and re-compression.
#
# Invoke this script with a since argument - the path to the local clone of
# ruby.

require 'fileutils'
require 'rugged'

class RubyBuilder
  attr_reader :repo_path

  def initialize(base_dir, repo_path)
    @base_dir = base_dir
    FileUtils.mkdir_p(base_dir)
    @repo_path = repo_path
    @src_archive_path = File.join(@base_dir, "ruby-dev.tar.gz")
    @builds_repo_path = File.join(@base_dir, "ruby-builds")
    @definitions_path = File.join(@base_dir, "definitions")

    @ruby_package_name = "ruby-dev"
    @build_definition_path = File.join(@definitions_path, @ruby_package_name)
    ensure_builds_repo
  end

  def ensure_builds_repo
    if !File.exist?(@builds_repo_path)
      FileUtils.mkdir_p(@builds_repo_path)
      system("cd #{@builds_repo_path} && git init")
    end
  end

  def archive_source(commit)
    prefix = "#{@ruby_package_name}/"
    puts "Archiving from #{@repo_path} to #{@src_archive_path}"
    cmd = "cd #{@repo_path} && git archive --format tar #{commit.oid} --prefix #{prefix} | gzip >#{@src_archive_path}"
    puts "  #{cmd}"
    system(cmd)
  end

  def generate_build_definition
    definition = <<-END
      install_package "openssl-1.0.1e" "https://www.openssl.org/source/openssl-1.0.1e.tar.gz#66bf6f10f060d561929de96f9dfe5b8c" mac_openssl --if has_broken_mac_openssl
      install_package "#{@ruby_package_name}" "#{@src_archive_path}" ldflags_dirs autoconf standard verify_openssl
    END
    FileUtils.mkdir_p(@definitions_path)
    File.open(@build_definition_path, "w") do |f|
      f.write(definition)
    end
  end

  def with_env(env)
    old_values = {}
    env.keys.each { |key| old_values[key] = ENV[key] }
    env.each { |key, value| ENV[key] = value }
    yield
    old_values.each { |key, value| ENV[key] = value }
  end

  def invoke_ruby_build(commit)
    puts "Building Ruby at #{commit.oid} from #{@build_definition_path} into #{@builds_repo_path}"
    build_invocation = "ruby-build #{@build_definition_path} #{@builds_repo_path}"
    puts "  #{build_invocation}"
    env = {
      'RUBY_BUILD_CACHE_PATH' => @base_dir,
      'CONFIGURE_OPTS' => '--disable-install-doc'
    }
    with_env(env) { system(build_invocation) }
  end

  def clean_builds_repo
    puts "Cleaning #{@builds_repo_path}"
    Dir[File.join(@builds_repo_path, "*")].each { |e| FileUtils.rmtree(e) }
  end

  def commit_to_builds_repo(src_commit)
    commit_msg  = "Build of ruby from #{src_commit.oid}\n"
    commit_msg << "\n"
    commit_msg << "Original commit message:\n"
    commit_msg << "#{src_commit.message}"
    commit_msg_file = File.join(@base_dir, "commit.msg")
    File.open(commit_msg_file, "w") { |f| f.write(commit_msg) }

    puts "Committing to #{@builds_repo_path}"
    add_cmd = "cd #{@builds_repo_path} && git add --all ."
    system(add_cmd)
    commit_cmd = "cd #{@builds_repo_path} && git commit -F #{commit_msg_file}"
    system(commit_cmd)
  end

  def build_ruby_version(commit)
    archive_source(commit)
    generate_build_definition
    clean_builds_repo
    invoke_ruby_build(commit)
    commit_to_builds_repo(commit)
  end
end

local_ruby_repo_path = ARGV[0]
work_dir = File.expand_path('./work')
builder = RubyBuilder.new(work_dir, local_ruby_repo_path)

repo = Rugged::Repository.new(local_ruby_repo_path)
head_commit = Rugged::Branch.lookup(repo, "trunk").tip
head_sha = head_commit.oid
time_period = (60 * 60 * 24 * 365) # 1 year
earliest_timestamp = head_commit.time.to_f - time_period

walker = Rugged::Walker.new(repo)
walker.sorting(Rugged::SORT_TOPO)
walker.push(head_sha)
walker.each do |c|
  break if c.time.to_f < earliest_timestamp
  builder.build_ruby_version(c)
end