convert-project.rb

# Converts changesets bidirectionally between two repos, presumably between a
# subdirectory in a main repo and a separate project repo. The hg convert
# extension is used once in each direction. The convert extension's shamaps
# are tracked, so you need two of them plus two filemaps to map the file names
# between repos.
#
# Required args are paths and revisions for each repo. Revs are required to avoid
# syncing unwanted branches.
#
# An optional name is used as a suffix for all the map files. It defaults to
# the basename of the project repo.
#
# shamap and filemap names are picked up (and written) in the working
# directory. shamaps will be created if they don't exist but filemaps must
# exist. Names are implied using the following scheme:
#
# - shamap-to-PROJECT
# - shamap-from-PROJECT
# - filemap-to-PROJECT
# - filemap-from-PROJECT
#
# Once you get a sync set up, just run this anytime you want to sync:
#
#    sync-project.rb MAIN-REPO REV1 PROJECT-REPO REV2 PROJECT-NAME
#
# Examples that follow show how to set up typical scenarios.
#
#   ## To create a new sync repo for foo from ./foo.
#
#   cd tools/sync/foo
#   touch shamap-to-foo shamap-from-foo
#   edit filemap-to-foo filemap-from-foo # map between ./foo and .
#   sync-project.rb `hg root` 1.0.1 foo-synced tip foo
#
#   # creates foo-synced as a subdirectory and converts all changes from the
#   # current repo's ./foo to the new repo.
#
#   # Now push both repos to their servers
#   hg push
#   cd bar-synced && hg push https://example.com/bar
#
#
#   ## To add a project repo into the main repo, clone into a subdirectory first.
#   
#   cd tools/sync/bar
#   touch shamap-to-bar shamap-from-bar
#   edit filemap-to-bar filemap-from-bar # map between ./bar and .
#   hg clone https://example.com/bar bar-synced
#   sync-project.rb `hg root` 1.0.1 bar-synced tip bar

require 'fileutils'

if not (4..5).member?(ARGV.length)
  abort "Usage: convert.rb MAIN_REPO MAIN_REV PROJECT_REPO PROJECT_REV [PROJECT]"
end

main_repo, main_rev, project_repo, project_rev, project = ARGV
project = project || File.basename(project_repo)

sha_outf = "shamap-to-#{project}"
sha_inf = "shamap-from-#{project}"
filemap_outf = "filemap-to-#{project}"
filemap_inf = "filemap-from-#{project}"

FileUtils.touch(sha_outf)
FileUtils.touch(sha_inf)

# Parses a shapmap into an array of to/from tuples.
def parse_mapfile(f)
  lines = File.readlines(f)
  pairs = lines.map{|pair|pair.split(' ')[0..1]}
  pairs
end

# Merges reversed changeset ID maps from other into base and writes back to
# base. Order is preserved.
#
# Only new keys not already in base are inserted. This helps with two
# situations:
#
# a) if we blow away the base map every time, we lose the mapping of
# changesets in from base to other that never made real changesets in other
# (excluded by the filemap, e.g. everything not in the subdirectory of
# interest) and have to try to convert them every time. This is usually
# harmless because they are noop conversions, but it means you never see a
# clean conversion output--it always thinks it has to try to convert those.
#
# b) changesets unique to other (filemap excluded from other to base) will
# map to a single changeset in base. We want protect the first instance of
# that destination changeset.
def merge_shamap(basef, otherf)
  other_pairs = parse_mapfile(otherf)

  # We want to use a hash for checking has_key? and also preserve order. Order
  # is preserved in 1.9. For earlier Ruby versions, base_map import needs to
  # be rewritten to maintain order and unique keys.
  abort "This script requires Ruby >= 1.9" unless RUBY_VERSION =~ /^(\d+)\.(\d+)\./ and ($1.to_i >= 2 or ($1.to_i >= 1 and $2.to_i >= 9))

  base_map = (File.exists?(basef) and Hash[parse_mapfile(basef)]) || {}

  # Reverse each pair in the other shamap and merge with the base. Merge only
  # new keys.
  other_pairs.each do |x,y|
    base_map[y] = x unless base_map.has_key?(y) or y == 'SKIP'
  end

  File.open(basef, 'w') do |f|
    base_map.each do |from,to|
      f.puts "#{from} #{to}"
    end
  end
end

all_hg = "-s hg -d hg"

# Pick up any ID pairs not already in the outgoing file. This should be a noop
# every time after it's first seeded.
merge_shamap(sha_inf, sha_outf)

# Convert from the project to main first, only because it was the right thing
# to do when starting with a shamap created converting a project into a
# monorepo. Once you have both shamaps seeded, the order doesn't matter.
#
# When creating a synced project repo from scratch from the monorepo, this
# command will fail the first time, but the script proceeds and the next
# command will create the project repo from scratch. Then you're good to go
# for subsequent syncs.
puts "---> converting from #{project_repo} -r #{project_rev} to #{main_repo}"
system "hg convert #{all_hg} --filemap #{filemap_inf} --rev #{project_rev} #{project_repo} #{main_repo} #{sha_inf}"
puts ''

merge_shamap(sha_outf, sha_inf)
puts "---> converting from #{main_repo} -r #{main_rev} to #{project_repo}"
system "hg convert #{all_hg} --config convert.skiptags=True --filemap #{filemap_outf} --rev #{main_rev} #{main_repo} #{project_repo} #{sha_outf}"