git_mirror

#!/usr/bin/env bash
# Auto export all variables and functions, so foreach can use them
set -eu

SELF_SOURCE="$(cd "$(dirname "${BASH_SOURCE[0]}")"; pwd)"/"$(basename "${BASH_SOURCE[0]}")"

: ${GIT=git}

function os_compatible_mktemp()
{  # Stupid mac compatibility
  local dir="${1}"
  while [[ -e ${dir} ]]; do
    dir="${1}/$(basename "$(mktemp -u)")"
  done
  mkdir -p "${dir}"
  echo "${dir}"
}

#########################
# Auto Colored sections #
#########################
function cleanup_files()
{
  if [[ -d ${TEMP_DIR-} ]]; then
    rm -rf "${TEMP_DIR}"
  fi
  if [[ -f ${COLOR_FILE-} ]]; then
    rm "${COLOR_FILE}"
  fi
  echo -n $'\e[0m'
}

function next_section()
{
  if [[ -z ${COLOR_FILE+set} ]]; then
    export COLOR_FILE="$(mktemp)"
  fi
  local COLORS=($'\e[31m' $'\e[32m' $'\e[33m' $'\e[34m' $'\e[0m')
  local -i COLOR_INDEX="$(cat "${COLOR_FILE}" || echo 0)"
  COLOR_INDEX+=1
  if [[ ${COLOR_INDEX} -ge ${#COLORS[@]} ]]; then
    COLOR_INDEX=0
  fi
  echo "${COLOR_INDEX}" > "${COLOR_FILE}"
  echo "${COLORS[$COLOR_INDEX]}"${@+"${@}"}
}

####################################################
# Get list of initialize submodules, non-recursive #
####################################################

GIT_VERSION="$(${GIT} --version)"
git_version_pattern='git version ([0-9]*)\.([0-9]*)\..*'
[[ ${GIT_VERSION} =~ ${git_version_pattern} ]]
GIT_VERSION=("${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}")

# If newer than 2.6
if [ "${GIT_VERSION[0]}" = "2" -a "${GIT_VERSION[1]}" -ge "6" ] || [[ ${GIT_VERSION[0]} -gt 2 ]]; then
  function get_submodule_names()
  {
    ${GIT} config --name-only --get-regexp '^submodule\..*\.url$' | sed 's|.url$||'
  }
else
  function get_submodule_names()
  {
    local names=()
    local line
    # Get null terminated lines

    while IFS= read -r -d '' record; do
      # Get first newline terminated line of that record, it's the name
      IFS= read -r -d $'\n' record <<< "${record}"
      names+=("${record}")
    done < <(${GIT} config -z --get-regexp '^submodule\..*\.url$')

    for line in ${names[@]+"${names[@]}"}; do
      echo "${line%.url}"
    done
  }
fi

#############
# LFS Check #
#############

HAS_LFS=1
${GIT} lfs &> /dev/null || HAS_LFS=0

#############
# Functions #
#############

# Recursive submodule helper: WARNING this would have run in sh because git is weird.
# However I start bash, and source this script for it's vars and functions, so it's
# really bash again
function clone_submodules()
{
  # Init (any) submodules
  ${GIT} submodule init

  # Submodule names can't contain newlines
  local IFS=$'\n'
  # This does not work for init only modules
  #   submodule_names=($(${GIT} submodule foreach --quiet 'echo "${name}"'))
  # This does
  local submodule_names=($(get_submodule_names))

  local prepped_submodule_path
  local full_relative_path
  local submodule

  local base_submodule_path="${base_submodule_path-}${base_submodule_path:+/}${prefix-${displaypath-}}"

  # Update submodule urls to use PREP_DIR
  for submodule in ${submodule_names[@]+"${submodule_names[@]}"}; do
    # Calculate full submodule path wrt superproject (not just parent submodule)
    full_relative_path="${base_submodule_path}${base_submodule_path:+/}$(${GIT} config -f .gitmodules ${submodule}.path)"

    # Search for existing prepped submodule
    # https://stackoverflow.com/a/52657447/4166604
    prepped_submodule_path="$(echo "${PREP_DIR}"/*/"${full_relative_path}/config")"
    # If I already have this submodule, use it
    if [[ -f ${prepped_submodule_path} ]]; then
      # Re-establish url in case it changed
      prepped_submodule_path="$(dirname "${prepped_submodule_path}")"
      pushd "${prepped_submodule_path}"&> /dev/null
        ${GIT} remote set-url origin "$(${GIT} config "${submodule}.url")"
      popd &> /dev/null
    else
      prepped_submodule_path="$(os_compatible_mktemp "${PREP_DIR}")/${full_relative_path}"
      next_section "Cloning a fresh copy of ${full_relative_path}"
      ${GIT} clone --mirror "$(${GIT} config "${submodule}.url")" "${prepped_submodule_path}"
    fi

    ${GIT} config "${submodule}.url" "${prepped_submodule_path}"
  done

  # Checkout submodule
  next_section "Updating submodules for ${prefix-${displaypath-${MAIN_REPO}}}"
  # The local non-bare doesn't need to waste time copying LFS objects
  GIT_LFS_SKIP_SMUDGE=1 ${GIT} submodule update
  # Restore the origin urls after update, so that relative URLs work
  ${GIT} submodule sync

  if [[ ${HAS_LFS} = 1 ]]; then
    next_section "Fetching lfs objects for ${prefix-${displaypath-${MAIN_REPO}}}"
    # Determine this (sub)modules' prepared path
    if [[ -z ${base_submodule_path} ]]; then
      prepped_submodule_path="${PREP_DIR}/${MAIN_DIR}"
    else
      prepped_submodule_path="$(dirname "$(echo "${PREP_DIR}"/*/"${base_submodule_path}/config")")"
    fi

    local lfs_dir="$(${GIT} rev-parse --git-dir)/lfs"
    # Incase it it doesn't exist
    mkdir -p "${lfs_dir}" "${prepped_submodule_path}/lfs/"

    # In the initial case, the non-bare repo will have an lfs folder, with
    # the current branch's objects in it. Move them to the prepped location
    if [[ -n $(ls -A "${lfs_dir}") ]]; then
      # combine the two
      cp -ra "${lfs_dir}"/* "${prepped_submodule_path}/lfs/"
    fi

    if [[ ${OS-} = Windows_NT && -n $(ls -A "${prepped_submodule_path}/lfs") ]]; then
      cp -ra "${prepped_submodule_path}"/lfs/* "${lfs_dir}/"
    else
      # Replace with symlink
      rm -rf "${lfs_dir}"
      ln -s "${prepped_submodule_path}/lfs" "${lfs_dir}"
    fi

    ${GIT} lfs fetch --all

    if [[ ${OS-} = "Windows_NT" && -n $(ls -A "${lfs_dir}") ]]; then
      cp -ra "${lfs_dir}"/* "${prepped_submodule_path}/lfs/"
    fi
  fi

  # And the recursion goes on... foreach runs in sh, I'm forcing bash
  # I need to pass prefix/displaypath on to the bash function, so export it.
  PREP_DIR="${PREP_DIR}" base_submodule_path="${base_submodule_path}" ${GIT} submodule foreach --quiet "export prefix displaypath; bash -euc 'source \"${SELF_SOURCE[0]}\"; unset GIT_DIR; clone_submodules'"
  # The "unset GIT_DIR" is due to the fact that somewhere between git 2.17 and
  # 2.21, git-submodule-foreach started setting the GIT_DIR, and this will undo
  # that. The logic here does not need GIT_DIR set, in fact it is constantly
  # cd'ing so that the commands work as expected, and this breaks those
}

function clone_from_mirror()
{
  # Arrays aren't exported, reload
  local -A repos
  source "${1}"

  local submodule_name
  local full_relative_path
  local base_submodule_path="${base_submodule_path-}${base_submodule_path:+/}${prefix-${displaypath-}}"
  # Remove trailing slashes, because on some versions of git (2.17.1), prefix
  # has a trailing slash
  base_submodule_path="${base_submodule_path%/}"
  local current_repo="${repos[${base_submodule_path:-.}]}"
  next_section "Cloning repo: ${current_repo}"

  ${GIT} submodule init
  local IFS=$'\n'
  local submodule_names=($(get_submodule_names))

  for submodule_name in ${submodule_names[@]+"${submodule_names[@]}"}; do
    full_relative_path="${base_submodule_path}${base_submodule_path:+/}$(${GIT} config -f .gitmodules ${submodule_name}.path)"
    ${GIT} config "${submodule_name}.url" "${repos[${full_relative_path}]}"
  done
  next_section "Updating ${current_repo}'s submodules"
  ${GIT} submodule update
  base_submodule_path="${base_submodule_path-}" ${GIT} submodule foreach --quiet "export prefix displaypath; bash -euc 'source \"${SELF_SOURCE[0]}\"; unset GIT_DIR; clone_from_mirror \"${1}\"'"
}

#########
# Mains #
#########

function git_mirror_main()
{
  if [[ $# = 0 ]]; then
    echo "Usage:" >&2
    echo "${BASH_SOURCE[0]} {git_repo or prep dir of last mirror clone} [branch/sha to base submodules off of]" >&2
    exit 1
  fi

  if [ -f "${1}/"*"/config" ]; then
    pushd "$(dirname "${1}/"*"/config")" &> /dev/null
      MAIN_REPO="$(${GIT} config --get remote.origin.url)"
    popd &> /dev/null
  else
    MAIN_REPO="${1}"
  fi
  BRANCH="${2-master}"

  MAIN_DIR="$(basename "${MAIN_REPO}")"
  MAIN_DIR="${MAIN_DIR%.*}"
  PREP_DIR="$(pwd)/${MAIN_DIR}_prep"

  #########################
  # Get the super project #
  #########################
  mkdir -p "${PREP_DIR}"
  if [ ! -e "${PREP_DIR}/${MAIN_DIR}" ]; then
    next_section "Cloning super project ${MAIN_REPO}..."
    ${GIT} clone --mirror "${MAIN_REPO}" "${PREP_DIR}/${MAIN_DIR}"
  else
    next_section "Fetching for super project ${MAIN_REPO} using last run..."
    pushd "${PREP_DIR}/${MAIN_DIR}" &> /dev/null
      # Re-establish url in case it changed
      ${GIT} remote set-url origin "${MAIN_REPO}"
      if [ "${GIT_VERSION[0]}" = "2" -a "${GIT_VERSION[1]}" -ge "17" ] || [[ ${GIT_VERSION[0]} -gt 2 ]]; then
        ${GIT} fetch -pP origin
      else
        ${GIT} fetch -p origin
      fi
    popd &> /dev/null
  fi

  next_section "Re-cloning locally"
  TEMP_DIR="$(mktemp -d)"
  # The local non-bare doesn't need to waste time copying LFS objects
  GIT_LFS_SKIP_SMUDGE=1 ${GIT} clone "${PREP_DIR}/${MAIN_DIR}" "${TEMP_DIR}"

  # git submodule init
  pushd "${TEMP_DIR}" &> /dev/null
    # The local non-bare doesn't need to waste time copying LFS objects
    GIT_LFS_SKIP_SMUDGE=1 ${GIT} checkout "${BRANCH}"

    # Restore origin to not point to mirror, so relative submodules work right
    ${GIT} remote set-url origin "${MAIN_REPO}"

    # This effectively does git submodule update --recursive --init,
    # But plumbs the submodules to use the "${PREP_DIR}" instead
    clone_submodules
  popd &> /dev/null

  temp_file="$(mktemp)"
  TAR_INCREMENTAL=1
  tar -cf /dev/null -g "${temp_file}" /dev/null &> /dev/null || TAR_INCREMENTAL=0
  rm "${temp_file}"


  pushd "${PREP_DIR}" &> /dev/null
    next_section "Creating tar file..."
    tar_file="transfer_$(date '+%Y_%m_%d_%H_%M_%S')"
    if [[ ${TAR_INCREMENTAL} = 1 ]]; then
      # Get the last one, alphabetically speaking
      last_tar_file="$(ls "${PREP_DIR}"/transfer_*.snar 2>/dev/null | tail -n1)"
      last_tar_file="$(basename "${last_tar_file%.snar}")"
      tar czf "${tar_file}.tgz" -g "${tar_file}.snar" */
    else
      tar czf "${tar_file}.tgz" */
    fi

    if [[ ${TAR_INCREMENTAL} = 1 ]] && [[ ${last_tar_file} != "" ]]; then
      next_section "Creating an incremental tar file too, based on ${last_tar_file}"
      tar czf "${tar_file}_${last_tar_file}.tgz" -g "${last_tar_file}.snar" */
      next_section "Your new tar file is ready:"
      echo "${PREP_DIR}/${tar_file}"
      echo "and you have an incremental file:"
      echo "${tar_file}_${last_tar_file}.tgz"
    else
      next_section "Your new tar file is ready:"
      echo "${PREP_DIR}/${tar_file}"
    fi
  popd &> /dev/null
}

function git_clone_main()
{
  local -A repos
  source "${1}"
  mkdir -p "${2-.}"
  pushd "${2-.}" &> /dev/null
    if [ ! -d ./.git ]; then
      ${GIT} clone "${repos[.]}" .
    fi
    clone_from_mirror "${1}"
  popd &> /dev/null
}

function git_push_main()
{
  local -A repos
  source "${1}"
  pushd "${2}" &> /dev/null
    repo_dir="$(dirname */config)"
    next_section "Processing main repo: ${repo_dir}"
    pushd "${repo_dir}" &> /dev/null
      ${GIT} push --mirror "${repos[.]}"

      if [[ ${HAS_LFS} = 1 ]]; then
        next_section "Pushing lfs objects for ${repo_dir}"
        ${GIT} remote add mirror "${repos[.]}" 2>/dev/null || ${GIT} remote set-url mirror "${repos[.]}"
        ${GIT} lfs push mirror --all || :
        # Does not work on file systems, only with real lfs servers
      fi
    popd &> /dev/null
    # Remove this, since we are done with it
    unset repos[.]
    # Loop through the remaining repos
    for repo_path in ${repos[@]+"${!repos[@]}"}; do
      repo_dir="$(dirname */"${repo_path}/config")"
      next_section "Processing submodule repo: ${repo_dir}"
      if [[ -d ${repo_dir} ]]; then
        pushd "${repo_dir}" &> /dev/null
          ${GIT} push --mirror "${repos[${repo_path}]}"
          if [[ ${HAS_LFS} = 1 ]]; then
            next_section "Pushing lfs objects for ${repo_dir}"
            ${GIT} remote add mirror "${repos[${repo_path}]}" 2>/dev/null || ${GIT} remote set-url mirror "${repos[${repo_path}]}"
            # Does not work on file systems, only with real lfs servers, unless you set up lfs-filestore
            ${GIT} lfs push mirror --all || :
          fi
        popd &> /dev/null
      else
        echo "No dir found for submodule ${repo_path}"
      fi
    done
  popd &> /dev/null
}

if [ "${BASH_SOURCE[0]}" = "${0}" ] || [ "$(basename "${BASH_SOURCE[0]}")" = "${0}" ]; then
  trap 'cleanup_files' exit

  arg="${1-}"
  shift 1

  case "${arg}" in
    mirror)
      git_mirror_main ${@+"${@}"}
      ;;
    push)
      git_push_main ${@+"${@}"}
      ;;
    clone)
      git_clone_main ${@+"${@}"}
      ;;
    *)
      echo "Usage:" >&2
      echo "    ${BASH_SOURCE[0]} mirror {url or prep dir}" >&2
      echo "---------------------------" >&2
      echo "    ${BASH_SOURCE[0]} push {file containting repo locations} {prep dir of git_mirror clone}" >&2
      echo >&2
      echo "Example repo location file:" >&2
      echo "---------------------------" >&2
      echo "repos[.]='https://mygithub.com/foo/superproject.git'">&2
      echo "repos[externa/submodule1]='https://mygithub.com/foo/submodule1.git'">&2
      echo "repos[externa/submodule1/submodule2]='https://mygithub.com/foo/subsubmodule2.git'">&2
      echo "---------------------------" >&2
      echo "    ${BASH_SOURCE[0]} clone {file containting repo locations} [clone location, . by default]}" >&2
      echo >&2
      echo "Example repo location file:" >&2
      echo "---------------------------" >&2
      echo "repos[.]='https://mygithub.com/foo/superproject.git'">&2
      echo "repos[externa/submodule1]='https://mygithub.com/foo/submodule1.git'">&2
      echo "repos[externa/submodule1/submodule2]='https://mygithub.com/foo/subsubmodule2.git'">&2
      exit 1

      ;;
  esac
fi

GIT_MIRROR.md

Git Mirror with submodules

While creating a git mirror is as simple as git clone --mirror, unfortunately this git command does not support git submodules (difficult) or lfs (easier). These scripts help in creating a mirror of a project with submodule and/or git lfs.

General user case:

1. You have a git repo with submodule on a git server

2. git_mirror https://github.com/visionsystemsinc/vsi_common.git master, recursively create mirrors of all submodule currently in the master branch.

3. Transfer vsi_common_prep/transfer_{date}.tgz to your destination

4. Write an info.env file:

   repos[.]=https://my_repo.com/foobar/vsi_common.git
   repos[docker/recipes]=https://my_repo.com/foobar/recipes.git

5. push_to_mirror ./info.env ./transfer_extracted_dir/

6. Now to clone from your new mirror: clone_from_mirror ./info.env ./my_project_dir/

git_mirror

Downloads a mirror of a git repo and all of its submodules. The normal git clone --mirror command does not support submodules at all. This will at least clone all the submodules available in the specified branch (master by default).

git_mirror {git repo or prep dir on subsequent calls} [git branch]

The script creates a directory: {repo_name}_prep. That directory will contain all of the repositories plus a single transfer_{date}.tgz file containing all the repositories, lfs objects, etc... Only this tgz file needs to be transferred to your destination.

Subsequent calls to git_mirror will use the existing {repo_name}_prep directory as cache, updating faster than the last fist time.

Subsequent calls also create a second tgz file, transfer_{date1}_transfer_{date2}.tgz. These are incremental files. Instead of having to bring in an entire transfer, only the incremental files are needed, plus the original full archive.

All three of these examples result in identical repos:

tar zxf transfer_2020_03_02_14_24_12.tgz

###

tar zxf transfer_2020_03_02_14_16_09.tgz
tar --incremental zxf transfer_2020_03_02_14_24_12_transfer_2020_03_02_14_16_09.tgz

###

tar zxf transfer_2020_03_02_14_07_59.tgz
tar --incremental transfer_2020_03_02_14_16_09_transfer_2020_03_02_14_07_59.tgz
tar --incremental transfer_2020_03_02_14_24_12_transfer_2020_03_02_14_16_09.tgz

push_to_mirror

Push the contents of the prep dir from git_mirror to your own mirrors.

Usage:

push_to_mirror {file setting repos array} {extracted prep dir}

Since the urls for your mirrors will different from the original repo urls, you need to create an environment file to specify the repo URLs. The main repo is referred to as . while the rest of the repos are referred to by the relative path with respect to the main repo (e.g. external/vsi_common). These need to be stored in an associative array called repos. For example:

repos[.]=https://my_repo.com/foobar/vsi_common.git
repos[docker/recipes]=https://my_repo.com/foobar/recipes.git

In this example, the main repo's mirror url is https://my_repo.com/foobar/vsi_common.git, and the submodule stored at ./docker/recipes has the url https://my_repo.com/foobar/recipes.git. This file will also be used for clone_from_mirror.

clone_from_mirror

Since the .gitmodules file will point to different urls than your mirrors, and changing the .gitmodules file will change the repo, which we don't want to do, we need to use a third script to make the initial clone of from the mirror.

Usage:

clone_from_mirror {file setting repos array} [clone path, . by default]

The main repo needs to be cloned, submodules inited, then submodule urls updated before fetching the submodules, and this has to be done one layer of submodules at a time. This can be very tedious, so this script will do all of this for you.

Limitations

• It will not pull all submodules from every version, only from a specific branch/sha/tag you specify (master by default). This is because trying to pull all submodules from the past could be very lengthy, and is very likely to include urls that do not exist anymore.

  • If you know you need submodules from multiple branches, you can always run git_mirror multiple times, deleting the transfer_{date}.tgz file each time (except the last)

• Does not support git older than 1.8. Neither does Atlassian.

• Uses bash, tested on alpine, debian, centos, and fedora. So should be fairly universal.

• Handles all valid characters for submodule paths, including spaces, tabs, and newlines, however git will not handle newlines nor paths that start or end with white space

• Handles all valid character for submodule names, including spaces, and tabs

• Support git lfs, but if you need extra configuration options to use particular lfs store or other git or lfs options, then set the environment variable GIT to call your own git script. For example git2:

#!/usr/bin/env bash
git -c lfs.customtransfer.lfs-folder.path=/tmp/lfs-folderstore-linux-amd64/lfs-folderstore"'" -c lfs.customtransfer.lfs-folder.args=/tmp/lfs/objects"'" -c lfs.standalonetransferagent=lfs-folder ${@+"${@}"}

Bugs

• Does not support sub-projects. Anyone need that? Didn't think so...

• Does not de-duplicate a submodule that may be used multiple times by other submodules

test.bsh

#!/usr/bin/env bash
set -eu

# Smoke test for testing ouy git_mirror + push_to_mirror + clone_from_mirror
# Only works on git 1.8 or newer (Uses the git -c option near the end)

CWD="$(cd "$(dirname "${BASH_SOURCE[0]}")"; pwd)"

function cleanup()
{
  IFS= read -d '' -n 1 -rsp "Press any key to continue..."
  rm -rf "${temp_dir}"
}
temp_dir="$(mktemp -d)"
trap 'cleanup' EXIT

# Better way to take care of committers on vanilla
export GIT_COMMITTER_NAME='Foo Bar'
export GIT_COMMITTER_EMAIL='[email protected]'

# 1 - Simulate a git repo
BUILD_REPO="${temp_dir}/build_repo"
mkdir -p "${BUILD_REPO}"
pushd "${BUILD_REPO}" &> /dev/null
  git init
  touch readme
  git add readme
  git commit -m "Initial commit"

  if [ "${OS-}" = "Windows_NT" ]; then
    git submodule add https://github.com/visionsystemsinc/vsi_common.git 'diff  i  cult'
    difficult=$'diff  i  cult'
  else
    # This won't actually create a newline in the name, it just ignores it
    git submodule add https://github.com/visionsystemsinc/vsi_common.git $'  diff  i \t cult'
    # git submodule add https://github.com/visionsystemsinc/vsi_common $'  diff\ticult\n'
    # This will.
    git mv $'  diff  i \t cult' $'diff  i \t cult'

    difficult=$'diff  i \t cult'
  fi
  git submodule add https://bitbucket.org/andyneff/lfs_sample.git lfs
  # git mv lfs $'\nlfs\t\t'
  # git mv lfs $'lfs\t\t'
  git commit -m "Added submodules"
popd &> /dev/null

# 2 - Simulate github
PRETEND_REPO="${temp_dir}/pretend_repo"
mkdir -p "${PRETEND_REPO}"
pushd "${PRETEND_REPO}" &> /dev/null
  git clone --mirror "${BUILD_REPO}" .
popd &> /dev/null

# 3 - Simulate mirror
pushd "${temp_dir}" &> /dev/null
  "${CWD}/git_mirror" mirror "${PRETEND_REPO}"
  PREP_DIR="${temp_dir}/pretend_repo_prep"
  PREP_FILE=("${PREP_DIR}/"transfer_*.tgz)
  "${CWD}/git_mirror" mirror "${PREP_DIR}"
popd &> /dev/null

# 4 - Simulate transfer
TRANSFER_DIR="${temp_dir}/transfer"
mkdir -p "${TRANSFER_DIR}"
pushd "${TRANSFER_DIR}" &> /dev/null
  tar zxf "${PREP_FILE[0]}"
popd &> /dev/null

# 5 - Simulate a mirror upload
MIRROR_DIR="${temp_dir}/mirror"
cat - << EOF > "${TRANSFER_DIR}/info.env"
repos[.]="${MIRROR_DIR}/main"
repos[lfs]="${MIRROR_DIR}/lfs"
repos['${difficult}']="${MIRROR_DIR}"/'${difficult}'
repos['${difficult}/docker/recipes']="${MIRROR_DIR}/recipes"
EOF

for d in main lfs vxl "${difficult}" recipes; do
  mkdir -p "${MIRROR_DIR}/${d}"
  pushd "${MIRROR_DIR}/${d}" &> /dev/null
    git init --bare
  popd &> /dev/null
done

# source /opt/projects/terra/terra_dsm/external/terra/external/vsi_common/linux/debug.bsh
# JUST_DEBUG_READ_METHOD=python
# trap set_trace ERR

# Setup lfs-folderstore

if [ "${OS-}" = "Windows_NT" ]; then
  os=windows-amd64
elif [[ ${OSTYPE} = darwin* ]]; then
  os=darwin-amd64
else
  os=linux-amd64
fi

pushd "${temp_dir}" &> /dev/null
  curl -Lo "lfs.zip" "https://github.com/sinbad/lfs-folderstore/releases/download/v1.0.0/lfs-folderstore-${os}-v1.0.0.zip"
  unzip lfs.zip
  chmod 755 lfs-folderstore-${os}/lfs-folderstore
popd &> /dev/null

echo '#!/usr/bin/env bash
git -c lfs.customtransfer.lfs-folder.path="'"${temp_dir}/lfs-folderstore-${os}/lfs-folderstore"'" -c lfs.customtransfer.lfs-folder.args="'"${temp_dir}/mirror/lfs/lfs/objects"'" -c lfs.standalonetransferagent=lfs-folder ${@+"${@}"}' > "${temp_dir}/git2"
chmod 755 "${temp_dir}/git2"
export GIT="${temp_dir}/git2"
# The lfs-folderstore will not work unless the dir exists :(
mkdir -p "${temp_dir}/mirror/lfs/lfs/objects"

"${CWD}/git_mirror" push "${TRANSFER_DIR}/info.env" "${TRANSFER_DIR}"

# # Fake an LFS push, using lfs-folderstore for the repo called "lfs"
# # Create a local lfs store
# # cp -ra "${PREP_DIR}"/*/lfs/lfs "${temp_dir}/"
# # This only works in newer git
# echo '#!/usr/bin/env bash
# git -c lfs.customtransfer.lfs-folder.path="'"${temp_dir}/lfs-folderstore-${os}/lfs-folderstore"'" -c lfs.customtransfer.lfs-folder.args="'"${temp_dir}/lfs/objects"'" -c lfs.standalonetransferagent=lfs-folder ${@+"${@}"}' > "${temp_dir}/git2"
# chmod 755 "${temp_dir}/git2"
# export GIT="${temp_dir}/git2"

# 6 - Simulate cloning from mirror

CLONE_DIR="${temp_dir}/clone"
"${CWD}/git_mirror" clone "${TRANSFER_DIR}/info.env" "${CLONE_DIR}"

echo $'\x1b[0m'"Checking..."
set -xv
[[ -f "${CLONE_DIR}/${difficult}/setup.env" ]]
[[ -f "${CLONE_DIR}/${difficult}/docker/recipes/README.rst" ]]
# The following test fails on Ubuntu 16.04 git 2.17.1 git-lfs. The reason is
# that git submodule update (and git clone --recursive) do not do a lfs
# checkout... the exact cause of this is unknown, but it's probably a known
# bug that's been fixed, or that's my guess
# git clone https://bitbucket.org/andyneff/lfs_sample.git doesn't even work!
[[ $(sha256sum ${CLONE_DIR}/lfs/heart.jpg | awk '{print $1}') == 259e5122c42220369b8ee2f3570858e08db668bda9832cd96233ed4e11ae9e25 ]]
set +xv
echo "Success!"

Wow! Impressive

It would be cool if next_section was called if an error occurs so that the error message was highlighted. I have a submodule that requires a password; if that fails it would cool to see the error highlighted by a different color.