Monorepo merger for the Kurento project

kurento-monorepo.sh

#!/usr/bin/env bash
# Checked with ShellCheck (https://www.shellcheck.net/)

# Shell setup
# ===========

# Bash options for strict error checking.
set -o errexit -o errtrace -o pipefail -o nounset
shopt -s inherit_errexit 2>/dev/null || true

# Trace all commands (to stderr).
#set -o xtrace



# Merge functions
# ===============

# Merge function based on the method used by GStreamer for their monorepo.
#
# This worked acceptably but it has several shortcomings, such as difficulties
# when having a collision between old and new paths. Otherwise, a good thing is
# that the preparation steps are kept in the monorepo itself.
#
# Sources:
# * https://blogs.gnome.org/tsaunier/2021/09/29/gstreamer-one-repository-to-rule-them-all/
#   - Archive: https://web.archive.org/web/20221226070556/https://blogs.gnome.org/tsaunier/2021/09/29/gstreamer-one-repository-to-rule-them-all/
# * https://gitlab.freedesktop.org/gstreamer/gstreamer/-/issues/474
#
# This is mostly a direct translation of the procedure from source code:
# https://gitlab.freedesktop.org/thiblahute/gst-merger/blob/master/merge.py
function merge_repo_gstreamer {
    if [[ $# -ne 4 ]]; then
        echo "Expected arguments: <Name> <URL> <Branch> <DestDir>"
        return 1
    fi

    MODULE_NAME="$1"
    MODULE_URL="$2"
    MODULE_BRANCH="$3"
    MODULE_DIR="$4"

    git remote add "$MODULE_NAME" "$MODULE_URL"
    git remote update "$MODULE_NAME"

    git merge --no-edit --allow-unrelated-histories --message "Merge '$MODULE_NAME' into monorepo" "$MODULE_NAME/$MODULE_BRANCH"
    mapfile -t FILES < <(git diff --name-only 'HEAD^..HEAD')
    for FILE in "${FILES[@]}"; do
        FILE_SRC="$(dirname "$FILE")"
        FILE_DST="$MODULE_DIR/$FILE_SRC"
        mkdir --parents "$FILE_DST"
        git mv "$FILE" "$FILE_DST"
    done
    git commit --no-edit --no-verify --message "Move files from '$MODULE_NAME' into $MODULE_DIR/"

    git clean -xdf
    git remote remove "$MODULE_NAME"
}

# Merge function based on the method shown by the JDriven blog.
#
# Doing monorepo preparation separately on the source repository is much less
# likely to hit a name collision between old and new paths.
#
# This function implements the original idea, and multiple safeguards to keep
# an eye on corner cases such as spaces in filenames, hidden destination
# dirs, and git submodules.
#
# Sources:
# * https://blog.jdriven.com/2021/04/how-to-merge-multiple-git-repositories/
#   - Archive: https://web.archive.org/web/20210518082201/https://blog.jdriven.com/2021/04/how-to-merge-multiple-git-repositories/
function merge_repo_jdriven {
    if [[ $# -ne 4 ]]; then
        echo "Expected arguments: <Name> <URL> <Branch> <DestDir>"
        return 1
    fi

    MODULE_NAME="$1"
    MODULE_URL="$2"
    MODULE_BRANCH="$3"
    MODULE_DIR="$4"

    git clone --branch "$MODULE_BRANCH" "$MODULE_URL" "/tmp/$MODULE_NAME"
    pushd "/tmp/$MODULE_NAME"
    {
        git switch --create monorepo-preparation

        # Create a unique temporary working directory. This avoids the chance of
        # collision between the <DestDir> and a possibly already existing dir.
        # For example, "kurento-media-server" already contains a "server" subdir
        # which collides with the "server" destination in monorepo.
        TEMP_DIR="$(mktemp --directory --tmpdir=.)"
        DEST_DIR="$TEMP_DIR/$MODULE_DIR"
        mkdir --parents "$DEST_DIR"

        # Specific per-repo fixes.
        if [[ "$MODULE_NAME" == "kms-omni-build" ]]; then
            # Delete all submodules, because they will be individually added
            # later under their new locations.
            mapfile -t SUBMODULES < <(git submodule status | awk '{print $2}')
            git rm "${SUBMODULES[@]}"
            git rm .gitmodules
            git clean --force .
            git commit --no-verify --message "dummy"
        fi

        # Make an array with the names of all top-level git entries.
        mapfile -t FILES < <(git ls-tree --name-only HEAD)

        # Exclude files that git requires to keep in the repo's root.
        FILES_AUX=()
        for FILE in "${FILES[@]}"; do
            if [[ "$FILE" != ".gitmodules" ]]; then
                FILES_AUX+=("$FILE")
            fi
        done
        FILES=("${FILES_AUX[@]}")
        unset FILES_AUX

        # Move all git entries to their destinations.
        git mv "${FILES[@]}" "$DEST_DIR"

        # Now move all files out of the temp dir.
        # `.[!.]*` is to avoid including `.` and `..` in the wildcard for hidden
        # files. It would also exclude things like `..file`, but that's a very
        # uncommon file name anyways.
        git mv "$TEMP_DIR"/* ./ || true
        git mv "$TEMP_DIR"/.[!.]* ./ || true
        rmdir "$TEMP_DIR"

        # `--amend` used here in case a previous dummy commit was done.
        git commit --amend --no-verify \
            --message "monorepo: move files into $MODULE_DIR/"

        # Push this preparation to the remote, so we leave a traceable branch.
        # WARNING: ONLY DO THIS IN THE DEFINITIVE RUN OF THE SCRIPT.
        # LEAVE COMMENTED OTHERWISE.
        #git push --set-upstream origin HEAD
    }
    popd

    git remote add "$MODULE_NAME" "/tmp/$MODULE_NAME"
    {
        git fetch "$MODULE_NAME"

        git merge \
            --allow-unrelated-histories \
            --message "monorepo: merge '$MODULE_NAME' into $MODULE_DIR/" \
            "$MODULE_NAME/monorepo-preparation"

        # Specific per-repo fixes.
        if [[ "$MODULE_NAME" == "kurento-java" ]]; then
            # Resolve the conflict caused by a past removal of maven-plugin.
            git restore --staged --worktree clients/java/maven-plugin/
            git add . && git commit --no-edit
        fi
    }
    git remote remove "$MODULE_NAME"
}

# Merge function based on the "Subtree Merging" technique.
#
# This seems to be the best monorepo merge technique for the needs of a Kurento
# monorepo. It is much simpler than the other methods, while keeping nice
# properties such as being safe against name collisions (which the GStreamer
# way doesn't prevent) and also keeping the whole migration in the commit
# history of the monorepo itself (which the JDriven method lacks).
#
# This way of merging requires, however, manually adjusting the git submodules
# file (.gitmodules) that each subproject might bring to the monorepo.
#
# Sources:
# * https://mirrors.edge.kernel.org/pub/software/scm/git/docs/howto/using-merge-subtree.html
# * https://docs.github.com/en/get-started/using-git/about-git-subtree-merges
# * https://stackoverflow.com/questions/1425892/how-do-you-merge-two-git-repositories
function merge_repo_subtree {
    if [[ $# -ne 4 ]]; then
        echo "Expected arguments: <Name> <URL> <Branch> <DestDir>"
        return 1
    fi

    MODULE_NAME="$1"
    MODULE_URL="$2"
    MODULE_BRANCH="$3"
    MODULE_DIR="$4"

    if [[ "$MODULE_DIR" -ef . ]]; then
        echo "ERROR: <DestDir> must be a subdirectory in the monorepo"
        exit 1
    fi

    # Add the source URL as a git remote. This simplifies the next commands.
    git remote add "$MODULE_NAME" "$MODULE_URL"

    # Fetch only commits.
    # Uses the default refspec, which is "refs/remotes/<RemoteName>/". This way,
    # when the remote is removed, its unmerged branches are removed too.
    git fetch --no-tags "$MODULE_NAME"

    # Fetch only tags.
    # Use an explicit refspec, in order to save tags under the given prefix.
    git fetch --no-tags "$MODULE_NAME" "+refs/tags/*:refs/tags/$MODULE_NAME/*"

    # Create a merge commit, but leave it empty ("ours" strategy).
    # The actual addition of files will be done next.
    git merge \
        --strategy=ours \
        --no-commit \
        --allow-unrelated-histories \
        "$MODULE_NAME/$MODULE_BRANCH"

    # Add files from the given branch, under the given prefix.
    # Because we are in the middle of an unfinished merge, this will show up as
    # files added by the merge commit.
    git read-tree -u --prefix="$MODULE_DIR" "$MODULE_NAME/$MODULE_BRANCH"

    # Specific per-repo fixes.
    if [[ "$MODULE_NAME" == "kms-omni-build" ]]; then
        # Delete all submodules, because they will be individually added
        # later under their new locations.
        mapfile -t SUBMODULES < <(sed --quiet --regexp-extended "s|^.*path = (.*)|$MODULE_DIR/\1|p" "$MODULE_DIR/.gitmodules")
        git rm --force "${SUBMODULES[@]}"
        git rm --force "$MODULE_DIR/.gitmodules"
        git clean --force .
    fi

    # If the repo comes with a `.gitmodules`, integrate it in the monorepo.
    if [[ -f "$MODULE_DIR/.gitmodules" ]]; then
        sed --regexp-extended \
            --expression "s|submodule \"(.*)\"|submodule \"$MODULE_DIR/\1\"|" \
            --expression "s|path = (.*)|path = $MODULE_DIR/\1|" \
            "$MODULE_DIR/.gitmodules" \
            >>.gitmodules
        git rm --force "$MODULE_DIR/.gitmodules"
        git add .gitmodules
    fi

    # Commit changes, finishing the ongoing merge.
    git commit --message "monorepo: merge '$MODULE_NAME' into $MODULE_DIR/"

    # Remove the git remote, which also removes all of its unmerged branches.
    # This leaves a clean repo graph.
    git remote remove "$MODULE_NAME"
}



# Entrypoint functions
# ====================

# NOT TESTED: I went directly to the other two methods, they seem better to me.
function prepare_merge_gstreamer {
    mkdir monorepo-gstreamer
    cd monorepo-gstreamer/
    git init --initial-branch "main" .
    git commit --allow-empty --message "monorepo: Initial commit"

    function merge_repo {
        merge_repo_gstreamer "$@"
    }
}

function prepare_merge_jdriven {
    mkdir monorepo-jdriven
    cd monorepo-jdriven/
    git init --initial-branch "main" .

    # Use the merge driver "union" for conflict resolution of `.gitmodules` file.
    # The "union" driver simply appends new content to the file.
    # https://git-scm.com/docs/gitattributes#Documentation/gitattributes.txt-union
    echo '.gitmodules merge=union' >.gitattributes

    git add .
    git commit --message "monorepo: Initial commit"

    function merge_repo {
        merge_repo_jdriven "$@"
    }
}

function prepare_merge_subtree {
    mkdir monorepo-subtree
    cd monorepo-subtree/
    git init --initial-branch "main" .
    git commit --allow-empty --message "monorepo: Initial commit"

    function merge_repo {
        merge_repo_subtree "$@"
    }
}



# Choose only one...
#prepare_merge_gstreamer
#prepare_merge_jdriven
prepare_merge_subtree



merge_repo "kms-omni-build" "https://github.com/Kurento/kms-omni-build.git" "7.0.0" "server"

merge_repo "kurento-module-creator" "https://github.com/Kurento/kurento-module-creator.git" "7.0.0" "server/module-creator"
merge_repo "kms-cmake-utils" "https://github.com/Kurento/kms-cmake-utils.git" "7.0.0" "server/cmake-utils"
merge_repo "kms-jsonrpc" "https://github.com/Kurento/kms-jsonrpc.git" "7.0.0" "server/jsonrpc"
merge_repo "kms-core" "https://github.com/Kurento/kms-core.git" "7.0.0" "server/plugin-core"
merge_repo "kms-elements" "https://github.com/Kurento/kms-elements.git" "7.0.0" "server/plugin-elements"
merge_repo "kms-filters" "https://github.com/Kurento/kms-filters.git" "7.0.0" "server/plugin-filters"
merge_repo "kurento-media-server" "https://github.com/Kurento/kurento-media-server.git" "7.0.0" "server/media-server"
merge_repo "kurento-maven-plugin" "https://github.com/Kurento/kurento-maven-plugin.git" "7.0.0" "clients/java/maven-plugin"

merge_repo "kms-chroma" "https://github.com/Kurento/kms-chroma.git" "7.0.0" "server/plugin-examples/chroma"
merge_repo "kms-crowddetector" "https://github.com/Kurento/kms-crowddetector.git" "7.0.0" "server/plugin-examples/crowddetector"
merge_repo "kms-datachannelexample" "https://github.com/Kurento/kms-datachannelexample.git" "7.0.0" "server/plugin-examples/datachannelexample"
merge_repo "kms-gstreamer-plugin-sample" "https://github.com/Kurento/kms-gstreamer-plugin-sample.git" "7.0.0" "server/plugin-examples/gstreamer-sample"
merge_repo "kms-markerdetector" "https://github.com/Kurento/kms-markerdetector.git" "7.0.0" "server/plugin-examples/markerdetector"
merge_repo "kms-opencv-plugin-sample" "https://github.com/Kurento/kms-opencv-plugin-sample.git" "7.0.0" "server/plugin-examples/opencv-sample"
merge_repo "kms-platedetector" "https://github.com/Kurento/kms-platedetector.git" "7.0.0" "server/plugin-examples/platedetector"
merge_repo "kms-pointerdetector" "https://github.com/Kurento/kms-pointerdetector.git" "7.0.0" "server/plugin-examples/pointerdetector"

merge_repo "adm-scripts" "https://github.com/Kurento/adm-scripts.git" "master" "ci-scripts"
merge_repo "doc-kurento" "https://github.com/Kurento/doc-kurento.git" "7.0.0" "doc-kurento"
merge_repo "kurento-client-js" "https://github.com/Kurento/kurento-client-js.git" "master" "clients/javascript/client"
merge_repo "kurento-docker" "https://github.com/Kurento/kurento-docker.git" "master" "docker"
merge_repo "kurento-java" "https://github.com/Kurento/kurento-java.git" "7.0.0" "clients/java"
merge_repo "kurento-jsonrpc-js" "https://github.com/Kurento/kurento-jsonrpc-js.git" "master" "clients/javascript/jsonrpc"
merge_repo "kurento-qa-pom" "https://github.com/Kurento/kurento-qa-pom.git" "master" "clients/java/qa-pom"
merge_repo "kurento-tutorial-java" "https://github.com/Kurento/kurento-tutorial-java.git" "7.0.0" "tutorials/java"
merge_repo "kurento-tutorial-js" "https://github.com/Kurento/kurento-tutorial-js.git" "7.0.0" "tutorials/javascript-browser"
merge_repo "kurento-tutorial-node" "https://github.com/Kurento/kurento-tutorial-node.git" "7.0.0" "tutorials/javascript-node"
merge_repo "kurento-tutorial-test" "https://github.com/Kurento/kurento-tutorial-test.git" "master" "test/tutorial"
merge_repo "kurento-utils-js" "https://github.com/Kurento/kurento-utils-js.git" "master" "browser/kurento-utils-js"

echo "All done! Enjoy your monorepo ;-)"