A utility script for managing svn "super projects" that consist of numerous separate svn projects (depends on zsh and parallel)

svnaid.zsh

#!/usr/bin/zsh
#
# https://creativecommons.org/licenses/by/4.0/
# Original Author: Lonnie VanZandt
# Creation Date: 04 November 2016
# Summary: Simplify using Subversion SVN for Super Projects consisting of multiple independent sub-projects

# try to deduce the remote repo URL, superproject root, and local workspace root from available local files
SUPERURL=$'( [[ -d .svn ]] && echo `svn info --show-item repos-root-url` ) || echo "https://server.foo/svn/superproject"'
SUPERURL=`eval $SUPERURL`
SUPERROOT=$'( [[ -d .svn ]] && echo `svn info --show-item relative-url| cut -d / -f 2` ) || echo `basename ${SUPERURL}`'
SUPERROOT=`eval $SUPERROOT`
SUPERPATH=$'( [[ -d .svn ]] && echo `svn info --show-item wc-root|sed -e "s!${SUPERROOT}.*!${SUPERROOT}!"` ) || echo "/superpath/${SUPERROOT}"'
SUPERPATH=`eval $SUPERPATH`
SUPERURL=$SUPERURL/$SUPERROOT

TODAY=`date +"%d%b%Y"|tr \[:upper:\] \[:lower:\]`

BRANCHPATTERN=".*${USER}.*"

status_phrase()
{
	last_status=( "Unsuccessfully" "Successfully" )

	last=$?

	if [[ $last -eq 254 ]]; then
		jobs="No"
		last=0
	else
		jobs="$last"
	fi

	echo "$jobs jobs failed. $last_status[ $(( $last == 0 ? 2 : 1 )) ]"
}

echoerr() { echo "$@" 1>&2; }

parse_options()
{
	o_goal=(-g "ls.branches")
	o_superroot=(-r ${SUPERROOT})
	o_superpath=(-p ${SUPERPATH})
	o_branchpattern=(-b ${BRANCHPATTERN})
	o_superurl=(-u ${SUPERURL})

	zparseopts -K -- g:=o_goal u:=o_superurl r:=o_superroot b:=o_branchpattern p:=o_superpath h=o_help

	if [[ $? != 0 || "$o_help" != "" ]]; then
	   echo Usage: $(basename "$0") "[-g GOAL] [-u SUPERURL] [-p SUPERPATH] [-r SUPERROOT] [-b PATTERN]"
	   echo ""
	   echo "SUPERURL      is an svn URL like ${SUPERURL}"
	   echo "SUPERPATH     is a local path to a workspace like ${SUPERPATH}"
           echo "SUPERROOT     is a SVN root project folder like ${SUPERROOT}"
           echo "GOAL          is a goal like ls.branches or stat.all"
	   echo "BRANCHPATTERN is a regex to pick out branches like ${BRANCHPATTERN}"

	   exit 1
	fi

	goal=$o_goal[2]
	superroot=$o_superroot[2]
	superpath=`readlink -f $o_superpath[2]`
	branchpattern=$o_branchpattern[2]
	superurl=$o_superurl[2]
}

parse_options $*

SUPERURL=$superurl
[[ $superroot =~ "superproject" ]] && SUPERROOT=`basename $SUPERURL`
[[ $superroot =~ "superproject" ]] || SUPERROOT=$superroot

[[ $superpath =~ "superpath" ]] && SUPERPATH=`pwd`
[[ $superpath =~ "superpath" ]] || SUPERPATH=$superpath

BRANCHPATTERN=$branchpattern

case $goal in

( "init" )

[[ $SUPERURL =~ "server.foo" ]] && echoerr "[$goal] specify a valid SUPERURL on the command-line." && exit 1

if [[ -d ${SUPERPATH} ]]; then
	echoerr "[$goal] doing nothing because ${SUPERPATH} already exists."
	exit 1
else
	svn -N co ${SUPERURL} ${SUPERPATH}
fi

if [[ ! -d ${SUPERPATH} ]]; then
	echoerr "[$goal] unable to svn -N co ${SUPERURL} ${SUPERPATH}."
	exit 1
fi

echoerr "[$goal] $(status_phrase) initialized ${SUPERPATH} from ${SUPERURL}."

;;

( "co.all.trunks" )

# svn co all trunks across all projects within the superproject
# run within the superproject root after svn -N co ${SUPERURL} ${SUPERROOT}
echoerr "[$goal] Starting query of ${SUPERURL}..."
echoerr "[$goal] Checking out a Working Copy into ${SUPERPATH}..."

cd ${SUPERPATH} && \
svn ls -R ${SUPERURL} | parallel --no-notice "[[ {} =~ $'"/trunk/\$"' ]] && svn co ${SUPERURL}/{} {//}"

echoerr "[$goal] $(status_phrase) checked out all trunks."

;;

( "switch.all" )

# svn switch all branched projects across all projects within the superproject to the matching branch
# run within the superproject root after svn -N co rooturl
echoerr "[$goal] Starting query of ${SUPERURL}..."
echoerr "[$goal] Switching a Working Copy in ${SUPERPATH} to branch matching .../branches/${BRANCHPATTERN}/..."

cd ${SUPERPATH} && \
svn ls -R ${SUPERURL} | parallel --no-notice "[[ {} =~ $'"/branches/${BRANCHPATTERN}/\$"' ]] && svn switch ${SUPERURL}/{} {= s!/branches/${BRANCHPATTERN}/\$!!g =}"

echoerr "[$goal] $(status_phrase) switched all working copies to branches/${BRANCHPATTERN}/."

;;

( "switch.trunks" )

# svn switch all trunked projects across all projects within the superproject to the trunk
# run within the superproject root after svn -N co rooturl
echoerr "[$goal] Starting query of ${SUPERURL}..."
echoerr "[$goal] Switching a Working Copy in ${SUPERPATH} to trunks..."

cd ${SUPERPATH} && \
svn ls -R ${SUPERURL} | parallel --no-notice "[[ {} =~ $'"/trunk/\$"' ]] && svn switch ${SUPERURL}/{} {//}"

echoerr "[$goal] $(status_phrase) switched all working copies to trunk/."

;;

( "up.all" )

# svn update all projects that have a trunk-brank-tag fork in them
# run in the root of the superproject
echoerr "[$goal] Starting query of ${SUPERURL}..."
echoerr "[$goal] Updating the Working Copy in ${SUPERPATH}..."

cd ${SUPERPATH} && \
svn ls -R ${SUPERURL} | parallel --no-notice "[[ {} =~ $'"/trunk/\$"' ]] && svn update ${SUPERPATH}/{//}"

echoerr "[$goal] $(status_phrase) updated all working copies."

;;

( "info.all" )

# svn info all projects that have a trunk-brank-tag fork in them
# run in the root of the superproject
echoerr "[$goal] Starting query of ${SUPERURL}..."
echoerr "[$goal] Getting info on the Working Copy in ${SUPERPATH}..."

cd ${SUPERPATH} && \
svn ls -R ${SUPERURL} | parallel --no-notice "[[ {} =~ $'"/trunk/\$"' ]] && svn info --show-item url ${SUPERPATH}/{//}"

echoerr "[$goal] $(status_phrase) info'ed all working copies."

;;

( "clean.all" )
# svn cleanup all projects that have a trunk-branch-tag fork in them
# run in the root of the superproject
echoerr "[$goal] Starting query of ${SUPERURL}..."
echoerr "[$goal] Cleaning the Working Copy in ${SUPERPATH}..."

cd ${SUPERPATH} && \
svn ls -R ${SUPERURL} | parallel --no-notice "[[ {} =~ $'"/trunk/\$"' ]] && echo "Cleaning ${SUPERPATH}/{//}..." && svn cleanup ${SUPERPATH}/{//}"

echoerr "[$goal] $(status_phrase) cleaned all working copies."

;;

( "stat.all" )

# svn status for all projects that have a trunk-branch-tag fork in them
# run in the root of the superproject
echoerr "[$goal] Starting query of ${SUPERURL}..."
echoerr "[$goal] Retrieving status of the Working Copy in ${SUPERPATH}..."

cd ${SUPERPATH} && \
svn ls -R ${SUPERURL} | parallel --no-notice "[[ {} =~ $'"/trunk/\$"' ]] && svn status -u ${SUPERPATH}/{//}"

echoerr "[$goal] $(status_phrase) statused all trunks."

;;

( "ls.trunks" )

# discover all trunks across all projects
# run anywhere
echoerr "[$goal] Starting query of ${SUPERURL}..."
echoerr "[$goal] Looking for any trunks matching .../trunk/."

svn ls -R ${SUPERURL} | parallel --no-notice "[[ {} =~ $'"/trunk/\$"' ]] && echo ${SUPERURL}/{}"

echoerr "[$goal] $(status_phrase) listed all trunks."

;;

( "ls.trunks.matching" )

# discover all trunks across all projects
# run anywhere
echoerr "[$goal] Starting query of ${SUPERURL}..."
echoerr "[$goal] Looking for any trunks matching .../trunk/ and ${BRANCHPATTERN} ."

svn ls -R ${SUPERURL} | parallel --no-notice "[[ {} =~ $'"/trunk/\$"' ]] && [[ {} =~ $'"${BRANCHPATTERN}"' ]] && echo ${SUPERURL}/{}"

echoerr "[$goal] $(status_phrase) listed trunks matching ${BRANCHPATTERN}."

;;

( "ls.branches" )

# discover all branches across all projects with branches matching a pattern
# run anywhere
echoerr "[$goal] Starting query of ${SUPERURL}..."
echoerr "[$goal] Looking for any branches matching .../branches/${BRANCHPATTERN}/..."

svn ls -R ${SUPERURL} | parallel --no-notice "[[ {} =~ $'"/branches/${BRANCHPATTERN}/\$"' ]] && echo ${SUPERURL}/{}"

echoerr "[$goal] $(status_phrase) listed branches/${BRANCHPATTERN}/."

;;

( "co.branches" )

# svn co for all branches matching a named pattern
# run within the superproject root directory
echoerr "[$goal] Starting query of ${SUPERURL}..."
echoerr "[$goal] Checking out all branches matching .../branches/${BRANCHPATTERN}/... into ${SUPERPATH}..."

cd ${SUPERPATH} && \
svn ls -R ${SUPERURL} | parallel --no-notice "[[ {} =~ $'"/branches/${BRANCHPATTERN}/\$"' ]] && svn co ${SUPERURL}/{} {= s!/branches/${BRANCHPATTERN}/\$!!g =}"

echoerr "[$goal] $(status_phrase) checked out branches/${BRANCHPATTERN}/."

;;

( "diff.branch.trunk" )

# svn diff old trunk vs new branch for all branches matching a pattern
# run anywhere
echoerr "[$goal] Starting query of ${SUPERURL}..."
echoerr "[$goal] Diffing all branches matching .../branches/${BRANCHPATTERN}/... with ${SUPERURL} trunks..."

svn ls -R ${SUPERURL} | parallel --no-notice "[[ {} =~ $'"/branches/${BRANCHPATTERN}/\$"' ]] && svn diff --summarize --new ${SUPERURL}/{} --old ${SUPERURL}/{= s!/branches/${BRANCHPATTERN}/\$!/trunk!g =} -r HEAD:HEAD"

echoerr "[$goal] $(status_phrase) diffed branches/${BRANCHPATTERN}/ with trunks."

;;

( "new.branch" )

# svn copy a new branch for the current user and date across all trunk-branch projects
# run anywhere
echoerr "[$goal] Starting query of ${SUPERURL}..."
echoerr "[$goal] Creating a new branch named .../branches/${USER}/${TODAY}... in all ${SUPERURL} trunks..."

svn ls -R ${SUPERURL} | parallel --no-notice "[[ {} =~ $'"/trunk/\$"' ]] && [[ {} =~ $'"\(/trunk/\)\?"' ]] && svn copy ${SUPERURL}/{} ${SUPERURL}/{//}/branches/${USER}/${TODAY} --parents -m 'creating developer private branch.'"

echoerr "[$goal] $(status_phrase) completed creation of branches/${USER}/${TODAY}/."

;;

( "rm.branches" )

# svn rm existing branches matching a branch pattern across all trunk-branch projects
# run anywhere
echoerr "[$goal] Starting query of ${SUPERURL}..."
echoerr "[$goal] Interactively removing each branch matching .../branches/${BRANCHPATTERN}/... in all ${SUPERURL} projects..."

svn ls -R ${SUPERURL} | parallel --no-notice "[[ {} =~ $'"/branches/${BRANCHPATTERN}/\$"' ]] && echo {}" | parallel -p --no-notice "svn rm ${SUPERURL}/{} -m 'removing developer private branch.'"

echoerr "[$goal] $(status_phrase) removed branches/${BRANCHPATTERN}/ from ${SUPERURL}."

;;

( "merge.to.trunk" )

# svn integration merge from branches into working copy (of the trunk, typically) for all branches matching a pattern
# run in the root of the superproject
echoerr "[$goal] Starting query of ${SUPERURL}..."
echoerr "[$goal] merging into ${SUPERURL} the branch matching .../branches/${BRANCHPATTERN}/... from ${SUPERPATH}..."

cd ${SUPERPATH} && \
svn ls -R ${SUPERURL} | parallel --no-notice "[[ {} =~ $'"/branches/${BRANCHPATTERN}/\$"' ]] && echo '${SUPERURL}/{}' && ( cd ${SUPERPATH}/{= s!/branches/${BRANCHPATTERN}/\$!!g =} && svn merge --reintegrate ${SUPERURL}/{} . )"

;;

( "merge.from.trunk" )

# svn merge from trunk into working copy (of the branches, typically)
# run in the root of the superproject
echoerr "[$goal] Starting query of ${SUPERURL}..."
echoerr "[$goal] merging into ${SUPERPATH} all ${SUPERURL} trunks projects..."

cd ${SUPERPATH} && \
svn ls -R ${SUPERURL} | parallel --no-notice "[[ {} =~ $'"/trunk/\$"' ]] && echo '${SUPERURL}/{}' && svn merge ${SUPERURL}/{} ^/${SUPERROOT}/{}"

;;

esac

svn ls -R is very slow on a large remote repository. I continue to look for ways to get the remote http svn server to report only its directory structure. All advice I have seen so far use client-side filtering to pick the directories out of the long listing. The "parallel" filters in this script also do that kind of selection. However, anything client-side is too late: all the delay is on the server as it visits and then sends the long list of directories and files.