Skip to content

Instantly share code, notes, and snippets.

@SR-G
Last active October 25, 2024 11:44
Show Gist options
  • Save SR-G/8583cdbcdcf8ea93a9f83a4d33c61031 to your computer and use it in GitHub Desktop.
Save SR-G/8583cdbcdcf8ea93a9f83a4d33c61031 to your computer and use it in GitHub Desktop.
Snippets about backuping GIT repositories (in my case : forgejo usage with backups managed in an "external" way through this script, and not with "forgejo" dump command) - see here for extra details : https://codeberg.org/forgejo/forgejo/issues/2253
#!/bin/bash
BACKUP_DIRECTORY="/data/backups/"
LFS_DIRECTORY="/data/git/lfs/"
REPOSITORY_LOCATION="/data/git/repositories/"
MAX_BUNDLE_BACKUPS_TO_BE_KEPT=2 # have +1 compared to number of files to be kept ("2" to keep 1 file, ...)
rm -f /tmp/bundles.created.$$ /tmp/bundles.deleted.$$ /tmp/lfs.$$ >/dev/null 2>&1
find "${REPOSITORY_LOCATION}" -name "*.git" | sort | while read REPOSITORY ; do
PARENT=$(basename $(dirname "${REPOSITORY}"))
REPOSITORY_NEED_TO_BE_BACKUPED=0
REPOSITORY_NAME="${REPOSITORY##*/}" # myrepo.git
REPOSITORY_NAME="${PARENT}-${REPOSITORY_NAME%%.git*}" # owner-myrepo
echo ""
echo " === Working on [$REPOSITORY] => [$REPOSITORY_NAME]"
PREVIOUS_BACKUP_FOUND=$(find "${BACKUP_DIRECTORY}" -regex ".*${REPOSITORY_NAME}-[0-9]*\.bundle" 2>/dev/null | wc -l)
cd "${REPOSITORY}"
LAST_COMMIT_FROM_REPOSITORY=$(git --no-pager log -1 --format="%at")
if [[ "$PREVIOUS_BACKUP_FOUND" -eq 0 ]] ; then
REPOSITORY_NEED_TO_BE_BACKUPED=1
echo " - No previous backup found, backup needed"
# TODO this is not optimal : we will backup whole LFS folder
# as soon as only one repository has changes (but they may not be LFS related)
NB_LFS_FILES=$(git lfs ls-files --all 2>/dev/null | wc -l)
[[ "$NB_LFS_FILES" -gt 0 ]] && echo "${REPOSITORY_NAME}" >> /tmp/lfs.$$
else
LAST_COMMIT_FROM_BACKUP=$(find "${BACKUP_DIRECTORY}" -regex ".*${REPOSITORY_NAME}-[0-9]*\.bundle" 2>/dev/null | tail -1 | sed "s|.*-\([0-9]*\)\.bundle|\1|")
if [[ "${LAST_COMMIT_FROM_REPOSITORY}" -gt "${LAST_COMMIT_FROM_BACKUP}" ]] ; then
echo " - Repo [$REPOSITORY_NAME] - last commit from repository [$LAST_COMMIT_FROM_REPOSITORY], last commit backuped [$LAST_COMMIT_FROM_BACKUP] ==> BACKUP NEEDED"
REPOSITORY_NEED_TO_BE_BACKUPED=1
NB_LFS_FILES=$(git lfs ls-files --all 2>/dev/null | wc -l)
[[ "$NB_LFS_FILES" -gt 0 ]] && echo "${REPOSITORY_NAME}" >> /tmp/lfs.$$
else
echo " - Repo [$REPOSITORY_NAME] - last commit from repository [$LAST_COMMIT_FROM_REPOSITORY], last commit backuped [$LAST_COMMIT_FROM_BACKUP] ==> BACKUP NOT NEEDED"
fi
fi
if [[ "$REPOSITORY_NEED_TO_BE_BACKUPED" -eq 1 ]] ; then
echo " - BACKUPING REPOSITORY [$REPOSITORY_NAME]"
git bundle create "${BACKUP_DIRECTORY}/${REPOSITORY_NAME}-${LAST_COMMIT_FROM_REPOSITORY}.bundle" --all
echo "${REPOSITORY_NAME}-${LAST_COMMIT_FROM_REPOSITORY}.bundle" >> /tmp/bundles.created.$$
# Delete old bundle backups to keep only "max" of them
ls -t "${BACKUP_DIRECTORY}/${REPOSITORY_NAME}"*.bundle | tail -n +${MAX_BUNDLE_BACKUPS_TO_BE_KEPT} >> /tmp/bundles.deleted.$$
ls -t "${BACKUP_DIRECTORY}/${REPOSITORY_NAME}"*.bundle | tail -n +${MAX_BUNDLE_BACKUPS_TO_BE_KEPT} | xargs -I {} rm {}
fi
cd - >/dev/null 2>&1
done
# LFS to be tarr'ed (only if needed)
if [[ -f /tmp/lfs.$$ ]] ; then
echo "LFS backup needed for these repositories : "
cat /tmp/lfs.$$ | sort
tar czvf "${BACKUP_DIRECTORY}/lfs-backup.tar.gz" "${LFS_DIRECTORY}"
fi
# Summaries
if [[ -f /tmp/bundles.created.$$ ]] ; then
echo ""
echo "Bundles created : "
echo "====================================="
cat /tmp/bundles.created.$$ | sort
fi
if [[ -f /tmp/bundles.deleted.$$ ]] ; then
echo ""
echo "Bundles removed : "
echo "====================================="
cat /tmp/bundles.deleted.$$ | sort
fi
30 1 * * * /usr/bin/docker run --rm --name forgejo-backuper -e DATE=$(date "+\%Y-\%m-\%d") --link postgresql:database_postgres -v volume_forgejo:/data/ -v volume_forgejo_backup_dumps:/data/backups/ -w /data/backups/ --entrypoint "" -e USER=git --user git codeberg.org/forgejo/forgejo:9 sh -c "forgejo dump --verbose --database postgres --skip-repository --skip-lfs-data --config /data/gitea/conf/app.ini --type tar.gz --file /data/backups/\$DATE-forgejo-dump.tar.gz" > /var/log/cron-backup-docker-forgejo.log 2>&1
5 1 * * * /usr/bin/docker run --rm --name "forgejo-backup-repositories" -v volume_forgejo:/data/ -v volume_forgejo_backup_repositories:/data/backups/ forgejo-backuper > /var/log/cron-backup-docker-forgejo-repositories.log 2>&1
FROM debian:bookworm-slim
CMD ["/backup.sh"]
RUN apt-get update && apt-get install -y git git-lfs && \
git lfs install && \
groupadd -g 1000 git && \
useradd -u 1000 -d /git/ -s /usr/bin/bash -g git -c "git" git && \
cp -Rp /root/ /git/ && chown -R 1000:1000 /git/ && \
apt-clean
COPY backup.sh /backup.sh
ENV HOME=/git/
USER git
WORKDIR /data/
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment