Last active
October 25, 2024 11:44
-
-
Save SR-G/8583cdbcdcf8ea93a9f83a4d33c61031 to your computer and use it in GitHub Desktop.
Snippets about backuping GIT repositories (in my case : forgejo usage with backups managed in an "external" way through this script, and not with "forgejo" dump command) - see here for extra details : https://codeberg.org/forgejo/forgejo/issues/2253
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
BACKUP_DIRECTORY="/data/backups/" | |
LFS_DIRECTORY="/data/git/lfs/" | |
REPOSITORY_LOCATION="/data/git/repositories/" | |
MAX_BUNDLE_BACKUPS_TO_BE_KEPT=2 # have +1 compared to number of files to be kept ("2" to keep 1 file, ...) | |
rm -f /tmp/bundles.created.$$ /tmp/bundles.deleted.$$ /tmp/lfs.$$ >/dev/null 2>&1 | |
find "${REPOSITORY_LOCATION}" -name "*.git" | sort | while read REPOSITORY ; do | |
PARENT=$(basename $(dirname "${REPOSITORY}")) | |
REPOSITORY_NEED_TO_BE_BACKUPED=0 | |
REPOSITORY_NAME="${REPOSITORY##*/}" # myrepo.git | |
REPOSITORY_NAME="${PARENT}-${REPOSITORY_NAME%%.git*}" # owner-myrepo | |
echo "" | |
echo " === Working on [$REPOSITORY] => [$REPOSITORY_NAME]" | |
PREVIOUS_BACKUP_FOUND=$(find "${BACKUP_DIRECTORY}" -regex ".*${REPOSITORY_NAME}-[0-9]*\.bundle" 2>/dev/null | wc -l) | |
cd "${REPOSITORY}" | |
LAST_COMMIT_FROM_REPOSITORY=$(git --no-pager log -1 --format="%at") | |
if [[ "$PREVIOUS_BACKUP_FOUND" -eq 0 ]] ; then | |
REPOSITORY_NEED_TO_BE_BACKUPED=1 | |
echo " - No previous backup found, backup needed" | |
# TODO this is not optimal : we will backup whole LFS folder | |
# as soon as only one repository has changes (but they may not be LFS related) | |
NB_LFS_FILES=$(git lfs ls-files --all 2>/dev/null | wc -l) | |
[[ "$NB_LFS_FILES" -gt 0 ]] && echo "${REPOSITORY_NAME}" >> /tmp/lfs.$$ | |
else | |
LAST_COMMIT_FROM_BACKUP=$(find "${BACKUP_DIRECTORY}" -regex ".*${REPOSITORY_NAME}-[0-9]*\.bundle" 2>/dev/null | tail -1 | sed "s|.*-\([0-9]*\)\.bundle|\1|") | |
if [[ "${LAST_COMMIT_FROM_REPOSITORY}" -gt "${LAST_COMMIT_FROM_BACKUP}" ]] ; then | |
echo " - Repo [$REPOSITORY_NAME] - last commit from repository [$LAST_COMMIT_FROM_REPOSITORY], last commit backuped [$LAST_COMMIT_FROM_BACKUP] ==> BACKUP NEEDED" | |
REPOSITORY_NEED_TO_BE_BACKUPED=1 | |
NB_LFS_FILES=$(git lfs ls-files --all 2>/dev/null | wc -l) | |
[[ "$NB_LFS_FILES" -gt 0 ]] && echo "${REPOSITORY_NAME}" >> /tmp/lfs.$$ | |
else | |
echo " - Repo [$REPOSITORY_NAME] - last commit from repository [$LAST_COMMIT_FROM_REPOSITORY], last commit backuped [$LAST_COMMIT_FROM_BACKUP] ==> BACKUP NOT NEEDED" | |
fi | |
fi | |
if [[ "$REPOSITORY_NEED_TO_BE_BACKUPED" -eq 1 ]] ; then | |
echo " - BACKUPING REPOSITORY [$REPOSITORY_NAME]" | |
git bundle create "${BACKUP_DIRECTORY}/${REPOSITORY_NAME}-${LAST_COMMIT_FROM_REPOSITORY}.bundle" --all | |
echo "${REPOSITORY_NAME}-${LAST_COMMIT_FROM_REPOSITORY}.bundle" >> /tmp/bundles.created.$$ | |
# Delete old bundle backups to keep only "max" of them | |
ls -t "${BACKUP_DIRECTORY}/${REPOSITORY_NAME}"*.bundle | tail -n +${MAX_BUNDLE_BACKUPS_TO_BE_KEPT} >> /tmp/bundles.deleted.$$ | |
ls -t "${BACKUP_DIRECTORY}/${REPOSITORY_NAME}"*.bundle | tail -n +${MAX_BUNDLE_BACKUPS_TO_BE_KEPT} | xargs -I {} rm {} | |
fi | |
cd - >/dev/null 2>&1 | |
done | |
# LFS to be tarr'ed (only if needed) | |
if [[ -f /tmp/lfs.$$ ]] ; then | |
echo "LFS backup needed for these repositories : " | |
cat /tmp/lfs.$$ | sort | |
tar czvf "${BACKUP_DIRECTORY}/lfs-backup.tar.gz" "${LFS_DIRECTORY}" | |
fi | |
# Summaries | |
if [[ -f /tmp/bundles.created.$$ ]] ; then | |
echo "" | |
echo "Bundles created : " | |
echo "=====================================" | |
cat /tmp/bundles.created.$$ | sort | |
fi | |
if [[ -f /tmp/bundles.deleted.$$ ]] ; then | |
echo "" | |
echo "Bundles removed : " | |
echo "=====================================" | |
cat /tmp/bundles.deleted.$$ | sort | |
fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
30 1 * * * /usr/bin/docker run --rm --name forgejo-backuper -e DATE=$(date "+\%Y-\%m-\%d") --link postgresql:database_postgres -v volume_forgejo:/data/ -v volume_forgejo_backup_dumps:/data/backups/ -w /data/backups/ --entrypoint "" -e USER=git --user git codeberg.org/forgejo/forgejo:9 sh -c "forgejo dump --verbose --database postgres --skip-repository --skip-lfs-data --config /data/gitea/conf/app.ini --type tar.gz --file /data/backups/\$DATE-forgejo-dump.tar.gz" > /var/log/cron-backup-docker-forgejo.log 2>&1 | |
5 1 * * * /usr/bin/docker run --rm --name "forgejo-backup-repositories" -v volume_forgejo:/data/ -v volume_forgejo_backup_repositories:/data/backups/ forgejo-backuper > /var/log/cron-backup-docker-forgejo-repositories.log 2>&1 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
FROM debian:bookworm-slim | |
CMD ["/backup.sh"] | |
RUN apt-get update && apt-get install -y git git-lfs && \ | |
git lfs install && \ | |
groupadd -g 1000 git && \ | |
useradd -u 1000 -d /git/ -s /usr/bin/bash -g git -c "git" git && \ | |
cp -Rp /root/ /git/ && chown -R 1000:1000 /git/ && \ | |
apt-clean | |
COPY backup.sh /backup.sh | |
ENV HOME=/git/ | |
USER git | |
WORKDIR /data/ |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment