Last active
January 4, 2021 11:43
-
-
Save Tjitse-E/c9c77f5aa5ff73ee229fe245938120f9 to your computer and use it in GitHub Desktop.
Github action recipe to backup a remote DB via n98-magerun2, anonymize with Masquerade and push the file to DigitalOcean Spaces (S3)
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Fetch DB, anonymize and upload to S3 | |
env: | |
MASQUERADE_DOWNLOAD_URL: 'https://github.com/elgentos/masquerade/releases/latest/download/masquerade.phar' | |
DO_SPACES_HOST: 'ams3.digitaloceanspaces.com' | |
ANONYMIZED_DB_NAME: 'db_anonymized.sql.gz' | |
MYQSL_HOST: '127.0.0.1' | |
MYSQL_PWD: 'root' | |
REMOTE_HOST: ${{ secrets.REMOTE_HOST }} # source server host | |
REMOTE_USER: ${{ secrets.REMOTE_USER }} # source server use | |
REMOTE_PORT: ${{ secrets.REMOTE_PORT }} # source server port | |
DO_SPACES_DB_LOCATION: ${{ secrets.DIGITAL_OCEAN_SPACES_DB_LOCATION }} # db/your_database.sql.gz | |
DO_SPACES_NAME: ${{ secrets.DIGITAL_OCEAN_SPACES_NAME }} # vendic | |
on: | |
schedule: | |
- cron: '0 3 * * *' | |
jobs: | |
anonymize: | |
name: Anonymize DB and push to S3 | |
runs-on: ubuntu-latest | |
services: | |
mysql: | |
image: mysql:5.7 | |
env: | |
MYSQL_DATABASE: db_anonymized | |
MYSQL_USER: root | |
MYSQL_ROOT_PASSWORD: root | |
ports: | |
- 3306 | |
options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 | |
steps: | |
- uses: actions/checkout@v2 | |
- uses: actions/setup-python@v1 | |
- uses: BSFishy/pip-action@v1 | |
with: | |
packages: | | |
s3cmd | |
python-dateutil | |
python-magic | |
- name: Check out source code | |
uses: actions/checkout@v2 | |
- name: Install SSH key | |
uses: shimataro/ssh-key-action@v2 | |
with: | |
key: ${{ secrets.MASQUERADE_SSH_KEY }} | |
name: id_rsa # optional | |
known_hosts: 'github.com' | |
- name: Create DB backup via SSH | |
run: | | |
export REMOTE_DB_BACKUP=${REMOTE_USER}_$(date +'%m_%d_%Y') | |
ssh -o StrictHostKeyChecking=no ${REMOTE_USER}@${REMOTE_HOST} -p ${REMOTE_PORT} \ | |
'source ~/.bash_profile && n98-magerun2 db:dump --compression="gzip" --strip="@stripped" ${USER}_$(date +'%m_%d_%Y')' | |
- name: Download DB backup via SSH | |
run: | | |
export REMOTE_DB_BACKUP=${REMOTE_USER}_$(date +'%m_%d_%Y').sql.gz | |
scp -P ${REMOTE_PORT} ${REMOTE_USER}@${REMOTE_HOST}:${REMOTE_DB_BACKUP} ${REMOTE_DB_BACKUP} | |
test -f ${REMOTE_DB_BACKUP} && echo "$FILE exists" | |
- name: Remove DB backup via SSH | |
run: | | |
export REMOTE_DB_BACKUP=${REMOTE_USER}_$(date +'%m_%d_%Y').sql.gz | |
ssh -o StrictHostKeyChecking=no ${REMOTE_USER}@${REMOTE_HOST} -p ${REMOTE_PORT} "rm ${REMOTE_DB_BACKUP}" | |
- name: Start MySQL server | |
run: sudo service mysql start | |
- name: Prepare test database | |
run: | | |
export MYSQL_TCP_PORT=${{ job.services.mysql.ports['3306'] }} | |
mysql -e 'CREATE DATABASE IF NOT EXISTS db_anonymized;' -uroot -proot | |
mysql -e 'GRANT ALL PRIVILEGES ON db_anonymized.* TO "db_anonymized"@"127.0.0.1" IDENTIFIED BY "password1"' -uroot -proot | |
mysql -e 'GRANT ALL PRIVILEGES ON db_anonymized_scaffold.* TO "db_anonymized"@"127.0.0.1" IDENTIFIED BY "password1"' -uroot -proot | |
- name: Import database | |
run: | | |
export LOCAL_DB_BACKUP=${REMOTE_USER}_$(date +'%m_%d_%Y').sql.gz | |
export MYSQL_TCP_PORT=${{ job.services.mysql.ports['3306'] }} | |
zcat ${LOCAL_DB_BACKUP} | mysql -uroot -proot db_anonymized | |
- name: Anonymize db with masquerade | |
run: | | |
export MYSQL_TCP_PORT=${{ job.services.mysql.ports['3306'] }} | |
curl -L -o masquerade.phar ${MASQUERADE_DOWNLOAD_URL} | |
chmod +x masquerade.phar | |
./masquerade.phar run --platform=magento2 \ | |
--database=db_anonymized \ | |
--host=$MYQSL_HOST \ | |
--username=root \ | |
--password=root | |
- name: Dump DB | |
run: | | |
export MYSQL_TCP_PORT=${{ job.services.mysql.ports['3306'] }} | |
mysqldump -P $MYSQL_TCP_PORT -h $MYQSL_HOST -u root db_anonymized | gzip > ${ANONYMIZED_DB_NAME} | |
test -f ${ANONYMIZED_DB_NAME} && echo "$FILE exists" | |
- name: Upload anonymized db to S3 | |
run: | | |
s3cmd --access_key=${{ secrets.DIGITALOCEAN_SPACES_ACCESS_KEY }} \ | |
--secret_key=${{ secrets.DIGITALOCEAN_SPACES_SECRET_KEY }} \ | |
--host-bucket="%(bucket)s.${DO_SPACES_HOST}" \ | |
--host="${DO_SPACES_HOST}" \ | |
put ${ANONYMIZED_DB_NAME} s3://${DO_SPACES_NAME}/${DO_SPACES_DB_LOCATION} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment