etl snapshot script.

etl_backups.sh

#!/bin/bash

# Variables
ETL_DIR=/root/blockchain-etl
ETL_USER=etl
ETL_DB=etl
ETL_PASS=etl
EXPORT_CORES=32
PSQL_HOST=localhost
EXPORT_DIR=/root/export
GCP_BUCKET=dewi_etl_snapshots
DATE_MIN=$(date +'%Y_%m_%d_%H_%M')
STARTTIME=$(date +%s)

# Logs
# exec 3>&1 4>&2
# trap 'exec 2>&4 1>&3' 0 1 2 3
# exec 1>log_$DATE_MIN.out 2>&1

# Start
echo "Starting ETL Snapshot"
echo "========================"

echo "Stopping ETL"
echo "Compressing ETL"
cd $ETL_DIR && make stop

# Get blockchain-etl git head
HEAD=$(/usr/bin/git rev-parse --short HEAD)
  
zip -r $EXPORT_DIR/etl_snapshot_$HEAD-$DATE_MIN.zip $ETL_DIR

echo "Exporting Database As Directory"
PGPASSWORD=$ETL_PASS pg_dump -h $PSQL_HOST -U $ETL_USER -x -Fd -j $EXPORT_CORES $ETL_DB -f $EXPORT_DIR/database

echo "Compressing Database"
zip -r $EXPORT_DIR/database_snapshot_$HEAD-$DATE_MIN.zip $EXPORT_DIR/database

echo "Removing Data Directory"
rm -rf $EXPORT_DIR/database

echo "Restarting ETL"
cd $ETL_DIR && make start

echo "Upload Files to CGP"
gsutil cp $EXPORT_DIR/etl_snapshot_$HEAD-$DATE_MIN.zip gs://$GCP_BUCKET
gsutil cp $EXPORT_DIR/database_snapshot_$HEAD-$DATE_MIN.zip gs://$GCP_BUCKET

echo "Delete Files"
rm $EXPORT_DIR/etl_snapshot_$HEAD-$DATE_MIN.zip $EXPORT_DIR/database_snapshot_$HEAD-$DATE_MIN.zip

echo "  _    ___  _  _  ___   _    _____   _____   ___ _____ _    _ "
echo " | |  / _ \| \| |/ __| | |  |_ _\ \ / / __| | __|_   _| |  | |"
echo " | |_| (_) | .  | (_ | | |__ | | \ V /| _|  | _|  | | | |__|_|"
echo " |____\___/|_|\_|\___| |____|___| \_/ |___| |___| |_| |____(_)"
                                                              
ENDTIME=$(date +%s)
echo "It took $(($ENDTIME - $STARTTIME)) seconds to complete the backup..."