circleci ha database export script

circleci-database-export

#!/bin/bash

##
# CircleCI Database Export Script
#
#   This script is for installations running CircleCI server 2.0 or higher.
#
# 	This script will create a tar ball of the PostgreSQL and Mongo databases.
# 	This should generally be used when you are planning on switching from
# 	the default embedded databases to an external database source.
#
# 	This script should be run as root from the CircleCI Services Box. CircleCI and any
# 	additional postgresql or mongo containers should be shut down to eliminate
# 	any chances of data corruption.
##

set -e

# Constants
DATE=$(date +"%Y-%m-%d-%s")
MONGO_VERSION="3.2.11"
MONGO_DATA="/data/circle/mongo"
MONGO_BU="circleci-mongo-export-${DATE}"
TMP_MONGO="circle-mongo-export"
PGDATA="/data/circle/postgres/9.5/data"
PGVERSION="9.5.8"
TMP_PSQL="circle-postgres-export"
PG_BU="circleci-postgres-export-${DATE}"


##
# Preflight checks
# 	make sure script is running as root
#	make sure circle, mongo, and postgres are shut down
##
function preflight_checks() {

    if [ $(id -u) -ne 0 ]
    then
        echo "Please run this script as root"
        exit 1

    elif [ -n "$(docker ps | grep circleci-frontend | head -n1 )" ]
    then
        echo "Please shut down CircleCI from the replicated console at https://<YOUR_CIRCLE_URL>:8800 before running this script."
        exit 1

    elif [ -n "$(docker ps | grep mongo | head -n1 )" ]
    then
        echo "Please shut down any other Mongo containers before running this script"
        exit 1

    elif [ -n "$(docker ps | grep postgres | head -n1 )" ]
    then
        echo "Please shut down any other PostgreSQL containers before running this script"
        exit 1
    fi
}

##
# Start temporary postgreSQL container with mounted data volumen
##
function start_postgres() {
    echo ... starting new postgres container with existing volume
    docker run --rm --name $TMP_PSQL -d -v $PGDATA:/var/lib/postgresql/data postgres:$PGVERSION
}

##
# Stop the PostgreSQL container that we created with the start_postgres function
##
function stop_postgres() {
    echo ... stoping postgresql container
    docker stop $TMP_PSQL
}

##
# Export PostgreSQL database once the container has started and is accepting connections
##
function export_postgres() {
    echo ... exporting postgresql database

    until docker exec $TMP_PSQL psql -U postgres -c '\l'; do
        >&2 echo "Postgres is starting up ... will try again momentarily"
        sleep 3
    done

    # note that this file is generated on in the current working directory
    # where this script is ran from.
    docker exec $TMP_PSQL pg_dump -U postgres circle > circle.sql
    docker exec $TMP_PSQL pg_dump -U postgres vms > vms.sql
    docker exec $TMP_PSQL pg_dump -U postgres conductor_production > conductor_production.sql
    docker exec $TMP_PSQL pg_dump -U postgres contexts_service_production > contexts_service_production.sql
    # @TODO uncomment when cron-service is released
    # docker exec $TMP_PSQL pg_dump -U postgres cron_service_production > cron_service_production.sql
}

##
# Basic sanity check / smoke test to ensure that the postgresql export performed by
# the export_postgres function contains what we expect it to contain
##
function check_postgres() {
    echo ... verifying postgres export file
    CHECK=$(cat $PG_BU.sql | grep circle_migrations | head -n1)
    if [ -z "$CHECK" ]
    then
        echo "[FATAL] Something is wrong with the postgresql export file, please contact CircleCI support at [email protected] for further assistance."
        stop_postgres
        exit 1
    fi
}

##
# Start temporary Mongo container with mounted data volume
##
function start_mongo() {
    echo ... starting new mongo container with existing volume
    docker run --rm --name $TMP_MONGO -d -v $MONGO_DATA:/data/db mongo:$MONGO_VERSION
}

##
# Stop the Mongo container tht we created with the start_mongo function
##
function stop_mongo() {
    echo ... stoping mongo container
    docker stop $TMP_MONGO
}

##
# Export Mongo database once the container has started and is accepting connections
##
function export_mongo() {
    echo ... exporting mongo database

    until docker exec $TMP_MONGO mongo --eval "db.stats()"; do
        >&2 echo "Mongo is starting up ... will try again momentarily"
        sleep 3
    done

    # note that this file is generated inside of the container and then moved to the
    # current working directory.
    docker exec $TMP_MONGO bash -c "mkdir -p /data/db/dump-${DATE} && cd /data/db/dump-${DATE} && mongodump"
    mv /data/circle/mongo/dump-${DATE}/dump $(pwd)/$MONGO_BU
}

##
# Basic santiy check / smoke test to ensure that the mongo export performed by
# the export_mongo function contains what we expect it to contain.
##
function check_mongo() {
    echo ... verifying mongo export files
    CHECK=$(ls -al $MONGO_BU | grep circle_ghe)
    if [ -z "$CHECK" ]
    then
        echo "[FATAL] Something is wrong with the mongo export, please contact CircleCI support at [email protected] for further assistance."
        exit 1
    fi
}

##
# Create tar ball with the postgreSQL and Mongo exports
##
function compress() {
    echo ... compressing exported files
    tar cvfz circleci_database_export_${DATE}.tar.gz *.sql $MONGO_BU
    rm *.sql
    rm -rf $MONGO_BU
}

##
# Main function
##
function circleci_database_export() {
    echo "Starting CircleCI Database Export"

    preflight_checks
    start_mongo
    export_mongo
    check_mongo
    stop_mongo
    start_postgres
    export_postgres
    check_postgres
    stop_postgres
    compress

    echo "CircleCI Database Export Complete."
    echo "Your exported files can be found at $(pwd)/circleci_database_export_${DATE}.tar.gz"
}

circleci_database_export