Create AWS DMS source endpoints

dms_source_endpoints.sh

#! /bin/bash
# Connect to mongo db's via an ssh tunnel,
# get a list of dbs to create DMS source endpoints
# dms_source_endpoints.sh port proxy [ name:dns:port ... ]

create-ssh-tunnel() {
    local endpoint=$1

    ssh -L $tunnel_port:$endpoint $proxy -N &
    echo $!
}

create-parameters() {
    echo $* | awk -F":" '{
        printf("identifier=%s;\n", $1)
        printf("server_name=%s;\n", $2)
        if (NF > 2) {
            printf("endpoint=%s:%s;\n", $2, $3)
            printf("port=%s;\n", $3)
        } else {
            printf("endpoint=%s;\n", $2)
        }
    }'
}

cleanup() {
    exec 2> /dev/null

    rm -f $tmpfile*

    if [ ! -z $tunnel_pid ]
    then
        kill $tunnel_pid
    fi
}

tmpfile=$(mktemp)
tunnel_port=$1
proxy=$2

trap 'cleanup' EXIT

shift 2

for host in $*
do
    eval $(create-parameters $host)
    tunnel_pid=$(create-ssh-tunnel $endpoint)

    mongo --host localhost \
          --port $tunnel_port \
          --quiet \
          --eval  "let response = db.adminCommand({listDatabases: 1, nameOnly: 1});
                   response.databases;" |
    jq --raw-output \
       '.[] | select(.name != "admin") |
        select(.name != "config") |
        select(.name != "local") | .name' > $tmpfile.db.list

    while read db
    do
        command="aws dms create-endpoint \
                --endpoint-identifier $identifier-$db \
                --endpoint-type source \
                --engine-name mongodb \
                --server-name $server_name \
                --port $port \
                --database-name $db \
                --mongo-db-settings AuthType=no"

        if ! $command
        then
            echo $command
        fi

    done < $tmpfile.db.list

    kill $tunnel_pid
done

dms_target_endpoints.sh

#! /bin/bash
# Create target docdb endpoints from a list of source mongodb endpoints
# dms_target_endpoints.sh username password host port certificate_arn

get-sources() {
    aws dms describe-endpoints \
            --filters Name=endpoint-type,Values=source |
    jq -r '.Endpoints[].EndpointIdentifier' |
    sort
}

get-databases() {
    awk '{
        position = index($0, "-")

        identifier = substr($0, 0, position - 1)
        db = substr($0, position + 1)

        if (!(db in endpoints) || (db in endpoints && identifier == "production")) {
            endpoints[db] = identifier
        }

    }END{
        for (database in endpoints) {
            print(database)
        }
    }' $*
}

cleanup() {
    exec >&2

    rm -f $tmpfile*
}

username=$1
password=$2
server_name=$3
port=$4
certificate=$5
tmpfile=$(mktemp)

trap 'cleanup' EXIT

get-sources > $tmpfile.source.list
get-databases $tmpfile.source.list |
while read db
do
    command="aws dms create-endpoint \
            --endpoint-identifier $db \
            --endpoint-type target \
            --engine-name docdb \
            --username $username \
            --password $password \
            --server-name $server_name \
            --port $port \
            --database-name $db \
            --ssl-mode verify-full \
            --certificate-arn $certificate"

    if ! $command
    then
        echo $command
    fi
done