kikori - a MySQL binary log archive utility

kikori.sh

#!/bin/bash

### kikori - a MySQL binary log archive utility

# Support config options

format="${KIKORI_FORMAT:-directory}"
archive_path="${KIKORI_ARCHIVE_PATH:-/var/log/mysql/archive}"
compression_cmd="${KIKORI_COMPRESSION_CMD:-gzip -1}"
compression_ext="${KIKORI_COMPRESSION_EXT:-.gz}"
cloud_auth_url="${KIKORI_CLOUD_AUTH_URL:-https://identity.api.rackspacecloud.com}"
cloud_files_retention="${KIKORI_CLOUD_RETENTION:-1 week}"
mysql_master_opts="${KIKORI_MYSQL_MASTER_OPTS:---defaults-group-suffix=-master}"
mysql_slave_opts="${KIKORI_MYSQL_SLAVE_OPTS:---defaults-group-suffix=-slave}"
binlog_directory="${KIKORI_BINLOG_DIRECTORY:-/var/log/mysql/}"
use_syslog="${KIKORI_USE_SYSLOG:-0}"
config_file="${KIKORI_CONFIG:-$HOME/.kikori}"

# source a config file, if available
[[ -f $config_file ]] && source $config_file

set -o pipefail

abort() {
    echo "$@"
    exit 1
}

usage() {
    echo "$0 <archive|download> [options]"
    echo
    echo "Options:"
    echo " -?/--help                          Show this help"
    echo " -F/--format <name>                 Format for archive: cloud or directory"
    echo "                                    (default: directory)"
    echo " -u/--cloud-user <name>             Username to auth to cloud"
    echo " -k/--cloud-apikey <key>            API key to auth to cloud"
    echo " -C/--cloud-files-container <name>  Container to archive to (when format=cloud)"
    echo " -I/--cloud-auth-url <url>          URL to use for cloud authentication"
    echo "                                    (default: https://identity.api.rackspacecloud.com/v2.0/)"
    echo " -M/--mysql-master-opts <opts>      Command line options for auth to master"
    echo "                                    (default: --default-group-suffix=-master)"
    echo " --skip-slave-check                 Don't check the slave log position"
    echo " -S/--mysql-slave-opts <opts>       Command line options for auth to slave"
    echo "                                    (default: --default-group-suffix=-slave)"
    echo " -L/--binlog-directory <path>       Path to binary log directory"
    echo " -D/--archive-path <path>           Path where binary logs should be archived"
    echo " -Z/--compression-command <args>    Command to use for compressing binary logs"
    echo " -E/--compression-ext <name>        Filename extension to use for compressed files"
    echo " --syslog                           Send script output to syslog"
}

parse_options() {
    local arg
    cmd=$1
    [[ -z "$cmd" ]] && { echo "Command must be specified"; usage 1>&2; exit 1; }
    [[ $cmd = "archive" || $cmd = "download" ]] || abort "Unknown command '$cmd'"
    shift
    while [[ $# > 0 ]]
    do
        case $1 in
            -\?|--help)
                usage
                exit 0
                ;;
            -F|--format)
                shift
                format=$1
                ;;
            -D|--archive-path)
                shift
                archive_path=$1
                ;;
            -Z|--compression-commnad)
                shift
                compression_cmd=$1
                ;;
            -E|--compression-ext)
                shift
                compression_ext=$1
                ;;
            -u|--cloud-user)
                shift
                cloud_user=$1
                ;;
            -k|--cloud-apikey)
                shift
                cloud_apikey=$1
                ;;
            -C|--cloud-files-container)
                shift
                cloud_files_container=$1
                ;;
            -I|--cloud-auth-url)
                shift
                cloud_auth_url=$1
                ;;
            -M|--mysql-master-options)
                shift
                mysql_master_opts=$1
                ;;
            -S|--mysql-slave-options)
                shift
                mysql_slave_opts=$1
                ;;
            -L|--binlog-directory)
                shift
                binlog_directory=$1
                ;;
            --syslog)
                use_syslog=1
                ;;
            *)
                echo "Unknown option: '$1'" 1>&2
                ;;
        esac
        shift
    done
}

## Cloud support stuff
cf_authenticate() {
    local payload="
    {
        \"auth\": {
            \"RAX-KSKEY:apiKeyCredentials\": {
                \"username\": \"$cloud_user\",
                \"apiKey\": \"$cloud_apikey\"
            }
        }
    }"
    jq '.' <<<"${payload}" | \
    curl -s \
         -X POST \
         -H "Content-type: application/json" \
         -d @- \
         "${cloud_auth_url}/v2.0/tokens"
}

cf_service_url() {
    local region=$1
    local name=$2
    jq -r ".access.serviceCatalog[] |
           select(.type == \"${name}\").endpoints[] |
           select(.region == \"${region}\").publicURL" <<< "${cloud_registry}"
}

# cf_upload <path>
cf_upload() {
    local path=$1
    local mimetype=$(file -bi "${path}")
    local rpath="${path##*/}${compression_ext}"
    local cpath=$(readlink -f ${path})
    local url="${cloud_files_url}/${cloud_files_container}/${rpath}"
    #echo "${url} [${mimetype}]"
    local config="
    request = PUT
    header = \"X-Auth-Token: ${cloud_auth_token}\"
    header = \"Content-Type: ${mimetype}\"
    header = \"Cache-Control: public, max-age=900\"
    header = \"Transfer-Encoding: chunked\"
    url = \"${url}\"
    "
    if [[ -n "${cloud_files_retention}" ]]
    then
        local delete_after
        local path_ts
        path_ts=$(TZ=UTC stat -c '%y' "$cpath")
        delete_after=$(date -u -d "$path_ts + ${cloud_files_retention}" '+%s')
        config="${config}header = \"X-Delete-At: ${delete_after}\""
    fi

    ${compression_cmd} < "${cpath}" | \
        curl --fail --silent \
             --data-binary @- \
             --config <(cat <<< "${config}")
    rc=$?
    return $rc
}

## Logging support
enable_syslog() {
    exec > >(logger -is -t "$0") 2>&1
}

## Helper methods
check_config() {
    echo "Archive format:       $format"
    echo "Binary log path:      $binlog_directory"
    echo "Compression command:  $compression_cmd"
    echo "MySQL Master Host:    $(mysql $mysql_master_opts -sse 'SELECT CONCAT(USER(), "/", @@hostname, ":", @@datadir)')"
    echo "MySQL Slave Host:     $(mysql $mysql_slave_opts -sse 'SELECT CONCAT(USER(), "/", @@hostname, ":", @@datadir)')"
    if [[ $format = "cloud" ]]
    then
        echo "Cloud Container:      $cloud_files_container"
        date -d "${cloud_files_retention}" &> /dev/null || \
            abort "Invalid retention '${cloud_files_retention}'" && \
        echo "Binlog Retention:     ${cloud_files_retention}"
    elif [[ $format = "directory" ]]
        echo "Backup directory:     $archive_path"
        [[ -d ${archive_path} ]] || abort "${archive_path} is not a directory."
    else
        abort "Unsupported format '$format'"
    fi
}

## Main archiving method
archive_to_directory() {
    local name
    name=$1
    # canonicalize the binlog we are copying
    src_path=$(readlink -f ${binlog_directory}/${name})
    # canonlicalize the path to the archived binlog we're writing
    dst_path=$(readlink -m ${archive_path}/${name})${compression_ext}
    # compress the src binlog and pipe through dd so we fsync at
    # the end
    ${compression_cmd} < ${src_path} | \
        dd of=${dst_path} bs=4k conv=fsync status=none
}

archive_to_cloud() {
    local name
    name=$1
    cf_upload "$(readlink -f ${binlog_directory}/${name})"
    rc=$?
    return $rc
}


process_binlogs() {
    if [[ "${skip_slave_check}" = 0 ]]
    then
        required_binlog=$(mysql $mysql_master_opts -sse 'SHOW MASTER STATUS' | cut -f1)
    else
        required_binlog=$(mysql $mysql_slave_opts -sse 'SHOW SLAVE STATUS' | cut -f10)
    fi
    [[ $? -eq 0 && -n "$required_binlog" ]] || abort "Failed to discover binary log to retain"

    echo "Binary log to keep: $required_binlog"

    local n_processed
    local name
    local size

    n_processed=0
    while read name size
    do
        # avoid a noop if processing the first binary log
        if [[ n_processed  -gt 0 ]]
        then
            echo "Purging binary logs prior to '${name}'"
            # Note: This purges everything before $name, but not the named binlog
            mysql $mysql_master_opts -sse "PURGE BINARY LOGS TO '$name'" || \
                abort "Failed to purge binary logs."
        fi
        # stop processing as soon as we hit the target binlog
        [[ $name = $required_binlog ]] && { echo "Stopping at binlog: $name"; break; }

        echo -n "Archiving: $name ($size bytes)..."

        if [[ $format = "directory" ]]
        then
            archive_to_directory "$name"
        elif [[ $format = "cloud" ]]
        then
            archive_to_cloud "$name"
        fi
        rc=$?
        [[ $rc -eq 0 ]] && echo "OK" || \
            { echo "FAILED"; abort "Failed to archive $name. Aborting."; }
        (( n_processed++ ))
    done < <(mysql $mysql_master_opts -sse 'SHOW BINARY LOGS')
    echo "Processed ${n_processed} binary logs."
}

download_binlogs() {
    [[ $format = "cloud" ]] || \
        abort "Download of binary logs requested, but format != 'cloud'"
    local name
    local config
    config="header=\"X-Auth-Token: ${cloud_auth_token}\""
    rm -f "${archive_path}/bin-log.index"
    for name in $(curl -s "${cloud_files_url}/${cloud_files_container}/" -K - <<< "${config}")
    do
        echo -n "Downloading '${name}'..."
        curl -f -s "${cloud_files_url}/${cloud_files_container}/${name}" -K - <<< "${config}" | \
            zcat > ${archive_path}/$(basename "${name}" .gz)
        [[ $? -eq 0 ]] && echo "OK" || { echo "Failed"; break; }
        echo "$(basename "${name}" .gz)" >> ${archive_path}/bin-log.index
    done
}

parse_options "$@"

[[ $use_syslog = "1" ]] && enable_syslog

check_config

if [[ $format = "cloud" ]]
then
    cloud_registry=$(cf_authenticate)
    [[ $? = 0 ]] || abort "Failed to auth to Rackspace Cloud"
    cloud_auth_token=$(jq -r '.access.token.id' <<< "$cloud_registry")
    [[ $? = 0 ]] || abort "Failed to find authentication token in registry"
    cloud_files_url=$(cf_service_url "${OS_REGION_NAME:-IAD}" 'object-store')
    [[ $? = 0 ]] || abort "Failed to find Cloud Files URL in registry"
fi

if [[ $cmd = "archive" ]]
then
    mkdir -p "$archive_path"
    process_binlogs
elif [[ $cmd = "download" ]]
then
    download_binlogs
fi