Skip to content

Instantly share code, notes, and snippets.

Show Gist options
  • Save BeshoyAtef/b4d2f99e2980f9e78d59e2824653b367 to your computer and use it in GitHub Desktop.
Save BeshoyAtef/b4d2f99e2980f9e78d59e2824653b367 to your computer and use it in GitHub Desktop.
Bash scripts to backup all databases in a MySQL server with the option to exclude some databases.
#!/usr/bin/env bash
# Combined MySQL Database Management Script
# Allows EITHER deleting selected databases OR importing selected/all databases from .sql.gz files.
# WARNING: The delete function is EXTREMELY DANGEROUS and IRREVERSIBLE. Use with caution.
# REQUIREMENTS
## mysql client, gunzip, find, grep, sed (standard Linux tools)
## Bash v4+ recommended for mapfile (can be adapted for older Bash if needed)
## Script assumes MySQL is running locally.
# --- Configuration ---
# TODO: Review and update these settings before running!
# Set MySQL User (Needs sufficient privileges for chosen action)
# - DROP for delete action
# - CREATE DATABASE, privileges on imported DBs for import action
DBUSER="root"
# Set MySQL Password (leave empty for no password or prompt)
# WARNING: Hardcoding passwords is insecure. Consider ~/.my.cnf or environment variables.
DBPASS=""
# Set path to the folder containing the .sql.gz backup files (for IMPORT action)
IMPORTPATH="/backups/mysql/" # TODO: Update this path if importing
# Set file extension pattern to look for (for IMPORT action)
# Used to find files and extract DB name (filename minus this extension)
EXT=".sql.gz"
# --- Critical System Databases to Exclude (for DELETE action) ---
# DO NOT REMOVE SYSTEM DBS UNLESS YOU ARE AN EXPERT.
# Add any other databases you MUST keep to this list.
excludedDatabases=( mysql information_schema performance_schema sys Database )
# Example: excludedDatabases=( mysql information_schema performance_schema sys Database my_important_app_db )
# --- Global Variables ---
# Build MySQL command options dynamically later
MYSQL_OPTS=""
# --- Script Enhancements ---
# Exit immediately if a command exits with a non-zero status.
# set -e # Optional: uncomment for stricter error handling, but can sometimes mask issues in loops.
# Treat unset variables as an error when substituting.
set -u
# Ensure pipeline commands fail if any command in the pipe fails
set -o pipefail
# --- Function Definitions ---
# Function to print errors to stderr
error_exit() {
echo "ERROR: $1" >&2
exit 1
}
# Function to setup and test MySQL connection
# Sets the global MYSQL_OPTS variable
setup_mysql_connection() {
echo "Attempting to configure MySQL connection for user '${DBUSER}'..."
MYSQL_OPTS="-u${DBUSER}"
local connection_ok=false
# Prefer ~/.my.cnf if it exists and no password is explicitly set
if [[ -z "$DBPASS" && -f "$HOME/.my.cnf" ]]; then
# Check if .my.cnf defines user/password for the client section
if grep -q "user=" "$HOME/.my.cnf" && grep -q "password=" "$HOME/.my.cnf"; then
echo "Using credentials from ~/.my.cnf"
# Unset DBPASS if we rely on .my.cnf to avoid confusion later
DBPASS="" # Ensure it's empty
MYSQL_OPTS="" # Reset opts, rely entirely on .my.cnf default group [client]
# Test connection using .my.cnf
if mysql ${MYSQL_OPTS} -e "SELECT 1;" &>/dev/null; then
connection_ok=true
else
echo "WARN: ~/.my.cnf found, but connection failed. Check config or try explicit user/pass." >&2
# Fall through to try explicit user/prompt method
fi
fi
fi
# If not using .my.cnf or it failed, proceed with configured user/pass/prompt
if [[ "$connection_ok" == "false" ]]; then
MYSQL_OPTS="-u${DBUSER}" # Re-initialize if .my.cnf failed
if [ -n "$DBPASS" ]; then
MYSQL_OPTS+=" -p${DBPASS}"
# Test connection with password
if mysql ${MYSQL_OPTS} -e "SELECT 1;" &>/dev/null; then
connection_ok=true
fi
else
# Test connection without password first (if no DBPASS set and .my.cnf wasn't successfully used)
if mysql ${MYSQL_OPTS} -e "SELECT 1;" &>/dev/null; then
connection_ok=true
echo "Connection successful without password."
else
# If no password provided AND connection failed, add prompt flag and test again
echo "Connection without password failed. Will try prompting for password..."
MYSQL_OPTS+=" -p"
# Test connection with prompt (will hang if no interactive terminal)
if mysql ${MYSQL_OPTS} -e "SELECT 1;" &>/dev/null; then
connection_ok=true
echo "Connection successful (likely after password prompt)."
fi
fi
fi
fi
if [[ "$connection_ok" == "false" ]]; then
echo "------------------------------------------------------------------" >&2
echo "ERROR: Failed to connect to MySQL as user '${DBUSER}'." >&2
echo "Please check:" >&2
echo " - MySQL server status." >&2
echo " - User '${DBUSER}' exists and has correct privileges." >&2
echo " - Password (if required) is correct (set in DBPASS, ~/.my.cnf or entered at prompt)." >&2
echo " - Host connection settings (script assumes local connection)." >&2
echo "------------------------------------------------------------------" >&2
return 1 # Indicate failure
fi
# Add batch mode for cleaner output during operations
MYSQL_OPTS+=" -B"
echo "MySQL connection configured successfully."
return 0 # Indicate success
}
# Function to delete selected databases
perform_delete() {
echo "########################################### Start Database Deletion ##############################################"
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!! WARNING !!!!!!!!!!!!!!!!!!!!!!!!!!!!"
echo "This function will list databases for potential permanent deletion."
echo "It will EXCLUDE the following system/critical databases:"
printf " - %s\n" "${excludedDatabases[@]}"
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
# Get list of all databases using mapfile for safety with names
echo "Fetching list of all databases..."
mapfile -t allDatabases < <(mysql $MYSQL_OPTS -Nse "SHOW DATABASES;")
# Capture exit codes immediately after mapfile/process substitution
local mapfile_exit_code=$?
local mysql_show_exit_code=${PIPESTATUS[0]}
if [[ $mapfile_exit_code -ne 0 || $mysql_show_exit_code -ne 0 ]]; then
echo "ERROR: Failed to retrieve database list (mysql exit: $mysql_show_exit_code, mapfile exit: $mapfile_exit_code)." >&2
return 1
fi
# Determine which databases are candidates for dropping
candidateDatabases=()
echo "Identifying databases eligible for deletion (excluding configured list):"
for db in "${allDatabases[@]}"; do
isExcluded="false"
for excluded in "${excludedDatabases[@]}"; do
if [[ "$db" == "$excluded" ]]; then
isExcluded="true"
break
fi
done
if [[ "$isExcluded" == "false" ]]; then
candidateDatabases+=("$db")
fi
done
# Check if any databases are candidates for deletion
if [[ ${#candidateDatabases[@]} -eq 0 ]]; then
echo "No user databases found to delete (only system/excluded databases exist)."
echo "Deletion process finished. No databases were dropped."
echo "############################################ End Database Deletion ###############################################"
return 0
fi
# --- Interactive Selection ---
echo
echo "Select databases to DELETE:"
local i=1
declare -A selected_indices # Use associative array for easy lookup
for db in "${candidateDatabases[@]}"; do
echo " $i) $db"
((i++))
done
echo " all) Delete ALL listed databases"
echo " n) No deletion (cancel)"
local selection=()
local choice
while true; do
read -p "Enter numbers (space-separated), 'all', or 'n': " choice
choice=$(echo "$choice" | tr '[:upper:]' '[:lower:]') # Lowercase for easier matching
if [[ "$choice" == "n" ]]; then
echo "Deletion cancelled by user."
echo "############################################ End Database Deletion ###############################################"
return 0
elif [[ "$choice" == "all" ]]; then
selection=("${candidateDatabases[@]}")
echo "Selected ALL eligible databases for deletion."
break
# Use POSIX character classes for broader compatibility with =~
elif [[ "$choice" =~ ^[[:digit:][:space:]]+$ ]]; then
# Validate and collect selected numbers
selection=()
local invalid_choice=false
for num in $choice; do
# Ensure it's a valid number within the range
if [[ "$num" -ge 1 && "$num" -le ${#candidateDatabases[@]} ]]; then
# Avoid duplicates if user enters same number twice
# Use :- default value construct for safety with set -u if key doesn't exist yet
if [[ -z "${selected_indices[$num]:-}" ]]; then
selection+=("${candidateDatabases[$((num-1))]}")
selected_indices[$num]=1 # Mark as selected
fi
else
echo "Invalid selection: '$num'. Please enter numbers between 1 and ${#candidateDatabases[@]}." >&2
invalid_choice=true
break # Exit inner loop on first invalid number
fi
done
# If no invalid choices were found in the loop, break the outer loop
if [[ "$invalid_choice" == "false" ]]; then
if [[ ${#selection[@]} -gt 0 ]]; then
echo "Selected databases for deletion:"
printf " - %s\n" "${selection[@]}"
break # Exit while loop
else
echo "No valid numbers entered. Please try again." >&2
fi
fi
else
echo "Invalid input. Please enter space-separated numbers, 'all', or 'n'." >&2
fi
done
if [[ ${#selection[@]} -eq 0 ]]; then
echo "No databases selected for deletion."
echo "############################################ End Database Deletion ###############################################"
return 0
fi
# FINAL CONFIRMATION for selected databases
echo
echo "!!!!!!!!!!!!!!!!!!!!!!!! FINAL WARNING !!!!!!!!!!!!!!!!!!!!!!!!!" >&2
echo "The following ${#selection[@]} databases WILL BE PERMANENTLY DELETED:" >&2
printf " -> %s\n" "${selection[@]}" >&2
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" >&2
read -p "Type EXACTLY 'YES' (all uppercase) to confirm deletion: " confirmation
if [[ "$confirmation" != "YES" ]]; then
echo "Confirmation not received. No databases will be deleted."
echo "############################################ End Database Deletion ###############################################"
return 1
fi
# Proceed with deletion
echo "Confirmation received. Proceeding with database deletion..."
local errors_occurred=0
for db in "${selection[@]}"; do
echo "$(date): Attempting to drop database: '$db'"
# Use backticks for safety with special characters in DB names
local drop_sql="DROP DATABASE \`$db\`;"
if mysql $MYSQL_OPTS -e "$drop_sql"; then
echo "$(date): Successfully dropped database: '$db'"
else
# Log error but continue trying to drop others
echo "$(date): ERROR: Failed to drop database: '$db'. Check permissions or if it's in use." >&2
errors_occurred=1
fi
done
echo "$(date): Database deletion process finished."
if [[ $errors_occurred -ne 0 ]]; then
echo "WARNING: One or more errors occurred during deletion. Please review logs." >&2
echo "############################################ End Database Deletion (with errors) #####################################"
return 1
fi
echo "############################################ End Database Deletion ###############################################"
return 0
}
# Function to import databases (decompress directly via pipe)
perform_import() {
echo "########################################### Start Database Import (Piped Decompression) ##############################################"
local STARTDATE
STARTDATE=$(date)
echo "$STARTDATE: Starting database import process."
echo "Searching for files matching '*${EXT}' in directory: $IMPORTPATH"
# Check import directory
if [[ ! -d "$IMPORTPATH" ]]; then
echo "ERROR: Import directory '$IMPORTPATH' not found." >&2
echo "############################################ End Database Import (Error) ##############################################"
return 1
fi
# Find backup files using find and store in an array (mapfile is safer for special chars)
local -a backup_files
# *** CORRECTED ERROR HANDLING FOR mapfile + process substitution ***
local mapfile_exit_code
local find_exit_code
# Use null delimiter for extreme safety with filenames containing newlines (rare but possible)
mapfile -d '' -t backup_files < <(find "$IMPORTPATH" -maxdepth 1 -name "*${EXT}" -print0)
mapfile_exit_code=$? # Capture mapfile's exit status IMMEDIATELY
find_exit_code=${PIPESTATUS[0]} # Capture find's exit status from process substitution IMMEDIATELY
# Check if either mapfile or find failed
# Note: find_exit_code might be non-zero if find fails (e.g., bad permissions on IMPORTPATH)
# Note: mapfile_exit_code might be non-zero if mapfile itself has an issue (less common)
if [[ $mapfile_exit_code -ne 0 || $find_exit_code -ne 0 ]]; then
echo "ERROR: Failed to list files in '$IMPORTPATH'. Check permissions or find command (find exit: $find_exit_code, mapfile exit: $mapfile_exit_code)." >&2
echo "############################################ End Database Import (Error) ##############################################"
return 1
fi
# *** END CORRECTION ***
if [[ ${#backup_files[@]} -eq 0 ]]; then
echo "No backup files found matching '*${EXT}' in '$IMPORTPATH'."
echo "############################################ End Database Import ###############################################"
return 0
fi
# --- Interactive Selection ---
echo
echo "Select backup files to import:"
local i=1
for file in "${backup_files[@]}"; do
# Display only the basename for clarity
echo " $i) $(basename "$file")"
((i++))
done
echo " all) Import ALL listed files"
echo " n) No import (cancel)"
local -a files_to_process
local -A selected_indices # Use associative array for easy lookup
local choice
while true; do
read -p "Enter numbers (space-separated), 'all', or 'n': " choice
choice=$(echo "$choice" | tr '[:upper:]' '[:lower:]') # Lowercase for easier matching
if [[ "$choice" == "n" ]]; then
echo "Import cancelled by user."
echo "############################################ End Database Import ###############################################"
return 0
elif [[ "$choice" == "all" ]]; then
files_to_process=("${backup_files[@]}")
echo "Selected ALL backup files for import."
break
# Use POSIX character classes for broader compatibility with =~
elif [[ "$choice" =~ ^[[:digit:][:space:]]+$ ]]; then
# Validate and collect selected numbers
files_to_process=()
local invalid_choice=false
for num in $choice; do
# Ensure it's a valid number within the range
if [[ "$num" -ge 1 && "$num" -le ${#backup_files[@]} ]]; then
# Avoid duplicates if user enters same number twice
# Use :- default value construct for safety with set -u if key doesn't exist yet
if [[ -z "${selected_indices[$num]:-}" ]]; then
# Store the *full path* from the original array
files_to_process+=("${backup_files[$((num-1))]}")
selected_indices[$num]=1 # Mark as selected
fi
else
echo "Invalid selection: '$num'. Please enter numbers between 1 and ${#backup_files[@]}." >&2
invalid_choice=true
break # Exit inner loop on first invalid number
fi
done
# If no invalid choices were found in the loop, break the outer loop
if [[ "$invalid_choice" == "false" ]]; then
if [[ ${#files_to_process[@]} -gt 0 ]]; then
echo "Selected files for import:"
for file in "${files_to_process[@]}"; do
echo " - $(basename "$file")" # Display only basename
done
break # Exit while loop
else
echo "No valid numbers entered. Please try again." >&2
fi
fi
else
echo "Invalid input. Please enter space-separated numbers, 'all', or 'n'." >&2
fi
done
if [[ ${#files_to_process[@]} -eq 0 ]]; then
echo "No files selected for import."
echo "############################################ End Database Import ###############################################"
return 0
fi
echo
echo "Starting processing of selected files..."
local errors_occurred=0
# Process only the selected files
for filepath in "${files_to_process[@]}"; do
local filename
filename=$(basename "$filepath")
echo "-------------------------------------------------"
echo "$(date): Processing file: $filename"
# Extract database name: filename minus the extension
# Assumes filenames like: databasename.sql.gz or databasename_timestamp.sql.gz
# It will extract 'databasename' or 'databasename_timestamp' respectively. Adjust if needed.
local dbname
# Use bash parameter expansion to remove the configured extension from the end
dbname="${filename%$EXT}"
if [[ -z "$dbname" ]]; then
echo "$(date): ERROR: Could not extract database name from '$filename' (after removing '$EXT'). Skipping." >&2
errors_occurred=1
continue
fi
# Optional: Sanitize dbname further if needed (e.g., remove other patterns)
# dbname=$(echo "$dbname" | sed 's/_timestamp_pattern//') # Example
echo "$(date): Target database name: '$dbname'"
# Check if database exists / Create database
# Use USE command to check existence, safer than SHOW DATABASES | grep
# Suppress stderr for the check, handle failure explicitly
if mysql $MYSQL_OPTS -e "USE \`$dbname\`;" 2>/dev/null; then
echo "$(date): Database '$dbname' already exists. Importing into existing database."
# Add prompt here if overwrite confirmation is desired
# read -p "Database '$dbname' exists. Overwrite? (y/N): " overwrite_confirm
# if [[ "${overwrite_confirm,,}" != "y" ]]; then
# echo "$(date): Skipping import for '$dbname'."
# continue
# fi
else
# Check if failure was due to non-existence (exit code 1 usually) vs other connection error
local mysql_check_exit_code=$?
# Crude check, might vary slightly across mysql versions
if [[ $mysql_check_exit_code -eq 1 ]]; then
echo "$(date): Database '$dbname' does not exist. Creating..."
local create_sql="CREATE DATABASE \`$dbname\`;"
if mysql $MYSQL_OPTS -e "$create_sql"; then
echo "$(date): Database '$dbname' created successfully."
else
echo "$(date): ERROR: Failed to create database '$dbname'. Check MySQL user permissions. Skipping import for this file." >&2
errors_occurred=1
continue # Skip to the next file
fi
else
echo "$(date): ERROR: Failed to check status of database '$dbname' (MySQL exit code: $mysql_check_exit_code). Check connection/permissions. Skipping import." >&2
errors_occurred=1
continue # Skip to next file
fi
fi
# Import using a direct pipe (MUCH FASTER)
echo "$(date): Decompressing and importing '$filename' into database '$dbname' via pipe..."
if gunzip -c "$filepath" | mysql $MYSQL_OPTS "$dbname"; then
# Pipe succeeded (both gunzip and mysql returned 0)
echo "$(date): Successfully imported database '$dbname'."
else
# Pipe failed. Check PIPESTATUS to see which command failed.
# PIPESTATUS is an array holding exit status of commands in the last pipe.
# Requires bash. Index 0 is gunzip, Index 1 is mysql.
local pipe_status=("${PIPESTATUS[@]}")
local gunzip_exit_code=${pipe_status[0]}
local mysql_exit_code=${pipe_status[1]}
echo "$(date): ERROR: Import failed for database '$dbname'." >&2
if [[ $gunzip_exit_code -ne 0 ]]; then
echo " -> Decompression (gunzip) failed with exit code: $gunzip_exit_code." >&2
fi
if [[ $mysql_exit_code -ne 0 ]]; then
echo " -> Database import (mysql) failed with exit code: $mysql_exit_code." >&2
echo " -> Check MySQL logs and ensure user '$DBUSER' has necessary privileges on '$dbname'." >&2
fi
errors_occurred=1
# Optional: Decide whether to continue with next file or stop on error
# continue # Continue processing next file
# break # Stop processing files on first error
fi
done
# Final message for import
echo "-------------------------------------------------"
echo "$(date): Database import process finished."
echo "Started at: $STARTDATE"
if [[ $errors_occurred -ne 0 ]]; then
echo "WARNING: One or more errors occurred during import. Please review logs above." >&2
echo "############################################ End Database Import (with errors) ##############################################"
return 1
fi
echo "############################################ End Database Import ###############################################"
return 0
}
# --- Main Script Execution ---
echo "=================================================="
echo " MySQL Database Management Script"
echo "=================================================="
echo
# Check prerequisites (basic check)
for cmd in mysql gunzip find grep sed date basename mapfile tr; do
if ! command -v "$cmd" &> /dev/null; then
# Check specifically for mapfile if tr exists (as mapfile is Bash builtin >= 4)
if [[ "$cmd" == "mapfile" && $(command -v tr) ]]; then
bash_major_version=$(bash --version | head -n1 | sed 's/.*version \([0-9]*\)\..*/\1/')
# Default to 0 if sed fails to extract version
if [[ ${bash_major_version:-0} -lt 4 ]]; then
error_exit "Required command '$cmd' (Bash builtin) requires Bash version 4+. You have Bash $bash_major_version."
else
# If mapfile exists but command -v fails, something weird is happening, but proceed cautiously.
# Or it might indicate a non-Bash shell identified as bash.
echo "WARN: 'command -v mapfile' failed, but Bash version seems >= 4. Proceeding..." >&2
fi
else
error_exit "Required command '$cmd' not found. Please install it or add it to your PATH."
fi
fi
done
# Setup and Test Connection FIRST
if ! setup_mysql_connection; then
# Error message already printed by the function
exit 1
fi
echo
# Ask user for action
echo "Please choose an action:"
echo " 1) DELETE selected Databases (USE WITH EXTREME CAUTION!)"
echo " 2) IMPORT selected/all Databases from '$IMPORTPATH' (*${EXT})"
echo " *) Any other key to Exit"
echo
read -p "Enter your choice (1 or 2): " choice
echo # Newline for readability
# Execute chosen action
last_exit_code=0
case "$choice" in
1)
perform_delete
last_exit_code=$?
;;
2)
perform_import
last_exit_code=$?
;;
*)
echo "Invalid choice or exiting."
last_exit_code=0
;;
esac
echo
echo "Script finished."
exit $last_exit_code
@BeshoyAtef
Copy link
Author

Updated to include also the reimporting if needed

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment