完整部署Hadoop三大核心组件

hadoop_deploy_full.sh

#!/bin/bash

#####################################################
# Hadoop 伪分布式集群一键部署脚本 - CentOS 7
# 功能：
#   1. 检查并安装必要依赖 (wget, curl, tar, ssh)
#   2. 获取本机IP (优先内网) 并配置 /etc/hosts
#   3. 设置主机名为 'master'
#   4. 下载、解压并重命名 Java 和 Hadoop 安装包
#   5. 配置 Java 和 Hadoop 环境变量到 /etc/profile
#   6. 自动配置 SSH 免密登录 (本机到本机)
#   7. 配置核心文件 core-site.xml
#   8. 配置HDFS相关文件 hdfs-site.xml
#   9. 配置YARN相关文件 mapred-site.xml, yarn-site.xml
#  10. 修改环境变量文件 hadoop-env.sh
#  11. 格式化NameNode并启动Hadoop集群
#  12. 验证服务状态并提供Web界面信息
# 作者：kangvcar@gmail.com
#####################################################

# 定义颜色
RED='\033[0;31m'
GREEN='\033[0;32m'
LIGHT_GREEN='\033[1;32m'
YELLOW='\033[0;33m'
LIGHT_YELLOW='\033[1;33m'
BLUE='\033[0;34m'
LIGHT_BLUE='\033[1;34m'
PURPLE='\033[0;35m'
LIGHT_PURPLE='\033[1;35m'
CYAN='\033[0;36m'
LIGHT_CYAN='\033[1;36m'
WHITE='\033[1;37m'
BOLD='\033[1m'
BLINK='\033[5m'
NC='\033[0m' # 无颜色
RESET='\033[0m'

# 边框字符定义（仅保留水平线）
BORDER_H="=" # 水平线

# 框架宽度常量
FRAME_WIDTH=70

# 初始化
SCRIPT_NAME=$(basename "$0")
CURRENT_USER=$(whoami)
LOG_FILE="/tmp/hadoop_deploy_$(date +%Y%m%d%H%M%S).log"
PROGRESS=0
TOTAL_STEPS=12
JAVA_HOME="/opt/java8"
HADOOP_HOME="/opt/hadoop"

# 保存当前目录
CURRENT_DIR=$(pwd)

# 初始化日志文件
touch "$LOG_FILE"

# 进度条函数
show_progress() {
    local percent=$1
    local description=$2
    local width=50
    local num_chars=$(($width * $percent / 100))
    local progress_bar=""
    
    # 使用更醒目的字符表示进度
    for ((i=0; i<$num_chars; i++)); do
        progress_bar+="="
    done
    
    for ((i=$num_chars; i<$width; i++)); do
        progress_bar+=" "
    done
    
    echo -e "\r${LIGHT_CYAN}[${WHITE} $percent% ${LIGHT_CYAN}]${RESET} ${LIGHT_CYAN}|${LIGHT_GREEN}$progress_bar${LIGHT_CYAN}| ${WHITE}$description${RESET}"
}

# 更新进度
update_progress() {
    local step_number=$1
    local description=$2
    
    # 计算准确的百分比 - 只基于主步骤数量
    PROGRESS=$(( (step_number * 100) / TOTAL_STEPS ))
    
    if [ $PROGRESS -gt 100 ]; then
        PROGRESS=100
    fi
    
    show_progress $PROGRESS "$description"
}

# 记录日志
log() {
    local level=$1
    local message=$2
    local timestamp=$(date "+%Y-%m-%d %H:%M:%S")
    
    case $level in
        "INFO") level_color=$CYAN ;;
        "SUCCESS") level_color=$GREEN ;;
        "WARNING") level_color=$YELLOW ;;
        "ERROR") level_color=$RED ;;
        *) level_color=$NC ;;
    esac
    
    echo -e "${level_color}[$level]${NC} $message"
    echo "[$timestamp] [$level] $message" >> "$LOG_FILE"
}

# 计算字符串的可见长度（去除ANSI颜色代码）
string_visible_length() {
    local string="$1"
    # 移除所有ANSI颜色代码
    local stripped=$(echo -e "$string" | sed 's/\x1B\[[0-9;]*[a-zA-Z]//g')
    echo ${#stripped}
}

# 创建填充空格
create_padding() {
    local length=$1
    printf '%*s' "$length" ''
}

# 创建带内容的对齐行
create_aligned_line() {
    local prefix="$1"  # 左边框
    local content="$2" # 内容
    local suffix="$3"  # 右边框
    
    local visible_length=$(string_visible_length "$content")
    local inner_width=$((FRAME_WIDTH - 2)) # 边框内部宽度
    local padding=$((inner_width - visible_length))
    
    echo "${prefix}${content}$(create_padding $padding)${suffix}"
}

# 打印带边框的标题 (修改为仅上下边框)
print_title() {
    local text="$1"
    local visible_length=$(string_visible_length "${WHITE}${BOLD}$text${RESET}")
    local padding=$(( (FRAME_WIDTH - visible_length) / 2 ))
    local line="${LIGHT_BLUE}$(printf '%*s' "$FRAME_WIDTH" '' | tr ' ' "${BORDER_H}")${RESET}"
    
    echo -e "\n$line"
    echo -e "$(create_padding $padding)${WHITE}${BOLD}$text${RESET}"
    echo -e "$line\n"
}

# 打印脚本功能摘要 (修改为仅上下边框)
print_script_summary() {
    local line="${CYAN}$(printf '%*s' "$FRAME_WIDTH" '' | tr ' ' "${BORDER_H}")${RESET}"
    local header="脚本主要功能"
    local header_len=$(string_visible_length "$header")
    local header_pad=$(( (FRAME_WIDTH - header_len) / 2))
    
    echo -e "$line"
    echo -e "$(create_padding $header_pad)${BOLD}$header${RESET}"
    echo -e "$line"
    
    local features=(
        " 1. 检查并安装必要依赖 (wget, curl, tar, ssh)"
        " 2. 获取本机IP (优先内网) 并配置 /etc/hosts"
        " 3. 设置主机名为 'master'"
        " 4. 下载、解压并重命名 Java 和 Hadoop 安装包"
        " 5. 配置 Java 和 Hadoop 环境变量到 /etc/profile"
        " 6. 自动配置 SSH 免密登录 (本机到本机)"
        " 7. 配置核心文件 core-site.xml"
        " 8. 配置HDFS相关文件 hdfs-site.xml"
        " 9. 配置YARN相关文件 mapred-site.xml, yarn-site.xml"
        "10. 修改环境变量文件 hadoop-env.sh"
        "11. 格式化NameNode并启动Hadoop集群"
        "12. 验证服务状态并提供Web界面信息"
    )
    
    for feature in "${features[@]}"; do
        echo -e " ${WHITE}${feature}${RESET}" # 左对齐输出
    done
    
    echo -e "$line\n"
}

# 打印带边框的完成消息 (修改为仅上下边框)
print_completion() {
    local text="${WHITE}${BOLD} Hadoop 伪分布式集群部署完成！${RESET}"
    local visible_length=$(string_visible_length "$text")
    local padding=$(( (FRAME_WIDTH - visible_length) / 2 ))
    local line="${LIGHT_GREEN}$(printf '%*s' "$FRAME_WIDTH" '' | tr ' ' "${BORDER_H}")${RESET}"
    
    echo -e "\n$line"
    echo -e "$(create_padding $padding)$text"
    echo -e "$line\n"
}

# 打印成功消息
print_success() {
    log "SUCCESS" "$1"
}

# 打印信息消息
print_info() {
    log "INFO" "$1"
}

# 打印警告消息
print_warning() {
    log "WARNING" "$1"
}

# 打印错误消息
print_error() {
    log "ERROR" "$1"
}

# 打印分隔线
print_separator() {
    echo -e "${LIGHT_BLUE}$(printf '%*s' "$FRAME_WIDTH" '' | tr ' ' "${BORDER_H}")${NC}" # 使用 BORDER_H (=)
}

# 打印步骤标题
print_step() {
    local step_num=$1
    local total=$2
    local description=$3
    
    print_separator
    echo -e "${LIGHT_YELLOW}[ 步骤 $step_num/$total ]${WHITE}${BOLD} $description${RESET}"
    print_separator
}

# 检查命令是否成功执行
check_success() {
    if [ $? -eq 0 ]; then
        print_success "$1"
        return 0
    else
        print_error "$2"
        return 1
    fi
}

# 检查命令是否存在
command_exists() {
    command -v "$1" >/dev/null 2>&1
}

# 检查并安装必要的包
install_required_packages() {
    print_info "检查并安装必要的软件包..."
    
    local packages=("wget" "curl" "tar" "openssh-server" "openssh-clients")
    local to_install=()
    
    for pkg in "${packages[@]}"; do
        if ! command_exists "$pkg"; then
            to_install+=("$pkg")
        fi
    done
    
    if [ ${#to_install[@]} -ne 0 ]; then
        print_info "正在安装缺失的软件包: ${to_install[*]}"
        yum install -y "${to_install[@]}" >/dev/null 2>&1
        check_success "软件包安装成功" "软件包安装失败" || exit 1
    else
        print_info "所有必要的软件包已安装"
    fi
}

# 检查磁盘空间
check_disk_space() {
    print_info "检查可用磁盘空间..."
    local required_space=3000000  # 约3GB，单位KB
    local available_space=$(df -k /opt | tail -1 | awk '{print $4}')
    
    if [ "$available_space" -lt "$required_space" ]; then
        print_error "磁盘空间不足！需要至少3GB空间，当前可用: $(($available_space/1024))MB"
        return 1
    else
        print_success "磁盘空间充足: $(($available_space/1024))MB"
        return 0
    fi
}

# 备份文件
backup_file() {
    local file="$1"
    if [ -f "$file" ]; then
        local backup="${file}.bak.$(date +%Y%m%d%H%M%S)"
        cp "$file" "$backup"
        print_info "已备份文件 $file 到 $backup"
    fi
}

# 添加到文件末尾（避免重复）
append_to_file() {
    local file="$1"
    local content="$2"
    local pattern="$3"
    
    backup_file "$file"
    
    if ! grep -q "$pattern" "$file"; then
        echo "$content" >> "$file"
        print_success "内容已添加到 $file"
    else
        print_info "$file 中已存在相似内容，跳过"
    fi
}

# 使环境变量生效的函数
apply_env() {
    print_info "正在使环境变量生效..."
    
    # 将环境变量添加到当前会话
    if [ -n "$JAVA_HOME" ]; then
        export JAVA_HOME
        export PATH=$PATH:$JAVA_HOME/bin
    fi
    
    if [ -n "$HADOOP_HOME" ]; then
        export HADOOP_HOME
        export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
    fi
    
    # 对当前shell生效
    source /etc/profile
    print_success "环境变量已在当前会话中生效"
    
    # 验证环境变量是否生效
    if [ -n "$JAVA_HOME" ] && [ -z "$(echo $PATH | grep -o "$JAVA_HOME/bin")" ]; then
        print_warning "Java环境变量在PATH中不可见，手动添加到当前会话"
        export PATH=$PATH:$JAVA_HOME/bin
    fi
    
    if [ -n "$HADOOP_HOME" ] && [ -z "$(echo $PATH | grep -o "$HADOOP_HOME/bin")" ]; then
        print_warning "Hadoop环境变量在PATH中不可见，手动添加到当前会话"
        export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
    fi
}

# 自动SSH免密登录设置
setup_passwordless_ssh() {
    print_info "设置SSH免密登录..."
    
    # 确保SSH服务正在运行
    systemctl is-active --quiet sshd
    if [ $? -ne 0 ]; then
        print_info "正在启动SSH服务..."
        systemctl start sshd
        systemctl enable sshd
    fi
    
    # 创建SSH密钥（如果不存在）
    if [ ! -f ~/.ssh/id_rsa ]; then
        print_info "生成SSH密钥对..."
        mkdir -p ~/.ssh
        ssh-keygen -t rsa -P "" -f ~/.ssh/id_rsa >/dev/null 2>&1
        chmod 700 ~/.ssh
        check_success "SSH密钥生成成功" "SSH密钥生成失败" || return 1
    else
        print_info "SSH密钥已存在，跳过生成"
    fi
    
    # 确保known_hosts存在
    touch ~/.ssh/known_hosts
    
    # 自动接受主机密钥并配置免密登录
    print_info "配置本地免密登录..."
    
    # 移除可能存在的旧主机密钥
    ssh-keygen -R master >/dev/null 2>&1
    ssh-keygen -R localhost >/dev/null 2>&1
    ssh-keygen -R 127.0.0.1 >/dev/null 2>&1
    ssh-keygen -R ${HOST_IP} >/dev/null 2>&1
    
    # 更新known_hosts（自动接受主机密钥）
    ssh-keyscan -H master localhost 127.0.0.1 ${HOST_IP} 2>/dev/null >> ~/.ssh/known_hosts
    
    # 配置authorized_keys
    mkdir -p ~/.ssh
    touch ~/.ssh/authorized_keys
    chmod 600 ~/.ssh/authorized_keys
    
    if ! grep -q "$(cat ~/.ssh/id_rsa.pub)" ~/.ssh/authorized_keys; then
        cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
        print_success "已将公钥添加到authorized_keys"
    else
        print_info "公钥已存在于authorized_keys中"
    fi
    
    # 测试SSH免密登录
    print_info "测试SSH免密登录..."
    ssh -o BatchMode=yes -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@master exit
    if check_success "SSH免密登录测试成功" "SSH免密登录测试失败"; then
        return 0
    else
        # 如果失败，尝试另一种方式
        print_info "尝试替代方法设置SSH免密登录..."
        cat ~/.ssh/id_rsa.pub | ssh -o StrictHostKeyChecking=no root@master "mkdir -p ~/.ssh && cat >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys"
        
        # 再次测试
        ssh -o BatchMode=yes -o ConnectTimeout=5 root@master exit
        check_success "SSH免密登录重试成功" "SSH免密登录仍然失败，脚本将继续，但后续步骤可能出错" || return 1
    fi
}

# 配置环境变量（避免重复）
configure_env_var() {
    local name="$1"
    local value="$2"
    local pattern="export $name="
    local content="export $name=$value"
    
    if grep -q "$pattern" /etc/profile; then
        print_info "$name 环境变量已存在"
        # 更新已存在的环境变量
        sed -i "s|$pattern.*|$content|" /etc/profile
        print_success "$name 环境变量已更新"
    else
        echo "$content" >> /etc/profile
        print_success "$name 环境变量已添加"
    fi
    
    # 在当前Shell中立即设置环境变量
    eval "export $name=$value"
}

# 清理临时文件
cleanup() {
    print_info "清理临时文件..."
    cd "$CURRENT_DIR"
}

# 错误处理
handle_error() {
    print_error "脚本执行过程中出错，请查看日志文件: $LOG_FILE"
    cleanup
    exit 1
}

# 优雅退出
graceful_exit() {
    print_warning "收到中断信号，正在清理..."
    cleanup
    exit 130
}

# 创建XML配置文件
create_xml_config() {
    local file="$1"
    local configs="$2"
    
    # 备份原始文件
    backup_file "$file"
    
    # 创建新的XML文件
    cat > "$file" << EOF
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
$configs
</configuration>
EOF

    check_success "成功创建配置文件: $file" "创建配置文件失败: $file" 
}

# 获取本机IP
get_host_ip() {
    local ip=$(hostname -I | awk '{print $1}')
    if [ -z "$ip" ]; then
        print_warning "无法通过hostname -I获取IP，尝试其他方法..."
        ip=$(curl -s --connect-timeout 5 ip.me)
        if [ -z "$ip" ]; then
            print_error "无法获取本机IP地址"
            return 1
        fi
    fi
    echo "$ip"
}


# 注册信号处理
trap graceful_exit INT TERM
trap handle_error ERR

# 主函数
main() {
    # 确保脚本以root权限运行
    if [ "$(id -u)" != "0" ]; then
        print_error "此脚本必须以root权限运行"
        exit 1
    fi
    
    # 开始部署过程
    print_title "Hadoop 伪分布式集群一键部署脚本"
    print_script_summary
    echo -e "${LIGHT_CYAN}开始时间: $(date '+%Y-%m-%d %H:%M:%S')${RESET}"
    echo -e "${LIGHT_CYAN}日志文件: $LOG_FILE${RESET}\n"
    
    # 初始化进度为0
    PROGRESS=0
    # 显示初始进度
    show_progress 0 "开始部署"
    
    ###########################################
    # 第一部分: 系统环境准备和基础依赖安装
    ###########################################
    
    # 预检查
    print_step 1 $TOTAL_STEPS "系统预检查"
    check_disk_space || exit 1
    install_required_packages
    update_progress 1 "系统预检查完成"
    
    # 配置主机名和IP
    print_step 2 $TOTAL_STEPS "环境准备"
    print_info "正在获取本机主要IP地址(优先内网)..."
    HOST_IP=$(get_host_ip)
    if [ $? -ne 0 ]; then
        print_error "无法获取主机IP地址，退出脚本"
        exit 1
    fi
    check_success "成功获取本机IP: ${HOST_IP}" "获取IP失败" || exit 1
    
    # 修改主机名
    print_info "正在修改主机名为master..."
    if [ "$(hostname)" != "master" ]; then
        hostnamectl set-hostname master
        check_success "成功将主机名设置为master" "设置主机名失败" || exit 1
    else
        print_info "主机名已经是master，跳过设置"
    fi
    
    # 配置hosts文件
    print_info "正在配置IP与主机名映射关系..."
    backup_file "/etc/hosts"
    
    # 先删除可能存在的旧 master 映射
    if grep -q "[[:space:]]master[[:space:]]*$" /etc/hosts; then
        sed -i "/[[:space:]]master[[:space:]]*$/d" /etc/hosts
        print_info "已移除旧的 master 映射"
    fi
    
    # 检查 IP 是否已存在于其他映射中，如果存在则移除旧映射
    if grep -q "^${HOST_IP}[[:space:]]" /etc/hosts; then
         sed -i "/^${HOST_IP}[[:space:]]/d" /etc/hosts
         print_info "已移除旧的 ${HOST_IP} 映射"
    fi
    
    # 添加新的映射
    echo "${HOST_IP} master" >> /etc/hosts
    print_info "已添加新的映射: ${HOST_IP} master"
    
    check_success "成功配置IP与主机名映射关系" "配置IP与主机名映射关系失败" || exit 1
    
    # 创建软件包存放目录
    print_info "正在创建软件包存放目录..."
    mkdir -p /opt/software
    check_success "成功创建目录：/opt/software" "创建目录失败" || exit 1
    update_progress 2 "环境准备完成"
    
    ###########################################
    # 第二部分: 安装和配置Java
    ###########################################
    
    # 下载Java
    print_step 3 $TOTAL_STEPS "安装配置 Java 环境"
    cd /opt/software || exit 1
    
    print_info "正在检查Java软件包..."
    if [ ! -f "openjdk-8u41-b04-linux-x64-14_jan_2020.tar.gz" ]; then
        print_info "正在下载Java软件包，这可能需要一些时间..."
        wget --no-check-certificate --progress=dot:giga https://download.java.net/openjdk/jdk8u41/ri/openjdk-8u41-b04-linux-x64-14_jan_2020.tar.gz 2>&1 | grep --line-buffered -o "[0-9]*%" | awk '{printf "\r下载进度: %s", $1}'
        echo ""
        check_success "Java软件包下载成功" "Java软件包下载失败" || exit 1
    else
        print_info "Java软件包已存在，跳过下载"
    fi
    
    # 解压Java
    print_info "正在解压Java软件包到/opt目录..."
    if [ ! -d "/opt/java-se-8u41-ri" ]; then
        tar -zxf openjdk-8u41-b04-linux-x64-14_jan_2020.tar.gz -C /opt/
        check_success "Java软件包解压成功" "Java软件包解压失败" || exit 1
    else
        print_info "Java目录已存在，跳过解压"
    fi
    
    # 重命名Java目录
    print_info "正在检查Java目录..."
    if [ ! -d "/opt/java8" ]; then
        if [ -d "/opt/java-se-8u41-ri" ]; then
            print_info "正在重命名Java目录..."
            mv /opt/java-se-8u41-ri /opt/java8
            check_success "Java目录重命名成功" "Java目录重命名失败" || exit 1
        else
            print_error "Java解压目录不存在！"
            exit 1
        fi
    else
        print_info "Java8目录已存在，无需重命名"
    fi
    
    # 配置Java环境变量
    print_info "正在配置Java环境变量..."
    backup_file "/etc/profile"
    
    # 配置JAVA_HOME环境变量
    configure_env_var "JAVA_HOME" "/opt/java8"
    
    # 添加PATH变量（如果不存在）
    if ! grep -q "PATH=.*JAVA_HOME" /etc/profile; then
        append_to_file "/etc/profile" 'export PATH=$PATH:$JAVA_HOME/bin' 'PATH=.*JAVA_HOME'
        check_success "PATH环境变量配置成功" "PATH环境变量配置失败" || exit 1
    fi
    
    # 立即应用环境变量
    apply_env
    
    # 验证Java环境
    print_info "正在验证Java环境..."
    java -version >/dev/null 2>&1
    check_success "Java环境验证成功" "Java环境验证失败，请检查安装" || exit 1
    JAVA_VERSION=$(java -version 2>&1 | head -n 1)
    print_info "检测到Java版本: $JAVA_VERSION"
    update_progress 3 "Java环境配置完成"
    
    ###########################################
    # 第三部分: 配置SSH免密登录
    ###########################################
    
    # 配置SSH免密登录
    print_step 4 $TOTAL_STEPS "配置 SSH 免密登录"
    setup_passwordless_ssh
    update_progress 4 "SSH免密登录配置完成"
    
    ###########################################
    # 第四部分: 安装和配置Hadoop
    ###########################################
    
    # 下载Hadoop
    print_step 5 $TOTAL_STEPS "安装 Hadoop 环境"
    cd /opt/software || exit 1
    
    print_info "正在检查Hadoop软件包..."
    if [ ! -f "hadoop-3.3.6.tar.gz" ]; then
        print_info "正在下载Hadoop软件包，这可能需要较长时间..."
        curl -# -o hadoop-3.3.6.tar.gz https://mirrors.tuna.tsinghua.edu.cn/apache/hadoop/common/hadoop-3.3.6/hadoop-3.3.6.tar.gz 2>&1 | stdbuf -o0 awk -v RS='\r' '/[0-9]*%/ {printf("\r下载进度: %s", $0)}'
        echo ""
        check_success "Hadoop软件包下载成功" "Hadoop软件包下载失败" || exit 1
    else
        print_info "Hadoop软件包已存在，跳过下载"
    fi
    
    # 解压Hadoop
    print_info "正在解压Hadoop软件包到/opt目录..."
    if [ ! -d "/opt/hadoop-3.3.6" ]; then
        tar -zxf hadoop-3.3.6.tar.gz -C /opt/
        check_success "Hadoop软件包解压成功" "Hadoop软件包解压失败" || exit 1
    else
        print_info "Hadoop解压目录已存在，跳过解压"
    fi
    
    # 重命名Hadoop目录
    print_info "正在检查Hadoop目录..."
    if [ ! -d "/opt/hadoop" ]; then
        if [ -d "/opt/hadoop-3.3.6" ]; then
            print_info "正在重命名Hadoop目录..."
            mv /opt/hadoop-3.3.6 /opt/hadoop
            check_success "Hadoop目录重命名成功" "Hadoop目录重命名失败" || exit 1
        else
            print_error "Hadoop解压目录不存在！"
            exit 1
        fi
    else
        print_info "Hadoop目录已存在，无需重命名"
    fi
    
    # 配置Hadoop环境变量
    print_info "正在配置Hadoop环境变量..."
    
    # 配置HADOOP_HOME环境变量
    configure_env_var "HADOOP_HOME" "/opt/hadoop"
    
    # 添加PATH变量（如果不存在）
    if ! grep -q "PATH=.*HADOOP_HOME/bin" /etc/profile; then
        append_to_file "/etc/profile" 'export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin' 'PATH=.*HADOOP_HOME'
        check_success "Hadoop PATH环境变量配置成功" "Hadoop PATH环境变量配置失败" || exit 1
    fi
    
    # 立即应用环境变量
    apply_env
    
    # 验证Hadoop环境
    print_info "正在验证Hadoop环境..."
    hadoop version >/dev/null 2>&1
    check_success "Hadoop环境验证成功" "Hadoop环境验证失败，请检查安装" || exit 1
    HADOOP_VERSION=$(hadoop version | head -n 1)
    print_info "检测到Hadoop版本: $HADOOP_VERSION"
    update_progress 5 "Hadoop环境安装完成"
    
    ###########################################
    # 第五部分: 配置 core-site.xml
    ###########################################
    
    # 配置core-site.xml
    print_step 6 $TOTAL_STEPS "配置 core-site.xml"
    HADOOP_CONF_DIR="$HADOOP_HOME/etc/hadoop"
    CORE_SITE_XML="$HADOOP_CONF_DIR/core-site.xml"
    print_info "开始配置 core-site.xml..."
    
    # 创建临时目录
    TMP_DIR="$HADOOP_HOME/hdfs/tmp"
    mkdir -p "$TMP_DIR"
    check_success "创建临时目录: $TMP_DIR" "创建临时目录失败"
    
    # 配置core-site.xml
    CORE_CONFIGS="
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://master:8020</value>
    </property>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/opt/hadoop/hdfs/tmp</value>
    </property>
    <property>
        <name>hadoop.proxyuser.root.hosts</name>
        <value>*</value>
    </property>
    <property>
        <name>hadoop.proxyuser.root.groups</name>
        <value>*</value>
    </property>"
    
    create_xml_config "$CORE_SITE_XML" "$CORE_CONFIGS"
    update_progress 6 "core-site.xml 配置完成"
    
    ###########################################
    # 第六部分: 配置 hdfs-site.xml
    ###########################################
    
    # 配置hdfs-site.xml
    print_step 7 $TOTAL_STEPS "配置 hdfs-site.xml"
    HDFS_SITE_XML="$HADOOP_CONF_DIR/hdfs-site.xml"
    print_info "开始配置 hdfs-site.xml..."
    
    # 创建HDFS目录
    mkdir -p "$HADOOP_HOME/hdfs/name"
    mkdir -p "$HADOOP_HOME/hdfs/data"
    check_success "创建HDFS目录" "创建HDFS目录失败"
    
    # 配置hdfs-site.xml
    HDFS_CONFIGS="
    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>
    <property>
        <name>dfs.namenode.name.dir</name>
        <value>/opt/hadoop/hdfs/name</value>
        <final>true</final>
    </property>
    <property>
        <name>dfs.datanode.data.dir</name>
        <value>/opt/hadoop/hdfs/data</value>
        <final>true</final>
    </property>
    <property>
        <name>dfs.namenode.http-address</name>
        <value>0.0.0.0:9870</value>
    </property>
    <property>
        <name>dfs.permissions</name>
        <value>false</value>
    </property>"
    
    create_xml_config "$HDFS_SITE_XML" "$HDFS_CONFIGS"
    update_progress 7 "hdfs-site.xml 配置完成"
    
    ###########################################
    # 第七部分: 配置 YARN 相关文件
    ###########################################
    
    # 配置YARN相关文件
    print_step 8 $TOTAL_STEPS "配置 YARN 相关文件"
    MAPRED_SITE_XML="$HADOOP_CONF_DIR/mapred-site.xml"
    YARN_SITE_XML="$HADOOP_CONF_DIR/yarn-site.xml"
    
    # 配置mapred-site.xml
    print_info "开始配置 mapred-site.xml..."
    MAPRED_CONFIGS="
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
    <property>
        <name>yarn.app.mapreduce.am.env</name>
        <value>HADOOP_MAPRED_HOME=$HADOOP_HOME</value>
    </property>
    <property>
        <name>mapreduce.map.env</name>
        <value>HADOOP_MAPRED_HOME=$HADOOP_HOME</value>
    </property>
    <property>
        <name>mapreduce.reduce.env</name>
        <value>HADOOP_MAPRED_HOME=$HADOOP_HOME</value>
    </property>"
    
    create_xml_config "$MAPRED_SITE_XML" "$MAPRED_CONFIGS"
    check_success "mapred-site.xml 配置完成" "mapred-site.xml 配置失败"
    
    # 配置yarn-site.xml
    print_info "开始配置 yarn-site.xml..."
    YARN_CONFIGS="
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
    <property>
        <name>yarn.resourcemanager.hostname</name>
        <value>master</value>
    </property>"
    
    create_xml_config "$YARN_SITE_XML" "$YARN_CONFIGS"
    check_success "yarn-site.xml 配置完成" "yarn-site.xml 配置失败"
    update_progress 8 "YARN相关文件配置完成"
    
    ###########################################
    # 第八部分: 配置环境变量文件 hadoop-env.sh
    ###########################################
    
    # 配置环境变量文件
    print_step 9 $TOTAL_STEPS "配置环境变量文件 hadoop-env.sh"
    HADOOP_ENV_SH="$HADOOP_CONF_DIR/hadoop-env.sh"
    print_info "开始配置 hadoop-env.sh..."
    
    # 备份原始文件
    backup_file "$HADOOP_ENV_SH"
    
    # 添加环境变量到hadoop-env.sh
    cat >> "$HADOOP_ENV_SH" << EOF
# 添加自定义环境变量 - 由脚本生成于 $(date '+%Y-%m-%d %H:%M:%S')
export JAVA_HOME=$JAVA_HOME
export HADOOP_HOME=$HADOOP_HOME
export HDFS_NAMENODE_USER=root
export HDFS_DATANODE_USER=root
export HDFS_SECONDARYNAMENODE_USER=root
export YARN_RESOURCEMANAGER_USER=root
export YARN_NODEMANAGER_USER=root
EOF
    
    check_success "hadoop-env.sh 配置完成" "hadoop-env.sh 配置失败"
    update_progress 9 "环境变量文件配置完成"
    
    ###########################################
    # 第九部分: 格式化NameNode
    ###########################################
    
    # 格式化NameNode
    print_step 10 $TOTAL_STEPS "格式化NameNode"
    print_info "正在格式化NameNode..."
    
    # 加载最新的环境变量
    source /etc/profile
    
    # 格式化NameNode
    hdfs namenode -format > /dev/null
    check_success "NameNode格式化成功" "NameNode格式化失败" || exit 1
    update_progress 10 "NameNode格式化完成"
    
    ###########################################
    # 第十部分: 启动Hadoop集群
    ###########################################
    
    # 启动Hadoop集群
    print_step 11 $TOTAL_STEPS "启动Hadoop集群"
    
    print_info "启动HDFS服务..."
    $HADOOP_HOME/sbin/start-dfs.sh
    if [ $? -eq 0 ]; then
        print_success "HDFS服务启动成功"
    else
        print_error "HDFS服务启动失败"
        exit 1
    fi
    
    print_info "启动YARN服务..."
    $HADOOP_HOME/sbin/start-yarn.sh
    if [ $? -eq 0 ]; then
        print_success "YARN服务启动成功"
    else
        print_error "YARN服务启动失败"
        exit 1
    fi
    
    update_progress 11 "Hadoop集群启动完成"
    
    ###########################################
    # 第十一部分: 验证服务状态
    ###########################################
    
    # 验证服务状态
    print_step 12 $TOTAL_STEPS "验证服务状态"
    print_info "检查Hadoop进程状态..."
    jps_output=$(jps)
    echo "$jps_output"
    
    expected_processes=("NameNode" "DataNode" "SecondaryNameNode" "ResourceManager" "NodeManager")
    missing_processes=()
    
    for process in "${expected_processes[@]}"; do
        if ! echo "$jps_output" | grep -q "$process"; then
            missing_processes+=("$process")
        fi
    done
    
    if [ ${#missing_processes[@]} -eq 0 ]; then
        print_success "所有Hadoop进程都在运行"
    else
        print_warning "以下进程未运行: ${missing_processes[*]}"
    fi
    
    update_progress 12 "服务状态验证完成"
    
    # 部署完成
    print_completion
    
    # 显示访问信息
    local info_line="${LIGHT_PURPLE}$(printf '%*s' "$FRAME_WIDTH" '' | tr ' ' "${BORDER_H}")${RESET}"
    echo -e "$info_line"
    local header_text="${BOLD}Hadoop 集群访问信息${RESET}"
    local header_visible_length=$(string_visible_length "$header_text")
    local header_padding=$(( (FRAME_WIDTH - header_visible_length) / 2 ))
    echo -e "$(create_padding $header_padding)$header_text"
    echo -e "$info_line"
    
    # 打印信息行
    print_info_row() {
        local label="$1"
        local value="$2"
        echo -e " ${WHITE}$label${GREEN}$value${RESET}"
    }
    
    print_info_row "HDFS NameNode Web界面:    " "http://${HOST_IP}:9870/"
    print_info_row "YARN资源管理界面:   " "http://${HOST_IP}:8088/"
    print_info_row "主机名:             " "$(hostname)"
    print_info_row "IP地址:             " "${HOST_IP}"
    print_info_row "HDFS地址:           " "hdfs://master:8020"
    print_info_row "Hadoop版本:         " "${HADOOP_VERSION}"
    print_info_row "Java版本:           " "${JAVA_VERSION}"
    print_info_row "日志文件:           " "${LOG_FILE}"
    
    echo -e "$info_line\n"
    
    # 添加使用示例
    local example_line="${LIGHT_YELLOW}$(printf '%*s' "$FRAME_WIDTH" '' | tr ' ' "${BORDER_H}")${RESET}"
    echo -e "$example_line"
    local example_title="${BOLD}Hadoop 使用示例${RESET}"
    local example_title_visible_length=$(string_visible_length "$example_title")
    local example_title_padding=$(( (FRAME_WIDTH - example_title_visible_length) / 2 ))
    echo -e "$(create_padding $example_title_padding)$example_title"
    echo -e "$example_line"
    
    echo -e "${WHITE} 1. 创建HDFS目录:${RESET}"
    echo -e "    ${CYAN}hdfs dfs -mkdir -p /user/root${RESET}"
    echo -e "${WHITE} 2. 上传文件到HDFS:${RESET}"
    echo -e "    ${CYAN}hdfs dfs -put /etc/passwd /user/root/${RESET}"
    echo -e "${WHITE} 3. 列出HDFS目录内容:${RESET}"
    echo -e "    ${CYAN}hdfs dfs -ls /user/root/${RESET}"
    echo -e "${WHITE} 4. 查看Hadoop集群状态:${RESET}"
    echo -e "    ${CYAN}hdfs dfsadmin -report${RESET}"
    echo -e "${WHITE} 5. 运行示例MapReduce作业:${RESET}"
    echo -e "    ${CYAN}hadoop jar \$HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar pi 10 100${RESET}"
    
    echo -e "$example_line\n"
    
    echo -e "${LIGHT_CYAN}结束时间: $(date '+%Y-%m-%d %H:%M:%S')${RESET}"
    echo -e "${LIGHT_GREEN}Hadoop伪分布式集群部署全部完成！${RESET}\n"
    
    # 返回到原始目录
    cd "$CURRENT_DIR" > /dev/null 2>&1
}

# 执行主函数
main