attach_node.sh

gistfile1.txt

#!/bin/bash

set -x

export ACK_ATTACH_START_TIME=$(date +%s)
export ACK_ATTACH_START_MONOTONIC_TIME=$(cat /proc/uptime|awk '{print $1}')
# 和灵骏约定俗成好的文件
# aone:https://aone.alibaba-inc.com/v2/project/1174072/req#viewIdentifier=d7f112f9d023e2108fa1b0d8&openWorkitemIdentifier=51816276
export LINGJUN_CONFIG="/etc/eflo_config/lingjun_config"

set -e

echo_args_to_console(){
    set +e
    args_msg=""
    while [[ $# -gt 0 ]]; do
        local key="$1"
        case $key in
        --token)
            shift
            ;;
        --openapi-token)
            shift
            ;;
        *)
            args_msg+=$key
            args_msg+=" "
            ;;
        esac
        shift
    done
    echo "" > /dev/console
    echo "[ACK-NODE-INIT-ARGS] [$(date -R)] [$args_msg]" > /dev/console
    set -e
}

get_ai_node_nicname() {
    local nicname="bond0"
    if [[ ! -f $LINGJUN_CONFIG || "$NODE_TYPE" == "bmcp" ]]; then
        echo ${nicname}
        return
    fi
    nicname=$(cat ${LINGJUN_CONFIG} |grep AckNicName |sed 's/ //g;s/,//g;s/"//g'|awk -F ':' '{print $2}')
    echo $nicname
}

install_exit_callback()
{
    # trap EXIT to upload execution results
    trap 'exit_code=$?; set +x; set +e; exit_callback $exit_code ; exit $exit_code' EXIT
    set -o errtrace
}

exit_callback(){
    cat > /tmp/ack-callback.sh << 'EOF'
#!/bin/bash
msg="[ACK-NODE-INIT] "
if [ -n "$CLUSTER_ID" ]; then
    msg+="[$CLUSTER_ID] "
else
    msg+="[unknown] "
fi
if [ -n "$INSTANCE_ID" ]; then
    msg+="[$INSTANCE_ID] "
else
    msg+="[unknown] "
fi

# cost time
elapsed_time=$(($(date +%s)- $ACK_ATTACH_START_TIME))
msg+="[elapsed_time:$elapsed_time] "
msg+="[exit_code:$1] "
# timestamp
msg+="[$(date -R)] "
addition_info="addition_info:"
if [ -n "$IMAGE_ID" ]; then
    addition_info+="imageId:$IMAGE_ID,"
fi
if [ -n "$OS" ]; then
    addition_info+="os:$OS,"
fi
if [ -n "$KUBE_VERSION" ]; then
    addition_info+="kubeVersion:$KUBE_VERSION,"
fi
if [ -n "$RUNTIME" ]; then
    addition_info+="runtime:$RUNTIME,"
fi
if [ -n "$RUNTIME_VERSION" ]; then
    addition_info+="runtimeVersion:$RUNTIME_VERSION,"
fi

if [ -f "/etc/image-id" ]; then
    if grep -q "ACK-Optimized-OS" /etc/image-id ; then
        addition_info+="ACKOptimizedOS:true,"
    fi
fi

error_code=""
if [ -f "/var/log/ack-deploy-error-code.log" ]; then
    error_code=$(cat /var/log/ack-deploy-error-code.log)
    addition_info+="error_code:$error_code,"
fi

if [ -n "$AUTO_SCALE_NODE" ]; then
    addition_info+="autoScaleNode:$AUTO_SCALE_NODE,"
fi

if [ -n "$REGION" ]; then
    addition_info+="region:$REGION,"
fi

if [ -f /tmp/ack_node_type ]; then
    instance_type=$(cat /tmp/ack_node_type)
    addition_info+="instanceType:$instance_type,"
fi

if [[ $1 -ne 0 ]]; then
    if [[ -z $error_code ]]; then
        error_code="FailedInitNode"
    fi

    err_msg=""
    if [ -f "/var/log/ack-deploy-error-msg.log" ]; then
        err_msg=$(cat /var/log/ack-deploy-error-msg.log)
    fi


    data_str=$(date +%Y-%m-%dT%T%:z)
    echo "{\"reason\":\"${error_code}\",\"message\":\"${err_msg}\",\"timestamp\":\"${data_str}\"}" > /var/log/ack-deploy-error-output.log

    exit_msg="$(tail -c 1024 /var/log/ack-deploy.log)"
    exit_msg="${exit_msg//$'\n'/'\n'}"
    msg+="[$addition_info] "
    msg+="[exit msg: $exit_msg"
    msg+="]"
else
    if /usr/local/bin/ack-tool node-capacity --resource cpu; then
      cpu_capacity=$(/usr/local/bin/ack-tool node-capacity --resource cpu)
      addition_info+="cpu:$cpu_capacity,"
    fi

    if /usr/local/bin/ack-tool node-capacity --resource memory; then
      cpu_capacity=$(/usr/local/bin/ack-tool node-capacity --resource memory)
      addition_info+="memory:$cpu_capacity,"
    fi

    msg+="[$addition_info] "
    msg+="[succeed to execute]"
fi
echo "" > /dev/console
echo "${msg}" > /dev/console
EOF
    # 'cd' somewhere, avoid "shell-init: error retrieving current directory"
    cd /tmp
    chmod +x /tmp/ack-callback.sh
    /tmp/ack-callback.sh $1 &
}

setup_private_env() {
    export OPENAPI="http://{{ .OPENAPI_ADDRESS}}"
    export INTERNET_DOMAIN="{{ .INTERNET_DOMAIN}}"
    export ADDITION_INSECURE_REGISTRY="{{ .ADDITION_INSECURE_REGISTRY }}"
    export CENTER_REGION="{{ .CENTER_REGION }}"
    if [[ "${CENTER_REGION}" == "${REGION}" ]];
    then
       export PKG_FILE_SERVER="{{ .BIZ_CHEST_ENDPOINT }}/static/k8s/"
       export KUBE_REPO_PREFIX="{{ .REGISTRY_ENDPOINT }}/acs"
       export INSECURE_REGISTRY="{{ .CR_REGISTRY_DOMAIN }}"
       export SLB_ENDPOINT="{{ .SLB_ENDPOINT }}"
       export ECS_ENDPOINT="{{ .ECS_ENDPOINT }}"
       export VPC_ENDPOINT="{{ .VPC_ENDPOINT }}"
    else
       export PKG_FILE_SERVER=`echo "{{ .CHEST_ENDPOINT_TEMPLATE }}"|sed "s/\[regionid\]/$REGION/g"`/static/k8s/
       export KUBE_REPO_PREFIX=`echo "{{ .REGISTRY_ENDPOINT_TEMPLATE }}"|sed "s/\[regionid\]/$REGION/g"`/acs
       export INSECURE_REGISTRY=`echo "{{ .CR_REGISTRY_DOMAIN_TEMPLATE }}"|sed "s/\[regionid\]/$REGION/g"`
       export SLB_ENDPOINT=`echo "{{ .SLB_ENDPOINT_TEMPLATE }}"|sed "s/\[regionid\]/$REGION/g"`
       export ECS_ENDPOINT=`echo "{{ .ECS_ENDPOINT_TEMPLATE }}"|sed "s/\[regionid\]/$REGION/g"`
       export VPC_ENDPOINT=`echo "{{ .VPC_ENDPOINT_TEMPLATE }}"|sed "s/\[regionid\]/$REGION/g"`
    fi
}

metadata() {
    if [[ "$NODE_TYPE" == "lingjun" || "$NODE_TYPE" == "bmcp" ]]; then
        case $1 in
            "region-id" )
            echo "$REGION_ID"
            ;;
            "instance-id" )
            echo "$INSTANCE_ID"
            ;;
            "private-ipv4" )
            echo $(ip -o -4 addr list $(get_ai_node_nicname) | awk '{print $4}' | cut -d/ -f1)
            ;;
            "zone-id" )
            echo "$REGION_ID-$NODE_TYPE"
            ;;
            "instance/instance-type" )
            echo "$INSTANCE_TYPE"
            ;;
            * )
            echo ""
            ;;
        esac
        return
    fi
    local path=100.100.100.200/latest/meta-data/$1
    local start_time=$(date +%s)
    for (( i=0; i<5; i++));
    do
        # shellcheck disable=SC2086
        response=$(curl --max-time 5 -sSL $path)

        # shellcheck disable=SC2181
        if [[ $? -gt 0 ]];
        then
            sleep 3; continue
        fi
        if echo "$response"|grep -E "<title>.*</title>" >/dev/null;
        then
            sleep 3; continue
        fi
        local cost_time=$(($(date +%s)- $start_time))
        if [ $cost_time -gt 1 ]; then
            log_trace "${1}MetaSlow${i}T${cost_time}"
        fi
        echo "$response"
        # return from metadata succeed.
        return
    done
    # function will return empty string when failed
}

# try run multi times for cmd
retry() {
    local n=0
    local try="$1"

    while [[ $try -gt $n ]]; do

        # shellcheck disable=SC2068
        if ${@:2}; then
            return 0
        else
            n=$((n + 1))
            # shellcheck disable=SC2145
            echo "failed to run cmd, retry $n :: [${@:2}]"
            sleep 1
        fi
    done

    # shellcheck disable=SC2145
    echo "failed to run with exceeded max retry count: ${@:2}"
    return 1
}

arch_env() {
    export OS_ARCH=amd64
    case $(uname -m) in
    "sw_64")
        export OS_ARCH=sw64
        ;;
    "x86_64")
        export OS_ARCH=amd64
        ;;
    "arm64")
        export OS_ARCH=arm64
        ;;
    "aarch64")
        export OS_ARCH=arm64
        ;;
    *)
        echo "unknown arch $(uname -a), default to amd64 "
        ;;
    esac
}

setup_env() {
    arch_env
    os_env
    OS_TYPE="linux"
    export PKG=pkg
    export RUN=run

    export GPU_FOUNDED=0
    export DISABLE_SWAP=0
    export DEPLOY_DIR="/root/ack-deploy"
    get_region
    get_image_id

    if [[ "$BETA_VERSION" ]]; then
        export BETA_PATH=\/$BETA_VERSION
    fi
    if [[ -z "$CLOUD_TYPE" ]]; then
        export CLOUD_TYPE=public
        export OPENAPI="http://cs-anony-vpc.${REGION}.aliyuncs.com"
        # 公有云场景下设置默认 FILE SERVER 和 REGISTRY ADDRESS
        if [[ -z "$PKG_FILE_SERVER" ]]; then
            export PKG_FILE_SERVER=http://aliacs-k8s-$REGION.oss-$REGION-internal.aliyuncs.com$BETA_PATH
            export PKG_FILE_SERVER_NO_BETA=http://aliacs-k8s-$REGION.oss-$REGION-internal.aliyuncs.com
        fi

        # setup k8s pull image prefix
        if [[ -z "$KUBE_REPO_PREFIX" && -n "$REGION" ]]; then
            export KUBE_REPO_PREFIX=registry-vpc.$REGION.aliyuncs.com/acs
        fi
    fi

    mkdir -p $DEPLOY_DIR
    cd $DEPLOY_DIR

    if [[ "$CLOUD_TYPE" == "private" ]]; then
        setup_private_env
    fi
}

parse_args() {
    while [[ $# -gt 0 ]]; do
        local key="$1"
        case $key in
        --kube-version)
            shift
            ;;
        --runtime)
            export RUNTIME=$2
            shift
            ;;
        --runtime-version)
            export RUNTIME_VERSION=$2
            shift
            ;;
        --docker-version)
            export RUNTIME_VERSION=$2
            shift
            ;;
        --cluster-dns)
            export CLUSTER_DNS=$2
            shift
            ;;
        --token)
            export TOKEN=$2
            shift
            ;;
        --ip-vlan-enabled)
            export IPVLAN_ENABLED=$2
            shift
            ;;
        --endpoint)
            export ENDPOINT=$2
            export APISERVER_LB=${ENDPOINT//:6443/}
            shift
            ;;
        --openapi-token)
            export OPENAPI_TOKEN=$2
            shift
            ;;
        --node-config)
            export NODE_CONFIG=$2
            shift
            ;;
        --disk-topology)
            export DISK_TOPOLOGY=$2
            shift
            ;;
        --node-name-mode)
            export NAME_MODE=$2
            shift
            ;;
        --knode-version)
            export KNODE_VERSION=$2
            shift
            ;;
        --beta-version)
            export BETA_PATH=\/$2
            shift
            ;;
        --disable-swap)
            export DISABLE_SWAP=1
            ;;
        --cluster-id)
            export CLUSTER_ID=$2
            shift
            ;;
        --registry-url)
            export KUBE_REPO_PREFIX=$2/acs
            shift
            ;;
        --ip-stack)
            export IP_STACK=${2:-ipv4}
            if [[ "$IP_STACK" = "dual" || "$IP_STACK" = "ipv6" ]]; then
                export IPV6_ENABLE="true"
            fi
            shift
            ;;
        --is-lingjun-node)
            if [[ "$2" != "" ]]; then
                export NODE_TYPE="lingjun"
            fi
            ;;
        --is-bmcp-node)
            if [[ "$2" != "" ]]; then
                export NODE_TYPE="bmcp"
            fi
            ;;
        --region-id)
            export REGION_ID=$2
            ;;
        --instance-id)
            export INSTANCE_ID=$2
            ;;
        --instance-type)
            export INSTANCE_TYPE=$2
            ;;
        --instance-metadata)
            export INSTANCE_META_DATA=$2
            ;;
        --labels)
            labels=$2
            if [[ $labels == *"k8s.aliyun.com=true"* ]]; then
              export AUTO_SCALE_NODE="true"
            fi
            shift
            ;;
        esac
        shift
    done
}

set_lingjun_metadata() {
    if [[ "$NODE_TYPE" == "lingjun" ]]; then
        if [[ "$REGION_ID" == "" ]]; then
            log_error "Parameter region-id is null"
            exit 1
        fi
        if [[ "$INSTANCE_ID" == "" ]]; then
            log_error "Parameter instance-id is null"
            exit 1
        fi
        if [[ "$INSTANCE_TYPE" == "" ]]; then
            log_error "Parameter instance-type is null"
            exit 1
        fi
        if [[ "$INSTANCE_ID" =~ [A-Z] ]]; then
            log_error "The instance-id contains uppercase letters"
            exit 1
        fi
        echo "region-id=$REGION_ID
instance-id=$INSTANCE_ID
zone-id=$REGION_ID-lingjun
instance/instance-type=$INSTANCE_TYPE
instance/meta-data=$INSTANCE_META_DATA" > /etc/lingjun_metadata
    fi
}

log_init() {
    local log_dir="/var/log"
    local log_file="$log_dir/ack-deploy.log"
    mkdir -p $log_dir || return

    if [[ -f /var/log/ack-deploy-error-code.log ]]; then
      rm /var/log/ack-deploy-error-code.log
    fi

    if [[ -f /var/log/ack-deploy-error-msg.log ]]; then
      rm /var/log/ack-deploy-error-msg.log
    fi

    if [[ -f /var/log/ack-deploy-error-output.log ]]; then
      rm /var/log/ack-deploy-error-output.log
    fi

    if [[ -f /var/log/ack-deploy-trace.log ]]; then
        rm /var/log/ack-deploy-trace.log
    fi

    # redirect stdout and stderr to $log_file and print
    exec &> >(tee -ia $log_file)
}

log() {
  echo -e "\\x1b[1;36m[$(date +%F\ %T)] $*\\x1b[0m" >&2
}

log_trace()
{
    stage=$1
    ts=$(date +%s)
    echo "$stage:$ts" >> /var/log/ack-deploy-trace.log
}

log_info() {
  echo -e "\\x1b[1;33mINFO: $*\\x1b[0m" >&2
}

log_warn() {
  echo -e "\\x1b[1;33mWARN: $*\\x1b[0m" >&2
}

log_error() {
  echo -e "\\x1b[1;31mERROR: $* \\x1b[0m" >&2
}

log_error_and_exit()
{
    error_code=$1
    error_msg=$2
    echo -n $error_code > /var/log/ack-deploy-error-code.log
    echo -n $error_msg > /var/log/ack-deploy-error-msg.log
    echo -e "\\x1b[1;31merror: $* \\x1b[0m" >&2
    exit 1
}

get_region() {
    region=$(metadata "region-id")
    if [[ "$region" ]]; then
        export REGION=$region
    else
       log_error_and_exit "FailedGetRegion" "can not get regionid"
    fi
}

get_image_id() {
    image_id=$(metadata "image-id")
    if [[ "$image_id" ]]; then
        export IMAGE_ID=$image_id
    fi
}

# download deploy script pakcage
download_package() {
    log "download package $1..."

    local pkg_type="$1"
    local pkg_version="$2"
    local pkg_name="${pkg_type}-${pkg_version}-${OS_TYPE}-${OS_ARCH}.tar.gz"

    for ((i = 1; i < 4; i++)); do
      retry 3 wget --no-check-certificate \
           -O "${pkg_name}" --tries 1 --connect-timeout 5 \
           "$PKG_FILE_SERVER/$CLOUD_TYPE/pkg/$pkg_type/$pkg_name"
      if  tar xvf "$pkg_name"; then
        return 0
      else
        log_warn "failed to untar pkg, retry ${i}th after 5s"
        rm -rf "$pkg_name"
        sleep 5
      fi
    done

    log_error_and_exit "FailedDownloadPackage" "failed to download package: $pkg_name"
}

# get k8s cluster related info
get_node_info() {
    log "get node info ..."

    local instance_id

    instance_id=$(metadata "instance-id")
    if [[ -z "$instance_id" ]];
    then
        echo "can not get instance id"; exit 1
    fi
    export INSTANCE_ID=$instance_id

    [[ "$OPENAPI_TOKEN" ]] || return 0

    log_trace "getNodeInfo"

    # node_info output:
    # callback_url=http://cs-anony.cn-beijing.aliyuncs.com/token/xxx/req_once/FF1AADE0-8424-4AD5-B561-CD9114E75A99/callback
    # name_mode=nodeip
    # network=flannel
    # docker_version=19.03.5
    # token=xxx
    # endpoint=192.168.37.117:6443
    # cluster_dns=172.21.0.10
    local node_info

    for (( i=0; i<10; i++));
    do
        set +e
        # test code for lingjun
        # if [[ "$NODE_TYPE" == "lingjun" ]]; then
        #     node_info="
        # callback_url=$CALLBACK_URL
        # name_mode=$NAME_MODE
        # network=flannel
        # docker_version=$RUNTIME_VERSION
        # token=$TOKEN
        # endpoint=$ENDPOINT
        # cluster_dns=$CLUSTER_DNS
        # "
        # else
        #     node_info=$(curl -k --retry 5 -H "Date:$(date -R)" -sSL "$OPENAPI/token/${OPENAPI_TOKEN}/instance/${instance_id}/node_info")
        # fi

        # online code
        node_info=$(curl -k --retry 5 -H "Date:$(date -R)" -sSL "$OPENAPI/token/${OPENAPI_TOKEN}/instance/${instance_id}/node_info")
        
        set -e;
        # shellcheck disable=SC2181
        if [[ $? -gt 0 || "x$node_info" == "x" ]];
        then
            sleep 3; continue
        fi

        if echo "$node_info"|grep -E "<Error>.*</Error>" >/dev/null;
        then
            sleep 3; continue
        fi

        if ! echo "$node_info" | grep "token=" >/dev/null;
        then
            sleep 3; continue
        fi
        break
    done

    if [[ "x$node_info" == "x" ]];
    then
      log_error_and_exit "GetNodeInfoFailed" "failed to get node info"
    fi
    if echo "$node_info"|grep -E "<Error>.*</Error>" >/dev/null;
    then
      log_error_and_exit "GetNodeInfoFailed" "failed to get node info"
    fi

    if ! echo "$node_info" | grep "token=" >/dev/null;
    then
      log_error_and_exit "GetNodeInfoFailed" "failed to get node info"
    fi

    CALLBACK_URL=$(echo "$node_info" | grep "callback_url=" | cut -f2 -d"=")
    export CALLBACK_URL

    if [[ -z "$NAME_MODE" ]]; then
        NAME_MODE=$(echo "$node_info" | grep "name_mode=" | cut -f2 -d"=")
        export NAME_MODE
    fi

    if [[ -z "$NETWORK" ]]; then
        NETWORK=$(echo "$node_info" | grep "network=" | cut -f2 -d"=")
        export NETWORK
    fi

    if echo "$node_info" | grep -q "EniTotalQuantity="; then
        ENI_TOTAL_QUANTITY=$(echo "$node_info" | grep "EniTotalQuantity=" | cut -f2 -d"=")
        export ENI_TOTAL_QUANTITY
        ENI_QUANTITY=$(echo "$node_info" | grep "EniQuantity=" | cut -f2 -d"=")
        export ENI_QUANTITY
        ENI_PRIVATE_IP_ADDRESS_QUANTITY=$(echo "$node_info" | grep "EniPrivateIpAddressQuantity=" | cut -f2 -d"=")
        export ENI_PRIVATE_IP_ADDRESS_QUANTITY
        ENI_IPV6_ADDRESS_QUANTITY=$(echo "$node_info" | grep "EniIpv6AddressQuantity=" | cut -f2 -d"=")
        export ENI_IPV6_ADDRESS_QUANTITY
    fi

    if [[ -z "$RUNTIME_VERSION" ]];
    then
        RUNTIME_VERSION=$(echo "$node_info" | grep "docker_version=" | cut -f2 -d"=")
        export RUNTIME_VERSION
    fi

    if [[ -z "$TOKEN" ]];
    then
      TOKEN=$(echo "$node_info" | grep "token=" | cut -f2 -d"=")
      export TOKEN
    fi

    CLUSTER_DNS=$(echo "$node_info" | grep "cluster_dns=" | cut -f2 -d"=")
    export  CLUSTER_DNS

    if [[ $node_info =~ "internal_endpoint=" ]]; then
        ENDPOINT=$(echo "$node_info" | grep "internal_endpoint=" | cut -f2 -d"=")
        export ENDPOINT
    else
        ENDPOINT=$(echo "$node_info" | grep "endpoint=" | cut -f2 -d"=")
        export ENDPOINT
    fi

    APISERVER_LB=${ENDPOINT//:6443/}
    export APISERVER_LB
}

# validate env var be set
validate_env() {
    if [[ -z "$TOKEN" ]]; then
        log_error_and_exit "WorkerTokenEmpty" "TOKEN must be provided"
    fi

    if [[ -z "$CLUSTER_DNS" ]]; then
        log_error_and_exit "WorkerClusterDnsEmpty" "CLUSTER_DNS must be provided"
    fi

    ## retry to get kubernetes version information in case of apiserver temp failure
    if [[ -z "$KUBE_VERSION" ]]; then
        log_trace "getKubeVersion"
        local cnt=1
        while ((cnt <= 120)); do
            # output: "gitVersion": "v1.18.8-aliyun.1"   => 1.18.8-aliyun.1
            KUBE_VERSION=$(curl -k --connect-timeout 4 "https://${APISERVER_LB}:6443/version" | grep gitVersion | awk '{print $2}' | cut -f2 -d \")
            if [[ "$KUBE_VERSION" ]]; then
                export KUBE_VERSION=${KUBE_VERSION:1}
                export RELEASE_VERSION=$(echo $KUBE_VERSION | awk -F. '{print $1"."$2}')
                return 0
            else
                log_warn "can not get kubeversion from apiserver, retry ${cnt}th after 2s"
                sleep 2
                ((cnt += 1))
            fi
        done
        log_error_and_exit "FailGetKubeVersion" "failed: unable to get kube version from apiserver"
    fi

    log "use KUBE_VERSION ${KUBE_VERSION}"
    return 0
}

validate_node() {
    if [[ "$CLOUD_TYPE" == "private" ]];then
        # private cloud has no yum support, skip
        return 0
    fi
    # Aone #37545945
    if [[ $PKG_MGR == "deb" ]]; then
        return 0
    fi
    if [[ `rpm -qa systemd` < "systemd-219-67" ]]; then
        log_warn "systemd version must more then 219.67, try update"
        yum update -y systemd
    fi
}

# check os type
os_env() {
    grep -q "Red Hat" /etc/redhat-release && export OS="RedHat" && return
    grep -q "CentOS Linux" /etc/os-release && export OS="CentOS" && return
#    grep -q "CentOS Stream" /etc/os-release && export OS="CentOS" && return
    grep -q "Aliyun Linux" /etc/os-release && export OS="AliyunOS" && return
    grep -q "Alibaba Cloud Linux Lifsea" /etc/os-release && export OS="ContainerOS" && return
    grep -q "Alibaba Cloud Linux" /etc/os-release && export OS="AliyunOS" && return
    grep -q "Alibaba Group Enterprise Linux" /etc/os-release && export OS="AliOS" && return
    grep -q "Kylin Linux Advanced Server V10" /etc/os-release && export OS="KylinV10" && return
    grep -q "UnionTech OS Server" /etc/os-release && export OS="CentOS" && return
    grep -q "Anolis OS" /etc/os-release && export OS="CentOS" && return

    grep -q "Ubuntu" /etc/os-release && export OS="Ubuntu" && export PKG_MGR="deb" && return

    export OS="unknown"
    log_error_and_exit "OSNotCompatible" "unknown os... exit"
}

cleanup_env() {
    log "cleanup env ..."
    log_trace "cleanupEnv"

    # not cleanup for ACK-Optmized-OS and ContainerOS
    if grep -q "ACK-Optimized-OS" /etc/image-id || [[ $OS = "ContainerOS" ]]; then
        return 0
    fi

    ip link del cni0 || true
    rm -rf /var/lib/cni/* || true
    rm -rf /etc/cni/net.d/* || true
    rm -rf /run/flannel/* || true
    rm -rf /etc/kubernetes
    docker ps | grep k8s_ | awk '{print $1}' | xargs -I '{}' docker stop {} || true
    if [[ -d "/var/lib/kubelet/pki/" ]]; then
        now=$(date "+%Y-%m-%d-%H-%M-%S")
        mv -f /var/lib/kubelet/pki/ /var/lib/kubelet/pki-$now
    fi

    systemctl stop kubelet || true
    rm -rf /var/lib/kubelet/cpu_manager_state
    log_trace "yumClean"
    rm /usr/bin/kubelet /etc/systemd/system/kubelet.service || true
    rm -rf /opt/cni || true
    rm /usr/bin/kubeadm || true
    rm -rf /etc/systemd/system/kubelet.service.d/ || true
    rm /usr/bin/kubectl || true
    # it needs to wait yum lock
#    timeout 60 yum remove -y kubectl kubeadm kubelet kubernetes-cni || true
#    rpm -ev --nodeps $(rpm -qa|grep kubelet) || true
#    rpm -ev --nodeps $(rpm -qa|grep kubeadm) || true
#    rpm -ev --nodeps $(rpm -qa|grep kubernetes-cni) || true
    log_trace "yumCleanDone"
}

disable_swap() {
    if [[ "$DISABLE_SWAP" = "1" ]]; then
        if [[ $(swapon -s) ]]; then
            swapoff -a
            sed -i '/swap/d' /etc/fstab
        fi
    fi

    if [[ $(swapon -s) ]]; then
        log_error_and_exit "SwapNotSupport" "swap is enabled, not supported"
    fi
}

attach_node() {
    download_package "run" "$RELEASE_VERSION"
    log_trace "startAttachNode"
    ROLE="deploy-nodes" $PKG/$RUN/$RELEASE_VERSION/bin/kubernetes.sh $ALL_ARGS
}

callback() {
    log  "callback ..."

    [[ "$OPENAPI_TOKEN" ]] || return 0
    [[ "$CLOUD_TYPE" = "public" ]] || [[ "$CLOUD_TYPE" = "private" ]] || return 0
    set +e
    curl -k -H "Date:$(date -R)" -X POST -sfSL "${CALLBACK_URL}"
    if [[ $? -gt 0 ]];
    then
        log_error_and_exit "CallbackFail" "failed to callback troopers"
    fi
    set -e

    echo "====================================================================="
    echo "                              SUCCESS                                "
    echo "====================================================================="
}

rewrite_args() {
    args=""
    while [[ $# -gt 0 ]]; do
        local key="$1"
        case $key in
        --kube-version)
            shift
            ;;
        *)
            args+=$key
            args+=" "
            ;;
        esac
        shift
    done
    export ALL_ARGS="$args"
}

main() {
    log_init
    install_exit_callback
    echo_args_to_console "$@"

    ALL_ARGS="$@"
    parse_args "$@"
    rewrite_args "$@"
    set_lingjun_metadata

    disable_swap
    setup_env
    get_node_info
    validate_env
    validate_node
    cleanup_env
    attach_node
    callback
}

main "$@"