High Availability K3s and Kube-vip with Cilium

install.sh

#!/bin/sh
set -e
set -o noglob

# Usage: ./install.sh [options]
#
# Example:
#   Installing on first master node run:
#     ./install --num 3 --vip 192.168.2.10 --iface eth0
#   Installing on other master nodes run:
#     ./install --token <token> --discovery <discovery>

SCRIPT_URL=https://gist.githubusercontent.com/hydrz/1364043149992f4f5bcc1fb12acf02f8/raw/install.sh

# print usage
usage() {
    echo "Usage: $0 [options] 
    -h, --help          Display this help message
    -v, --version       Display version information
    -d, --debug         Enable debug mode
    -n, --num           Number of initial nodes(default: 3)(required on first node)
    --vip               The virtual IP address for high availability(required)
    -i, --interface     The interface for the virtual IP address(required)
    --token             The token for join the cluster(required on other nodes)
    --discovery         The discovery url for join the cluster(required on other nodes)
    " >&2
    exit 1
}

# --- print version ---
version() {
    echo "Version: 0.1.0" >&2
    exit 1
}

# --- parse arguments ---
setup_env() {
    while [ $# -gt 0 ]; do
        case "$1" in
        -h | --help)
            usage
            ;;
        -v | --version)
            version
            ;;
        -d | --debug)
            set -x
            ;;
        -n | --num)
            shift
            NUM=$1
            ;;
        --vip)
            shift
            VIP=$1
            ;;
        -i | --interface)
            shift
            INTERFACE=$1
            ;;
        --token)
            shift
            TOKEN=$1
            ;;
        --discovery)
            shift
            DISCOVERY=$1
            ;;
        *)
            echo "Unknown argument: $1" >&2
            usage
            exit 1
            ;;
        esac
        shift
    done

    # check if the script is running as root
    if [ "$(id -u)" != "0" ]; then
        echo "This script must be run as root" >&2
        exit 1
    fi

    # check if the virtual ip address is set
    if [ -z "$VIP" ]; then
        echo "The virtual ip address is not set" >&2
        exit 1
    fi

    # check vip is a valid IP address
    if ! echo $VIP | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$'; then
        echo "Invalid IP address: $VIP" >&2
        exit 1
    fi

    # check if the interface is set
    if [ -z "$INTERFACE" ]; then
        echo "The interface is not set" >&2
        exit 1
    fi

    # check iface is a valid interface
    if ! ip addr show $INTERFACE >/dev/null 2>&1; then
        echo "Invalid interface: $INTERFACE" >&2
        exit 1
    fi

    # if the discovery url is not set, set IS_FIRST to true
    IS_FIRST="false"
    if [ -z "$DISCOVERY" ]; then
        IS_FIRST="true"
    fi

    if [ "$IS_FIRST" = "true" ]; then
        # check if the number of initial nodes is set
        if [ -z "$NUM" ]; then
            echo "The number of initial nodes is not set" >&2
            exit 1
        fi

        # generate discovery url
        DISCOVERY=$(curl -s https://discovery.etcd.io/new?size=$NUM)

        # generate a token for join the cluster
        TOKEN=$(head -c 16 /dev/urandom | od -An -t x | tr -d ' ')
    else
        # check if the token is set
        if [ -z "$TOKEN" ]; then
            echo "The token is not set" >&2
            usage
            exit 1
        fi

        # check if the discovery url is set
        if [ -z "$DISCOVERY" ]; then
            echo "The discovery url is not set" >&2
            usage
            exit 1
        fi
    fi
}

# --- print the cluster join command ---
print_join_command() {
    echo "Join command:"
    echo "Join other master nodes by running the following command on each node:"
    echo ""
    echo "curl -sfL ${SCRIPT_URL} | sh -s - --vip ${VIP} --interface ${INTERFACE} --token ${TOKEN} --discovery ${DISCOVERY}"
    echo ""
    echo ""
    echo "Join worker nodes by running the following command on each node:"
    echo ""
    echo "curl -sfL https://get.k3s.io | sh -s - agent --server https://${VIP}:6443 --token ${TOKEN}"
}

# --- install etcd ---
install_etcd() {
    # stop etcd service, ignore error if etcd is not installed
    systemctl stop etcd || true
    # clean up etcd data directory
    rm -rf /var/lib/etcd

    # download etcd if not exists
    if [ ! -f /usr/local/bin/etcd ]; then
        ETCD_VER=$(curl -s https://api.github.com/repos/etcd-io/etcd/releases/latest | grep tag_name | cut -d '"' -f 4)
        ETCD_ARCH=amd64
        if [ "$(uname -m)" = "aarch64" ]; then ETCD_ARCH=arm64; fi
        curl -L --fail --remote-name-all https://github.com/etcd-io/etcd/releases/download/${ETCD_VER}/etcd-${ETCD_VER}-linux-${ETCD_ARCH}.tar.gz
        tar xzvf etcd-${ETCD_VER}-linux-${ETCD_ARCH}.tar.gz
        mv etcd-${ETCD_VER}-linux-${ETCD_ARCH}/etcd /usr/local/bin/
        mv etcd-${ETCD_VER}-linux-${ETCD_ARCH}/etcdctl /usr/local/bin/
        rm -rf etcd-${ETCD_VER}-linux-${ETCD_ARCH}*
    fi

    # create etcd directories
    mkdir -p /var/lib/etcd
    chmod 700 /var/lib/etcd

    # generate the etcd configuration file
    cat <<EOF >/etc/etcd.conf.yml
name: $(hostname)
data-dir: /var/lib/etcd
listen-peer-urls: http://0.0.0.0:2380
listen-client-urls: http://0.0.0.0:2379
initial-advertise-peer-urls: http://$(hostname):2380
advertise-client-urls: http://$(hostname):2379
discovery: $DISCOVERY
logger: zap
EOF

    # generate the systemd unit file
    cat <<EOF >/etc/systemd/system/etcd.service
[Unit]
Description=etcd
Documentation=https://github.com/etcd-io/etcd
After=network-online.target local-fs.target remote-fs.target time-sync.target
Wants=network-online.target local-fs.target remote-fs.target time-sync.target

[Service]
Type=notify
Restart=always
RestartSec=5s
LimitNOFILE=40000
TimeoutStartSec=0
ExecStart=/usr/local/bin/etcd --config-file /etc/etcd.conf.yml

[Install]
WantedBy=multi-user.target
EOF

    # becase etcd will wait for the other nodes to join the cluster, we need to run join command on other nodes first
    echo "Becase etcd will wait for the other nodes to join the cluster, we need to run join command on other nodes first"
    echo ""
    print_join_command

    # start etcd
    systemctl daemon-reload
    systemctl enable etcd --now
}

# --- install k3s ---
install_k3s() {
    # uninstall k3s if exists
    if [ -f /usr/local/bin/k3s ]; then
        /usr/local/bin/k3s-uninstall.sh
    fi

    # create k3s config directory
    mkdir -p /etc/rancher/k3s

    # generate the k3s configuration file
    cat <<EOF >/etc/rancher/k3s/config.yaml
flannel-backend: none
disable-network-policy: true
disable: servicelb,traefik,local-storage,metrics-server
token: $TOKEN
tls-san: $VIP
datastore-endpoint: http://127.0.0.1:2379
EOF

    # install k3s
    curl -sfL https://get.k3s.io | sh -s - server --config /etc/rancher/k3s/config.yaml
}

# --- install kube-vip ---
install_kube_vip() {
    KVVERSION=$(curl -s https://api.github.com/repos/kube-vip/kube-vip/releases/latest | grep tag_name | cut -d '"' -f 4)
    ctr image pull ghcr.io/kube-vip/kube-vip:$KVVERSION
    mkdir -p /var/lib/rancher/k3s/server/manifests
    curl https://kube-vip.io/manifests/rbac.yaml | tee /var/lib/rancher/k3s/server/manifests/kube-vip.yaml
    echo '---' | tee -a /var/lib/rancher/k3s/server/manifests/kube-vip.yaml
    ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:$KVVERSION vip \
        /kube-vip manifest daemonset \
        --interface ${INTERFACE} \
        --address ${VIP} \
        --inCluster \
        --taint \
        --controlplane \
        --services \
        --arp \
        --leaderElection | tee -a /var/lib/rancher/k3s/server/manifests/kube-vip.yaml
    systemctl restart k3s
}

# --- install cilium ---
install_cilium() {
    # download cilium if not exists
    if [ ! -f /usr/local/bin/cilium ]; then
        CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/master/stable.txt)
        CLI_ARCH=amd64
        if [ "$(uname -m)" = "aarch64" ]; then CLI_ARCH=arm64; fi
        curl -L --fail --remote-name-all https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum}
        sha256sum --check cilium-linux-${CLI_ARCH}.tar.gz.sha256sum
        tar xzvfC cilium-linux-${CLI_ARCH}.tar.gz /usr/local/bin
        rm -f cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum}
    fi

    # install cilium
    cilium install
}

# --- copy kubeconfig ---
copy_kubeconfig() {
    # copy kube config
    mkdir -p ~/.kube
    cp /etc/rancher/k3s/k3s.yaml ~/.kube/config
    chown $(id -u):$(id -g) ~/.kube/config
    chmod 600 ~/.kube/config
}

# --- run ---
{
    setup_env "$@"
    install_etcd
    install_k3s
    install_kube_vip
    copy_kubeconfig

    if [ "$IS_FIRST" = true ]; then
        install_cilium
    fi

    sed -i 's/server: https:\/\/127.0.0.1:6443/server: https:\/\/'${VIP}':6443/g' ~/.kube/config

    print_join_command

    echo ""
    echo "Done!"
}

run on first node

curl -sfL https://gist.githubusercontent.com/hydrz/1364043149992f4f5bcc1fb12acf02f8/raw/install.sh | sh -s - -n 3 -i eth0 --vip 192.168.2.10