ubuntu-prep-k8s-worker.sh

ubuntu-prep-k8s.sh

#!/usr/bin/env bash

# This script is supposed to run right after having installed the OS
# With only one NIC configured

shopt -s extglob
set -o errtrace
set -o errexit
set +o noclobber

# Defaults {{{
ASSUMEYES=0
NOOP=
FORCE=0
VERBOSE=1
SHOW_TRACE=0
LOG_ROOT=.
LOG_BASENAME="ubuntu-prep-k8s"
LOG="${LOG_ROOT}/${LOG_BASENAME}.log"

MGT_NIC=

K8S_CERT_HASH=$K8S_CERT_HASH
K8S_TOKEN=$K8S_TOKEN
K8S_JOIN=$K8S_JOIN
# }}}

# DO NOT MODIFY ANYTHING AFTER THIS LINE #################################################
DOCKER_REPO_FINGERPRINT="0EBFCD88"
DOCKER_EDITION=stable # or edge
KUBERNETES_REPO_FINGERPRINT="BA07F4FB"
KUBERNETES_EDITION=main

# tracing {{{

trap trace_end EXIT

function trace() # {{{2
{
  local caller_index=1

  while :; do # Parse arguments {{{3
    case $1 in
      --trace-member)
      caller_index=2
      ;;
      --noop|-n)
        return
      ;;
     *)  # End of options
       break
       ;;
    esac
    shift
  done # }}}3

  echo -e "[$(date +'%Y%m%dT%H%M%S')]${BASH_SOURCE[$caller_index]}::${FUNCNAME[$caller_index]}@${BASH_LINENO[(($caller_index - 1))]}: $@" >> "$LOG"
} # 2}}}

function trace_init() # {{{2
{
  local log_file=$(basename $LOG)
  local log_group="wheel"
  local result

  if [ -w $LOG_ROOT ]; then
    export LOG="${LOG_ROOT}/${LOG_BASENAME}-$(date +'%Y%m%d%H%M%S').log"
  else
    export LOG="${HOME}/${LOG_BASENAME}-$(date +'%Y%m%d%H%M%S').log"
  fi

  while :; do # Parse arguments {{{3
    case $1 in
      --logdest)
        [[ -z $2 || ${2:0:1} == '-' ]] && die -n "Argument for option $1 is missing"
        LOG="$2/$log_file"
        shift 2
        continue
      ;;
      --logdest=*?)
        LOG="${1#*=}/$log_file"
      ;;
      --logdest=)
        die -n "Argument for option $1 is missing"
      ;;
      --loggroup)
        [[ -z $2 || ${2:0:1} == '-' ]] && die -n "Argument for option $1 is missing"
        log_group="$2"
        shift 2
        continue
      ;;
      --loggroup=*?)
        log_group=${1#*=}
      ;;
      --loggroup=)
        die -n "Argument for option $1 is missing"
      ;;
     -?*) # Invalid options
       ;;
     --) # Force end of options
       shift
       break
       ;;
     *)  # End of options
       break
       ;;
    esac
    shift
  done # }}}3

  if [[ ! -w $LOG ]]; then
    if [[ ! -w $(dirname $LOG) ]]; then
      echo "NOTE: You might have to enter your password to allow the script to modify your system!"
      if [[ ! -d $(dirname $LOG) ]]; then
        sudo mkdir -p $(dirname $LOG) 2>&1 | tee /dev/null > /dev/null
        result=$?
        [[ $result ]] && die -n "Could not create folder $(dirname $LOG)" $result
      fi
      sudo touch $LOG 2>&1 | tee /dev/null > /dev/null
      [[ $result ]] && die -n "Could not create $LOG" $result
      sudo chown $(whoami):${log_group} $LOG
      [[ $result ]] && die -n "Could not change owner for $LOG" $result
      sudo chmod 640 $LOG 2>&1 | tee /dev/null > /dev/null
      [[ $result ]] && die -n "Could not change permissions for $LOG" $result
    else
      touch $LOG 2>&1 | tee /dev/null > /dev/null
      [[ $result ]] && die -n "Could not create $LOG" $result
      chgrp ${log_group} $LOG 2>&1 | tee /dev/null > /dev/null
      [[ $result ]] && die -n "Could not change group for $LOG" $result
      chmod 640 $LOG 2>&1 | tee /dev/null > /dev/null
      [[ $result ]] && die -n "Could not change permissions for $LOG" $result
    fi
  fi
  trace --trace-member "[BEGIN] --8<----------------------8<------------------------8<------------  [BEGIN]"
} # 2}}}

function trace_end() # {{{2
{
  for cache_mount in ${CACHE_MOUNTS[@]}; do
    trace --trace-member "Removing CIFS mount point: $cache_mount"
    umount $cache_mount 2>&1 > /dev/null
  done

  for vpn_id in ${CONNECTED_VPNS[@]}; do
    vpn_stop --id=$vpn_id
  done

  trace --trace-member "[END]   --8<----------------------8<------------------------8<------------    [END]"
} # 2}}}

function trace_output() ## {{{2
{
  tee -a "$LOG"
} # 2}}}

function verbose() { # {{{2
  trace --trace-member "$@"
  [[ $VERBOSE > 0 ]] && echo -e "$@"
} # 2}}}

function warn() # {{{2
{
  trace --trace-member "[WARNING] $@"
  echo -e "Warning: $@"
} # 2}}}

function error() # {{{2
{
  trace --trace-member "[ERROR] $@"
  echo -e "\e[0;31mError: $@\e[0m" >&2
} # 2}}}

function die() { # {{{2
  local trace_noop=
  if [[ $1 == '-n' ]]; then
   trace_noop=:
   shift
  fi
  local message=$1
  local errorlevel=$2

  [[ -z $message    ]] && message='Died'
  [[ -z $errorlevel ]] && errorlevel=1
  $trace_noop trace --trace-member "[FATALERROR] $errorlevel $message"
  echo -e "\e[0;31m$message\e[0m" >&2
  echo Installation logs: $LOG
  exit $errorlevel
} # 2}}}

function on_error_die() { # {{{2
  status=$? ; [[ $status == 0 ]] || die "$@. Error: $status" $status
  return 0
} # 2}}}

# }}}

function add_repo_ansible() { # {{{2
  local status

  if [[ $(lsb_release -cs) == "bionic" ]]; then
    trace "Bionic Beaver already contains Ansible > 2.3"
    return 0
  fi
  if [[ -z $(grep "ansible" /etc/apt/sources.list) ]]; then
    verbose "Adding repository for Ansible"
    sudo add-apt-repository ppa:ansible/ansible
    status=$? ; [[ $status == 0 ]] || ( trace "Error: $status, while adding repository" ; return $status)
  fi
  return 0
} # 2}}}

function add_repo_docker() { # {{{2
  local status

  if [[ -z $(apt-key fingerprint $DOCKER_REPO_FINGERPRINT) ]]; then
    verbose "Adding repository key for Docker"
    curl -sSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - > /dev/null
    status=$? ; [[ $status == 0 ]] || ( trace "Error: $status, while adding repository key" ; return $status)
  fi

  if [[ -z $(grep "docker.com" /etc/apt/sources.list) ]]; then
    verbose "Adding repository for Docker"
    sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) $DOCKER_EDITION"
    status=$? ; [[ $status == 0 ]] || ( trace "Error: $status, while adding repository" ; return $status)
    # TODO: remove conditionally. Since 18.04, this is done by the add-aot-repository...
    sudo apt update --quiet --quiet
    status=$? ; [[ $status == 0 ]] || ( trace "Error: $status, while updating" ; return $status)
  fi
  return 0
} # 2}}}

function add_repo_kubernetes() { # {{{2
  local status

  if [[ -z $(apt-key fingerprint $KUBERNETES_REPO_FINGERPRINT) ]]; then
    verbose "Adding repository key for Kubernetes"
    curl -sSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - > /dev/null
    status=$? ; [[ $status == 0 ]] || ( trace "Error: $status, while adding repository key" ; return $status)
  fi

  if [[ -z $(grep "kubernetes.io" /etc/apt/sources.list) ]]; then
    verbose "Adding repository for Kubernetes (Note: Kubernetes for Bionic Beaver does not exist yet)"
    sudo add-apt-repository "deb http://apt.kubernetes.io/ kubernetes-xenial $KUBERNETES_EDITION"
    status=$? ; [[ $status == 0 ]] || ( trace "Error: $status, while adding repository" ; return $status)
    # TODO: remove conditionally. Since 18.04, this is done by the add-aot-repository...
    sudo apt update --quiet --quiet
    status=$? ; [[ $status == 0 ]] || ( trace "Error: $status, while updating" ; return $status)
  fi
  return 0
} # 2}}}

function install() { # {{{2
  dpkg -s $1 >&/dev/null || sudo -H apt install --yes --quiet $1
  on_error_die "Failed to install $1"
} # 2}}}

function parse_args() { # {{{2
  while :; do
    trace "Analyzing option \"$1\""
    case $1 in
      --discovery-token-ca-cert-hash|--hash)
        [[ -z $2 || ${2:0:1} == '-' ]] && die "Argument for option $1 is missing"
        K8S_CERT_HASH=$2
        shift 2
        continue
      ;;
      --discovery-token-ca-cert-hash=*?|--hash=*?)
        K8S_CERT_HASH=${1#*=} # delete everything up to =
      ;;
      --discovery-token-ca-cert-hash=|--hash=)
        die "Argument for option $1 is missing"
        ;;

      --join)
        [[ -z $2 || ${2:0:1} == '-' ]] && die "Argument for option $1 is missing"
        K8S_JOIN=$2
        shift 2
        continue
      ;;
      --join=*?)
        K8S_JOIN=${1#*=} # delete everything up to =
        [[ -z $1 || ${1:0:1} == '-' ]] && die "Argument #2 for option $1 is missing"
      ;;
      --join=)
        die "Argument for option $1 is missing"
        ;;

      --token)
        [[ -z $2 || ${2:0:1} == '-' ]] && die "Argument for option $1 is missing"
        K8S_TOKEN=$2
        shift 2
        continue
      ;;
      --token=*?)
        K8S_TOKEN=${1#*=} # delete everything up to =
      ;;
      --token=)
        die "Argument for option $1 is missing"
        ;;

      --force)
        warn "This program will overwrite the current configuration"
        FORCE=1
        ;;
      -h|-\?|--help)
       trace "Showing usage"
       usage
       exit 0
       ;;
      --noop|--dry_run|--dry-run)
        warn "This program will execute in dry mode, your system will not be modified"
        NOOP=:
        ;;
     --quiet)
       VERBOSE=0
       trace "Verbose level: $VERBOSE"
       ;;
     --show_trace|--show-trace)
       SHOW_TRACE=1
       ;;
     -v|--verbose)
       VERBOSE=$((VERBOSE + 1))
       trace "Verbose level: $VERBOSE"
       ;;
     -y|--yes|--assumeyes|--assume_yes|--assume-yes) # All questions will get a "yes"  answer automatically
       ASSUMEYES=1
       trace "All prompts will be answered \"yes\" automatically"
       ;;
     -?*) # Invalid options
       warn "Unknown option $1 will be ignored"
       ;;
     --) # Force end of options
       shift
       break
       ;;
     *)  # End of options
       break
       ;;
    esac
    shift
  done

  # Validation
  if [[ -n $K8S_JOIN ]]; then
    [[ -z $K8S_TOKEN     ]] && die "Missing token to join $K8S_JOIN"
    [[ -z $K8S_CERT_HASH ]] && die "Missing Certificate Hash to join $K8S_JOIN"
    trace "Join Cluster with Master: $K8S_JOIN"
    trace "Token: $K8S_TOKEN"
    trace "Cert Hash: $K8S_CERT_HASH"
  fi
  return 0
} # 2}}}

function main() { # {{{2
  trace_init      ; on_error_die "Failed to initialize tracing"
  
  [[ -r /etc/lsb-release ]] || die "This script supports only Ubuntu"
  trace "Ubuntu version:"
  lsb_release -a >> $LOG 2>&1
  verbose "Installing Kubernetes on $(lsb_release -ds) ($(lsb_release -cs))"

  MGT_NIC=$(ip -o link show | awk '{print $2,$9}' | grep -v docker | grep UP | cut -f1 -d:)

  parse_args "$@" ; on_error_die "Failed to parse command line"

  if ! dpkg -s docker-ce >&/dev/null; then
    # We have not installed anything, let's update the packages
    verbose "Updating Apt..."
    sudo apt update --quiet --quiet
  fi

  add_repo_ansible    ; on_error_die "Failed to add Ansible repository"
  add_repo_docker     ; on_error_die "Failed to add Docker repository"
  add_repo_kubernetes ; on_error_die "Failed to add Kubernetes repository"

  verbose "Installing common pre-requisite stuff"
  install apt-transport-https
  install ca-certificates
  install curl
  install jq
  install software-properties-common
  install ansible
  install ansible-lint

  if [[ -n $(sudo sysctl net.ipv4.ip_forward | grep 0) ]]; then
    verbose "Turning on IPV4 Forwarding"
    sudo tee /etc/sysctl.d/90-bridge.conf >/dev/null <<-EOF
net.ipv4.ip_forward=1

net.bridge.bridge-nf-call-arptables=1
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
EOF
    sudo systemctl restart procps
  fi

  if ! dpkg -s docker-ce >&/dev/null; then
    # We have not installed anything, let's upgrade the packages
    verbose "Upgrading packages"
    sudo UCF_CONFNEW=yes DEBIAN_FRONTEND=noninteractive apt upgrade -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold"
  fi

  if [[ -n $K8S_JOIN ]]; then
    if [[ -n $(grep swap /etc/fstab) ]]; then
      verbose "Turning off swap partitions as Docker does not support them"
      sudo swapoff -a
      sudo sed -i '/ swap /s/^/#/' /etc/fstab
    fi

    if [[ -z $(grep docker /etc/fstab) ]]; then
      used_partitions=( $(blkid | cut -f1 -d:) )
      trace "Used partitions: ${used_partitions[@]}"
      used_disks=( $(blkid | cut -f1 -d: | sed -e 's/[0-9]\+$//' -e 's;^/dev/;;' | sort | uniq) )
      trace "Used disks: ${used_disks[@]}"
      used_disks=$(printf "\|%s" "${used_disks[@]}")
      used_disks=${used_disks:2}
      trace "Used disks: ${used_disks[@]}"
      docker_disk=$(sudo fdisk --list --bytes | grep '^Disk \/dev' | grep -v "$used_disks" | sort -k 5n | tail -1 | cut -f1 -d: | cut -f2 -d' ')
      trace "Docker disk: ${docker_disk}"
      if [[ -n $docker_disk ]]; then
        verbose "Preparing disk $docker_disk for Docker storage"
        verbose "  partitioning..."
        sudo parted --script --align=optimal $docker_disk -- \
          mklabel msdos \
          mkpart primary ext4 0% 100% \
          set 1 lvm on
        sudo parted --script $docker_disk print
        verbose "  creating Physical Volume..."
        sudo pvcreate ${docker_disk}1
        trace "Physical volumes: $(sudo pvs)"
        verbose "  creating Volume Group docker-pool..."
        sudo vgcreate docker-pool ${docker_disk}1
        trace "Physical groups: $(sudo vgs)"
        verbose "  creating Logical Volume docker-data..."
        sudo lvcreate --extents 100%FREE --name docker-data docker-pool
        trace "Logical groups: $(sudo lvs)"
        verbose "  formating Logical Volume docker-data..."
        sudo mke2fs -t ext4 /dev/docker-pool/docker-data | tee -a $LOG
        verbose "  binding docker-data to /var/lib/docker..."
        sudo tee -a /etc/fstab > /dev/null <<EOF
/dev/docker-pool/docker-data /var/lib/docker ext4 defaults 0 0
EOF
        trace "fstab: $(cat /etc/fstab)"
        verbose "  mounting /var/lib/docker..."
        sudo mkdir -p /var/lib/docker
        sudo mount /var/lib/docker
      fi
    fi

    verbose "Installing Docker Community Edition"
    #install docker-ce=17.03.2~ce-0~ubuntu-xenial
    #sudo apt-mark hold docker-ce
    install docker-ce
    sudo usermod -aG docker ${USER}

    echo "Installing Kubernetes"
    install kubelet
    install kubeadm
    install kubectl
    install kubernetes-cni

    verbose "Kubernetes Worker will join its cluster at $K8S_JOIN"
    sudo kubeadm join ${K8S_JOIN} --token $K8S_TOKEN --discovery-token-ca-cert-hash $K8S_CERT_HASH
    on_error_die "Failed to join ${K8S_JOIN}"
  else
    verbose "Installing Docker Community Edition"
    #install docker-ce=17.03.2~ce-0~ubuntu-xenial
    #sudo apt-mark hold docker-ce
    install docker-ce
    sudo usermod -aG docker ${USER}

    echo "Installing Kubernetes"
    install kubeadm
    install kubectl
    install kubernetes-cni

    #if [[ -z $(docker ps | grep k8s) ]]; then
      verbose "Creating Master Node"
      #sudo kubeadm init --pod-network-cidr=10.81.0.0/16 --apiserver-advertise-address=w.x.y.z
      sudo kubeadm init | tee -a kubeadm-init.log
    #fi
    [[ -d $HOME/.kube        ]] || mkdir -p $HOME/.kube
    [[ -r $HOME/.kube/config ]] || \
      (sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config && \
       sudo chown $(id -u):$(id -g) $HOME/.kube/config)

    if ! kubectl get daemonset --namespace kube-system weave-net >& /dev/null; then
      verbose "Installing Weave-net"
      sudo mkdir -p /etc/cni/net.d
      sudo tee /etc/cni/net.d/00-weave-portmap.conflist > /dev/null <<-EOF
{
    "cniVersion": "0.3.0",
    "name": "mynet",
      "plugins": [
        {
            "name": "weave",
            "type": "weave-net",
            "hairpinMode": true
        },
        {
            "type": "portmap",
            "capabilities": {"portMappings": true},
            "snat": true
        }
    ]
}
EOF
      kubectl apply -f \
        "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')"
    fi
    kubectl get nodes

    if [[ -z $(grep 'tinyurl' ./kubeadm-init.log) ]]; then
      echo "To join a worker node with this script, run:" >> ./kubeadm-init.log 
      grep 'kubeadm join' kubeadm-init.log | sed -e 's/^\s\+kubeadm join/curl -sSL https:\/\/tinyurl.com\/ubuntu-prep-k8s | bash -s -- --join /' | tee -a ./kubeadm-init.log
    else
      echo "To join a worker node with this script, run:"
      grep 'tinyurl' kubeadm-init.log
    fi
  fi
  verbose "Installation Logs:  ${LOG}"
} # 2}}}

main "$@"

To run it, on the (future) master:

$ curl -sSL https://tinyurl.com/ubuntu-prep-k8s | bash -s
Then follow the instructions to install the worker nodes.