Skip to content

Instantly share code, notes, and snippets.

@ThinGuy

ThinGuy/microclouds.sh

Last active February 4, 2019 05:00
Show Gist options
  • Save ThinGuy/4531a39413a128689c1b116863ddc4cb to your computer and use it in GitHub Desktop.
Save ThinGuy/4531a39413a128689c1b116863ddc4cb to your computer and use it in GitHub Desktop.
Download ZIP
Microcloud demo using MAAS 2.5, KVM Pods and LXD Clustering
Raw
microclouds.sh
microcloud-tag-cleanup() {
local OTAG=$1
local NTAG=$2
# remove ready machines from a given tag ${OTAG}
# delete given tags ${OTAG} that do not contain machines
# Exclude a given tag ${NTAG} from deletion
[[ -z $OTAG || $OTAG =~ [-]+h ]] && { printf "${FUNCNAME}: <tag_name> [ exclude_name ]\n";return 1; }
local MAAS_INFO=$(maas 2>/dev/null ${MAAS_PROFILE} machines read)
jq 2>/dev/null -r '.[]|"\(.tag_names|to_entries[]|select(.value|startswith("'${OTAG}'")).value) \(select(.status==4).system_id)"' <<< ${MAAS_INFO}|xargs -n2 -P0 bash -c 'maas 2>/dev/null ${MAAS_PROFILE} tag update-nodes $0 remove=$1'
if [[ -n $NTAG ]];then
maas 2>/dev/null ${MAAS_PROFILE} tags read |jq 2>/dev/null -r '.[]|select(.name|startswith("'${OTAG}'"))|select(.name|startswith("'${NTAG}'")|not).name'|xargs -n1 -P0 bash -c 'printf "$0 $(maas 2>/dev/null ${MAAS_PROFILE} tag nodes $0|jq length==0)\n"'|awk '/true/{print $1}'|xargs -n1 -P0 maas 2>/dev/null ${MAAS_PROFILE} tag delete
else
maas 2>/dev/null ${MAAS_PROFILE} tags read |jq 2>/dev/null -r '.[]|select(.name|startswith("'${OTAG}'")).name'|xargs -n1 -P0 bash -c 'printf "$0 $(maas 2>/dev/null ${MAAS_PROFILE} tag nodes $0|jq length==0)\n"'|awk '/true/{print $1}'|xargs -n1 -P0 maas 2>/dev/null ${MAAS_PROFILE} tag delete
fi
} &> /dev/null
export -f microcloud-tag-cleanup
ul-msg() {
[[ $1 = '--desc' ]] && { printf "\e[2G${FUNCNAME}: Underlines the provided string with line style, indent, and bold options.\n";return; };
local INDENT= BOLD=
local STYLE=s
local UL=$(printf '\u2501')
ARGS=$(getopt -o dbi: --long double,bold,indent: -- "$@")
eval set -- "$ARGS"
while true; do
case "$1" in
-i | --indent) local INDENT='\e['${2}'G';shift 2;;
-b | --bold) local BOLD='\e[1m';shift 1;;
-d | --double) local STYLE=d;shift 1;;
--) shift;break;;
esac
done
[[ ${STYLE} = d ]] && local UL=$(printf '\u2550') || local UL=$(printf '\u2501')
printf "\n${BOLD}${INDENT}${@}";
local StrLen=$(($(echo ${@}|sed -E -e 's,\x1B\[[0-9;]*[a-zA-Z],,g;s/^[ \t]*|[ \t]*$//g'|wc -c)-2));
[[ ${@} =~ .*\\n$ || ${@} =~ .*\\n.$ ]] || echo;
printf "${BOLD}${INDENT}"
eval printf "%.3s" ${UL}{0..7};printf "\e[0m\n\n"
}
maas-microcloud-lxd() {
local DESC="${FUNCNAME}: Deploy a LXD Cluster via MAAS command line"
local CI_TZ='America/Los_Angeles'
local CI_LC='en_US.UTF-8'
# Test for Ubuntu Country Mirror based on Locale
local CI_TC=${CI_LC:3:2}
if [[ -n ${CI_TC} && $(curl -slSL -w %{http_code} -o /dev/null ${CI_TC,,}.archive.ubuntu.com) -eq 200 ]];then
local CI_CC=${CI_TC}
local CI_APT_MIRROR=${CI_CC,,}.archive.ubuntu.com
else
local CI_APT_MIRROR=archive.ubuntu.com
fi
local CI_PKG_UPG=true
local CI_PKG_UPD=true
local ENABLE_GPU=false
local BASE_TAG="microcloud-lxd"
local STORAGE_POOL_DEVICE=
local STORAGE_POOL_TYPE=file
local ENFORE_HA=true
local -a CI_PKG_LIST=(
bridge-utils
build-essential
jq
prips
squashfuse
unzip
zfsutils-linux
)
maas-microcloud-lxd_usage() {
printf "\n\e[2G${DESC}\n\n"
printf "\e[2G\e[1mUsage\e[0m: ${FUNCNAME%%_*} [-c <count>] [-t <tag>] [-d <distro>] ( -g gpu-passthrough support ) (-p storage-pool block device)\n\n"
printf "\e[4G -c, --count\e[20GNumber of nodes (Min: 3) \n"
printf "\e[4G -t, --tag \e[20GExisting tag to use when selecting nodes\n"
printf "\e[4G -d, --distro\e[20GName of Ubuntu Distro (Default: bionic)\n"
printf "\e[4G -p, --pool-device\e[20GPhysical disk to use for Storage Pool (Optional: default uses file method)\n"
printf "\e[4G -g, --gpu\e[20GEnable GPU passthrough support\n"
printf "\e[4G -H, --ha\e[20GEnsure n+2 servers are available (default: True)\n"
printf "\e[4G -N, --no-ha\e[20GAllow less than n+2 servers in a cloud (default: False)\n"
printf "\e[4G -h, --help\e[20GThis message\n"
printf "\n\n"
}
ARGS=$(getopt -o n:c:t:d:p:gdhHN -l count:,tag:,distro:,pool-device:,gpu,help,desc,ha,no-ha -n ${FUNCNAME} -- "$@")
eval set -- "$ARGS"
while true ; do
case "$1" in
-n|-c|--count) local COUNT=${2};shift 2;;
-t|--tag) local TAG=${2};local TAG=${TAG,,};shift 2;;
-d|--distro) local DISTRO=${2};local DISTRO=${DISTRO,,};shift 2;;
-p|--pool-device) local STORAGE_POOL_DEVICE=${2};local STORAGE_POOL_DEVICE=${STORAGE_POOL_DEVICE,,};local STORAGE_POOL_TYPE=device;shift 2;;
-g|--gpu) local ENABLE_GPU=true;shift 1;;
-H|--ha) local ENFORCE_HA=true;shift 1;;
-N|--no-ha) local ENFORCE_HA=false;shift 1;;
--desc) printf "\n\e[2G${DESC}\n";return 0;;
-h|--help) ${FUNCNAME}_usage;return 2;;
--) shift;break;;
esac
done
command -v maas &>/dev/null || { printf "\n\n\e[2G\e[2G\e[38;2;255;255;0mSorry! \e[0m\e[38;2;0;255;0m maas-cli \e[0m\e[1m2.5+\e[0m is required for this demo.\n\e[9GPlease install and configure version 2.5 or later\n\n";return 1; }
[[ $(dpkg-query -s maas|grep -oP '(?<=Version: )[^?]{3}'|sed 's/\.//') -lt 25 ]] && { printf "\n\n\e[2G\e[2G\e[38;2;255;255;0mSorry! \e[0mMAAS \e[38;2;0;255;0m2.5\e[0m\e[1m+\e[0m is required for this demo.\n\e[9GYou are running version \e[38;2;255;255;0m$(dpkg-query -s maas|grep -oP '(?<=Version: )[^~]+')\e[0m\n\n";return 1; }
command -v jq &>/dev/null || sudo apt install jq -y
local OK='\u00A0\e[38;2;0;255;0m\u2713\e[0m\u00A0\n'
local FAILED='\u00A0\e[38;2;255;0;0m\u2718\u00A0\n'
printf -- "\n\e[1mMicro-cloud Demo - LXD\e[0m\n\n"
printf -- " - Powered by \e[1m$(lsb_release -ds)\e[0m${OK}\n"
printf -- " - Provisioning by \e[1mMAAS $(dpkg-query -s maas|grep -oP '(?<=Version: )[^~]+')\e[0m${OK}\n"
printf -- " - Featuring \e[1mLXD $(snap info lxd|awk '/- </{print $2}') with clustering\e[0m${OK}\n"
[[ ${ENABLE_GPU} = true ]] && printf -- " -\e[1m GPGPU\e[0m Passthrough enabled\e[0m${OK}\n"
[[ -z ${MAAS_PROFILE} ]] && { read -srp "$(printf "Please enter your MAAS profile name: ")" MAAS_PROFILE_INPUT;local MAAS_PROFILE=$MAAS_PROFILE_INPUT; }
#Validation and error handling
local -a VALID_TAGS=($(maas ${MAAS_PROFILE} tags read|jq -r '.[].name'))
local -a VALID_DISTROS=($(maas ${MAAS_PROFILE} boot-resources read|jq -r '.[]|"\(select((.name|startswith("grub")|not) and (.name|startswith("ubuntu")) and (.name|startswith("pxe")|not)).name|sub("ubuntu/"; ""))"'|sort -uV))
# If not a physical disk, set storage pool to use a file
[[ ${STORAGE_POOL_TYPE} = device ]] && { printf "\e[3G- Storage Pool on Cluster Hosts will be using block device ${STORAGE_POOL_DEVICE} \n\n"; local STORAGE_POOL_TYPE=device; } || { local STORAGE_POOL_TYPE=file;local STORAGE_POOL_DEVICE="/var/snap/lxd/common/lxd/disks/local.img";printf "\e[3G- Storage Pool on Cluster Hosts will be file-based \n\n"; }
[[ ${COUNT} =~ ^[0-9]+$ ]] || { local COUNT=3;printf "\e[3G- Setting Node count to ${COUNT}\n\n"; }
[[ ${COUNT} =~ ^[0-9]+$ && ${COUNT} -lt 3 ]] && { local COUNT=3;printf "\e[3G- Minimum LXD Cluster node count is ${COUNT}. Setting Node count to ${COUNT}.\n\n"; }
[[ -z ${TAG} ]] && { printf "\e[2GERROR: No tag name given ${TAG}\n\n\e[2GValid tags are:\n";printf "\e[4G- %s\n" ${VALID_TAGS[@]};return 1; }
[[ -n ${TAG} && -n $(grep -P '(^|\s)\K'${TAG}'(?=\s|$)' <<< ${VALID_TAGS[@]}) ]] || { printf "\e[2GERROR: No tags exist named ${TAG}\n\n\e[2GValid tags are:\n";printf "\e[4G- %s\n" ${VALID_TAGS[@]};return 1; }
[[ -n ${TAG} ]] && { printf "\e[3G- Using tag \"${TAG}\" as primary machine constraint\n\n"; }
[[ -n ${DISTRO} ]] || { local DISTRO="bionic"; }
[[ -n ${DISTRO} && -n $(grep -P '(^|\s)\K'${DISTRO}'(?=\s|$)' <<< ${VALID_DISTROS[@]}) ]] || { printf "\e[2GERROR: invalid distro: ${DISTRO}\n\n\e[2GValid distros for LXD Clusters are:\n";printf "\e[4G- %s\n" ${VALID_DISTROS[@]};return 1; }
[[ -n ${DISTRO} ]] && { printf "\e[3G- Setting distro to ${DISTRO}\n\n"; }
printf "\n\e[3G- Gathering values from to use for setting up the lxd-cluster. Please wait..."
#Get values we'll be replacing in lxd-cluster script template
local MAAS_LIST=$(maas list|awk '/'${MAAS_PROFILE}'/')
local MAAS_API=$(echo ${MAAS_LIST}|awk '{print $3}')
local MAAS_IP=$(echo ${MAAS_LIST}|awk -F'(//|:)' '{print $3}')
local MAAS_URL=$(echo ${MAAS_LIST}|awk '{gsub(/\/api.*/,"");print $2}')
local MAAS_SUBNET=$(maas ${MAAS_PROFILE} subnets read|jq -r '.[]|select(.vlan.fabric_id==0).cidr')
local MAAS_DOMAIN=$(maas ${MAAS_PROFILE} domains read|jq -r '.[]|select((.authoritative==true) and .id==0).name')
IFS=$'\n' && declare -ag MAAS_SSH_HOST_KEYS=($(maas ${MAAS_PROFILE} sshkeys read|jq -r '.[]|"\(.key)"'))
if [[ -n ${MAAS_API} && -n ${MAAS_URL} && -n ${MAAS_SUBNET} && -n ${MAAS_IP} && -n ${MAAS_DOMAIN} ]];then
printf "${OK}"
else
printf "${FAILED}\e[2GThere was an issue gathering lxd-cluster values from maas. Make you are running this while logged into the maas-cli. Quitting\n\n"
echo
for v in API URL SUBNET IP DOMAIN;do eval echo MAAS_$v=\$MAAS_$v;done
echo
return 1
fi
[[ ${COUNT} -eq 1 ]] && W= || W=s
if [[ ${STORAGE_POOL_TYPE} = device ]];then
printf "\n\e[3G- Finding ${COUNT} machine${W} :\n\e[5G- Marked as \"Ready\"\n\e[5G- Tagged with \"${TAG}\"\n"
local READY_TAGGED_MACHINES=($(maas ${MAAS_PROFILE} machines read|jq -r '.[]|select(.physicalblockdevice_set[].name|contains("'${STORAGE_POOL_DEVICE##*/}'"))|select((.tag_names[]|contains("'"${TAG}"'")) and .status == 4)|"\(.hostname):\(.system_id)"'|head -n${COUNT}|sort -V))
[[ ${#READY_TAGGED_MACHINES[@]} -lt 3 && ${ENFORCE_HA} = true ]] && { printf "\e[2G\e[2G\e[38;2;255;255;0mSorry! \e[0mCould not find enough machines to create a LXD Cluster.\n\n\e[9GMachines Required: 3\n\e[9GMachines Found:\e[4C\e[38;2;255;0;0m${#READY_TAGGED_MACHINES[@]}\n\n";return 1; }
[[ ${#READY_TAGGED_MACHINES[@]} -lt ${COUNT} && ${ENFORCE_HA} = true && ${#READY_TAGGED_MACHINES[@]} -ge 3 ]] && { read -erp "$(printf "\n\e[2G - Could only find ${#READY_TAGGED_MACHINES[@]} machine${W} with the ${TAG} tag. Continue? [y/n] : ")" CONT; [[ ${CONT,,} =~ y ]] && echo || return 0; }
[[ ${#READY_TAGGED_MACHINES[@]} -lt ${COUNT} && ${ENFORCE_HA} = false ]] && { read -erp "$(printf "\n\e[2G - Could only find ${#READY_TAGGED_MACHINES[@]} machines with the ${TAG} tag. Continue? [y/n] : ")" CONT; [[ ${CONT,,} =~ y ]] && echo || return 0; }
local COUNT=${#READY_TAGGED_MACHINES[@]}
else
printf "\n\e[3G- Finding ${COUNT} machine${W} :\n\e[5G- Marked as \"Ready\"\e[5G- Tagged with \"${TAG}\"\n\e[5G- Has physical disk named ${STORAGE_POOL_DEVICE}\n"
local -a READY_TAGGED_MACHINES=($(maas ${MAAS_PROFILE} machines read|jq -r '.[]|select((.tag_names[]|contains("'"${TAG}"'")) and .status == 4)|"\(.hostname):\(.system_id)"'|head -n${COUNT}|sort -V))
[[ ${#READY_TAGGED_MACHINES[@]} -lt 3 && ${ENFORCE_HA} = true ]] && { printf "\e[2G\e[2G\e[38;2;255;255;0mSorry! \e[0mCould not find enough machines to create a LXD Cluster.\n\n\e[9GMachines Required: 3\n\e[9GMachines Found:\e[4C\e[38;2;255;0;0m${#READY_TAGGED_MACHINES[@]}\n\n";return 1; }
[[ ${#READY_TAGGED_MACHINES[@]} -lt ${COUNT} && ${ENFORCE_HA} = true && ${#READY_TAGGED_MACHINES[@]} -ge 3 ]] && { read -erp "$(printf "\n\e[2G - Could only find ${#READY_TAGGED_MACHINES[@]} machine${W} with the ${TAG} tag. Continue? [y/n] : ")" CONT; [[ ${CONT,,} =~ y ]] && echo || return 0; }
[[ ${#READY_TAGGED_MACHINES[@]} -lt ${COUNT} && ${ENFORCE_HA} = false ]] && { read -erp "$(printf "\n\e[2G - Could only find ${#READY_TAGGED_MACHINES[@]} machine${W} with the ${TAG} tag. Continue? [y/n] : ")" CONT; [[ ${CONT,,} =~ y ]] && echo || return 0; }
local COUNT=${#READY_TAGGED_MACHINES[@]}
fi
printf "\n\e[2GAllocating the following Host${W} to the LXD Pods:\n"
printf -- "\e[2G - %s\n" ${READY_TAGGED_MACHINES[@]}|sed 's/:/ \(/g;s/.$/&\)/g';echo
local -a ALLOCATED_MACHINES=($(printf "%s\n" ${READY_TAGGED_MACHINES[@]##*:}|xargs -I{} -n1 -P0 maas ${MAAS_PROFILE} machines allocate system_id={}|jq -r '"\(.hostname):\(.system_id)"'|sort -uV))
[[ ${#ALLOCATED_MACHINES[@]} -lt 3 && ${ENFORCE_HA} = true ]] && { printf "\e[2G\e[2G\e[38;2;255;255;0mSorry! \e[0mCould not allocate enough machines to create a LXD Cluster.\n\n\e[9GMachines Required: 3\n\e[9GMachines Found:\e[4C\e[38;2;255;0;0m${#ALLOCATED_MACHINES[@]}\n\n";return 1; }
[[ ${#ALLOCATED_MACHINES[@]} -lt ${COUNT} && ${ENFORCE_HA} = true && ${#ALLOCATED_MACHINES[@]} -ge 3 ]] && { read -erp "$(printf "\n\e[2G - Could only allocate ${#ALLOCATED_MACHINES[@]} machine${W}. Continue? [y/n] : ")" CONT; [[ ${CONT,,} =~ y ]] && echo || return 0; }
[[ ${#ALLOCATED_MACHINES[@]} -lt ${COUNT} && ${ENFORCE_HA} = false ]] && { read -erp "$(printf "\n\e[2G - Could only allocate ${#ALLOCATED_MACHINES[@]} machines. Continue? [y/n] : ")" CONT; [[ ${CONT,,} =~ y ]] && echo || return 0; }
local COUNT=${#ALLOCATED_MACHINES[@]}
# The first element in the array will be the Cluster Primary
local LXD_PRIMARY=${ALLOCATED_MACHINES[0]%%:*}
# Preconfigure the system keys in /etc/ssh so we can setup the cluster without any prompts or exchanging ssh keys
local -a KEY_TYPES=(rsa dsa ecdsa)
xargs -n1 -P0 bash -c 'ssh-keygen &>/dev/null -t $0 -f /tmp/lxd_cluster_ssh_host_$0_key -P ""' <<< ${KEY_TYPES[@]}
local SSH_HOST_KEYS="$(printf "\nssh_keys:\n";xargs -n1 -P1 bash -c 'printf " $0_private: |\n$(cat /tmp/lxd_cluster_ssh_host_$0_key|sed '"'"'s/^.*$/ &/g'"'"')\n\n $0_public: $(cat /tmp/lxd_cluster_ssh_host_$0_key.pub)\n\n"' <<< ${KEY_TYPES[@]})"
#Remove preseeded ssh keys from local machine
rm -rf /tmp/lxd_cluster_ssh_host* /tmp/lxd_cluster_user_ssh*
# Get SSH Public keys from local accounts and maas. Remove duplicates
local SSH_PUB_KEYS=$((if [[ -n $(find 2>/dev/null ~/.ssh -iname "*.pub") ]];then
echo ssh_authorized_keys:
find 2>/dev/null ~/.ssh -iname "*.pub"|xargs -n1 -P1 bash -c 'printf -- '"'"' - %s\n'"'"' "$(cat $0)"'
fi
if [[ -n $(command -v maas 2>/dev/null) ]];then
if [[ $(maas 2>/dev/null ${MAAS_PROFILE} sshkeys read|jq 2>/dev/null length) -ge 1 ]];then
echo "ssh_authorized_keys:"
maas 2>/dev/null ${MAAS_PROFILE} sshkeys read|jq 2>/dev/null -r '.[]|"\(.key)"'| \
while IFS= read -r line;do
sed 's/^/ - /g'
done
fi
fi
)|awk '!seen[$0]++')
# Create correct the stanza for disk or file based storage pool
[[ ${STORAGE_POOL_DEVICE} =~ dev || ${STORAGE_POOL_DEVICE} =~ sd ]] && local LXD_STORAGE_POOL=$(cat <<EOF
storage_pools:
- config:
source: ${STORAGE_POOL_DEVICE}
description: 'Default LXD Storage Pool'
name: local
driver: zfs
EOF
)
[[ ! ${STORAGE_POOL_DEVICE} =~ dev && ! ${STORAGE_POOL_DEVICE} =~ sd ]] && local LXD_STORAGE_POOL=$(cat <<EOF
storage_pools:
- config:
size: 100GB
source: /var/snap/lxd/common/lxd/disks/local.img
zfs.pool_name: local
description: ""
name: local
driver: zfs
EOF
)
#Indent yaml for use in LXD preeseed yaml
local LXD_SSH_PUB_KEYS=$(echo;echo "${SSH_PUB_KEYS}"|sed 's/^/ /g')
# Create a string of the Cluster hosts that we can for SSH config files (thus the added asterisk)
# (easier to remove the asterisk in a variable than to add it, so it's added to main variable)
local CLUSTER_HOST_LIST=$(printf "%s*\n" ${ALLOCATED_MACHINES[@]%%:*}|paste -sd" "|sort -uV)
# Create Package List for Cluster hosts
local CI_PACKAGES=$([[ ${#CI_PKG_LIST[@]} -ge 1 ]] && printf "packages: [$(printf '%s\n' ${CI_PKG_LIST[@]}|paste -sd',')]\n")
# xpath auto tags for varius different nVidia GPGPUs
if [[ ${ENABLE_GPU} = true ]];then
printf "\e[2GConfiguring xpath auto-tags for GPGPU Detection at the hardware level in MAAS...\n\n"
NVDA_3D_CONTROLLER_TAG_DEF='//node[@id="display"]/description = "3D controller" and //node[@id="display"]/vendor = "NVDA Corporation" and //node[@id="display"]/product = "NVIDIA Corporation"' \
NVDA_3D_CONTROLLER_TAG_COMMENT='xpath auto-tag for nvidia general-purpose GPUs that have been claimed by noveau or nvidiafb module'
declare -ag NVDA_GPGPU_TESLA_PRODUCTS=(P4 P40 P100 V100 T4)
NVDA_GPGPU_TESLA_PROD_DEF="$(for p in ${NVDA_GPGPU_TESLA_PRODUCTS[@]};do printf '%s\n' '//node[@id="display"]/product[starts-with(.,"'${p}'")] or '; done|sed '$ s/ or//g')"
NVDA_GPGPU_TESLA_TAG_DEF='contains(//node[@id="display"]/vendor,"nVidia") and contains(//node[@id="display"]/product,"Tesla") and '"${NVDA_GPGPU_TESLA_PROD_DEF}"''
NVDA_GPGPU_TESLA_TAG_COMMENT="xpath auto-tag for nVidia Tesla-based general-purpose GPUs"
declare -ag NVDA_GPGPU_QUADRO_PRODUCTS=(K M P2000 P4000 P5000 P6000)
NVDA_GPGPU_QUADRO_PROD_DEF="$(for p in ${NVDA_GPGPU_QUADRO_PRODUCTS[@]};do printf '%s\n' '//node[@id="display"]/product[starts-with(.,"'${p}'")] or '; done|sed '$ s/ or//g')"
NVDA_GPGPU_QUADRO_TAG_DEF='contains(//node[@id="display"]/vendor,"nVidia") and contains(//node[@id="display"]/product,"Quadro") and '"${NVDA_GPGPU_QUADRO_PROD_DEF}"''
NVDA_GPGPU_QUADRO_TAG_COMMENT="xpath auto-tag for nVidia Quadro-based general-purpose GPUs"
declare -ag NVDA_GPGPU_ALL_PRODUCTS=($(printf "%s\n" ${NVDA_GPGPU_TESLA_PRODUCTS[@]} ${NVDA_GPGPU_QUADRO_PRODUCTS[@]}|paste -sd" "))
NVDA_GPGPU_ALL_PROD_DEF="$(for p in ${NVDA_GPGPU_TESLA_PRODUCTS[@]} ${NVDA_GPGPU_QUADRO_PRODUCTS[@]};do printf '%s\n' '//node[@id="display"]/product[starts-with(.,"'${p}'")] or '; done|sed '$ s/ or//g')"
NVDA_GPGPU_ALL_TAG_DEF='contains(//node[@id="display"]/vendor,"nVidia") and contains(//node[@id="display"]/product,"Quadro") or contains(//node[@id="display"]/product,"Tesla") and '"${NVDA_GPGPU_ALL_PROD_DEF}"''
NVDA_GPGPU_ALL_TAG_COMMENT="xpath auto-tag for nVidia Quadro and Tesla-based general-purpose GPUs"
NVDA_GPGPU_ALL_KERNEL_OPTS='nomodeset modprobe.blacklist=nouveau modprobe.blacklist=nvidiafb intel_iommu=on iommu=pt rd.driver.pre=vfio-pci video=efifb:off vfio_iommu_type1.allow_unsafe_interrupts=1'
declare -ag GPGPU_ARR=(NVDA_3D_CONTROLLER NVDA_GPGPU_TESLA NVDA_GPGPU_QUADRO NVDA_GPGPU_ALL)
MAAS_TAGS=$(maas ${MAAS_PROFILE} tags read)
for G in ${GPGPU_ARR[@]};do
if [[ -n $(jq -r '.[]|select(.name=="'${G,,}'").name' <<< ${MAAS_TAGS}) ]];then
printf "\e[2G - Updating xpath auto tag for ${G} products...\n"
maas ${MAAS_PROFILE} tag update ${G,,} name=${G,,} definition=$(eval echo \$${G}_TAG_DEF) comment=$(eval echo \$${G}_TAG_COMMENT)
[[ ${G} = NVDA_GPGPU_ALL || ${G} = NVDA_3D_CONTROLLER ]] && maas ${MAAS_PROFILE} tag update ${G,,} name=${G,,} kernel_opts="${NVDA_GPGPU_ALL_KERNEL_OPTS}"
printf '\n\n'
else
printf "\e[2G - Creating xpath auto tag for ${G} products...\n\n"
maas ${MAAS_PROFILE} tags create name=${G,,} definition=$(eval echo \$${G}_TAG_DEF) comment=$(eval echo \$${G}_TAG_COMMENT)
[[ ${G} = NVDA_GPGPU_ALL || ${G} = NVDA_3D_CONTROLLER ]] && maas ${MAAS_PROFILE} tag update ${G,,} name=${G,,} kernel_opts="${NVDA_GPGPU_ALL_KERNEL_OPTS}"
printf '\n\n'
fi
done
fi
if [[ ${ENABLE_GPU} = true ]];then
printf "\e[2G - Adding GPGPU Detection at the OS level using cloud-init...\n"
local CI_GPU_COMMANDS=" - echo Adding GPGPU Support
- if [ -n \$(lspci -nn|awk -vIGNORECASE=1 -F'[ \\\\]\\\\[]' '/\\[03/&&/nvidia/{print \$(NF-3)}') ];then lspci -nn|awk -vIGNORECASE=1 -F'[ \\\\]\\\\[]' '/\\[03/&&/nvidia/{printf \"export NVDA_GPU_%02d_ID=%s\nexport NVDA_GPU_%02d_BUS=%s\n\",NR,\$(NF-3),NR,\$1}'|tee -a /etc/gpgpu-pci-ids.conf;fi
- if [ -n \$(lspci -nn|awk -vIGNORECASE=1 -F'[ \\\\]\\\\[]' '/\\[03/&&/nvidia/{print \$(NF-3)}') ];then lspci -nn|awk -vIGNORECASE=1 -F'[ \\\\]\\\\[]' 'BEGIN {printf \"%s\",\"options vfio-pci ids=\"} /\\[03/&&/nvidia/{printf \"%s,\",\$(NF-3)} END {print \"\\n\"}'|sed 's/,$//'|tee /etc/modprobe.d/vfio.conf;fi
- if [ -n \$(lspci -nn|awk -vIGNORECASE=1 -F'[ \\\\]\\\\[]' '/\\[03/&&/nvidia/{print \$(NF-3)}') ];then printf 'vfio\nvfio_pci\n' > /etc/modules-load.d/vfio.conf;fi
"
fi
#Make LXD Image Script to preseed LXD Clusters image store
local LXD_IMAGE_SCRIPT=$(cat <<EOF|base64 -w0
#!/bin/bash
printf "\e[1mLXD Image Copy Tool\e[0m\n"
#Add ubuntu minimal remotes if they are not defined
printf "\e[2GChecking if ubuntu-minimal repos have been added...\n"
[[ -n \$(/snap/bin/lxc 2>/dev/null remote list|awk '/ubuntu.com\/minimal\/daily/{print $2}') ]] || { printf "\e[2G - Adding ubuntu-minimal-daily repo\n";/snap/bin/lxc remote add ubuntu-min-daily https://cloud-images.ubuntu.com/minimal/daily --protocol simplestreams --public; }
[[ -n \$(/snap/bin/lxc 2>/dev/null remote list|awk '/ubuntu.com\/minimal\/releases/{print $2}') ]] || { printf "\e[2G - Adding ubuntu-minimal-releases repo\n";/snap/bin/lxc remote add ubuntu-min-releases https://cloud-images.ubuntu.com/minimal/releases --protocol simplestreams --public; }
printf "\e[2GCreating array of simplestreams-based remotes\n"
declare -ag REPOS=(\$(/snap/bin/lxc remote list --format=json|jq -r 'to_entries[]|select((.key|startswith("fcb")|not) and .value.Protocol=="simplestreams").key'))
printf "\e[2GCreating array of unique LXD images from ${#REPOS[@]} repos...\n"
declare -ag IMAGES=(\$(xargs -n1 -P0 bash -c '/snap/bin/lxc image list \${0}: a=amd64 --format=json|jq -r '"'"'.[]|select(.aliases!=null)|"'"'"'\$0'"'"':\(.fingerprint|sub("\\\\s.*"; ""))|'"'"'\$0-'"'"'\(.properties.os|ascii_downcase)-\(.properties.release|ascii_downcase)"'"'"'' <<< \${REPOS[@]}|\
sed \
-e 's/ubuntu\(-min-daily\)-ubuntu-\(.*\)/\2\1/' \
-e 's/ubuntu\(-min\)-ubuntu-\(.*\)/\2\1/' \
-e 's/ubuntu\(-daily\)-ubuntu-\(.*\)/\2\1/' \
-e '/images-ubuntu-.*[a-z]$/d' \
-e 's/ubuntu-ubuntu-/ubuntu-r-/g' \
-e 's/ubuntu\(-r\)-\(.*\)/\2\1/' \
-e 's/-daily$/-d/g' \
-e 's/images-//g' \
-e 's/-current//g' \
-e 's/min$/&-r/g'))
printf "\e[2G - Discovered ${#IMAGES[@]} unique LXD OS images\n\n\e[4G - Note:\n\e[6G - aliases that end with \"-r\" are \"release\" builds (aka GA)\n\e[6G - aliases that end with \"-d\" are \"daily\" builds\n\e[2GCommencing downloads...\n\n"
for ((i=0; i<\${#IMAGES[@]}; i++));do printf '%s %s\n' \${IMAGES[i]%%|*} \${IMAGES[i]##*|};done|xargs -n2 -P0 bash -c '/snap/bin/lxc image copy \$0 --alias \$1 --public --auto-update local:'
EOF
)
#Make LXD Cluster Script and convert to base64 to include in cloud-init user-data
local LXD_CLUSTER_SCRIPT=$(cat <<EOF|base64 -w0
#!/bin/bash
if [[ \$(hostname 2>/dev/null -s) = ${LXD_PRIMARY} ]];then
cat <<EOT|sudo lxd init --preseed
config:
core.https_address: \$(dig +short @${MAAS_IP} \$(hostname -f)):8443
core.trust_password: ubuntu
maas.api.key: ${MAAS_API}
maas.api.url: ${MAAS_URL}
networks:
- config:
ipv4.address: auto
ipv4.nat: 'true'
ipv6.address: none
ipv6.nat: 'false'
name: lxdbr0
type: bridge
${LXD_STORAGE_POOL}
profiles:
- name: default
description: 'Default LXD Profile'
config:
user.user-data: |
#cloud-config
${LXD_SSH_PUB_KEYS}
devices:
eth0:
name: eth0
nictype: bridged
parent: lxdbr0
type: nic
root:
path: /
pool: local
type: disk
- name: privileged
description: 'Privileged LXD Profile'
config:
linux.kernel_modules: ip_tables,ip6_tables,netlink_diag,nf_nat,overlay
migration.incremental.memory: 'true'
raw.lxc: |-
lxc.cgroup.devices.allow = c 10:237 rwm
lxc.apparmor.profile = unconfined
lxc.cgroup.devices.allow = b 7:* rwm
security.nesting: 'true'
security.privileged: 'true'
user.user-data: |
#cloud-config
${LXD_SSH_PUB_KEYS}
devices:
eth0:
name: eth0
nictype: bridged
parent: lxdbr0
type: nic
root:
path: /
pool: local
type: disk
kvm:
path: /dev/kvm
type: unix-char
mem:
path: /dev/mem
type: unix-char
loop-control:
path: /dev/loop-control
type: unix-char
loop0:
path: /dev/loop0
type: unix-block
loop1:
path: /dev/loop1
type: unix-block
loop2:
path: /dev/loop2
type: unix-block
loop3:
path: /dev/loop3
type: unix-block
loop4:
path: /dev/loop4
type: unix-block
loop5:
path: /dev/loop5
type: unix-block
loop6:
path: /dev/loop6
type: unix-block
loop7:
path: /dev/loop7
type: unix-block
- name: maas
description: 'MAAS LXD Profile'
config:
user.user-data: |
#cloud-config
${LXD_SSH_PUB_KEYS}
devices:
eth0:
maas.subnet.ipv4: ${MAAS_SUBNET}
name: eth0
nictype: bridged
parent: br0
type: nic
root:
path: /
pool: local
type: disk
- name: maas-privileged
description: 'MAAS Privileged LXD Profile'
config:
linux.kernel_modules: ip_tables,ip6_tables,netlink_diag,nf_nat,overlay
migration.incremental.memory: 'true'
raw.lxc: |-
lxc.cgroup.devices.allow = c 10:237 rwm
lxc.apparmor.profile = unconfined
lxc.cgroup.devices.allow = b 7:* rwm
security.nesting: 'true'
security.privileged: 'true'
user.user-data: |
#cloud-config
${LXD_SSH_PUB_KEYS}
devices:
eth0:
maas.subnet.ipv4: ${MAAS_SUBNET}
name: eth0
nictype: bridged
parent: br0
type: nic
root:
path: /
pool: local
type: disk
kvm:
path: /dev/kvm
type: unix-char
mem:
path: /dev/mem
type: unix-char
loop-control:
path: /dev/loop-control
type: unix-char
loop0:
path: /dev/loop0
type: unix-block
loop1:
path: /dev/loop1
type: unix-block
loop2:
path: /dev/loop2
type: unix-block
loop3:
path: /dev/loop3
type: unix-block
loop4:
path: /dev/loop4
type: unix-block
loop5:
path: /dev/loop5
type: unix-block
loop6:
path: /dev/loop6
type: unix-block
loop7:
path: /dev/loop7
type: unix-block
cluster:
server_name: \$(hostname 2>/dev/null -s)
enabled: true
member_config: []
cluster_address: ''
cluster_certificate: ''
server_address: ''
cluster_password: ''
EOT
elif [[ \$(hostname 2>/dev/null -s) != ${LXD_PRIMARY} ]];then
while [[ \$(sudo ssh 2>/dev/null -i /etc/ssh/ssh_host_rsa_key -tt -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ubuntu@${LXD_PRIMARY} 'test -f /var/snap/lxd/common/lxd/server.crt';echo $?) -ne 0 ]];do sleep 5;done
cat <<EOT|sudo lxd init --preseed
config: {}
networks: []
storage_pools: []
profiles: []
cluster:
server_name: \$(hostname -s)
enabled: true
member_config:
- entity: storage-pool
name: local
key: source
value: ${STORAGE_POOL_DEVICE}
description: ''
- entity: storage-pool
name: local
key: volatile.initial_source
value: ${STORAGE_POOL_DEVICE}
description: ''
- entity: storage-pool
name: local
key: zfs.pool_name
value: local
description: ''
cluster_address: ${LXD_PRIMARY}.${MAAS_DOMAIN}:8443
cluster_certificate: |
\$(sudo ssh 2>/dev/null -i /etc/ssh/ssh_host_rsa_key -tt -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ubuntu@${LXD_PRIMARY} 'sed -e ":a;N;$!ba;s/\n/\n\n/g" /var/snap/lxd/common/lxd/server.crt'|sed 's/^.*$/ &/g')
server_address: \$(dig +short @${MAAS_IP} \$(hostname -f)):8443
cluster_password: ubuntu
EOT
fi
EOF
)
local CI_USER_DATA=$(cat <<EOF|tee /tmp/${FUNCNAME}.cloud-init.yaml|base64 -w0
#cloud-config
bootcmd:
- ip route add default via ${MAAS_IP}
final_message: LXD-Cluster Install complete on \$(hostname -f)
timezone: ${CI_TZ}
locale: ${CI_LC}
apt:
proxy: http://${MAAS_IP}:8000/
primary:
- arches: [amd64]
uri: http://${CI_APT_MIRROR}/ubuntu
security:
- arches: [amd64]
uri: http://security.ubuntu.com/ubuntu
package_update: ${CI_PKG_UPD}
package_upgrade: ${CI_PKG_UPG}
${CI_PACKAGES}
write_files:
- encoding: b64
content: ${LXD_CLUSTER_SCRIPT}
path: /usr/local/bin/lxd-cluster.sh
permissions: '755'
owner: root:root
- encoding: b64
content: ${LXD_IMAGE_SCRIPT}
path: /usr/local/bin/lxc-image-copy.sh
permissions: '755'
owner: root:root
runcmd:
- set -x
- for i in \$(seq \$(find /dev -iname 'loop[0-9]*'|wc -l) 1 256);do mknod -m0660 /dev/loop\${i} b 7 \${i} && chown root.disk /dev/loop\${i};done
- if [ \$(lsb_release -sr|sed 's/\.//g') -le 1804 ];then apt purge lxd lxd-client -y;fi
- if [ \$(lsb_release -sr|sed 's/\.//g') -ge 1810 ];then snap remove lxd;fi
- apt autoremove -y
- snap install lxd --candidate
- adduser \$(id -un 1000) lxd
- if [ ! \$(id -un 1000) = ubuntu -a -n \$(id 2>/dev/null -u ubuntu) ];then adduser ubuntu lxd;fi
- if [ ! -f \$(awk -F":" '/'\$(id -un 0)'/{print \$6}' /etc/passwd)/.ssh/id_rsa ];then printf "y\n"|ssh-keygen -f ~/.ssh/id_rsa -P "";fi
- if [ ! -f \$(awk -F":" '/'\$(id -un 1000)'/{print \$6}' /etc/passwd)/.ssh/id_rsa ];then su - \$(id -un 1000) -c 'printf "y\n"|ssh-keygen -f ~/.ssh/id_rsa -P ""' ;fi
- if [ ! \$(id -un 1000) = ubuntu -a -n \$(id 2>/dev/null -u ubuntu) ];then if [ ! -f \$(awk -F":" '/ubuntu/{print \$6}' /etc/passwd)/.ssh/id_rsa ];then su - ubuntu -c 'printf "y\n"|ssh-keygen -f ~/.ssh/id_rsa -P ""' ;fi;fi
- cp /etc/ssh/ssh_host_rsa_key* \$(awk -F\":\" '/'\$(id -un 1000)'/{print \$6}'/etc/passwd)/.ssh/
- chown -R \$(id -un 1000):\$(id -un 1000) \$(awk -F\":\" '/'\$(id -un 1000)'/{print \$6}'/etc/passwd)/.ssh/
- cat /etc/ssh/ssh_host_rsa_key.pub|tee 1>/dev/null -a \$(awk -F":" '/'\$(id -un 0)'/{print \$6}' /etc/passwd)/.ssh/authorized_keys \$(awk -F":" '/'\$(id -un 1000)'/{print \$6}' /etc/passwd)/.ssh/authorized_keys
- printf -- "\nHost ${CLUSTER_HOST_LIST}\n\tAddressFamily inet\n\tCheckHostIP no\n\tForwardX11Trusted yes\n\tForwardX11 yes\n\tIdentityFile /etc/ssh/ssh_host_rsa_key\n LogLevel FATAL\n SendEnv LANG LC_*\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n User ubuntu\n\tXAuthLocation /usr/bin/xauth\n"|tee -a 1>/dev/null /etc/ssh/ssh_config
- printf -- "%s\n" ${CLUSTER_HOST_LIST//\*/}|xargs -I{} -n1 -P0 ssh-keyscan 2>/dev/null -H {}|su - \$(id -un 1000) -c "tee -a ~/.ssh/known_hosts"
- printf -- "%s\n" ${CLUSTER_HOST_LIST//\*/}|xargs -I{} -n1 -P0 ssh-keyscan 2>/dev/null -H {}|tee -a \$(awk -F\":\" '/'\$(id -un 0)'/{print \$6}' /etc/passwd)/.ssh/known_hosts
- if [ -f /usr/local/bin/lxd-cluster.sh ];then /usr/local/bin/lxd-cluster.sh;fi
- if [ \$(hostname -s) = ${LXD_PRIMARY} -a -f /usr/local/bin/lxc-image-copy.sh ];then /usr/local/bin/lxc-image-copy.sh;fi
- snap install conjure-up --classic
- lxc 2>/dev/null profile device set maas eth0 parent \$(ip route get ${MAAS_IP}|/bin/grep -oP '(?<=dev )[^ ]+')
- lxc 2>/dev/null profile device set maas-privileged eth0 parent \$(ip route get ${MAAS_IP}|/bin/grep -oP '(?<=dev )[^ ]+')
- export GPU_CMD='lspci -nn|awk -vIGNORECASE=1 '"'"'/\[03.*intel/{ORS=",";print substr(\$(NF-2),2,9)}'"'"'|sed "s/,$/\\n/"'
- export GPU_CONF_FILE='lspci -nn|awk -vIGNORECASE=1 '"'"'/\[03.*intel/{printf "export NVDA_GPU_%02d_ID=%s\nexport NVDA_GPU_%02d_BUS=%s\n",NR,substr(\$(NF-2),2,9),NR,\$1}'"'"'|tee -a /etc/gpgpu-pci-ids.conf'
- export GPU_PCI_LIST='lspci -nn|awk -vIGNORECASE=1 '"'"'BEGIN {printf "%s","options vfio-pci ids="} /\[03.*intel/{ORS=",";print substr(\$(NF-2),2,9)}'"'"'|sed "s/,$/\\n/"|tee /etc/modprobe.d/vfio.conf'
- if [ -n \$(eval \$GPU_CMD) ];then echo GPGPUs Detected;printf '%s\n' vfio vfio_pci > /etc/modules-load.d/vfio.conf;eval \$GPU_CONF_FILE;eval \$GPU_PCI_LIST;fi
- touch /var/tmp/install-complete
${SSH_HOST_KEYS}
${SSH_PUB_KEYS}
EOF
)
#Remove old tags on redeployment
printf "\n\e[2GCleaning up any existing microcloud tags that these machines may have belonged to...\n"
printf '%s\n' ${ALLOCATED_MACHINES[@]##*:}|xargs -n1 -P0 bash -c 'maas 2>/dev/null ${MAAS_PROFILE} machine read $0|jq 2>/dev/null -r '"'"'.tag_names[]|select(startswith("'${BASE_TAG}'"))'"'"''|xargs -n1 -P0 maas 2>/dev/null ${MAAS_PROFILE} tag delete &>/dev/null
local MICROCLOUD_TAG="${BASE_TAG}-$(grep -m1 -oaE '[a-z0-9]{6}' /dev/random|head -n1)"
printf "\n\e[2GCreating tag \"${MICROCLOUD_TAG}\" for this microcloud...\n"
maas ${MAAS_PROFILE} tags create name=${MICROCLOUD_TAG} comment="$(echo -n ${MICROCLOUD_TAG^^}|sed 's/-/_/g')_HOSTS=(`printf '%s\n' ${ALLOCATED_MACHINES[@]##*:}|paste -sd" "` ); $(echo -n ${MICROCLOUD_TAG^^}|sed 's/-/_/g')_BUILD_DATE=$(date --iso-8601=seconds)" &>/dev/null
printf "\e[2G- Tagging ${#ALLOCATED_MACHINES[@]} machines with \"${MICROCLOUD_TAG}\"...\n"
ADD=$(printf "add=%s\n" ${ALLOCATED_MACHINES[@]##*:}|paste -sd' ')
local ADDED=$(eval "maas ${MAAS_PROFILE} tag update-nodes ${MICROCLOUD_TAG} "$ADD""|jq -r .added)
[[ ${ADDED} -eq ${#ALLOCATED_MACHINES[@]} ]] && { printf "\e[4G - Successfully tagged ${ADDED}/${#ALLOCATED_MACHINES[@]} machines with the \"${MICROCLOUD_TAG}\" tag ${OK} "; } || { printf "\e[2G - Could only tag ${ADDED}/${#ALLOCATED_MACHINES[@]} machines with ${MICROCLOUD_TAG} ${FAILED} \n"; }
echo
printf "\n\e[2GDeploying the following hosts to ${MICROCLOUD_TAG} :\n\n"
printf -- "\e[2G - %s\n" ${ALLOCATED_MACHINES[@]}|sed 's/:/ \(/g;s/.$/&\)/g';echo
local -a DEPLOYED_MACHINES=($(printf "%s\n" ${ALLOCATED_MACHINES[@]##*:}|xargs -I{} -n1 -P1 maas ${MAAS_PROFILE} machine deploy {} distro_series=${DISTRO} user_data=${CI_USER_DATA}|jq 2>/dev/null -r '"\(.hostname):\(.system_id)"'))
printf "\n\e[4G - A copy of the cloud-init used for this deployment can be found at\n\e[2G/tmp/${FUNCNAME}.cloud-init.yaml\n\n"
microcloud-tag-cleanup ${BASE_TAG} ${MICROCLOUD_TAG}
}
export -f maas-microcloud-lxd
maas-microcloud-kvm() {
local DESC="${FUNCNAME}: Deploy a KVM Pods with Cloud-Init customizations via MAAS command line"
local CI_TZ='America/Los_Angeles'
local CI_LC='en_US.UTF-8'
# Test for Ubuntu Country Mirror based on Locale
local CI_TC=${CI_LC:3:2}
if [[ -n ${CI_TC} && $(curl -slSL -w %{http_code} -o /dev/null ${CI_TC,,}.archive.ubuntu.com) -eq 200 ]];then
local CI_CC=${CI_TC}
local CI_APT_MIRROR=${CI_CC,,}.archive.ubuntu.com
else
local CI_APT_MIRROR=archive.ubuntu.com
fi
local CI_PKG_UPG=true
local CI_PKG_UPD=true
local BASE_TAG="microcloud-kvm"
local ENABLE_GPU=false
local STORAGE_POOL_DEVICE=
local STORAGE_POOL_TYPE=file
local ENFORCE_HA=true
local -a CI_PKG_LIST=(
apt-utils
build-essential
debconf-utils
jq
libvirt-daemon-driver-storage-zfs
libvirt-daemon-system
ovmf
squashfuse
virtinst
virt-manager
zfsutils
)
maas-microcloud-kvm_usage() {
printf "\n\e[2G${DESC}\n\n"
printf "\e[2G\e[1mUsage\e[0m: ${FUNCNAME%%_*} [-c <count>] [-t <tag>] [-d <distro>] ( -g gpu-passthrough support ) (-p storage-pool block device)\n\n"
printf "\e[4G -c, --count\e[20GNumber of nodes (Min: 3) \n"
printf "\e[4G -t, --tag \e[20GExisting tag to use when selecting nodes\n"
printf "\e[4G -d, --distro\e[20GName of Ubuntu Distro (Default: bionic)\n"
printf "\e[4G -p, --pool-device\e[20GPhysical disk to use for KVM Storage Pool (Default: File-based Pool)\n"
printf "\e[4G -g, --gpu\e[20GEnable GPU passthrough support\n"
printf "\e[4G -H, --ha\e[20GEnsure n+2 servers are available (default: True)\n"
printf "\e[4G -N, --no-ha\e[20GAllow less than n+2 servers in a cloud (default: False)\n"
printf "\e[4G -h, --help\e[20GThis message\n"
printf "\n\n"
}
ARGS=$(getopt -o n:c:t:d:p:gdhHN -l count:,tag:,distro:,pool-device:,gpu,help,desc,ha,no-ha -n ${FUNCNAME} -- "$@")
eval set -- "$ARGS"
while true ; do
case "$1" in
-n|-c|--count) local COUNT=${2};shift 2;;
-t|--tag) local TAG=${2};local TAG=${TAG,,};shift 2;;
-d|--distro) local DISTRO=${2};local DISTRO=${DISTRO,,};shift 2;;
-p|--pool-device) local STORAGE_POOL_DEVICE=${2};local STORAGE_POOL_DEVICE=${STORAGE_POOL_DEVICE,,};local STORAGE_POOL_TYPE=device;shift 2;;
-g|--gpu) local ENABLE_GPU=true;shift 1;;
-H|--ha) local ENFORCE_HA=true;shift 1;;
-N|--no-ha) local ENFORCE_HA=false;shift 1;;
--desc) printf "\n\e[2G${DESC}\n";return 0;;
-h|--help) ${FUNCNAME}_usage;return 2;;
--) shift;break;;
esac
done
command -v maas &>/dev/null || { printf "\n\n\e[2G\e[2G\e[38;2;255;255;0mSorry! \e[0m\e[38;2;0;255;0m maas-cli \e[0m\e[1m2.5+\e[0m is required for this demo.\n\e[9GPlease install and configure version 2.5 or later\n\n";return 1; }
[[ $(dpkg-query -s maas|grep -oP '(?<=Version: )[^?]{3}'|sed 's/\.//') -lt 25 ]] && { printf "\n\n\e[2G\e[2G\e[38;2;255;255;0mSorry! \e[0mMAAS \e[38;2;0;255;0m2.5\e[0m\e[1m+\e[0m is required for this demo.\n\e[9GYou are running version \e[38;2;255;255;0m$(dpkg-query -s maas|grep -oP '(?<=Version: )[^~]+')\e[0m\n\n";return 1; }
command -v jq &>/dev/null || sudo apt install jq -y
local OK='\u00A0\e[38;2;0;255;0m\u2713\e[0m\u00A0\n'
local FAILED='\u00A0\e[38;2;255;0;0m\u2718\u00A0\n'
printf -- "\n\e[1mMicro-cloud Demo - KVM\e[0m\n\n"
printf -- "\e[3G- Powered by \e[1m$(lsb_release -ds)\e[0m${OK}\n"
printf -- "\e[3G- Provisioning by \e[1mMAAS $(dpkg-query -s maas|grep -oP '(?<=Version: )[^~]+')\e[0m${OK}\n"
printf -- "\e[3G- Featuring \e[1mKVM Pods \e[0m${OK}\n"
[[ ${ENABLE_GPU} = true ]] && printf -- " -\e[1m GPGPU\e[0m Passthrough enabled\e[0m${OK}\n"
printf '\n\n'
[[ -z ${MAAS_PROFILE} ]] && { read -srp "$(printf "\e[2GPlease enter your MAAS profile name: ")" MAAS_PROFILE_INPUT;local MAAS_PROFILE=$MAAS_PROFILE_INPUT; }
local MAAS_IP=$(maas list|awk -F"//|:" '/^'${MAAS_PROFILE}'/{print $3}')
#Validation and error handling
local -a VALID_TAGS=($(maas ${MAAS_PROFILE} tags read|jq -r '.[].name'))
local -a VALID_DISTROS=($(maas ${MAAS_PROFILE} boot-resources read|jq -r '.[]|"\(select((.name|startswith("grub")|not) and (.name|startswith("ubuntu")) and (.name|startswith("pxe")|not)).name|sub("ubuntu/"; ""))"'|sort -uV))
[[ ${ENFORCE_HA} = true ]] && { local COUNT=3;printf "\e[3G- Enforcing n+2 Node count\n"; }
# If not a physical disk, set storage pool to use a file
[[ ${STORAGE_POOL_TYPE} = device ]] && { printf "\e[3G- Storage Pool will use block device ${STORAGE_POOL_DEVICE} \n"; local STORAGE_POOL_TYPE=device; } || { local STORAGE_POOL_TYPE=file;local STORAGE_POOL_DEVICE="/var/lib/libvirt/images/";printf "\e[3G- Storage Pool will be file-based \n"; }
[[ -z ${COUNT} ]] && { [[ ${ENFORCE_HA} = true ]] && { local COUNT=3;printf "\e[3G- Setting Node count to ${COUNT}\n"; } || { local COUNT=1;printf "\e[3G- Setting Node count to ${COUNT}\n"; }; }
[[ ${COUNT} =~ ^[0-9]+$ && ${COUNT} -lt 3 && ${ENFORCE_HA} = true ]] && { local COUNT=3;printf "\e[3G- Minimum Node count for HA is ${COUNT}. Setting Node count to ${COUNT}.\n\n"; }
[[ -z ${TAG} ]] && { printf "\e[2GERROR: No tag name given ${TAG}\n\n\e[2GValid tags are:\n";printf "\e[4G- %s\n" ${VALID_TAGS[@]};return 1; }
[[ -n ${TAG} && -n $(grep -P '(^|\s)\K'${TAG}'(?=\s|$)' <<< ${VALID_TAGS[@]}) ]] || { printf "\e[2GERROR: No tags exist named ${TAG}\n\n\e[2GValid tags are:\n";printf "\e[4G- %s\n" ${VALID_TAGS[@]};return 1; }
[[ -n ${DISTRO} ]] && { printf "\e[3G- Setting distro to ${DISTRO}\n"; } || { local DISTRO="bionic";printf "\e[3G- Setting distro to ${DISTRO}\n"; }
[[ -n ${DISTRO} && -n $(grep -P '(^|\s)\K'${DISTRO}'(?=\s|$)' <<< ${VALID_DISTROS[@]}) ]] || { printf "\e[2GERROR: Invalid Distro: ${DISTRO}\n\n\e[2GValid distros for KVM Pods are:\n";printf "\e[4G- %s\n" ${VALID_DISTROS[@]};return 1; }
[[ ${COUNT} -eq 1 ]] && W= || W=s
if [[ ${STORAGE_POOL_TYPE} = device ]];then
printf "\n\e[3G- Finding ${COUNT} machine${W} :\n\e[5G- Marked as \"Ready\"\n\e[5G- Tagged with \"${TAG}\"\n"
local READY_TAGGED_MACHINES=($(maas ${MAAS_PROFILE} machines read|jq -r '.[]|select(.physicalblockdevice_set[].name|contains("'${STORAGE_POOL_DEVICE##*/}'"))|select((.tag_names[]|contains("'"${TAG}"'")) and .status == 4)|"\(.hostname):\(.system_id)"'|head -n${COUNT}|sort -V))
[[ ${#READY_TAGGED_MACHINES[@]} -lt 3 && ${ENFORCE_HA} = true ]] && { printf "\e[2G\e[2G\e[38;2;255;255;0mSorry! \e[0mCould not find enough machines to create a KVM Cluster.\n\n\e[9GMachines Required: 3\n\e[9GMachines Found:\e[4C\e[38;2;255;0;0m${#READY_TAGGED_MACHINES[@]}\n\n";return 1; }
[[ ${#READY_TAGGED_MACHINES[@]} -lt ${COUNT} && ${ENFORCE_HA} = true && ${#READY_TAGGED_MACHINES[@]} -ge 3 ]] && { read -erp "$(printf "\n\e[2G - Could only find ${#READY_TAGGED_MACHINES[@]} machine${W} with the ${TAG} tag. Continue? [y/n] : ")" CONT; [[ ${CONT,,} =~ y ]] && echo || return 0; }
[[ ${#READY_TAGGED_MACHINES[@]} -lt ${COUNT} && ${ENFORCE_HA} = false ]] && { read -erp "$(printf "\n\e[2G - Could only find ${#READY_TAGGED_MACHINES[@]} machines with the ${TAG} tag. Continue? [y/n] : ")" CONT; [[ ${CONT,,} =~ y ]] && echo || return 0; }
local COUNT=${#READY_TAGGED_MACHINES[@]}
else
printf "\n\e[3G- Finding ${COUNT} machine${W} :\n\e[5G- Marked as \"Ready\"\e[5G- Tagged with \"${TAG}\"\n\e[5G- Has physical disk named ${STORAGE_POOL_DEVICE}\n"
local -a READY_TAGGED_MACHINES=($(maas ${MAAS_PROFILE} machines read|jq -r '.[]|select((.tag_names[]|contains("'"${TAG}"'")) and .status == 4)|"\(.hostname):\(.system_id)"'|head -n${COUNT}|sort -V))
[[ ${#READY_TAGGED_MACHINES[@]} -lt 3 && ${ENFORCE_HA} = true ]] && { printf "\e[2G\e[2G\e[38;2;255;255;0mSorry! \e[0mCould not find enough machines to create a KVM Cluster.\n\n\e[9GMachines Required: 3\n\e[9GMachines Found:\e[4C\e[38;2;255;0;0m${#READY_TAGGED_MACHINES[@]}\n\n";return 1; }
[[ ${#READY_TAGGED_MACHINES[@]} -lt ${COUNT} && ${ENFORCE_HA} = true && ${#READY_TAGGED_MACHINES[@]} -ge 3 ]] && { read -erp "$(printf "\n\e[2G - Could only find ${#READY_TAGGED_MACHINES[@]} machine${W} with the ${TAG} tag. Continue? [y/n] : ")" CONT; [[ ${CONT,,} =~ y ]] && echo || return 0; }
[[ ${#READY_TAGGED_MACHINES[@]} -lt ${COUNT} && ${ENFORCE_HA} = false ]] && { read -erp "$(printf "\n\e[2G - Could only find ${#READY_TAGGED_MACHINES[@]} machine${W} with the ${TAG} tag. Continue? [y/n] : ")" CONT; [[ ${CONT,,} =~ y ]] && echo || return 0; }
local COUNT=${#READY_TAGGED_MACHINES[@]}
fi
printf "\n\e[2GAllocating the following Host${W} to the KVM Pods:\n"
printf -- "\e[2G - %s\n" ${READY_TAGGED_MACHINES[@]}|sed 's/:/ \(/g;s/.$/&\)/g';echo
local -a ALLOCATED_MACHINES=($(printf "%s\n" ${READY_TAGGED_MACHINES[@]##*:}|xargs -I{} -n1 -P0 maas ${MAAS_PROFILE} machines allocate system_id={}|jq -r '"\(.hostname):\(.system_id)"'|sort -uV))
[[ ${#ALLOCATED_MACHINES[@]} -lt 3 && ${ENFORCE_HA} = true ]] && { printf "\e[2G\e[2G\e[38;2;255;255;0mSorry! \e[0mCould not allocate enough machines to create a KVM Pods.\n\n\e[9GMachines Required: 3\n\e[9GMachines Found:\e[4C\e[38;2;255;0;0m${#ALLOCATED_MACHINES[@]}\n\n";return 1; }
[[ ${#ALLOCATED_MACHINES[@]} -lt ${COUNT} && ${ENFORCE_HA} = true && ${#ALLOCATED_MACHINES[@]} -ge 3 ]] && { read -erp "$(printf "\n\e[2G - Could only allocate ${#ALLOCATED_MACHINES[@]} machine${W}. Continue? [y/n] : ")" CONT; [[ ${CONT,,} =~ y ]] && echo || return 0; }
[[ ${#ALLOCATED_MACHINES[@]} -lt ${COUNT} && ${ENFORCE_HA} = false ]] && { read -erp "$(printf "\n\e[2G - Could only allocate ${#ALLOCATED_MACHINES[@]} machines. Continue? [y/n] : ")" CONT; [[ ${CONT,,} =~ y ]] && echo || return 0; }
local COUNT=${#ALLOCATED_MACHINES[@]}
# Get SSH Public keys from local accounts and maas. Remove duplicates
local SSH_PUB_KEYS=$((if [[ -n $(find 2>/dev/null ~/.ssh -iname "*.pub") ]];then
echo ssh_authorized_keys:
find 2>/dev/null ~/.ssh -iname "*.pub"|xargs -n1 -P1 bash -c 'printf -- '"'"' - %s\n'"'"' "$(cat $0)"'
fi
if [[ -n $(command -v maas 2>/dev/null) ]];then
if [[ $(maas 2>/dev/null ${MAAS_PROFILE} sshkeys read|jq 2>/dev/null length) -ge 1 ]];then
echo "ssh_authorized_keys:"
maas 2>/dev/null ${MAAS_PROFILE} sshkeys read|jq 2>/dev/null -r '.[]|"\(.key)"'| \
while IFS= read -r line;do
sed 's/^/ - /g'
done
fi
fi
)|awk '!seen[$0]++')
# Create a string of the Cluster hosts that we can for SSH config files (thus the added asterisk)
# (easier to remove the asterisk in a variable than to add it, so it's added to main variable)
local CLUSTER_HOST_LIST=$(printf "%s*\n" ${ALLOCATED_MACHINES[@]%%:*}|paste -sd" "|sort -uV)
# Create Package List for Cluster hosts
local CI_PACKAGES=$([[ ${#CI_PKG_LIST[@]} -ge 1 ]] && printf "packages: [$(printf '%s\n' ${CI_PKG_LIST[@]}|paste -sd',')]\n")
# xpath auto tags for varius different nVidia GPGPUs
if [[ ${ENABLE_GPU} = true ]];then
printf "\e[2GConfiguring xpath auto-tags for GPGPU Detection at the hardware level in MAAS...\n\n"
NVDA_3D_CONTROLLER_TAG_DEF='//node[@id="display"]/description = "3D controller" and //node[@id="display"]/vendor = "NVDA Corporation" and //node[@id="display"]/product = "NVIDIA Corporation"' \
NVDA_3D_CONTROLLER_TAG_COMMENT='xpath auto-tag for nvidia general-purpose GPUs that have been claimed by noveau or nvidiafb module'
declare -ag NVDA_GPGPU_TESLA_PRODUCTS=(P4 P40 P100 V100 T4)
NVDA_GPGPU_TESLA_PROD_DEF="$(for p in ${NVDA_GPGPU_TESLA_PRODUCTS[@]};do printf '%s\n' '//node[@id="display"]/product[starts-with(.,"'${p}'")] or '; done|sed '$ s/ or//g')"
NVDA_GPGPU_TESLA_TAG_DEF='contains(//node[@id="display"]/vendor,"nVidia") and contains(//node[@id="display"]/product,"Tesla") and '"${NVDA_GPGPU_TESLA_PROD_DEF}"''
NVDA_GPGPU_TESLA_TAG_COMMENT="xpath auto-tag for nVidia Tesla-based general-purpose GPUs"
declare -ag NVDA_GPGPU_QUADRO_PRODUCTS=(K M P2000 P4000 P5000 P6000)
NVDA_GPGPU_QUADRO_PROD_DEF="$(for p in ${NVDA_GPGPU_QUADRO_PRODUCTS[@]};do printf '%s\n' '//node[@id="display"]/product[starts-with(.,"'${p}'")] or '; done|sed '$ s/ or//g')"
NVDA_GPGPU_QUADRO_TAG_DEF='contains(//node[@id="display"]/vendor,"nVidia") and contains(//node[@id="display"]/product,"Quadro") and '"${NVDA_GPGPU_QUADRO_PROD_DEF}"''
NVDA_GPGPU_QUADRO_TAG_COMMENT="xpath auto-tag for nVidia Quadro-based general-purpose GPUs"
declare -ag NVDA_GPGPU_ALL_PRODUCTS=($(printf "%s\n" ${NVDA_GPGPU_TESLA_PRODUCTS[@]} ${NVDA_GPGPU_QUADRO_PRODUCTS[@]}|paste -sd" "))
NVDA_GPGPU_ALL_PROD_DEF="$(for p in ${NVDA_GPGPU_TESLA_PRODUCTS[@]} ${NVDA_GPGPU_QUADRO_PRODUCTS[@]};do printf '%s\n' '//node[@id="display"]/product[starts-with(.,"'${p}'")] or '; done|sed '$ s/ or//g')"
NVDA_GPGPU_ALL_TAG_DEF='contains(//node[@id="display"]/vendor,"nVidia") and contains(//node[@id="display"]/product,"Quadro") or contains(//node[@id="display"]/product,"Tesla") and '"${NVDA_GPGPU_ALL_PROD_DEF}"''
NVDA_GPGPU_ALL_TAG_COMMENT="xpath auto-tag for nVidia Quadro and Tesla-based general-purpose GPUs"
NVDA_GPGPU_ALL_KERNEL_OPTS="nomodeset modprobe.blacklist=nouveau modprobe.blacklist=nvidiafb intel_iommu=on iommu=pt rd.driver.pre=vfio-pci video=efifb:off vfio_iommu_type1.allow_unsafe_interrupts=1"
declare -ag GPGPU_ARR=(NVDA_3D_CONTROLLER NVDA_GPGPU_TESLA NVDA_GPGPU_QUADRO NVDA_GPGPU_ALL)
MAAS_TAGS=$(maas ${MAAS_PROFILE} tags read)
for G in ${GPGPU_ARR[@]};do
if [[ -n $(jq -r '.[]|select(.name=="'${G,,}'").name' <<< ${MAAS_TAGS}) ]];then
printf "\e[2G - Updating xpath auto tag for ${G} products...\n"
maas ${MAAS_PROFILE} tag update ${G,,} name="${G,,}" definition="$(eval echo \$${G}_TAG_DEF)" comment="$(eval echo \$${G}_TAG_COMMENT)"
[[ ${G} = NVDA_GPGPU_ALL || ${G} = NVDA_3D_CONTROLLER ]] && maas ${MAAS_PROFILE} tag update ${G,,} name="${G,,}" kernel_opts="${NVDA_GPGPU_ALL_KERNEL_OPTS}"
printf '\n\n'
else
printf "\e[2G - Creating xpath auto tag for ${G} products...\n\n"
maas ${MAAS_PROFILE} tags create name="${G,,}" definition="$(eval echo \$${G}_TAG_DEF)" comment="$(eval echo \$${G}_TAG_COMMENT)"
[[ ${G} = NVDA_GPGPU_ALL || ${G} = NVDA_3D_CONTROLLER ]] && maas ${MAAS_PROFILE} tag update ${G,,} name="${G,,}" kernel_opts="${NVDA_GPGPU_ALL_KERNEL_OPTS}"
printf '\n\n'
fi
done
fi
local CI_USER_DATA=$(cat <<EOF|tee /tmp/${FUNCNAME}.cloud-init.yaml|base64 -w0
#cloud-config-2
bootcmd:
- ip route add default via 172.27.21.254
final_message: KVM Pod Deployment Complete on \$(hostname -f)
timezone: ${CI_TZ}
locale: ${CI_LC}
apt:
proxy: http://${MAAS_IP}:8000/
primary:
- arches: [amd64]
uri: http://${CI_APT_MIRROR}/ubuntu
security:
- arches: [amd64]
uri: http://security.ubuntu.com/ubuntu
package_update: ${CI_PKG_UPD}
package_upgrade: ${CI_PKG_UPG}
${CI_PACKAGES}
${SSH_PUB_KEYS}
runcmd:
- set -x
- mkdir -p /home/virsh/bin
- ln -s /usr/bin/virsh /home/virsh/bin/virsh
- sh -c echo 'PATH=/home/virsh/bin' >> /home/virsh/.bashrc
- sh -c printf "Match user virsh\\n X11Forwarding no\\n AllowTcpForwarding no\\n PermitTTY no\\n ForceCommand nc -q 0 -U /var/run/libvirt/libvirt-sock\\n" >> /etc/ssh/sshd_config
- /usr/sbin/usermod --append --groups libvirt,libvirt-qemu virsh
- systemctl restart sshd
- /bin/sleep 10
- for i in \$(seq \$(find /dev -iname 'loop[0-9]*'|wc -l) 1 256);do mknod -m0660 /dev/loop\${i} b 7 \${i} && chown root.disk /dev/loop\${i};done
- export GPU_CMD='lspci -nn|awk -vIGNORECASE=1 '"'"'/\[03.*intel/{ORS=",";print substr(\$(NF-2),2,9)}'"'"'|sed "s/,$/\\n/"'
- export GPU_CONF_FILE='lspci -nn|awk -vIGNORECASE=1 '"'"'/\[03.*intel/{printf "export NVDA_GPU_%02d_ID=%s\nexport NVDA_GPU_%02d_BUS=%s\n",NR,substr(\$(NF-2),2,9),NR,\$1}'"'"'|tee -a /etc/gpgpu-pci-ids.conf'
- export GPU_PCI_LIST='lspci -nn|awk -vIGNORECASE=1 '"'"'BEGIN {printf "%s","options vfio-pci ids="} /\[03.*intel/{ORS=",";print substr(\$(NF-2),2,9)}'"'"'|sed "s/,$/\\n/"|tee /etc/modprobe.d/vfio.conf'
- if [ -n \$(eval \$GPU_CMD) ];then echo GPGPUs Detected;printf '%s\n' vfio vfio_pci > /etc/modules-load.d/vfio.conf;eval \$GPU_CONF_FILE;eval \$GPU_PCI_LIST;fi
- systemctl restart libvirtd
- zpool create local /dev/sdb
- zfs create local/maas
- zfs set dedup=on local
- zfs set compression=on local
- |-
virsh pool-define /dev/stdin <<EOF
<pool type="zfs">
<name>maas-zfs</name>
<source>
<name>local</name>
<device path="/dev/sdb"/>
</source>
</pool>
EOF
- virsh pool-start maas-zfs
- virsh pool-autostart maas-zfs
- touch /var/tmp/install-complete
EOF
)
#Remove old tags on redeployment
printf "\n\e[2GCleaning up any existing microcloud tags previously assigned host${W}...\n"
printf '%s\n' ${ALLOCATED_MACHINES[@]##*:}|xargs -n1 -P0 bash -c 'maas 2>/dev/null ${MAAS_PROFILE} machine read $0|jq 2>/dev/null -r '"'"'.tag_names[]|select(startswith("'${BASE_TAG}'"))'"'"''|xargs -n1 -P0 maas 2>/dev/null ${MAAS_PROFILE} tag delete &>/dev/null
local MICROCLOUD_TAG="${BASE_TAG}-$(grep -m1 -oaE '[a-z0-9]{6}' /dev/random|head -n1)"
printf "\n\e[2GCreating tag \"${MICROCLOUD_TAG}\" for this microcloud...\n"
maas ${MAAS_PROFILE} tags create name=${MICROCLOUD_TAG} comment="$(echo -n ${MICROCLOUD_TAG^^}|sed 's/-/_/g')_HOSTS=(`printf '%s\n' ${ALLOCATED_MACHINES[@]##*:}|paste -sd" "` ); $(echo -n ${MICROCLOUD_TAG^^}|sed 's/-/_/g')_BUILD_DATE=$(date --iso-8601=seconds)" &>/dev/null
printf "\e[2G- Tagging ${#ALLOCATED_MACHINES[@]} machines with \"${MICROCLOUD_TAG}\"...\n"
ADD=$(printf "add=%s\n" ${ALLOCATED_MACHINES[@]##*:}|paste -sd' ')
local ADDED=$(eval "maas ${MAAS_PROFILE} tag update-nodes ${MICROCLOUD_TAG} "$ADD""|jq -r .added)
[[ ${ADDED} -eq ${#ALLOCATED_MACHINES[@]} ]] && { printf "\e[4G - Successfully tagged ${ADDED}/${#ALLOCATED_MACHINES[@]} machines with the \"${MICROCLOUD_TAG}\" tag ${OK} "; } || { printf "\e[2G - Could only tag ${ADDED}/${#ALLOCATED_MACHINES[@]} machines with ${MICROCLOUD_TAG} ${FAILED} \n"; }
echo
printf "\n\e[2GDeploying the following host${W} to ${MICROCLOUD_TAG}:\n\n"
printf -- "\e[2G - %s\n" ${ALLOCATED_MACHINES[@]}|sed 's/:/ \(/g;s/.$/&\)/g';echo
local -a DEPLOYED_MACHINES=($(printf "%s\n" ${ALLOCATED_MACHINES[@]##*:}|xargs -I{} -n1 -P1 maas ${MAAS_PROFILE} machine deploy {} install_kvm=true distro_series=${DISTRO} user_data=${CI_USER_DATA}|jq 2>/dev/null -r '"\(.hostname):\(.system_id)"'))
printf "\n\e[4G - A copy of the cloud-init used for this deployment can be found at\n\e[2G/tmp/${FUNCNAME}.cloud-init.yaml\n\n"
microcloud-tag-cleanup ${BASE_TAG} ${MICROCLOUD_TAG}
}
export -f maas-microcloud-lxd
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment