Skip to content

Instantly share code, notes, and snippets.

@ThinGuy

ThinGuy/microclouds2.sh

Created January 16, 2019 18:07
Show Gist options
  • Save ThinGuy/b4e1922d8939653d72eaa1f30712a73b to your computer and use it in GitHub Desktop.
Save ThinGuy/b4e1922d8939653d72eaa1f30712a73b to your computer and use it in GitHub Desktop.
Download ZIP
Microcloud with virtip
Raw
microclouds2.sh
gen-sa-ip() {
gen-sa-ip_usage() { printf "\nFunction: ${FUNCNAME%_*}\n\nPurpose: Generates a random IP address using the self-assigned range (169.254.0.0-169.254.255.255)\n\n\tUsage: ${FUNCNAME%_*}\n\n" 1>&2;return; }
[[ $1 = '-h' || $1 = '--help' ]] && { ${FUNCNAME}_usage; return; }
printf '169.254.%01d.%01d\n' $((RANDOM%256)) $((RANDOM%256))
}
export -f gen-sa-ip
wrap() {
local DESC="${FUNCNAME}: Like fold, but with indents\n"
[[ $1 = '--desc' ]] && { printf "${DESC}";return; }
wrap_usage() {
printf "${DESC}\n"
printf "\e[2GUsage: ${FUNCNAME%%_*} [-i #] [-w #]\n"
printf "\e[4G -i, --indent NUM\e[20GNumber of spaces to indent (default 0)\n"
printf "\e[4G -w, --wrap NUM\e[20GColumn number to wrap text at (default 80)\n"
printf "\n\e[2GEx:\n"
printf "\e[4G <STDOUT>|${FUNCNAME%%_*} -i 5 -w 75 \n"
}
local INDENT=0
local WRAP=80
ARGS=`getopt -o w:i:h -l wrap:,indent:,help -n ${FUNCNAME} -- "$@"`
eval set -- "$ARGS"
while true ; do
case "$1" in
-i|--indent) local INDENT=${2};shift 2;;
-w|--wrap) local WRAP=${2};shift 2;;
-h|--help) ${FUNCNAME}_usage;return 2;;
--) shift;break;;
esac
done
fold -sw $((${WRAP}-${INDENT}))|sed 's/^./'$(printf "\e[${INDENT}G")'&/g'
}
export -f wrap
ul-msg() {
[[ $1 = '--desc' ]] && { printf "\e[2G${FUNCNAME}: Underlines the provided string with line style, indent, and bold options.\n";return; };
local INDENT= BOLD=
local STYLE=s
local UL=$(printf '\u2501')
ARGS=$(getopt -o dbi: --long double,bold,indent: -- "$@")
eval set -- "$ARGS"
while true; do
case "$1" in
-i | --indent) local INDENT='\e['${2}'G';shift 2;;
-b | --bold) local BOLD='\e[1m';shift 1;;
-d | --double) local STYLE=d;shift 1;;
--) shift;break;;
esac
done
[[ ${STYLE} = d ]] && local UL=$(printf '\u2550') || local UL=$(printf '\u2501')
printf "\n${BOLD}${INDENT}${@}";
local StrLen=$(($(echo ${@}|sed -E -e 's,\x1B\[[0-9;]*[a-zA-Z],,g;s/^[ \t]*|[ \t]*$//g'|wc -c)-2));
[[ ${@} =~ .*\\n$ || ${@} =~ .*\\n.$ ]] || echo;
printf "${BOLD}${INDENT}"
eval printf "%.3s" ${UL}{0..7};printf "\e[0m\n\n"
}
export -f ul-msg
spin-trap() { trap 'tput sgr0;tput cnorm; tput rmcup;eval kill -9 \$spid &>/dev/null;trap - INT TERM EXIT; return 0' INT TERM EXIT; }
export -f spin-trap
spin-trap-clear() { trap - INT TERM EXIT;tput sgr0;tput cnorm;kill -9 \$spid &>/dev/null;return; }
export -f spin-trap-clear
microcloud-tag-cleanup() {
local DESC="${FUNCNAME}: Removes tags that start with a given value from machines that not deployed, also deletes tag if not associated with any machines\n"
microcloud-tag-cleanup_usage() {
printf "\e[2G${DESC}\n"|wrap -i$((2+${#FUNCNAME[1]}))
printf "\e[2GUsage: ${#FUNCNAME[1]} -p <name> [-i <name> ]\n"
printf "\e[5G-p, --prefix\e[20GPerform cleanup on tags that start with provided prefix\n"
printf "\e[5G-i, --ignore\e[20GIgnore tags that start with the the provided prefix\n"
printf "\e[5G-h, --help\e[20GThis message\n"
printf "\n\e[2GEx:\n"
printf "\e[7G${#FUNCNAME[1]} -p east-edge -i east-edge-prod\n"
printf "\e[7GWill perform cleanup on all tags starting with east-edge, except for those that start with east-edge-prod\n"|wrap -i$((7+${#FUNCNAME[1]}))
printf "\n\n"
}
unset PTAG ITAG
ARGS=$(getopt -o p:i:h --long prefix:,ignore:,help,desc -- "$@")
eval set -- "$ARGS"
while true; do
case "$1" in
-p | --prefix) local PTAG=${2};shift 2;;
-i | --ignore) local ITAG=${2};shift 2;;
--desc) printf "\n\e[2G${DESC}\n";return 0;;
-h|--help) ${FUNCNAME}_usage;return 2;;
--) shift;break;;
esac
done
[[ -n $PTAG ]] || { printf "\e[2GNo tag prefix provided\n\n";${FUNCNAME}_usage;return 2; }
[[ -n $PTAG ]] && { printf "\e[6G - Cleaning up tags starting with: ${PTAG}\n"; }
[[ -n $ITAG ]] && { printf "\e[6G - Ignoring tags starting with: ${PTAG}\n"; }
maas 2>/dev/null ${MAAS_PROFILE} machines read|jq 2>/dev/null -r '.[]|"\(.tag_names|to_entries[]|select(.value|startswith("'${PTAG}'")).value) \(select(.status==4).system_id)"'|xargs -n2 -P1 bash -c 'maas 2>/dev/null ${MAAS_PROFILE} tag update-nodes $0 remove=$1|wrap -i 5 -w 80'
if [[ -n $ITAG ]];then
(maas 2>/dev/null ${MAAS_PROFILE} tags read |jq 2>/dev/null -r '.[]|select(.name|startswith("'${PTAG}'"))|select(.name|startswith("'${ITAG}'")|not).name')|(xargs -n1 -P0 bash -c '[[ -n $0 ]] && printf "$0 $(maas 2>/dev/null ${MAAS_PROFILE} tag nodes $0|jq '"'"'length==0'"'"')\n"|sed /false/d'|[[ -n $1 && -n $2 ]] && xargs -n2 -P1 bash -c 'printf -- "\e[3G - Deleting tag $0 (no machines assigned)\n";maas 2>/dev/null ${MAAS_PROFILE} tag delete $0|wrap -i 5 -w 80' )
else
(maas 2>/dev/null ${MAAS_PROFILE} tags read |jq 2>/dev/null -r '.[]|select(.name|startswith("'${PTAG}'")).name')|(xargs -n1 -P0 bash -c '[[ -n $0 ]] && printf "$0 $(maas 2>/dev/null ${MAAS_PROFILE} tag nodes $0|jq '"'"'length==0'"'"')\n"|sed /false/d'|[[ -n $1 && -n $2 ]] && xargs -n2 -P1 bash -c 'printf -- "\e[3G - Deleting tag $0 (no machines assigned)\n";maas 2>/dev/null ${MAAS_PROFILE} tag delete $|wrap -i 5 -w 80' )
fi
}
export -f microcloud-tag-cleanup
wait-pods-ready() {
local DESC="${FUNCNAME}: Waits in back until Pod hosts are deployed then performs final configuration \n"
wait-pods-ready_usage() {
printf "\e[2G${DESC}\n"|wrap -i $((2+${#FUNCNAME[1]}))
printf "\e[2GUsage: ${FUNCNAME[1]} -c <cpu overcommit> -m <memory overcommit> hostname:system_id pair (multiple pairs allowed)\n"
printf "\e[5G-c|--cpu-ocr\e[20GConfigure pods with given cpu overcommit ratio (Min:1.0 Max:10)\n\e[20Recommed Value: <= 10\n\n"
printf "\e[5G-m|--mem-ocr\e[20GConfigure pods with given memory overcommit ratio (Min:1.0 Max:10)\n\e[20Recommed Value: <= 1.5\n\n"
printf "\e[5G-t|--tag\e[20GMachines with this tag will be processes\n\e[20Recommed Value: <= 1.5\n\n"
printf "\e[5G-h, --help\e[20GThis message\n"
printf "\n\e[2GEx:\n"
printf "\e[7G${FUNCNAME} -c 10 -i 1.25 -t microcloud-kvm-xzypdq\n"|wrap -i$((7+${#FUNCNAME[1]}))
printf "\e[7GWill wait until all the machines associated with the give tag are in deployed state\n"
printf "\e[7Gthen will configure the cpu and memory overcommit.\n\m"
printf "\e[7GUsing the example values, 56 core/512 GiB RAM will have virtual 560 Cores and allow memory ballooning up to 640 GiB\n"|wrap -i$((7+${#FUNCNAME[1]}))
printf "\n\n"
}
ARGS=$(getopt -o c:m:t:hd --long tag:,cpu-ocr:,mem-ocr:,help,desc -- "$@")
eval set -- "$ARGS"
while true; do
case "$1" in
-c|-cpu-ocr) local CPU_OCR=${2};shift 2;;
-m|--mem-ocr) local MEM_OCR=${2};shift 2;;
-t|--tag) local TAG=${2};shift 2;;
--) shift;break;;
esac
done
#Remaining args are processed as hostname/system_id value pair. They should be presented in format of "hostname:system_id"
local -a TAGGED_MACHINES=($(maas admin tag nodes ${TAG}|jq -r '.[]|"\(.hostname):\(.system_id)"'))
[[ -z ${CPU_OCR} ]] && local CPU_OCR=1
[[ -z ${MEM_OCR} ]] && local MEM_OCR=1
# Check for integer and use bc to check ge/le of integers due to possible decimal points
[[ ${CPU_OCR} =~ ^[0-9]+[\.][0-9]+$|^[0-9]+ ]] || { printf "Expected a postive integer for CPU overcommit. Setting to 10\n";local CPU_OC=10; }
[[ ${MEM_OCR} =~ ^[0-9]+[\.][0-9]+$|^[0-9]+ ]] || { printf "Expected a positive integer for Memory overcommit. Setting to a safe 1.5\n";local MEM_OCR=1.5; }
[[ $(bc -l <<< "${CPU_OCR} <= 10") -eq 1 ]] || { printf "Maximum CPU overcommit is 10\n. Setting safe value of 4\n";local CPU_OC=4; }
[[ $(bc -l <<< "${MEM_OCR} <= 10") -eq 1 ]] || { printf "Maximum Memory overcommit is 10\n. Setting safe value of 1.25\n";local MEM_OCR=1.25; }
[[ $(bc -l <<< "${CPU_OCR} >= 1") -eq 1 ]] || { printf "Minimum CPU overcommit is 1 (no-overcommit). Setting to 1.\n";local CPU_OC=1; }
[[ $(bc -l <<< "${MEM_OCR} >= 1") -eq 1 ]] || { printf "Minimum Memory overcommit is 1 (no-overcommit). Setting to 1.\n";local MEM_OCR=1; }
local -a INSTALLED_MACHINES=($(sudo -u postgres psql -F":" --no-align -P pager=off -t maasdb -c "SELECT hostname,system_id from maasserver_node WHERE system_id SIMILAR TO '%($(printf '%s\n' ${TAGGED_MACHINES[@]##*:}|paste -sd\|))%' and status = 6 ORDER BY hostname"))
MSG="Waiting for ${#TAGGED_MACHINES[@]} machines ($(printf "%s\n" ${TAGGED_MACHINES[@]%%:*}|sort -uV|paste -sd,)) to finish deploying"
while [[ -n $(maas admin tag nodes ${TAG}|jq -r '.[].status_name'|grep -v Deployed) ]];do
MSG="Waiting for ${#TAGGED_MACHINES[@]} machines ($(maas admin tag nodes ${TAG}|jq -r '.[].hostname'|paste -sd,)) to finish deploying"
printf "${MSG}"
for i in {1..3..1};do
printf '\e[1m%0.1s\e[0m' .${i}
sleep .5
done
printf "\r${MSG}\e[K\r"
sleep 1
done
printf "\nInstallation Complete!Setting CPU and Memory over commit ratios on ($(printf "%s\n" ${TAGGED_MACHINES[@]%%:*}|sort -uV|paste -sd,))\n\n";sleep 1
spin-trap
query-pods() {
tput civis
declare -ag POD_ELEMENT_L1=('Pod Name' 'Pod ID' 'Hostname' 'System ID' 'Cores' 'CPU RO' 'CPU (GHz)' ' Mem (GiB)' 'Memory RO' 'Storage (GB)' 'Storage ID' 'Tags' 'Power Addr')
declare -ag POD_ELEMENT_L2=("$(printf '\u200A')" "$(printf '\u200A')" ' ' "$(printf '\u200A')" ' p/v' "$(printf '\u200A')" "$(printf '\u200A')" ' p/v' "$(printf '\u200A')" "$(printf '\u200A')" "$(printf '\u200A')" "$(printf '\u200A')" "$(printf '\u200A')")
export PI=$(printf "%s\n" "${POD_ELEMENT_L1[@]}"|paste -sd'|';printf "%s\n" "${POD_ELEMENT_L2[@]}"|paste -sd'|'; sudo -u postgres psql -F"|" --no-align -P pager=off -t maasdb -c "SELECT
bmc.name,bmc.id,hostname,system_id,concat_ws('/',bmc.cores,(bmc.cores*bmc.cpu_over_commit_ratio)),bmc.cpu_over_commit_ratio,bmc.cpu_speed::TEXT,concat_ws('/',round(bmc.memory/2^10),round((bmc.memory/2^10)*bmc.memory_over_commit_ratio)),bmc.cpu_over_commit_ratio,round(bmc.local_storage/2^30),
bmc.default_storage_pool_id,bmc.tags,bmc.power_parameters
FROM maasserver_bmc bmc LEFT JOIN maasserver_node node ON bmc.name = node.hostname WHERE cores > 0 and node.system_id SIMILAR TO '%($(printf '%s\n' ${INSTALLED_MACHINES[@]##*:}|paste -sd"|"))%' ORDER BY node.hostname"|sed -Er 's/."power_pass": .* |"power_address":/ /g;s/[",:]|\{ |.}$| //g;s/ |\| /|/g')
declare -ag PL=($(echo "${PI}"|awk -F"|" '{ for (i=1; i<=NF; i++) { max[i] = length($i) > max[i] ? length($i) : max[i] ;ncols = i > ncols ? i : ncols }} END { for (col=1; col <= ncols; col++) {printf "%d\n", max[col]}}'))
echo "$PI"|sed '3i'$(for l in ${PL[@]};do eval printf '%0.3s' "${LS}{0..${l}}";echo;done|paste -sd'|')''|column -nexts'|'|sed '1,3s/^.*$/'$(printf "\e[1;38;2;255;255;255m&\e[0m")'/;2s/^.*$/'$(printf "\e[38;2;175;175;175m&\e[0m")'/'
}
local -a POD_IDS=($(printf '%s\n' ${INSTALLED_MACHINES[@]##*:}|xargs -n1 -P1 bash -c 'maas ${MAAS_PROFILE} pods read|jq -r '"'"'.[]|select(.host.system_id=="'"'"'$0'"'"'").id'"'"''))
SET_OC=$(printf "%s\n" ${POD_IDS[@]}|xargs -n1 -P1 bash -c 'maas ${MAAS_PROFILE} pod update $0 memory_over_commit_ratio='${MEM_OCR}' cpu_over_commit_ratio='${CPU_OCR}'') &
export spid=$!
while kill -0 $spid 2>/dev/null;do
printf '\e[s'
query-pods
printf '\e[u'
sleep .25
done
wait $spid
spin-trap-clear
}
export -f wait-pods-ready
create-lxd-microcloud() {
local DESC="${FUNCNAME}: Deploy a LXD Cluster via MAAS command line"
#ENV Setttings
local CI_TZ='America/Los_Angeles'
local CI_LC='en_US.UTF-8'
#Package Settings
local CI_PKG_UPG=true
local CI_PKG_UPD=true
# Test for Ubuntu Country Mirror based on Locale
local CI_TC=${CI_LC:3:2}
if [[ -n ${CI_TC} && $(curl -slSL -w %{http_code} -o /dev/null ${CI_TC,,}.archive.ubuntu.com) -eq 200 ]];then
local CI_CC=${CI_TC}
local CI_APT_MIRROR=${CI_CC,,}.archive.ubuntu.com
else
local CI_APT_MIRROR=archive.ubuntu.com
fi
#Generic KeepAliveD Settings for a Cluster VIP using Keepalived
local VIP_VRRP_CIDR=
local VIP_VRRP_PASS="$(cat /dev/urandom|tr -dc 'a-z0-9'|fold -w6|head -n1)"
local VIP_VRRP_INSTANCE="MICROCLOUD-VIP"
local VIP_VRRP_ENABLE=false
#Defaults if not chosen
local ENABLE_GPU=false
local BASE_TAG="microcloud-lxd"
local STORAGE_POOL_DEVICE=
local STORAGE_POOL_TYPE=file
#Enforce 3 node min
local ENFORE_HA=true
local -a CI_PKG_LIST=(
bridge-utils
build-essential
jq
prips
squashfuse
unzip
zfsutils-linux
keepalived
)
create-lxd-microcloud_usage() {
printf "\n\e[2G${DESC}\n\n"
printf "\e[2G\e[1mUsage\e[0m: ${FUNCNAME%%_*} [-n <count>] [-t <tag>] [-d <distro>] ( -g gpu-passthrough support ) (-p storage-pool block device)\n\n"
printf "\e[4G -n, --count\e[20GNumber of nodes\n"
printf "\e[4G -t, --tag \e[20GExisting tag to use when selecting nodes\n"
printf "\e[4G -d, --distro\e[20GName of Ubuntu Distro (Default: bionic)\n"
printf "\e[4G -p, --pool-device\e[20GBlock device to use for Storage Pool (Optional: default uses file method)\n"
printf "\e[4G -g, --gpu\e[20GEnable GPU passthrough support\n"
printf "\e[4G -H, --ha\e[20GEnsure n+2 servers are available (default: True)\n"
printf "\e[4G -N, --no-ha\e[20GAllow less than n+2 servers in a cloud (default: False)\n"
printf "\e[4G -T, --cloud-name\e[20Name of cloud for tagging purposes (default: ${BASE_TAG}-xxxxxx)\n"
printf "\e[4G -c, --comment\e[20Comment to apply to tags related to the microcloud\n"
printf "\e[4G -v, --vrrp\e[20Enable a VIP managed by KeepAliveD (default: false)\n"
printf "\e[4G -V, --VIP\e[20IP Address to use for the VIP\n"
printf "\e[4G -i, --instance\e[20KeepAliveD instance name for VRRP_VIP service (default: MICROCLOUD_VIP)\n"
printf "\e[4G -P, --vrrp-password\e[20Password used by KeepAliveD to track state (default: Random 6 digit alpha numeric\n"
printf "\e[4G -h, --help\e[20GThis message\n"
printf "\n\n"
}
ARGS=$(getopt -o n:c:t:d:p:C:T:P:v:o:gdhHNV -l count:,tag:,distro:,pool-device:,comment:,cloud-name:.vip:,vrrp-password:,gpu,help,desc,enable-vrrp,ha,no-ha -n ${FUNCNAME} -- "$@")
eval set -- "$ARGS"
while true ; do
case "$1" in
-n|-c|--count) local COUNT=${2};shift 2;;
-t|--tag) local TAG=${2};local TAG=${TAG,,};shift 2;;
-d|--distro) local DISTRO=${2};local DISTRO=${DISTRO,,};shift 2;;
-p|--pool-device) local STORAGE_POOL_DEVICE=${2};local STORAGE_POOL_DEVICE=${STORAGE_POOL_DEVICE,,};local STORAGE_POOL_TYPE=device;shift 2;;
-g|--gpu) local ENABLE_GPU=true;shift 1;;
-H|--ha) local ENFORCE_HA=true;shift 1;;
-N|--no-ha) local ENFORCE_HA=false;shift 1;;
-C|--comment) local CLOUD_COMMENT=${2};shift 2;;
-T|--cloud-name) local CLOUD_NAME=${2};shift 2;;
-V|--enable-vrrp) local VIP_VRRP_ENABLE=true;shift 1;;
-v|--vip) local VIP_VRRP_CIDR=${2};shift 2;;
-i|--vrrp-instance) local VIP_VRRP_INSTANCE=${2};shift 2;;
-P|--vrrp-password) local VIP_VRRP_PASS=${2};shift 2;;
--desc) printf "\n\e[2G${DESC}\n";return 2;;
-h|--help) ${FUNCNAME}_usage;return 2;;
--) shift;break;;
esac
done
command -v maas &>/dev/null || { printf "\n\n\e[2G\e[2G\e[38;2;255;255;0mSorry! \e[0m\e[38;2;0;255;0m maas-cli \e[0m\e[1m2.5+\e[0m is required for this demo.\n\e[9GPlease install and configure version 2.5 or later\n\n";return 1; }
[[ $(dpkg-query -s maas|grep -oP '(?<=Version: )[^?]{3}'|sed 's/\.//') -lt 25 ]] && { printf "\n\n\e[2G\e[2G\e[38;2;255;255;0mSorry! \e[0mMAAS \e[38;2;0;255;0m2.5\e[0m\e[1m+\e[0m is required for this demo.\n\e[9GYou are running version \e[38;2;255;255;0m$(dpkg-query -s maas|grep -oP '(?<=Version: )[^~]+')\e[0m\n\n";return 1; }
command -v jq &>/dev/null || { printf "Installing pre-req utils...";sudo apt install jq -yqq|tee 1>/dev/null /tmp/${FUNCNAME}.log; }
local OK='\u00A0\e[38;2;0;255;0m\u2713\e[0m\u00A0\n'
local FAILED='\u00A0\e[38;2;255;0;0m\u2718\e[0m\u00A0\n'
local ERR_MSG='\e[0m\e[1;38;2;255;255;255m\e[1;48;2;255;0;0m***\ufeffERROR\ufeff***\e[0m'
local WRN_MSG='\e[0m\e[1;38;2;255;255;255m\e[1;48;2;255;255;0m***\ufeffERROR\ufeff***\e[0m'
[[ -z ${MAAS_PROFILE} ]] && { read -srp "$(printf "Please enter your MAAS profile name: ")" MAAS_PROFILE_INPUT;local MAAS_PROFILE=$MAAS_PROFILE_INPUT; }
printf -- "\n\e[1mMicro-cloud Demo - LXD\e[0m\n\n"
printf -- "\e[3G- Provisioning by \e[1mMAAS $(dpkg-query -s maas|grep -oP '(?<=Version: )[^~]+')\e[0m${OK}\n"
printf -- "\e[6G- Powered by \e[1m$(lsb_release -ds)\e[0m${OK}\n"
printf -- "\e[3G- Featuring \e[1mLXD $(snap info lxd|awk '/- </{print $2}') with clustering\e[0m${OK}\n"
[[ ${ENABLE_GPU} = true ]] && printf -- "\e[3G- \e[1m GPGPU\e[0m Passthrough enabled\e[0m${OK}\n"
#Validation and error handling
local -a VALID_TAGS=($(maas ${MAAS_PROFILE} tags read|jq -r '.[].name'))
local -a VALID_DISTROS=($(maas ${MAAS_PROFILE} boot-resources read|jq -r '.[]|"\(select((.name|startswith("grub")|not) and (.name|startswith("ubuntu")) and (.name|startswith("pxe")|not)).name|sub("ubuntu/"; ""))"'|sort -uV))
if [[ ${VIP_VRRP_ENABLE} = true ]];then
printf -- "\e[3GSimple Virtual IP Address Failover enabled\n"
[[ -n ${VIP_VRRP_CIDR} ]] && { printf -- "\e[5G- Virtual IP: ${VIP_VRRP_CIDR}\n"; } || { local VIP_VRRP_CIDR_=$(gen-sa-ip);printf -- "\e[5G- VIP_VRRP_CIDR: ${VIP_VRRP_CIDR} (auto-generated)\n"; }
[[ -n ${VIP_VRRP_INSTANCE} ]] && { printf -- "\e[5G- VRRP Instance Name: ${VIP_VRRP_INSTANCE}}\n"; } || { local VIP_VRRP_INSTANCE="VIP-$(cat /dev/urandom|tr -dc 'A-Z0-9'|fold -w6|head -n1)";printf -- "\e[5G- VRRP INSTANCE: ${VIP_VRRP_INSTANCE} (auto-generated)\n"; }
[[ -n ${VIP_VRRP_PASS} ]] && { printf -- "\e[5G- VRRP Passwd: ${VIP_VRRP_PASS}\n"; } || { local VIP_VRRP_PASS="$(cat /dev/urandom|tr -dc 'a-z0-9'|fold -w6|head -n1)";printf -- "\e[5G- VRRP Passwd: ${VIP_VRRP_PASS} (auto-generated)\n"; }
fi
# If not a physical disk, set storage pool to use a file
[[ ${STORAGE_POOL_TYPE} = device ]] && { printf "\e[3G- Storage Pool will use block device ${STORAGE_POOL_DEVICE} \n\n"; local STORAGE_POOL_TYPE=device; } || { local STORAGE_POOL_TYPE=file;local STORAGE_POOL_DEVICE="/var/snap/lxd/common/lxd/disks/local.img";printf "\e[3G- Storage Pool on Cluster Hosts will be file-based \n\n"; }
[[ ${COUNT} =~ ^[0-9]+$ ]] || { local COUNT=3;printf "\e[3G- Setting Node count to ${COUNT}\n\n"; }
[[ ${COUNT} =~ ^[0-9]+$ && ${COUNT} -lt 3 ]] && { local COUNT=3;printf "\e[3G- Minimum LXD Cluster node count is ${COUNT}. Setting Node count to ${COUNT}.\n\n"; }
[[ -z ${TAG} ]] && { printf "\e[2GERROR: No tag name given ${TAG}\n\n\e[2GValid tags are:\n";printf "\e[4G- %s\n" ${VALID_TAGS[@]};return 1; }
[[ -n ${TAG} && -n $(grep -P '(^|\s)\K'${TAG}'(?=\s|$)' <<< ${VALID_TAGS[@]}) ]] || { printf "\e[2GERROR: No tags exist named ${TAG}\n\n\e[2GValid tags are:\n";printf "\e[4G- %s\n" ${VALID_TAGS[@]};return 1; }
[[ -n ${TAG} ]] && { printf "\e[3G- Using tag \"${TAG}\" as primary machine constraint\n\n"; }
[[ -n ${DISTRO} ]] || { local DISTRO="bionic"; }
[[ -n ${DISTRO} && -n $(grep -P '(^|\s)\K'${DISTRO}'(?=\s|$)' <<< ${VALID_DISTROS[@]}) ]] || { printf "\e[2GERROR: invalid distro: ${DISTRO}\n\n\e[2GValid distros for LXD Clusters are:\n";printf "\e[4G- %s\n" ${VALID_DISTROS[@]};return 1; }
[[ -n ${DISTRO} ]] && { printf "\e[3G- Setting distro to ${DISTRO}\n\n"; }
printf "\n\e[3G- Gathering values from to use for setting up the lxd-cluster. Please wait..."
#Get values we'll be replacing in lxd-cluster script template
local MAAS_LIST=$(maas list|awk '/'${MAAS_PROFILE}'/')
local MAAS_API=$(echo ${MAAS_LIST}|awk '{print $3}')
local MAAS_IP=$(echo ${MAAS_LIST}|awk -F'(//|:)' '{print $3}')
local MAAS_URL=$(echo ${MAAS_LIST}|awk '{gsub(/\/api.*/,"");print $2}')
local MAAS_SUBNET=$(maas ${MAAS_PROFILE} subnets read|jq -r '.[]|select(.vlan.fabric_id==0).cidr')
local MAAS_DOMAIN=$(maas ${MAAS_PROFILE} domains read|jq -r '.[]|select((.authoritative==true) and .id==0).name')
IFS=$'\n' && declare -ag MAAS_SSH_HOST_KEYS=($(maas ${MAAS_PROFILE} sshkeys read|jq -r '.[]|"\(.key)"'))
if [[ -n ${MAAS_API} && -n ${MAAS_URL} && -n ${MAAS_SUBNET} && -n ${MAAS_IP} && -n ${MAAS_DOMAIN} ]];then
printf "${OK}"
else
printf "${FAILED}\e[2GThere was an issue gathering lxd-cluster values from maas. Make you are running this while logged into the maas-cli. Quitting\n\n"
echo
for v in API URL SUBNET IP DOMAIN;do eval echo MAAS_$v=\$MAAS_$v;done
echo
return 1
fi
[[ ${COUNT} -eq 1 ]] && W= || W=s
if [[ ${STORAGE_POOL_TYPE} = device ]];then
printf "\n\e[3G- Finding ${COUNT} machine${W} :\n\e[5G- Marked as \"Ready\"\n\e[5G- Tagged with \"${TAG}\"\n"
local READY_TAGGED_MACHINES=($(maas ${MAAS_PROFILE} machines read|jq -r '.[]|select(.physicalblockdevice_set[].name|contains("'${STORAGE_POOL_DEVICE##*/}'"))|select((.tag_names[]|contains("'"${TAG}"'")) and .status == 4)|"\(.hostname):\(.system_id)"'|head -n${COUNT}|sort -V))
[[ ${#READY_TAGGED_MACHINES[@]} -lt 3 && ${ENFORCE_HA} = true ]] && { printf "\e[2G\e[2G\e[38;2;255;255;0mSorry! \e[0mCould not find enough machines to create a LXD Cluster.\n\n\e[9GMachines Required: 3\n\e[9GMachines Found:\e[4C\e[38;2;255;0;0m${#READY_TAGGED_MACHINES[@]}\n\n";return 1; }
[[ ${#READY_TAGGED_MACHINES[@]} -lt ${COUNT} && ${ENFORCE_HA} = true && ${#READY_TAGGED_MACHINES[@]} -ge 3 ]] && { read -erp "$(printf "\n\e[2G - Could only find ${#READY_TAGGED_MACHINES[@]} machine${W} with the ${TAG} tag. Continue? [y/n] : ")" CONT; [[ ${CONT,,} =~ y ]] && echo || return 0; }
[[ ${#READY_TAGGED_MACHINES[@]} -lt ${COUNT} && ${ENFORCE_HA} = false ]] && { read -erp "$(printf "\n\e[2G - Could only find ${#READY_TAGGED_MACHINES[@]} machines with the ${TAG} tag. Continue? [y/n] : ")" CONT; [[ ${CONT,,} =~ y ]] && echo || return 0; }
local COUNT=${#READY_TAGGED_MACHINES[@]}
else
printf "\n\e[3G- Finding ${COUNT} machine${W} :\n\e[5G- Marked as \"Ready\"\e[5G- Tagged with \"${TAG}\"\n\e[5G- Has physical disk named ${STORAGE_POOL_DEVICE}\n"
local -a READY_TAGGED_MACHINES=($(maas ${MAAS_PROFILE} machines read|jq -r '.[]|select((.tag_names[]|contains("'"${TAG}"'")) and .status == 4)|"\(.hostname):\(.system_id)"'|head -n${COUNT}|sort -V))
[[ ${#READY_TAGGED_MACHINES[@]} -lt 3 && ${ENFORCE_HA} = true ]] && { printf "\e[2G\e[2G\e[38;2;255;255;0mSorry! \e[0mCould not find enough machines to create a LXD Cluster.\n\n\e[9GMachines Required: 3\n\e[9GMachines Found:\e[4C\e[38;2;255;0;0m${#READY_TAGGED_MACHINES[@]}\n\n";return 1; }
[[ ${#READY_TAGGED_MACHINES[@]} -lt ${COUNT} && ${ENFORCE_HA} = true && ${#READY_TAGGED_MACHINES[@]} -ge 3 ]] && { read -erp "$(printf "\n\e[2G - Could only find ${#READY_TAGGED_MACHINES[@]} machine${W} with the ${TAG} tag. Continue? [y/n] : ")" CONT; [[ ${CONT,,} =~ y ]] && echo || return 0; }
[[ ${#READY_TAGGED_MACHINES[@]} -lt ${COUNT} && ${ENFORCE_HA} = false ]] && { read -erp "$(printf "\n\e[2G - Could only find ${#READY_TAGGED_MACHINES[@]} machine${W} with the ${TAG} tag. Continue? [y/n] : ")" CONT; [[ ${CONT,,} =~ y ]] && echo || return 0; }
local COUNT=${#READY_TAGGED_MACHINES[@]}
fi
printf "\n\e[3G- Allocating the following Host${W} to the LXD Pods:\n"
printf -- "\e[6G - %s\n" ${READY_TAGGED_MACHINES[@]}|sed 's/:/ \(/g;s/.$/&\)/g';echo
local -a ALLOCATED_MACHINES=($(printf "%s\n" ${READY_TAGGED_MACHINES[@]##*:}|xargs -I{} -n1 -P0 maas ${MAAS_PROFILE} machines allocate system_id={}|jq -r '"\(.hostname):\(.system_id)"'|sort -uV))
[[ ${#ALLOCATED_MACHINES[@]} -lt 3 && ${ENFORCE_HA} = true ]] && { printf "\e[2G\e[2G\e[38;2;255;255;0mSorry! \e[0mCould not allocate enough machines to create a LXD Cluster.\n\n\e[9GMachines Required: 3\n\e[9GMachines Found:\e[4C\e[38;2;255;0;0m${#ALLOCATED_MACHINES[@]}\n\n";return 1; }
[[ ${#ALLOCATED_MACHINES[@]} -lt ${COUNT} && ${ENFORCE_HA} = true && ${#ALLOCATED_MACHINES[@]} -ge 3 ]] && { read -erp "$(printf "\n\e[2G - Could only allocate ${#ALLOCATED_MACHINES[@]} machine${W}. Continue? [y/n] : ")" CONT; [[ ${CONT,,} =~ y ]] && echo || return 0; }
[[ ${#ALLOCATED_MACHINES[@]} -lt ${COUNT} && ${ENFORCE_HA} = false ]] && { read -erp "$(printf "\n\e[2G - Could only allocate ${#ALLOCATED_MACHINES[@]} machines. Continue? [y/n] : ")" CONT; [[ ${CONT,,} =~ y ]] && echo || return 0; }
local COUNT=${#ALLOCATED_MACHINES[@]}
# The first element in the array will be the Cluster Primary
#
local CLOUD_PRIMARY=${ALLOCATED_MACHINES[0]%%:*}
printf "\e[3G- First cluster member will be ${CLOUD_PRIMARY}\n\n"
# Preconfigure the system keys in /etc/ssh so we can setup the cluster without any prompts or exchanging ssh keys
local -a KEY_TYPES=(rsa dsa ecdsa)
printf "\e[3G- Creating pre-generated ssh private keys \($(printf '%s\n' ${KEY_TYPES[@]}|paste -sd',') \) for the cluster\n"
xargs -n1 -P0 bash -c 'ssh-keygen &>/dev/null -t $0 -f /tmp/lxd_cluster_ssh_host_$0_key -P ""' <<< ${KEY_TYPES[@]}
local SSH_HOST_KEYS="$(printf "\nssh_keys:\n";xargs -n1 -P1 bash -c 'printf " $0_private: |\n$(cat /tmp/lxd_cluster_ssh_host_$0_key|sed '"'"'s/^.*$/ &/g'"'"')\n\n $0_public: $(cat /tmp/lxd_cluster_ssh_host_$0_key.pub)\n\n"' <<< ${KEY_TYPES[@]})"
#Remove preseeded ssh keys from local machine
printf "\e[5G- Deleting local copies of pregenerated keys \n\n"
rm -rf /tmp/lxd_cluster_ssh_host* /tmp/lxd_cluster_user_ssh*
# Get SSH Public keys from local accounts and maas. Remove duplicates
printf "\e[3G- Looking for public keys to add to LXD profiles on Cluster\n"
local SSH_PUB_KEYS=$((if [[ -n $(find 2>/dev/null ~/.ssh -iname "*.pub") ]];then
echo ssh_authorized_keys:
find 2>/dev/null ~/.ssh -iname "*.pub"|xargs -n1 -P1 bash -c 'printf -- '"'"' - %s\n'"'"' "$(cat $0)"'
fi
if [[ -n $(command -v maas 2>/dev/null) ]];then
if [[ $(maas 2>/dev/null ${MAAS_PROFILE} sshkeys read|jq 2>/dev/null length) -ge 1 ]];then
echo "ssh_authorized_keys:"
maas 2>/dev/null ${MAAS_PROFILE} sshkeys read|jq 2>/dev/null -r '.[]|"\(.key)"'| \
while IFS= read -r line;do
sed 's/^/ - /g'
done
fi
fi
)|awk '!seen[$0]++')
echo
# Create correct the stanza for disk or file based storage pool
[[ ${STORAGE_POOL_TYPE} = device ]] && local STORAGE_POOL_CONF=$(cat <<EOF
storage_pools:
- config:
source: ${STORAGE_POOL_DEVICE}
description: LXD Storage Pool on ${STORAGE_POOL_DEVICE,,}
name: local
driver: zfs
- config:
size: 100GB
source: /var/snap/lxd/common/lxd/disks/local.img
zfs.pool_name: default
description: File Based LXD Storage Pool
name: default
driver: zfs
EOF
)
[[ ${STORAGE_POOL_TYPE} != device ]] && local STORAGE_POOL_CONF=$(cat <<EOF
storage_pools:
- config:
size: 100GB
source: /var/snap/lxd/common/lxd/disks/local.img
zfs.pool_name: local
description: File Based LXD Storage Pool
name: local
driver: zfs
EOF
)
#Indent yaml for use in LXD preeseed yaml
local LXD_SSH_PUB_KEYS=$(echo;echo "${SSH_PUB_KEYS}"|sed 's/^/ /g')
# Create a string of the Cluster hosts that we can for SSH config files (thus the added asterisk)
# (easier to remove the asterisk in a variable than to add it, so it's added to main variable)
local CLUSTER_HOST_LIST=$(printf "%s*\n" ${ALLOCATED_MACHINES[@]%%:*}|paste -sd" "|sort -uV)
# Create Package List for Cluster hosts
local CI_PACKAGES=$([[ ${#CI_PKG_LIST[@]} -ge 1 ]] && printf "packages: [$(printf '%s\n' ${CI_PKG_LIST[@]}|paste -sd',')]\n")
# xpath auto tags for varius different nVidia GPGPUs
if [[ ${ENABLE_GPU} = true ]];then
printf "\e[3G- Configuring xpath auto-tags for GPGPU Detection at the hardware level in MAAS...\n\n"
NVDA_3D_CONTROLLER_TAG_DEF='//node[@id="display"]/description = "3D controller" and //node[@id="display"]/vendor = "NVDA Corporation" and //node[@id="display"]/product = "NVIDIA Corporation"'
NVDA_3D_CONTROLLER_TAG_COMMENT='xpath auto-tag to prevent nvidia general-purpose GPUs that have been claimed by nouveau or nvidiafb module'
declare -ag NVDA_GPGPU_TESLA_PRODUCTS=(P4 P40 P100 V100 T4)
NVDA_GPGPU_TESLA_PROD_DEF="$(for p in ${NVDA_GPGPU_TESLA_PRODUCTS[@]};do printf '%s\n' '//node[@id="display"]/product[starts-with(.,"'${p}'")] or '; done|sed '$ s/ or//g')"
NVDA_GPGPU_TESLA_TAG_DEF='contains(//node[@id="display"]/vendor,"nVidia") and contains(//node[@id="display"]/product,"Tesla") and '"${NVDA_GPGPU_TESLA_PROD_DEF}"''
NVDA_GPGPU_TESLA_TAG_COMMENT="xpath auto-tag for nVidia Tesla-based general-purpose GPUs"
declare -ag NVDA_GPGPU_QUADRO_PRODUCTS=(K M P2000 P4000 P5000 P6000)
NVDA_GPGPU_QUADRO_PROD_DEF="$(for p in ${NVDA_GPGPU_QUADRO_PRODUCTS[@]};do printf '%s\n' '//node[@id="display"]/product[starts-with(.,"'${p}'")] or '; done|sed '$ s/ or//g')"
NVDA_GPGPU_QUADRO_TAG_DEF='contains(//node[@id="display"]/vendor,"nVidia") and contains(//node[@id="display"]/product,"Quadro") and '"${NVDA_GPGPU_QUADRO_PROD_DEF}"''
NVDA_GPGPU_QUADRO_TAG_COMMENT="xpath auto-tag for nVidia Quadro-based general-purpose GPUs"
declare -ag NVDA_GPGPU_ALL_PRODUCTS=($(printf "%s\n" ${NVDA_GPGPU_TESLA_PRODUCTS[@]} ${NVDA_GPGPU_QUADRO_PRODUCTS[@]}|paste -sd" "))
NVDA_GPGPU_ALL_PROD_DEF="$(for p in ${NVDA_GPGPU_TESLA_PRODUCTS[@]} ${NVDA_GPGPU_QUADRO_PRODUCTS[@]};do printf '%s\n' '//node[@id="display"]/product[starts-with(.,"'${p}'")] or '; done|sed '$ s/ or//g')"
NVDA_GPGPU_ALL_TAG_DEF='contains(//node[@id="display"]/vendor,"nVidia") and contains(//node[@id="display"]/product,"Quadro") or contains(//node[@id="display"]/product,"Tesla") and '"${NVDA_GPGPU_ALL_PROD_DEF}"''
NVDA_GPGPU_ALL_TAG_COMMENT="xpath auto-tag for nVidia Quadro and Tesla-based general-purpose GPUs"
NVDA_GPGPU_ALL_KERNEL_OPTS='nomodeset modprobe.blacklist=nouveau modprobe.blacklist=nvidiafb intel_iommu=on iommu=pt rd.driver.pre=vfio-pci video=efifb:off vfio_iommu_type1.allow_unsafe_interrupts=1'
declare -ag GPGPU_ARR=(NVDA_3D_CONTROLLER NVDA_GPGPU_TESLA NVDA_GPGPU_QUADRO NVDA_GPGPU_ALL)
MAAS_TAGS=$(maas ${MAAS_PROFILE} tags read)
for G in ${GPGPU_ARR[@]};do
if [[ -n $(jq -r '.[]|select(.name=="'${G,,}'").name' <<< ${MAAS_TAGS}) ]];then
printf "\e[5G - Updating xpath auto tag for ${G} products...\n"
maas ${MAAS_PROFILE} tag update ${G,,} name=${G,,} definition=$(eval echo \$${G}_TAG_DEF) comment=$(eval echo \$${G}_TAG_COMMENT)|wrap -i8 -w80
[[ ${G} = NVDA_GPGPU_ALL || ${G} = NVDA_3D_CONTROLLER ]] && maas ${MAAS_PROFILE} tag update ${G,,} name=${G,,} kernel_opts="${NVDA_GPGPU_ALL_KERNEL_OPTS}"|wrap -i8 -w80
printf '\n\n'
else
printf "\e[5G - Creating xpath auto tag for ${G} products...\n\n"
maas ${MAAS_PROFILE} tags create name=${G,,} definition=$(eval echo \$${G}_TAG_DEF) comment=$(eval echo \$${G}_TAG_COMMENT)|wrap -i8 -w80
[[ ${G} = NVDA_GPGPU_ALL || ${G} = NVDA_3D_CONTROLLER ]] && maas ${MAAS_PROFILE} tag update ${G,,} name=${G,,} kernel_opts="${NVDA_GPGPU_ALL_KERNEL_OPTS}"|wrap -i8 -w80
printf '\n\n'
fi
done
fi
#Make LXD Image Script to preseed LXD Clusters image store
printf "\e[3G- Writing LXD Image Copy Script to Cloud-Init file...\n\n"
local LXD_IMAGE_SCRIPT=$(
cat <<EOF|base64 -w0
#!/bin/bash
printf "\e[1mLXD Image Copy Tool\e[0m\n"
#Add ubuntu minimal remotes if they are not defined
printf "\e[2GChecking if ubuntu-minimal repos have been added...\n"
[[ -n \$(/snap/bin/lxc 2>/dev/null remote list|awk '/ubuntu.com\/minimal\/daily/{print $2}') ]] || { printf "\e[2G - Adding ubuntu-minimal-daily repo\n";/snap/bin/lxc remote add ubuntu-min-daily https://cloud-images.ubuntu.com/minimal/daily --protocol simplestreams --public; }
[[ -n \$(/snap/bin/lxc 2>/dev/null remote list|awk '/ubuntu.com\/minimal\/releases/{print $2}') ]] || { printf "\e[2G - Adding ubuntu-minimal-releases repo\n";/snap/bin/lxc remote add ubuntu-min-releases https://cloud-images.ubuntu.com/minimal/releases --protocol simplestreams --public; }
printf "\e[2GCreating array of simplestreams-based remotes\n"
declare -ag REPOS=(\$(/snap/bin/lxc remote list --format=json|jq -r 'to_entries[]|select((.key|startswith("fcb")|not) and .value.Protocol=="simplestreams").key'))
printf "\e[2GCreating array of unique LXD images from ${#REPOS[@]} repos...\n"
declare -ag IMAGES=(\$(xargs -n1 -P0 bash -c '/snap/bin/lxc image list \${0}: a=amd64 --format=json|jq -r '"'"'.[]|select(.aliases!=null)|"'"'"'\$0'"'"':\(.fingerprint|sub("\\\\s.*"; ""))|'"'"'\$0-'"'"'\(.properties.os|ascii_downcase)-\(.properties.release|ascii_downcase)"'"'"'' <<< \${REPOS[@]}|\
sed \
-e 's/ubuntu\(-min-daily\)-ubuntu-\(.*\)/\2\1/' \
-e 's/ubuntu\(-min\)-ubuntu-\(.*\)/\2\1/' \
-e 's/ubuntu\(-daily\)-ubuntu-\(.*\)/\2\1/' \
-e '/images-ubuntu-.*[a-z]$/d' \
-e 's/ubuntu-ubuntu-/ubuntu-r-/g' \
-e 's/ubuntu\(-r\)-\(.*\)/\2\1/' \
-e 's/-daily$/-d/g' \
-e 's/images-//g' \
-e 's/-current//g' \
-e 's/min$/&-r/g'))
printf "\e[2G - Discovered \${#IMAGES[@]} unique LXD OS images\n\n\e[4G - Note:\n\e[6G - aliases that end with \"-r\" are \"release\" builds (aka GA)\n\e[6G - aliases that end with \"-d\" are \"daily\" builds\n\e[2GCommencing downloads...\n\n"
for ((i=0; i<\${#IMAGES[@]}; i++));do printf '%s %s\n' \${IMAGES[i]%%|*} \${IMAGES[i]##*|};done|xargs -n2 -P0 bash -c '/snap/bin/lxc image copy \$0 --alias \$1 --public --auto-update local:'
EOF
)
[[ ${VIP_VRRP_ENABLE} = true ]] && printf "\e[3G- Writing KeepAliveD configuration Script to Cloud-Init file...\n\n"
[[ ${VIP_VRRP_ENABLE} = true ]] && local VIP_VRRP_SCRIPT=$(
cat <<EOS|base64 -w0
#!/bin/bash
#root check
if [[ \$EUID -ne 0 ]]; then
printf '\n\e[1;33mThis script requires admin privileges.\e[0m\e[1;37m Please run via sudo.\e[0m\n\n'
exit 1
fi
if [[ \$(hostname 2>/dev/null -s) = ${CLOUD_PRIMARY} ]];then
cat <<EOM|/etc/keepalived/keepalived.conf
global_defs {
notification_email {
root@\$(hostname -d)
}
notification_email_from \$(hostname -f)
smtp_server localhost
smtp_connect_timeout 30
}
vrrp_instance ${VIP_VRRP_INSTANCE} {
state MASTER
# Specify the network interface to which the virtual address is assigned
interface \$(ip route get ${MAAS_IP}|grep -oP '(?<=dev )[^ ]+')
# The virtual router ID must be unique to each VRRP instance that you define
virtual_router_id 41
# Set the value of priority higher on the master server than on a backup server
priority 200
advert_int 1
authentication {
auth_type PASS
auth_pass ${VIP_VRRP_PASS}
}
virtual_ipaddress {
$VIP_VRRP_CIDR
}
}
EOM
else
cat <<EOB|/etc/keepalived/keepalived.conf
global_defs {
notification_email {
root@\$(hostname -d)
}
notification_email_from \$(hostname -f)
smtp_server localhost
smtp_connect_timeout 30
}
vrrp_instance ${VIP_VRRP_INSTANCE} {
state BACKUP
# Specify the network interface to which the virtual address is assigned
\$(ip route get ${MAAS_IP}|grep -oP '"'"'(?<=dev )[^ ]+'"'"')
virtual_router_id 41
# Set the value of priority lower on the backup server than on the master server
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass ${VIP_VRRP_PASS}
}
virtual_ipaddress {
${VIP_VRRP_CIDR}
}
}
EOB
fi
EOS
)
#Make LXD Cluster Script and convert to base64 to include in cloud-init user-data
printf "\e[3G- Writing LXD Cluster Script to Cloud-Init file...\n\n"
local LXD_CLUSTER_SCRIPT=$(
cat <<EOF|base64 -w0
#!/bin/bash
if [[ \$(hostname 2>/dev/null -s) = ${CLOUD_PRIMARY} ]];then
cat <<EOT|sudo lxd init --preseed
config:
core.https_address: \$(dig +short @${MAAS_IP} \$(hostname -f)):8443
core.trust_password: ubuntu
maas.api.key: ${MAAS_API}
maas.api.url: ${MAAS_URL}
networks:
- config:
ipv4.address: auto
ipv4.nat: 'true'
ipv6.address: auto
ipv6.nat: 'true'
name: lxdbr0
type: bridge
${STORAGE_POOL_CONF}
profiles:
- name: default
description: 'Default LXD Profile'
config:
user.user-data: |
#cloud-config
${LXD_SSH_PUB_KEYS}
devices:
eth0:
name: eth0
nictype: bridged
parent: lxdbr0
type: nic
root:
path: /
pool: local
type: disk
- name: privileged
description: 'Privileged LXD Profile'
config:
linux.kernel_modules: ip_tables,ip6_tables,netlink_diag,nf_nat,overlay
migration.incremental.memory: 'true'
raw.lxc: |-
lxc.cgroup.devices.allow = c 10:237 rwm
lxc.apparmor.profile = unconfined
lxc.cgroup.devices.allow = b 7:* rwm
security.nesting: 'true'
security.privileged: 'true'
user.user-data: |
#cloud-config
${LXD_SSH_PUB_KEYS}
devices:
eth0:
name: eth0
nictype: bridged
parent: lxdbr0
type: nic
root:
path: /
pool: local
type: disk
kvm:
path: /dev/kvm
type: unix-char
mem:
path: /dev/mem
type: unix-char
loop-control:
path: /dev/loop-control
type: unix-char
loop0:
path: /dev/loop0
type: unix-block
loop1:
path: /dev/loop1
type: unix-block
loop2:
path: /dev/loop2
type: unix-block
loop3:
path: /dev/loop3
type: unix-block
loop4:
path: /dev/loop4
type: unix-block
loop5:
path: /dev/loop5
type: unix-block
loop6:
path: /dev/loop6
type: unix-block
loop7:
path: /dev/loop7
type: unix-block
- name: maas
description: 'MAAS LXD Profile'
config:
user.user-data: |
#cloud-config
${LXD_SSH_PUB_KEYS}
devices:
eth0:
maas.subnet.ipv4: ${MAAS_SUBNET}
name: eth0
nictype: bridged
parent: br0
type: nic
root:
path: /
pool: local
type: disk
- name: maas-privileged
description: 'MAAS Privileged LXD Profile'
config:
linux.kernel_modules: ip_tables,ip6_tables,netlink_diag,nf_nat,overlay
migration.incremental.memory: 'true'
raw.lxc: |-
lxc.cgroup.devices.allow = c 10:237 rwm
lxc.apparmor.profile = unconfined
lxc.cgroup.devices.allow = b 7:* rwm
security.nesting: 'true'
security.privileged: 'true'
user.user-data: |
#cloud-config
${LXD_SSH_PUB_KEYS}
devices:
eth0:
maas.subnet.ipv4: ${MAAS_SUBNET}
name: eth0
nictype: bridged
parent: br0
type: nic
root:
path: /
pool: local
type: disk
kvm:
path: /dev/kvm
type: unix-char
mem:
path: /dev/mem
type: unix-char
loop-control:
path: /dev/loop-control
type: unix-char
loop0:
path: /dev/loop0
type: unix-block
loop1:
path: /dev/loop1
type: unix-block
loop2:
path: /dev/loop2
type: unix-block
loop3:
path: /dev/loop3
type: unix-block
loop4:
path: /dev/loop4
type: unix-block
loop5:
path: /dev/loop5
type: unix-block
loop6:
path: /dev/loop6
type: unix-block
loop7:
path: /dev/loop7
type: unix-block
cluster:
server_name: \$(hostname 2>/dev/null -s)
enabled: true
member_config: []
cluster_address: ''
cluster_certificate: ''
server_address: ''
cluster_password: ''
EOT
elif [[ \$(hostname 2>/dev/null -s) != ${CLOUD_PRIMARY} ]];then
while [[ \$(sudo ssh 2>/dev/null -i /etc/ssh/ssh_host_rsa_key -tt -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ubuntu@${CLOUD_PRIMARY} 'test -f /var/snap/lxd/common/lxd/server.crt';echo $?) -ne 0 ]];do sleep 5;done
cat <<EOT|sudo lxd init --preseed
config: {}
networks: []
storage_pools: []
profiles: []
cluster:
server_name: \$(hostname -s)
enabled: true
member_config:
- entity: storage-pool
name: local
key: source
value: ${STORAGE_POOL_DEVICE}
description: ''
- entity: storage-pool
name: local
key: volatile.initial_source
value: ${STORAGE_POOL_DEVICE}
description: ''
- entity: storage-pool
name: local
key: zfs.pool_name
value: local
description: ''
cluster_address: ${CLOUD_PRIMARY}.${MAAS_DOMAIN}:8443
cluster_certificate: |
\$(sudo ssh 2>/dev/null -i /etc/ssh/ssh_host_rsa_key -tt -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ubuntu@${CLOUD_PRIMARY} 'sed -e ":a;N;$!ba;s/\n/\n\n/g" /var/snap/lxd/common/lxd/server.crt'|sed 's/^.*$/ &/g')
server_address: \$(dig +short @${MAAS_IP} \$(hostname -f)):8443
cluster_password: ubuntu
EOT
fi
EOF
)
#Remove old tags on redeployment
printf "\n\e[2GCleaning up tags on machines which are no longer deployed...\n"
microcloud-tag-cleanup -p ${BASE_TAG} -i ${MICROCLOUD_TAG}
local MICROCLOUD_TAG="${BASE_TAG}-$(cat /dev/urandom|tr -dc 'a-z0-9'|fold -w6|head -n1)"
printf "\n\e[3G- Creating tag \"${MICROCLOUD_TAG}\" for this microcloud...\n"
maas ${MAAS_PROFILE} tags create name=${MICROCLOUD_TAG} comment="${CLOUD_COMMENT};$(echo -n ${MICROCLOUD_TAG^^}|sed 's/-/_/g')_HOSTS=(`printf '%s\n' ${ALLOCATED_MACHINES[@]##*:}|paste -sd" "` ); $(echo -n ${MICROCLOUD_TAG^^}|sed 's/-/_/g')_BUILD_DATE=$(date --iso-8601=seconds)" &>/dev/null
printf "\e[5G- Tagging ${#ALLOCATED_MACHINES[@]} machine${W} with \"${MICROCLOUD_TAG}\"...\n"
ADD=$(printf "add=%s\n" ${ALLOCATED_MACHINES[@]##*:}|paste -sd' ')
local ADDED=$(eval "maas ${MAAS_PROFILE} tag update-nodes ${MICROCLOUD_TAG} "$ADD""|jq -r .added)
[[ ${ADDED} -eq ${#ALLOCATED_MACHINES[@]} ]] && { printf "\e[8G- Successfully tagged ${ADDED}/${#ALLOCATED_MACHINES[@]} machines with the \"${MICROCLOUD_TAG}\" tag ${OK} "; } || { printf "\e[8G- Could only tag ${ADDED}/${#ALLOCATED_MACHINES[@]} machines with ${MICROCLOUD_TAG} ${FAILED} \n"; }
echo
printf "\n\e[3G- Deploying the following hosts to ${MICROCLOUD_TAG}:\n"
printf -- "\e[5G - %s\n" ${ALLOCATED_MACHINES[@]}|sed 's/:/ \(/g;s/.$/&\)/g';echo
local -a SYSID_ARR=($(printf "%s\n" ${ALLOCATED_MACHINES[@]##*:}))
local -a DEPLOYED_MACHINES=($(printf "%s\n" ${SYSID_ARR[@]}|xargs -I{} -n1 -P1 maas ${MAAS_PROFILE} machine deploy {} distro_series=${DISTRO} user_data="$(echo;cat <<EOF|tee /tmp/${FUNCNAME}.cloud-init.yaml|base64 -w0
#cloud-config-2
bootcmd:
- (ip route get ${MAAS_IP}|grep -oP '(?<=dev )[^ ]+')|xargs -n1 ip route add default dev
final_message: LXD-Cluster Install complete on \$(hostname -f)
timezone: ${CI_TZ}
locale: ${CI_LC}
apt:
proxy: http://${MAAS_IP}:8000/
primary:
- arches: [amd64]
uri: http://${CI_APT_MIRROR}/ubuntu
security:
- arches: [amd64]
uri: http://security.ubuntu.com/ubuntu
package_update: ${CI_PKG_UPD}
package_upgrade: ${CI_PKG_UPG}
${CI_PACKAGES}
write_files:
- encoding: b64
content: ${LXD_CLUSTER_SCRIPT}
path: /usr/local/bin/lxd-cluster.sh
permissions: '755'
owner: root:root
- encoding: b64
content: ${LXD_IMAGE_SCRIPT}
path: /usr/local/bin/lxc-image-copy.sh
permissions: '755'
owner: root:root
- encoding: b64
content: ${VIP_VRRP_SCRIPT}
path: /usr/local/bin/configure-vrrp.sh
permissions: '755'
owner: root:root
${SSH_HOST_KEYS}
${SSH_PUB_KEYS}
runcmd:
- set -x
- for i in \$(seq \$(find /dev -iname 'loop[0-9]*'|wc -l) 1 256);do mknod -m0660 /dev/loop\${i} b 7 \${i} && chown root.disk /dev/loop\${i};done
- if [ \$(lsb_release -sr|sed 's/\.//g') -le 1804 ];then apt purge lxd lxd-client -y;fi
- if [ \$(lsb_release -sr|sed 's/\.//g') -ge 1810 ];then snap remove lxd;fi
- apt autoremove -y
- snap install lxd --candidate
- adduser \$(id -un 1000) lxd
- if [ ! \$(id -un 1000) = ubuntu -a -n \$(id 2>/dev/null -u ubuntu) ];then adduser ubuntu lxd;fi
- if [ ! -f \$(awk -F":" '/'\$(id -un 0)'/{print \$6}' /etc/passwd)/.ssh/id_rsa ];then printf "y\n"|ssh-keygen -f ~/.ssh/id_rsa -P "";fi
- if [ ! -f \$(awk -F":" '/'\$(id -un 1000)'/{print \$6}' /etc/passwd)/.ssh/id_rsa ];then su - \$(id -un 1000) -c 'printf "y\n"|ssh-keygen -f ~/.ssh/id_rsa -P ""' ;fi
- if [ ! \$(id -un 1000) = ubuntu -a -n \$(id 2>/dev/null -u ubuntu) ];then if [ ! -f \$(awk -F":" '/ubuntu/{print \$6}' /etc/passwd)/.ssh/id_rsa ];then su - ubuntu -c 'printf "y\n"|ssh-keygen -f ~/.ssh/id_rsa -P ""' ;fi;fi
- cp /etc/ssh/ssh_host_rsa_key* \$(awk -F\":\" '/'\$(id -un 1000)'/{print \$6}'/etc/passwd)/.ssh/
- chown -R \$(id -un 1000):\$(id -un 1000) \$(awk -F\":\" '/'\$(id -un 1000)'/{print \$6}'/etc/passwd)/.ssh/
- cat /etc/ssh/ssh_host_rsa_key.pub|tee 1>/dev/null -a \$(awk -F":" '/'\$(id -un 0)'/{print \$6}' /etc/passwd)/.ssh/authorized_keys \$(awk -F":" '/'\$(id -un 1000)'/{print \$6}' /etc/passwd)/.ssh/authorized_keys
- printf -- '\nHost ${CLUSTER_HOST_LIST}\n\tAddressFamily inet\n\tCheckHostIP no\n\tForwardX11Trusted yes\n\tForwardX11 yes\n\tIdentityFile /etc/ssh/ssh_host_rsa_key\n LogLevel FATAL\n SendEnv LANG LC_*\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n User ubuntu\n\tXAuthLocation /usr/bin/xauth\n'|tee -a 1>/dev/null /etc/ssh/ssh_config
- printf -- '%s\n' ${CLUSTER_HOST_LIST//\*/}|xargs -I{} -n1 -P0 ssh-keyscan 2>/dev/null -H {}|su - \$(id -un 1000) -c 'tee -a ~/.ssh/known_hosts'
- printf -- '%s\n' ${CLUSTER_HOST_LIST//\*/}|xargs -I{} -n1 -P0 ssh-keyscan 2>/dev/null -H {}|tee -a \$(awk -F\":\" '/'\$(id -un 0)'/{print \$6}' /etc/passwd)/.ssh/known_hosts
- if [ -f /usr/local/bin/lxd-cluster.sh ];then /usr/local/bin/lxd-cluster.sh;fi
- if [ \$(hostname -s) = ${CLOUD_PRIMARY} -a -f /usr/local/bin/lxc-image-copy.sh ];then /usr/local/bin/lxc-image-copy.sh;fi
- snap install conjure-up --classic
- lxc 2>/dev/null profile device set maas eth0 parent \$(ip route get ${MAAS_IP}|/bin/grep -oP '(?<=dev )[^ ]+')
- lxc 2>/dev/null profile device set maas-privileged eth0 parent \$(ip route get ${MAAS_IP}|/bin/grep -oP '(?<=dev )[^ ]+')
- export ENABLE_GPU_SUPPORT=${ENABLE_GPU}
- if [ \${ENABLE_GPU_SUPPORT} = true ];then echo Detecting GPGPU devices...;fi
- if [ \${ENABLE_GPU_SUPPORT} = true ];then export GPU_CMD='lspci -nn|awk -vIGNORECASE=1 '"'"'/\[03.*intel/{ORS=",";print substr(\$(NF-2),2,9)}'"'"'|sed "s/,$/\\n/"';fi
- if [ \${ENABLE_GPU_SUPPORT} = true ];then export GPU_CONF_FILE='lspci -nn|awk -vIGNORECASE=1 '"'"'/\[03.*intel/{printf "export NVDA_GPU_%02d_ID=%s\nexport NVDA_GPU_%02d_BUS=%s\n",NR,substr(\$(NF-2),2,9),NR,\$1}'"'"'|tee -a /etc/gpgpu-pci-ids.conf';fi
- if [ \${ENABLE_GPU_SUPPORT} = true ];then export GPU_PCI_LIST='lspci -nn|awk -vIGNORECASE=1 '"'"'BEGIN {printf "%s","options vfio-pci ids="} /\[03.*intel/{ORS=",";print substr(\$(NF-2),2,9)}'"'"'|sed "s/,$/\\n/"|tee /etc/modprobe.d/vfio.conf';fi
- if [ \${ENABLE_GPU_SUPPORT} = true ];then if [ -n \$(eval \$GPU_CMD) ];then echo ' GPGPUs Detected!';printf '%s\n' vfio vfio_pci > /etc/modules-load.d/vfio.conf;eval \$GPU_CONF_FILE;eval \$GPU_PCI_LIST;fi;fi
- export VIP_VRRP_ENABLE=${VIP_VRRP_ENABLE}
- if [ \${VIP_VRRP_ENABLE} = true ];then echo 'net.ipv4.ip_forward = 1' >> /etc/sysctl.conf;fi
- if [ \${VIP_VRRP_ENABLE} = true ];then sysctl -p;fi
- if [ \${VIP_VRRP_ENABLE} = true ];then install -D -m 644 /dev/null '/etc/systemd/system/keepalived.service';fi
- |-
if [ \${VIP_VRRP_ENABLE} = true ];then
cat > '/etc/systemd/system/keepalived.service' << 'EOF'
[Unit]
Description=LVS and VRRP High Availability monitor
After=network.target
ConditionFileNotEmpty=/etc/keepalived/keepalived.conf
[Service]
Type=simple
EnvironmentFile=-/etc/default/keepalived
ExecStart=/usr/sbin/keepalived --dont-fork
ExecReload=/bin/kill -s HUP \$MAINPID
KillMode=process
[Install]
WantedBy=multi-user.target
EOF
fi
- if [ \${VIP_VRRP_ENABLE} = true ];then /bin/systemctl enable '/etc/systemd/system/keepalived.service';fi
- if [ \${VIP_VRRP_ENABLE} = true ];then /bin/systemctl start '/etc/systemd/system/keepalived.service';fi
- touch /var/tmp/install-complete
EOF
)"|jq 2>/dev/null -r '"\(.hostname):\(.system_id)"'))
printf "\n\e[2GA copy of the cloud-init used for this deployment can be found at\n\e[2G/tmp/${FUNCNAME}.cloud-init.yaml\n\n"
}
export -f create-lxd-microcloud
create-kvm-microcloud() {
local DESC="${FUNCNAME}: Deploy a KVM Pods with Cloud-Init customizations via MAAS command line"
# Env Setttings
local CI_TZ='America/Los_Angeles'
local CI_LC='en_US.UTF-8'
#Package/Mirror Settings
local CI_PKG_UPG=true
local CI_PKG_UPD=true
# Test for Ubuntu Country Mirror based on Locale
local CI_TC=${CI_LC:3:2}
if [[ -n ${CI_TC} && $(curl -slSL -w %{http_code} -o /dev/null ${CI_TC,,}.archive.ubuntu.com) -eq 200 ]];then
local CI_CC=${CI_TC}
local CI_APT_MIRROR=${CI_CC,,}.archive.ubuntu.com
else
local CI_APT_MIRROR=archive.ubuntu.com
fi
#Generic KeepAliveD Settings for a Cluster VIP using Keepalived
#ENV Setttings
local CI_TZ='America/Los_Angeles'
local CI_LC='en_US.UTF-8'
#Package Settings
local CI_PKG_UPG=true
local CI_PKG_UPD=true
# Test for Ubuntu Country Mirror based on Locale
local CI_TC=${CI_LC:3:2}
if [[ -n ${CI_TC} && $(curl -slSL -w %{http_code} -o /dev/null ${CI_TC,,}.archive.ubuntu.com) -eq 200 ]];then
local CI_CC=${CI_TC}
local CI_APT_MIRROR=${CI_CC,,}.archive.ubuntu.com
else
local CI_APT_MIRROR=archive.ubuntu.com
fi
#Generic KeepAliveD Settings for a Cluster VIP using Keepalived
local VIP_VRRP_CIDR=
local VIP_VRRP_ENABLE=false
#Defaults if not chosen
local ENABLE_GPU=false
local BASE_TAG="microcloud-kvm"
local STORAGE_POOL_DEVICE=
local STORAGE_POOL_TYPE=file
local ENFORE_HA=true
local -a CI_PKG_LIST=(
bridge-utils
build-essential
jq
prips
squashfuse
unzip
zfsutils-linux
keepalived
)
local -a CI_PKG_LIST_BIONIC=(
apt-utils
build-essential
debconf-utils
jq
libvirt-bin
libvirt-daemon-driver-storage*
libvirt-daemon-system
ovmf
qemu-kvm
squashfuse
virtinst
virt-manager
virt-viewer
zfsutils
keepalived
)
local -a CI_PKG_LIST_COSMIC=(
apt-utils
build-essential
debconf-utils
jq
qemu-kvm
libvirt-daemon*
ovmf
squashfuse
virtinst
virt-manager
zfsutils-linux
keepalived
)
create-kvm-microcloud_usage() {
printf "\n\e[2G${DESC}\n\n"
printf "\e[2G\e[1mUsage\e[0m: ${FUNCNAME%%_*} [-c <count>] [-t <tag>] [-d <distro>] ( -g gpu-passthrough support ) (-p storage-pool block device)\n\n"
printf "\e[4G -n, --count\e[20GNumber of nodes (Min: 3) \n"
printf "\e[4G -t, --tag \e[20GExisting tag to use when selecting nodes\n"
printf "\e[4G -d, --distro\e[20GName of Ubuntu Distro (Default: bionic)\n"
printf "\e[4G -p, --pool-device\e[20GBlock device to use for KVM Storage Pool (Default: File-based Pool)\n"
printf "\e[4G -g, --gpu\e[20GEnable GPU passthrough support\n"
printf "\e[4G -H, --ha\e[20GEnsure n+2 servers are available (default: True)\n"
printf "\e[4G -N, --no-ha\e[20GAllow less than n+2 servers in a cloud (default: False)\n"
printf "\e[4G -T, --cloud-name\e[20GUse a custom tag to indentify this microcloud\n"
printf "\e[4G -c, --comment\e[20GComment to apply to tags related to the microcloud\n"
printf "\e[4G -v, --vrrp\e[20GEnable simple Virtual IP address failover using Keepalived (default: false)\n"
printf "\e[4G -V, --VIP\e[20GIP Address to use for the VIP\n"
printf "\e[4G -i, --instance\e[20GKeepAliveD instance name for VRRP_VIP service (default: MICROCLOUD_VIP)\n"
printf "\e[4G -P, --vrrp-password\e[20GPassword used by KeepAliveD to track state (default: Random 6 digit alpha numeric\n"
printf "\e[4G --cpu-ocr\e[20G CPU overcommit ratio for KVM 1.0 >= <= 10.0 (default: 1 recommended: 10\n"
printf "\e[4G --mem-ocr\e[20G memory overcommit ratio for KVM 1.0 >= <= 10.0 (default: 1 recommended: < 1.5\n"
printf "\e[4G -h, --help\e[20GThis message\n"
printf "\n\n"
}
ARGS=$(getopt -o n:c:t:d:p:C:T:P:v:o:gdhHNV -l count:,tag:,distro:,pool-device:,comment:,cloud-name:.vip:,vrrp-password:,cpu-ocr:,mem-ocr:,gpu,help,desc,enable-vrrp,ha,no-ha -n ${FUNCNAME} -- "$@")
eval set -- "$ARGS"
while true ; do
case "$1" in
-n|-c|--count) local COUNT=${2};shift 2;;
-t|--tag) local TAG=${2};local TAG=${TAG,,};shift 2;;
-d|--distro) local DISTRO=${2};local DISTRO=${DISTRO,,};shift 2;;
-p|--pool-device) local STORAGE_POOL_DEVICE=${2};local STORAGE_POOL_DEVICE=${STORAGE_POOL_DEVICE,,};local STORAGE_POOL_TYPE=device;shift 2;;
-g|--gpu) local ENABLE_GPU=true;shift 1;;
-H|--ha) local ENFORCE_HA=true;shift 1;;
-N|--no-ha) local ENFORCE_HA=false;shift 1;;
-C|--comment) local CLOUD_COMMENT="${2}";shift 2;;
-T|--cloud-name) local CLOUD_NAME="${2,,}";shift 2;;
-V|--enable-vrrp) local VIP_VRRP_ENABLE=true;shift 1;;
-v|--vip) local VIP_VRRP_CIDR=${2};shift 2;;
-i|--vrrp-instance) local VIP_VRRP_INSTANCE=${2};shift 2;;
-P|--vrrp-password) local VIP_VRRP_PASS=${2};shift 2;;
--cpu-ocr) local CPU_OCR=${2};shift 2;;
--mem-ocr) local MEM_OCR=${2};shift 2;;
--desc) printf "\n\e[2G${DESC}\n";return 0;;
-h|--help) ${FUNCNAME}_usage;return 2;;
--) shift;break;;
esac
done
command -v maas &>/dev/null || { printf "\n\n\e[2G\e[2G\e[38;2;255;255;0mSorry! \e[0m\e[38;2;0;255;0m maas-cli \e[0m\e[1m2.5+\e[0m is required for this demo.\n\e[9GPlease install and configure version 2.5 or later\n\n";return 1; }
[[ $(dpkg-query -s maas|grep -oP '(?<=Version: )[^?]{3}'|sed 's/\.//') -lt 25 ]] && { printf "\n\n\e[2G\e[2G\e[38;2;255;255;0mSorry! \e[0mMAAS \e[38;2;0;255;0m2.5\e[0m\e[1m+\e[0m is required for this demo.\n\e[9GYou are running version \e[38;2;255;255;0m$(dpkg-query -s maas|grep -oP '(?<=Version: )[^~]+')\e[0m\n\n";return 1; }
command -v jq &>/dev/null || sudo apt install jq -y
local OK='\u00A0\e[38;2;0;255;0m\u2713\e[0m\u00A0\n'
local FAILED='\u00A0\e[38;2;255;0;0m\u2718\u00A0\n'
printf -- "\n\e[1mMicro-cloud Demo - KVM\e[0m\n\n"
printf -- "\e[3G- Provisioning by \e[1mMAAS $(dpkg-query -s maas|grep -oP '(?<=Version: )[^~]+')\e[0m${OK}\n"
printf -- "\e[6G- Powered by \e[1m$(lsb_release -ds)\e[0m${OK}\n"
printf -- "\e[3G- Featuring \e[1mKVM Pods \e[0m${OK}\n"
printf '\n\n'
[[ -z ${MAAS_PROFILE} ]] && { read -srp "$(printf "\e[2GPlease enter your MAAS profile name: ")" MAAS_PROFILE_INPUT;local MAAS_PROFILE=$MAAS_PROFILE_INPUT; }
local MAAS_IP=$(maas list|awk -F"//|:" '/^'${MAAS_PROFILE}'/{print $3}')
#Validation and error handling
local -a VALID_TAGS=($(maas ${MAAS_PROFILE} tags read|jq -r '.[].name'))
local -a VALID_DISTROS=($(maas ${MAAS_PROFILE} boot-resources read|jq -r '.[]|"\(select((.name|startswith("grub")|not) and (.name|startswith("ubuntu")) and (.name|startswith("pxe")|not)).name|sub("ubuntu/"; ""))"'|sort -uV))
printf -- "\e[1m\e[2GProcessing selections\e[0m:\n\n"
[[ ${ENABLE_GPU} = true ]] && printf -- "\e[3G- GPGPU Passthrough enabled\n"
if [[ ${VIP_VRRP_ENABLE} = true ]];then
printf -- "\e[3GSimple Virtual IP Address Failover enabled\n"
[[ -n ${VIP_VRRP_CIDR} ]] && { printf -- "\e[5G- Virtual IP: ${VIP_VRRP_CIDR}\n"; } || { local VIP_VRRP_CIDR_=$(gen-sa-ip);printf -- "\e[5G- VIP_VRRP_CIDR: ${VIP_VRRP_CIDR} (auto-generated)\n"; }
[[ -n ${VIP_VRRP_INSTANCE} ]] && { printf -- "\e[5G- VRRP Instance Name: ${VIP_VRRP_INSTANCE}}\n"; } || { local VIP_VRRP_INSTANCE="VIP-$(cat /dev/urandom|tr -dc 'A-Z0-9'|fold -w6|head -n1)";printf -- "\e[5G- VRRP INSTANCE: ${VIP_VRRP_INSTANCE} (auto-generated)\n"; }
[[ -n ${VIP_VRRP_PASS} ]] && { printf -- "\e[5G- VRRP Passwd: ${VIP_VRRP_PASS}\n"; } || { local VIP_VRRP_PASS="$(cat /dev/urandom|tr -dc 'a-z0-9'|fold -w6|head -n1)";printf -- "\e[5G- VRRP Passwd: ${VIP_VRRP_PASS} (auto-generated)\n"; }
fi
[[ ${ENFORCE_HA} = true ]] && { printf "\e[3G- Enforcing n+2 Rule\n"; }
[[ ${ENFORCE_HA} = true && $COUNT -lt 3 ]] && { local COUNT=3;printf "\e[3G- Setting node count to 3\n"; }
# If not a physical disk, set storage pool to use a file
[[ ${STORAGE_POOL_TYPE} = device ]] && { printf "\e[3G- Storage Pool will use block device ${STORAGE_POOL_DEVICE} \n"; local STORAGE_POOL_TYPE=device; } || { local STORAGE_POOL_TYPE=file;local STORAGE_POOL_DEVICE="/var/lib/libvirt/maas-images/";printf "\e[3G- Storage Pool will be file-based \n"; }
[[ -z ${COUNT} ]] && { [[ ${ENFORCE_HA} = true ]] && { local COUNT=3;printf "\e[3G- Setting Node count to ${COUNT}\n"; } || { local COUNT=1;printf "\e[3G- Setting Node count to ${COUNT}\n"; }; }
[[ ${COUNT} =~ ^[0-9]+$ && ${COUNT} -lt 3 && ${ENFORCE_HA} = true ]] && { local COUNT=3;printf "\e[3G- Minimum Node count for HA is ${COUNT}. Setting Node count to ${COUNT}.\n\n"; }
[[ -z ${TAG} ]] && { printf "\e[2GERROR: No tag name given ${TAG}\n\n\e[2GValid tags are:\n";printf "\e[4G- %s\n" ${VALID_TAGS[@]};return 1; }
[[ -n ${TAG} && -n $(grep -P '(^|\s)\K'${TAG}'(?=\s|$)' <<< ${VALID_TAGS[@]}) ]] || { printf "\e[2GERROR: No tags exist named ${TAG}\n\n\e[2GValid tags are:\n";printf "\e[4G- %s\n" ${VALID_TAGS[@]};return 1; }
[[ -n ${DISTRO} ]] && { printf "\e[3G- Setting distro to ${DISTRO}\n"; } || { local DISTRO="bionic";printf "\e[3G- Setting distro to ${DISTRO}\n"; }
[[ -n ${DISTRO} && -n $(grep -P '(^|\s)\K'${DISTRO}'(?=\s|$)' <<< ${VALID_DISTROS[@]}) ]] || { printf "\e[2GERROR: Invalid Distro: ${DISTRO}\n\n\e[2GValid distros for KVM Pods are:\n";printf "\e[4G- %s\n" ${VALID_DISTROS[@]};return 1; }
[[ ${COUNT} -eq 1 ]] && W= || W=s
if [[ ${STORAGE_POOL_TYPE} = device ]];then
printf "\n\e[3G- Finding ${COUNT} machine${W} :\n\e[5G- Marked as \"Ready\"\n\e[5G- Tagged with \"${TAG}\"\n"
local READY_TAGGED_MACHINES=($(maas ${MAAS_PROFILE} machines read|jq -r '.[]|select(.physicalblockdevice_set[].name|contains("'${STORAGE_POOL_DEVICE##*/}'"))|select((.tag_names[]|contains("'"${TAG}"'")) and .status == 4)|"\(.hostname):\(.system_id)"'|head -n${COUNT}|sort -V))
[[ ${#READY_TAGGED_MACHINES[@]} -lt 3 && ${ENFORCE_HA} = true ]] && { printf "\e[2G\e[2G\e[38;2;255;255;0mSorry! \e[0mCould not find enough machines to create a KVM Cluster.\n\n\e[9GMachines Required: 3\n\e[9GMachines Found:\e[4C\e[38;2;255;0;0m${#READY_TAGGED_MACHINES[@]}\n\n";return 1; }
[[ ${#READY_TAGGED_MACHINES[@]} -lt ${COUNT} && ${ENFORCE_HA} = true && ${#READY_TAGGED_MACHINES[@]} -ge 3 ]] && { read -erp "$(printf "\n\e[2G - Could only find ${#READY_TAGGED_MACHINES[@]} machine${W} with the ${TAG} tag. Continue? [y/n] : ")" CONT; [[ ${CONT,,} =~ y ]] && echo || return 0; }
[[ ${#READY_TAGGED_MACHINES[@]} -lt ${COUNT} && ${ENFORCE_HA} = false ]] && { read -erp "$(printf "\n\e[2G - Could only find ${#READY_TAGGED_MACHINES[@]} machines with the ${TAG} tag. Continue? [y/n] : ")" CONT; [[ ${CONT,,} =~ y ]] && echo || return 0; }
local COUNT=${#READY_TAGGED_MACHINES[@]}
else
printf "\n\e[3G- Finding ${COUNT} machine${W} :\n\e[5G- Marked as \"Ready\"\e[5G- Tagged with \"${TAG}\"\n\e[5G- Has physical disk named ${STORAGE_POOL_DEVICE}\n"
local -a READY_TAGGED_MACHINES=($(maas ${MAAS_PROFILE} machines read|jq -r '.[]|select((.tag_names[]|contains("'"${TAG}"'")) and .status == 4)|"\(.hostname):\(.system_id)"'|head -n${COUNT}|sort -V))
[[ ${#READY_TAGGED_MACHINES[@]} -lt 3 && ${ENFORCE_HA} = true ]] && { printf "\e[2G\e[2G\e[38;2;255;255;0mSorry! \e[0mCould not find enough machines to create a KVM Cluster.\n\n\e[9GMachines Required: 3\n\e[9GMachines Found:\e[4C\e[38;2;255;0;0m${#READY_TAGGED_MACHINES[@]}\n\n";return 1; }
[[ ${#READY_TAGGED_MACHINES[@]} -lt ${COUNT} && ${ENFORCE_HA} = true && ${#READY_TAGGED_MACHINES[@]} -ge 3 ]] && { read -erp "$(printf "\n\e[2G - Could only find ${#READY_TAGGED_MACHINES[@]} machine${W} with the ${TAG} tag. Continue? [y/n] : ")" CONT; [[ ${CONT,,} =~ y ]] && echo || return 0; }
[[ ${#READY_TAGGED_MACHINES[@]} -lt ${COUNT} && ${ENFORCE_HA} = false ]] && { read -erp "$(printf "\n\e[2G - Could only find ${#READY_TAGGED_MACHINES[@]} machine${W} with the ${TAG} tag. Continue? [y/n] : ")" CONT; [[ ${CONT,,} =~ y ]] && echo || return 0; }
local COUNT=${#READY_TAGGED_MACHINES[@]}
fi
printf "\n\e[2GAllocating the following Host${W} to the KVM Pods:\n"
printf -- "\e[2G - %s\n" ${READY_TAGGED_MACHINES[@]}|sed 's/:/ \(/g;s/.$/&\)/g';echo
local -a ALLOCATED_MACHINES=($(printf "%s\n" ${READY_TAGGED_MACHINES[@]##*:}|xargs -I{} -n1 -P0 maas ${MAAS_PROFILE} machines allocate system_id={}|jq -r '"\(.hostname):\(.system_id)"'|sort -uV))
[[ ${#ALLOCATED_MACHINES[@]} -lt 3 && ${ENFORCE_HA} = true ]] && { printf "\e[2G\e[2G\e[38;2;255;255;0mSorry! \e[0mCould not allocate enough machines to create a KVM Pods.\n\n\e[9GMachines Required: 3\n\e[9GMachines Found:\e[4C\e[38;2;255;0;0m${#ALLOCATED_MACHINES[@]}\n\n";return 1; }
[[ ${#ALLOCATED_MACHINES[@]} -lt ${COUNT} && ${ENFORCE_HA} = true && ${#ALLOCATED_MACHINES[@]} -ge 3 ]] && { read -erp "$(printf "\n\e[2G - Could only allocate ${#ALLOCATED_MACHINES[@]} machine${W}. Continue? [y/n] : ")" CONT; [[ ${CONT,,} =~ y ]] && echo || return 0; }
[[ ${#ALLOCATED_MACHINES[@]} -lt ${COUNT} && ${ENFORCE_HA} = false ]] && { read -erp "$(printf "\n\e[2G - Could only allocate ${#ALLOCATED_MACHINES[@]} machines. Continue? [y/n] : ")" CONT; [[ ${CONT,,} =~ y ]] && echo || return 0; }
local COUNT=${#ALLOCATED_MACHINES[@]}
# Get SSH Public keys from local accounts and maas. Remove duplicates
local SSH_PUB_KEYS=$((if [[ -n $(find 2>/dev/null ~/.ssh -iname "*.pub") ]];then
echo ssh_authorized_keys:
find 2>/dev/null ~/.ssh -iname "*.pub"|xargs -n1 -P1 bash -c 'printf -- '"'"' - %s\n'"'"' "$(cat $0)"'
fi
if [[ -n $(command -v maas 2>/dev/null) ]];then
if [[ $(maas 2>/dev/null ${MAAS_PROFILE} sshkeys read|jq 2>/dev/null length) -ge 1 ]];then
echo "ssh_authorized_keys:"
maas 2>/dev/null ${MAAS_PROFILE} sshkeys read|jq 2>/dev/null -r '.[]|"\(.key)"'| \
while IFS= read -r line;do
sed 's/^/ - /g'
done
fi
fi
)|awk '!seen[$0]++')
# Create a string of the Cluster hosts that we can for SSH config files (thus the added asterisk)
# (easier to remove the asterisk in a variable than to add it, so it's added to main variable)
local CLUSTER_HOST_LIST=$(printf "%s*\n" ${ALLOCATED_MACHINES[@]%%:*}|paste -sd" "|sort -uV)
# Create Package List for Cluster hosts
[[ ${DISTRO,,} = xenial || ${DISTRO,,} = bionic ]] && local CI_PACKAGES=$([[ ${#CI_PKG_LIST_BIONIC[@]} -ge 1 ]] && printf "packages: [$(printf '%s\n' ${CI_PKG_LIST_BIONIC[@]}|paste -sd',')]\n")
[[ ${DISTRO,,} = cosmic || ${DISTRO,,} = disco ]] && local CI_PACKAGES=$([[ ${#CI_PKG_LIST_COSMIC[@]} -ge 1 ]] && printf "packages: [$(printf '%s\n' ${CI_PKG_LIST_COSMIC[@]}|paste -sd',')]\n")
# xpath auto tags for varius different nVidia GPGPUs
if [[ ${ENABLE_GPU} = true ]];then
printf "\e[2GConfiguring xpath auto-tags for GPGPU Detection at the hardware level in MAAS...\n\n"
NVDA_3D_CONTROLLER_TAG_DEF='//node[@id="display"]/description = "3D controller" and //node[@id="display"]/vendor = "NVDA Corporation" and //node[@id="display"]/product = "NVIDIA Corporation"'
NVDA_3D_CONTROLLER_TAG_COMMENT='xpath auto-tag for nvidia general-purpose GPUs that have been claimed by noveau or nvidiafb module'
declare -ag NVDA_GPGPU_TESLA_PRODUCTS=(P4 P40 P100 V100 T4)
NVDA_GPGPU_TESLA_PROD_DEF="$(for p in ${NVDA_GPGPU_TESLA_PRODUCTS[@]};do printf '%s\n' '//node[@id="display"]/product[starts-with(.,"'${p}'")] or '; done|sed '$ s/ or//g')"
NVDA_GPGPU_TESLA_TAG_DEF='contains(//node[@id="display"]/vendor,"nVidia") and contains(//node[@id="display"]/product,"Tesla") and '"${NVDA_GPGPU_TESLA_PROD_DEF}"''
NVDA_GPGPU_TESLA_TAG_COMMENT="xpath auto-tag for nVidia Tesla-based general-purpose GPUs"
declare -ag NVDA_GPGPU_QUADRO_PRODUCTS=(K M P2000 P4000 P5000 P6000)
NVDA_GPGPU_QUADRO_PROD_DEF="$(for p in ${NVDA_GPGPU_QUADRO_PRODUCTS[@]};do printf '%s\n' '//node[@id="display"]/product[starts-with(.,"'${p}'")] or '; done|sed '$ s/ or//g')"
NVDA_GPGPU_QUADRO_TAG_DEF='contains(//node[@id="display"]/vendor,"nVidia") and contains(//node[@id="display"]/product,"Quadro") and '"${NVDA_GPGPU_QUADRO_PROD_DEF}"''
NVDA_GPGPU_QUADRO_TAG_COMMENT="xpath auto-tag for nVidia Quadro-based general-purpose GPUs"
declare -ag NVDA_GPGPU_ALL_PRODUCTS=($(printf "%s\n" ${NVDA_GPGPU_TESLA_PRODUCTS[@]} ${NVDA_GPGPU_QUADRO_PRODUCTS[@]}|paste -sd" "))
NVDA_GPGPU_ALL_PROD_DEF="$(for p in ${NVDA_GPGPU_TESLA_PRODUCTS[@]} ${NVDA_GPGPU_QUADRO_PRODUCTS[@]};do printf '%s\n' '//node[@id="display"]/product[starts-with(.,"'${p}'")] or '; done|sed '$ s/ or//g')"
NVDA_GPGPU_ALL_TAG_DEF='contains(//node[@id="display"]/vendor,"nVidia") and contains(//node[@id="display"]/product,"Quadro") or contains(//node[@id="display"]/product,"Tesla") and '"${NVDA_GPGPU_ALL_PROD_DEF}"''
NVDA_GPGPU_ALL_TAG_COMMENT="xpath auto-tag for nVidia Quadro and Tesla-based general-purpose GPUs"
NVDA_GPGPU_ALL_KERNEL_OPTS="nomodeset modprobe.blacklist=nouveau modprobe.blacklist=nvidiafb intel_iommu=on iommu=pt rd.driver.pre=vfio-pci video=efifb:off vfio_iommu_type1.allow_unsafe_interrupts=1"
declare -ag GPGPU_ARR=(NVDA_3D_CONTROLLER NVDA_GPGPU_TESLA NVDA_GPGPU_QUADRO NVDA_GPGPU_ALL)
MAAS_TAGS=$(maas ${MAAS_PROFILE} tags read)
for G in ${GPGPU_ARR[@]};do
if [[ -n $(jq -r '.[]|select(.name=="'${G,,}'").name' <<< ${MAAS_TAGS}) ]];then
printf "\e[2G - Updating xpath auto tag for ${G} products...\n"
maas ${MAAS_PROFILE} tag update ${G,,} name="${G,,}" definition="$(eval echo \$${G}_TAG_DEF)" comment="$(eval echo \$${G}_TAG_COMMENT)"
[[ ${G} = NVDA_GPGPU_ALL || ${G} = NVDA_3D_CONTROLLER ]] && maas ${MAAS_PROFILE} tag update ${G,,} name="${G,,}" kernel_opts="${NVDA_GPGPU_ALL_KERNEL_OPTS}"
printf '\n\n'
else
printf "\e[2G - Creating xpath auto tag for ${G} products...\n\n"
maas ${MAAS_PROFILE} tags create name="${G,,}" definition="$(eval echo \$${G}_TAG_DEF)" comment="$(eval echo \$${G}_TAG_COMMENT)"
[[ ${G} = NVDA_GPGPU_ALL || ${G} = NVDA_3D_CONTROLLER ]] && maas ${MAAS_PROFILE} tag update ${G,,} name="${G,,}" kernel_opts="${NVDA_GPGPU_ALL_KERNEL_OPTS}"
printf '\n\n'
fi
done
fi
[[ ${VIP_VRRP_ENABLE} = true ]] && printf "\e[3G- Writing KeepAliveD configuration Script to Cloud-Init file...\n\n"
[[ ${VIP_VRRP_ENABLE} = true ]] && local VIP_VRRP_SCRIPT=$(cat <<EOS|base64 -w0
#!/bin/bash
#root check
if [[ $EUID -ne 0 ]]; then
printf '\n\e[1;33mThis script requires admin privileges.\e[0m\e[1;37m Please run via sudo.\e[0m\n\n'
exit 1
fi
if [[ \$(hostname 2>/dev/null -s) = ${CLOUD_PRIMARY} ]];then
cat <<EOM|/etc/keepalived/keepalived.conf
global_defs {
notification_email {
root@\$(hostname -d)
}
notification_email_from \$(hostname -f)
smtp_server localhost
smtp_connect_timeout 30
}
vrrp_instance ${VIP_VRRP_INSTANCE} {
state MASTER
# Specify the network interface to which the virtual address is assigned
interface \$(ip route get ${MAAS_IP}|grep -oP '(?<=dev )[^ ]+')
# The virtual router ID must be unique to each VRRP instance that you define
virtual_router_id 81
# Set the value of priority higher on the master server than on a backup server
priority 200
advert_int 1
authentication {
auth_type PASS
auth_pass ${VIP_VRRP_PASS}
}
virtual_ipaddress {
$VIP_VRRP_CIDR
}
}
EOM
else
cat <<EOB|/etc/keepalived/keepalived.conf
global_defs {
notification_email {
root@\$(hostname -d)
}
notification_email_from \$(hostname -f)
smtp_server localhost
smtp_connect_timeout 30
}
vrrp_instance ${VIP_VRRP_INSTANCE} {
state BACKUP
# Specify the network interface to which the virtual address is assigned
\$(ip route get ${MAAS_IP}|grep -oP '"'"'(?<=dev )[^ ]+'"'"')
virtual_router_id 81
# Set the value of priority lower on the backup server than on the master server
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass ${VIP_VRRP_PASS}
}
virtual_ipaddress {
${VIP_VRRP_CIDR}
}
}
EOB
fi
EOS
)
local MICROCLOUD_TAG="${BASE_TAG}-$(cat /dev/urandom|tr -dc 'a-z0-9'|fold -w6|head -n1)"
printf "\n\e[2GCreating tag \"${MICROCLOUD_TAG}\" for this microcloud...\n"
#Remove old tags on redeployment
printf "\n\e[2GCleaning up tags on machines that no longer deployed...\n"
microcloud-tag-cleanup -p ${BASE_TAG} -i ${MICROCLOUD_TAG}
maas ${MAAS_PROFILE} tags create name=${MICROCLOUD_TAG} comment="$(echo -n ${MICROCLOUD_TAG^^}|sed 's/-/_/g')_HOSTS=(`printf '%s\n' ${ALLOCATED_MACHINES[@]##*:}|paste -sd" "` ); $(echo -n ${MICROCLOUD_TAG^^}|sed 's/-/_/g')_BUILD_DATE=$(date --iso-8601=seconds)" &>/dev/null
printf "\e[2G- Tagging ${#ALLOCATED_MACHINES[@]} machines with \"${MICROCLOUD_TAG}\"...\n"
ADD=$(printf "add=%s\n" ${ALLOCATED_MACHINES[@]##*:}|paste -sd' ')
local ADDED=$(eval "maas ${MAAS_PROFILE} tag update-nodes ${MICROCLOUD_TAG} "$ADD""|jq -r .added)
[[ ${ADDED} -eq ${#ALLOCATED_MACHINES[@]} ]] && { printf "\e[4G - Successfully tagged ${ADDED}/${#ALLOCATED_MACHINES[@]} machines with the \"${MICROCLOUD_TAG}\" tag ${OK} "; } || { printf "\e[2G - Could only tag ${ADDED}/${#ALLOCATED_MACHINES[@]} machines with ${MICROCLOUD_TAG} ${FAILED} \n"; }
echo
printf "\n\e[2GDeploying the following host${W} to ${MICROCLOUD_TAG}:\n\n"
printf -- "\e[2G - %s\n" ${ALLOCATED_MACHINES[@]}|sed 's/:/ \(/g;s/.$/&\)/g';echo
local -a SYSID_ARR=($(printf "%s\n" ${ALLOCATED_MACHINES[@]##*:}))
local -a DEPLOYED_MACHINES=($(printf "%s\n" ${SYSID_ARR[@]}|xargs -I{} -n1 -P1 maas ${MAAS_PROFILE} machine deploy {} install_kvm=true distro_series=${DISTRO} user_data="$(echo;cat <<EOF|tee /tmp/${FUNCNAME}.cloud-init.yaml|base64 -w0
#cloud-config-2
bootcmd:
- (ip route get ${MAAS_IP}|grep -oP '(?<=dev )[^ ]+')|xargs -n1 ip route add default dev
final_message: KVM Pod Deployment Complete on \$(hostname -f)
timezone: ${CI_TZ}
locale: ${CI_LC}
apt:
proxy: http://${MAAS_IP}:8000/
primary:
- arches: [amd64]
uri: http://${CI_APT_MIRROR}/ubuntu
security:
- arches: [amd64]
uri: http://security.ubuntu.com/ubuntu
write_files:
- encoding: b64
content: ${VIP_VRRP_SCRIPT}
path: /usr/local/bin/configure-vrrp.sh
permissions: '755'
owner: root:root
package_update: ${CI_PKG_UPD}
package_upgrade: ${CI_PKG_UPG}
${CI_PACKAGES}
${SSH_PUB_KEYS}
runcmd:
- set -x
- /usr/sbin/usermod --append --groups libvirt,libvirt-qemu virsh
- systemctl restart sshd
- /bin/sleep 10
- for i in \$(seq \$(find /dev -iname 'loop[0-9]*'|wc -l) 1 256);do mknod -m0660 /dev/loop\${i} b 7 \${i} && chown root.disk /dev/loop\${i};done
- export ENABLE_GPU_SUPPORT=${ENABLE_GPU}
- if [[ \${ENABLE_GPU_SUPPORT} = true ]];then export GPU_CMD='lspci -nn|awk -vIGNORECASE=1 '"'"'/\[03.*intel/{ORS=",";print substr(\$(NF-2),2,9)}'"'"'|sed "s/,$/\\n/"';fi
- if [[ \${ENABLE_GPU_SUPPORT} = true ]];then export GPU_CONF_FILE='lspci -nn|awk -vIGNORECASE=1 '"'"'/\[03.*intel/{printf "export NVDA_GPU_%02d_ID=%s\nexport NVDA_GPU_%02d_BUS=%s\n",NR,substr(\$(NF-2),2,9),NR,\$1}'"'"'|tee -a /etc/gpgpu-pci-ids.conf';fi
- if [[ \${ENABLE_GPU_SUPPORT} = true ]];then export GPU_PCI_LIST='lspci -nn|awk -vIGNORECASE=1 '"'"'BEGIN {printf "%s","options vfio-pci ids="} /\[03.*intel/{ORS=",";print substr(\$(NF-2),2,9)}'"'"'|sed "s/,$/\\n/"|tee /etc/modprobe.d/vfio.conf';fi
- if [ -n \$(eval \$GPU_CMD) ];then echo GPGPUs Detected;printf '%s\n' vfio vfio_pci > /etc/modules-load.d/vfio.conf;eval \$GPU_CONF_FILE;eval \$GPU_PCI_LIST;fi
- systemctl restart libvirtd
- export STORAGE_POOL_TYPE=${STORAGE_POOL_TYPE}
- if [ \${STORAGE_POOL_TYPE} = device ];then zpool create local ${STORAGE_POOL_DEVICE};fi
- if [ \${STORAGE_POOL_TYPE} = device ];then zfs create local/maas-zfs;fi
- if [ \${STORAGE_POOL_TYPE} = device ];then zfs set dedup=on local;fi
- if [ \${STORAGE_POOL_TYPE} = device ];then zfs set compression=on local;fi
- if [ \${STORAGE_POOL_TYPE} = device ];then virsh pool-define-as --name maas-images-zfs --type zfs --source-name local --source-dev=${STORAGE_POOL_DEVICE};fi
- if [ \${STORAGE_POOL_TYPE} = device ];then virsh pool-start maas-images-zfs;fi
- if [ \${STORAGE_POOL_TYPE} = device ];then virsh pool-autostart maas-images-zfs;fi
- if [ \${STORAGE_POOL_TYPE} = device ];then mkdir -p /var/lib/libvirt/extra-images && virsh pool-define-as extra-images-dir dir - - - - '/var/lib/libvirt/extra-images';fi
- if [ \${STORAGE_POOL_TYPE} = device ];then virsh pool-start extra-images-dir;fi
- if [ \${STORAGE_POOL_TYPE} = device ];then virsh pool-autostart extra-images-dir;fi
- export VIP_VRRP_ENABLE=${VIP_VRRP_ENABLE}
- if [ \${VIP_VRRP_ENABLE} = true ];then echo 'net.ipv4.ip_forward = 1' >> /etc/sysctl.conf;fi
- if [ \${VIP_VRRP_ENABLE} = true ];then sysctl -p;fi
- if [ \${VIP_VRRP_ENABLE} = true ];then install -D -m 644 /dev/null '/etc/systemd/system/keepalived.service';fi
- |-
if [ \${VIP_VRRP_ENABLE} = true ];then
cat > '/etc/systemd/system/keepalived.service' << 'EOF'
[Unit]
Description=LVS and VRRP High Availability monitor
After=network.target
ConditionFileNotEmpty=/etc/keepalived/keepalived.conf
[Service]
Type=simple
EnvironmentFile=-/etc/default/keepalived
ExecStart=/usr/sbin/keepalived --dont-fork
ExecReload=/bin/kill -s HUP \$MAINPID
KillMode=process
[Install]
WantedBy=multi-user.target
EOF
fi
- if [ \${VIP_VRRP_ENABLE} = true ];then /bin/systemctl enable '/etc/systemd/system/keepalived.service';fi
- touch /var/tmp/install-complete
EOF
)" |jq 2>/dev/null -r '"\(.hostname):\(.system_id)"'))
printf "\n\e[4G - A copy of the cloud-init used for this deployment can be found at\n\e[2G/tmp/${FUNCNAME}.cloud-init.yaml\n\n"
wait-pods-ready -c ${CPU_OCR} -m ${MEM_OCR} -t ${MICROCLOUD_TAG}
}
export -f create-kvm-microcloud
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment