Kargo2.0

README.md

FYI: this gist is a research sandbox for manual methods to build the 100DaysOfHomelab hypervisor

Kargo 2.0 KubeVirt Intel Nuc HomeLab

0. Join the ContainerCraft CodeCtl Slack

1. Create/Have GitHub Account & Upload SSH Public Key

2. Download & Write Fedora 36+ to USB

3. Install Fedora

4. Install ssh key from GitHub

mkdir ~/.ssh
curl -L github.com/${GH_USERNAME}.keys | tee -a .ssh/authorized_keys
chmod 700 ~/.ssh
chmod 600 ~/.ssh/authorized_keys

5. Configure Host Bridge br0 & Static IP(s)

6. Follow the directions below

2.a) Install Dependencies

sudo dnf remove -y zram-generator-defaults # disable swap
sudo dnf install -y keepalived haproxy dnf-automatic python3 python3-pip screenfetch glances lm_sensors htop tmux vim git tar
sudo sed -i 's/^apply_updates = no/apply_updates = yes/g' /etc/dnf/automatic.conf
sudo systemctl enable --now dnf-automatic.timer
sudo python3 -m pip install --upgrade pip
python3 -m pip install --upgrade glances
sudo dnf -y update

2.b) Disable FirewallD (LAB USE ONLY)

sudo systemctl disable firewalld
sudo systemctl stop firewalld

## TESTING
sudo firewall-cmd --add-rich-rule='rule protocol value="vrrp" accept' --permanent
sudo firewall-cmd --permanent --add-port={16443,8443,6443,2379-2380,10250-10252,30000-32767,179}/tcp --add-port=4789/udp
sudo firewall-cmd --reload

2.c) Enable nested virtualization (SELINUX DISABLED -- LAB USE ONLY)

sudo grubby --update-kernel=ALL --args 'selinux=0 cgroup_memory=1 cgroup_enable=cpuset cgroup_enable=memory systemd.unified_cgroup_hierarchy=0 intel_iommu=on iommu=pt rd.driver.pre=vfio-pci pci=realloc'

2.d) Install virtctl binary

export VIRTCTL_RELEASE=$(curl -s https://api.github.com/repos/kubevirt/kubevirt/releases/latest | awk -F '["v,]' '/tag_name/{print $5}')
sudo curl --output /tmp/virtctl -L https://github.com/kubevirt/kubevirt/releases/download/v${VIRTCTL_RELEASE}/virtctl-v${VIRTCTL_RELEASE}-linux-amd64
sudo install -o root -g root -m 0755 /tmp/virtctl /usr/local/bin/virtctl

2.e) Install kubectl binary

curl --output /tmp/kubectl -L "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
sudo install -o root -g root -m 0755 /tmp/kubectl /usr/local/bin/kubectl

2.f) Move ResolveD to port 5353 (required only on CentOS Workstation / Server with GUI)

cat <<EOF | sudo tee /etc/systemd/resolved.conf
[Resolve]
DNS=1.1.1.1
DNSStubListener=no
EOF

sudo mkdir -p /run/systemd/resolve && sudo ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf
sudo systemctl disable dnsmasq ; sudo systemctl stop dnsmasq ; sudo pkill -KILL dnsmasq
sudo systemctl enable --now systemd-resolved
sudo systemctl restart systemd-resolved.service
sudo systemctl restart NetworkManager

2.g) Create SSH Keys & Enable SSH to self

ssh-keygen -b 2048 -t rsa -f ~/.ssh/id_rsa -q -N ""
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys
ssh $(ip a s $(ip r | awk '/default/{print $5}') | awk -F'[/ ]' '/inet /{print $6}' | head -n 1) whoami

2.h) Reboot

sudo reboot

2.i) Install Kubespray ansible dependencies

git clone https://github.com/kubernetes-sigs/kubespray.git ~/kubespray && cd ~/kubespray/
python3 -m pip install --upgrade -r requirements.txt

2.x) Export Variables

export VIPADDR=192.168.1.30
export IPADDR1=192.168.1.31
export IPADDR2=192.168.1.32
export IPADDR3=192.168.1.33
export CLUSTER="kargo"

2.j) Create ansible hosts inventory file

cp -rfp inventory/sample inventory/kargo
declare -a IPS="${IPADDR1} ${IPADDR2} ${IPADDR3}"
CONFIG_FILE=inventory/kargo/hosts.yaml python3 contrib/inventory_builder/inventory.py ${IPS[@]}

2.k) Write override variables to file

cat <<EOF >vars.yml
container_manager: crio
kube_encrypt_secret_data: true
kube_network_plugin_multus: true
kubelet_deployment_type: host
kubelet_shutdown_grace_period: 60s
kubelet_shutdown_grace_period_critical_pods: 20s
auto_renew_certificates: true
kubeconfig_localhost: true
etcd_deployment_type: host
download_container: true
kubectl_localhost: true
ping_access_ip: true
#######################################################
# Support HAPROXY & VRRP for High Availability
loadbalancer_apiserver_localhost: false
apiserver_loadbalancer_domain_name: "kargo.home.arpa"
loadbalancer_apiserver:
  address: ${VIPADDR}
  port: 16443
#######################################################
# EXPERIMENTAL
#kube_network_plugin: kube-ovn
EOF

2.l) Test hosts file && Run ansible playbook

ansible -i inventory/kargo/hosts.yaml -m ping all && time ansible-playbook -i inventory/kargo/hosts.yaml --become --become-user=root --ask-become-pass --extra-vars @vars.yml --user=fedora cluster.yml

2.m) Link kubectl into path && Optimize for single node

mkdir -p ~/.kube && cp inventory/kargo/artifacts/admin.conf ~/.kube/config && chmod 600 ~/.kube/config
kubectl patch node node1 -p '{"spec":{"taints":[]}}'
curl -L https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/images/multus-daemonset-crio.yml | kubectl apply -f -
kubectl patch deployment -n kube-system coredns --patch='{"spec":{"template":{"spec":{"tolerations":[]}}}}'
kubectl -n kube-system rollout restart deployment/coredns
sleep 6
kubectl patch configmap -n kube-system dns-autoscaler --patch '{"data":{"linear":"{\"coresPerReplica\":256,\"min\":1,\"nodesPerReplica\":16,\"preventSinglePointFailure\":true}"}}'

2.n) Install Kargo

kubectl create namespace kargo
kubectl taint nodes --overwrite --all node-role.kubernetes.io/master-
kubectl label nodes --all --overwrite node-role.kubernetes.io/worker=''
kubectl label nodes --all --overwrite node-role.kubernetes.io/kubevirt=''
kubectl get nodes -owide
kubectl apply -f https://git.io/JCVmB; sleep 10; kubectl apply -f https://git.io/JCVmB; sleep 10; kubectl apply -f https://git.io/JCVmB
kubectl patch storageclass hostpath-provisioner -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'

• Wait for all pods to start up

watch kubectl get po -A

---

Create & Test a new VM attached to your LAN -- USER:PASS = ubuntu:ubuntu

kubectl apply -f https://git.io/JCVoI
kubectl get events -n kargo
virtctl console -n kargo testbr0vm

nmcli_br0.md

Build Linux Bridge br0 as primary host interface

Build Primary Bridge br0

export IFACE="enp0s31f6"
export ADDRESS="192.168.1.51"
export GATEWAY="192.168.1.1"
export DNS1="192.168.1.1"
export DNS2="1.1.1.1"
sudo nmcli con add type bridge autoconnect yes ifname br0 con-name br0
sudo nmcli con mod br0 ipv4.address "${ADDRESS}/24" 
sudo nmcli con mod br0 ipv4.gateway ${GATEWAY}
sudo nmcli con mod br0 ipv4.dns "${DNS1}" +ipv4.dns "$DNS2"
sudo nmcli con mod br0 bridge.multicast-snooping no
sudo nmcli con mod br0 bridge.stp no
sudo nmcli connection modify br0 ipv4.dns-search 'codectl.lab'
sudo nmcli connection add type bridge-slave autoconnect yes con-name ${IFACE} ifname ${IFACE} master br0
sudo nmcli con mod br0 ipv4.method manual
sudo nmcli connection delete ${IFACE}

DO NOT USE OVS for br0

ovs should not be used for default host networking due to conflict with kube-ovn saving method for posterity

sudo nmcli con add type ovs-bridge conn.interface br0 con-name ovs-br0-bridge
sudo nmcli con add type ovs-port conn.interface ovs-br0-port master ovs-br0-bridge con-name ovs-br0-port
sudo nmcli con mod ovs-br0-interface ipv4.method static ipv4.address 192.168.1.51/24
sudo nmcli con mod ovs-br0-interface ipv4.method static ipv4.gateway 192.168.1.1
sudo nmcli con mod ovs-br0-interface ipv4.method static ipv4.dns "192.168.1.1"
sudo nmcli con mod ovs-br0-interface ipv4.method static ipv4.method manual
sudo nmcli con add type ovs-interface slave-type ovs-port conn.interface br0 master ovs-br0-port con-name ovs-br0-interface
export IFACE="enp0s31f6"
sudo nmcli con add type ovs-port conn.interface ${IFACE} master ovs-br0-bridge con-name ovs-br0-port-${IFACE}
sudo nmcli con add type ethernet conn.interface ${IFACE} master ovs-br0-port-${IFACE} con-name ovs-br0-interface-${IFACE}

wip_haproxy.md

cat <<EOF | sudo tee /etc/haproxy/haproxy.cfg
listen kubernetes-apiserver-https
bind *:18443
mode tcp
option log-health-checks
timeout connect         10s
timeout client          1m
timeout server          1m
server prod-master1 ${IPADDR1}:16443 check check-ssl verify none inter 10000
server prod-master2 ${IPADDR2}:16443 check check-ssl verify none inter 10000
server prod-master3 ${IPADDR3}:16443 check check-ssl verify none inter 10000
balance roundrobin
EOF
sudo systemctl enable --now haproxy ; sudo systemctl restart haproxy
sudo systemctl status haproxy
sudo firewall-cmd --permanent --add-port=18443/tcp ; sudo firewall-cmd --reload

wip_vrrp.md

• master 1 (VRRP PRIMARY)

cat <<EOF | sudo tee /etc/keepalived/keepalived.conf
vrrp_script chk_haproxy {
  script "killall -0 haproxy"
  interval 2
  weight 2
}
vrrp_instance VI_1 {
  interface br0
  state MASTER
  advert_int 1
  virtual_router_id 51
  priority 101
  unicast_src_ip ${IPADDR1}    ##Master 1 IP Address
  unicast_peer {
      ${IPADDR2}               ##Master 2 IP Address
      ${IPADDR3}               ##Master 3 IP Address
   }
  virtual_ipaddress {
      ${VIPADDR}               ##Shared Virtual IP address
  }
  track_script {
    chk_haproxy
  }
}
EOF

• master 2 (VRRP BACKUP)

cat <<EOF | sudo tee /etc/keepalived/keepalived.conf
vrrp_script chk_haproxy {
  script "killall -0 haproxy"
  interval 2
  weight 2
}
vrrp_instance VI_1 {
  interface br0
  state BACKUP
  advert_int 3
  virtual_router_id 50
  priority 100
  unicast_src_ip ${IPADDR2}    ##Master 2 IP Address
  unicast_peer {
      ${IPADDR1}               ##Master 1 IP Address
      ${IPADDR3}               ##Master 3 IP Address
   }
  virtual_ipaddress {
    ${VIPADDR}                 ##Shared Virtual IP address
  }
  track_script {
    chk_haproxy
  }
}
EOF

• master 3 (VRRP BACKUP)

cat <<EOF | sudo tee /etc/keepalived/keepalived.conf
vrrp_script chk_haproxy {
  script "killall -0 haproxy"
  interval 2
  weight 2
}
vrrp_instance VI_1 {
  interface br0
  state BACKUP
  advert_int 3
  virtual_router_id 49
  priority 99
  unicast_src_ip ${IPADDR3}    ##Master 3 IP Address
  unicast_peer {
      ${IPADDR1}               ##Master 1 IP Address
      ${IPADDR2}               ##Master 2 IP Address
   }
  virtual_ipaddress {
    ${VIPADDR}                 ##Shared Virtual IP address
  }
  track_script {
    chk_haproxy
  }
}
EOF

sudo systemctl enable --now keepalived ; sleep 2; sudo systemctl restart keepalived ; sleep 2
sudo systemctl restart NetworkManager ; sleep 3
sudo systemctl status keepalived
ip a s br0
firewall-cmd --add-rich-rule='rule protocol value="vrrp" accept' --permanent && firewall-cmd --reload

consider adopting openstack's os-net-config https://computingforgeeks.com/configuring-open-vswitch-on-centos-rhel-fedora/