Deploying a Kubernetes Cluster on VMWare vSphere with CSI and CPI. Prerequisites configuration example

vsphere_cpi_csi_prerequisites.sh

## !!! DO NOT RUN THIS SCRIPT AS IS !!!
## SOME VALUES ARE JUST AN EXAMPLE AND MUST BE CHANGED
## ACCORDING TO YOUR INFRASTRUCTURE CONFIGURATION

## This example script uses the following considerations:
# - vSphere 6.7U3 (or later) is a prerequisite for using CSI and CPI at the time of writing
# - vCenter Datacenter name: DC01
# - vCenter Cluster name: CLS01
# - vCenter Datastore names: DATASTORE01, DATASTORE02, DATASTORE03
# - K8S nodes VMs hostname pattern: example-k8s* (e.g.: example-k8s-master01.example.com)


## References:
# - https://github.com/kubernetes/cloud-provider-vsphere/blob/master/docs/book/tutorials/kubernetes-on-vsphere-with-kubeadm.md


## Install and configure govc tool
# https://github.com/vmware/govmomi/tree/master/govc
export GOVC_URL="https://vcenter-url.example.com"
export GOVC_USERNAME="[email protected]"
export GOVC_PASSWORD="sup3rS3cr3t(p4ss"
export GOVC_INSECURE=1


## Configure VMs with disk.EnableUUID=1
# This step is necessary so that the VMDK always presents a consistent UUID to the VM
# thus allowing the disk to be mounted properly
export CLUSTER_NAME=example-k8s
for vm in $(govc ls /DC01/vm/"$CLUSTER_NAME"*)
do
  govc vm.change -vm "$vm" -e="disk.enableUUID=1"
done


## !! This step is needed ONLY when K8S nodes VMs have HW version <15!!
# VM Hardware should be at version 15 or higher
export CLUSTER_NAME=example-k8s

# shutdown VM's
for vm in $(govc ls /DC01/vm/"$CLUSTER_NAME"*)
do
  govc vm.power -off "$vm"
done

# upgrade hardware version
for vm in $(govc ls /DC01/vm/"$CLUSTER_NAME"*)
do
  govc vm.upgrade -version=15 -vm "$vm"
done

# power on VM's
10for vm in $(govc ls /DC01/vm/"$CLUSTER_NAME"*)
do
  govc vm.power -on "$vm"
done


## The vSphere user for CSI driver requires a set of privileges to perform Cloud Native Storage operations
# Set of roles below is the essential minimum in case of topology/zone UNAWARE K8S cluster

# CNS-SEARCH-AND-SPBM role - allows viewing of defined storage policies - required on Root vCenter Server
govc role.create CNS-SEARCH-AND-SPBM \
  Cns.Searchable \
  StorageProfile.View \
  System.Anonymous \
  System.Read \
  System.View

# CNS-VM role - allows adding an existing virtual disk to a virtual machine - required on all k8s cluster node VMs
govc role.create CNS-VM \
  System.Anonymous \
  System.Read \
  System.View \
  VirtualMachine.Config.AddExistingDisk \
  VirtualMachine.Config.AddRemoveDevice
  
# CNS-DATASTORE role - allows performing read, write, delete, and rename operations in the datastore browser
# required on shared datastore where persistent volumes reside
govc role.create CNS-DATASTORE \
  Datastore.FileManagement \
  System.Anonymous \
  System.Read \
  System.View


## Create local vCenter user for the CPI/CSI
# !! Be careful with using special symbols in the password !!
# !! Some of them (e.g. backslash '\') might cause issues with Kubespray and kubectl !!
govc sso.user.create -p 'sup3rS3cr3t(p4ss' -d 'User for the Kubernetes CPI/CSI integration' k8s-csi


## Assign permissions to the created user
# CNS-SEARCH-AND-SPBM role on the root vCenter level
govc permissions.set -principal VSPHERE.LOCAL\\k8s-csi -role CNS-SEARCH-AND-SPBM -propagate=false /

# CNS-VM role on each K8S cluster node VM
export CLUSTER_NAME=uat-k8s
govc permissions.set -principal VSPHERE.LOCAL\\k8s-csi -role CNS-VM -propagate=false $(govc ls /DC01/vm/"$CLUSTER_NAME"*)

# CNS-DATASTORE role on each shared datastore where persistent volumes reside
govc permissions.set -principal VSPHERE.LOCAL\\k8s-csi -role CNS-DATASTORE -propagate=false $(govc ls /DC01/datastore/DATASTORE*)

## ReadOnly role
# ReadOnly role on the K8S nodes VMs Datacenter
govc permissions.set -principal VSPHERE.LOCAL\\k8s-csi -role ReadOnly  -propagate=false /DC01

# ReadOnly role on the K8S nodes VMs Cluster
govc permissions.set -principal VSPHERE.LOCAL\\k8s-csi -role ReadOnly  -propagate=false /DC01/host/CLS01

# ReadOnly role for k8s-csi user for each host where K8S nodes VMs reside. ! Example for the esx01.example.com host !:
govc permissions.set -principal VSPHERE.LOCAL\\k8s-csi -role ReadOnly  -propagate=false /DC01/host/CLS01/esx01.example.com