vcenter-api-shutdown.py

from ast import Try
import socket
import sys
import logging
import atexit
import argparse
import getpass
from time import sleep
import vsanapiutils
import requests
import paramiko

from colorama import Fore, Style
from pyVmomi import vim
import vsanmgmtObjects
from pyVim.connect import SmartConnect, Disconnect
from ssl import _create_unverified_context
from vmware.vapi.vsphere.client import create_vsphere_client
from com.vmware.vcenter_client import VM
from com.vmware.vcenter.vm_client import Power

CATEGORY_NAME = "ShutdownTier"
CATEGORY_ID = "urn:vmomi:InventoryServiceCategory:02368c0a-3e88-4147-b743-069a466ad592:GLOBAL"
VCENTER_NAME = "VCENTER"
ESX_ROOT_USER = "root"
LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s"

logging.root.handlers = []
logging.basicConfig(format=LOG_FORMAT, level=logging.DEBUG , filename='vcenter-api-shutdown.log')

streamHandler = logging.StreamHandler()
streamHandler.setLevel(logging.INFO)
formatter = logging.Formatter(LOG_FORMAT)
streamHandler.setFormatter(formatter)
logger = logging.getLogger("")
logger.addHandler(streamHandler)
logger.info("Application started.")

def GetArgs():
    """
    Supports the command-line arguments listed below.
    """
    parser = argparse.ArgumentParser(
        description='Process args for vSAN SDK sample application')
    parser.add_argument('-s',
                        '--host',
                        required=True,
                        action='store',
                        help='Remote host to connect to')
    parser.add_argument('-o',
                        '--port',
                        type=int,
                        default=443,
                        action='store',
                        help='Port to connect on')
    parser.add_argument('-u',
                        '--vcenter-username',
                        required=True,
                        action='store',
                        help='User name to use when connecting to vCenter Host')
    parser.add_argument('-p',
                        '--password',
                        required=False,
                        action='store',
                        help='Password to use when connecting to vCenter host')
    parser.add_argument('--cluster',
                        dest='clusterName',
                        metavar="CLUSTER",
                        default='VSAN-Cluster')
    args = parser.parse_args()
    return args


def get_status(status):
    message = "%s%s%s"
    if status == "green":
        message = message % (Fore.GREEN, status, Style.RESET_ALL)
    if status == "info":
        message = message % (Fore.LIGHTYELLOW_EX, status, Style.RESET_ALL)
    if status == "skipped":
        message = message % (Fore.LIGHTYELLOW_EX, status, Style.RESET_ALL)
    if status == "red":
        message = message % (Fore.RED, status, Style.RESET_ALL)
    return message


def get_unverified_session():
    session = requests.session()
    session.verify = False
    requests.packages.urllib3.disable_warnings()
    return session


def getComputeInstance(entityName, serviceInstance):
   content = serviceInstance.RetrieveContent()
   searchIndex = content.searchIndex
   datacenters = content.rootFolder.childEntity
   for datacenter in datacenters:
      instance = searchIndex.FindChild(datacenter.hostFolder, entityName)
      if instance is not None:
         return instance
   return None


def getClusterInstance(cluster_name, serviceInstance):
    content = serviceInstance.RetrieveContent()
    searchIndex = content.searchIndex
    datacenters = content.rootFolder.childEntity
    for datacenter in datacenters:
        cluster = searchIndex.FindChild(datacenter.hostFolder, cluster_name)
        if cluster is not None:
            return cluster
    return None


def getVmInstance(vm_name, serviceInstance):
    content = serviceInstance.RetrieveContent()
    container = content.rootFolder
    recursive = True
    viewType = [vim.VirtualMachine]
    containerView = content.viewManager.CreateContainerView(container, viewType, recursive)
    vms = containerView.view
    for vm in vms:
        vm = vm.config.name == vm_name
        if vm is not None:
            return vm
    return None


def evaluateVSanClusterHealth(service_instance, context, host, cluster_name):
    logger.info("Evaluating vSAN cluster health prior to shutdown...")
    apiVersion = vsanapiutils.GetLatestVmodlVersion(host)
    vcMos = vsanapiutils.GetVsanVcMos(service_instance._stub,
                                        context=context,
                                        version=apiVersion)
    vhs = vcMos['vsan-cluster-health-system']
    cluster = getClusterInstance(cluster_name, service_instance)
    fetchFromCache = True
    healthSummary = vhs.QueryClusterHealthSummary(
        cluster=cluster,
        includeObjUuids=True,
        fetchFromCache=fetchFromCache)
    clusterStatus = healthSummary.clusterStatus
    logger.info("Cluster %s Status: %s" %
            (cluster_name, get_status(clusterStatus.status)))
    for hostStatus in clusterStatus.trackedHostsStatus:
        logger.info("Host %s Status: %s" %
                (hostStatus.hostname, get_status(hostStatus.status)))
    vsanClusterHealthGroups = healthSummary.groups
    healthy = True
    for group in vsanClusterHealthGroups:
        logger.info("%s - %s" % (group.groupName.upper(), get_status(group.groupHealth)))
        if  group.groupHealth not in ["green", "info", "skipped"]:
            healthy = False
        for test in group.groupTests:
            logger.info(
                "%s - %s - %s" %
                (group.groupName.upper(), test.testName,  get_status(test.testHealth)))
            if  test.testHealth not in ["green", "info", "skipped"]:
                healthy = False
    if not healthy:
        logger.warning("Exiting automated shutdown: vSAN cluster is unhealthy")
        sys.exit()


def waitForGuestShutdownsToComplete(vsphere_client, vm_tasks):
    for vm_task in vm_tasks:
        status = vsphere_client.vcenter.vm.Power.get(vm_task["vm_id"])
        if status.state == Power.State.POWERED_OFF:
            vm_task["power_state"] = Power.State.POWERED_OFF
    powered_on_vms = list(filter(lambda vm: vm["power_state"] == Power.State.POWERED_ON, vm_tasks))
    if len(powered_on_vms) > 0:
        logger.info("Waiting on %s VMs to properly shutdown." % (len(powered_on_vms)))
        sleep(2)
        waitForGuestShutdownsToComplete(vsphere_client, vm_tasks)


def shutdownAllVmsInCluster(vsphere_client, tiers):
    logger.info("Shutting down VMs in cluster based on SHUTDOWN_TIER.")
    tag_ids = vsphere_client.tagging.Tag.list_tags_for_category(CATEGORY_ID)
    tags = []
    vm_tasks = []
    for tag_id in tag_ids:
        tags.append(vsphere_client.tagging.Tag.get(tag_id))
    for tag in sorted(tags, reverse=False, key=lambda t: t.name):
        objects = vsphere_client.tagging.TagAssociation.list_attached_objects(tag.id)
        for object in objects:
            vm = vsphere_client.vcenter.VM.get(object.id)
            if tag.name in tiers:
                status = vsphere_client.vcenter.vm.Power.get(object.id)
                if status.state == Power.State.POWERED_ON:
                    logger.info("Shutting down VM %s in Tier %s" % (vm.name, tag.name))
                    vsphere_client.vcenter.vm.guest.Power.shutdown(object.id)
                    vm_tasks.append({ "vm_id": object.id, "power_state": Power.State.POWERED_ON})
                else:
                    logger.info("VM %s is already in the %s state." % (vm.name, Power.State.POWERED_OFF))
        waitForGuestShutdownsToComplete(vsphere_client, vm_tasks)
        vm_tasks = []


def disableVcenterHighAvailabilityMode(service_instance, cluster_name):
    logger.info("Disabling vSphere HA Mode.")
    cluster = getClusterInstance(cluster_name, service_instance)
    clusterConfigSpec = vim.cluster.ConfigSpec()
    clusterConfigSpec.dasConfig = cluster.configuration.dasConfig
    clusterConfigSpec.dasConfig.enabled = False
    tasks = []
    task = cluster.ReconfigureCluster_Task(clusterConfigSpec, True)
    tasks.append(task)
    vsanapiutils.WaitForTasks(tasks, service_instance)


def waitForVclsVmsToRetreat(vsphere_client):
    vms = vsphere_client.vcenter.VM.list()
    powered_on_vcls_vms = list(filter(lambda vm: vm.name.startswith ("vCLS"), vms))
    if len(powered_on_vcls_vms) > 0:
        logger.info("Waiting on %s vCLS VMs to retreat." % (len(powered_on_vcls_vms)))
        sleep(2)
        waitForVclsVmsToRetreat(vsphere_client)


def enableVclsRetreatMode(vsphere_client, service_instance, cluster_name):
    logger.info("Enabling vCLS Retreat Mode.")
    cluster = getClusterInstance(cluster_name, service_instance)
    key = "config.vcls.clusters.{}.enabled".format(cluster._moId)
    value = "false"
    option_value = vim.option.OptionValue
    vim_values = [option_value(key=key, value=str(value))]
    opt_mgr = service_instance.RetrieveContent().setting
    opt_mgr.UpdateValues(vim_values)
    waitForVclsVmsToRetreat(vsphere_client)


def ensureVsanResyncingCompleted(service_instance, context, host, cluster_name):
    logger.info("Ensuring vSAN Resyncing Objects has completed.")
    apiVersion = vsanapiutils.GetLatestVmodlVersion(host)
    vcMos = vsanapiutils.GetVsanVcMos(
        service_instance._stub, context=context, version=apiVersion)
    vhs = vcMos['vsan-cluster-object-system']
    cluster = getClusterInstance(cluster_name, service_instance)
    object_filter = vim.cluster.VsanSyncingObjectFilter()
    object_filter.resyncStatus = 'active'
    object_filter.resyncType = None
    object_filter.offset = 0
    object_filter.numberOfObjects = 100
    resyincing_objects = vhs.QuerySyncingVsanObjectsSummary(cluster, object_filter)
    if resyincing_objects.totalObjectsToSync > 0:
        logger.info("Exiting automated shutdown: vSAN cluster is resyncing objects")
        sys.exit()


def waitForVcenterVmGuestShutdownsToComplete(host, username, password):
    ssh_client = paramiko.SSHClient()
    ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    ssh_client.connect(host.name, username=username, password=password)
    stdin, stdout, stderr = ssh_client.exec_command('esxcli vm process list')
    response = stdout.read().decode()
    ssh_client.close()
    logger.info(response)
    if "Display Name: %s" % (VCENTER_NAME) in response:
        logger.info("Waiting for vCenter VM on host %s to shutdown." % (host.name))
        sleep(10)
        waitForVcenterVmGuestShutdownsToComplete(host, username, password)


def returnVCenterHost(vsphere_client):
    logger.info("Shutting down vCenter VM in cluster.")
    hosts = vsphere_client.vcenter.Host.list()
    for host in hosts:
        vms = vsphere_client.vcenter.VM.list(VM.FilterSpec(hosts={host.host}))
        for vm in vms:
            if vm.name == VCENTER_NAME:
                status = vsphere_client.vcenter.vm.Power.get(vm.vm)
                if status == Power.Info(state=Power.State.POWERED_ON):
                    logger.info("%s is running on: %s" % (VCENTER_NAME, host.name))
                    return host


def returnAllPweredOnHosts(vsphere_client):
    logger.info("Getting a list of all powered on hosts in cluster.")
    all_hosts = []
    hosts = vsphere_client.vcenter.Host.list()
    for host in hosts:
        if host.power_state == "POWERED_ON":
            all_hosts.append(host)
    return all_hosts


def shutdownVCenterVmInCluster(vsphere_client, vcenter_host):
    logger.info("Shutting down vCenter VM in cluster.")
    vms = vsphere_client.vcenter.VM.list(VM.FilterSpec(hosts={vcenter_host.host}))
    for vm in vms:
        if vm.name == VCENTER_NAME:
            status = vsphere_client.vcenter.vm.Power.get(vm.vm)
            if status == Power.Info(state=Power.State.POWERED_ON):
                logger.info("Shutting down %s on host: %s" % (VCENTER_NAME, vcenter_host.name))
                vsphere_client.vcenter.vm.guest.Power.shutdown(vm.vm)
            break


def waitForServiceToBeEnabled(cluster_name, service_instance):
    cluster = getClusterInstance(cluster_name, service_instance)
    for host in cluster.host:
        serviceInfo = host.configManager.serviceSystem.serviceInfo
        for service in serviceInfo.service:
            if service.label == "SSH" and service.running == "false":
                logger.info("Waiting for SSH to be started on host%s." % (host))
                sleep(2)
                waitForServiceToBeEnabled(cluster_name, service_instance)


def startSshServiceOnHosts(service_instance, cluster_name, start=True):
    cluster = getClusterInstance(cluster_name, service_instance)
    for host in cluster.host:
        serviceInfo = host.configManager.serviceSystem.serviceInfo
        for service in serviceInfo.service:
            if service.label == "SSH":
                if start:
                    host.configManager.serviceSystem.StartService(service.key)
                    waitForServiceToBeEnabled(cluster_name, service_instance)
                else:
                    host.configManager.serviceSystem.StopService(service.key)


def disableClusterMemberUpdates(all_powered_on_hosts, username, password):
    logger.info("Disabling cluster member updates.")
    for host in all_powered_on_hosts:
        logger.info("Disabling cluster member updates on host %s." % (host.name))
        ssh_client = paramiko.SSHClient()
        ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        ssh_client.connect(host.name, username=username, password=password)
        stdin, stdout, stderr = ssh_client.exec_command('esxcfg-advcfg -s 1 /VSAN/IgnoreClusterMemberListUpdates')
        response = stdout.read().decode()
        ssh_client.close()
        logger.info(response)


def prepareClusterForReboot(all_powered_on_hosts, username, password):
    logger.info("Preparing cluster for reboot.")
    # Run reboot prepare script only on one host
    for host in all_powered_on_hosts:
        logger.info("Running reboot preparation script on host %s." % (host.name))
        ssh_client = paramiko.SSHClient()
        ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        ssh_client.connect(host.name, username=username, password=password)
        stdin, stdout, stderr = ssh_client.exec_command('python /usr/lib/vmware/vsan/bin/reboot_helper.py prepare')
        response = stdout.read().decode()
        ssh_client.close()
        logger.info(response)
        break


def waitForHostToEnterMaintenanceMode(host, username, password):
    logger.info("Waiting for host %s to enter maintenance mode." % (host.name))
    ssh_client = paramiko.SSHClient()
    ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    ssh_client.connect(host.name, username=username, password=password)
    stdin, stdout, stderr = ssh_client.exec_command('vim-cmd /hostsvc/hostsummary | grep inMaintenanceMode')
    response = stdout.read().decode()
    ssh_client.close()
    if "inMaintenanceMode = true" not in response:
        sleep(2)
        waitForHostToEnterMaintenanceMode(host, username, password)


def placeHostsInMaintenanceMode(all_powered_on_hosts, username, password):
    logger.info("Placing hosts into maintenance mode.")
    for host in all_powered_on_hosts:
        logger.info("Placing host %s into maintenance mode." % (host.name))
        ssh_client = paramiko.SSHClient()
        ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        ssh_client.connect(host.name, username=username, password=password)
        stdin, stdout, stderr = ssh_client.exec_command('esxcli system maintenanceMode set -e true -m noAction')
        response = stdout.read().decode()
        ssh_client.close()
        logger.info(response)
        waitForHostToEnterMaintenanceMode(host, username, password)


def waitForHostToShutDown(host):
    HOST_ONLINE = True
    while HOST_ONLINE:
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            s.connect((host.name, 22))
            logger.info("Waiting for host %s to shutdown." % (host.name))
        except socket.error as e:
           HOST_ONLINE = False
        finally:
            s.close()
        sleep(2)


def performRollingHostShutdown(all_powered_on_hosts, username, password):
    logger.info("Performing rolling host shutdown.")
    for host in all_powered_on_hosts:
        waitForHostToEnterMaintenanceMode(host, username, password)
        ssh_client = paramiko.SSHClient()
        ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        ssh_client.connect(host.name, username=username, password=password)
        stdin, stdout, stderr = ssh_client.exec_command('esxcli system shutdown poweroff --reason AUTOMATED_SHUTDOWN_ON_UPS_BACKUP')
        ssh_client.close()
        waitForHostToShutDown(host)


def main():
    """
    This application automates shutdown of vSphere, vCenter, vSAN, VMs and ESXi hosts.
    https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.vsan-monitoring.doc/GUID-31B4F958-30A9-4BEC-819E-32A18A685688.html
    """

    args = GetArgs()

    tiers = ["0", "1", "2", "3"]

    password = args.password

    if not password:
        password = getpass.getpass(prompt="Password: ")

    context = _create_unverified_context()
    vsphere_client = create_vsphere_client(server=args.host,
                                        username=args.vcenter_username,
                                        password=password,
                                        session=get_unverified_session())
    service_instance = SmartConnect(host=args.host,
                      user=args.vcenter_username,
                      pwd=password,
                      port=int(args.port),
                      sslContext=context)

    atexit.register(Disconnect, service_instance)
    about_info = service_instance.content.about
    vcenter_host = None
    all_powered_on_hosts = []

    if about_info.apiType == 'VirtualCenter':

        evaluateVSanClusterHealth(service_instance, context, args.host, args.clusterName)
        shutdownAllVmsInCluster(vsphere_client, tiers)
        disableVcenterHighAvailabilityMode(service_instance, args.clusterName)
        enableVclsRetreatMode(vsphere_client, service_instance, args.clusterName)
        ensureVsanResyncingCompleted(service_instance, context, args.host, args.clusterName)
        startSshServiceOnHosts(service_instance, args.clusterName)
        vcenter_host = returnVCenterHost(vsphere_client)
        all_powered_on_hosts = returnAllPweredOnHosts(vsphere_client)
        shutdownVCenterVmInCluster(vsphere_client, vcenter_host)

        # At this point, access to vCenter is now unavailable.  The remaining functions will use SSH.
        waitForVcenterVmGuestShutdownsToComplete(vcenter_host, ESX_ROOT_USER, password)
        disableClusterMemberUpdates(all_powered_on_hosts, ESX_ROOT_USER, password)
        prepareClusterForReboot(all_powered_on_hosts, ESX_ROOT_USER, password)
        placeHostsInMaintenanceMode(all_powered_on_hosts, ESX_ROOT_USER, password)
        performRollingHostShutdown(all_powered_on_hosts, ESX_ROOT_USER, password)

    else:
        logger.info("Server provided is not a vCenter server.")

    logger.info("All vCenter shutdown tasks are complete.")


if __name__ == "__main__":
    main()

This code shutdowns vCenter and ESX hosts running vSAN and VCHA. It essentially follows this guide. It has the added benefit of shuttung down VMs in tiers, which is handy so some VMs can shutdown ahead of others.

Requirements:

requests
pyvmomi
defusedxml
lxml
colorama
paramiko

Note: also needs vsanapiutils.py and vsanmgmtObjects.py which you can download from VMWare.

To run: python vcenter-api-shutdown.py -s <vcenter-ip> -u <vcenter-username> -p <vcenter-password> --cluster <vcenter-cluster>

Dockerfile Example:

FROM python:alpine
WORKDIR /src
COPY vsan-sdk-python/samplecode/vsanapiutils.py /src/
COPY vsan-sdk-python/bindings/vsanmgmtObjects.py /src/
COPY requirements.txt vcenter-api-shutdown.py /src/
RUN apk update && apk add git
RUN pip install --upgrade pip \
    && pip install --upgrade setuptools==62.0.0 \
    && pip install --upgrade git+https://github.com/vmware/vsphere-automation-sdk-python.git
RUN pip install --no-cache-dir -r requirements.txt
RUN pip list
ENV PYTHONUNBUFFERED=1
ENTRYPOINT ["python", "-u", "/src/vcenter-api-shutdown.py"]

Assumptions:

• Each host is named by IP
• Each VM has a tag of 0-3 within a custom category called ShutdownTier
• The login to vCenter uses the same pass as logging in through SSH

Example Output: